def test_rest_api_error(self):
apix = ApiException(status=400, reason="boom")
e = RestApiError(cause=apix)
assert_that(e.status, equal_to(400))
assert_that(e.reason, equal_to("boom"))
assert_that(e.body, equal_to(None))
assert_that(e.cause, equal_to(apix))
def _open_for_read(self):
"""open the file in read mode"""
ownerid, datasetid = parse_dataset_key(self._dataset_key)
response = requests.get(
'{}/file_download/{}/{}/{}'.format(
self._query_host, ownerid, datasetid, self._file_name),
headers={
'User-Agent': self._user_agent,
'Authorization': 'Bearer {}'.format(
self._config.auth_token)
}, stream=True)
try:
response.raise_for_status()
except Exception as e:
raise RestApiError(cause=e)
self._read_response = response
def close(self):
"""
in write mode, closing the handle adds the sentinel value into the
queue and joins the thread executing the HTTP request. in read mode,
this clears out the read response object so there are no references
to it, and the resources can be reclaimed.
"""
if self._mode.find('w') >= 0:
self._queue.put(self._sentinel)
self._thread.join(timeout=self._timeout)
if self._thread.is_alive():
raise RemoteFileException("Closing file timed out.")
response = self._response_queue.get_nowait()
try:
response.raise_for_status()
except Exception as e:
raise RestApiError(cause=e)
else:
self._read_response = None
def open_remote_file(self, dataset_key, file_name,
mode='w', **kwargs):
"""Open a remote file object that can be used to write to or read from
a file in a data.world dataset
:param dataset_key: Dataset identifier, in the form of owner/id
:type dataset_key: str
:param file_name: The name of the file to open
:type file_name: str
:param mode: the mode for the file - must be 'w', 'wb', 'r', or 'rb' -
indicating read/write ('r'/'w') and optionally "binary"
handling of the file data. (Default value = 'w')
:type mode: str, optional
:param chunk_size: size of chunked bytes to return when reading
streamed bytes in 'rb' mode
:type chunk_size: int, optional
:param decode_unicode: whether to decode textual responses as unicode
when returning streamed lines in 'r' mode
:type decode_unicode: bool, optional
:param **kwargs:
Examples
--------
>>> import datadotworld as dw
>>>
>>> # write a text file
>>> with dw.open_remote_file('username/test-dataset',
... 'test.txt') as w:
... w.write("this is a test.")
>>>
>>> # write a jsonlines file
>>> import json
>>> with dw.open_remote_file('username/test-dataset',
... 'test.jsonl') as w:
... json.dump({'foo':42, 'bar':"A"}, w)
... w.write("\\n")
... json.dump({'foo':13, 'bar':"B"}, w)
... w.write("\\n")
>>>
>>> # write a csv file
>>> import csv
>>> with dw.open_remote_file('username/test-dataset',
... 'test.csv') as w:
... csvw = csv.DictWriter(w, fieldnames=['foo', 'bar'])
... csvw.writeheader()
... csvw.writerow({'foo':42, 'bar':"A"})
... csvw.writerow({'foo':13, 'bar':"B"})
>>>
>>> # write a pandas dataframe as a csv file
>>> import pandas as pd
>>> df = pd.DataFrame({'foo':[1,2,3,4],'bar':['a','b','c','d']})
>>> with dw.open_remote_file('username/test-dataset',
... 'dataframe.csv') as w:
... df.to_csv(w, index=False)
>>>
>>> # write a binary file
>>> with dw.open_remote_file('username/test-dataset',
>>> 'test.txt', mode='wb') as w:
... w.write(bytes([100,97,116,97,46,119,111,114,108,100]))
>>>
>>> # read a text file
>>> with dw.open_remote_file('username/test-dataset',
... 'test.txt', mode='r') as r:
... print(r.read())
>>>
>>> # read a csv file
>>> with dw.open_remote_file('username/test-dataset',
... 'test.csv', mode='r') as r:
... csvr = csv.DictReader(r)
... for row in csvr:
... print(row['column a'], row['column b'])
>>>
>>> # read a binary file
>>> with dw.open_remote_file('username/test-dataset',
... 'test', mode='rb') as r:
... bytes = r.read()
"""
try:
return RemoteFile(self._config, dataset_key, file_name,
mode=mode, **kwargs)
except Exception as e:
raise RestApiError(cause=e)