def save_content(self, file):
data = io.BytesIO()
stream.stream_response_to_file(file, path=data)
created = PhotoStationService.session.query('SYNO.PhotoStation.File', {
'method': 'upload' + self.filetype,
'version': '1',
'dest_folder_path': self.album.path,
'duplicate': 'overwrite', # rename, ignore
'filename': self.filename,
'mtime': str(self.created),
'title': self.title,
'description': self.description,
'ps_username': PhotoStationService.session.username,
'original': (self.filename, data)
})
if self.rating > 0 or (self.latitude and self.longitude):
self.update({
'rating': self.rating,
'gps_lat': self.latitude,
'gps_lng': self.longitude
})
self.album.add_item(self.filename, self)
def stream_zip_directory_over_http(url, directory, members=None, timeout=(9.05, 31.1)):
''' Supply an http get request and stream the response to a file.
Parameters
----------
url : str
Send the request to this url
directory : str
Extract the response to this directory
members : list of str, optional
Extract only these files
timeout : float or tuple of float, optional
Specify a timeout for the request. If a tuple, specify seperate connect
and read timeouts.
'''
buf = io.BytesIO()
with closing( requests.get(url, stream=True, timeout=timeout) ) as request:
stream.stream_response_to_file( request, buf )
zipper = zipfile.ZipFile(buf)
zipper.extractall(path=directory, members=members)
zipper.close()
def test_stream_response_to_file_chunksize():
s = requests.Session()
recorder = get_betamax(s)
url = ('https://api.github.com/repos/sigmavirus24/github3.py/releases/'
'assets/37944')
class FileWrapper(io.BytesIO):
def __init__(self):
super(FileWrapper, self).__init__()
self.chunk_sizes = []
def write(self, data):
self.chunk_sizes.append(len(data))
return super(FileWrapper, self).write(data)
file_obj = FileWrapper()
chunksize = 1231
with recorder.use_cassette('stream_response_to_file', **preserve_bytes):
r = s.get(url, headers={'Accept': 'application/octet-stream'},
stream=True)
stream.stream_response_to_file(r, path=file_obj, chunksize=chunksize)
assert 0 < file_obj.tell()
assert len(file_obj.chunk_sizes) >= 1
assert file_obj.chunk_sizes[0] == chunksize
def test_stream_response_to_file_chunksize():
s = requests.Session()
recorder = get_betamax(s)
url = ('https://api.github.com/repos/sigmavirus24/github3.py/releases/'
'assets/37944')
class FileWrapper(io.BytesIO):
def __init__(self):
super(FileWrapper, self).__init__()
self.chunk_sizes = []
def write(self, data):
self.chunk_sizes.append(len(data))
return super(FileWrapper, self).write(data)
file_obj = FileWrapper()
chunksize = 1231
with recorder.use_cassette('stream_response_to_file', **preserve_bytes):
r = s.get(url,
headers={'Accept': 'application/octet-stream'},
stream=True)
stream.stream_response_to_file(r, path=file_obj, chunksize=chunksize)
assert 0 < file_obj.tell()
assert len(file_obj.chunk_sizes) >= 1
assert file_obj.chunk_sizes[0] == chunksize
def bulk_export(self, config_ids=None, device_ids=None, package_ids=None, result_ids=None, exclude_captures=False):
"""Bulk export a set of configs, devices, packages and results.
:param config_ids: (optional) Int list of config IDs.
:param device_ids: (optional) Int list of device IDs.
:param package_ids: (optional) Int list of package IDs.
:param result_ids: (optional) Int list of result IDs.
:param exclude_captures: (optional) Exclude capture files if bool `True`.
:rtype: tuple `(io.BytesIO, 'filename')`
"""
if config_ids is None:
config_ids = []
if device_ids is None:
device_ids = []
if package_ids is None:
package_ids = []
if result_ids is None:
result_ids = []
json = {
'configs': map(int, config_ids),
'devices': map(int, device_ids),
'packages': map(int, package_ids),
'results': map(int, result_ids),
'options': {'exclude_captures': exclude_captures}
}
resp = self.service.post(self.base, json=json, stream=True)
b = io.BytesIO()
stream.stream_response_to_file(resp, path=b)
resp.close()
b.seek(0)
return (b, self.service.filename(resp))
def retrieve_file_over_http(self, url, file_path):
'''Get a file from the data api and save it.
Parameters
----------
url : string
Url[1]_ from which to get the file.
file_path : string
Absolute path including the file name to save.
See Also
--------
construct_well_known_file_download_url: Can be used to construct the url.
References
----------
.. [1] Allen Brain Atlas Data Portal: `Downloading a WellKnownFile <http://help.brain-map.org/display/api/Downloading+a+WellKnownFile>`_.
'''
self._file_download_log.info("Downloading URL: %s", url)
from requests_toolbelt import exceptions
from requests_toolbelt.downloadutils import stream
try:
with closing(requests.get(url, stream=True,
timeout=(9.05, 31.1))) as response:
response.raise_for_status()
with open(file_path, 'wb') as f:
stream.stream_response_to_file(response, path=f)
except exceptions.StreamingError as e:
self._file_download_log.error(
"Couldn't retrieve file %s from %s (streaming)." %
(file_path, url))
self.cleanup_truncated_file(file_path)
raise
except requests.exceptions.ConnectionError as e:
self._file_download_log.error(
"Couldn't retrieve file %s from %s (connection)." %
(file_path, url))
self.cleanup_truncated_file(file_path)
raise
except requests.exceptions.ReadTimeout as e:
self._file_download_log.error(
"Couldn't retrieve file %s from %s (timeout)." %
(file_path, url))
self.cleanup_truncated_file(file_path)
raise
except requests.exceptions.RequestException as e:
self._file_download_log.error(
"Couldn't retrieve file %s from %s (request)." %
(file_path, url))
self.cleanup_truncated_file(file_path)
raise
except Exception as e:
self._file_download_log.error("Couldn't retrieve file %s from %s" %
(file_path, url))
self.cleanup_truncated_file(file_path)
raise
def export(self, base, id, format='gz', params=None): # pylint: disable=invalid-name,redefined-builtin
if params is None:
params = {}
params.update({'format': format})
resp = self.get(base + str(id) + '/', params=params, stream=True)
b = io.BytesIO()
stream.stream_response_to_file(resp, path=b)
resp.close()
b.seek(0)
return (b, self.filename(resp))
def bulk_export(self, base, ids, params=None):
if params is None:
params = {}
params.update({'bulk': 'export', 'ids': map(str, ids)})
resp = self.get(base, params=params, stream=True)
b = io.BytesIO()
stream.stream_response_to_file(resp, path=b)
resp.close()
b.seek(0)
return (b, self.filename(resp))
def diagnostics(self):
"""Get system diagnostics from cdrouter-diag output.
:rtype: tuple `(io.BytesIO, 'filename')`
"""
resp = self.service.get(self.base+'diag/', stream=True)
b = io.BytesIO()
stream.stream_response_to_file(resp, path=b)
resp.close()
b.seek(0)
return (b, self.service.filename(resp))
def test_stream_response_to_file_like_object():
s = requests.Session()
recorder = get_betamax(s)
url = ('https://api.github.com/repos/sigmavirus24/github3.py/releases/'
'assets/37944')
file_obj = io.BytesIO()
with recorder.use_cassette('stream_response_to_file', **preserve_bytes):
r = s.get(url, headers={'Accept': 'application/octet-stream'},
stream=True)
stream.stream_response_to_file(r, path=file_obj)
assert 0 < file_obj.tell()
def test_stream_response_to_specific_filename():
s = requests.Session()
recorder = get_betamax(s)
url = ('https://api.github.com/repos/sigmavirus24/github3.py/releases/'
'assets/37944')
filename = 'github3.py.whl'
with recorder.use_cassette('stream_response_to_file', **preserve_bytes):
r = s.get(url, headers={'Accept': 'application/octet-stream'},
stream=True)
stream.stream_response_to_file(r, path=filename)
assert os.path.exists(filename)
os.unlink(filename)
def get_logdir_file(self, id, filename): # pylint: disable=invalid-name,redefined-builtin
"""Download a logdir file.
:param id: Result ID as an int.
:param filename: Logdir filename as string.
:rtype: tuple `(io.BytesIO, 'filename')`
"""
resp = self.service.get(self.base+str(id)+'/logdir/'+filename+'/', stream=True)
b = io.BytesIO()
stream.stream_response_to_file(resp, path=b)
resp.close()
b.seek(0)
return (b, self.service.filename(resp))
def test_stream_response_to_file_like_object():
s = requests.Session()
recorder = get_betamax(s)
url = ('https://api.github.com/repos/sigmavirus24/github3.py/releases/'
'assets/37944')
file_obj = io.BytesIO()
with recorder.use_cassette('stream_response_to_file', **preserve_bytes):
r = s.get(url,
headers={'Accept': 'application/octet-stream'},
stream=True)
stream.stream_response_to_file(r, path=file_obj)
assert 0 < file_obj.tell()
def download(self, id, attid): # pylint: disable=invalid-name,redefined-builtin
"""Download a device's attachment.
:param id: Device ID as an int.
:param attid: Attachment ID as an int.
:rtype: tuple `(io.BytesIO, 'filename')`
"""
resp = self.service.get_id(self._base(id), attid, params={'format': 'download'}, stream=True)
b = io.BytesIO()
stream.stream_response_to_file(resp, path=b)
resp.close()
b.seek(0)
return (b, self.service.filename(resp))
def fetch(self, url, use_proxy=False, allow_ftp=False, allow_file=False,
output_file=None, **request_kwargs):
"""Fetch the URL using a custom HTTP handler supporting timeout.
:param url: The URL to fetch.
:param use_proxy: If True, use Launchpad's configured proxy.
:param allow_ftp: If True, allow ftp:// URLs.
:param allow_file: If True, allow file:// URLs. (Be careful to only
pass this if the URL is trusted.)
:param output_file: If not None, download the response content to
this file object or path.
:param request_kwargs: Additional keyword arguments passed on to
`Session.request`.
"""
self.session = Session()
# Always ignore proxy/authentication settings in the environment; we
# configure that sort of thing explicitly.
self.session.trust_env = False
# Mount our custom adapters.
self.session.mount("https://", CleanableHTTPAdapter())
self.session.mount("http://", CleanableHTTPAdapter())
# We can do FTP, but currently only via an HTTP proxy.
if allow_ftp and use_proxy:
self.session.mount("ftp://", CleanableHTTPAdapter())
if allow_file:
self.session.mount("file://", FileAdapter())
request_kwargs.setdefault("method", "GET")
if use_proxy and config.launchpad.http_proxy:
request_kwargs.setdefault("proxies", {})
request_kwargs["proxies"]["http"] = config.launchpad.http_proxy
request_kwargs["proxies"]["https"] = config.launchpad.http_proxy
if allow_ftp:
request_kwargs["proxies"]["ftp"] = config.launchpad.http_proxy
if output_file is not None:
request_kwargs["stream"] = True
response = self.session.request(url=url, **request_kwargs)
response.raise_for_status()
if output_file is None:
# Make sure the content has been consumed before returning.
response.content
else:
# Download the content to the given file.
stream.stream_response_to_file(response, path=output_file)
# The responses library doesn't persist cookies in the session
# (https://github.com/getsentry/responses/issues/80). Work around
# this.
session_cookies = request_kwargs.get("cookies")
if session_cookies is not None and response.cookies:
session_cookies.update(response.cookies)
return response
def retrieve_file_over_http(self, url, file_path):
'''Get a file from the data api and save it.
Parameters
----------
url : string
Url[1]_ from which to get the file.
file_path : string
Absolute path including the file name to save.
See Also
--------
construct_well_known_file_download_url: Can be used to construct the url.
References
----------
.. [1] Allen Brain Atlas Data Portal: `Downloading a WellKnownFile <http://help.brain-map.org/display/api/Downloading+a+WellKnownFile>`_.
'''
self._log.info("Downloading URL: %s", url)
from requests_toolbelt import exceptions
from requests_toolbelt.downloadutils import stream
try:
with closing(requests.get(url,
stream=True,
timeout=(9.05,31.1))) as response:
response.raise_for_status()
with open(file_path, 'wb') as f:
stream.stream_response_to_file(response, path=f)
except exceptions.StreamingError as e:
self._log.error("Couldn't retrieve file %s from %s (streaming)." % (file_path,url))
self.cleanup_truncated_file(file_path)
raise
except requests.exceptions.ConnectionError as e:
self._log.error("Couldn't retrieve file %s from %s (connection)." % (file_path,url))
self.cleanup_truncated_file(file_path)
raise
except requests.exceptions.ReadTimeout as e:
self._log.error("Couldn't retrieve file %s from %s (timeout)." % (file_path,url))
self.cleanup_truncated_file(file_path)
raise
except requests.exceptions.RequestException as e:
self._log.error("Couldn't retrieve file %s from %s (request)." % (file_path,url))
self.cleanup_truncated_file(file_path)
raise
except Exception as e:
self._log.error("Couldn't retrieve file %s from %s" % (file_path, url))
self.cleanup_truncated_file(file_path)
raise
def download_logdir_archive(self, id, format='zip', exclude_captures=False): # pylint: disable=invalid-name,redefined-builtin
"""Download logdir archive in tgz or zip format.
:param id: Result ID as an int.
:param format: (optional) Format to download, must be string `zip` or `tgz`.
:param exclude_captures: If bool `True`, don't include capture files
:rtype: tuple `(io.BytesIO, 'filename')`
"""
resp = self.service.get(self.base+str(id)+'/logdir/', params={'format': format, 'exclude_captures': exclude_captures}, stream=True)
b = io.BytesIO()
stream.stream_response_to_file(resp, path=b)
resp.close()
b.seek(0)
return (b, self.service.filename(resp))
def test_stream_response_to_specific_filename():
s = requests.Session()
recorder = get_betamax(s)
url = ('https://api.github.com/repos/sigmavirus24/github3.py/releases/'
'assets/37944')
filename = 'github3.py.whl'
with recorder.use_cassette('stream_response_to_file', **preserve_bytes):
r = s.get(url,
headers={'Accept': 'application/octet-stream'},
stream=True)
stream.stream_response_to_file(r, path=filename)
assert os.path.exists(filename)
os.unlink(filename)
def download(self, id, seq, intf, inline=False): # pylint: disable=invalid-name,redefined-builtin
"""Download a capture as a PCAP file.
:param id: Result ID as an int.
:param seq: TestResult sequence ID as an int.
:param intf: Interface name as string.
:param inline: (optional) Use inline version of capture file.
:rtype: tuple `(io.BytesIO, 'filename')`
"""
resp = self.service.get_id(self._base(id, seq), intf, params={'format': 'cap', 'inline': inline}, stream=True)
b = io.BytesIO()
stream.stream_response_to_file(resp, path=b)
resp.close()
b.seek(0)
return (b, self.service.filename(resp))
def download_data(self, path, filename=None, dir=None):
"""
:param path: REST URL API path
:param filename: file name
:param dir: File download directory
:return None
"""
try:
# RESTAPI url path
self.path = path
# Storing GET request response to variable `res`
res = self.__get_request(self.path)
if filename is None:
filename = 'File_' + str(random.randrange(1, 100))
# Saving file to location
if dir:
file = os.path.join(dir, filename)
# Creating downloads folder to project and saving file
else:
self.dir = os.path.join(self.cw, 'downloads')
if os.path.exists(self.dir):
os.makedirs(self.dir)
file = os.path.join(self.dir, filename)
# Saving response data to file
with open(file, 'wb') as d_file:
filename = stream.stream_response_to_file(res, path=d_file)
self.log.info('File'
' Downloaded to location {}'.format(filename))
except Exception as e:
self.log.exception('Got exception file download the file'
' {}'.format(e))
def thumbnail(self, id, attid, size=None): # pylint: disable=invalid-name,redefined-builtin
"""Download thumbnail of a device's attachment. Attachment must be a
GIF, JPEG or PNG image.
:param id: Device ID as an int.
:param attid: Attachment ID as an int.
:param size: (optional) Height in pixels of generated thumbnail.
:rtype: tuple `(io.BytesIO, 'filename')`
"""
resp = self.service.get_id(self._base(id), attid, params={'format': 'thumbnail', 'size': size}, stream=True)
b = io.BytesIO()
stream.stream_response_to_file(resp, path=b)
resp.close()
b.seek(0)
return (b, self.service.filename(resp))
def exportEAD(resourceID, identifier, headers):
if not os.path.exists(os.path.join(EADdestination, resourceID)):
os.makedirs(os.path.join(EADdestination, resourceID))
try:
with open(
os.path.join(EADdestination, resourceID, resourceID + '.xml'),
'wb') as fd:
ead = requests.get(
repositoryBaseURL + 'resource_descriptions/' +
str(identifier) +
'.xml?include_unpublished={exportUnpublished}&include_daos={exportDaos}&numbered_cs={exportNumbered}&print_pdf={exportPdf}'
.format(exportUnpublished=exportUnpublished,
exportDaos=exportDaos,
exportNumbered=exportNumbered,
exportPdf=exportPdf),
headers=headers,
stream=True)
filename = stream.stream_response_to_file(ead, path=fd)
fd.close
logging.info('%s.xml exported to %s', resourceID,
os.path.join(EADdestination, resourceID))
resourceExportList.append(resourceID)
except exceptions.StreamingError as e:
logging.warning(e.message)
#validate here
prettyPrintXml(
os.path.join(EADdestination, resourceID, resourceID + '.xml'),
resourceID, headers)
def download_worker():
logger.info("Downloading updated worker")
r = requests.get(cfg.SERVER_URL + 'worker.zip', stream=True, auth=auth, timeout=60)
r.raise_for_status()
logger.info("Getting the data")
filename = stream.stream_response_to_file(r, path='worker.zip')
logger.info("Download complete")
return filename
def test_stream_response_to_directory():
s = requests.Session()
recorder = get_betamax(s)
url = ('https://api.github.com/repos/sigmavirus24/github3.py/releases/'
'assets/37944')
td = tempfile.mkdtemp()
try:
filename = 'github3.py-0.7.1-py2.py3-none-any.whl'
expected_path = os.path.join(td, filename)
with recorder.use_cassette('stream_response_to_file', **preserve_bytes):
r = s.get(url, headers={'Accept': 'application/octet-stream'},
stream=True)
stream.stream_response_to_file(r, path=td)
assert os.path.exists(expected_path)
finally:
shutil.rmtree(td)
def download_worker():
logger.info("Downloading updated worker")
r = requests.get(cfg.SERVER_URL + 'worker.zip',
stream=True,
auth=auth,
timeout=60)
r.raise_for_status()
logger.info("Getting the data")
filename = stream.stream_response_to_file(r, path='worker.zip')
logger.info("Download complete")
return filename
def test_stream_response_to_directory():
s = requests.Session()
recorder = get_betamax(s)
url = ('https://api.github.com/repos/sigmavirus24/github3.py/releases/'
'assets/37944')
td = tempfile.mkdtemp()
try:
filename = 'github3.py-0.7.1-py2.py3-none-any.whl'
expected_path = os.path.join(td, filename)
with recorder.use_cassette('stream_response_to_file',
**preserve_bytes):
r = s.get(url,
headers={'Accept': 'application/octet-stream'},
stream=True)
stream.stream_response_to_file(r, path=td)
assert os.path.exists(expected_path)
finally:
shutil.rmtree(td)
def test_stream_response_to_existing_file():
s = requests.Session()
recorder = get_betamax(s)
url = ('https://api.github.com/repos/sigmavirus24/github3.py/releases/'
'assets/37944')
filename = 'github3.py.whl'
with open(filename, 'w') as f_existing:
f_existing.write('test')
with recorder.use_cassette('stream_response_to_file', **preserve_bytes):
r = s.get(url, headers={'Accept': 'application/octet-stream'},
stream=True)
try:
stream.stream_response_to_file(r, path=filename)
except stream.exc.StreamingError as e:
assert str(e).startswith('File already exists:')
else:
assert False, "Should have raised a FileExistsError"
finally:
os.unlink(filename)
def stream_file_over_http(url, file_path, timeout=(9.05, 31.1)):
''' Supply an http get request and stream the response to a file.
Parameters
----------
url : str
Send the request to this url
file_path : str
Stream the response to this path
timeout : float or tuple of float, optional
Specify a timeout for the request. If a tuple, specify seperate connect
and read timeouts.
'''
with closing(requests.get(url, stream=True, timeout=timeout)) as response:
response.raise_for_status()
with open(file_path, 'wb') as fil:
stream.stream_response_to_file(response, path=fil)
def exportMETS(doID, d, headers):
if not os.path.exists(os.path.join(METSdestination,doID)):
os.makedirs(os.path.join(METSdestination,doID))
try:
with open(os.path.join(METSdestination,doID,doID+'.xml'), 'wb') as fd:
mets = requests.get(repositoryBaseURL+'digital_objects/mets/'+str(d)+'.xml', headers=headers, stream=True)
filename = stream.stream_response_to_file(mets, path=fd)
fd.close
logging.info('%s.xml exported to %s', doID, os.path.join(METSdestination,doID))
doExportList.append(doID)
except exceptions.StreamingError as e:
logging.warning(e.message)
def test_stream_response_to_existing_file():
s = requests.Session()
recorder = get_betamax(s)
url = ('https://api.github.com/repos/sigmavirus24/github3.py/releases/'
'assets/37944')
filename = 'github3.py.whl'
with open(filename, 'w') as f_existing:
f_existing.write('test')
with recorder.use_cassette('stream_response_to_file', **preserve_bytes):
r = s.get(url,
headers={'Accept': 'application/octet-stream'},
stream=True)
try:
stream.stream_response_to_file(r, path=filename)
except stream.exc.StreamingError as e:
assert str(e).startswith('File already exists:')
else:
assert False, "Should have raised a FileExistsError"
finally:
os.unlink(filename)
def download_replay(resultid, filename):
r = requests.get(cfg.SERVER_URL + 'get_replay/%d/' % resultid, stream=True, auth=auth)
r.raise_for_status()
gz_filename = stream.stream_response_to_file(r, path=filename + '.gz')
logger.info("GUnzipping the replay")
f_in = gzip.open(gz_filename, 'rb')
f_out = open(filename, 'wb')
f_out.writelines(f_in)
f_out.close()
f_in.close()
os.remove(gz_filename)
def exportEAD(resourceID, identifier, headers):
if not os.path.exists(os.path.join(EADdestination,resourceID)):
os.makedirs(os.path.join(EADdestination,resourceID))
try:
with open(os.path.join(EADdestination,resourceID,resourceID+'.xml'), 'wb') as fd:
ead = requests.get(repositoryBaseURL+'resource_descriptions/'+str(identifier)+'.xml?include_unpublished={exportUnpublished}&include_daos={exportDaos}&numbered_cs={exportNumbered}&print_pdf={exportPdf}'.format(exportUnpublished=exportUnpublished, exportDaos=exportDaos, exportNumbered=exportNumbered, exportPdf=exportPdf), headers=headers, stream=True)
filename = stream.stream_response_to_file(ead, path=fd)
fd.close
logging.info('%s.xml exported to %s', resourceID, os.path.join(EADdestination,resourceID))
resourceExportList.append(resourceID)
except exceptions.StreamingError as e:
logging.warning(e.message)
#validate here
prettyPrintXml(os.path.join(EADdestination,resourceID,resourceID+'.xml'), resourceID, headers)
def get_xml(self, fp, format=FORMAT_NATIVE):
"""
Returns the XML metadata for this source, converted to the requested format.
Converted metadata may not contain all the same information as the native format.
:param file fp: A path, or an open file-like object which the content should be written to.
:param str format: desired format for the output. This should be one of the available
formats from :py:meth:`.get_formats`, or :py:attr:`.FORMAT_NATIVE` for the native format.
If you pass this function an open file-like object as the fp parameter, the function will
not close that file for you.
"""
r = self._client.request('GET', getattr(self, format), stream=True)
filename = stream.stream_response_to_file(r, path=fp)
return filename
def download_replay(resultid, filename):
r = requests.get(cfg.SERVER_URL + 'get_replay/%d/' % resultid,
stream=True,
auth=auth)
r.raise_for_status()
gz_filename = stream.stream_response_to_file(r, path=filename + '.gz')
logger.info("GUnzipping the replay")
f_in = gzip.open(gz_filename, 'rb')
f_out = open(filename, 'wb')
f_out.writelines(f_in)
f_out.close()
f_in.close()
os.remove(gz_filename)
def exportMETS(doID, d, headers):
if not os.path.exists(os.path.join(METSdestination, doID)):
os.makedirs(os.path.join(METSdestination, doID))
try:
with open(os.path.join(METSdestination, doID, doID + '.xml'),
'wb') as fd:
mets = requests.get(repositoryBaseURL + 'digital_objects/mets/' +
str(d) + '.xml',
headers=headers,
stream=True)
filename = stream.stream_response_to_file(mets, path=fd)
fd.close
logging.info('%s.xml exported to %s', doID,
os.path.join(METSdestination, doID))
doExportList.append(doID)
except exceptions.StreamingError as e:
logging.warning(e.message)
def stream_download(self, path):
buf = BytesIO()
if self.local_server:
for d in self.local_server.load(path, dtype="byte"):
buf.write(d)
buf.seek(0)
return buf
else:
from requests_toolbelt.downloadutils import stream
json = LoadEntry(path, "byte")._asdict()
# note: reading stuff from the server is always synchronous.
# with (future) sessions, this is forced by the result call.
r = self.session.get(self.stream_url, json=json, stream=True)
assert stream.stream_response_to_file(r.result(), path=buf) is None
buf.seek(0)
return buf
