def save_content(self, file): data = io.BytesIO() stream.stream_response_to_file(file, path=data) created = PhotoStationService.session.query('SYNO.PhotoStation.File', { 'method': 'upload' + self.filetype, 'version': '1', 'dest_folder_path': self.album.path, 'duplicate': 'overwrite', # rename, ignore 'filename': self.filename, 'mtime': str(self.created), 'title': self.title, 'description': self.description, 'ps_username': PhotoStationService.session.username, 'original': (self.filename, data) }) if self.rating > 0 or (self.latitude and self.longitude): self.update({ 'rating': self.rating, 'gps_lat': self.latitude, 'gps_lng': self.longitude }) self.album.add_item(self.filename, self)
def stream_zip_directory_over_http(url, directory, members=None, timeout=(9.05, 31.1)): ''' Supply an http get request and stream the response to a file. Parameters ---------- url : str Send the request to this url directory : str Extract the response to this directory members : list of str, optional Extract only these files timeout : float or tuple of float, optional Specify a timeout for the request. If a tuple, specify seperate connect and read timeouts. ''' buf = io.BytesIO() with closing( requests.get(url, stream=True, timeout=timeout) ) as request: stream.stream_response_to_file( request, buf ) zipper = zipfile.ZipFile(buf) zipper.extractall(path=directory, members=members) zipper.close()
def test_stream_response_to_file_chunksize(): s = requests.Session() recorder = get_betamax(s) url = ('https://api.github.com/repos/sigmavirus24/github3.py/releases/' 'assets/37944') class FileWrapper(io.BytesIO): def __init__(self): super(FileWrapper, self).__init__() self.chunk_sizes = [] def write(self, data): self.chunk_sizes.append(len(data)) return super(FileWrapper, self).write(data) file_obj = FileWrapper() chunksize = 1231 with recorder.use_cassette('stream_response_to_file', **preserve_bytes): r = s.get(url, headers={'Accept': 'application/octet-stream'}, stream=True) stream.stream_response_to_file(r, path=file_obj, chunksize=chunksize) assert 0 < file_obj.tell() assert len(file_obj.chunk_sizes) >= 1 assert file_obj.chunk_sizes[0] == chunksize
def test_stream_response_to_file_chunksize(): s = requests.Session() recorder = get_betamax(s) url = ('https://api.github.com/repos/sigmavirus24/github3.py/releases/' 'assets/37944') class FileWrapper(io.BytesIO): def __init__(self): super(FileWrapper, self).__init__() self.chunk_sizes = [] def write(self, data): self.chunk_sizes.append(len(data)) return super(FileWrapper, self).write(data) file_obj = FileWrapper() chunksize = 1231 with recorder.use_cassette('stream_response_to_file', **preserve_bytes): r = s.get(url, headers={'Accept': 'application/octet-stream'}, stream=True) stream.stream_response_to_file(r, path=file_obj, chunksize=chunksize) assert 0 < file_obj.tell() assert len(file_obj.chunk_sizes) >= 1 assert file_obj.chunk_sizes[0] == chunksize
def bulk_export(self, config_ids=None, device_ids=None, package_ids=None, result_ids=None, exclude_captures=False): """Bulk export a set of configs, devices, packages and results. :param config_ids: (optional) Int list of config IDs. :param device_ids: (optional) Int list of device IDs. :param package_ids: (optional) Int list of package IDs. :param result_ids: (optional) Int list of result IDs. :param exclude_captures: (optional) Exclude capture files if bool `True`. :rtype: tuple `(io.BytesIO, 'filename')` """ if config_ids is None: config_ids = [] if device_ids is None: device_ids = [] if package_ids is None: package_ids = [] if result_ids is None: result_ids = [] json = { 'configs': map(int, config_ids), 'devices': map(int, device_ids), 'packages': map(int, package_ids), 'results': map(int, result_ids), 'options': {'exclude_captures': exclude_captures} } resp = self.service.post(self.base, json=json, stream=True) b = io.BytesIO() stream.stream_response_to_file(resp, path=b) resp.close() b.seek(0) return (b, self.service.filename(resp))
def retrieve_file_over_http(self, url, file_path): '''Get a file from the data api and save it. Parameters ---------- url : string Url[1]_ from which to get the file. file_path : string Absolute path including the file name to save. See Also -------- construct_well_known_file_download_url: Can be used to construct the url. References ---------- .. [1] Allen Brain Atlas Data Portal: `Downloading a WellKnownFile <http://help.brain-map.org/display/api/Downloading+a+WellKnownFile>`_. ''' self._file_download_log.info("Downloading URL: %s", url) from requests_toolbelt import exceptions from requests_toolbelt.downloadutils import stream try: with closing(requests.get(url, stream=True, timeout=(9.05, 31.1))) as response: response.raise_for_status() with open(file_path, 'wb') as f: stream.stream_response_to_file(response, path=f) except exceptions.StreamingError as e: self._file_download_log.error( "Couldn't retrieve file %s from %s (streaming)." % (file_path, url)) self.cleanup_truncated_file(file_path) raise except requests.exceptions.ConnectionError as e: self._file_download_log.error( "Couldn't retrieve file %s from %s (connection)." % (file_path, url)) self.cleanup_truncated_file(file_path) raise except requests.exceptions.ReadTimeout as e: self._file_download_log.error( "Couldn't retrieve file %s from %s (timeout)." % (file_path, url)) self.cleanup_truncated_file(file_path) raise except requests.exceptions.RequestException as e: self._file_download_log.error( "Couldn't retrieve file %s from %s (request)." % (file_path, url)) self.cleanup_truncated_file(file_path) raise except Exception as e: self._file_download_log.error("Couldn't retrieve file %s from %s" % (file_path, url)) self.cleanup_truncated_file(file_path) raise
def export(self, base, id, format='gz', params=None): # pylint: disable=invalid-name,redefined-builtin if params is None: params = {} params.update({'format': format}) resp = self.get(base + str(id) + '/', params=params, stream=True) b = io.BytesIO() stream.stream_response_to_file(resp, path=b) resp.close() b.seek(0) return (b, self.filename(resp))
def bulk_export(self, base, ids, params=None): if params is None: params = {} params.update({'bulk': 'export', 'ids': map(str, ids)}) resp = self.get(base, params=params, stream=True) b = io.BytesIO() stream.stream_response_to_file(resp, path=b) resp.close() b.seek(0) return (b, self.filename(resp))
def diagnostics(self): """Get system diagnostics from cdrouter-diag output. :rtype: tuple `(io.BytesIO, 'filename')` """ resp = self.service.get(self.base+'diag/', stream=True) b = io.BytesIO() stream.stream_response_to_file(resp, path=b) resp.close() b.seek(0) return (b, self.service.filename(resp))
def test_stream_response_to_file_like_object(): s = requests.Session() recorder = get_betamax(s) url = ('https://api.github.com/repos/sigmavirus24/github3.py/releases/' 'assets/37944') file_obj = io.BytesIO() with recorder.use_cassette('stream_response_to_file', **preserve_bytes): r = s.get(url, headers={'Accept': 'application/octet-stream'}, stream=True) stream.stream_response_to_file(r, path=file_obj) assert 0 < file_obj.tell()
def test_stream_response_to_specific_filename(): s = requests.Session() recorder = get_betamax(s) url = ('https://api.github.com/repos/sigmavirus24/github3.py/releases/' 'assets/37944') filename = 'github3.py.whl' with recorder.use_cassette('stream_response_to_file', **preserve_bytes): r = s.get(url, headers={'Accept': 'application/octet-stream'}, stream=True) stream.stream_response_to_file(r, path=filename) assert os.path.exists(filename) os.unlink(filename)
def get_logdir_file(self, id, filename): # pylint: disable=invalid-name,redefined-builtin """Download a logdir file. :param id: Result ID as an int. :param filename: Logdir filename as string. :rtype: tuple `(io.BytesIO, 'filename')` """ resp = self.service.get(self.base+str(id)+'/logdir/'+filename+'/', stream=True) b = io.BytesIO() stream.stream_response_to_file(resp, path=b) resp.close() b.seek(0) return (b, self.service.filename(resp))
def test_stream_response_to_file_like_object(): s = requests.Session() recorder = get_betamax(s) url = ('https://api.github.com/repos/sigmavirus24/github3.py/releases/' 'assets/37944') file_obj = io.BytesIO() with recorder.use_cassette('stream_response_to_file', **preserve_bytes): r = s.get(url, headers={'Accept': 'application/octet-stream'}, stream=True) stream.stream_response_to_file(r, path=file_obj) assert 0 < file_obj.tell()
def download(self, id, attid): # pylint: disable=invalid-name,redefined-builtin """Download a device's attachment. :param id: Device ID as an int. :param attid: Attachment ID as an int. :rtype: tuple `(io.BytesIO, 'filename')` """ resp = self.service.get_id(self._base(id), attid, params={'format': 'download'}, stream=True) b = io.BytesIO() stream.stream_response_to_file(resp, path=b) resp.close() b.seek(0) return (b, self.service.filename(resp))
def fetch(self, url, use_proxy=False, allow_ftp=False, allow_file=False, output_file=None, **request_kwargs): """Fetch the URL using a custom HTTP handler supporting timeout. :param url: The URL to fetch. :param use_proxy: If True, use Launchpad's configured proxy. :param allow_ftp: If True, allow ftp:// URLs. :param allow_file: If True, allow file:// URLs. (Be careful to only pass this if the URL is trusted.) :param output_file: If not None, download the response content to this file object or path. :param request_kwargs: Additional keyword arguments passed on to `Session.request`. """ self.session = Session() # Always ignore proxy/authentication settings in the environment; we # configure that sort of thing explicitly. self.session.trust_env = False # Mount our custom adapters. self.session.mount("https://", CleanableHTTPAdapter()) self.session.mount("http://", CleanableHTTPAdapter()) # We can do FTP, but currently only via an HTTP proxy. if allow_ftp and use_proxy: self.session.mount("ftp://", CleanableHTTPAdapter()) if allow_file: self.session.mount("file://", FileAdapter()) request_kwargs.setdefault("method", "GET") if use_proxy and config.launchpad.http_proxy: request_kwargs.setdefault("proxies", {}) request_kwargs["proxies"]["http"] = config.launchpad.http_proxy request_kwargs["proxies"]["https"] = config.launchpad.http_proxy if allow_ftp: request_kwargs["proxies"]["ftp"] = config.launchpad.http_proxy if output_file is not None: request_kwargs["stream"] = True response = self.session.request(url=url, **request_kwargs) response.raise_for_status() if output_file is None: # Make sure the content has been consumed before returning. response.content else: # Download the content to the given file. stream.stream_response_to_file(response, path=output_file) # The responses library doesn't persist cookies in the session # (https://github.com/getsentry/responses/issues/80). Work around # this. session_cookies = request_kwargs.get("cookies") if session_cookies is not None and response.cookies: session_cookies.update(response.cookies) return response
def retrieve_file_over_http(self, url, file_path): '''Get a file from the data api and save it. Parameters ---------- url : string Url[1]_ from which to get the file. file_path : string Absolute path including the file name to save. See Also -------- construct_well_known_file_download_url: Can be used to construct the url. References ---------- .. [1] Allen Brain Atlas Data Portal: `Downloading a WellKnownFile <http://help.brain-map.org/display/api/Downloading+a+WellKnownFile>`_. ''' self._log.info("Downloading URL: %s", url) from requests_toolbelt import exceptions from requests_toolbelt.downloadutils import stream try: with closing(requests.get(url, stream=True, timeout=(9.05,31.1))) as response: response.raise_for_status() with open(file_path, 'wb') as f: stream.stream_response_to_file(response, path=f) except exceptions.StreamingError as e: self._log.error("Couldn't retrieve file %s from %s (streaming)." % (file_path,url)) self.cleanup_truncated_file(file_path) raise except requests.exceptions.ConnectionError as e: self._log.error("Couldn't retrieve file %s from %s (connection)." % (file_path,url)) self.cleanup_truncated_file(file_path) raise except requests.exceptions.ReadTimeout as e: self._log.error("Couldn't retrieve file %s from %s (timeout)." % (file_path,url)) self.cleanup_truncated_file(file_path) raise except requests.exceptions.RequestException as e: self._log.error("Couldn't retrieve file %s from %s (request)." % (file_path,url)) self.cleanup_truncated_file(file_path) raise except Exception as e: self._log.error("Couldn't retrieve file %s from %s" % (file_path, url)) self.cleanup_truncated_file(file_path) raise
def download_logdir_archive(self, id, format='zip', exclude_captures=False): # pylint: disable=invalid-name,redefined-builtin """Download logdir archive in tgz or zip format. :param id: Result ID as an int. :param format: (optional) Format to download, must be string `zip` or `tgz`. :param exclude_captures: If bool `True`, don't include capture files :rtype: tuple `(io.BytesIO, 'filename')` """ resp = self.service.get(self.base+str(id)+'/logdir/', params={'format': format, 'exclude_captures': exclude_captures}, stream=True) b = io.BytesIO() stream.stream_response_to_file(resp, path=b) resp.close() b.seek(0) return (b, self.service.filename(resp))
def test_stream_response_to_specific_filename(): s = requests.Session() recorder = get_betamax(s) url = ('https://api.github.com/repos/sigmavirus24/github3.py/releases/' 'assets/37944') filename = 'github3.py.whl' with recorder.use_cassette('stream_response_to_file', **preserve_bytes): r = s.get(url, headers={'Accept': 'application/octet-stream'}, stream=True) stream.stream_response_to_file(r, path=filename) assert os.path.exists(filename) os.unlink(filename)
def download(self, id, seq, intf, inline=False): # pylint: disable=invalid-name,redefined-builtin """Download a capture as a PCAP file. :param id: Result ID as an int. :param seq: TestResult sequence ID as an int. :param intf: Interface name as string. :param inline: (optional) Use inline version of capture file. :rtype: tuple `(io.BytesIO, 'filename')` """ resp = self.service.get_id(self._base(id, seq), intf, params={'format': 'cap', 'inline': inline}, stream=True) b = io.BytesIO() stream.stream_response_to_file(resp, path=b) resp.close() b.seek(0) return (b, self.service.filename(resp))
def download_data(self, path, filename=None, dir=None): """ :param path: REST URL API path :param filename: file name :param dir: File download directory :return None """ try: # RESTAPI url path self.path = path # Storing GET request response to variable `res` res = self.__get_request(self.path) if filename is None: filename = 'File_' + str(random.randrange(1, 100)) # Saving file to location if dir: file = os.path.join(dir, filename) # Creating downloads folder to project and saving file else: self.dir = os.path.join(self.cw, 'downloads') if os.path.exists(self.dir): os.makedirs(self.dir) file = os.path.join(self.dir, filename) # Saving response data to file with open(file, 'wb') as d_file: filename = stream.stream_response_to_file(res, path=d_file) self.log.info('File' ' Downloaded to location {}'.format(filename)) except Exception as e: self.log.exception('Got exception file download the file' ' {}'.format(e))
def thumbnail(self, id, attid, size=None): # pylint: disable=invalid-name,redefined-builtin """Download thumbnail of a device's attachment. Attachment must be a GIF, JPEG or PNG image. :param id: Device ID as an int. :param attid: Attachment ID as an int. :param size: (optional) Height in pixels of generated thumbnail. :rtype: tuple `(io.BytesIO, 'filename')` """ resp = self.service.get_id(self._base(id), attid, params={'format': 'thumbnail', 'size': size}, stream=True) b = io.BytesIO() stream.stream_response_to_file(resp, path=b) resp.close() b.seek(0) return (b, self.service.filename(resp))
def exportEAD(resourceID, identifier, headers): if not os.path.exists(os.path.join(EADdestination, resourceID)): os.makedirs(os.path.join(EADdestination, resourceID)) try: with open( os.path.join(EADdestination, resourceID, resourceID + '.xml'), 'wb') as fd: ead = requests.get( repositoryBaseURL + 'resource_descriptions/' + str(identifier) + '.xml?include_unpublished={exportUnpublished}&include_daos={exportDaos}&numbered_cs={exportNumbered}&print_pdf={exportPdf}' .format(exportUnpublished=exportUnpublished, exportDaos=exportDaos, exportNumbered=exportNumbered, exportPdf=exportPdf), headers=headers, stream=True) filename = stream.stream_response_to_file(ead, path=fd) fd.close logging.info('%s.xml exported to %s', resourceID, os.path.join(EADdestination, resourceID)) resourceExportList.append(resourceID) except exceptions.StreamingError as e: logging.warning(e.message) #validate here prettyPrintXml( os.path.join(EADdestination, resourceID, resourceID + '.xml'), resourceID, headers)
def download_worker(): logger.info("Downloading updated worker") r = requests.get(cfg.SERVER_URL + 'worker.zip', stream=True, auth=auth, timeout=60) r.raise_for_status() logger.info("Getting the data") filename = stream.stream_response_to_file(r, path='worker.zip') logger.info("Download complete") return filename
def test_stream_response_to_directory(): s = requests.Session() recorder = get_betamax(s) url = ('https://api.github.com/repos/sigmavirus24/github3.py/releases/' 'assets/37944') td = tempfile.mkdtemp() try: filename = 'github3.py-0.7.1-py2.py3-none-any.whl' expected_path = os.path.join(td, filename) with recorder.use_cassette('stream_response_to_file', **preserve_bytes): r = s.get(url, headers={'Accept': 'application/octet-stream'}, stream=True) stream.stream_response_to_file(r, path=td) assert os.path.exists(expected_path) finally: shutil.rmtree(td)
def download_worker(): logger.info("Downloading updated worker") r = requests.get(cfg.SERVER_URL + 'worker.zip', stream=True, auth=auth, timeout=60) r.raise_for_status() logger.info("Getting the data") filename = stream.stream_response_to_file(r, path='worker.zip') logger.info("Download complete") return filename
def test_stream_response_to_directory(): s = requests.Session() recorder = get_betamax(s) url = ('https://api.github.com/repos/sigmavirus24/github3.py/releases/' 'assets/37944') td = tempfile.mkdtemp() try: filename = 'github3.py-0.7.1-py2.py3-none-any.whl' expected_path = os.path.join(td, filename) with recorder.use_cassette('stream_response_to_file', **preserve_bytes): r = s.get(url, headers={'Accept': 'application/octet-stream'}, stream=True) stream.stream_response_to_file(r, path=td) assert os.path.exists(expected_path) finally: shutil.rmtree(td)
def test_stream_response_to_existing_file(): s = requests.Session() recorder = get_betamax(s) url = ('https://api.github.com/repos/sigmavirus24/github3.py/releases/' 'assets/37944') filename = 'github3.py.whl' with open(filename, 'w') as f_existing: f_existing.write('test') with recorder.use_cassette('stream_response_to_file', **preserve_bytes): r = s.get(url, headers={'Accept': 'application/octet-stream'}, stream=True) try: stream.stream_response_to_file(r, path=filename) except stream.exc.StreamingError as e: assert str(e).startswith('File already exists:') else: assert False, "Should have raised a FileExistsError" finally: os.unlink(filename)
def stream_file_over_http(url, file_path, timeout=(9.05, 31.1)): ''' Supply an http get request and stream the response to a file. Parameters ---------- url : str Send the request to this url file_path : str Stream the response to this path timeout : float or tuple of float, optional Specify a timeout for the request. If a tuple, specify seperate connect and read timeouts. ''' with closing(requests.get(url, stream=True, timeout=timeout)) as response: response.raise_for_status() with open(file_path, 'wb') as fil: stream.stream_response_to_file(response, path=fil)
def exportMETS(doID, d, headers): if not os.path.exists(os.path.join(METSdestination,doID)): os.makedirs(os.path.join(METSdestination,doID)) try: with open(os.path.join(METSdestination,doID,doID+'.xml'), 'wb') as fd: mets = requests.get(repositoryBaseURL+'digital_objects/mets/'+str(d)+'.xml', headers=headers, stream=True) filename = stream.stream_response_to_file(mets, path=fd) fd.close logging.info('%s.xml exported to %s', doID, os.path.join(METSdestination,doID)) doExportList.append(doID) except exceptions.StreamingError as e: logging.warning(e.message)
def test_stream_response_to_existing_file(): s = requests.Session() recorder = get_betamax(s) url = ('https://api.github.com/repos/sigmavirus24/github3.py/releases/' 'assets/37944') filename = 'github3.py.whl' with open(filename, 'w') as f_existing: f_existing.write('test') with recorder.use_cassette('stream_response_to_file', **preserve_bytes): r = s.get(url, headers={'Accept': 'application/octet-stream'}, stream=True) try: stream.stream_response_to_file(r, path=filename) except stream.exc.StreamingError as e: assert str(e).startswith('File already exists:') else: assert False, "Should have raised a FileExistsError" finally: os.unlink(filename)
def download_replay(resultid, filename): r = requests.get(cfg.SERVER_URL + 'get_replay/%d/' % resultid, stream=True, auth=auth) r.raise_for_status() gz_filename = stream.stream_response_to_file(r, path=filename + '.gz') logger.info("GUnzipping the replay") f_in = gzip.open(gz_filename, 'rb') f_out = open(filename, 'wb') f_out.writelines(f_in) f_out.close() f_in.close() os.remove(gz_filename)
def exportEAD(resourceID, identifier, headers): if not os.path.exists(os.path.join(EADdestination,resourceID)): os.makedirs(os.path.join(EADdestination,resourceID)) try: with open(os.path.join(EADdestination,resourceID,resourceID+'.xml'), 'wb') as fd: ead = requests.get(repositoryBaseURL+'resource_descriptions/'+str(identifier)+'.xml?include_unpublished={exportUnpublished}&include_daos={exportDaos}&numbered_cs={exportNumbered}&print_pdf={exportPdf}'.format(exportUnpublished=exportUnpublished, exportDaos=exportDaos, exportNumbered=exportNumbered, exportPdf=exportPdf), headers=headers, stream=True) filename = stream.stream_response_to_file(ead, path=fd) fd.close logging.info('%s.xml exported to %s', resourceID, os.path.join(EADdestination,resourceID)) resourceExportList.append(resourceID) except exceptions.StreamingError as e: logging.warning(e.message) #validate here prettyPrintXml(os.path.join(EADdestination,resourceID,resourceID+'.xml'), resourceID, headers)
def get_xml(self, fp, format=FORMAT_NATIVE): """ Returns the XML metadata for this source, converted to the requested format. Converted metadata may not contain all the same information as the native format. :param file fp: A path, or an open file-like object which the content should be written to. :param str format: desired format for the output. This should be one of the available formats from :py:meth:`.get_formats`, or :py:attr:`.FORMAT_NATIVE` for the native format. If you pass this function an open file-like object as the fp parameter, the function will not close that file for you. """ r = self._client.request('GET', getattr(self, format), stream=True) filename = stream.stream_response_to_file(r, path=fp) return filename
def download_replay(resultid, filename): r = requests.get(cfg.SERVER_URL + 'get_replay/%d/' % resultid, stream=True, auth=auth) r.raise_for_status() gz_filename = stream.stream_response_to_file(r, path=filename + '.gz') logger.info("GUnzipping the replay") f_in = gzip.open(gz_filename, 'rb') f_out = open(filename, 'wb') f_out.writelines(f_in) f_out.close() f_in.close() os.remove(gz_filename)
def exportMETS(doID, d, headers): if not os.path.exists(os.path.join(METSdestination, doID)): os.makedirs(os.path.join(METSdestination, doID)) try: with open(os.path.join(METSdestination, doID, doID + '.xml'), 'wb') as fd: mets = requests.get(repositoryBaseURL + 'digital_objects/mets/' + str(d) + '.xml', headers=headers, stream=True) filename = stream.stream_response_to_file(mets, path=fd) fd.close logging.info('%s.xml exported to %s', doID, os.path.join(METSdestination, doID)) doExportList.append(doID) except exceptions.StreamingError as e: logging.warning(e.message)
def stream_download(self, path): buf = BytesIO() if self.local_server: for d in self.local_server.load(path, dtype="byte"): buf.write(d) buf.seek(0) return buf else: from requests_toolbelt.downloadutils import stream json = LoadEntry(path, "byte")._asdict() # note: reading stuff from the server is always synchronous. # with (future) sessions, this is forced by the result call. r = self.session.get(self.stream_url, json=json, stream=True) assert stream.stream_response_to_file(r.result(), path=buf) is None buf.seek(0) return buf