def test_retry(mock_retry): """ Test that you can use Fido.fetch to retry failed downloads. """ res = Results() res.data.append("/this/worked.fits") err1 = FailedDownload("This is not a filename", "http://not.url/test", None) err2 = FailedDownload("This is not a filename2", "http://not.url/test2", None) res.errors.append(err1) res.errors.append(err2) mock_retry.return_value._errors += [err2] res2 = Fido.fetch(res, Results(["/this/also/worked.fits"])) assert res2 is not res # Assert that the result of retry ends up in the returned Results() object assert res2.data == [ "/this/worked.fits", "/tmp/test", "/this/also/worked.fits", "/tmp/test" ] assert res2.errors == [err2, err2]
def results_generator(dl): http = list(dl.http_queue._queue) ftp = list(dl.ftp_queue._queue) outputs = [] for url in http + ftp: outputs.append(pathlib.Path(url.keywords['url'].split("/")[-1])) return Results(outputs)
def results_generator(dl): http = dl.http_queue ftp = dl.ftp_queue # Handle compatibility with parfive 1.0 if not isinstance(dl.http_queue, list): http = list(dl.http_queue._queue) ftp = list(dl.ftp_queue._queue) outputs = [] for url in http + ftp: outputs.append(pathlib.Path(url.keywords['url'].split("/")[-1])) return Results(outputs)
def test_no_download(client): """ Test for https://github.com/sunpy/sunpy/issues/3292 """ class MockDownloader: download_called = False def __init__(self): pass def download(self, *args, **kwargs): self.download_called = True # this should fail stereo = (core_attrs.Detector('STEREO_B') & core_attrs.Instrument('EUVI') & core_attrs.Time('1900-01-01', '1900-01-01T00:10:00')) qr = client.search(stereo, response_format="table") downloader = MockDownloader() res = client.fetch(qr, wait=False, downloader=downloader) assert downloader.download_called is False assert res == Results()
def get_request(self, requests, path=None, overwrite=False, progress=True, downloader=None, wait=True): """ Query JSOC to see if the request(s) is ready for download. If the request is ready for download, it will then download it. Parameters ---------- requests : `~drms.ExportRequest`, `str`, `list` `~drms.ExportRequest` objects or `str` request IDs or lists returned by `~sunpy.net.jsoc.jsoc.JSOCClient.request_data`. path : `str` Path to save data to, defaults to SunPy download dir. progress : `bool`, optional If `True` show a progress bar showing how many of the total files have been downloaded. If `False`, no progress bar will be shown. overwrite : `bool` or `str`, optional Determine how to handle downloading if a file already exists with the same name. If `False` the file download will be skipped and the path returned to the existing file, if `True` the file will be downloaded and the existing file will be overwritten, if `'unique'` the filename will be modified to be unique. downloader : `parfive.Downloader`, optional The download manager to use. wait : `bool`, optional If `False` ``downloader.download()`` will not be called. Only has any effect if `downloader` is not `None`. Returns ------- res: `~sunpy.net.download.Results` A `~sunpy.net.download.Results` instance or `None` if no URLs to download """ c = drms.Client() # Convert Responses to a list if not already if isinstance(requests, str) or not isiterable(requests): requests = [requests] # Ensure all the requests are drms ExportRequest objects for i, request in enumerate(requests): if isinstance(request, str): r = c.export_from_id(request) requests[i] = r # We only download if all are finished if not all([r.has_succeeded() for r in requests]): raise NotExportedError("Can not download as not all the requests " "have been exported for download yet.") # Ensure path has a {file} in it if path is None: default_dir = config.get("downloads", "download_dir") path = os.path.join(default_dir, '{file}') elif isinstance(path, str) and '{file}' not in path: path = os.path.join(path, '{file}') paths = [] for request in requests: for filename in request.data['filename']: # Ensure we don't duplicate the file extension ext = os.path.splitext(filename)[1] if path.endswith(ext): fname = path.strip(ext) else: fname = path fname = fname.format(file=filename) fname = os.path.expanduser(fname) paths.append(fname) dl_set = True if not downloader: dl_set = False downloader = Downloader(progress=progress, overwrite=overwrite) urls = [] for request in requests: if request.status == 0: for index, data in request.data.iterrows(): url_dir = request.request_url + '/' urls.append(urllib.parse.urljoin(url_dir, data['filename'])) if urls: if progress: print_message = "{0} URLs found for download. Full request totalling {1}MB" print(print_message.format(len(urls), request._d['size'])) for aurl, fname in zip(urls, paths): downloader.enqueue_file(aurl, filename=fname) if dl_set and not wait: return Results() results = downloader.download() return results
def test_mixed_retry_error(): with pytest.raises(TypeError): Fido.fetch([], Results())
def test_downloader_type_error(): with pytest.raises(TypeError): Fido.fetch([], downloader=Results())
@settings(deadline=50000) @given(offline_query()) def test_repr2(query): res = Fido.search(query) for rep_meth in (res.__repr__, res.__str__, res._repr_html_): if len(res) == 1: assert "Provider" in rep_meth() assert "Providers" not in rep_meth() else: assert "Provider" not in rep_meth() assert "Providers" in rep_meth() @mock.patch("parfive.Downloader.download", return_value=Results(["/tmp/test"])) def test_retry(mock_retry): """ Test that you can use Fido.fetch to retry failed downloads. """ res = Results() res.data.append("/this/worked.fits") err1 = FailedDownload("This is not a filename", "http://not.url/test", None) err2 = FailedDownload("This is not a filename2", "http://not.url/test2", None) res.errors.append(err1) res.errors.append(err2) mock_retry.return_value._errors += [err2]
def fetch(self, *query_results, path=None, max_conn=5, progress=True, overwrite=False, downloader=None, **kwargs): """ Download the records represented by `~sunpy.net.fido_factory.UnifiedResponse` objects. Parameters ---------- query_results : `sunpy.net.fido_factory.UnifiedResponse` Container returned by query method, or multiple. path : `str` The directory to retrieve the files into. Can refer to any fields in `UnifiedResponse.response_block_properties` via string formatting, moreover the file-name of the file downloaded can be referred to as file, e.g. "{source}/{instrument}/{time.start}/{file}". max_conn : `int`, optional The number of parallel download slots. progress : `bool`, optional If `True` show a progress bar showing how many of the total files have been downloaded. If `False`, no progress bars will be shown at all. overwrite : `bool` or `str`, optional Determine how to handle downloading if a file already exists with the same name. If `False` the file download will be skipped and the path returned to the existing file, if `True` the file will be downloaded and the existing file will be overwritten, if `'unique'` the filename will be modified to be unique. downloader : `parfive.Downloader`, optional The download manager to use. If specified the ``max_conn``, ``progress`` and ``overwrite`` arguments are ignored. Returns ------- `parfive.Results` Examples -------- >>> from sunpy.net.attrs import Time, Instrument >>> unifresp = Fido.search(Time('2012/3/4','2012/3/5'), Instrument('EIT')) # doctest: +REMOTE_DATA >>> filepaths = Fido.fetch(unifresp) # doctest: +SKIP If any downloads fail, they can be retried by passing the `parfive.Results` object back into ``fetch``. >>> filepaths = Fido.fetch(filepaths) # doctest: +SKIP """ if path is not None: exists = list(filter(lambda p: p.exists(), Path(path).parents)) if not os.access(exists[0], os.W_OK): raise PermissionError('You do not have permission to write' f' to the directory {exists[0]}.') if "wait" in kwargs: raise ValueError("wait is not a valid keyword argument to Fido.fetch.") if downloader is None: downloader = Downloader(max_conn=max_conn, progress=progress, overwrite=overwrite) elif not isinstance(downloader, Downloader): raise TypeError("The downloader argument must be a parfive.Downloader object.") # Handle retrying failed downloads retries = [isinstance(arg, Results) for arg in query_results] if all(retries): results = Results() for retry in query_results: dr = downloader.retry(retry) results.data += dr.data results._errors += dr._errors return results elif any(retries): raise TypeError("If any arguments to fetch are " "`parfive.Results` objects, all arguments must be.") reslist = [] for query_result in query_results: for block in query_result.responses: reslist.append(block.client.fetch(block, path=path, downloader=downloader, wait=False, **kwargs)) results = downloader.download() # Combine the results objects from all the clients into one Results # object. for result in reslist: if result is None: continue if not isinstance(result, Results): raise TypeError( "If wait is False a client must return a parfive.Downloader and either None" " or a parfive.Results object.") results.data += result.data results._errors += result.errors return results
def download_all(self, response, methods, downloader, path, qr, info=None): results = Results() GET_VERSION = [ ('0.8', (5, 8)), ('0.7', (1, 4)), ('0.6', (0, 3)), ] for dresponse in response.getdataresponseitem: for version, (from_, to) in GET_VERSION: if getattr(dresponse, version, '0.6') >= version: break else: results.add_error('', UnknownVersion(dresponse)) continue # If from_ and to are uninitialized, the else block of the loop # continues the outer loop and thus this code is never reached. # pylint: disable=W0631 code = (dresponse.status[from_:to] if getattr( dresponse, 'status', None) else '200') if code == '200': for dataitem in dresponse.getdataitem.dataitem: try: self.download(dresponse.method.methodtype[0], dataitem.url, downloader, path, qr[dataitem.fileiditem.fileid[0]]) except NoData: results.add_error('', '', DownloadFailed(dresponse)) continue elif code == '300' or code == '412' or code == '405': if code == '300': try: methods = self.multiple_choices( dresponse.method.methodtype, dresponse) except NoData: results.add_error('', '', MultipleChoices(dresponse)) continue elif code == '412': try: info = self.missing_information(info, dresponse.info) except NoData: results.add_error('', '', MissingInformation(dresponse)) continue elif code == '405': try: methods = self.unknown_method(dresponse) except NoData: results.add_error('', '', UnknownMethod(dresponse)) continue files = [] for dataitem in dresponse.getdataitem.dataitem: files.extend(dataitem.fileiditem.fileid) request = self.create_getdatarequest( {dresponse.provider: files}, methods, info) self.download_all(self.api.service.GetData(request), methods, downloader, path, qr, info) else: results.add_error('', '', UnknownStatus(dresponse)) return results
def fetch(self, query_response, path=None, methods=None, site=None, progress=True, overwrite=False, downloader=None, wait=True): """ Download data specified in the query_response. Parameters ---------- query_response : sunpy.net.vso.QueryResponse QueryResponse containing the items to be downloaded. path : str Specify where the data is to be downloaded. Can refer to arbitrary fields of the QueryResponseItem (instrument, source, time, ...) via string formatting, moreover the file-name of the file downloaded can be referred to as file, e.g. "{source}/{instrument}/{time.start}/{file}". methods : {list of str} Download methods, defaults to URL-FILE_Rice then URL-FILE. Methods are a concatenation of one PREFIX followed by any number of SUFFIXES i.e. `PREFIX-SUFFIX_SUFFIX2_SUFFIX3`. The full list of `PREFIXES <https://sdac.virtualsolar.org/cgi/show_details?keyword=METHOD_PREFIX>`_ and `SUFFIXES <https://sdac.virtualsolar.org/cgi/show_details?keyword=METHOD_SUFFIX>`_ are listed on the VSO site. site : str There are a number of caching mirrors for SDO and other instruments, some available ones are listed below. =============== ======================================================== NSO National Solar Observatory, Tucson (US) SAO (aka CFA) Smithonian Astronomical Observatory, Harvard U. (US) SDAC (aka GSFC) Solar Data Analysis Center, NASA/GSFC (US) ROB Royal Observatory of Belgium (Belgium) MPS Max Planck Institute for Solar System Research (Germany) UCLan University of Central Lancashire (UK) IAS Institut Aeronautique et Spatial (France) KIS Kiepenheuer-Institut fur Sonnenphysik Germany) NMSU New Mexico State University (US) =============== ======================================================== progress : `bool`, optional If `True` show a progress bar showing how many of the total files have been downloaded. If `False`, no progress bars will be shown at all. overwrite : `bool` or `str`, optional Determine how to handle downloading if a file already exists with the same name. If `False` the file download will be skipped and the path returned to the existing file, if `True` the file will be downloaded and the existing file will be overwritten, if `'unique'` the filename will be modified to be unique. downloader : `parfive.Downloader`, optional The download manager to use. wait : `bool`, optional If `False` ``downloader.download()`` will not be called. Only has any effect if `downloader` is not `None`. Returns ------- out : `parfive.Results` Object that supplies a list of filenames and any errors. Examples -------- >>> files = fetch(qr) # doctest:+SKIP """ if path is None: path = os.path.join(config.get('downloads', 'download_dir'), '{file}') elif isinstance(path, str) and '{file}' not in path: path = os.path.join(path, '{file}') path = os.path.expanduser(path) dl_set = True if not downloader: dl_set = False downloader = Downloader(progress=progress) fileids = VSOClient.by_fileid(query_response) if not fileids: return downloader.download() if wait else Results() # Adding the site parameter to the info info = {} if site is not None: info['site'] = site VSOGetDataResponse = self.api.get_type("VSO:VSOGetDataResponse") data_request = self.make_getdatarequest(query_response, methods, info) data_response = VSOGetDataResponse( self.api.service.GetData(data_request)) err_results = self.download_all(data_response, methods, downloader, path, fileids) if dl_set and not wait: return err_results results = downloader.download() results += err_results results._errors += err_results.errors return results
def download_all(self, response, methods, downloader, path, qr, info=None): results = Results() GET_VERSION = [ ('0.8', (5, 8)), ('0.7', (1, 4)), ('0.6', (0, 3)), ] for dresponse in response.getdataresponseitem: for version, (from_, to) in GET_VERSION: if getattr(dresponse, version, '0.6') >= version: break else: results.add_error('', UnknownVersion(dresponse)) continue # If from_ and to are uninitialized, the else block of the loop # continues the outer loop and thus this code is never reached. # pylint: disable=W0631 code = ( dresponse.status[from_:to] if getattr(dresponse, 'status', None) else '200' ) if code == '200': for dataitem in dresponse.getdataitem.dataitem: try: self.download( dresponse.method.methodtype[0], dataitem.url, downloader, path, qr[dataitem.fileiditem.fileid[0]] ) except NoData: results.add_error('', DownloadFailed(dresponse)) continue elif code == '300' or code == '412' or code == '405': if code == '300': try: methods = self.multiple_choices( dresponse.method.methodtype, dresponse ) except NoData: results.add_error('', MultipleChoices(dresponse)) continue elif code == '412': try: info = self.missing_information( info, dresponse.info ) except NoData: results.add_error('', MissingInformation(dresponse)) continue elif code == '405': try: methods = self.unknown_method(dresponse) except NoData: results.add_error('', UnknownMethod(dresponse)) continue files = [] for dataitem in dresponse.getdataitem.dataitem: files.extend(dataitem.fileiditem.fileid) request = self.create_getdatarequest( {dresponse.provider: files}, methods, info ) self.download_all( self.api.service.GetData(request), methods, downloader, path, qr, info ) else: results.add_error(UnknownStatus(dresponse)) return results