def test_retry(mock_retry):
"""
Test that you can use Fido.fetch to retry failed downloads.
"""
res = Results()
res.data.append("/this/worked.fits")
err1 = FailedDownload("This is not a filename", "http://not.url/test",
None)
err2 = FailedDownload("This is not a filename2", "http://not.url/test2",
None)
res.errors.append(err1)
res.errors.append(err2)
mock_retry.return_value._errors += [err2]
res2 = Fido.fetch(res, Results(["/this/also/worked.fits"]))
assert res2 is not res
# Assert that the result of retry ends up in the returned Results() object
assert res2.data == [
"/this/worked.fits", "/tmp/test", "/this/also/worked.fits", "/tmp/test"
]
assert res2.errors == [err2, err2]
def results_generator(dl):
http = list(dl.http_queue._queue)
ftp = list(dl.ftp_queue._queue)
outputs = []
for url in http + ftp:
outputs.append(pathlib.Path(url.keywords['url'].split("/")[-1]))
return Results(outputs)
def results_generator(dl):
http = dl.http_queue
ftp = dl.ftp_queue
# Handle compatibility with parfive 1.0
if not isinstance(dl.http_queue, list):
http = list(dl.http_queue._queue)
ftp = list(dl.ftp_queue._queue)
outputs = []
for url in http + ftp:
outputs.append(pathlib.Path(url.keywords['url'].split("/")[-1]))
return Results(outputs)
def test_no_download(client):
"""
Test for https://github.com/sunpy/sunpy/issues/3292
"""
class MockDownloader:
download_called = False
def __init__(self):
pass
def download(self, *args, **kwargs):
self.download_called = True
# this should fail
stereo = (core_attrs.Detector('STEREO_B') & core_attrs.Instrument('EUVI')
& core_attrs.Time('1900-01-01', '1900-01-01T00:10:00'))
qr = client.search(stereo, response_format="table")
downloader = MockDownloader()
res = client.fetch(qr, wait=False, downloader=downloader)
assert downloader.download_called is False
assert res == Results()
def get_request(self,
requests,
path=None,
overwrite=False,
progress=True,
downloader=None,
wait=True):
"""
Query JSOC to see if the request(s) is ready for download.
If the request is ready for download, it will then download it.
Parameters
----------
requests : `~drms.ExportRequest`, `str`, `list`
`~drms.ExportRequest` objects or `str` request IDs or lists
returned by `~sunpy.net.jsoc.jsoc.JSOCClient.request_data`.
path : `str`
Path to save data to, defaults to SunPy download dir.
progress : `bool`, optional
If `True` show a progress bar showing how many of the total files
have been downloaded. If `False`, no progress bar will be shown.
overwrite : `bool` or `str`, optional
Determine how to handle downloading if a file already exists with the
same name. If `False` the file download will be skipped and the path
returned to the existing file, if `True` the file will be downloaded
and the existing file will be overwritten, if `'unique'` the filename
will be modified to be unique.
downloader : `parfive.Downloader`, optional
The download manager to use.
wait : `bool`, optional
If `False` ``downloader.download()`` will not be called. Only has
any effect if `downloader` is not `None`.
Returns
-------
res: `~sunpy.net.download.Results`
A `~sunpy.net.download.Results` instance or `None` if no URLs to download
"""
c = drms.Client()
# Convert Responses to a list if not already
if isinstance(requests, str) or not isiterable(requests):
requests = [requests]
# Ensure all the requests are drms ExportRequest objects
for i, request in enumerate(requests):
if isinstance(request, str):
r = c.export_from_id(request)
requests[i] = r
# We only download if all are finished
if not all([r.has_succeeded() for r in requests]):
raise NotExportedError("Can not download as not all the requests "
"have been exported for download yet.")
# Ensure path has a {file} in it
if path is None:
default_dir = config.get("downloads", "download_dir")
path = os.path.join(default_dir, '{file}')
elif isinstance(path, str) and '{file}' not in path:
path = os.path.join(path, '{file}')
paths = []
for request in requests:
for filename in request.data['filename']:
# Ensure we don't duplicate the file extension
ext = os.path.splitext(filename)[1]
if path.endswith(ext):
fname = path.strip(ext)
else:
fname = path
fname = fname.format(file=filename)
fname = os.path.expanduser(fname)
paths.append(fname)
dl_set = True
if not downloader:
dl_set = False
downloader = Downloader(progress=progress, overwrite=overwrite)
urls = []
for request in requests:
if request.status == 0:
for index, data in request.data.iterrows():
url_dir = request.request_url + '/'
urls.append(urllib.parse.urljoin(url_dir,
data['filename']))
if urls:
if progress:
print_message = "{0} URLs found for download. Full request totalling {1}MB"
print(print_message.format(len(urls), request._d['size']))
for aurl, fname in zip(urls, paths):
downloader.enqueue_file(aurl, filename=fname)
if dl_set and not wait:
return Results()
results = downloader.download()
return results
def test_mixed_retry_error():
with pytest.raises(TypeError):
Fido.fetch([], Results())
def test_downloader_type_error():
with pytest.raises(TypeError):
Fido.fetch([], downloader=Results())
@settings(deadline=50000)
@given(offline_query())
def test_repr2(query):
res = Fido.search(query)
for rep_meth in (res.__repr__, res.__str__, res._repr_html_):
if len(res) == 1:
assert "Provider" in rep_meth()
assert "Providers" not in rep_meth()
else:
assert "Provider" not in rep_meth()
assert "Providers" in rep_meth()
@mock.patch("parfive.Downloader.download", return_value=Results(["/tmp/test"]))
def test_retry(mock_retry):
"""
Test that you can use Fido.fetch to retry failed downloads.
"""
res = Results()
res.data.append("/this/worked.fits")
err1 = FailedDownload("This is not a filename", "http://not.url/test",
None)
err2 = FailedDownload("This is not a filename2", "http://not.url/test2",
None)
res.errors.append(err1)
res.errors.append(err2)
mock_retry.return_value._errors += [err2]
def fetch(self, *query_results, path=None, max_conn=5, progress=True,
overwrite=False, downloader=None, **kwargs):
"""
Download the records represented by
`~sunpy.net.fido_factory.UnifiedResponse` objects.
Parameters
----------
query_results : `sunpy.net.fido_factory.UnifiedResponse`
Container returned by query method, or multiple.
path : `str`
The directory to retrieve the files into. Can refer to any fields
in `UnifiedResponse.response_block_properties` via string formatting,
moreover the file-name of the file downloaded can be referred to as file,
e.g. "{source}/{instrument}/{time.start}/{file}".
max_conn : `int`, optional
The number of parallel download slots.
progress : `bool`, optional
If `True` show a progress bar showing how many of the total files
have been downloaded. If `False`, no progress bars will be shown at all.
overwrite : `bool` or `str`, optional
Determine how to handle downloading if a file already exists with the
same name. If `False` the file download will be skipped and the path
returned to the existing file, if `True` the file will be downloaded
and the existing file will be overwritten, if `'unique'` the filename
will be modified to be unique.
downloader : `parfive.Downloader`, optional
The download manager to use. If specified the ``max_conn``,
``progress`` and ``overwrite`` arguments are ignored.
Returns
-------
`parfive.Results`
Examples
--------
>>> from sunpy.net.attrs import Time, Instrument
>>> unifresp = Fido.search(Time('2012/3/4','2012/3/5'), Instrument('EIT')) # doctest: +REMOTE_DATA
>>> filepaths = Fido.fetch(unifresp) # doctest: +SKIP
If any downloads fail, they can be retried by passing the `parfive.Results` object back into ``fetch``.
>>> filepaths = Fido.fetch(filepaths) # doctest: +SKIP
"""
if path is not None:
exists = list(filter(lambda p: p.exists(), Path(path).parents))
if not os.access(exists[0], os.W_OK):
raise PermissionError('You do not have permission to write'
f' to the directory {exists[0]}.')
if "wait" in kwargs:
raise ValueError("wait is not a valid keyword argument to Fido.fetch.")
if downloader is None:
downloader = Downloader(max_conn=max_conn, progress=progress, overwrite=overwrite)
elif not isinstance(downloader, Downloader):
raise TypeError("The downloader argument must be a parfive.Downloader object.")
# Handle retrying failed downloads
retries = [isinstance(arg, Results) for arg in query_results]
if all(retries):
results = Results()
for retry in query_results:
dr = downloader.retry(retry)
results.data += dr.data
results._errors += dr._errors
return results
elif any(retries):
raise TypeError("If any arguments to fetch are "
"`parfive.Results` objects, all arguments must be.")
reslist = []
for query_result in query_results:
for block in query_result.responses:
reslist.append(block.client.fetch(block, path=path,
downloader=downloader,
wait=False, **kwargs))
results = downloader.download()
# Combine the results objects from all the clients into one Results
# object.
for result in reslist:
if result is None:
continue
if not isinstance(result, Results):
raise TypeError(
"If wait is False a client must return a parfive.Downloader and either None"
" or a parfive.Results object.")
results.data += result.data
results._errors += result.errors
return results
def download_all(self, response, methods, downloader, path, qr, info=None):
results = Results()
GET_VERSION = [
('0.8', (5, 8)),
('0.7', (1, 4)),
('0.6', (0, 3)),
]
for dresponse in response.getdataresponseitem:
for version, (from_, to) in GET_VERSION:
if getattr(dresponse, version, '0.6') >= version:
break
else:
results.add_error('', UnknownVersion(dresponse))
continue
# If from_ and to are uninitialized, the else block of the loop
# continues the outer loop and thus this code is never reached.
# pylint: disable=W0631
code = (dresponse.status[from_:to] if getattr(
dresponse, 'status', None) else '200')
if code == '200':
for dataitem in dresponse.getdataitem.dataitem:
try:
self.download(dresponse.method.methodtype[0],
dataitem.url, downloader, path,
qr[dataitem.fileiditem.fileid[0]])
except NoData:
results.add_error('', '', DownloadFailed(dresponse))
continue
elif code == '300' or code == '412' or code == '405':
if code == '300':
try:
methods = self.multiple_choices(
dresponse.method.methodtype, dresponse)
except NoData:
results.add_error('', '', MultipleChoices(dresponse))
continue
elif code == '412':
try:
info = self.missing_information(info, dresponse.info)
except NoData:
results.add_error('', '',
MissingInformation(dresponse))
continue
elif code == '405':
try:
methods = self.unknown_method(dresponse)
except NoData:
results.add_error('', '', UnknownMethod(dresponse))
continue
files = []
for dataitem in dresponse.getdataitem.dataitem:
files.extend(dataitem.fileiditem.fileid)
request = self.create_getdatarequest(
{dresponse.provider: files}, methods, info)
self.download_all(self.api.service.GetData(request), methods,
downloader, path, qr, info)
else:
results.add_error('', '', UnknownStatus(dresponse))
return results
def fetch(self,
query_response,
path=None,
methods=None,
site=None,
progress=True,
overwrite=False,
downloader=None,
wait=True):
"""
Download data specified in the query_response.
Parameters
----------
query_response : sunpy.net.vso.QueryResponse
QueryResponse containing the items to be downloaded.
path : str
Specify where the data is to be downloaded. Can refer to arbitrary
fields of the QueryResponseItem (instrument, source, time, ...) via
string formatting, moreover the file-name of the file downloaded can
be referred to as file, e.g.
"{source}/{instrument}/{time.start}/{file}".
methods : {list of str}
Download methods, defaults to URL-FILE_Rice then URL-FILE.
Methods are a concatenation of one PREFIX followed by any number of
SUFFIXES i.e. `PREFIX-SUFFIX_SUFFIX2_SUFFIX3`.
The full list of
`PREFIXES <https://sdac.virtualsolar.org/cgi/show_details?keyword=METHOD_PREFIX>`_
and `SUFFIXES <https://sdac.virtualsolar.org/cgi/show_details?keyword=METHOD_SUFFIX>`_
are listed on the VSO site.
site : str
There are a number of caching mirrors for SDO and other
instruments, some available ones are listed below.
=============== ========================================================
NSO National Solar Observatory, Tucson (US)
SAO (aka CFA) Smithonian Astronomical Observatory, Harvard U. (US)
SDAC (aka GSFC) Solar Data Analysis Center, NASA/GSFC (US)
ROB Royal Observatory of Belgium (Belgium)
MPS Max Planck Institute for Solar System Research (Germany)
UCLan University of Central Lancashire (UK)
IAS Institut Aeronautique et Spatial (France)
KIS Kiepenheuer-Institut fur Sonnenphysik Germany)
NMSU New Mexico State University (US)
=============== ========================================================
progress : `bool`, optional
If `True` show a progress bar showing how many of the total files
have been downloaded. If `False`, no progress bars will be shown at all.
overwrite : `bool` or `str`, optional
Determine how to handle downloading if a file already exists with the
same name. If `False` the file download will be skipped and the path
returned to the existing file, if `True` the file will be downloaded
and the existing file will be overwritten, if `'unique'` the filename
will be modified to be unique.
downloader : `parfive.Downloader`, optional
The download manager to use.
wait : `bool`, optional
If `False` ``downloader.download()`` will not be called. Only has
any effect if `downloader` is not `None`.
Returns
-------
out : `parfive.Results`
Object that supplies a list of filenames and any errors.
Examples
--------
>>> files = fetch(qr) # doctest:+SKIP
"""
if path is None:
path = os.path.join(config.get('downloads', 'download_dir'),
'{file}')
elif isinstance(path, str) and '{file}' not in path:
path = os.path.join(path, '{file}')
path = os.path.expanduser(path)
dl_set = True
if not downloader:
dl_set = False
downloader = Downloader(progress=progress)
fileids = VSOClient.by_fileid(query_response)
if not fileids:
return downloader.download() if wait else Results()
# Adding the site parameter to the info
info = {}
if site is not None:
info['site'] = site
VSOGetDataResponse = self.api.get_type("VSO:VSOGetDataResponse")
data_request = self.make_getdatarequest(query_response, methods, info)
data_response = VSOGetDataResponse(
self.api.service.GetData(data_request))
err_results = self.download_all(data_response, methods, downloader,
path, fileids)
if dl_set and not wait:
return err_results
results = downloader.download()
results += err_results
results._errors += err_results.errors
return results
def download_all(self, response, methods, downloader, path, qr, info=None):
results = Results()
GET_VERSION = [
('0.8', (5, 8)),
('0.7', (1, 4)),
('0.6', (0, 3)),
]
for dresponse in response.getdataresponseitem:
for version, (from_, to) in GET_VERSION:
if getattr(dresponse, version, '0.6') >= version:
break
else:
results.add_error('', UnknownVersion(dresponse))
continue
# If from_ and to are uninitialized, the else block of the loop
# continues the outer loop and thus this code is never reached.
# pylint: disable=W0631
code = (
dresponse.status[from_:to]
if getattr(dresponse, 'status', None) else '200'
)
if code == '200':
for dataitem in dresponse.getdataitem.dataitem:
try:
self.download(
dresponse.method.methodtype[0],
dataitem.url,
downloader,
path,
qr[dataitem.fileiditem.fileid[0]]
)
except NoData:
results.add_error('', DownloadFailed(dresponse))
continue
elif code == '300' or code == '412' or code == '405':
if code == '300':
try:
methods = self.multiple_choices(
dresponse.method.methodtype, dresponse
)
except NoData:
results.add_error('', MultipleChoices(dresponse))
continue
elif code == '412':
try:
info = self.missing_information(
info, dresponse.info
)
except NoData:
results.add_error('', MissingInformation(dresponse))
continue
elif code == '405':
try:
methods = self.unknown_method(dresponse)
except NoData:
results.add_error('', UnknownMethod(dresponse))
continue
files = []
for dataitem in dresponse.getdataitem.dataitem:
files.extend(dataitem.fileiditem.fileid)
request = self.create_getdatarequest(
{dresponse.provider: files}, methods, info
)
self.download_all(
self.api.service.GetData(request), methods, downloader, path,
qr, info
)
else:
results.add_error(UnknownStatus(dresponse))
return results
