def fetch(self, jsoc_response, path=None, overwrite=False, progress=True,
max_conn=5, downloader=None, sleep=10):
"""
Make the request for the data in jsoc_response and wait for it to be
staged and then download the data.
Parameters
----------
jsoc_response : JSOCResponse object
A response object
path : string
Path to save data to, defaults to SunPy download dir
overwrite : bool
Replace files with the same name if True
progress : bool
Print progress info to terminal
max_conns : int
Maximum number of download connections.
downloader: `sunpy.download.Downloader` instance
A Custom downloader to use
sleep : int
The number of seconds to wait between calls to JSOC to check the status
of the request.
Returns
-------
results : a :class:`sunpy.net.vso.Results` instance
A Results object
"""
# Make staging request to JSOC
requestIDs = self.request_data(jsoc_response)
# Add them to the response for good measure
jsoc_response.requestIDs = requestIDs
time.sleep(sleep/2.)
r = Results(lambda x: None, done=lambda maps: [v['path'] for v in maps.values()])
while requestIDs:
for i, request_id in enumerate(requestIDs):
u = self._request_status(request_id)
if progress:
self.check_request(request_id)
if u.status_code == 200 and u.json()['status'] == '0':
rID = requestIDs.pop(i)
r = self.get_request(rID, path=path, overwrite=overwrite,
progress=progress, results=r)
else:
time.sleep(sleep)
return r
def fetch(self, qres, path=None, error_callback=None, **kwargs):
"""
Download a set of results.
Parameters
----------
qres : `~sunpy.net.dataretriever.QueryResponse`
Results to download.
path : string or pathlib.Path
Path to the download directory
error_callback : Function
Callback function for error during downloads
Returns
-------
Results Object
"""
# Check for type of path
if path is not None:
if isinstance(path, pathlib.Path):
path = str(path.absolute())
elif not isinstance(path, str):
err = "path should be either 'pathlib.Path' or 'str'. "\
"Got '{}'.".format(type(path))
raise TypeError(err)
urls = [qrblock.url for qrblock in qres]
filenames = [url.split('/')[-1] for url in urls]
paths = self._get_full_filenames(qres, filenames, path)
res = Results(lambda x: None, 0, lambda map_: self._link(map_))
dobj = Downloader(max_conn=len(urls), max_total=len(urls))
# We cast to list here in list(zip... to force execution of
# res.require([x]) at the start of the loop.
for aurl, ncall, fname in list(
zip(urls, map(lambda x: res.require([x]), urls), paths)):
dobj.download(aurl, fname, ncall, error_callback)
return res
def fetch(self, qres, path=None, error_callback=None, **kwargs):
"""
Download a set of results.
Parameters
----------
qres : `~sunpy.net.dataretriever.QueryResponse`
Results to download.
path : string or pathlib.Path
Path to the download directory
error_callback : Function
Callback function for error during downloads
Returns
-------
Results Object
"""
# Check for type of path
if path is not None:
if HAS_PATHLIB and isinstance(path, pathlib.Path):
path = str(path.absolute())
elif not isinstance(path, six.string_types):
err = "path should be either 'pathlib.Path' or 'str'. "\
"Got '{}'.".format(type(path))
raise TypeError(err)
urls = [qrblock.url for qrblock in qres]
filenames = [url.split('/')[-1] for url in urls]
paths = self._get_full_filenames(qres, filenames, path)
res = Results(lambda x: None, 0, lambda map_: self._link(map_))
dobj = Downloader(max_conn=len(urls), max_total=len(urls))
# We cast to list here in list(zip... to force execution of
# res.require([x]) at the start of the loop.
for aurl, ncall, fname in list(zip(urls, map(lambda x: res.require([x]),
urls), paths)):
dobj.download(aurl, fname, ncall, error_callback)
return res
def fetch(self, jsoc_response, path=None, overwrite=False, progress=True,
max_conn=5, downloader=None, sleep=10):
"""
Make the request for the data in a JSOC response and wait for it to be
staged and then download the data.
Parameters
----------
jsoc_response : `~sunpy.net.jsoc.jsoc.JSOCResponse` object
A response object
path : `str`
Path to save data to, defaults to SunPy download dir
overwrite : `bool`
Replace files with the same name if True
progress : `bool`
Print progress info to terminal
max_conns : `int`
Maximum number of download connections.
downloader: `~sunpy.net.download.Downloader` instance
A Custom downloader to use
sleep : `int`
The number of seconds to wait between calls to JSOC to check the status
of the request.
Returns
-------
results : a `~sunpy.net.download.Results` instance
A Results object
"""
# Make staging request to JSOC
responses = self.request_data(jsoc_response)
# Make response iterable
if not isiterable(responses):
responses = [responses]
# Add them to the response for good measure
jsoc_response.requests = [r for r in responses]
time.sleep(sleep/2.)
r = Results(lambda x: None, done=lambda maps: [v['path'] for v in maps.values()])
for response in responses:
response.wait(verbose=progress)
r = self.get_request(response, path=path, overwrite=overwrite,
progress=progress, results=r)
return r
def get_request(self,
requests,
path=None,
overwrite=False,
progress=True,
max_conn=5,
downloader=None,
results=None):
"""
Query JSOC to see if the request(s) is ready for download.
If the request is ready for download, it will then download it.
Parameters
----------
requests : `~drms.ExportRequest`, `str`, `list`
`~drms.ExportRequest` objects or `str` request IDs or lists
returned by `~sunpy.net.jsoc.jsoc.JSOCClient.request_data`.
path : `str`
Path to save data to, defaults to SunPy download dir.
overwrite : `bool`
Replace files with the same name if True.
progress : `bool`
Print progress info to terminal.
max_conns : `int`
Maximum number of download connections.
downloader : `~sunpy.net.download.Downloader`
A Custom downloader to use
results: `~sunpy.net.download.Results`
A `~sunpy.net.download.Results` manager to use.
Returns
-------
res: `~sunpy.net.download.Results`
A `~sunpy.net.download.Results` instance or `None` if no URLs to download
"""
c = drms.Client()
# Convert Responses to a list if not already
if isinstance(requests, str) or not isiterable(requests):
requests = [requests]
# Ensure all the requests are drms ExportRequest objects
for i, request in enumerate(requests):
if isinstance(request, str):
r = c.export_from_id(request)
requests[i] = r
# We only download if all are finished
if not all([r.has_succeeded() for r in requests]):
raise NotExportedError("Can not download as not all the requests "
"have been exported for download yet.")
# Ensure path has a {file} in it
if path is None:
default_dir = config.get("downloads", "download_dir")
path = os.path.join(default_dir, '{file}')
elif isinstance(path, str) and '{file}' not in path:
path = os.path.join(path, '{file}')
paths = []
for request in requests:
for filename in request.data['filename']:
# Ensure we don't duplicate the file extension
ext = os.path.splitext(filename)[1]
if path.endswith(ext):
fname = path.strip(ext)
else:
fname = path
fname = fname.format(file=filename)
fname = os.path.expanduser(fname)
fname = partial(simple_path, fname)
paths.append(fname)
if downloader is None:
downloader = Downloader(max_conn=max_conn, max_total=max_conn)
# A Results object tracks the number of downloads requested and the
# number that have been completed.
if results is None:
results = Results(
lambda _: downloader.stop(),
done=lambda maps: [v['path'] for v in maps.values()])
urls = []
for request in requests:
if request.status == 0:
for index, data in request.data.iterrows():
is_file = os.path.isfile(paths[index].args[0])
if overwrite or not is_file:
url_dir = request.request_url + '/'
urls.append(
urllib.parse.urljoin(url_dir, data['filename']))
if not overwrite and is_file:
print_message = "Skipping download of file {} as it " \
"has already been downloaded. " \
"If you want to redownload the data, "\
"please set overwrite to True"
print(print_message.format(data['filename']))
# Add the file on disk to the output
results.map_.update(
{data['filename']: {
'path': paths[index].args[0]
}})
if urls:
if progress:
print_message = "{0} URLs found for download. Full request totalling {1}MB"
print(print_message.format(len(urls), request._d['size']))
for i, url in enumerate(urls):
downloader.download(url,
callback=results.require([url]),
errback=lambda x: print(x),
path=paths[i])
else:
# Make Results think it has finished.
results.require([])
results.poke()
return results
def get_request(self,
requestIDs,
path=None,
overwrite=False,
progress=True,
max_conn=5,
downloader=None,
results=None):
"""
Query JSOC to see if request_id is ready for download.
If the request is ready for download, download it.
Parameters
----------
requestIDs : list or string
One or many requestID strings
path : string
Path to save data to, defaults to SunPy download dir
overwrite : bool
Replace files with the same name if True
progress : bool
Print progress info to terminal
max_conns : int
Maximum number of download connections.
downloader : `sunpy.download.Downloader` instance
A Custom downloader to use
results: Results instance
A Results manager to use.
Returns
-------
res: Results
A Results instance or None if no URLs to download
"""
# Convert IDs to a list if not already
if not isiterable(requestIDs) or isinstance(requestIDs,
six.string_types):
requestIDs = [requestIDs]
if path is None:
path = config.get('downloads', 'download_dir')
path = os.path.expanduser(path)
if downloader is None:
downloader = Downloader(max_conn=max_conn, max_total=max_conn)
# A Results object tracks the number of downloads requested and the
# number that have been completed.
if results is None:
results = Results(lambda _: downloader.stop())
urls = []
for request_id in requestIDs:
u = self._request_status(request_id)
if u.status_code == 200 and u.json()['status'] == '0':
for ar in u.json()['data']:
is_file = os.path.isfile(os.path.join(
path, ar['filename']))
if overwrite or not is_file:
url_dir = BASE_DL_URL + u.json()['dir'] + '/'
urls.append(
urllib.parse.urljoin(url_dir, ar['filename']))
else:
print_message = "Skipping download of file {} as it " \
"has already been downloaded"
print(print_message.format(ar['filename']))
# Add the file on disk to the output
results.map_.update({
ar['filename']: {
'path': os.path.join(path, ar['filename'])
}
})
if progress:
print_message = "{0} URLs found for download. Totalling {1}MB"
print(print_message.format(len(urls), u.json()['size']))
else:
if progress:
self.check_request(request_id)
if urls:
for url in urls:
downloader.download(url,
callback=results.require([url]),
errback=lambda x: print(x),
path=path)
else:
# Make Results think it has finished.
results.require([])
results.poke()
return results
def fetch(self, qres, path=None, error_callback=None, **kwargs):
"""
Download a set of results.
Parameters
----------
qres : `~sunpy.net.dataretriever.QueryResponse`
Results to download.
Returns
-------
Results Object
"""
urls = [qrblock.url for qrblock in qres]
filenames = []
local_filenames = []
for i, [url, qre] in enumerate(zip(urls, qres)):
name = url.split('/')[-1]
# temporary fix !!! coz All QRBs have same start_time values
day = qre.time.start.date() + datetime.timedelta(days=i)
if name not in filenames:
filenames.append(name)
if name.endswith('.gz'):
local_filenames.append('{date:%Y%m%d}SRS.txt'.format(date=day))
else:
local_filenames.append(name)
# Files to be actually downloaded
paths = self._get_full_filenames(qres, filenames, path)
# Those files that will be present after get returns
local_paths = self._get_full_filenames(qres, local_filenames, path)
res = Results(lambda x: None, 0, lambda map_: self._link(map_))
# remove duplicate urls. This will make paths and urls to have same number of elements.
# OrderedDict is required to maintain ordering because it will be zipped with paths later
urls = list(OrderedDict.fromkeys(urls))
dobj = Downloader(max_conn=len(urls), max_total=len(urls))
# We cast to list here in list(zip... to force execution of
# res.require([x]) at the start of the loop.
for aurl, ncall, fname in list(
zip(urls, map(lambda x: res.require([x]), urls), paths)):
dobj.download(aurl, fname, ncall, error_callback)
res.wait()
res2 = Results(lambda x: None, 0)
for fname, srs_filename in zip(local_paths, local_filenames):
fname = fname.args[0]
name = fname.split('/')[-1]
past_year = False
for i, fname2 in enumerate(paths):
fname2 = fname2.args[0]
if fname2.endswith('.txt'):
continue
year = fname2.split('/')[-1]
year = year.split('_SRS')[0]
if year in name:
TarFile = tarfile.open(fname2)
filepath = fname.rpartition('/')[0]
member = TarFile.getmember('SRS/' + srs_filename)
member.name = name
TarFile.extract(member, path=filepath)
TarFile.close()
callback = res2.require([fname])
callback({'path': fname})
past_year = True
break
if past_year is False:
callback = res2.require([fname])
callback({'path': fname})
return res2
def fetch(self, qres, path=None, error_callback=None, **kwargs):
"""
Download a set of results.
Parameters
----------
qres : `~sunpy.net.dataretriever.QueryResponse`
Results to download.
Returns
-------
Results Object
"""
urls = [qrblock.url for qrblock in qres]
filenames = []
local_filenames = []
for i, [url, qre] in enumerate(zip(urls, qres)):
name = url.split('/')[-1]
# temporary fix !!! coz All QRBs have same start_time values
day = Time(qre.time.start.strftime('%Y-%m-%d')) + TimeDelta(i*u.day)
if name not in filenames:
filenames.append(name)
if name.endswith('.gz'):
local_filenames.append('{}SRS.txt'.format(day.strftime('%Y%m%d')))
else:
local_filenames.append(name)
# Files to be actually downloaded
paths = self._get_full_filenames(qres, filenames, path)
# Those files that will be present after get returns
local_paths = self._get_full_filenames(qres, local_filenames, path)
res = Results(lambda x: None, 0, lambda map_: self._link(map_))
# remove duplicate urls. This will make paths and urls to have same number of elements.
# OrderedDict is required to maintain ordering because it will be zipped with paths later
urls = list(OrderedDict.fromkeys(urls))
dobj = Downloader(max_conn=len(urls), max_total=len(urls))
# We cast to list here in list(zip... to force execution of
# res.require([x]) at the start of the loop.
for aurl, ncall, fname in list(zip(urls, map(lambda x: res.require([x]),
urls), paths)):
dobj.download(aurl, fname, ncall, error_callback)
res.wait()
res2 = Results(lambda x: None, 0)
for fname, srs_filename in zip(local_paths, local_filenames):
fname = fname.args[0]
name = fname.split('/')[-1]
past_year = False
for i, fname2 in enumerate(paths):
fname2 = fname2.args[0]
if fname2.endswith('.txt'):
continue
year = fname2.split('/')[-1]
year = year.split('_SRS')[0]
if year in name:
TarFile = tarfile.open(fname2)
filepath = fname.rpartition('/')[0]
member = TarFile.getmember('SRS/' + srs_filename)
member.name = name
TarFile.extract(member, path=filepath)
TarFile.close()
callback = res2.require([fname])
callback({'path': fname})
past_year = True
break
if past_year is False:
callback = res2.require([fname])
callback({'path': fname})
return res2
def get_request(self, requestIDs, path=None, overwrite=False, progress=True,
max_conn=5, downloader=None, results=None):
"""
Query JSOC to see if request_id is ready for download.
If the request is ready for download, download it.
Parameters
----------
requestIDs : list or string
One or many requestID strings
path : string
Path to save data to, defaults to SunPy download dir
overwrite : bool
Replace files with the same name if True
progress : bool
Print progress info to terminal
max_conns : int
Maximum number of download connections.
downloader : `sunpy.download.Downloader` instance
A Custom downloader to use
results: Results instance
A Results manager to use.
Returns
-------
res: Results
A Results instance or None if no URLs to download
"""
# Convert IDs to a list if not already
if not isiterable(requestIDs) or isinstance(requestIDs, six.string_types):
requestIDs = [requestIDs]
if path is None:
path = config.get('downloads', 'download_dir')
path = os.path.expanduser(path)
if downloader is None:
downloader = Downloader(max_conn=max_conn, max_total=max_conn)
# A Results object tracks the number of downloads requested and the
# number that have been completed.
if results is None:
results = Results(lambda _: downloader.stop())
urls = []
for request_id in requestIDs:
u = self._request_status(request_id)
if u.status_code == 200 and u.json()['status'] == '0':
for ar in u.json()['data']:
is_file = os.path.isfile(os.path.join(path, ar['filename']))
if overwrite or not is_file:
url_dir = BASE_DL_URL + u.json()['dir'] + '/'
urls.append(urllib.parse.urljoin(url_dir, ar['filename']))
else:
print_message = "Skipping download of file {} as it " \
"has already been downloaded"
print(print_message.format(ar['filename']))
# Add the file on disk to the output
results.map_.update({ar['filename']:
{'path': os.path.join(path, ar['filename'])}})
if progress:
print_message = "{0} URLs found for download. Totalling {1}MB"
print(print_message.format(len(urls), u.json()['size']))
else:
if progress:
self.check_request(request_id)
if urls:
for url in urls:
downloader.download(url, callback=results.require([url]),
errback=lambda x: print(x), path=path)
else:
# Make Results think it has finished.
results.require([])
results.poke()
return results
def get_request(self, requests, path=None, overwrite=False, progress=True,
max_conn=5, downloader=None, results=None):
"""
Query JSOC to see if the request(s) is ready for download.
If the request is ready for download, it will then download it.
Parameters
----------
requests : `~drms.ExportRequest`, `str`, `list`
`~drms.ExportRequest` objects or `str` request IDs or lists
returned by `~sunpy.net.jsoc.jsoc.JSOCClient.request_data`.
path : `str`
Path to save data to, defaults to SunPy download dir.
overwrite : `bool`
Replace files with the same name if True.
progress : `bool`
Print progress info to terminal.
max_conns : `int`
Maximum number of download connections.
downloader : `~sunpy.net.download.Downloader`
A Custom downloader to use
results: `~sunpy.net.download.Results`
A `~sunpy.net.download.Results` manager to use.
Returns
-------
res: `~sunpy.net.download.Results`
A `~sunpy.net.download.Results` instance or `None` if no URLs to download
"""
c = drms.Client()
# Convert Responses to a list if not already
if isinstance(requests, six.string_types) or not isiterable(requests):
requests = [requests]
# Ensure all the requests are drms ExportRequest objects
for i, request in enumerate(requests):
if isinstance(request, six.string_types):
r = c.export_from_id(request)
requests[i] = r
# We only download if all are finished
if not all([r.has_succeeded() for r in requests]):
raise NotExportedError("Can not download as not all the requests"
"have been exported for download yet.")
# Ensure path has a {file} in it
if path is None:
default_dir = config.get("downloads", "download_dir")
path = os.path.join(default_dir, '{file}')
elif isinstance(path, six.string_types) and '{file}' not in path:
path = os.path.join(path, '{file}')
paths = []
for request in requests:
for filename in request.data['filename']:
# Ensure we don't duplicate the file extension
ext = os.path.splitext(filename)[1]
if path.endswith(ext):
fname = path.strip(ext)
else:
fname = path
fname = fname.format(file=filename)
fname = os.path.expanduser(fname)
fname = partial(simple_path, fname)
paths.append(fname)
if downloader is None:
downloader = Downloader(max_conn=max_conn, max_total=max_conn)
# A Results object tracks the number of downloads requested and the
# number that have been completed.
if results is None:
results = Results(lambda _: downloader.stop(),
done=lambda maps: [v['path'] for v in maps.values()])
urls = []
for request in requests:
if request.status == 0:
for index, data in request.data.iterrows():
is_file = os.path.isfile(paths[index].args[0])
if overwrite or not is_file:
url_dir = request.request_url + '/'
urls.append(urllib.parse.urljoin(url_dir, data['filename']))
if not overwrite and is_file:
print_message = "Skipping download of file {} as it " \
"has already been downloaded. " \
"If you want to redownload the data, "\
"please set overwrite to True"
print(print_message.format(data['filename']))
# Add the file on disk to the output
results.map_.update({data['filename']:
{'path': paths[index].args[0]}})
if urls:
if progress:
print_message = "{0} URLs found for download. Full request totalling {1}MB"
print(print_message.format(len(urls), request._d['size']))
for i, url in enumerate(urls):
downloader.download(url, callback=results.require([url]),
errback=lambda x: print(x), path=paths[i])
else:
# Make Results think it has finished.
results.require([])
results.poke()
return results