def test_path_exception():
x = threading.Event()
dw = Downloader(1, 2)
dw.download("http://google.at", path_fun, errback=wait_for(1, lambda a: x.set()))
threading.Thread(target=dw.reactor.run).start()
assert x.wait(10)
dw.reactor.stop()
def test_path_exception():
x = threading.Event()
dw = Downloader(1, 2)
dw.download(
"http://google.at", path_fun, errback=wait_for(1, lambda a: x.set())
)
threading.Thread(target=dw.reactor.run).start()
assert x.wait(10)
dw.reactor.stop()
def test_path_exception():
x = threading.Event()
dw = Downloader(1, 2)
dw.download("http://google.at", path_fun, errback=wait_for(1, lambda a: x.set()))
th = threading.Thread(target=dw.wait)
th.daemon = True
th.start()
x.wait(10)
assert x.isSet()
dw.stop()
def fetch(self, qres, path=None, error_callback=None, **kwargs):
"""
Download a set of results.
Parameters
----------
qres : `~sunpy.net.dataretriever.QueryResponse`
Results to download.
path : string or pathlib.Path
Path to the download directory
error_callback : Function
Callback function for error during downloads
Returns
-------
Results Object
"""
# Check for type of path
if path is not None:
if HAS_PATHLIB and isinstance(path, pathlib.Path):
path = str(path.absolute())
elif not isinstance(path, six.string_types):
err = "path should be either 'pathlib.Path' or 'str'. "\
"Got '{}'.".format(type(path))
raise TypeError(err)
urls = [qrblock.url for qrblock in qres]
filenames = [url.split('/')[-1] for url in urls]
paths = self._get_full_filenames(qres, filenames, path)
res = Results(lambda x: None, 0, lambda map_: self._link(map_))
dobj = Downloader(max_conn=len(urls), max_total=len(urls))
# We cast to list here in list(zip... to force execution of
# res.require([x]) at the start of the loop.
for aurl, ncall, fname in list(zip(urls, map(lambda x: res.require([x]),
urls), paths)):
dobj.download(aurl, fname, ncall, error_callback)
return res
def test_download_default_dir():
_config = sunpy.config
try:
tmpdir = tempfile.mkdtemp()
sunpy.config = MockConfig()
sunpy.config.add_section("downloads", {"download_dir": tmpdir})
dw = Downloader(1, 1)
_stop = lambda _: dw.stop()
timeout = CalledProxy(dw.stop)
errback = CalledProxy(_stop)
dw.download(
'http://ajax.googleapis.com/ajax/libs/jquery/2.0.0/jquery.min.js',
callback=_stop,
errback=errback)
timer = threading.Timer(10, timeout)
timer.start()
dw.wait()
timer.cancel()
assert not timeout.fired
assert not errback.fired
assert os.path.exists(os.path.join(tmpdir, 'jquery.min.js'))
finally:
sunpy.config = _config
def fetch(self, qres, path=None, error_callback=None, **kwargs):
"""
Download a set of results.
Parameters
----------
qres : `~sunpy.net.dataretriever.QueryResponse`
Results to download.
path : string or pathlib.Path
Path to the download directory
error_callback : Function
Callback function for error during downloads
Returns
-------
Results Object
"""
# Check for type of path
if path is not None:
if isinstance(path, pathlib.Path):
path = str(path.absolute())
elif not isinstance(path, str):
err = "path should be either 'pathlib.Path' or 'str'. "\
"Got '{}'.".format(type(path))
raise TypeError(err)
urls = [qrblock.url for qrblock in qres]
filenames = [url.split('/')[-1] for url in urls]
paths = self._get_full_filenames(qres, filenames, path)
res = Results(lambda x: None, 0, lambda map_: self._link(map_))
dobj = Downloader(max_conn=len(urls), max_total=len(urls))
# We cast to list here in list(zip... to force execution of
# res.require([x]) at the start of the loop.
for aurl, ncall, fname in list(
zip(urls, map(lambda x: res.require([x]), urls), paths)):
dobj.download(aurl, fname, ncall, error_callback)
return res
def test_download_default_dir():
_config = sunpy.config
try:
tmpdir = tempfile.mkdtemp()
sunpy.config = MockConfig()
sunpy.config.add_section(
"downloads", {"download_dir": tmpdir}
)
dw = Downloader(1, 1)
_stop = lambda _: dw.stop()
timeout = CalledProxy(dw.stop)
errback = CalledProxy(_stop)
dw.download(
'http://ajax.googleapis.com/ajax/libs/jquery/2.0.0/jquery.min.js',
callback=_stop,
errback=errback
)
timer = threading.Timer(10, timeout)
timer.start()
dw.wait()
timer.cancel()
assert not timeout.fired
assert not errback.fired
assert os.path.exists(os.path.join(tmpdir, 'jquery.min.js'))
finally:
sunpy.config = _config
def test_download_http():
items = []
lck = threading.Lock()
def wait_for(n, callback): # pylint: disable=W0613
def _fun(handler):
with lck:
items.append(handler)
if len(items) == n:
callback(items)
return _fun
tmp = tempfile.mkdtemp()
path_fun = partial(default_name, tmp)
dw = Downloader(1, 1)
timeout = CalledProxy(dw.stop)
timer = threading.Timer(60, timeout)
timer.start()
on_finish = wait_for(3, lambda _: dw.stop())
dw.download("http://ajax.googleapis.com/ajax/libs/jquery/2.0.0/jquery.min.js", path_fun, on_finish)
dw.download("http://ajax.googleapis.com/ajax/libs/webfont/1.4.2/webfont.js", path_fun, on_finish)
dw.download("https://raw.github.com/sunpy/sunpy/master/INSTALL.txt", path_fun, on_finish)
# dw.download('ftp://speedtest.inode.at/speedtest-100mb', path_fun, on_finish)
dw.wait()
timer.cancel()
assert len(items) == 3
assert not timeout.fired
for item in items:
assert os.path.exists(item["path"])
def test_path_exception():
x = threading.Event()
dw = Downloader(1, 2)
dw.download("http://google.at",
path_fun,
errback=wait_for(1, lambda a: x.set()))
th = threading.Thread(target=dw.wait)
th.daemon = True
th.start()
x.wait(10)
assert x.isSet()
dw.stop()
def test_download_dir():
tmpdir = tempfile.mkdtemp()
dw = Downloader(1, 1)
_stop = lambda _: dw.stop()
timeout = CalledProxy(dw.stop)
errback = CalledProxy(_stop)
dw.download(
'http://ajax.googleapis.com/ajax/libs/jquery/2.0.0/jquery.min.js',
tmpdir,
callback=_stop,
errback=errback)
timer = threading.Timer(10, timeout)
timer.start()
dw.wait()
timer.cancel()
assert not timeout.fired
assert not errback.fired
assert os.path.exists(os.path.join(tmpdir, 'jquery.min.js'))
def test_download_dir():
tmpdir = tempfile.mkdtemp()
dw = Downloader(1, 1)
_stop = lambda _: dw.stop()
timeout = CalledProxy(dw.stop)
errback = CalledProxy(_stop)
dw.download(
"http://ajax.googleapis.com/ajax/libs/jquery/2.0.0/jquery.min.js", tmpdir, callback=_stop, errback=errback
)
timer = threading.Timer(10, timeout)
timer.start()
dw.wait()
timer.cancel()
assert not timeout.fired
assert not errback.fired
assert os.path.exists(os.path.join(tmpdir, "jquery.min.js"))
def get_request(self, requests, path=None, overwrite=False, progress=True,
max_conn=5, downloader=None, results=None):
"""
Query JSOC to see if the request(s) is ready for download.
If the request is ready for download, it will then download it.
Parameters
----------
requests : `~drms.ExportRequest`, `str`, `list`
`~drms.ExportRequest` objects or `str` request IDs or lists
returned by `~sunpy.net.jsoc.jsoc.JSOCClient.request_data`.
path : `str`
Path to save data to, defaults to SunPy download dir.
overwrite : `bool`
Replace files with the same name if True.
progress : `bool`
Print progress info to terminal.
max_conns : `int`
Maximum number of download connections.
downloader : `~sunpy.net.download.Downloader`
A Custom downloader to use
results: `~sunpy.net.download.Results`
A `~sunpy.net.download.Results` manager to use.
Returns
-------
res: `~sunpy.net.download.Results`
A `~sunpy.net.download.Results` instance or `None` if no URLs to download
"""
c = drms.Client()
# Convert Responses to a list if not already
if isinstance(requests, six.string_types) or not isiterable(requests):
requests = [requests]
# Ensure all the requests are drms ExportRequest objects
for i, request in enumerate(requests):
if isinstance(request, six.string_types):
r = c.export_from_id(request)
requests[i] = r
# We only download if all are finished
if not all([r.has_succeeded() for r in requests]):
raise NotExportedError("Can not download as not all the requests"
"have been exported for download yet.")
# Ensure path has a {file} in it
if path is None:
default_dir = config.get("downloads", "download_dir")
path = os.path.join(default_dir, '{file}')
elif isinstance(path, six.string_types) and '{file}' not in path:
path = os.path.join(path, '{file}')
paths = []
for request in requests:
for filename in request.data['filename']:
# Ensure we don't duplicate the file extension
ext = os.path.splitext(filename)[1]
if path.endswith(ext):
fname = path.strip(ext)
else:
fname = path
fname = fname.format(file=filename)
fname = os.path.expanduser(fname)
fname = partial(simple_path, fname)
paths.append(fname)
if downloader is None:
downloader = Downloader(max_conn=max_conn, max_total=max_conn)
# A Results object tracks the number of downloads requested and the
# number that have been completed.
if results is None:
results = Results(lambda _: downloader.stop(),
done=lambda maps: [v['path'] for v in maps.values()])
urls = []
for request in requests:
if request.status == 0:
for index, data in request.data.iterrows():
is_file = os.path.isfile(paths[index].args[0])
if overwrite or not is_file:
url_dir = request.request_url + '/'
urls.append(urllib.parse.urljoin(url_dir, data['filename']))
if not overwrite and is_file:
print_message = "Skipping download of file {} as it " \
"has already been downloaded. " \
"If you want to redownload the data, "\
"please set overwrite to True"
print(print_message.format(data['filename']))
# Add the file on disk to the output
results.map_.update({data['filename']:
{'path': paths[index].args[0]}})
if urls:
if progress:
print_message = "{0} URLs found for download. Full request totalling {1}MB"
print(print_message.format(len(urls), request._d['size']))
for i, url in enumerate(urls):
downloader.download(url, callback=results.require([url]),
errback=lambda x: print(x), path=paths[i])
else:
# Make Results think it has finished.
results.require([])
results.poke()
return results
def get_request(self,
requests,
path=None,
overwrite=False,
progress=True,
max_conn=5,
downloader=None,
results=None):
"""
Query JSOC to see if the request(s) is ready for download.
If the request is ready for download, it will then download it.
Parameters
----------
requests : `~drms.ExportRequest`, `str`, `list`
`~drms.ExportRequest` objects or `str` request IDs or lists
returned by `~sunpy.net.jsoc.jsoc.JSOCClient.request_data`.
path : `str`
Path to save data to, defaults to SunPy download dir.
overwrite : `bool`
Replace files with the same name if True.
progress : `bool`
Print progress info to terminal.
max_conns : `int`
Maximum number of download connections.
downloader : `~sunpy.net.download.Downloader`
A Custom downloader to use
results: `~sunpy.net.download.Results`
A `~sunpy.net.download.Results` manager to use.
Returns
-------
res: `~sunpy.net.download.Results`
A `~sunpy.net.download.Results` instance or `None` if no URLs to download
"""
c = drms.Client()
# Convert Responses to a list if not already
if isinstance(requests, str) or not isiterable(requests):
requests = [requests]
# Ensure all the requests are drms ExportRequest objects
for i, request in enumerate(requests):
if isinstance(request, str):
r = c.export_from_id(request)
requests[i] = r
# We only download if all are finished
if not all([r.has_succeeded() for r in requests]):
raise NotExportedError("Can not download as not all the requests "
"have been exported for download yet.")
# Ensure path has a {file} in it
if path is None:
default_dir = config.get("downloads", "download_dir")
path = os.path.join(default_dir, '{file}')
elif isinstance(path, str) and '{file}' not in path:
path = os.path.join(path, '{file}')
paths = []
for request in requests:
for filename in request.data['filename']:
# Ensure we don't duplicate the file extension
ext = os.path.splitext(filename)[1]
if path.endswith(ext):
fname = path.strip(ext)
else:
fname = path
fname = fname.format(file=filename)
fname = os.path.expanduser(fname)
fname = partial(simple_path, fname)
paths.append(fname)
if downloader is None:
downloader = Downloader(max_conn=max_conn, max_total=max_conn)
# A Results object tracks the number of downloads requested and the
# number that have been completed.
if results is None:
results = Results(
lambda _: downloader.stop(),
done=lambda maps: [v['path'] for v in maps.values()])
urls = []
for request in requests:
if request.status == 0:
for index, data in request.data.iterrows():
is_file = os.path.isfile(paths[index].args[0])
if overwrite or not is_file:
url_dir = request.request_url + '/'
urls.append(
urllib.parse.urljoin(url_dir, data['filename']))
if not overwrite and is_file:
print_message = "Skipping download of file {} as it " \
"has already been downloaded. " \
"If you want to redownload the data, "\
"please set overwrite to True"
print(print_message.format(data['filename']))
# Add the file on disk to the output
results.map_.update(
{data['filename']: {
'path': paths[index].args[0]
}})
if urls:
if progress:
print_message = "{0} URLs found for download. Full request totalling {1}MB"
print(print_message.format(len(urls), request._d['size']))
for i, url in enumerate(urls):
downloader.download(url,
callback=results.require([url]),
errback=lambda x: print(x),
path=paths[i])
else:
# Make Results think it has finished.
results.require([])
results.poke()
return results
def get_request(self,
requestIDs,
path=None,
overwrite=False,
progress=True,
max_conn=5,
downloader=None,
results=None):
"""
Query JSOC to see if request_id is ready for download.
If the request is ready for download, download it.
Parameters
----------
requestIDs : list or string
One or many requestID strings
path : string
Path to save data to, defaults to SunPy download dir
overwrite : bool
Replace files with the same name if True
progress : bool
Print progress info to terminal
max_conns : int
Maximum number of download connections.
downloader : `sunpy.download.Downloader` instance
A Custom downloader to use
results: Results instance
A Results manager to use.
Returns
-------
res: Results
A Results instance or None if no URLs to download
"""
# Convert IDs to a list if not already
if not isiterable(requestIDs) or isinstance(requestIDs,
six.string_types):
requestIDs = [requestIDs]
if path is None:
path = config.get('downloads', 'download_dir')
path = os.path.expanduser(path)
if downloader is None:
downloader = Downloader(max_conn=max_conn, max_total=max_conn)
# A Results object tracks the number of downloads requested and the
# number that have been completed.
if results is None:
results = Results(lambda _: downloader.stop())
urls = []
for request_id in requestIDs:
u = self._request_status(request_id)
if u.status_code == 200 and u.json()['status'] == '0':
for ar in u.json()['data']:
is_file = os.path.isfile(os.path.join(
path, ar['filename']))
if overwrite or not is_file:
url_dir = BASE_DL_URL + u.json()['dir'] + '/'
urls.append(
urllib.parse.urljoin(url_dir, ar['filename']))
else:
print_message = "Skipping download of file {} as it " \
"has already been downloaded"
print(print_message.format(ar['filename']))
# Add the file on disk to the output
results.map_.update({
ar['filename']: {
'path': os.path.join(path, ar['filename'])
}
})
if progress:
print_message = "{0} URLs found for download. Totalling {1}MB"
print(print_message.format(len(urls), u.json()['size']))
else:
if progress:
self.check_request(request_id)
if urls:
for url in urls:
downloader.download(url,
callback=results.require([url]),
errback=lambda x: print(x),
path=path)
else:
# Make Results think it has finished.
results.require([])
results.poke()
return results
def get_request(self, requestIDs, path=None, overwrite=False, progress=True,
max_conn=5, downloader=None, results=None):
"""
Query JSOC to see if request_id is ready for download.
If the request is ready for download, download it.
Parameters
----------
requestIDs: list or string
One or many requestID strings
path: string
Path to save data to, defaults to SunPy download dir
overwrite: bool
Replace files with the same name if True
progress: bool
Print progress info to terminal
max_conns: int
Maximum number of download connections.
downloader: sunpy.download.Downloader instance
A Custom downloader to use
results: Results instance
A Results manager to use.
Returns
-------
res: Results
A Results instance or None if no URLs to download
"""
# Convert IDs to a list if not already
if not astropy.utils.misc.isiterable(requestIDs) or isinstance(requestIDs, basestring):
requestIDs = [requestIDs]
if path is None:
path = config.get('downloads','download_dir')
path = os.path.expanduser(path)
if downloader is None:
downloader = Downloader(max_conn=max_conn, max_total=max_conn)
# A Results object tracks the number of downloads requested and the
# number that have been completed.
if results is None:
results = Results(lambda x: None)
urls = []
for request_id in requestIDs:
u = self._request_status(request_id)
if u.status_code == 200 and u.json()['status'] == '0':
for ar in u.json()['data']:
if overwrite or not os.path.isfile(os.path.join(path, ar['filename'])):
urls.append(urlparse.urljoin(BASE_DL_URL + u.json()['dir'] +
'/', ar['filename']))
if progress:
print("{0} URLs found for Download. Totalling {1}MB".format(
len(urls), u.json()['size']))
else:
if progress:
self.check_request(request_id)
if urls:
for url, rcall in list(zip(urls, list(map(lambda x: results.require([x]), urls)))):
downloader.download(url, callback=rcall, path=path)
else:
#Make Results think it has finished.
results.require([])
results.poke()
return results
def get_request(self, requestIDs, path=None, overwrite=False, progress=True,
max_conn=5, downloader=None, results=None):
"""
Query JSOC to see if request_id is ready for download.
If the request is ready for download, download it.
Parameters
----------
requestIDs : list or string
One or many requestID strings
path : string
Path to save data to, defaults to SunPy download dir
overwrite : bool
Replace files with the same name if True
progress : bool
Print progress info to terminal
max_conns : int
Maximum number of download connections.
downloader : `sunpy.download.Downloader` instance
A Custom downloader to use
results: Results instance
A Results manager to use.
Returns
-------
res: Results
A Results instance or None if no URLs to download
"""
# Convert IDs to a list if not already
if not isiterable(requestIDs) or isinstance(requestIDs, six.string_types):
requestIDs = [requestIDs]
if path is None:
path = config.get('downloads', 'download_dir')
path = os.path.expanduser(path)
if downloader is None:
downloader = Downloader(max_conn=max_conn, max_total=max_conn)
# A Results object tracks the number of downloads requested and the
# number that have been completed.
if results is None:
results = Results(lambda _: downloader.stop())
urls = []
for request_id in requestIDs:
u = self._request_status(request_id)
if u.status_code == 200 and u.json()['status'] == '0':
for ar in u.json()['data']:
is_file = os.path.isfile(os.path.join(path, ar['filename']))
if overwrite or not is_file:
url_dir = BASE_DL_URL + u.json()['dir'] + '/'
urls.append(urllib.parse.urljoin(url_dir, ar['filename']))
else:
print_message = "Skipping download of file {} as it " \
"has already been downloaded"
print(print_message.format(ar['filename']))
# Add the file on disk to the output
results.map_.update({ar['filename']:
{'path': os.path.join(path, ar['filename'])}})
if progress:
print_message = "{0} URLs found for download. Totalling {1}MB"
print(print_message.format(len(urls), u.json()['size']))
else:
if progress:
self.check_request(request_id)
if urls:
for url in urls:
downloader.download(url, callback=results.require([url]),
errback=lambda x: print(x), path=path)
else:
# Make Results think it has finished.
results.require([])
results.poke()
return results
def fetch(self, qres, path=None, error_callback=None, **kwargs):
"""
Download a set of results.
Parameters
----------
qres : `~sunpy.net.dataretriever.QueryResponse`
Results to download.
Returns
-------
Results Object
"""
urls = [qrblock.url for qrblock in qres]
filenames = []
local_filenames = []
for i, [url, qre] in enumerate(zip(urls, qres)):
name = url.split('/')[-1]
# temporary fix !!! coz All QRBs have same start_time values
day = qre.time.start.date() + datetime.timedelta(days=i)
if name not in filenames:
filenames.append(name)
if name.endswith('.gz'):
local_filenames.append('{date:%Y%m%d}SRS.txt'.format(date=day))
else:
local_filenames.append(name)
# Files to be actually downloaded
paths = self._get_full_filenames(qres, filenames, path)
# Those files that will be present after get returns
local_paths = self._get_full_filenames(qres, local_filenames, path)
res = Results(lambda x: None, 0, lambda map_: self._link(map_))
# remove duplicate urls. This will make paths and urls to have same number of elements.
# OrderedDict is required to maintain ordering because it will be zipped with paths later
urls = list(OrderedDict.fromkeys(urls))
dobj = Downloader(max_conn=len(urls), max_total=len(urls))
# We cast to list here in list(zip... to force execution of
# res.require([x]) at the start of the loop.
for aurl, ncall, fname in list(
zip(urls, map(lambda x: res.require([x]), urls), paths)):
dobj.download(aurl, fname, ncall, error_callback)
res.wait()
res2 = Results(lambda x: None, 0)
for fname, srs_filename in zip(local_paths, local_filenames):
fname = fname.args[0]
name = fname.split('/')[-1]
past_year = False
for i, fname2 in enumerate(paths):
fname2 = fname2.args[0]
if fname2.endswith('.txt'):
continue
year = fname2.split('/')[-1]
year = year.split('_SRS')[0]
if year in name:
TarFile = tarfile.open(fname2)
filepath = fname.rpartition('/')[0]
member = TarFile.getmember('SRS/' + srs_filename)
member.name = name
TarFile.extract(member, path=filepath)
TarFile.close()
callback = res2.require([fname])
callback({'path': fname})
past_year = True
break
if past_year is False:
callback = res2.require([fname])
callback({'path': fname})
return res2
def fetch(self, qres, path=None, error_callback=None, **kwargs):
"""
Download a set of results.
Parameters
----------
qres : `~sunpy.net.dataretriever.QueryResponse`
Results to download.
Returns
-------
Results Object
"""
urls = [qrblock.url for qrblock in qres]
filenames = []
local_filenames = []
for i, [url, qre] in enumerate(zip(urls, qres)):
name = url.split('/')[-1]
# temporary fix !!! coz All QRBs have same start_time values
day = Time(qre.time.start.strftime('%Y-%m-%d')) + TimeDelta(i*u.day)
if name not in filenames:
filenames.append(name)
if name.endswith('.gz'):
local_filenames.append('{}SRS.txt'.format(day.strftime('%Y%m%d')))
else:
local_filenames.append(name)
# Files to be actually downloaded
paths = self._get_full_filenames(qres, filenames, path)
# Those files that will be present after get returns
local_paths = self._get_full_filenames(qres, local_filenames, path)
res = Results(lambda x: None, 0, lambda map_: self._link(map_))
# remove duplicate urls. This will make paths and urls to have same number of elements.
# OrderedDict is required to maintain ordering because it will be zipped with paths later
urls = list(OrderedDict.fromkeys(urls))
dobj = Downloader(max_conn=len(urls), max_total=len(urls))
# We cast to list here in list(zip... to force execution of
# res.require([x]) at the start of the loop.
for aurl, ncall, fname in list(zip(urls, map(lambda x: res.require([x]),
urls), paths)):
dobj.download(aurl, fname, ncall, error_callback)
res.wait()
res2 = Results(lambda x: None, 0)
for fname, srs_filename in zip(local_paths, local_filenames):
fname = fname.args[0]
name = fname.split('/')[-1]
past_year = False
for i, fname2 in enumerate(paths):
fname2 = fname2.args[0]
if fname2.endswith('.txt'):
continue
year = fname2.split('/')[-1]
year = year.split('_SRS')[0]
if year in name:
TarFile = tarfile.open(fname2)
filepath = fname.rpartition('/')[0]
member = TarFile.getmember('SRS/' + srs_filename)
member.name = name
TarFile.extract(member, path=filepath)
TarFile.close()
callback = res2.require([fname])
callback({'path': fname})
past_year = True
break
if past_year is False:
callback = res2.require([fname])
callback({'path': fname})
return res2
def test_download_http():
items = []
lck = threading.Lock()
def wait_for(n, callback): # pylint: disable=W0613
def _fun(handler):
with lck:
items.append(handler)
if len(items) == n:
callback(items)
return _fun
tmp = tempfile.mkdtemp()
path_fun = partial(default_name, tmp)
dw = Downloader(1, 1)
timeout = CalledProxy(dw.stop)
timer = threading.Timer(60, timeout)
timer.start()
on_finish = wait_for(3, lambda _: dw.stop())
dw.download(
'http://ajax.googleapis.com/ajax/libs/jquery/2.0.0/jquery.min.js',
path_fun, on_finish)
dw.download(
'http://ajax.googleapis.com/ajax/libs/webfont/1.4.2/webfont.js',
path_fun, on_finish)
dw.download('https://raw.github.com/sunpy/sunpy/master/INSTALL.txt',
path_fun, on_finish)
# dw.download('ftp://speedtest.inode.at/speedtest-100mb', path_fun, on_finish)
dw.wait()
timer.cancel()
assert len(items) == 3
assert not timeout.fired
for item in items:
assert os.path.exists(item['path'])
def get(self, requestIDs, path=None, overwrite=False, progress=True,
max_conn=5, downloader=None, results=None):
"""
Query JSOC to see if request_id is ready for download.
If the request is ready for download download it.
Parameters
----------
requestIDs: list or string
One or many requestID strings
path: string
Path to save data to, defaults to SunPy download dir
overwrite: bool
Replace files with the same name if True
progress: bool
Print progress info to terminal
max_conns: int
Maximum number of download connections.
downloader: sunpy.download.Downloder instance
A Custom downloader to use
results: Results instance
A Results manager to use.
Returns
-------
res: Results
A Results instance or None if no URLs to download
"""
# Convert IDs to a list if not already
if not astropy.utils.misc.isiterable(requestIDs) or isinstance(requestIDs, basestring):
requestIDs = [requestIDs]
if path is None:
path = config.get('downloads','download_dir')
if downloader is None:
downloader = Downloader(max_conn=max_conn, max_total=max_conn)
# A Results object tracks the number of downloads requested and the
# number that have been completed.
if results is None:
results = Results(lambda x: None)
urls = []
for request_id in requestIDs:
u = self._request_status(request_id)
if u.status_code == 200 and u.json()['status'] == '0':
for ar in u.json()['data']:
if overwrite or not os.path.isfile(os.path.join(path, ar['filename'])):
urls.append(urlparse.urljoin(BASE_DL_URL + u.json()['dir']+'/', ar['filename']))
if progress:
print("{0} URLs found for Download. Totalling {1}MB".format(len(urls), u.json()['size']))
else:
if progress:
self.check_request(request_id)
if urls:
for url, rcall in list(zip(urls, list(map(lambda x: results.require([x]), urls)))):
downloader.download(url, callback=rcall, path=path)
else:
#Make Results think it has finished.
results.require([])
results.poke()
return results
