def test_download_http():
items = []
lck = threading.Lock()
def wait_for(n, callback): # pylint: disable=W0613
def _fun(handler):
with lck:
items.append(handler)
if len(items) == n:
callback(items)
return _fun
tmp = tempfile.mkdtemp()
path_fun = partial(default_name, tmp)
dw = Downloader(1, 1)
_stop = lambda _: dw.stop()
timeout = CalledProxy(dw.stop)
timer = threading.Timer(60, timeout)
timer.start()
on_finish = wait_for(3, lambda _: dw.stop())
dw.download('http://ajax.googleapis.com/ajax/libs/jquery/2.0.0/jquery.min.js', path_fun, on_finish)
dw.download('http://ajax.googleapis.com/ajax/libs/webfont/1.4.2/webfont.js', path_fun, on_finish)
dw.download('https://raw.githubusercontent.com/sunpy/sunpy/master/README.rst', path_fun, on_finish)
dw.wait()
timer.cancel()
assert len(items) == 3
assert not timeout.fired
for item in items:
assert os.path.exists(item['path'])
def test_path_exception():
x = threading.Event()
dw = Downloader(1, 2)
dw.download("http://google.at", path_fun, errback=wait_for(1, lambda a: x.set()))
th = threading.Thread(target=dw.wait)
th.daemon = True
th.start()
x.wait(10)
assert x.isSet()
dw.stop()
def test_path_exception():
x = threading.Event()
dw = Downloader(1, 2)
dw.download("http://google.at",
path_fun,
errback=wait_for(1, lambda a: x.set()))
th = threading.Thread(target=dw.wait)
th.daemon = True
th.start()
x.wait(10)
assert x.isSet()
dw.stop()
def test_download_default_dir():
_config = sunpy.config
try:
tmpdir = tempfile.mkdtemp()
sunpy.config = MockConfig()
sunpy.config.add_section("downloads", {"download_dir": tmpdir})
dw = Downloader(1, 1)
_stop = lambda _: dw.stop()
timeout = CalledProxy(dw.stop)
errback = CalledProxy(_stop)
dw.download(
'http://ajax.googleapis.com/ajax/libs/jquery/2.0.0/jquery.min.js',
callback=_stop,
errback=errback)
timer = threading.Timer(10, timeout)
timer.start()
dw.wait()
timer.cancel()
assert not timeout.fired
assert not errback.fired
assert os.path.exists(os.path.join(tmpdir, 'jquery.min.js'))
finally:
sunpy.config = _config
def test_download_default_dir():
_config = sunpy.config
try:
tmpdir = tempfile.mkdtemp()
sunpy.config = MockConfig()
sunpy.config.add_section(
"downloads", {"download_dir": tmpdir}
)
dw = Downloader(1, 1)
_stop = lambda _: dw.stop()
timeout = CalledProxy(dw.stop)
errback = CalledProxy(_stop)
dw.download(
'http://ajax.googleapis.com/ajax/libs/jquery/2.0.0/jquery.min.js',
callback=_stop,
errback=errback
)
timer = threading.Timer(10, timeout)
timer.start()
dw.wait()
timer.cancel()
assert not timeout.fired
assert not errback.fired
assert os.path.exists(os.path.join(tmpdir, 'jquery.min.js'))
finally:
sunpy.config = _config
def test_download_http():
items = []
lck = threading.Lock()
def wait_for(n, callback): # pylint: disable=W0613
def _fun(handler):
with lck:
items.append(handler)
if len(items) == n:
callback(items)
return _fun
tmp = tempfile.mkdtemp()
path_fun = partial(default_name, tmp)
dw = Downloader(1, 1)
_stop = lambda _: dw.stop()
timeout = CalledProxy(dw.stop)
timer = threading.Timer(60, timeout)
timer.start()
on_finish = wait_for(3, lambda _: dw.stop())
dw.download(
'http://ajax.googleapis.com/ajax/libs/jquery/2.0.0/jquery.min.js',
path_fun, on_finish)
dw.download(
'http://ajax.googleapis.com/ajax/libs/webfont/1.4.2/webfont.js',
path_fun, on_finish)
dw.download('https://raw.github.com/sunpy/sunpy/master/INSTALL.txt',
path_fun, on_finish)
# dw.download('ftp://speedtest.inode.at/speedtest-100mb', path_fun, on_finish)
dw.wait()
timer.cancel()
assert len(items) == 3
assert not timeout.fired
for item in items:
assert os.path.exists(item['path'])
def test_download_dir():
tmpdir = tempfile.mkdtemp()
dw = Downloader(1, 1)
_stop = lambda _: dw.stop()
timeout = CalledProxy(dw.stop)
errback = CalledProxy(_stop)
dw.download(
"http://ajax.googleapis.com/ajax/libs/jquery/2.0.0/jquery.min.js", tmpdir, callback=_stop, errback=errback
)
timer = threading.Timer(10, timeout)
timer.start()
dw.wait()
timer.cancel()
assert not timeout.fired
assert not errback.fired
assert os.path.exists(os.path.join(tmpdir, "jquery.min.js"))
def test_download_dir():
tmpdir = tempfile.mkdtemp()
dw = Downloader(1, 1)
_stop = lambda _: dw.stop()
timeout = CalledProxy(dw.stop)
errback = CalledProxy(_stop)
dw.download(
'http://ajax.googleapis.com/ajax/libs/jquery/2.0.0/jquery.min.js',
tmpdir,
callback=_stop,
errback=errback)
timer = threading.Timer(10, timeout)
timer.start()
dw.wait()
timer.cancel()
assert not timeout.fired
assert not errback.fired
assert os.path.exists(os.path.join(tmpdir, 'jquery.min.js'))
def get_request(self,
requests,
path=None,
overwrite=False,
progress=True,
max_conn=5,
downloader=None,
results=None):
"""
Query JSOC to see if the request(s) is ready for download.
If the request is ready for download, it will then download it.
Parameters
----------
requests : `~drms.ExportRequest`, `str`, `list`
`~drms.ExportRequest` objects or `str` request IDs or lists
returned by `~sunpy.net.jsoc.jsoc.JSOCClient.request_data`.
path : `str`
Path to save data to, defaults to SunPy download dir.
overwrite : `bool`
Replace files with the same name if True.
progress : `bool`
Print progress info to terminal.
max_conns : `int`
Maximum number of download connections.
downloader : `~sunpy.net.download.Downloader`
A Custom downloader to use
results: `~sunpy.net.download.Results`
A `~sunpy.net.download.Results` manager to use.
Returns
-------
res: `~sunpy.net.download.Results`
A `~sunpy.net.download.Results` instance or `None` if no URLs to download
"""
c = drms.Client()
# Convert Responses to a list if not already
if isinstance(requests, str) or not isiterable(requests):
requests = [requests]
# Ensure all the requests are drms ExportRequest objects
for i, request in enumerate(requests):
if isinstance(request, str):
r = c.export_from_id(request)
requests[i] = r
# We only download if all are finished
if not all([r.has_succeeded() for r in requests]):
raise NotExportedError("Can not download as not all the requests "
"have been exported for download yet.")
# Ensure path has a {file} in it
if path is None:
default_dir = config.get("downloads", "download_dir")
path = os.path.join(default_dir, '{file}')
elif isinstance(path, str) and '{file}' not in path:
path = os.path.join(path, '{file}')
paths = []
for request in requests:
for filename in request.data['filename']:
# Ensure we don't duplicate the file extension
ext = os.path.splitext(filename)[1]
if path.endswith(ext):
fname = path.strip(ext)
else:
fname = path
fname = fname.format(file=filename)
fname = os.path.expanduser(fname)
fname = partial(simple_path, fname)
paths.append(fname)
if downloader is None:
downloader = Downloader(max_conn=max_conn, max_total=max_conn)
# A Results object tracks the number of downloads requested and the
# number that have been completed.
if results is None:
results = Results(
lambda _: downloader.stop(),
done=lambda maps: [v['path'] for v in maps.values()])
urls = []
for request in requests:
if request.status == 0:
for index, data in request.data.iterrows():
is_file = os.path.isfile(paths[index].args[0])
if overwrite or not is_file:
url_dir = request.request_url + '/'
urls.append(
urllib.parse.urljoin(url_dir, data['filename']))
if not overwrite and is_file:
print_message = "Skipping download of file {} as it " \
"has already been downloaded. " \
"If you want to redownload the data, "\
"please set overwrite to True"
print(print_message.format(data['filename']))
# Add the file on disk to the output
results.map_.update(
{data['filename']: {
'path': paths[index].args[0]
}})
if urls:
if progress:
print_message = "{0} URLs found for download. Full request totalling {1}MB"
print(print_message.format(len(urls), request._d['size']))
for i, url in enumerate(urls):
downloader.download(url,
callback=results.require([url]),
errback=lambda x: print(x),
path=paths[i])
else:
# Make Results think it has finished.
results.require([])
results.poke()
return results
def get_request(self,
requestIDs,
path=None,
overwrite=False,
progress=True,
max_conn=5,
downloader=None,
results=None):
"""
Query JSOC to see if request_id is ready for download.
If the request is ready for download, download it.
Parameters
----------
requestIDs : list or string
One or many requestID strings
path : string
Path to save data to, defaults to SunPy download dir
overwrite : bool
Replace files with the same name if True
progress : bool
Print progress info to terminal
max_conns : int
Maximum number of download connections.
downloader : `sunpy.download.Downloader` instance
A Custom downloader to use
results: Results instance
A Results manager to use.
Returns
-------
res: Results
A Results instance or None if no URLs to download
"""
# Convert IDs to a list if not already
if not isiterable(requestIDs) or isinstance(requestIDs,
six.string_types):
requestIDs = [requestIDs]
if path is None:
path = config.get('downloads', 'download_dir')
path = os.path.expanduser(path)
if downloader is None:
downloader = Downloader(max_conn=max_conn, max_total=max_conn)
# A Results object tracks the number of downloads requested and the
# number that have been completed.
if results is None:
results = Results(lambda _: downloader.stop())
urls = []
for request_id in requestIDs:
u = self._request_status(request_id)
if u.status_code == 200 and u.json()['status'] == '0':
for ar in u.json()['data']:
is_file = os.path.isfile(os.path.join(
path, ar['filename']))
if overwrite or not is_file:
url_dir = BASE_DL_URL + u.json()['dir'] + '/'
urls.append(
urllib.parse.urljoin(url_dir, ar['filename']))
else:
print_message = "Skipping download of file {} as it " \
"has already been downloaded"
print(print_message.format(ar['filename']))
# Add the file on disk to the output
results.map_.update({
ar['filename']: {
'path': os.path.join(path, ar['filename'])
}
})
if progress:
print_message = "{0} URLs found for download. Totalling {1}MB"
print(print_message.format(len(urls), u.json()['size']))
else:
if progress:
self.check_request(request_id)
if urls:
for url in urls:
downloader.download(url,
callback=results.require([url]),
errback=lambda x: print(x),
path=path)
else:
# Make Results think it has finished.
results.require([])
results.poke()
return results
def get_request(self, requestIDs, path=None, overwrite=False, progress=True,
max_conn=5, downloader=None, results=None):
"""
Query JSOC to see if request_id is ready for download.
If the request is ready for download, download it.
Parameters
----------
requestIDs : list or string
One or many requestID strings
path : string
Path to save data to, defaults to SunPy download dir
overwrite : bool
Replace files with the same name if True
progress : bool
Print progress info to terminal
max_conns : int
Maximum number of download connections.
downloader : `sunpy.download.Downloader` instance
A Custom downloader to use
results: Results instance
A Results manager to use.
Returns
-------
res: Results
A Results instance or None if no URLs to download
"""
# Convert IDs to a list if not already
if not isiterable(requestIDs) or isinstance(requestIDs, six.string_types):
requestIDs = [requestIDs]
if path is None:
path = config.get('downloads', 'download_dir')
path = os.path.expanduser(path)
if downloader is None:
downloader = Downloader(max_conn=max_conn, max_total=max_conn)
# A Results object tracks the number of downloads requested and the
# number that have been completed.
if results is None:
results = Results(lambda _: downloader.stop())
urls = []
for request_id in requestIDs:
u = self._request_status(request_id)
if u.status_code == 200 and u.json()['status'] == '0':
for ar in u.json()['data']:
is_file = os.path.isfile(os.path.join(path, ar['filename']))
if overwrite or not is_file:
url_dir = BASE_DL_URL + u.json()['dir'] + '/'
urls.append(urllib.parse.urljoin(url_dir, ar['filename']))
else:
print_message = "Skipping download of file {} as it " \
"has already been downloaded"
print(print_message.format(ar['filename']))
# Add the file on disk to the output
results.map_.update({ar['filename']:
{'path': os.path.join(path, ar['filename'])}})
if progress:
print_message = "{0} URLs found for download. Totalling {1}MB"
print(print_message.format(len(urls), u.json()['size']))
else:
if progress:
self.check_request(request_id)
if urls:
for url in urls:
downloader.download(url, callback=results.require([url]),
errback=lambda x: print(x), path=path)
else:
# Make Results think it has finished.
results.require([])
results.poke()
return results
def get_request(self, requests, path=None, overwrite=False, progress=True,
max_conn=5, downloader=None, results=None):
"""
Query JSOC to see if the request(s) is ready for download.
If the request is ready for download, it will then download it.
Parameters
----------
requests : `~drms.ExportRequest`, `str`, `list`
`~drms.ExportRequest` objects or `str` request IDs or lists
returned by `~sunpy.net.jsoc.jsoc.JSOCClient.request_data`.
path : `str`
Path to save data to, defaults to SunPy download dir.
overwrite : `bool`
Replace files with the same name if True.
progress : `bool`
Print progress info to terminal.
max_conns : `int`
Maximum number of download connections.
downloader : `~sunpy.net.download.Downloader`
A Custom downloader to use
results: `~sunpy.net.download.Results`
A `~sunpy.net.download.Results` manager to use.
Returns
-------
res: `~sunpy.net.download.Results`
A `~sunpy.net.download.Results` instance or `None` if no URLs to download
"""
c = drms.Client()
# Convert Responses to a list if not already
if isinstance(requests, six.string_types) or not isiterable(requests):
requests = [requests]
# Ensure all the requests are drms ExportRequest objects
for i, request in enumerate(requests):
if isinstance(request, six.string_types):
r = c.export_from_id(request)
requests[i] = r
# We only download if all are finished
if not all([r.has_succeeded() for r in requests]):
raise NotExportedError("Can not download as not all the requests"
"have been exported for download yet.")
# Ensure path has a {file} in it
if path is None:
default_dir = config.get("downloads", "download_dir")
path = os.path.join(default_dir, '{file}')
elif isinstance(path, six.string_types) and '{file}' not in path:
path = os.path.join(path, '{file}')
paths = []
for request in requests:
for filename in request.data['filename']:
# Ensure we don't duplicate the file extension
ext = os.path.splitext(filename)[1]
if path.endswith(ext):
fname = path.strip(ext)
else:
fname = path
fname = fname.format(file=filename)
fname = os.path.expanduser(fname)
fname = partial(simple_path, fname)
paths.append(fname)
if downloader is None:
downloader = Downloader(max_conn=max_conn, max_total=max_conn)
# A Results object tracks the number of downloads requested and the
# number that have been completed.
if results is None:
results = Results(lambda _: downloader.stop(),
done=lambda maps: [v['path'] for v in maps.values()])
urls = []
for request in requests:
if request.status == 0:
for index, data in request.data.iterrows():
is_file = os.path.isfile(paths[index].args[0])
if overwrite or not is_file:
url_dir = request.request_url + '/'
urls.append(urllib.parse.urljoin(url_dir, data['filename']))
if not overwrite and is_file:
print_message = "Skipping download of file {} as it " \
"has already been downloaded. " \
"If you want to redownload the data, "\
"please set overwrite to True"
print(print_message.format(data['filename']))
# Add the file on disk to the output
results.map_.update({data['filename']:
{'path': paths[index].args[0]}})
if urls:
if progress:
print_message = "{0} URLs found for download. Full request totalling {1}MB"
print(print_message.format(len(urls), request._d['size']))
for i, url in enumerate(urls):
downloader.download(url, callback=results.require([url]),
errback=lambda x: print(x), path=paths[i])
else:
# Make Results think it has finished.
results.require([])
results.poke()
return results
