def test_download_http(): items = [] lck = threading.Lock() def wait_for(n, callback): # pylint: disable=W0613 def _fun(handler): with lck: items.append(handler) if len(items) == n: callback(items) return _fun tmp = tempfile.mkdtemp() path_fun = partial(default_name, tmp) dw = Downloader(1, 1) _stop = lambda _: dw.stop() timeout = CalledProxy(dw.stop) timer = threading.Timer(60, timeout) timer.start() on_finish = wait_for(3, lambda _: dw.stop()) dw.download('http://ajax.googleapis.com/ajax/libs/jquery/2.0.0/jquery.min.js', path_fun, on_finish) dw.download('http://ajax.googleapis.com/ajax/libs/webfont/1.4.2/webfont.js', path_fun, on_finish) dw.download('https://raw.githubusercontent.com/sunpy/sunpy/master/README.rst', path_fun, on_finish) dw.wait() timer.cancel() assert len(items) == 3 assert not timeout.fired for item in items: assert os.path.exists(item['path'])
def test_path_exception(): x = threading.Event() dw = Downloader(1, 2) dw.download("http://google.at", path_fun, errback=wait_for(1, lambda a: x.set())) th = threading.Thread(target=dw.wait) th.daemon = True th.start() x.wait(10) assert x.isSet() dw.stop()
def test_download_default_dir(): _config = sunpy.config try: tmpdir = tempfile.mkdtemp() sunpy.config = MockConfig() sunpy.config.add_section("downloads", {"download_dir": tmpdir}) dw = Downloader(1, 1) _stop = lambda _: dw.stop() timeout = CalledProxy(dw.stop) errback = CalledProxy(_stop) dw.download( 'http://ajax.googleapis.com/ajax/libs/jquery/2.0.0/jquery.min.js', callback=_stop, errback=errback) timer = threading.Timer(10, timeout) timer.start() dw.wait() timer.cancel() assert not timeout.fired assert not errback.fired assert os.path.exists(os.path.join(tmpdir, 'jquery.min.js')) finally: sunpy.config = _config
def test_download_default_dir(): _config = sunpy.config try: tmpdir = tempfile.mkdtemp() sunpy.config = MockConfig() sunpy.config.add_section( "downloads", {"download_dir": tmpdir} ) dw = Downloader(1, 1) _stop = lambda _: dw.stop() timeout = CalledProxy(dw.stop) errback = CalledProxy(_stop) dw.download( 'http://ajax.googleapis.com/ajax/libs/jquery/2.0.0/jquery.min.js', callback=_stop, errback=errback ) timer = threading.Timer(10, timeout) timer.start() dw.wait() timer.cancel() assert not timeout.fired assert not errback.fired assert os.path.exists(os.path.join(tmpdir, 'jquery.min.js')) finally: sunpy.config = _config
def test_download_http(): items = [] lck = threading.Lock() def wait_for(n, callback): # pylint: disable=W0613 def _fun(handler): with lck: items.append(handler) if len(items) == n: callback(items) return _fun tmp = tempfile.mkdtemp() path_fun = partial(default_name, tmp) dw = Downloader(1, 1) _stop = lambda _: dw.stop() timeout = CalledProxy(dw.stop) timer = threading.Timer(60, timeout) timer.start() on_finish = wait_for(3, lambda _: dw.stop()) dw.download( 'http://ajax.googleapis.com/ajax/libs/jquery/2.0.0/jquery.min.js', path_fun, on_finish) dw.download( 'http://ajax.googleapis.com/ajax/libs/webfont/1.4.2/webfont.js', path_fun, on_finish) dw.download('https://raw.github.com/sunpy/sunpy/master/INSTALL.txt', path_fun, on_finish) # dw.download('ftp://speedtest.inode.at/speedtest-100mb', path_fun, on_finish) dw.wait() timer.cancel() assert len(items) == 3 assert not timeout.fired for item in items: assert os.path.exists(item['path'])
def test_download_dir(): tmpdir = tempfile.mkdtemp() dw = Downloader(1, 1) _stop = lambda _: dw.stop() timeout = CalledProxy(dw.stop) errback = CalledProxy(_stop) dw.download( "http://ajax.googleapis.com/ajax/libs/jquery/2.0.0/jquery.min.js", tmpdir, callback=_stop, errback=errback ) timer = threading.Timer(10, timeout) timer.start() dw.wait() timer.cancel() assert not timeout.fired assert not errback.fired assert os.path.exists(os.path.join(tmpdir, "jquery.min.js"))
def test_download_dir(): tmpdir = tempfile.mkdtemp() dw = Downloader(1, 1) _stop = lambda _: dw.stop() timeout = CalledProxy(dw.stop) errback = CalledProxy(_stop) dw.download( 'http://ajax.googleapis.com/ajax/libs/jquery/2.0.0/jquery.min.js', tmpdir, callback=_stop, errback=errback) timer = threading.Timer(10, timeout) timer.start() dw.wait() timer.cancel() assert not timeout.fired assert not errback.fired assert os.path.exists(os.path.join(tmpdir, 'jquery.min.js'))
def get_request(self, requests, path=None, overwrite=False, progress=True, max_conn=5, downloader=None, results=None): """ Query JSOC to see if the request(s) is ready for download. If the request is ready for download, it will then download it. Parameters ---------- requests : `~drms.ExportRequest`, `str`, `list` `~drms.ExportRequest` objects or `str` request IDs or lists returned by `~sunpy.net.jsoc.jsoc.JSOCClient.request_data`. path : `str` Path to save data to, defaults to SunPy download dir. overwrite : `bool` Replace files with the same name if True. progress : `bool` Print progress info to terminal. max_conns : `int` Maximum number of download connections. downloader : `~sunpy.net.download.Downloader` A Custom downloader to use results: `~sunpy.net.download.Results` A `~sunpy.net.download.Results` manager to use. Returns ------- res: `~sunpy.net.download.Results` A `~sunpy.net.download.Results` instance or `None` if no URLs to download """ c = drms.Client() # Convert Responses to a list if not already if isinstance(requests, str) or not isiterable(requests): requests = [requests] # Ensure all the requests are drms ExportRequest objects for i, request in enumerate(requests): if isinstance(request, str): r = c.export_from_id(request) requests[i] = r # We only download if all are finished if not all([r.has_succeeded() for r in requests]): raise NotExportedError("Can not download as not all the requests " "have been exported for download yet.") # Ensure path has a {file} in it if path is None: default_dir = config.get("downloads", "download_dir") path = os.path.join(default_dir, '{file}') elif isinstance(path, str) and '{file}' not in path: path = os.path.join(path, '{file}') paths = [] for request in requests: for filename in request.data['filename']: # Ensure we don't duplicate the file extension ext = os.path.splitext(filename)[1] if path.endswith(ext): fname = path.strip(ext) else: fname = path fname = fname.format(file=filename) fname = os.path.expanduser(fname) fname = partial(simple_path, fname) paths.append(fname) if downloader is None: downloader = Downloader(max_conn=max_conn, max_total=max_conn) # A Results object tracks the number of downloads requested and the # number that have been completed. if results is None: results = Results( lambda _: downloader.stop(), done=lambda maps: [v['path'] for v in maps.values()]) urls = [] for request in requests: if request.status == 0: for index, data in request.data.iterrows(): is_file = os.path.isfile(paths[index].args[0]) if overwrite or not is_file: url_dir = request.request_url + '/' urls.append( urllib.parse.urljoin(url_dir, data['filename'])) if not overwrite and is_file: print_message = "Skipping download of file {} as it " \ "has already been downloaded. " \ "If you want to redownload the data, "\ "please set overwrite to True" print(print_message.format(data['filename'])) # Add the file on disk to the output results.map_.update( {data['filename']: { 'path': paths[index].args[0] }}) if urls: if progress: print_message = "{0} URLs found for download. Full request totalling {1}MB" print(print_message.format(len(urls), request._d['size'])) for i, url in enumerate(urls): downloader.download(url, callback=results.require([url]), errback=lambda x: print(x), path=paths[i]) else: # Make Results think it has finished. results.require([]) results.poke() return results
def get_request(self, requestIDs, path=None, overwrite=False, progress=True, max_conn=5, downloader=None, results=None): """ Query JSOC to see if request_id is ready for download. If the request is ready for download, download it. Parameters ---------- requestIDs : list or string One or many requestID strings path : string Path to save data to, defaults to SunPy download dir overwrite : bool Replace files with the same name if True progress : bool Print progress info to terminal max_conns : int Maximum number of download connections. downloader : `sunpy.download.Downloader` instance A Custom downloader to use results: Results instance A Results manager to use. Returns ------- res: Results A Results instance or None if no URLs to download """ # Convert IDs to a list if not already if not isiterable(requestIDs) or isinstance(requestIDs, six.string_types): requestIDs = [requestIDs] if path is None: path = config.get('downloads', 'download_dir') path = os.path.expanduser(path) if downloader is None: downloader = Downloader(max_conn=max_conn, max_total=max_conn) # A Results object tracks the number of downloads requested and the # number that have been completed. if results is None: results = Results(lambda _: downloader.stop()) urls = [] for request_id in requestIDs: u = self._request_status(request_id) if u.status_code == 200 and u.json()['status'] == '0': for ar in u.json()['data']: is_file = os.path.isfile(os.path.join( path, ar['filename'])) if overwrite or not is_file: url_dir = BASE_DL_URL + u.json()['dir'] + '/' urls.append( urllib.parse.urljoin(url_dir, ar['filename'])) else: print_message = "Skipping download of file {} as it " \ "has already been downloaded" print(print_message.format(ar['filename'])) # Add the file on disk to the output results.map_.update({ ar['filename']: { 'path': os.path.join(path, ar['filename']) } }) if progress: print_message = "{0} URLs found for download. Totalling {1}MB" print(print_message.format(len(urls), u.json()['size'])) else: if progress: self.check_request(request_id) if urls: for url in urls: downloader.download(url, callback=results.require([url]), errback=lambda x: print(x), path=path) else: # Make Results think it has finished. results.require([]) results.poke() return results
def get_request(self, requestIDs, path=None, overwrite=False, progress=True, max_conn=5, downloader=None, results=None): """ Query JSOC to see if request_id is ready for download. If the request is ready for download, download it. Parameters ---------- requestIDs : list or string One or many requestID strings path : string Path to save data to, defaults to SunPy download dir overwrite : bool Replace files with the same name if True progress : bool Print progress info to terminal max_conns : int Maximum number of download connections. downloader : `sunpy.download.Downloader` instance A Custom downloader to use results: Results instance A Results manager to use. Returns ------- res: Results A Results instance or None if no URLs to download """ # Convert IDs to a list if not already if not isiterable(requestIDs) or isinstance(requestIDs, six.string_types): requestIDs = [requestIDs] if path is None: path = config.get('downloads', 'download_dir') path = os.path.expanduser(path) if downloader is None: downloader = Downloader(max_conn=max_conn, max_total=max_conn) # A Results object tracks the number of downloads requested and the # number that have been completed. if results is None: results = Results(lambda _: downloader.stop()) urls = [] for request_id in requestIDs: u = self._request_status(request_id) if u.status_code == 200 and u.json()['status'] == '0': for ar in u.json()['data']: is_file = os.path.isfile(os.path.join(path, ar['filename'])) if overwrite or not is_file: url_dir = BASE_DL_URL + u.json()['dir'] + '/' urls.append(urllib.parse.urljoin(url_dir, ar['filename'])) else: print_message = "Skipping download of file {} as it " \ "has already been downloaded" print(print_message.format(ar['filename'])) # Add the file on disk to the output results.map_.update({ar['filename']: {'path': os.path.join(path, ar['filename'])}}) if progress: print_message = "{0} URLs found for download. Totalling {1}MB" print(print_message.format(len(urls), u.json()['size'])) else: if progress: self.check_request(request_id) if urls: for url in urls: downloader.download(url, callback=results.require([url]), errback=lambda x: print(x), path=path) else: # Make Results think it has finished. results.require([]) results.poke() return results
def get_request(self, requests, path=None, overwrite=False, progress=True, max_conn=5, downloader=None, results=None): """ Query JSOC to see if the request(s) is ready for download. If the request is ready for download, it will then download it. Parameters ---------- requests : `~drms.ExportRequest`, `str`, `list` `~drms.ExportRequest` objects or `str` request IDs or lists returned by `~sunpy.net.jsoc.jsoc.JSOCClient.request_data`. path : `str` Path to save data to, defaults to SunPy download dir. overwrite : `bool` Replace files with the same name if True. progress : `bool` Print progress info to terminal. max_conns : `int` Maximum number of download connections. downloader : `~sunpy.net.download.Downloader` A Custom downloader to use results: `~sunpy.net.download.Results` A `~sunpy.net.download.Results` manager to use. Returns ------- res: `~sunpy.net.download.Results` A `~sunpy.net.download.Results` instance or `None` if no URLs to download """ c = drms.Client() # Convert Responses to a list if not already if isinstance(requests, six.string_types) or not isiterable(requests): requests = [requests] # Ensure all the requests are drms ExportRequest objects for i, request in enumerate(requests): if isinstance(request, six.string_types): r = c.export_from_id(request) requests[i] = r # We only download if all are finished if not all([r.has_succeeded() for r in requests]): raise NotExportedError("Can not download as not all the requests" "have been exported for download yet.") # Ensure path has a {file} in it if path is None: default_dir = config.get("downloads", "download_dir") path = os.path.join(default_dir, '{file}') elif isinstance(path, six.string_types) and '{file}' not in path: path = os.path.join(path, '{file}') paths = [] for request in requests: for filename in request.data['filename']: # Ensure we don't duplicate the file extension ext = os.path.splitext(filename)[1] if path.endswith(ext): fname = path.strip(ext) else: fname = path fname = fname.format(file=filename) fname = os.path.expanduser(fname) fname = partial(simple_path, fname) paths.append(fname) if downloader is None: downloader = Downloader(max_conn=max_conn, max_total=max_conn) # A Results object tracks the number of downloads requested and the # number that have been completed. if results is None: results = Results(lambda _: downloader.stop(), done=lambda maps: [v['path'] for v in maps.values()]) urls = [] for request in requests: if request.status == 0: for index, data in request.data.iterrows(): is_file = os.path.isfile(paths[index].args[0]) if overwrite or not is_file: url_dir = request.request_url + '/' urls.append(urllib.parse.urljoin(url_dir, data['filename'])) if not overwrite and is_file: print_message = "Skipping download of file {} as it " \ "has already been downloaded. " \ "If you want to redownload the data, "\ "please set overwrite to True" print(print_message.format(data['filename'])) # Add the file on disk to the output results.map_.update({data['filename']: {'path': paths[index].args[0]}}) if urls: if progress: print_message = "{0} URLs found for download. Full request totalling {1}MB" print(print_message.format(len(urls), request._d['size'])) for i, url in enumerate(urls): downloader.download(url, callback=results.require([url]), errback=lambda x: print(x), path=paths[i]) else: # Make Results think it has finished. results.require([]) results.poke() return results