def download_file(url, dest, cachename=None): """Downloads a file using a local cache. If the file cannot be downloaded or if it wasn't modified, the cached version will be used instead. The cache lives in ``~/.cache/reprozip/``. """ if cachename is None: cachename = dest.name request = Request(url) if 'XDG_CACHE_HOME' in os.environ: cache = Path(os.environ['XDG_CACHE_HOME']) else: cache = Path('~/.cache').expand_user() cache = cache / 'reprozip' / cachename if cache.exists(): mtime = email.utils.formatdate(cache.mtime(), usegmt=True) request.add_header('If-Modified-Since', mtime) cache.parent.mkdir(parents=True) try: response = urlopen(request) except URLError as e: if cache.exists(): if isinstance(e, HTTPError) and e.code == 304: logging.info("Cached file %s is up to date", cachename) else: logging.warning("Couldn't download %s: %s", url, e) cache.copy(dest) return else: raise if response is None: logging.info("Cached file %s is up to date", cachename) cache.copy(dest) return logging.info("Downloading %s", url) try: CHUNK_SIZE = 4096 with cache.open('wb') as f: while True: chunk = response.read(CHUNK_SIZE) if not chunk: break f.write(chunk) response.close() except Exception as e: # pragma: no cover try: cache.remove() except OSError: pass raise e cache.copy(dest)
def download_file(url, dest, cachename=None): """Downloads a file using a local cache. If the file cannot be downloaded or if it wasn't modified, the cached version will be used instead. The cache lives in ``~/.cache/reprozip/``. """ if cachename is None: cachename = dest.components[-1] request = Request(url) if 'XDG_CACHE_HOME' in os.environ: cache = Path(os.environ['XDG_CACHE_HOME']) else: cache = Path('~/.cache').expand_user() cache = cache / 'reprozip' / cachename if cache.exists(): mtime = email.utils.formatdate(cache.mtime(), usegmt=True) request.add_header('If-Modified-Since', mtime) cache.parent.mkdir(parents=True) try: response = urlopen(request, timeout=2 if cache.exists() else 10) except URLError as e: if cache.exists(): if isinstance(e, HTTPError) and e.code == 304: logging.info("Download %s: cache is up to date", cachename) else: logging.warning("Download %s: error downloading %s: %s", cachename, url, e) cache.copy(dest) return else: raise if response is None: logging.info("Download %s: cache is up to date", cachename) cache.copy(dest) return logging.info("Download %s: downloading %s", cachename, url) try: with cache.open('wb') as f: copyfile(response, f) response.close() except Exception as e: # pragma: no cover try: cache.remove() except OSError: pass raise e logging.info("Downloaded %s successfully", cachename) cache.copy(dest)
def download_file(url, dest, cachename=None, ssl_verify=None): """Downloads a file using a local cache. If the file cannot be downloaded or if it wasn't modified, the cached version will be used instead. The cache lives in ``~/.cache/reprozip/``. """ if cachename is None: if dest is None: raise ValueError("One of 'dest' or 'cachename' must be specified") cachename = dest.components[-1] headers = {} if 'XDG_CACHE_HOME' in os.environ: cache = Path(os.environ['XDG_CACHE_HOME']) else: cache = Path('~/.cache').expand_user() cache = cache / 'reprozip' / cachename if cache.exists(): mtime = email.utils.formatdate(cache.mtime(), usegmt=True) headers['If-Modified-Since'] = mtime cache.parent.mkdir(parents=True) try: response = requests.get(url, headers=headers, timeout=2 if cache.exists() else 10, stream=True, verify=ssl_verify) response.raise_for_status() if response.status_code == 304: raise requests.HTTPError( '304 File is up to date, no data returned', response=response) except requests.RequestException as e: if cache.exists(): if e.response and e.response.status_code == 304: logging.info("Download %s: cache is up to date", cachename) else: logging.warning("Download %s: error downloading %s: %s", cachename, url, e) if dest is not None: cache.copy(dest) return dest else: return cache else: raise logging.info("Download %s: downloading %s", cachename, url) try: with cache.open('wb') as f: for chunk in response.iter_content(4096): f.write(chunk) response.close() except Exception as e: # pragma: no cover try: cache.remove() except OSError: pass raise e logging.info("Downloaded %s successfully", cachename) if dest is not None: cache.copy(dest) return dest else: return cache