def download_file(url, dest, cachename=None):
"""Downloads a file using a local cache.
If the file cannot be downloaded or if it wasn't modified, the cached
version will be used instead.
The cache lives in ``~/.cache/reprozip/``.
"""
if cachename is None:
cachename = dest.name
request = Request(url)
if 'XDG_CACHE_HOME' in os.environ:
cache = Path(os.environ['XDG_CACHE_HOME'])
else:
cache = Path('~/.cache').expand_user()
cache = cache / 'reprozip' / cachename
if cache.exists():
mtime = email.utils.formatdate(cache.mtime(), usegmt=True)
request.add_header('If-Modified-Since', mtime)
cache.parent.mkdir(parents=True)
try:
response = urlopen(request)
except URLError as e:
if cache.exists():
if isinstance(e, HTTPError) and e.code == 304:
logging.info("Cached file %s is up to date", cachename)
else:
logging.warning("Couldn't download %s: %s", url, e)
cache.copy(dest)
return
else:
raise
if response is None:
logging.info("Cached file %s is up to date", cachename)
cache.copy(dest)
return
logging.info("Downloading %s", url)
try:
CHUNK_SIZE = 4096
with cache.open('wb') as f:
while True:
chunk = response.read(CHUNK_SIZE)
if not chunk:
break
f.write(chunk)
response.close()
except Exception as e: # pragma: no cover
try:
cache.remove()
except OSError:
pass
raise e
cache.copy(dest)
def download_file(url, dest, cachename=None):
"""Downloads a file using a local cache.
If the file cannot be downloaded or if it wasn't modified, the cached
version will be used instead.
The cache lives in ``~/.cache/reprozip/``.
"""
if cachename is None:
cachename = dest.name
request = Request(url)
if 'XDG_CACHE_HOME' in os.environ:
cache = Path(os.environ['XDG_CACHE_HOME'])
else:
cache = Path('~/.cache').expand_user()
cache = cache / 'reprozip' / cachename
if cache.exists():
mtime = email.utils.formatdate(cache.mtime(), usegmt=True)
request.add_header('If-Modified-Since', mtime)
cache.parent.mkdir(parents=True)
try:
response = urlopen(request)
except URLError as e:
if cache.exists():
if isinstance(e, HTTPError) and e.code == 304:
logging.info("Cached file %s is up to date", cachename)
else:
logging.warning("Couldn't download %s: %s", url, e)
cache.copy(dest)
return
else:
raise
if response is None:
logging.info("Cached file %s is up to date", cachename)
cache.copy(dest)
return
logging.info("Downloading %s", url)
try:
CHUNK_SIZE = 4096
with cache.open('wb') as f:
while True:
chunk = response.read(CHUNK_SIZE)
if not chunk:
break
f.write(chunk)
response.close()
except Exception as e: # pragma: no cover
try:
cache.remove()
except OSError:
pass
raise e
cache.copy(dest)
def download_file(url, dest, cachename=None):
"""Downloads a file using a local cache.
If the file cannot be downloaded or if it wasn't modified, the cached
version will be used instead.
The cache lives in ``~/.cache/reprozip/``.
"""
if cachename is None:
cachename = dest.components[-1]
request = Request(url)
if 'XDG_CACHE_HOME' in os.environ:
cache = Path(os.environ['XDG_CACHE_HOME'])
else:
cache = Path('~/.cache').expand_user()
cache = cache / 'reprozip' / cachename
if cache.exists():
mtime = email.utils.formatdate(cache.mtime(), usegmt=True)
request.add_header('If-Modified-Since', mtime)
cache.parent.mkdir(parents=True)
try:
response = urlopen(request, timeout=2 if cache.exists() else 10)
except URLError as e:
if cache.exists():
if isinstance(e, HTTPError) and e.code == 304:
logging.info("Download %s: cache is up to date", cachename)
else:
logging.warning("Download %s: error downloading %s: %s",
cachename, url, e)
cache.copy(dest)
return
else:
raise
if response is None:
logging.info("Download %s: cache is up to date", cachename)
cache.copy(dest)
return
logging.info("Download %s: downloading %s", cachename, url)
try:
with cache.open('wb') as f:
copyfile(response, f)
response.close()
except Exception as e: # pragma: no cover
try:
cache.remove()
except OSError:
pass
raise e
logging.info("Downloaded %s successfully", cachename)
cache.copy(dest)
def download_file(url, dest, cachename=None, ssl_verify=None):
"""Downloads a file using a local cache.
If the file cannot be downloaded or if it wasn't modified, the cached
version will be used instead.
The cache lives in ``~/.cache/reprozip/``.
"""
if cachename is None:
if dest is None:
raise ValueError("One of 'dest' or 'cachename' must be specified")
cachename = dest.components[-1]
headers = {}
if 'XDG_CACHE_HOME' in os.environ:
cache = Path(os.environ['XDG_CACHE_HOME'])
else:
cache = Path('~/.cache').expand_user()
cache = cache / 'reprozip' / cachename
if cache.exists():
mtime = email.utils.formatdate(cache.mtime(), usegmt=True)
headers['If-Modified-Since'] = mtime
cache.parent.mkdir(parents=True)
try:
response = requests.get(url,
headers=headers,
timeout=2 if cache.exists() else 10,
stream=True,
verify=ssl_verify)
response.raise_for_status()
if response.status_code == 304:
raise requests.HTTPError(
'304 File is up to date, no data returned', response=response)
except requests.RequestException as e:
if cache.exists():
if e.response and e.response.status_code == 304:
logging.info("Download %s: cache is up to date", cachename)
else:
logging.warning("Download %s: error downloading %s: %s",
cachename, url, e)
if dest is not None:
cache.copy(dest)
return dest
else:
return cache
else:
raise
logging.info("Download %s: downloading %s", cachename, url)
try:
with cache.open('wb') as f:
for chunk in response.iter_content(4096):
f.write(chunk)
response.close()
except Exception as e: # pragma: no cover
try:
cache.remove()
except OSError:
pass
raise e
logging.info("Downloaded %s successfully", cachename)
if dest is not None:
cache.copy(dest)
return dest
else:
return cache
def download_file(url, dest, cachename=None, ssl_verify=None):
"""Downloads a file using a local cache.
If the file cannot be downloaded or if it wasn't modified, the cached
version will be used instead.
The cache lives in ``~/.cache/reprozip/``.
"""
if cachename is None:
if dest is None:
raise ValueError("One of 'dest' or 'cachename' must be specified")
cachename = dest.components[-1]
headers = {}
if 'XDG_CACHE_HOME' in os.environ:
cache = Path(os.environ['XDG_CACHE_HOME'])
else:
cache = Path('~/.cache').expand_user()
cache = cache / 'reprozip' / cachename
if cache.exists():
mtime = email.utils.formatdate(cache.mtime(), usegmt=True)
headers['If-Modified-Since'] = mtime
cache.parent.mkdir(parents=True)
try:
response = requests.get(url, headers=headers,
timeout=2 if cache.exists() else 10,
stream=True, verify=ssl_verify)
response.raise_for_status()
if response.status_code == 304:
raise requests.HTTPError(
'304 File is up to date, no data returned',
response=response)
except requests.RequestException as e:
if cache.exists():
if e.response and e.response.status_code == 304:
logging.info("Download %s: cache is up to date", cachename)
else:
logging.warning("Download %s: error downloading %s: %s",
cachename, url, e)
if dest is not None:
cache.copy(dest)
return dest
else:
return cache
else:
raise
logging.info("Download %s: downloading %s", cachename, url)
try:
with cache.open('wb') as f:
for chunk in response.iter_content(4096):
f.write(chunk)
response.close()
except Exception as e: # pragma: no cover
try:
cache.remove()
except OSError:
pass
raise e
logging.info("Downloaded %s successfully", cachename)
if dest is not None:
cache.copy(dest)
return dest
else:
return cache
