def local_md5(self):
"""Checksum for local file.
Returns
-------
str
"""
if self.local_exists() and self._local_md5 is None:
self._local_md5 = get_file_hash(filename=self.filename)
return self._local_md5
def md5(self):
"""Checksum for file.
Returns
-------
str
"""
if self.exists():
return get_file_hash(filename=self.filename)
else:
return None
def download(self):
"""Download remote file and save it as local file.
Returns
-------
self
"""
try:
if self.local_changed():
try:
from urllib.request import urlretrieve
except ImportError:
from urllib import urlretrieve
# Set socket timeout
socket.setdefaulttimeout(self.socket_timeout)
def progress_hook(t):
"""
Wraps tqdm instance. Don't forget to close() or __exit__()
the tqdm instance once you're done with it (easiest using `with` syntax).
"""
last_b = [0]
def inner(b=1, bsize=1, tsize=None):
"""
b : int, optional
Number of blocks just transferred [default: 1].
bsize : int, optional
Size of each block (in tqdm units) [default: 1].
tsize : int, optional
Total size (in tqdm units). If [default: None] remains unchanged.
"""
if tsize is not None:
t.total = tsize
t.update((b - last_b[0]) * bsize)
last_b[0] = b
return inner
tmp_file = self.filename + '.partial_download'
with tqdm(desc="{0: >25s}".format(os.path.splitext(self.remote_file.split('/')[-1])[0]),
file=sys.stdout,
unit='B',
unit_scale=True,
miniters=1,
leave=False,
disable=self.disable_progress_bar,
ascii=self.use_ascii_progress_bar) as t:
local_filename, headers = urlretrieve(
url=self.remote_file,
filename=tmp_file,
reporthook=progress_hook(t),
data=None
)
tmp_md5 = get_file_hash(filename=tmp_file)
file_valid = True
if self.remote_md5 is not None:
if tmp_md5 == self.remote_md5:
file_valid = True
else:
message = '{name}: Download failed [{filename}] [md5 mismatch]'.format(
name=self.__class__.__name__,
filename=self.remote_file,
)
self.logger.exception(message)
raise IOError(message)
if file_valid:
self._local_md5 = tmp_md5
os.rename(tmp_file, self.filename)
except Exception as e:
message = '{name}: Download failed [{filename}] [{error_number}: {strerror}]'.format(
name=self.__class__.__name__,
filename=self.remote_file,
error_number=e.errno if hasattr(e, 'errno') else '',
strerror=e.strerror if hasattr(e, 'strerror') else '',
)
self.logger.exception(message)
raise
return self