def local_md5(self): """Checksum for local file. Returns ------- str """ if self.local_exists() and self._local_md5 is None: self._local_md5 = get_file_hash(filename=self.filename) return self._local_md5
def md5(self): """Checksum for file. Returns ------- str """ if self.exists(): return get_file_hash(filename=self.filename) else: return None
def download(self): """Download remote file and save it as local file. Returns ------- self """ try: if self.local_changed(): try: from urllib.request import urlretrieve except ImportError: from urllib import urlretrieve # Set socket timeout socket.setdefaulttimeout(self.socket_timeout) def progress_hook(t): """ Wraps tqdm instance. Don't forget to close() or __exit__() the tqdm instance once you're done with it (easiest using `with` syntax). """ last_b = [0] def inner(b=1, bsize=1, tsize=None): """ b : int, optional Number of blocks just transferred [default: 1]. bsize : int, optional Size of each block (in tqdm units) [default: 1]. tsize : int, optional Total size (in tqdm units). If [default: None] remains unchanged. """ if tsize is not None: t.total = tsize t.update((b - last_b[0]) * bsize) last_b[0] = b return inner tmp_file = self.filename + '.partial_download' with tqdm(desc="{0: >25s}".format(os.path.splitext(self.remote_file.split('/')[-1])[0]), file=sys.stdout, unit='B', unit_scale=True, miniters=1, leave=False, disable=self.disable_progress_bar, ascii=self.use_ascii_progress_bar) as t: local_filename, headers = urlretrieve( url=self.remote_file, filename=tmp_file, reporthook=progress_hook(t), data=None ) tmp_md5 = get_file_hash(filename=tmp_file) file_valid = True if self.remote_md5 is not None: if tmp_md5 == self.remote_md5: file_valid = True else: message = '{name}: Download failed [{filename}] [md5 mismatch]'.format( name=self.__class__.__name__, filename=self.remote_file, ) self.logger.exception(message) raise IOError(message) if file_valid: self._local_md5 = tmp_md5 os.rename(tmp_file, self.filename) except Exception as e: message = '{name}: Download failed [{filename}] [{error_number}: {strerror}]'.format( name=self.__class__.__name__, filename=self.remote_file, error_number=e.errno if hasattr(e, 'errno') else '', strerror=e.strerror if hasattr(e, 'strerror') else '', ) self.logger.exception(message) raise return self