def download_file( url: str, file: Optional[str] = None, user_agent: Optional[str] = None, md5: Optional[str] = None, ) -> str: if file is None: file = path.basename(url) if user_agent is None: user_agent = "pystiche" else: warnings.warn( build_deprecation_message("The parameter user_agent", "0.6.0")) request = Request(url, headers={"User-Agent": user_agent}) try: with urlopen(request) as response, open(file, "wb") as fh: fh.write(response.read()) except HTTPError as error: msg = f"The server returned {error.code}: {error.reason}." raise RuntimeError(msg) from error if md5 is not None and not check_md5(file, md5): raise RuntimeError(f"The MD5 checksum of {file} mismatches.") return file
def download(self, root: Optional[str] = None, overwrite: bool = False) -> None: r"""Download the image and if applicable the guides from their URL. If the correct MD5 checksum is known, it is verified first. If it checks out the file not re-downloaded. Args: root: Optional root directory for the download if the file is a relative path. Defaults to :func:`pystiche.home`. overwrite: Overwrites files if they already exists or the MD5 checksum does not match. Defaults to ``False``. """ def _download(file: str) -> None: os.makedirs(path.dirname(file), exist_ok=True) download_file(self.url, file=file, md5=self.md5) if root is None: root = pystiche.home() if isinstance(self.guides, DownloadableImageCollection): self.guides.download(root=root, overwrite=overwrite) file = self.file if not path.isabs(file) and root is not None: file = path.join(root, file) if not path.isfile(file): _download(file) return msg_overwrite = "If you want to overwrite it, set overwrite=True." if self.md5 is None: if overwrite: _download(file) return else: msg = f"{path.basename(file)} already exists in {root}. {msg_overwrite}" raise FileExistsError(msg) if not check_md5(file, self.md5): if overwrite: _download(file) return else: msg = ( f"{path.basename(file)} with a different MD5 hash already exists " f"in {root}. {msg_overwrite}") raise FileExistsError(msg)
def download_and_extract(download_root: str, extract_root: str): file_name = os.path.join(download_root, 'DvsGesture.tar.gz') if os.path.exists(file_name): print('DvsGesture.tar.gz already exists, check md5') if utils.check_md5(file_name, resource[1]): print('md5 checked, extracting...') utils.extract_archive(file_name, extract_root) return else: print(f'{file_name} corrupted.') print(f'Please download from {resource[0]} and save to {download_root} manually.') raise NotImplementedError
def download_and_extract(download_root: str, extract_root: str): for key in resource.keys(): file_name = os.path.join(download_root, key + '.zip') if os.path.exists(file_name): print('Train.zip already exists, check md5') if utils.check_md5(file_name, resource[key][1]): print('md5 checked, extracting...') utils.extract_archive(file_name, extract_root) else: print(f'{file_name} corrupted.') print( f'Please re-download {file_name} from {resource[key]} and save to {download_root} manually.' ) raise NotImplementedError else: print( f'Please download from {resource[key]} and save to {download_root} manually.' ) raise NotImplementedError
def download_and_extract(download_root: str, extract_root: str): file_name = os.path.join(download_root, 'ICCV2019_DVS_dataset.zip') if os.path.exists(file_name): print('ICCV2019_DVS_dataset.zip already exists, check md5') if utils.check_md5(file_name, resource[1]): print('md5 checked, extracting...') temp_extract_root = os.path.join(download_root, 'temp_extract') os.mkdir(temp_extract_root) utils.extract_archive(file_name, temp_extract_root) for zip_file in tqdm.tqdm(utils.list_files(temp_extract_root, '.zip')): utils.extract_archive(os.path.join(temp_extract_root, zip_file), extract_root) shutil.rmtree(temp_extract_root) return else: print(f'{file_name} corrupted.') print(f'Please download from {resource[0]} and save to {download_root} manually.') raise NotImplementedError
def download_and_extract(download_root: str, extract_root: str): for key in resource.keys(): file_name = os.path.join(download_root, key + '.zip') if os.path.exists(file_name): if utils.check_md5(file_name, resource[key][1]): print(f'extract {file_name} to {extract_root}') utils.extract_archive(file_name, extract_root) else: print(f'{file_name} corrupted, re-download...') utils.download_and_extract_archive(resource[key][0], download_root, extract_root, filename=key + '.zip', md5=resource[key][1]) else: utils.download_and_extract_archive(resource[key][0], download_root, extract_root, filename=key + '.zip', md5=resource[key][1])
def download(self, root: Optional[str] = None, overwrite: bool = False) -> None: def _download(file: str) -> None: os.makedirs(path.dirname(file), exist_ok=True) download_file(self.url, file) if root is None: root = pystiche.home() if isinstance(self.guides, DownloadableImageCollection): self.guides.download(root=root, overwrite=overwrite) file = self.file if not path.isabs(file) and root is not None: file = path.join(root, file) if not path.isfile(file): _download(file) return msg_overwrite = "If you want to overwrite it, set overwrite=True." if self.md5 is None: if overwrite: _download(file) return else: msg = f"{path.basename(file)} already exists in {root}. {msg_overwrite}" raise FileExistsError(msg) if not check_md5(file, self.md5): if overwrite: _download(file) return else: msg = ( f"{path.basename(file)} with a different MD5 hash already exists " f"in {root}. {msg_overwrite}") raise FileExistsError(msg)
def download_file( url: str, file: Optional[str] = None, md5: Optional[str] = None, ) -> str: if file is None: file = path.basename(url) request = Request( url, headers={"User-Agent": f"pystiche/{pystiche.__version__}"}) try: with urlopen(request) as response, open(file, "wb") as fh: fh.write(response.read()) except HTTPError as error: msg = f"The server returned {error.code}: {error.reason}." raise RuntimeError(msg) from error if md5 is not None and not check_md5(file, md5): raise RuntimeError(f"The MD5 checksum of {file} mismatches.") return file
def test_check_md5(self): fpath = TEST_FILE correct_md5 = '9c0bb82894bb3af7f7675ef2b3b6dcdc' false_md5 = '' self.assertTrue(utils.check_md5(fpath, correct_md5)) self.assertFalse(utils.check_md5(fpath, false_md5))
def test_check_md5(self): fpath = TEST_FILE correct_md5 = '9c0bb82894bb3af7f7675ef2b3b6dcdc' false_md5 = '' assert utils.check_md5(fpath, correct_md5) assert not utils.check_md5(fpath, false_md5)