def download_file(
url: str,
file: Optional[str] = None,
user_agent: Optional[str] = None,
md5: Optional[str] = None,
) -> str:
if file is None:
file = path.basename(url)
if user_agent is None:
user_agent = "pystiche"
else:
warnings.warn(
build_deprecation_message("The parameter user_agent", "0.6.0"))
request = Request(url, headers={"User-Agent": user_agent})
try:
with urlopen(request) as response, open(file, "wb") as fh:
fh.write(response.read())
except HTTPError as error:
msg = f"The server returned {error.code}: {error.reason}."
raise RuntimeError(msg) from error
if md5 is not None and not check_md5(file, md5):
raise RuntimeError(f"The MD5 checksum of {file} mismatches.")
return file
def download(self,
root: Optional[str] = None,
overwrite: bool = False) -> None:
r"""Download the image and if applicable the guides from their URL. If the
correct MD5 checksum is known, it is verified first. If it checks out the file
not re-downloaded.
Args:
root: Optional root directory for the download if the file is a relative
path. Defaults to :func:`pystiche.home`.
overwrite: Overwrites files if they already exists or the MD5 checksum does
not match. Defaults to ``False``.
"""
def _download(file: str) -> None:
os.makedirs(path.dirname(file), exist_ok=True)
download_file(self.url, file=file, md5=self.md5)
if root is None:
root = pystiche.home()
if isinstance(self.guides, DownloadableImageCollection):
self.guides.download(root=root, overwrite=overwrite)
file = self.file
if not path.isabs(file) and root is not None:
file = path.join(root, file)
if not path.isfile(file):
_download(file)
return
msg_overwrite = "If you want to overwrite it, set overwrite=True."
if self.md5 is None:
if overwrite:
_download(file)
return
else:
msg = f"{path.basename(file)} already exists in {root}. {msg_overwrite}"
raise FileExistsError(msg)
if not check_md5(file, self.md5):
if overwrite:
_download(file)
return
else:
msg = (
f"{path.basename(file)} with a different MD5 hash already exists "
f"in {root}. {msg_overwrite}")
raise FileExistsError(msg)
def download_and_extract(download_root: str, extract_root: str):
file_name = os.path.join(download_root, 'DvsGesture.tar.gz')
if os.path.exists(file_name):
print('DvsGesture.tar.gz already exists, check md5')
if utils.check_md5(file_name, resource[1]):
print('md5 checked, extracting...')
utils.extract_archive(file_name, extract_root)
return
else:
print(f'{file_name} corrupted.')
print(f'Please download from {resource[0]} and save to {download_root} manually.')
raise NotImplementedError
def download_and_extract(download_root: str, extract_root: str):
for key in resource.keys():
file_name = os.path.join(download_root, key + '.zip')
if os.path.exists(file_name):
print('Train.zip already exists, check md5')
if utils.check_md5(file_name, resource[key][1]):
print('md5 checked, extracting...')
utils.extract_archive(file_name, extract_root)
else:
print(f'{file_name} corrupted.')
print(
f'Please re-download {file_name} from {resource[key]} and save to {download_root} manually.'
)
raise NotImplementedError
else:
print(
f'Please download from {resource[key]} and save to {download_root} manually.'
)
raise NotImplementedError
def download_and_extract(download_root: str, extract_root: str):
file_name = os.path.join(download_root, 'ICCV2019_DVS_dataset.zip')
if os.path.exists(file_name):
print('ICCV2019_DVS_dataset.zip already exists, check md5')
if utils.check_md5(file_name, resource[1]):
print('md5 checked, extracting...')
temp_extract_root = os.path.join(download_root, 'temp_extract')
os.mkdir(temp_extract_root)
utils.extract_archive(file_name, temp_extract_root)
for zip_file in tqdm.tqdm(utils.list_files(temp_extract_root, '.zip')):
utils.extract_archive(os.path.join(temp_extract_root, zip_file), extract_root)
shutil.rmtree(temp_extract_root)
return
else:
print(f'{file_name} corrupted.')
print(f'Please download from {resource[0]} and save to {download_root} manually.')
raise NotImplementedError
def download_and_extract(download_root: str, extract_root: str):
for key in resource.keys():
file_name = os.path.join(download_root, key + '.zip')
if os.path.exists(file_name):
if utils.check_md5(file_name, resource[key][1]):
print(f'extract {file_name} to {extract_root}')
utils.extract_archive(file_name, extract_root)
else:
print(f'{file_name} corrupted, re-download...')
utils.download_and_extract_archive(resource[key][0],
download_root,
extract_root,
filename=key + '.zip',
md5=resource[key][1])
else:
utils.download_and_extract_archive(resource[key][0],
download_root,
extract_root,
filename=key + '.zip',
md5=resource[key][1])
def download(self,
root: Optional[str] = None,
overwrite: bool = False) -> None:
def _download(file: str) -> None:
os.makedirs(path.dirname(file), exist_ok=True)
download_file(self.url, file)
if root is None:
root = pystiche.home()
if isinstance(self.guides, DownloadableImageCollection):
self.guides.download(root=root, overwrite=overwrite)
file = self.file
if not path.isabs(file) and root is not None:
file = path.join(root, file)
if not path.isfile(file):
_download(file)
return
msg_overwrite = "If you want to overwrite it, set overwrite=True."
if self.md5 is None:
if overwrite:
_download(file)
return
else:
msg = f"{path.basename(file)} already exists in {root}. {msg_overwrite}"
raise FileExistsError(msg)
if not check_md5(file, self.md5):
if overwrite:
_download(file)
return
else:
msg = (
f"{path.basename(file)} with a different MD5 hash already exists "
f"in {root}. {msg_overwrite}")
raise FileExistsError(msg)
def download_file(
url: str,
file: Optional[str] = None,
md5: Optional[str] = None,
) -> str:
if file is None:
file = path.basename(url)
request = Request(
url, headers={"User-Agent": f"pystiche/{pystiche.__version__}"})
try:
with urlopen(request) as response, open(file, "wb") as fh:
fh.write(response.read())
except HTTPError as error:
msg = f"The server returned {error.code}: {error.reason}."
raise RuntimeError(msg) from error
if md5 is not None and not check_md5(file, md5):
raise RuntimeError(f"The MD5 checksum of {file} mismatches.")
return file
def test_check_md5(self):
fpath = TEST_FILE
correct_md5 = '9c0bb82894bb3af7f7675ef2b3b6dcdc'
false_md5 = ''
self.assertTrue(utils.check_md5(fpath, correct_md5))
self.assertFalse(utils.check_md5(fpath, false_md5))
def test_check_md5(self):
fpath = TEST_FILE
correct_md5 = '9c0bb82894bb3af7f7675ef2b3b6dcdc'
false_md5 = ''
assert utils.check_md5(fpath, correct_md5)
assert not utils.check_md5(fpath, false_md5)
