def _download_raw_dataset(metadata):
if os.path.exists(metadata["filename"]):
return
print(f"Downloading raw dataset from {metadata['url']}...")
util.download_url(metadata["url"], metadata["filename"])
sha256 = util.compute_sha256(metadata["filename"])
if sha256 != metadata["sha256"]:
raise ValueError("Downloaded data file SHA-256 does not match that listed in metadata document.")
def _download_raw_dataset(metadata):
if os.path.exists(metadata['filename']):
return
print('Downloading raw dataset...')
util.download_url(metadata['url'], metadata['filename'])
print('Computing SHA-256...')
sha256 = util.compute_sha256(metadata['filename'])
if sha256 != metadata['sha256']:
raise ValueError('Downloaded data file SHA-256 does not match that listed in metadata document.')
def _download_raw_dataset(metadata: Dict, dl_dirname: Path) -> Path:
dl_dirname.mkdir(parents=True, exist_ok=True)
filename = dl_dirname / metadata["filename"]
if filename.exists():
return
print(f"Downloading raw dataset from {metadata['url']} to {filename}...")
util.download_url(metadata["url"], filename)
print("Computing SHA-256...")
sha256 = util.compute_sha256(filename)
if sha256 != metadata["sha256"]:
raise ValueError("Downloaded data file SHA-256 does not match that listed in metadata document.")
return filename
def main():
metadata_filename = _get_metadata_filename()
metadata = toml.load(metadata_filename)
data_filename = metadata_filename.parents[0] / metadata['filename']
supposed_data_sha256 = metadata['sha256']
actual_data_sha256 = util.compute_sha256(data_filename)
if supposed_data_sha256 == actual_data_sha256:
print('Nothing to update: SHA-256 matches')
return
print('Updating metadata SHA-256')
metadata['sha256'] = actual_data_sha256
with open(metadata_filename, 'w') as f:
toml.dump(metadata, f)
def main():
metadata_filename = _get_metadata_filename()
metadata = toml.load(metadata_filename)
data_filename = metadata_filename.parents[0] / metadata["filename"]
supposed_data_sha256 = metadata["sha256"]
actual_data_sha256 = util.compute_sha256(data_filename)
if supposed_data_sha256 == actual_data_sha256:
print("Nothing to update: SHA-256 matches")
return
print("Updating metadata SHA-256")
metadata["sha256"] = actual_data_sha256
with open(metadata_filename, "w") as f:
toml.dump(metadata, f)