def _download_raw_dataset(metadata): if os.path.exists(metadata["filename"]): return print(f"Downloading raw dataset from {metadata['url']}...") util.download_url(metadata["url"], metadata["filename"]) sha256 = util.compute_sha256(metadata["filename"]) if sha256 != metadata["sha256"]: raise ValueError("Downloaded data file SHA-256 does not match that listed in metadata document.")
def _download_raw_dataset(metadata): if os.path.exists(metadata['filename']): return print('Downloading raw dataset...') util.download_url(metadata['url'], metadata['filename']) print('Computing SHA-256...') sha256 = util.compute_sha256(metadata['filename']) if sha256 != metadata['sha256']: raise ValueError('Downloaded data file SHA-256 does not match that listed in metadata document.')
def _download_raw_dataset(metadata: Dict, dl_dirname: Path) -> Path: dl_dirname.mkdir(parents=True, exist_ok=True) filename = dl_dirname / metadata["filename"] if filename.exists(): return print(f"Downloading raw dataset from {metadata['url']} to {filename}...") util.download_url(metadata["url"], filename) print("Computing SHA-256...") sha256 = util.compute_sha256(filename) if sha256 != metadata["sha256"]: raise ValueError("Downloaded data file SHA-256 does not match that listed in metadata document.") return filename
def main(): metadata_filename = _get_metadata_filename() metadata = toml.load(metadata_filename) data_filename = metadata_filename.parents[0] / metadata['filename'] supposed_data_sha256 = metadata['sha256'] actual_data_sha256 = util.compute_sha256(data_filename) if supposed_data_sha256 == actual_data_sha256: print('Nothing to update: SHA-256 matches') return print('Updating metadata SHA-256') metadata['sha256'] = actual_data_sha256 with open(metadata_filename, 'w') as f: toml.dump(metadata, f)
def main(): metadata_filename = _get_metadata_filename() metadata = toml.load(metadata_filename) data_filename = metadata_filename.parents[0] / metadata["filename"] supposed_data_sha256 = metadata["sha256"] actual_data_sha256 = util.compute_sha256(data_filename) if supposed_data_sha256 == actual_data_sha256: print("Nothing to update: SHA-256 matches") return print("Updating metadata SHA-256") metadata["sha256"] = actual_data_sha256 with open(metadata_filename, "w") as f: toml.dump(metadata, f)