def _get_cached_path(
manually_downloaded_path: Optional[epath.Path],
checksum_path: Optional[epath.Path],
url_path: epath.Path,
expected_url_info: Optional[checksums.UrlInfo],
) -> downloader.DownloadResult:
"""Returns the downloaded path and computed url-info.
If the path is not cached, or that `url_path` does not match checksums,
the file will be downloaded again.
Path can be cached at three different locations:
Args:
manually_downloaded_path: Manually downloaded in `dl_manager.manual_dir`
checksum_path: Cached in the final destination (if checksum known)
url_path: Cached in the tmp destination (if checksum unknown).
expected_url_info: Registered checksum (if known)
"""
# User has manually downloaded the file.
if manually_downloaded_path and manually_downloaded_path.exists():
return downloader.DownloadResult(manually_downloaded_path,
url_info=None) # pytype: disable=wrong-arg-types
# Download has been cached (checksum known)
elif checksum_path and resource_lib.Resource.exists_locally(checksum_path):
# `path = f(checksum)` was found, so url_info match
return downloader.DownloadResult(checksum_path,
url_info=expected_url_info)
# Download has been cached (checksum unknown)
elif resource_lib.Resource.exists_locally(url_path):
# Info restored from `.INFO` file
computed_url_info = _read_url_info(url_path)
# If checksums are now registered but do not match, trigger a new
# download (e.g. previous file corrupted, checksums updated)
if expected_url_info and computed_url_info != expected_url_info:
return downloader.DownloadResult(path=None, url_info=None) # pytype: disable=wrong-arg-types
else:
return downloader.DownloadResult(url_path,
url_info=computed_url_info)
# Else file not found (or has bad checksums). (re)download.
else:
return downloader.DownloadResult(path=None, url_info=None) # pytype: disable=wrong-arg-types
def _download(url, tmpdir_path, verify):
del verify
self.downloaded_urls.append(url) # Record downloader.download() calls
# If the name isn't explicitly provided, then it is extracted from the
# url.
filename = self.dl_fnames.get(url, os.path.basename(url))
# Save the file in the tmp_dir
path = os.path.join(tmpdir_path, filename)
self.fs.add_file(path)
dl_result = downloader.DownloadResult(
path=utils.as_path(path),
url_info=self.dl_results[url],
)
return promise.Promise.resolve(dl_result)
