def _get_cached_path( manually_downloaded_path: Optional[epath.Path], checksum_path: Optional[epath.Path], url_path: epath.Path, expected_url_info: Optional[checksums.UrlInfo], ) -> downloader.DownloadResult: """Returns the downloaded path and computed url-info. If the path is not cached, or that `url_path` does not match checksums, the file will be downloaded again. Path can be cached at three different locations: Args: manually_downloaded_path: Manually downloaded in `dl_manager.manual_dir` checksum_path: Cached in the final destination (if checksum known) url_path: Cached in the tmp destination (if checksum unknown). expected_url_info: Registered checksum (if known) """ # User has manually downloaded the file. if manually_downloaded_path and manually_downloaded_path.exists(): return downloader.DownloadResult(manually_downloaded_path, url_info=None) # pytype: disable=wrong-arg-types # Download has been cached (checksum known) elif checksum_path and resource_lib.Resource.exists_locally(checksum_path): # `path = f(checksum)` was found, so url_info match return downloader.DownloadResult(checksum_path, url_info=expected_url_info) # Download has been cached (checksum unknown) elif resource_lib.Resource.exists_locally(url_path): # Info restored from `.INFO` file computed_url_info = _read_url_info(url_path) # If checksums are now registered but do not match, trigger a new # download (e.g. previous file corrupted, checksums updated) if expected_url_info and computed_url_info != expected_url_info: return downloader.DownloadResult(path=None, url_info=None) # pytype: disable=wrong-arg-types else: return downloader.DownloadResult(url_path, url_info=computed_url_info) # Else file not found (or has bad checksums). (re)download. else: return downloader.DownloadResult(path=None, url_info=None) # pytype: disable=wrong-arg-types
def _download(url, tmpdir_path, verify): del verify self.downloaded_urls.append(url) # Record downloader.download() calls # If the name isn't explicitly provided, then it is extracted from the # url. filename = self.dl_fnames.get(url, os.path.basename(url)) # Save the file in the tmp_dir path = os.path.join(tmpdir_path, filename) self.fs.add_file(path) dl_result = downloader.DownloadResult( path=utils.as_path(path), url_info=self.dl_results[url], ) return promise.Promise.resolve(dl_result)