def iter_tries(): yield lambda: download_by_skynet(resource_info, resource_file_name) if custom_fetcher: yield lambda: fetch_via_script(custom_fetcher, resource_id) # Don't try too hard here: we will get back to proxy later on yield lambda: fetch_from.fetch_url(proxy_link, False, resource_file_name, expected_md5, tries=2) for x in get_storage_links(): # Don't spend too much time connecting single host yield lambda: fetch_from.fetch_url(x, False, resource_file_name, expected_md5, tries=1) if mds_link is not None: # Don't try too hard here: we will get back to MDS later on yield lambda: fetch_from.fetch_url(mds_link, True, resource_file_name, expected_md5, tries=2) yield lambda: fetch_from.fetch_url(proxy_link, False, resource_file_name, expected_md5) if mds_link is not None: yield lambda: fetch_from.fetch_url(mds_link, True, resource_file_name, expected_md5)
def iter_tries(): yield lambda: download_by_skynet(resource_info, resource_file_name) if custom_fetcher: yield lambda: fetch_via_script(custom_fetcher, resource_id) for x in get_storage_links(): yield lambda: fetch_from.fetch_url(x, False, resource_file_name, expected_md5) yield lambda: fetch_from.fetch_url( proxy_link, False, resource_file_name, expected_md5) if mds_link is not None: yield lambda: fetch_from.fetch_url( mds_link, True, resource_file_name, expected_md5) yield lambda: fetch_from.fetch_url(proxy_link, False, resource_file_name, expected_md5) if mds_link is not None: yield lambda: fetch_from.fetch_url( mds_link, True, resource_file_name, expected_md5)
def fetch(key): parts = key.split("/") if len(parts) != 3: raise ValueError("Invalid MDS key '{}'".format(key)) _, sha1, file_name = parts fetched_file = fetch_from.fetch_url(MDS_PREFIX + key, False, file_name, expected_sha1=sha1) return fetched_file, file_name
def _fetch_via_http(name, version, integrity, integrity_algorithm, file_name): # Example: "http://npm.yandex-team.ru/@scope/name/-/name-0.0.1.tgz" for @scope/name v0.0.1. url = NPM_BASEURL + "/".join([name, "-", "{}-{}.tgz".format(name.split("/").pop(), version)]) hashobj = hashlib.new(integrity_algorithm) fetched_file = fetch_from.fetch_url(url, False, file_name, tries=1, writers=[hashobj.update]) if hashobj.hexdigest() != integrity: raise fetch_from.BadChecksumFetchError("Expected {}, but got {} for {}".format( integrity, hashobj.hexdigest(), file_name, )) return fetched_file