def iter_tries():
yield lambda: download_by_skynet(resource_info, resource_file_name)
if custom_fetcher:
yield lambda: fetch_via_script(custom_fetcher, resource_id)
# Don't try too hard here: we will get back to proxy later on
yield lambda: fetch_from.fetch_url(proxy_link, False, resource_file_name, expected_md5, tries=2)
for x in get_storage_links():
# Don't spend too much time connecting single host
yield lambda: fetch_from.fetch_url(x, False, resource_file_name, expected_md5, tries=1)
if mds_link is not None:
# Don't try too hard here: we will get back to MDS later on
yield lambda: fetch_from.fetch_url(mds_link, True, resource_file_name, expected_md5, tries=2)
yield lambda: fetch_from.fetch_url(proxy_link, False, resource_file_name, expected_md5)
if mds_link is not None:
yield lambda: fetch_from.fetch_url(mds_link, True, resource_file_name, expected_md5)
def iter_tries():
yield lambda: download_by_skynet(resource_info, resource_file_name)
if custom_fetcher:
yield lambda: fetch_via_script(custom_fetcher, resource_id)
for x in get_storage_links():
yield lambda: fetch_from.fetch_url(x, False, resource_file_name,
expected_md5)
yield lambda: fetch_from.fetch_url(
proxy_link, False, resource_file_name, expected_md5)
if mds_link is not None:
yield lambda: fetch_from.fetch_url(
mds_link, True, resource_file_name, expected_md5)
yield lambda: fetch_from.fetch_url(proxy_link, False,
resource_file_name, expected_md5)
if mds_link is not None:
yield lambda: fetch_from.fetch_url(
mds_link, True, resource_file_name, expected_md5)
def fetch(key):
parts = key.split("/")
if len(parts) != 3:
raise ValueError("Invalid MDS key '{}'".format(key))
_, sha1, file_name = parts
fetched_file = fetch_from.fetch_url(MDS_PREFIX + key, False, file_name, expected_sha1=sha1)
return fetched_file, file_name
def _fetch_via_http(name, version, integrity, integrity_algorithm, file_name):
# Example: "http://npm.yandex-team.ru/@scope/name/-/name-0.0.1.tgz" for @scope/name v0.0.1.
url = NPM_BASEURL + "/".join([name, "-", "{}-{}.tgz".format(name.split("/").pop(), version)])
hashobj = hashlib.new(integrity_algorithm)
fetched_file = fetch_from.fetch_url(url, False, file_name, tries=1, writers=[hashobj.update])
if hashobj.hexdigest() != integrity:
raise fetch_from.BadChecksumFetchError("Expected {}, but got {} for {}".format(
integrity,
hashobj.hexdigest(),
file_name,
))
return fetched_file
