def generate_local_path(url):
# this function generates a safe local file name corresponding to
# an S3 URL. URLs may be longer than maximum file length limit on Linux,
# so we mostly hash the URL but retain the leaf part as a convenience
# feature to ease eyeballing
quoted = url_quote(url)
fname = quoted.split(b'/')[-1].replace(b'.', b'_').replace(b'-', b'_')
sha = sha1(quoted).hexdigest()
return u'-'.join((sha, fname.decode('utf-8')))
def generate_local_path(url, suffix=None):
# this function generates a safe local file name corresponding to
# an S3 URL. URLs may be longer than maximum file length limit on Linux,
# so we mostly hash the URL but retain the leaf part as a convenience
# feature to ease eyeballing
quoted = url_quote(url)
fname = quoted.split(b"/")[-1].replace(b".", b"_").replace(b"-", b"_")
sha = sha1(quoted).hexdigest()
if suffix:
return "-".join((sha, fname.decode("utf-8"), suffix))
return "-".join((sha, fname.decode("utf-8")))
def generate_local_path(url, range="whole", suffix=None):
# this function generates a safe local file name corresponding to
# an S3 URL. URLs may be longer than maximum file length limit on Linux,
# so we mostly hash the URL but retain the leaf part as a convenience
# feature to ease eyeballing
# We also call out "range" specifically to allow multiple ranges for the same
# file to be downloaded in parallel.
if range is None:
range = "whole"
if range != "whole":
# It will be of the form `bytes=%d-` or `bytes=-%d` or `bytes=%d-%d`
range = range[6:].replace("-", "_")
quoted = url_quote(url)
fname = quoted.split(b"/")[-1].replace(b".", b"_").replace(b"-", b"_")
sha = sha1(quoted).hexdigest()
if suffix:
return "-".join((sha, fname.decode("utf-8"), range, suffix))
return "-".join((sha, fname.decode("utf-8"), range))
def format_triplet(prefix, url="", local=""):
return " ".join(url_quote(x).decode("utf-8") for x in (prefix, url, local))
def format_triplet(prefix, url='', local=''):
return u' '.join(
url_quote(x).decode('utf-8') for x in (prefix, url, local))
