def generate_local_path(url): # this function generates a safe local file name corresponding to # an S3 URL. URLs may be longer than maximum file length limit on Linux, # so we mostly hash the URL but retain the leaf part as a convenience # feature to ease eyeballing quoted = url_quote(url) fname = quoted.split(b'/')[-1].replace(b'.', b'_').replace(b'-', b'_') sha = sha1(quoted).hexdigest() return u'-'.join((sha, fname.decode('utf-8')))
def generate_local_path(url, suffix=None): # this function generates a safe local file name corresponding to # an S3 URL. URLs may be longer than maximum file length limit on Linux, # so we mostly hash the URL but retain the leaf part as a convenience # feature to ease eyeballing quoted = url_quote(url) fname = quoted.split(b"/")[-1].replace(b".", b"_").replace(b"-", b"_") sha = sha1(quoted).hexdigest() if suffix: return "-".join((sha, fname.decode("utf-8"), suffix)) return "-".join((sha, fname.decode("utf-8")))
def generate_local_path(url, range="whole", suffix=None): # this function generates a safe local file name corresponding to # an S3 URL. URLs may be longer than maximum file length limit on Linux, # so we mostly hash the URL but retain the leaf part as a convenience # feature to ease eyeballing # We also call out "range" specifically to allow multiple ranges for the same # file to be downloaded in parallel. if range is None: range = "whole" if range != "whole": # It will be of the form `bytes=%d-` or `bytes=-%d` or `bytes=%d-%d` range = range[6:].replace("-", "_") quoted = url_quote(url) fname = quoted.split(b"/")[-1].replace(b".", b"_").replace(b"-", b"_") sha = sha1(quoted).hexdigest() if suffix: return "-".join((sha, fname.decode("utf-8"), range, suffix)) return "-".join((sha, fname.decode("utf-8"), range))
def format_triplet(prefix, url="", local=""): return " ".join(url_quote(x).decode("utf-8") for x in (prefix, url, local))
def format_triplet(prefix, url='', local=''): return u' '.join( url_quote(x).decode('utf-8') for x in (prefix, url, local))