def _files():
for local, url in files:
yield url_unquote(local), url_unquote(url)
if filelist:
for line in open(filelist, mode='rb'):
local, url = map(url_unquote, line.split())
if not os.path.exists(local):
exit(ERROR_LOCAL_FILE_NOT_FOUND, local)
yield local, url
def _files():
for local, url in files:
yield url_unquote(local), url_unquote(url), None, None
if filelist:
for line in open(filelist, mode="rb"):
r = json.loads(line)
local = r["local"]
url = r["url"]
content_type = r.get("content_type", None)
metadata = r.get("metadata", None)
if not os.path.exists(local):
exit(ERROR_LOCAL_FILE_NOT_FOUND, local)
yield local, url, content_type, metadata
def _files():
for local, url in files:
yield url_unquote(local), url_unquote(url), None, None
if filelist:
for line in open(filelist, mode='rb'):
r = json.loads(line)
local = r['local']
url = r['url']
content_type = r.get('content_type', None)
metadata = r.get('metadata', None)
if not os.path.exists(local):
exit(ERROR_LOCAL_FILE_NOT_FOUND, local)
yield local, url, content_type, metadata
def _populate_prefixes(prefixes, inputs):
# Returns a tuple: first element is the prefix and second element
# is the optional range (or None if the entire prefix is requested)
if prefixes:
prefixes = [(url_unquote(p), None) for p in prefixes]
else:
prefixes = []
if inputs:
with open(inputs, mode="rb") as f:
for l in f:
s = l.split(b" ")
if len(s) > 1:
prefixes.append(
(url_unquote(s[0].strip()), url_unquote(s[1].strip())))
else:
prefixes.append((url_unquote(s[0].strip()), None))
return prefixes
