def _files(): for local, url in files: yield url_unquote(local), url_unquote(url) if filelist: for line in open(filelist, mode='rb'): local, url = map(url_unquote, line.split()) if not os.path.exists(local): exit(ERROR_LOCAL_FILE_NOT_FOUND, local) yield local, url
def _files(): for local, url in files: yield url_unquote(local), url_unquote(url), None, None if filelist: for line in open(filelist, mode="rb"): r = json.loads(line) local = r["local"] url = r["url"] content_type = r.get("content_type", None) metadata = r.get("metadata", None) if not os.path.exists(local): exit(ERROR_LOCAL_FILE_NOT_FOUND, local) yield local, url, content_type, metadata
def _files(): for local, url in files: yield url_unquote(local), url_unquote(url), None, None if filelist: for line in open(filelist, mode='rb'): r = json.loads(line) local = r['local'] url = r['url'] content_type = r.get('content_type', None) metadata = r.get('metadata', None) if not os.path.exists(local): exit(ERROR_LOCAL_FILE_NOT_FOUND, local) yield local, url, content_type, metadata
def _populate_prefixes(prefixes, inputs): # Returns a tuple: first element is the prefix and second element # is the optional range (or None if the entire prefix is requested) if prefixes: prefixes = [(url_unquote(p), None) for p in prefixes] else: prefixes = [] if inputs: with open(inputs, mode="rb") as f: for l in f: s = l.split(b" ") if len(s) > 1: prefixes.append( (url_unquote(s[0].strip()), url_unquote(s[1].strip()))) else: prefixes.append((url_unquote(s[0].strip()), None)) return prefixes