def multi_read(star_path, no_exist=None, unknown_type="RAISE", ignore_exceptions=None): return { x: read(x, no_exist, unknown_type, ignore_exceptions) for x in glob(star_path) }
def from_cache(url, cache_key, paths_only=False, compression=".gz"): domain = get_domain(url) obj_type, fname = get_obj_type(cache_key) initial_part = f"~/.just_requests/{domain}{obj_type}" partition = "*" paths = glob(f"{initial_part}/{partition}/{fname}.json{compression}") if paths_only: return paths return [read(x)["resp"] for x in paths]
def delete_from_cache(url, cache_key=None, compression=".gz"): domain_name = get_domain(url) if cache_key is not None: fname = get_cache_file_name_str(domain_name, cache_key, compression) parts = fname.split() parts[-2] = "*" fname = glob("/".join(parts))[0] else: fname = last_cache_fname[domain_name] remove(fname) return fname
def update_obj_count(domain, dir_name, obj_type): if (domain, obj_type) not in obj_counts: dir_name = make_path(dir_name) try: existing_partitions = glob(make_path(dir_name) + "/*") if existing_partitions: latest_folder = max(existing_partitions) base = int(latest_folder.split("/")[-1]) * PER_FOLDER count = base + len(os.listdir(latest_folder)) else: count = 0 except FileNotFoundError: count = 0 obj_counts[(domain, obj_type)] = count return obj_counts[(domain, obj_type)]