def multi_read(star_path,
no_exist=None,
unknown_type="RAISE",
ignore_exceptions=None):
return {
x: read(x, no_exist, unknown_type, ignore_exceptions)
for x in glob(star_path)
}
def from_cache(url, cache_key, paths_only=False, compression=".gz"):
domain = get_domain(url)
obj_type, fname = get_obj_type(cache_key)
initial_part = f"~/.just_requests/{domain}{obj_type}"
partition = "*"
paths = glob(f"{initial_part}/{partition}/{fname}.json{compression}")
if paths_only:
return paths
return [read(x)["resp"] for x in paths]
def delete_from_cache(url, cache_key=None, compression=".gz"):
domain_name = get_domain(url)
if cache_key is not None:
fname = get_cache_file_name_str(domain_name, cache_key, compression)
parts = fname.split()
parts[-2] = "*"
fname = glob("/".join(parts))[0]
else:
fname = last_cache_fname[domain_name]
remove(fname)
return fname
def update_obj_count(domain, dir_name, obj_type):
if (domain, obj_type) not in obj_counts:
dir_name = make_path(dir_name)
try:
existing_partitions = glob(make_path(dir_name) + "/*")
if existing_partitions:
latest_folder = max(existing_partitions)
base = int(latest_folder.split("/")[-1]) * PER_FOLDER
count = base + len(os.listdir(latest_folder))
else:
count = 0
except FileNotFoundError:
count = 0
obj_counts[(domain, obj_type)] = count
return obj_counts[(domain, obj_type)]