def touch_random_unused_file(base_dir:Path, ext:Optional[str]=None)->Path:
assert base_dir.is_dir()
if ext is None:
ext = ""
elif ext[0] != ".":
ext = f".{ext}"
lock = Lock(f"dir_lock:{base_dir.name}")
while(not lock.acquire(timeout=5)):
pass
# THREADSAFE
name = f"{get_random_ascii_str(10)}{ext}"
path = base_dir.joinpath(name)
while path.is_file():
name = f"{get_random_ascii_str(10)}{ext}"
path = base_dir.joinpath(name)
path.touch()
# End Threadsafe
lock.release()
return path
def _flush_to_batches(redis_key, name):
lock = Lock(redis_key)
# TODO set timeout and handle
if lock.acquire(timeout=1):
try:
processor = SchemaPreprocessor(name)
batch_writer = BatchWriter(name)
# Get the batch and remove the read range atomically
with rd.pipeline() as pipe:
pipe.multi()
pipe.lrange(redis_key, 0, Config.batches.size - 1)
pipe.ltrim(redis_key, Config.batches.size, -1)
batch = pipe.execute()[0]
batch_matrix = processor.json_blobs_to_matrix(batch)
batch_writer.write_batch_matrix(batch_matrix)
finally:
lock.release()
else:
raise Reschedule()