def create_event(self,
family_batch,
ep_name,
xtract_dir,
sys_path_add,
module_path,
metadata_write_path,
recursion_depth=None):
event = create_event(family_batch=family_batch,
ep_name=ep_name,
xtract_dir=xtract_dir,
sys_path_add=sys_path_add,
module_path=module_path,
metadata_write_path=metadata_write_path)
return event
def create_event(self,
family_batch,
ep_name,
xtract_dir,
sys_path_add,
module_path,
metadata_write_path,
recursion_depth=None):
if recursion_depth is not None:
rec_limit = recursion_depth
else: # otherwise, set to extractor default.
rec_limit = recursion_depth
event = create_event(family_batch=family_batch,
ep_name=ep_name,
xtract_dir=xtract_dir,
sys_path_add=sys_path_add,
module_path=module_path,
metadata_write_path=metadata_write_path)
return event
while not task_batches.empty():
# Temporary edge case fix.
current_batch = []
while len(current_batch) < fx_batch_size:
print(f"All events queue size: {task_batches.qsize()}")
if task_batches.empty():
break
event = task_batches.get()
# extractor_name = 'opener'
payload = create_event(ep_name="foobar",
family_batch=event['family_batch'],
xtract_dir="/home/tskluzac/.xtract",
sys_path_add="/",
module_path=f"xtract_{extractor_name}_main",
metadata_write_path=f'/home/tskluzac/{extractor_name}-{repo_name}-completed',
writer='json-np') # TODO: make an arg.
current_batch.append(payload)
# print(current_batch)
print(f"Len Current Batch: {len(current_batch)}")
for item in current_batch:
batch.add(item, endpoint_id=ep_id, function_id=fn_uuid)
# List of task_ids
ts = time.time()
batch_res = fxc.batch_run(batch)
# While my queue isn't empty
while not task_batches.empty():
# Temporary edge case fix.
current_batch = []
while len(current_batch) < fx_batch_size:
print(f"All events queue size: {task_batches.qsize()}")
if task_batches.empty():
break
event = task_batches.get()
payload = create_event(ep_name="foobar",
family_batch=event['family_batch'],
xtract_dir="/home/tskluzac/.xtract",
sys_path_add="/",
module_path="xtract_xpcs_main",
metadata_write_path=f'/home/tskluzac/{trimester}-completed')
current_batch.append(payload)
# print(current_batch)
print(f"Len Current Batch: {len(current_batch)}")
for item in current_batch:
batch.add(item, endpoint_id=ep_id, function_id=fn_uuid)
# List of task_ids
batch_res = fxc.batch_run(batch)
for item in batch_res: