class FuncXExecutor(Executor):
def __init__(self, endpoint_id, process_function="301f653b-40b6-449e-ad2e-e57d3aaa33cd"):
self.fxc = FuncXClient(asynchronous=True)
self.endpoint_id = endpoint_id
self.process_function = process_function
def run_async_analysis(self, file_url, tree_name, accumulator, process_func):
if not self.process_function:
self.process_function = self.fxc.register_function(run_coffea_processor)
pickled_process_func = pickle.dumps(process_func)
data_result = self.safe_run(file_url, tree_name,
accumulator,
pickled_process_func,
function_id=self.process_function)
# Pass this down to the next item in the stream.
return data_result
@retry(wait=wait_fixed(5), retry=retry_if_exception_type(MaxRequestsExceeded))
def safe_run(self, file_url, tree_name, accumulator, proc, function_id):
return self.fxc.run(file_url,
tree_name,
accumulator,
proc,
True,
endpoint_id=self.endpoint_id,
function_id=function_id)
def hello_container(event):
import os
return f"Container version: {os.environ['container_version']}"
for container in all_containers:
print(f"Using funcX version: {funcx.__version__}")
fxc = FuncXClient()
base_path = '/home/tskluzac/ext_repos/'
container_path = os.path.join(base_path, container)
print(f"Container path: {container_path}")
container_uuid = fxc.register_container(container_path, 'singularity')
fn_uuid = fxc.register_function(
hdf_extract,
container_uuid=container_uuid,
description="New sum function defined without string spec")
print(f"FN UUID: {fn_uuid}")
res = fxc.run(sample_hdf_1, endpoint_id=js_ep_id, function_id=fn_uuid)
print(res)
for i in range(100):
# TODO: break when successful
try:
x = fxc.get_result(res)
print(x)
break
except Exception as e:
print("Exception: {}".format(e))
time.sleep(2)
# # with open(item, 'r') as f:
# # f.close()
#
# time.sleep(sleep_s)
return "hello, world!"
# func_id = fxc.register_function(function=sleep_func, function_name='hpdc_sleep_extractor')
container_uuid = fxc.register_container('/home/tskluzac/xtract-matio.img', 'singularity')
print("Container UUID: {}".format(container_uuid))
func_id = fxc.register_function(matio_extract,
#ep_id, # TODO: We do not need ep id here
container_uuid=container_uuid,
description="New sum function defined without")
for fx_id in id_list:
for i in range(1,10):
task_id = fxc.run({'event'}, endpoint_id=fx_id, function_id=func_id)
while True:
result = fxc.get_batch_result([task_id])
print(result)
for tid in result:
if result[tid]['status'] == 'failed':
exc = result[tid]['exception']
exc.reraise()
time.sleep(2)
