class FuncXExecutor(Executor): def __init__(self, endpoint_id, process_function="301f653b-40b6-449e-ad2e-e57d3aaa33cd"): self.fxc = FuncXClient(asynchronous=True) self.endpoint_id = endpoint_id self.process_function = process_function def run_async_analysis(self, file_url, tree_name, accumulator, process_func): if not self.process_function: self.process_function = self.fxc.register_function(run_coffea_processor) pickled_process_func = pickle.dumps(process_func) data_result = self.safe_run(file_url, tree_name, accumulator, pickled_process_func, function_id=self.process_function) # Pass this down to the next item in the stream. return data_result @retry(wait=wait_fixed(5), retry=retry_if_exception_type(MaxRequestsExceeded)) def safe_run(self, file_url, tree_name, accumulator, proc, function_id): return self.fxc.run(file_url, tree_name, accumulator, proc, True, endpoint_id=self.endpoint_id, function_id=function_id)
def hello_container(event): import os return f"Container version: {os.environ['container_version']}" for container in all_containers: print(f"Using funcX version: {funcx.__version__}") fxc = FuncXClient() base_path = '/home/tskluzac/ext_repos/' container_path = os.path.join(base_path, container) print(f"Container path: {container_path}") container_uuid = fxc.register_container(container_path, 'singularity') fn_uuid = fxc.register_function( hdf_extract, container_uuid=container_uuid, description="New sum function defined without string spec") print(f"FN UUID: {fn_uuid}") res = fxc.run(sample_hdf_1, endpoint_id=js_ep_id, function_id=fn_uuid) print(res) for i in range(100): # TODO: break when successful try: x = fxc.get_result(res) print(x) break except Exception as e: print("Exception: {}".format(e)) time.sleep(2)
# # with open(item, 'r') as f: # # f.close() # # time.sleep(sleep_s) return "hello, world!" # func_id = fxc.register_function(function=sleep_func, function_name='hpdc_sleep_extractor') container_uuid = fxc.register_container('/home/tskluzac/xtract-matio.img', 'singularity') print("Container UUID: {}".format(container_uuid)) func_id = fxc.register_function(matio_extract, #ep_id, # TODO: We do not need ep id here container_uuid=container_uuid, description="New sum function defined without") for fx_id in id_list: for i in range(1,10): task_id = fxc.run({'event'}, endpoint_id=fx_id, function_id=func_id) while True: result = fxc.get_batch_result([task_id]) print(result) for tid in result: if result[tid]['status'] == 'failed': exc = result[tid]['exception'] exc.reraise() time.sleep(2)