class FuncXExecutor(Executor): def __init__(self, endpoint_id, process_function="301f653b-40b6-449e-ad2e-e57d3aaa33cd"): self.fxc = FuncXClient(asynchronous=True) self.endpoint_id = endpoint_id self.process_function = process_function def run_async_analysis(self, file_url, tree_name, accumulator, process_func): if not self.process_function: self.process_function = self.fxc.register_function(run_coffea_processor) pickled_process_func = pickle.dumps(process_func) data_result = self.safe_run(file_url, tree_name, accumulator, pickled_process_func, function_id=self.process_function) # Pass this down to the next item in the stream. return data_result @retry(wait=wait_fixed(5), retry=retry_if_exception_type(MaxRequestsExceeded)) def safe_run(self, file_url, tree_name, accumulator, proc, function_id): return self.fxc.run(file_url, tree_name, accumulator, proc, True, endpoint_id=self.endpoint_id, function_id=function_id)
def register_function(self, container_type='docker', location=None, ep_id=None, group=None): from funcx import FuncXClient assert self.extr_func is not None, "Extractor function must first be registered!" if location is None: location = self.store_url fxc = FuncXClient() container_id = fxc.register_container( location=location, container_type=container_type, name='kube-tabular', description='I don\'t think so!', ) self.func_id = fxc.register_function(self.extr_func, ep_id, group=group, container_uuid=container_id, description="A sum function") print(f"The function has been updated! " f"Please copy/paste the following code into {self.func_id} function class:\n") print(self.func_id) return self.func_id
def _register_function(): """Register the inference function with FuncX""" client = FuncXClient() # Get the Group UUID config = json.loads(_config_path.read_text()) function_id = client.register_function(_funcx_func, group=config['group_uuid']) _set_config(function_id=function_id)
# xpcs_x = HDFExtractor() # xpcs_x = ImagesExtractor() # xpcs_x = KeywordExtractor() # xpcs_x = PythonExtractor() # xpcs_x = TabularExtractor() xpcs_x = CCodeExtractor() # xpcs_x = TikaExtractor() ep_id = "2293034e-4c9f-459c-a6f0-0ed310a8e618" extractor_name = "matio" repo_name = "mdf" container_uuid = fxc.register_container(f'/home/tskluzac/.xtract/.containers/xtract-{extractor_name}.img', 'singularity') print("Container UUID: {}".format(container_uuid)) fn_uuid = fxc.register_function(base_extractor, container_uuid=container_uuid, description="Tabular test function.") print("FN_UUID : ", fn_uuid) task_batch_size = 1 # 128 fx_batch_size = 64 # 32 hdf_count = 0 task_batches = Queue() missing_file = "/Users/tylerskluzacek/missing_files.json" missing_file_0 = "/Users/tylerskluzacek/missing_mdf2.json" # missing_file = None # missing_file = None min_num = 300000 # 540000 max_count = 400000 # 560000
auth_token = tokens["petrel_https_server"]['access_token'] transfer_token = tokens['transfer.api.globus.org']['access_token'] funcx_token = tokens['funcx_service']['access_token'] headers = {'Authorization': f"Bearer {funcx_token}", 'Transfer': transfer_token, 'FuncX': funcx_token, 'Petrel': auth_token} print(f"Headers: {headers}") def hello_world(event): return "Hello World!" fxc = FuncXClient() func_uuid = fxc.register_function(hello_world) print(func_uuid) event = None endpoint = '68bade94-bf58-4a7a-bfeb-9c6a61fa5443' items_to_batch = [{"func_id": func_uuid, "event": {}}, {"func_id": func_uuid, "event": {}}] x = remote_extract_batch(items_to_batch, endpoint, headers=headers) fx_ser = FuncXSerializer() import time while True: a = remote_poll_batch(x, headers) print(f"The returned: {a}")
def hello_container(event): import os return f"Container version: {os.environ['container_version']}" for container in all_containers: print(f"Using funcX version: {funcx.__version__}") fxc = FuncXClient() base_path = '/home/tskluzac/ext_repos/' container_path = os.path.join(base_path, container) print(f"Container path: {container_path}") container_uuid = fxc.register_container(container_path, 'singularity') fn_uuid = fxc.register_function( hdf_extract, container_uuid=container_uuid, description="New sum function defined without string spec") print(f"FN UUID: {fn_uuid}") res = fxc.run(sample_hdf_1, endpoint_id=js_ep_id, function_id=fn_uuid) print(res) for i in range(100): # TODO: break when successful try: x = fxc.get_result(res) print(x) break except Exception as e: print("Exception: {}".format(e)) time.sleep(2)
# * Redistributions in binary form must reproduce the above copyright notice, # this list of conditions and the following disclaimer in the documentation # and/or other materials provided with the distribution. # # * Neither the name of the copyright holder nor the names of its # contributors may be used to endorse or promote products derived from # this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. from funcx import FuncXClient from sx_multi import run_coffea_processor fxc = FuncXClient() container_id = fxc.register_container( "bengal1/funcx_coffea:add_schema_to_notebooks", "docker", "Coffea Processor") function_id = fxc.register_function( run_coffea_processor, "Run your coffea process code in a setup environment", container_uuid=container_id) print("Function_ID is ", function_id)
def sleep_func(file_ls): import time # # # for item in file_ls: # # with open(item, 'r') as f: # # f.close() # # time.sleep(sleep_s) return "hello, world!" # func_id = fxc.register_function(function=sleep_func, function_name='hpdc_sleep_extractor') container_uuid = fxc.register_container('/home/tskluzac/xtract-matio.img', 'singularity') print("Container UUID: {}".format(container_uuid)) func_id = fxc.register_function(matio_extract, #ep_id, # TODO: We do not need ep id here container_uuid=container_uuid, description="New sum function defined without") for fx_id in id_list: for i in range(1,10): task_id = fxc.run({'event'}, endpoint_id=fx_id, function_id=func_id) while True: result = fxc.get_batch_result([task_id]) print(result) for tid in result: if result[tid]['status'] == 'failed': exc = result[tid]['exception']