def register_container():
from funcx.sdk.client import FuncXClient
fxc = FuncXClient()
from gladier_xpcs.tools.corr import eigen_corr
cont_dir = '/eagle/APSDataAnalysis/XPCS_test/containers/'
container_name = 'eigen_v2.simg'
eigen_cont_id = fxc.register_container(location=cont_dir+container_name,container_type='singularity')
corr_cont_fxid = fxc.register_function(eigen_corr, container_uuid=eigen_cont_id)
return corr_cont_fxid
def register_funcx(task):
"""Register the function and the container with funcX.
Parameters
----------
task : dict
A dict of the task to publish
Returns
-------
str
The funcX function id
"""
# Get the funcX dependent token
fx_token = task['dlhub']['funcx_token']
# Create a client using this token
fx_auth = globus_sdk.AccessTokenAuthorizer(fx_token)
fxc = FuncXClient(fx_authorizer=fx_auth, use_offprocess_checker=False)
description = f"A container for the DLHub model {task['dlhub']['shorthand_name']}"
try:
description = task['datacite']['descriptions'][0]['description']
except:
# It doesn't have a simple description
pass
# Register the container with funcX
container_id = fxc.register_container(task['dlhub']['ecr_uri'],
'docker',
name=task['dlhub']['shorthand_name'],
description=description)
# Register a function
funcx_id = fxc.register_function(dlhub_run,
function_name=task['dlhub']['name'],
container_uuid=container_id,
description=description,
public=True)
# Whitelist the function on DLHub's endpoint
# First create a new fxc client on DLHub's behalf
fxc = FuncXClient(use_offprocess_checker=False)
endpoint_uuid = '86a47061-f3d9-44f0-90dc-56ddc642c000'
res = fxc.add_to_whitelist(endpoint_uuid, [funcx_id])
print(res)
return funcx_id
from funcx.sdk.client import FuncXClient
fxc = FuncXClient()
location = '039706667969.dkr.ecr.us-east-1.amazonaws.com/xtract-bert'
description = 'Xtract text types container using pretrained BERT NER model'
container_type = 'docker'
name = "xtract/bert"
container_uuid = fxc.register_container(name, location, description,
container_type)
def bert_extract(event):
import sys
import time
from xtract_sdk.downloaders.globus_https import GlobusHttpsDownloader
from xtract_sdk.downloaders.google_drive import GoogleDriveDownloader
t0 = time.time()
sys.path.insert(1, '/')
import xtract_keyword_main
# except Exception as e:
# return e
# print("BUT NO HERE?")
new_mdata = None