def register_container():
    from funcx.sdk.client import FuncXClient
    fxc = FuncXClient()
    from gladier_xpcs.tools.corr import eigen_corr
    cont_dir = '/eagle/APSDataAnalysis/XPCS_test/containers/'
    container_name = 'eigen_v2.simg'
    eigen_cont_id = fxc.register_container(location=cont_dir+container_name,container_type='singularity')
    corr_cont_fxid = fxc.register_function(eigen_corr, container_uuid=eigen_cont_id)
    return corr_cont_fxid
示例#2
0
def register_funcx(task):
    """Register the function and the container with funcX.

    Parameters
    ----------
    task : dict
        A dict of the task to publish

    Returns
    -------
    str
        The funcX function id
    """

    # Get the funcX dependent token
    fx_token = task['dlhub']['funcx_token']
    # Create a client using this token
    fx_auth = globus_sdk.AccessTokenAuthorizer(fx_token)
    fxc = FuncXClient(fx_authorizer=fx_auth, use_offprocess_checker=False)
    description = f"A container for the DLHub model {task['dlhub']['shorthand_name']}"
    try:
        description = task['datacite']['descriptions'][0]['description']
    except:
        # It doesn't have a simple description
        pass
    # Register the container with funcX
    container_id = fxc.register_container(task['dlhub']['ecr_uri'],
                                          'docker',
                                          name=task['dlhub']['shorthand_name'],
                                          description=description)

    # Register a function
    funcx_id = fxc.register_function(dlhub_run,
                                     function_name=task['dlhub']['name'],
                                     container_uuid=container_id,
                                     description=description,
                                     public=True)

    # Whitelist the function on DLHub's endpoint
    # First create a new fxc client on DLHub's behalf
    fxc = FuncXClient(use_offprocess_checker=False)
    endpoint_uuid = '86a47061-f3d9-44f0-90dc-56ddc642c000'
    res = fxc.add_to_whitelist(endpoint_uuid, [funcx_id])
    print(res)
    return funcx_id
示例#3
0
from funcx.sdk.client import FuncXClient

fxc = FuncXClient()

location = '039706667969.dkr.ecr.us-east-1.amazonaws.com/xtract-bert'
description = 'Xtract text types container using pretrained BERT NER model'
container_type = 'docker'
name = "xtract/bert"
container_uuid = fxc.register_container(name, location, description,
                                        container_type)


def bert_extract(event):

    import sys
    import time

    from xtract_sdk.downloaders.globus_https import GlobusHttpsDownloader
    from xtract_sdk.downloaders.google_drive import GoogleDriveDownloader

    t0 = time.time()

    sys.path.insert(1, '/')

    import xtract_keyword_main
    # except Exception as e:
    #     return e
    # print("BUT NO HERE?")

    new_mdata = None