Ejemplo n.º 1
0
def run_function_wait_result(
        py_fn,
        py_fn_args,
        py_fn_kwargs={},
        endpoint_id="3c3f0b4f-4ae4-4241-8497-d7339972ff4a",
        print_status=True):
    """
    Register and run a function with FuncX, wait for execution,
        and return results when they are available

    :param py_fn: Handle of Python function
    :param py_fn_args: List of positional args for py function
    :param py_fn_kwargs: Dict of keyword args for py function,
    :param endpoint_id: ID of endpoint to run command on
        - must be configured in config.py
    """
    fxc = FuncXClient()
    func_uuid = fxc.register_function(py_fn)
    res = fxc.run(*py_fn_args,
                  **py_fn_kwargs,
                  endpoint_id=endpoint_id,
                  function_id=func_uuid)
    while True:
        try:
            if print_status:
                print("Waiting for results...")
            time.sleep(FUNCX_SLEEP_TIME)
            return str(fxc.get_result(res), encoding="utf-8")
            break
        except Exception as e:
            if "waiting-for-" in str(e):
                continue
            else:
                raise e
Ejemplo n.º 2
0
def run_real(payload):
    """Run a function with some raw json input.
    """
    fxc = FuncXClient(funcx_service_address='https://dev.funcx.org/api/v1')

    # register a function
    func_id = fxc.register_function(test_func)
    ep_id = '60ad46e1-c912-468b-8674-4d582e9dc9ee'

    res = fxc.run({'name': 'real'}, function_id=func_id, endpoint_id=ep_id)

    print(res)

    return res
Ejemplo n.º 3
0
def run_function_async(py_fn,
                       py_fn_args,
                       py_fn_kwargs={},
                       endpoint_id="3c3f0b4f-4ae4-4241-8497-d7339972ff4a"):
    """
    Asynchronously register and run a Python function on a FuncX endpoint

    :param py_fn: Handle of Python function
    :param py_fn_args: List of positional args for py function
    :param py_fn_kwargs: Dict of keyword args for py function,
    :param endpoint_id: ID of endpoint to run command on
        - must be configured in config.py
    """
    # Use return value for Funcx polling
    fxc = FuncXClient()
    func_uuid = fxc.register_function(py_fn)
    res = fxc.run(*py_fn_args,
                  **kwargs,
                  endpoint_id=endpoint_id,
                  function_id=func_uuid)
    return res
Ejemplo n.º 4
0
    'host': args.host
    # Other function specific params here.
}
funcx_endp_id = "a62a830a-5cd1-42a8-a4a8-a44fa552c899"  # merf.egs.anl.gov endpoint

# Running the function locally only
if args.run == 'local':
    basic_analysis(params)

# Use funcX to run the function
elif args.run == 'funcx':
    if not args.register:
        from funcx.sdk.client import FuncXClient
        fxc = FuncXClient()
    task_id = fxc.run(params,
                      endpoint_id=funcx_endp_id,
                      function_id=funcx_func_uuid)
    import time
    print("Sleeping for the computation.")
    time.sleep(2)
    result = fxc.get_result(task_id)
    print(result)

# Use funcX through MDML to run the function
elif args.run == 'mdml':
    exp.globus_login()
    exp.publish_analysis('TEMP_ANALYSIS',
                         funcx_func_uuid,
                         funcx_endp_id,
                         params,
                         trigger=['DATA1'])  # Change the analysis device ID
from funcx.sdk.client import FuncXClient
fxc = FuncXClient()


def funcx_test():
    while True:
        print("Viana")


func_uuid = fxc.register_function(funcx_test)
tutorial_endpoint = '70d29c21-66c3-4ba8-98fc-91490b522699'  # Public tutorial endpoint
res = fxc.run(endpoint_id=tutorial_endpoint, function_id=func_uuid)
funcx_test()
"""Run toxicity inference with FuncX"""
from funcx.sdk.client import FuncXClient
import json

# Get FuncX ready
fxc = FuncXClient()
theta_ep = 'd3a23590-3282-429a-8bce-e0ca0f4177f3'
with open('func_uuid.json') as fp:
    func_id = json.load(fp)

# Run the infernece
smiles = ['C', 'CC', 'CCC']
task_id = fxc.run(smiles, endpoint_id=theta_ep, function_id=func_id)
print(task_id)
def main(args):
    if args.config_file is not None:
        with open(args.config_file, "r") as infile:
            config = json.load(infile)

    backend = args.backend

    pallet_path = Path(config["input_prefix"]).joinpath(config["pallet_name"])

    # locally get pyhf pallet for analysis
    if not pallet_path.exists():
        download(config["pallet_url"], pallet_path)

    analysis_name = config["analysis_name"]
    analysis_prefix_str = "" if analysis_name is None else f"{analysis_name}_"
    if config["analysis_dir"] is not None:
        pallet_path = pallet_path.joinpath(config["analysis_dir"])

    with open(pallet_path.joinpath(
            f"{analysis_prefix_str}BkgOnly.json")) as bkgonly_json:
        bkgonly_workspace = json.load(bkgonly_json)

    # Initialize funcX client
    fxc = FuncXClient()
    fxc.max_requests = 200

    with open("endpoint_id.txt") as endpoint_file:
        pyhf_endpoint = str(endpoint_file.read().rstrip())

    # register functions
    prepare_func = fxc.register_function(prepare_workspace)
    infer_func = fxc.register_function(infer_hypotest)

    # execute background only workspace
    prepare_task = fxc.run(bkgonly_workspace,
                           backend,
                           endpoint_id=pyhf_endpoint,
                           function_id=prepare_func)

    # Read patchset in while background only workspace running
    with open(pallet_path.joinpath(
            f"{analysis_prefix_str}patchset.json")) as patchset_json:
        patchset = pyhf.PatchSet(json.load(patchset_json))

    workspace = None
    while not workspace:
        try:
            workspace = fxc.get_result(prepare_task)
        except Exception as excep:
            print(f"prepare: {excep}")
            sleep(10)

    print("--------------------")
    print(workspace)

    # execute patch fits across workers and retrieve them when done
    n_patches = len(patchset.patches)
    tasks = {}
    for patch_idx in range(n_patches):
        patch = patchset.patches[patch_idx]
        task_id = fxc.run(
            workspace,
            patch.metadata,
            [patch.patch],
            backend,
            endpoint_id=pyhf_endpoint,
            function_id=infer_func,
        )
        tasks[patch.name] = {"id": task_id, "result": None}

    while count_complete(tasks.values()) < n_patches:
        for task in tasks.keys():
            if not tasks[task]["result"]:
                try:
                    result = fxc.get_result(tasks[task]["id"])
                    print(
                        f"Task {task} complete, there are {count_complete(tasks.values())+1} results now"
                    )
                    tasks[task]["result"] = result
                except Exception as excep:
                    print(f"inference: {excep}")
                    sleep(15)

    print("--------------------")
    print(tasks.values())
sheet.write(0, 3, 'exec_time')
for i in range(0, n):
    #start1 = time.time()
    #hello_function = fxc.register_function(matrix)
    #end1 = time.time()
    #registertime = ((end1 - start1) * 1000)
    #print("The register time is:", registertime)
    #high = 10
    #low = 5
    #m,n = 2,2
    #a = (high - low)*np.random.rand(m,n) + low
    start2 = time.time()
    #a = (high - low)*np.random.rand(m,n) + low
    # res = fxc.run(items, endpoint_id='7601789e-8569-413f-be3e-e573d04c5799', function_id=sum_function)
    res = fxc.run(a,
                  endpoint_id='d25d7886-6112-4f05-b3bd-0625cc61e6e8',
                  function_id=hello_function)
    end2 = time.time()
    runtime = ((end2 - start2) * 1000)
    print("The runtime is:", runtime)
    # get the raw json response
    start = time.time()
    result = fxc.get(f"tasks/{res}")
    while result['status'] != 'success':
        time.sleep(1)
        result = fxc.get(f"tasks/{res}")
    completion_time = result['completion_t']
    exec_time = (float(completion_time) - start) * 1000
    print("The function execution time is:", exec_time)
    # add the result to the list
    estimate.append(a)  #overhead.append(Overheadtime)
Ejemplo n.º 9
0
class TestTutorial:
    def __init__(
        self,
        fx_auth,
        search_auth,
        openid_auth,
        endpoint_id,
        func,
        expected,
        args=None,
        timeout=15,
        concurrency=1,
        tol=1e-5,
    ):
        self.endpoint_id = endpoint_id
        self.func = func
        self.expected = expected
        self.args = args
        self.timeout = timeout
        self.concurrency = concurrency
        self.tol = tol
        self.fxc = FuncXClient(
            fx_authorizer=fx_auth,
            search_authorizer=search_auth,
            openid_authorizer=openid_auth,
        )
        self.func_uuid = self.fxc.register_function(self.func)

        self.logger = logging.getLogger(__name__)
        self.logger.setLevel(logging.DEBUG)
        handler = logging.StreamHandler(sys.stdout)
        handler.setLevel(logging.DEBUG)
        formatter = logging.Formatter(
            "%(asctime)s %(name)s:%(lineno)d [%(levelname)s]  %(message)s")
        handler.setFormatter(formatter)
        self.logger.addHandler(handler)

    def run(self):
        try:
            submissions = []
            for _ in range(self.concurrency):
                task = self.fxc.run(self.args,
                                    endpoint_id=self.endpoint_id,
                                    function_id=self.func_uuid)
                submissions.append(task)

            time.sleep(self.timeout)

            unfinished = copy.deepcopy(submissions)
            while True:
                unfinished[:] = [
                    task for task in unfinished
                    if self.fxc.get_task(task)["pending"]
                ]
                if not unfinished:
                    break
                time.sleep(self.timeout)

            success = 0
            for task in submissions:
                result = self.fxc.get_result(task)
                if abs(result - self.expected) > self.tol:
                    self.logger.exception(
                        f"Difference for task {task}. "
                        f"Returned: {result}, Expected: {self.expected}")
                else:
                    success += 1

            self.logger.info(
                f"{success}/{self.concurrency} tasks completed successfully")
        except KeyboardInterrupt:
            self.logger.info("Cancelled by keyboard interruption")
        except Exception as e:
            self.logger.exception(f"Encountered exception: {e}")
            raise
Ejemplo n.º 10
0
sheet.write(0, 2, 'runtime')
sheet.write(0, 3, 'exec_time')
for i in range(0, n):
    #funcx-endpoint start Test
    #funcx-endpoint stop Test
    #funcx-endpoint start Test
    #start1 = time.time()
    #hello_function = fxc.register_function(matrix)
    ress = np.dot(mat1, mat2)
    #end1 = time.time()
    #registertime = ((end1 - start1) * 1000)
    #print("The register time is:", registertime)
    start2 = time.time()
    # res = fxc.run(items, endpoint_id='7601789e-8569-413f-be3e-e573d04c5799', function_id=sum_function)
    res = fxc.run(ress,
                  endpoint_id='2e3061ea-ef19-4cbb-bc81-82f8cc47baa9',
                  function_id=hello_function)
    end2 = time.time()
    runtime = ((end2 - start2) * 1000)
    print("The runtime is:", runtime)
    # get the raw json response
    start = time.time()
    result = fxc.get(f"tasks/{res}")
    while result['status'] != 'success':
        time.sleep(1)
        result = fxc.get(f"tasks/{res}")

    completion_time = result['completion_t']
    exec_time = (float(completion_time) - start) * 1000
    print("The function execution time is:", exec_time)
    # add the result to the list
Ejemplo n.º 11
0
sheet.write(0, 2, 'runtime')
sheet.write(0, 3, 'exec_time')
for i in range(0, n):
    #funcx-endpoint start Test
    #funcx-endpoint stop Test
    #funcx-endpoint start Test
    start1 = time.time()
    hello_function = fxc.register_function(car)
    enh = ImageEnhance.Contrast(im)
    end1 = time.time()
    registertime = ((end1 - start1) * 1000)
    print("The register time is:", registertime)
    start2 = time.time()
    # res = fxc.run(items, endpoint_id='7601789e-8569-413f-be3e-e573d04c5799', function_id=sum_function)
    res = fxc.run(enh,
                  endpoint_id='d4b5b300-d12b-40d2-acff-fa54cb7dcfb2',
                  function_id=hello_function)
    end2 = time.time()
    runtime = ((end2 - start2) * 1000)
    print("The runtime is:", runtime)
    # get the raw json response
    start = time.time()
    result = fxc.get(f"tasks/{res}")
    while result['status'] != 'success':
        time.sleep(1)
        result = fxc.get(f"tasks/{res}")

    completion_time = result['completion_t']
    exec_time = (float(completion_time) - start) * 1000
    print("The function execution time is:", exec_time)
    # add the result to the list
Ejemplo n.º 12
0
    batch = fxc.create_batch()
    for _ in range(args.num_arrays):
        x = np.random.rand(args.size, args.size)
        if args.proxy:
            x = store.proxy(x)
        batch.add(x, endpoint_id=args.funcx_endpoint, function_id=double_uuid)

    batch_res = fxc.batch_run(batch)
    mapped_results = fxc.get_batch_result(batch_res)
    for res in mapped_results.values():
        while res["pending"]:
            time.sleep(0.1)

    mapped_results = [
        fxc.get_result(i) for i, status in mapped_results.items()
    ]

    if args.proxy:
        mapped_results = store.proxy(mapped_results)
    total = fxc.run(
        mapped_results,
        endpoint_id=args.funcx_endpoint,
        function_id=sum_uuid,
    )

    while fxc.get_task(total)["pending"]:
        time.sleep(0.1)

    print("Sum:", fxc.get_result(total))
    return pyhf.Workspace(data)


if __name__ == "__main__":
    # locally get pyhf pallet for analysis
    if not Path("1Lbb-pallet").exists():
        download("https://doi.org/10.17182/hepdata.90607.v3/r3", "1Lbb-pallet")
    with open("1Lbb-pallet/BkgOnly.json") as bkgonly_json:
        bkgonly_workspace = json.load(bkgonly_json)

    # Use privately assigned endpoint id
    with open("endpoint_id.txt") as endpoint_file:
        pyhf_endpoint = str(endpoint_file.read().rstrip())

    fxc = FuncXClient()

    # Register function and execute on worker node
    prepare_func = fxc.register_function(prepare_workspace)
    prepare_task = fxc.run(bkgonly_workspace,
                           endpoint_id=pyhf_endpoint,
                           function_id=prepare_func)

    # Wait for worker to finish and retrieve results
    workspace = None
    while not workspace:
        try:
            workspace = fxc.get_result(prepare_task)
        except Exception as excep:
            print(f"prepare: {excep}")
            sleep(10)
Ejemplo n.º 14
0
class DLHubClient(BaseClient):
    """Main class for interacting with the DLHub service

    Holds helper operations for performing common tasks with the DLHub service. For example,
    `get_servables` produces a list of all servables registered with DLHub.

    For most cases, we recommend creating a new DLHubClient by calling ``DLHubClient.login``.
    This operation will check if you have saved any credentials to disk before using the CLI or SDK
    and, if not, get new credentials and save them for later use.
    For cases where disk access is unacceptable, you can create the client by creating an authorizer
    following the
    `tutorial for the Globus SDK <https://globus-sdk-python.readthedocs.io/en/stable/tutorial/>`_
    and providing that authorizer to the initializer (e.g., ``DLHubClient(auth)``)"""
    def __init__(self,
                 dlh_authorizer=None,
                 search_client=None,
                 http_timeout=None,
                 force_login=False,
                 fx_authorizer=None,
                 **kwargs):
        """Initialize the client

        Args:
            dlh_authorizer (:class:`GlobusAuthorizer
                            <globus_sdk.authorizers.base.GlobusAuthorizer>`):
                An authorizer instance used to communicate with DLHub.
                If ``None``, will be created.
            search_client (:class:`SearchClient <globus_sdk.SearchClient>`):
                An authenticated SearchClient to communicate with Globus Search.
                If ``None``, will be created.
            http_timeout (int): Timeout for any call to service in seconds. (default is no timeout)
            force_login (bool): Whether to force a login to get new credentials.
                A login will always occur if ``dlh_authorizer`` or ``search_client``
                are not provided.
            no_local_server (bool): Disable spinning up a local server to automatically
                copy-paste the auth code. THIS IS REQUIRED if you are on a remote server.
                When used locally with no_local_server=False, the domain is localhost with
                a randomly chosen open port number.
                **Default**: ``True``.
            fx_authorizer (:class:`GlobusAuthorizer
                            <globus_sdk.authorizers.base.GlobusAuthorizer>`):
                An authorizer instance used to communicate with funcX.
                If ``None``, will be created.
            no_browser (bool): Do not automatically open the browser for the Globus Auth URL.
                Display the URL instead and let the user navigate to that location manually.
                **Default**: ``True``.
        Keyword arguments are the same as for BaseClient.
        """
        if force_login or not dlh_authorizer or not search_client or not fx_authorizer:

            fx_scope = "https://auth.globus.org/scopes/facd7ccc-c5f4-42aa-916b-a0e270e2c2a9/all"
            auth_res = login(services=["search", "dlhub", fx_scope, "openid"],
                             app_name="DLHub_Client",
                             client_id=CLIENT_ID,
                             clear_old_tokens=force_login,
                             token_dir=_token_dir,
                             no_local_server=kwargs.get(
                                 "no_local_server", True),
                             no_browser=kwargs.get("no_browser", True))
            # openid_authorizer = auth_res["openid"]
            dlh_authorizer = auth_res["dlhub"]
            fx_authorizer = auth_res[fx_scope]
            self._search_client = auth_res["search"]

        self._fx_client = FuncXClient(
            force_login=force_login,
            no_local_server=kwargs.get("no_local_server", True),
            no_browser=kwargs.get("no_browser", True),
            funcx_service_address='https://api.funcx.org/v1',
        )

        # funcX endpoint to use
        self.fx_endpoint = '86a47061-f3d9-44f0-90dc-56ddc642c000'
        # self.fx_endpoint = '2c92a06a-015d-4bfa-924c-b3d0c36bdad7'
        self.fx_serializer = FuncXSerializer()
        self.fx_cache = {}
        super(DLHubClient, self).__init__("DLHub",
                                          environment='dlhub',
                                          authorizer=dlh_authorizer,
                                          http_timeout=http_timeout,
                                          base_url=DLHUB_SERVICE_ADDRESS,
                                          **kwargs)

    def logout(self):
        """Remove credentials from your local system"""
        logout()

    @property
    def query(self):
        """Access a query of the DLHub Search repository"""
        return DLHubSearchHelper(search_client=self._search_client)

    def get_username(self):
        """Get the username associated with the current credentials"""

        res = self.get('/namespaces')
        return res.data['namespace']

    def get_servables(self, only_latest_version=True):
        """Get all of the servables available in the service

        Args:
            only_latest_version (bool): Whether to only return the latest version of each servable
        Returns:
            ([list]) Complete metadata for all servables found in DLHub
        """

        # Get all of the servables
        results, info = self.query.match_field('dlhub.type', 'servable')\
            .add_sort('dlhub.owner', ascending=True).add_sort('dlhub.name', ascending=False)\
            .add_sort('dlhub.publication_date', ascending=False).search(info=True)
        if info['total_query_matches'] > SEARCH_LIMIT:
            raise RuntimeError(
                'DLHub contains more servables than we can return in one entry. '
                'DLHub SDK needs to be updated.')

        if only_latest_version:
            # Sort out only the most recent versions (they come first in the sorted list
            names = set()
            output = []
            for r in results:
                name = r['dlhub']['shorthand_name']
                if name not in names:
                    names.add(name)
                    output.append(r)
            results = output

        # Add these to the cache
        for r in results:
            self.fx_cache[r['dlhub']
                          ['shorthand_name']] = r['dlhub']['funcx_id']

        return results

    def list_servables(self):
        """Get a list of the servables available in the service

        Returns:
            [string]: List of all servable names in username/servable_name format
        """

        servables = self.get_servables(only_latest_version=True)
        return [x['dlhub']['shorthand_name'] for x in servables]

    def get_task_status(self, task_id):
        """Get the status of a DLHub task.

        Args:
            task_id (string): UUID of the task
        Returns:
            dict: status block containing "status" key.
        """

        r = self._fx_client.get_task(task_id)
        return r

    def describe_servable(self, name):
        """Get the description for a certain servable

        Args:
            name (string): DLHub name of the servable of the form <user>/<servable_name>
        Returns:
            dict: Summary of the servable
        """
        split_name = name.split('/')
        if len(split_name) < 2:
            raise AttributeError(
                'Please enter name in the form <user>/<servable_name>')

        # Create a query for a single servable
        query = self.query.match_servable('/'.join(split_name[1:]))\
            .match_owner(split_name[0]).add_sort("dlhub.publication_date", False)\
            .search(limit=1)

        # Raise error if servable is not found
        if len(query) == 0:
            raise AttributeError('No such servable: {}'.format(name))
        return query[0]

    def describe_methods(self, name, method=None):
        """Get the description for the method(s) of a certain servable

        Args:
            name (string): DLHub name of the servable of the form <user>/<servable_name>
            method (string): Optional: Name of the method
        Returns:
             dict: Description of a certain method if ``method`` provided, all methods
                if the method name was not provided.
        """

        metadata = self.describe_servable(name)
        return get_method_details(metadata, method)

    def run(self,
            name,
            inputs,
            asynchronous=False,
            async_wait=5,
            timeout: Optional[float] = None) -> Union[Any, DLHubFuture]:
        """Invoke a DLHub servable

        Args:
            name (string): DLHub name of the servable of the form <user>/<servable_name>
            inputs: Data to be used as input to the function. Can be a string of file paths or URLs
            asynchronous (bool): Whether to return from the function immediately or
                wait for the execution to finish.
            async_wait (float): How many seconds to wait between checking async status
            timeout (float): How long to wait for a result to return.
                Only used for synchronous calls
        Returns:
            Results of running the servable. If asynchronous, then a DLHubFuture holding the result
        """

        if name not in self.fx_cache:
            # Look it up and add it to the cache, this will raise an exception if not found.
            serv = self.describe_servable(name)
            self.fx_cache.update({name: serv['dlhub']['funcx_id']})

        funcx_id = self.fx_cache[name]
        payload = {'data': inputs}
        task_id = self._fx_client.run(payload,
                                      endpoint_id=self.fx_endpoint,
                                      function_id=funcx_id)

        # Return the result
        future = DLHubFuture(self, task_id, async_wait)
        return future.result(timeout=timeout) if not asynchronous else future

    def run_serial(self, servables, inputs, async_wait=5):
        """Invoke each servable in a serial pipeline.
        This function accepts a list of servables and will run each one,
        passing the output of one as the input to the next.

        Args:
             servables (list): A list of servable strings
             inputs: Data to pass to the first servable
             asycn_wait (float): Seconds to wait between status checks
        Returns:
            Results of running the servable
        """
        if not isinstance(servables, list):
            print("run_serial requires a list of servables to invoke.")

        serv_data = inputs
        for serv in servables:
            serv_data = self.run(serv, serv_data, async_wait=async_wait)
        return serv_data

    def get_result(self, task_id, verbose=False):
        """Get the result of a task_id

        Args:
            task_id str: The task's uuid
            verbose bool: whether or not to return the full dlhub response
        Returns:
            Reult of running the servable
        """

        result = self._fx_client.get_result(task_id)
        if isinstance(result, tuple) and not verbose:
            result = result[0]
        return result

    def publish_servable(self, model):
        """Submit a servable to DLHub

        If this servable has not been published before, it will be assigned a unique identifier.

        If it has been published before (DLHub detects if it has an identifier), then DLHub
        will update the servable to the new version.

        Args:
            model (BaseMetadataModel): Servable to be submitted
        Returns:
            (string): Task ID of this submission, used for checking for success
        """

        # Get the metadata
        metadata = model.to_dict(simplify_paths=True)

        # Mark the method used to submit the model
        metadata['dlhub']['transfer_method'] = {'POST': 'file'}

        # Validate against the servable schema
        validate_against_dlhub_schema(metadata, 'servable')

        # Wipe the fx cache so we don't keep reusing an old servable
        self.clear_funcx_cache()

        # Get the data to be submitted as a ZIP file
        fp, zip_filename = mkstemp('.zip')
        os.close(fp)
        os.unlink(zip_filename)
        try:
            model.get_zip_file(zip_filename)

            # Get the authorization headers
            headers = {}
            self.authorizer.set_authorization_header(headers)

            # Submit data to DLHub service
            with open(zip_filename, 'rb') as zf:
                reply = requests.post(slash_join(self.base_url, 'publish'),
                                      headers=headers,
                                      files={
                                          'json':
                                          ('dlhub.json', json.dumps(metadata),
                                           'application/json'),
                                          'file': ('servable.zip', zf,
                                                   'application/octet-stream')
                                      })

            # Return the task id
            if reply.status_code != 200:
                raise Exception(reply.text)
            return reply.json()['task_id']
        finally:
            os.unlink(zip_filename)

    def publish_repository(self, repository):
        """Submit a repository to DLHub for publication

        Args:
            repository (string): Repository to publish
        Returns:
            (string): Task ID of this submission, used for checking for success
        """

        # Publish to DLHub
        metadata = {"repository": repository}

        # Wipe the fx cache so we don't keep reusing an old servable
        self.clear_funcx_cache()

        response = self.post('publish_repo', json_body=metadata)

        task_id = response.data['task_id']
        return task_id

    def search(self, query, advanced=False, limit=None, only_latest=True):
        """Query the DLHub servable library

        By default, the query is used as a simple plaintext search of all model metadata.
        Optionally, you can provided an advanced query on any of the indexed fields in
        the DLHub model metadata by setting :code:`advanced=True` and following the guide for
        constructing advanced queries found in the
        `Globus Search documentation <https://docs.globus.org/api/search/search/#query_syntax>`_.

        Args:
             query (string): Query to be performed
             advanced (bool): Whether to perform an advanced query
             limit (int): Maximum number of entries to return
             only_latest (bool): Whether to return only the latest version of the model
        Returns:
            ([dict]): All records matching the search query
        """

        results = self.query.search(query, advanced=advanced, limit=limit)
        return filter_latest(results) if only_latest else results

    def search_by_servable(self,
                           servable_name=None,
                           owner=None,
                           version=None,
                           only_latest=True,
                           limit=None,
                           get_info=False):
        """Search by the ownership, name, or version of a servable

        Args:
            servable_name (str): The name of the servable. **Default**: None, to match
                    all servable names.
            owner (str): The name of the owner of the servable. **Default**: ``None``,
                    to match all owners.
            version (int): Model version, which corresponds to the date when the
                servable was published. **Default**: ``None``, to match all versions.
            only_latest (bool): When ``True``, will only return the latest version
                    of each servable. When ``False``, will return all matching versions.
                    **Default**: ``True``.
            limit (int): The maximum number of results to return.
                    **Default:** ``None``, for no limit.
            get_info (bool): If ``False``, search will return a list of the results.
                    If ``True``, search will return a tuple containing the results list
                    and other information about the query.
                    **Default:** ``False``.

        Returns:
            If ``info`` is ``False``, *list*: The search results.
            If ``info`` is ``True``, *tuple*: The search results,
            and a dictionary of query information.
        """
        if not servable_name and not owner and not version:
            raise ValueError(
                "One of 'servable_name', 'owner', or 'publication_date' is required."
            )

        # Perform the query
        results, info = (self.query.match_servable(
            servable_name=servable_name, owner=owner,
            publication_date=version).search(limit=limit, info=True))

        # Filter out the latest models
        if only_latest:
            results = filter_latest(results)

        if get_info:
            return results, info
        return results

    def search_by_authors(self,
                          authors,
                          match_all=True,
                          limit=None,
                          only_latest=True):
        """Execute a search for servables from certain authors.

        Authors in DLHub may be different than the owners of the servable and generally are
        the people who developed functionality of a certain servable (e.g., the creators
        of the machine learning model used in a servable).

        If you want to search by ownership, see :meth:`search_by_servable`

        Args:
            authors (str or list of str): The authors to match. Names must be in
                "Family Name, Given Name" format
            match_all (bool): If ``True``, will require all authors be on any results.
                    If ``False``, will only require one author to be in results.
                    **Default**: ``True``.
            limit (int): The maximum number of results to return.
                    **Default:** ``None``, for no limit.
            only_latest (bool): When ``True``, will only return the latest version
                    of each servable. When ``False``, will return all matching versions.
                    **Default**: ``True``.

        Returns:
            [dict]: List of servables from the desired authors
        """
        results = self.query.match_authors(
            authors, match_all=match_all).search(limit=limit)
        return filter_latest(results) if only_latest else results

    def search_by_related_doi(self, doi, limit=None, only_latest=True):
        """Get all of the servables associated with a certain publication

        Args:
            doi (string): DOI of related paper
            limit (int): Maximum number of results to return
            only_latest (bool): Whether to return only the most recent version of the model
        Returns:
            [dict]: List of servables from the requested paper
        """

        results = self.query.match_doi(doi).search(limit=limit)
        return filter_latest(results) if only_latest else results

    def clear_funcx_cache(self, servable=None):
        """Remove functions from the cache. Either remove a specific servable or wipe the whole cache.

        Args:
            Servable: str
                The name of the servable to remove. Default None
        """

        if servable:
            del (self.fx_cache[servable])
        else:
            self.fx_cache = {}

        return self.fx_cache
Ejemplo n.º 15
0
# In[70]:


def hello_world(name):
    return f"Hello, {name}"


hello_func = fxc.register_function(hello_world,
                                   description="Test hello world.")
print(hello_func)

# In[71]:

name = "Ryan"
res = fxc.run(name=name, endpoint_id=local_ep, function_id=hello_func)

# In[72]:

fxc.get_result(res)

# In[74]:

name = "Cooley"
res = fxc.run(name=name, endpoint_id=cooley_ep, function_id=hello_func)

# In[75]:

fxc.get_result(res)

# In[63]:
    d = w.data(m)
    return {
        'metadata': metadata,
        'CLs_obs': float(pyhf.infer.hypotest(1.0, d, m, qtilde=True)),
        'Fit-Time': time.time() - tick
    }


infer_func = fxc.register_function(infer_hypotest)

data = requests.get(
    'https://gist.githubusercontent.com/lukasheinrich/75b80a2f8bc49e365bfb96e767c8a726/raw/a0946bc7590c76fec2b70de2f6f46208c0545c8d/BkgOnly.json'
).json()

prepare_task = fxc.run(data,
                       endpoint_id=pyhf_endpoint,
                       function_id=prepare_func)

# While this cooks, let's read in the patch set
patches = None
with open('patchset.json') as f:
    patches = json.load(f)
patch = patches['patches'][0]
name = patch['metadata']['name']

w = None

while not w:
    try:
        w = fxc.get_result(prepare_task)
    except Exception as e:
sheet.write(0,1,'registertime')
sheet.write(0,2,'runtime')
sheet.write(0,3,'exec_time')
for i in range (0, n):
    funcx-endpoint start Test
    funcx-endpoint stop Test
    funcx-endpoint start Test
    start1 = time.time()
    hello_function = fxc.register_function(hello_world)
    event = ("Hello World")
    end1 = time.time()
    registertime = ((end1 - start1) * 1000)
    print("The register time is:", registertime)
    start2 = time.time()
    # res = fxc.run(items, endpoint_id='7601789e-8569-413f-be3e-e573d04c5799', function_id=sum_function)
    res = fxc.run(event, endpoint_id='a4c93d82-58e0-4062-aa97-be34f6734e88', function_id=hello_function)
    end2 = time.time()
    runtime = ((end2 - start2) * 1000)
    print("The runtime is:", runtime)
    # get the raw json response
    start = time.time()
    result = fxc.get(f"tasks/{res}") 
    while result['status'] != 'success':
        time.sleep(1)
        result = fxc.get(f"tasks/{res}")

    completion_time = result['completion_t']
    exec_time = (float(completion_time) - start) * 1000
    print("The function execution time is:", exec_time)
    # add the result to the list
    estimate.append(exec_time)