def list_acls(endpoint, count_only=False, re_match=None): """List or count the ACLs on an endpoint, optionally matching some regex. Arguments: endpoint (str): The endpoint ID to check for ACLs. count_only (bool): When True, will only return the total count of ACLs found. When False, will print each ACL found. Default False. re_match (str): A regular expression to test the ACL's path. Any ACL rule that has a path matching this regex will be counted. Default None, to match all ACLs. Returns: int: The total ACLs matched. Note: You must be an Access Manager or higher on the target endpoint to list ACLs. """ tc = mdf_toolbox.login(services="transfer")["transfer"] count = 0 for rule in tc.endpoint_acl_list(endpoint): if re_match is None or re.match(re_match, rule["path"]): count += 1 if not count_only: print(rule) return count
def ingest_to_search(name, path, idx="dlhub-test"): """Ingests the model metadata into Globus Search Args: name (str): The model name. (generally the same as container directory name but not always) e.g. "cifar10" path (str): Path to the model metadata. e.g. "metadata/cifar10.json" idx (str): The Globus Index to upload search metadata to. Defaults=dlhub-test """ if path == None: return dl = client.DLHub() uuid = dl.get_id_by_name(name) iden = "https://dlhub.org/api/v1/servables/{}".format(uuid) index = mdf_toolbox.translate_index(idx) with open(path, 'r') as f: ingestable = json.load(f) ingestable = mdf_toolbox.format_gmeta(ingestable, acl="public", identifier=iden) ingestable = mdf_toolbox.format_gmeta( [ingestable]) # Make it a GIngest list of GMetaEntry ingest_client = mdf_toolbox.login( services=["search_ingest"])["search_ingest"] ingest_client.ingest(index, ingestable) print("Ingestion of {} to DLHub servables complete".format(name))
def __init__(self, authorizer=None, http_timeout=None, force_login=False, **kwargs): """Initialize the client Args: authorizer (:class:`GlobusAuthorizer <globus_sdk.authorizers.base.GlobusAuthorizer>`): An authorizer instance used to communicate with paropt. If ``None``, will be created. http_timeout (int): Timeout for any call to service in seconds. (default is no timeout) force_login (bool): Whether to force a login to get new credentials. A login will always occur if ``authorizer`` are not provided. Keyword arguments are the same as for BaseClient. """ if force_login or not authorizer or not search_client: dlhub_scope = "https://auth.globus.org/scopes/81fc4156-a623-47f2-93ad-7184118226ba/auth" auth_res = login(services=[dlhub_scope], app_name="paropt", client_id=CLIENT_ID, clear_old_tokens=force_login, token_dir=_token_dir) dlh_authorizer = auth_res['dlhub_org'] super(ParoptClient, self).__init__("paropt", authorizer=dlh_authorizer, http_timeout=http_timeout, base_url=PAROPT_SERVICE_ADDRESS, **kwargs)
def delete_acls(endpoint, re_match, dry_run=True): """Delete ACLs from an endpoint matching some regex on the rule path. Arguments: endpoint (str): The endpoint ID to delete ACLs from. re_match (str): A regular expression to test the ACL's path. Any ACL rule that has a path matching this regex will be deleted. dry_run (bool): When True, will not actually delete anything, and instead will print the ACL rules that would have been deleted. When False, will permanently delete the ACL rules. Default True. Returns: int: The number of ACL rules matched for deletion. Caution: This function deletes ACL rules, which removes permissions. You must be an Access Manager or higher on the target endpoint, and you must exercise caution. """ tc = mdf_toolbox.login(services="transfer")["transfer"] count = 0 for rule in tc.endpoint_acl_list(endpoint): if re.match(re_match, rule["path"]): count += 1 if dry_run: print(rule) else: tc.delete_endpoint_acl_rule(endpoint, rule["id"]) return count
def test_login(): # Login works # Impersonate Forge res1 = mdf_toolbox.login(services="search", app_name="MDF_Forge", client_id="b2b437c4-17c1-4e4b-8f15-e9783e1312d7") assert type(res1) is dict assert isinstance(res1.get("search"), globus_sdk.SearchClient) # Test other services # Use default "unknown app" # TODO: "groups" cannot be tested without whitelisting app res2 = mdf_toolbox.login(services=["search_ingest", "transfer", "data_mdf", "mdf_connect", "petrel"]) assert isinstance(res2.get("search_ingest"), globus_sdk.SearchClient) assert isinstance(res2.get("transfer"), globus_sdk.TransferClient) assert isinstance(res2.get("data_mdf"), globus_sdk.RefreshTokenAuthorizer) assert isinstance(res2.get("mdf_connect"), globus_sdk.RefreshTokenAuthorizer) assert isinstance(res2.get("petrel"), globus_sdk.RefreshTokenAuthorizer)
def __init__(self, dlh_authorizer=None, search_client=None, http_timeout=None, force_login=False, fx_authorizer=None, **kwargs): """Initialize the client Args: dlh_authorizer (:class:`GlobusAuthorizer <globus_sdk.authorizers.base.GlobusAuthorizer>`): An authorizer instance used to communicate with DLHub. If ``None``, will be created. search_client (:class:`SearchClient <globus_sdk.SearchClient>`): An authenticated SearchClient to communicate with Globus Search. If ``None``, will be created. http_timeout (int): Timeout for any call to service in seconds. (default is no timeout) force_login (bool): Whether to force a login to get new credentials. A login will always occur if ``dlh_authorizer`` or ``search_client`` are not provided. no_local_server (bool): Disable spinning up a local server to automatically copy-paste the auth code. THIS IS REQUIRED if you are on a remote server. When used locally with no_local_server=False, the domain is localhost with a randomly chosen open port number. **Default**: ``True``. fx_authorizer (:class:`GlobusAuthorizer <globus_sdk.authorizers.base.GlobusAuthorizer>`): An authorizer instance used to communicate with funcX. If ``None``, will be created. no_browser (bool): Do not automatically open the browser for the Globus Auth URL. Display the URL instead and let the user navigate to that location manually. **Default**: ``True``. Keyword arguments are the same as for BaseClient. """ if force_login or not dlh_authorizer or not search_client or not fx_authorizer: fx_scope = "https://auth.globus.org/scopes/facd7ccc-c5f4-42aa-916b-a0e270e2c2a9/all" auth_res = login(services=["search", "dlhub", fx_scope], app_name="DLHub_Client", client_id=CLIENT_ID, clear_old_tokens=force_login, token_dir=_token_dir, no_local_server=kwargs.get("no_local_server", True), no_browser=kwargs.get("no_browser", True)) dlh_authorizer = auth_res["dlhub"] fx_authorizer = auth_res[fx_scope] self._search_client = auth_res["search"] self._fx_client = FuncXClient(force_login=True,fx_authorizer=fx_authorizer, funcx_service_address='https://funcx.org/api/v1') # funcX endpoint to use self.fx_endpoint = '86a47061-f3d9-44f0-90dc-56ddc642c000' # self.fx_endpoint = '2c92a06a-015d-4bfa-924c-b3d0c36bdad7' self.fx_serializer = FuncXSerializer() self.fx_cache = {} super(DLHubClient, self).__init__("DLHub", environment='dlhub', authorizer=dlh_authorizer, http_timeout=http_timeout, base_url=DLHUB_SERVICE_ADDRESS, **kwargs)
def __init__(self, index, **kwargs): """Create a SearchHelper object. Arguments: index (str): The Globus Search index to search on. Keyword Arguments: search_client (globus_sdk.SearchClient): The Globus Search client to use for searching. If not provided, one will be created and the user may be asked to log in. **Default**: ``None``. anonymous (bool): If ``True``, will not authenticate with Globus Auth. If ``False``, will require authentication (either a SearchClient or a user-interactive login). **Default:** ``False``. Caution: Authentication is required to view non-public data in Search. An anonymous SearchHelper will only return public results. app_name (str): The application name to use. Should be changed for subclassed clients, and left alone otherwise. Only used if performing login flow. **Default**: ``"SearchHelper_Client"``. client_id (str): The ID of a native client to use when authenticating. Only used if performing login flow. **Default**: The default SearchHelper client ID. q (str): A query string to initialize the SearchHelper with. Intended for internal use. advanced (bool): The initial advanced state for thie SearchHelper. Intended for internal use. """ if kwargs.get("search_client"): self.__search_client = kwargs["search_client"] elif kwargs.get("anonymous"): self.__search_client = mdf_toolbox.anonymous_login(["search" ])["search"] else: self.__search_client = mdf_toolbox.login( app_name=kwargs.get("app_name", self.__app_name), client_id=kwargs.get("client_id", self.__client_id), services=["search"])["search"] # Get the UUID for the index if the name was provided self.index = mdf_toolbox.translate_index(index) # Query init self.__query = deepcopy(BLANK_QUERY) if kwargs.get("q"): self.__query["q"] = kwargs["q"] if kwargs.get("advanced"): self.__query["advanced"] = kwargs["advanced"]
def __init__(self, no_browser=False, no_local_server=False, search_index="mdf-test", **data): super().__init__(**data) auths = mdf_toolbox.login( services=[ "data_mdf", "search", "petrel", "transfer", "dlhub", "https://auth.globus.org/scopes/facd7ccc-c5f4-42aa-916b-a0e270e2c2a9/all", ], app_name="Foundry", make_clients=True, no_browser=no_browser, no_local_server=no_local_server, ) self.forge_client = Forge( index=search_index, services=None, search_client=auths["search"], transfer_client=auths["transfer"], data_mdf_authorizer=auths["data_mdf"], petrel_authorizer=auths["petrel"], ) self.dlhub_client = DLHubClient( dlh_authorizer=auths["dlhub"], search_client=auths["search"], fx_authorizer=auths[ "https://auth.globus.org/scopes/facd7ccc-c5f4-42aa-916b-a0e270e2c2a9/all"], force_login=False, ) self.xtract_tokens = { 'auth_token': auths['petrel'].access_token, 'transfer_token': auths['transfer'].authorizer.access_token, 'funx_token': auths[ 'https://auth.globus.org/scopes/facd7ccc-c5f4-42aa-916b-a0e270e2c2a9/all'] .access_token }
def __init__(self, http_timeout=None, funcx_home=os.path.join('~', '.funcx'), force_login=False, fx_authorizer=None, **kwargs): """ Initialize the client Parameters ---------- http_timeout: int Timeout for any call to service in seconds. Default is no timeout force_login: bool Whether to force a login to get new credentials. fx_authorizer:class:`GlobusAuthorizer <globus_sdk.authorizers.base.GlobusAuthorizer>`: A custom authorizer instance to communicate with funcX. Default: ``None``, will be created. Keyword arguments are the same as for BaseClient. """ self.ep_registration_path = 'register_endpoint_2' self.funcx_home = os.path.expanduser(funcx_home) if force_login or not fx_authorizer: fx_scope = "https://auth.globus.org/scopes/facd7ccc-c5f4-42aa-916b-a0e270e2c2a9/all" auth_res = login(services=[fx_scope], app_name="funcX_Client", client_id=self.CLIENT_ID, clear_old_tokens=force_login, token_dir=self.TOKEN_DIR) dlh_authorizer = auth_res['funcx_service'] super(FuncXClient, self).__init__("funcX", environment='funcx', authorizer=dlh_authorizer, http_timeout=http_timeout, base_url=self.FUNCX_SERVICE_ADDRESS, **kwargs) self.fx_serializer = FuncXSerializer()
def ingest_to_search(name, path, idx): if path == None: return dl = client.DLHub() uuid = dl.get_id_by_name(name) iden = "https://dlhub.org/api/v1/servables/{}".format(uuid) index = mdf_toolbox.translate_index(idx) with open(path, 'r') as f: ingestable = json.load(f) ingestable = mdf_toolbox.format_gmeta(ingestable, acl="public", identifier=iden) ingestable = mdf_toolbox.format_gmeta( [ingestable]) # Make it a GIngest list of GMetaEntry ingest_client = mdf_toolbox.login( services=["search_ingest"])["search_ingest"] ingest_client.ingest(index, ingestable) print("Ingestion of {} complete".format(name))
def __init__(self, dlh_authorizer=None, search_client=None, http_timeout=None, force_login=False, **kwargs): """Initialize the client Args: dlh_authorizer (:class:`GlobusAuthorizer <globus_sdk.authorizers.base.GlobusAuthorizer>`): An authorizer instance used to communicate with DLHub. If ``None``, will be created. search_client (:class:`SearchClient <globus_sdk.SearchClient>`): An authenticated SearchClient to communicate with Globus Search. If ``None``, will be created. http_timeout (int): Timeout for any call to service in seconds. (default is no timeout) force_login (bool): Whether to force a login to get new credentials. A login will always occur if ``dlh_authorizer`` or ``search_client`` are not provided. Keyword arguments are the same as for BaseClient. """ if force_login or not dlh_authorizer or not search_client: auth_res = login(services=["search", "dlhub"], app_name="DLHub_Client", client_id=CLIENT_ID, clear_old_tokens=force_login, token_dir=_token_dir) dlh_authorizer = auth_res["dlhub"] self._search_client = auth_res["search"] super(DLHubClient, self).__init__("DLHub", environment='dlhub', authorizer=dlh_authorizer, http_timeout=http_timeout, base_url=DLHUB_SERVICE_ADDRESS, **kwargs)
def search_ingest(task): """ Ingest the servable data into a Globus Search index. Args: task (dict): the task description. """ logging.debug("Ingesting servable into Search.") # idx = "dlhub" idx = '847c9105-18a0-4ffb-8a71-03dd76dfcc9d' iden = "https://dlhub.org/servables/{}".format(task['dlhub']['id']) index = mdf_toolbox.translate_index(idx) ingestable = task d = [convert_dict(ingestable, str)] glist = [] visible_to = task['dlhub'].get('visible_to', ['public']) # Add public so it isn't an empty list if len(visible_to) == 0: visible_to = ['public'] for document in d: gmeta_entry = mdf_toolbox.format_gmeta(document, visible_to, iden) glist.append(gmeta_entry) gingest = mdf_toolbox.format_gmeta(glist) ingest_client = mdf_toolbox.login(services=["search_ingest"], no_local_server=True, no_browser=True)["search_ingest"] logging.info("ingesting to search") logging.info(gingest) ingest_client.ingest(idx, gingest) logging.info("Ingestion of {} to DLHub servables complete".format(iden))
import pytest import re import mdf_toolbox SEARCH_CLIENT = mdf_toolbox.login(services=["search"], app_name="SearchHelper", client_id="878721f5-6b92-411e-beac-830672c0f69a")["search"] INDEX = "mdf" SCROLL_FIELD = "mdf.scroll_id" class DummyClient(mdf_toolbox.AggregateHelper, mdf_toolbox.SearchHelper): def __init__(self, *args, **kwargs): super().__init__(*args, scroll_field=SCROLL_FIELD, **kwargs) # Helper # Return codes: # -1: No match, the value was never found # 0: Exclusive match, no values other than argument found # 1: Inclusive match, some values other than argument found # 2: Partial match, value is found in some but not all results def check_field(res, field, regex): dict_path = "" for key in field.split("."): if key == "[]": dict_path += "[0]" else: dict_path += ".get('{}', {})".format(key, "{}") # If no results, set matches to false
def __init__(self, dlh_authorizer: Optional[GlobusAuthorizer] = None, search_authorizer: Optional[GlobusAuthorizer] = None, fx_authorizer: Optional[GlobusAuthorizer] = None, openid_authorizer: Optional[GlobusAuthorizer] = None, http_timeout: Optional[int] = None, force_login: bool = False, **kwargs): """Initialize the client Args: http_timeout (int): Timeout for any call to service in seconds. (default is no timeout) force_login (bool): Whether to force a login to get new credentials. A login will always occur if ``dlh_authorizer`` or ``search_client`` are not provided. no_local_server (bool): Disable spinning up a local server to automatically copy-paste the auth code. THIS IS REQUIRED if you are on a remote server. When used locally with no_local_server=False, the domain is localhost with a randomly chosen open port number. **Default**: ``True``. no_browser (bool): Do not automatically open the browser for the Globus Auth URL. Display the URL instead and let the user navigate to that location manually. **Default**: ``True``. dlh_authorizer (:class:`GlobusAuthorizer <globus_sdk.authorizers.base.GlobusAuthorizer>`): An authorizer instance used to communicate with DLHub. If ``None``, will be created from your account's credentials. search_authorizer (:class:`GlobusAuthorizer <globus_sdk.authorizers.base.GlobusAuthorizer>`): An authenticated SearchClient to communicate with Globus Search. If ``None``, will be created from your account's credentials. fx_authorizer (:class:`GlobusAuthorizer <globus_sdk.authorizers.base.GlobusAuthorizer>`): An authorizer instance used to communicate with funcX. If ``None``, will be created from your account's credentials. openid_authorizer (:class:`GlobusAuthorizer <globus_sdk.authorizers.base.GlobusAuthorizer>`): An authorizer instance used to communicate with OpenID. If ``None``, will be created from your account's credentials. Keyword arguments are the same as for :class:`BaseClient <globus_sdk.base.BaseClient>`. """ authorizers = [dlh_authorizer, search_authorizer, openid_authorizer, fx_authorizer] # Get authorizers through Globus login if any are not provided if not all(a is not None for a in authorizers): # If some but not all were provided, warn the user they could be making a mistake if any(a is not None for a in authorizers): logger.warning('You have defined some of the authorizers but not all. DLHub is falling back to login. ' 'You must provide authorizers for DLHub, Search, OpenID, FuncX.') fx_scope = "https://auth.globus.org/scopes/facd7ccc-c5f4-42aa-916b-a0e270e2c2a9/all" auth_res = login(services=["search", "dlhub", fx_scope, "openid"], app_name="DLHub_Client", make_clients=False, client_id=CLIENT_ID, clear_old_tokens=force_login, token_dir=_token_dir, no_local_server=kwargs.get("no_local_server", True), no_browser=kwargs.get("no_browser", True)) # Unpack the authorizers dlh_authorizer = auth_res["dlhub"] fx_authorizer = auth_res[fx_scope] openid_authorizer = auth_res['openid'] search_authorizer = auth_res['search'] # Define the subclients needed by the service self._fx_client = FuncXClient(fx_authorizer=fx_authorizer, search_authorizer=search_authorizer, openid_authorizer=openid_authorizer, no_local_server=kwargs.get("no_local_server", True), no_browser=kwargs.get("no_browser", True)) self._search_client = globus_sdk.SearchClient(authorizer=search_authorizer, http_timeout=5 * 60) # funcX endpoint to use self.fx_endpoint = '86a47061-f3d9-44f0-90dc-56ddc642c000' self.fx_cache = {} super(DLHubClient, self).__init__("DLHub", environment='dlhub', authorizer=dlh_authorizer, http_timeout=http_timeout, base_url=DLHUB_SERVICE_ADDRESS, **kwargs)