def list_acls(endpoint, count_only=False, re_match=None):
"""List or count the ACLs on an endpoint, optionally matching some regex.
Arguments:
endpoint (str): The endpoint ID to check for ACLs.
count_only (bool): When True, will only return the total count of ACLs found.
When False, will print each ACL found.
Default False.
re_match (str): A regular expression to test the ACL's path. Any ACL rule
that has a path matching this regex will be counted.
Default None, to match all ACLs.
Returns:
int: The total ACLs matched.
Note:
You must be an Access Manager or higher on the target endpoint to list ACLs.
"""
tc = mdf_toolbox.login(services="transfer")["transfer"]
count = 0
for rule in tc.endpoint_acl_list(endpoint):
if re_match is None or re.match(re_match, rule["path"]):
count += 1
if not count_only:
print(rule)
return count
def ingest_to_search(name, path, idx="dlhub-test"):
"""Ingests the model metadata into Globus Search
Args:
name (str): The model name. (generally the same as container directory name
but not always) e.g. "cifar10"
path (str): Path to the model metadata. e.g. "metadata/cifar10.json"
idx (str): The Globus Index to upload search metadata to. Defaults=dlhub-test
"""
if path == None:
return
dl = client.DLHub()
uuid = dl.get_id_by_name(name)
iden = "https://dlhub.org/api/v1/servables/{}".format(uuid)
index = mdf_toolbox.translate_index(idx)
with open(path, 'r') as f:
ingestable = json.load(f)
ingestable = mdf_toolbox.format_gmeta(ingestable,
acl="public",
identifier=iden)
ingestable = mdf_toolbox.format_gmeta(
[ingestable]) # Make it a GIngest list of GMetaEntry
ingest_client = mdf_toolbox.login(
services=["search_ingest"])["search_ingest"]
ingest_client.ingest(index, ingestable)
print("Ingestion of {} to DLHub servables complete".format(name))
def __init__(self,
authorizer=None,
http_timeout=None,
force_login=False,
**kwargs):
"""Initialize the client
Args:
authorizer (:class:`GlobusAuthorizer
<globus_sdk.authorizers.base.GlobusAuthorizer>`):
An authorizer instance used to communicate with paropt.
If ``None``, will be created.
http_timeout (int): Timeout for any call to service in seconds. (default is no timeout)
force_login (bool): Whether to force a login to get new credentials.
A login will always occur if ``authorizer``
are not provided.
Keyword arguments are the same as for BaseClient.
"""
if force_login or not authorizer or not search_client:
dlhub_scope = "https://auth.globus.org/scopes/81fc4156-a623-47f2-93ad-7184118226ba/auth"
auth_res = login(services=[dlhub_scope],
app_name="paropt",
client_id=CLIENT_ID,
clear_old_tokens=force_login,
token_dir=_token_dir)
dlh_authorizer = auth_res['dlhub_org']
super(ParoptClient, self).__init__("paropt",
authorizer=dlh_authorizer,
http_timeout=http_timeout,
base_url=PAROPT_SERVICE_ADDRESS,
**kwargs)
def delete_acls(endpoint, re_match, dry_run=True):
"""Delete ACLs from an endpoint matching some regex on the rule path.
Arguments:
endpoint (str): The endpoint ID to delete ACLs from.
re_match (str): A regular expression to test the ACL's path. Any ACL rule
that has a path matching this regex will be deleted.
dry_run (bool): When True, will not actually delete anything, and instead
will print the ACL rules that would have been deleted.
When False, will permanently delete the ACL rules.
Default True.
Returns:
int: The number of ACL rules matched for deletion.
Caution:
This function deletes ACL rules, which removes permissions.
You must be an Access Manager or higher on the target endpoint,
and you must exercise caution.
"""
tc = mdf_toolbox.login(services="transfer")["transfer"]
count = 0
for rule in tc.endpoint_acl_list(endpoint):
if re.match(re_match, rule["path"]):
count += 1
if dry_run:
print(rule)
else:
tc.delete_endpoint_acl_rule(endpoint, rule["id"])
return count
def test_login():
# Login works
# Impersonate Forge
res1 = mdf_toolbox.login(services="search", app_name="MDF_Forge",
client_id="b2b437c4-17c1-4e4b-8f15-e9783e1312d7")
assert type(res1) is dict
assert isinstance(res1.get("search"), globus_sdk.SearchClient)
# Test other services
# Use default "unknown app"
# TODO: "groups" cannot be tested without whitelisting app
res2 = mdf_toolbox.login(services=["search_ingest", "transfer", "data_mdf", "mdf_connect",
"petrel"])
assert isinstance(res2.get("search_ingest"), globus_sdk.SearchClient)
assert isinstance(res2.get("transfer"), globus_sdk.TransferClient)
assert isinstance(res2.get("data_mdf"), globus_sdk.RefreshTokenAuthorizer)
assert isinstance(res2.get("mdf_connect"), globus_sdk.RefreshTokenAuthorizer)
assert isinstance(res2.get("petrel"), globus_sdk.RefreshTokenAuthorizer)
def __init__(self, dlh_authorizer=None, search_client=None, http_timeout=None,
force_login=False, fx_authorizer=None, **kwargs):
"""Initialize the client
Args:
dlh_authorizer (:class:`GlobusAuthorizer
<globus_sdk.authorizers.base.GlobusAuthorizer>`):
An authorizer instance used to communicate with DLHub.
If ``None``, will be created.
search_client (:class:`SearchClient <globus_sdk.SearchClient>`):
An authenticated SearchClient to communicate with Globus Search.
If ``None``, will be created.
http_timeout (int): Timeout for any call to service in seconds. (default is no timeout)
force_login (bool): Whether to force a login to get new credentials.
A login will always occur if ``dlh_authorizer`` or ``search_client``
are not provided.
no_local_server (bool): Disable spinning up a local server to automatically
copy-paste the auth code. THIS IS REQUIRED if you are on a remote server.
When used locally with no_local_server=False, the domain is localhost with
a randomly chosen open port number.
**Default**: ``True``.
fx_authorizer (:class:`GlobusAuthorizer
<globus_sdk.authorizers.base.GlobusAuthorizer>`):
An authorizer instance used to communicate with funcX.
If ``None``, will be created.
no_browser (bool): Do not automatically open the browser for the Globus Auth URL.
Display the URL instead and let the user navigate to that location manually.
**Default**: ``True``.
Keyword arguments are the same as for BaseClient.
"""
if force_login or not dlh_authorizer or not search_client or not fx_authorizer:
fx_scope = "https://auth.globus.org/scopes/facd7ccc-c5f4-42aa-916b-a0e270e2c2a9/all"
auth_res = login(services=["search", "dlhub",
fx_scope],
app_name="DLHub_Client",
client_id=CLIENT_ID, clear_old_tokens=force_login,
token_dir=_token_dir, no_local_server=kwargs.get("no_local_server", True),
no_browser=kwargs.get("no_browser", True))
dlh_authorizer = auth_res["dlhub"]
fx_authorizer = auth_res[fx_scope]
self._search_client = auth_res["search"]
self._fx_client = FuncXClient(force_login=True,fx_authorizer=fx_authorizer,
funcx_service_address='https://funcx.org/api/v1')
# funcX endpoint to use
self.fx_endpoint = '86a47061-f3d9-44f0-90dc-56ddc642c000'
# self.fx_endpoint = '2c92a06a-015d-4bfa-924c-b3d0c36bdad7'
self.fx_serializer = FuncXSerializer()
self.fx_cache = {}
super(DLHubClient, self).__init__("DLHub", environment='dlhub', authorizer=dlh_authorizer,
http_timeout=http_timeout, base_url=DLHUB_SERVICE_ADDRESS,
**kwargs)
def __init__(self, index, **kwargs):
"""Create a SearchHelper object.
Arguments:
index (str): The Globus Search index to search on.
Keyword Arguments:
search_client (globus_sdk.SearchClient): The Globus Search client to use for
searching. If not provided, one will be created and the user may be asked
to log in. **Default**: ``None``.
anonymous (bool): If ``True``, will not authenticate with Globus Auth.
If ``False``, will require authentication (either a SearchClient or
a user-interactive login).
**Default:** ``False``.
Caution:
Authentication is required to view non-public data in Search.
An anonymous SearchHelper will only return public results.
app_name (str): The application name to use. Should be changed for
subclassed clients, and left alone otherwise.
Only used if performing login flow.
**Default**: ``"SearchHelper_Client"``.
client_id (str): The ID of a native client to use when authenticating.
Only used if performing login flow.
**Default**: The default SearchHelper client ID.
q (str): A query string to initialize the SearchHelper with.
Intended for internal use.
advanced (bool): The initial advanced state for thie SearchHelper.
Intended for internal use.
"""
if kwargs.get("search_client"):
self.__search_client = kwargs["search_client"]
elif kwargs.get("anonymous"):
self.__search_client = mdf_toolbox.anonymous_login(["search"
])["search"]
else:
self.__search_client = mdf_toolbox.login(
app_name=kwargs.get("app_name", self.__app_name),
client_id=kwargs.get("client_id", self.__client_id),
services=["search"])["search"]
# Get the UUID for the index if the name was provided
self.index = mdf_toolbox.translate_index(index)
# Query init
self.__query = deepcopy(BLANK_QUERY)
if kwargs.get("q"):
self.__query["q"] = kwargs["q"]
if kwargs.get("advanced"):
self.__query["advanced"] = kwargs["advanced"]
def __init__(self,
no_browser=False,
no_local_server=False,
search_index="mdf-test",
**data):
super().__init__(**data)
auths = mdf_toolbox.login(
services=[
"data_mdf",
"search",
"petrel",
"transfer",
"dlhub",
"https://auth.globus.org/scopes/facd7ccc-c5f4-42aa-916b-a0e270e2c2a9/all",
],
app_name="Foundry",
make_clients=True,
no_browser=no_browser,
no_local_server=no_local_server,
)
self.forge_client = Forge(
index=search_index,
services=None,
search_client=auths["search"],
transfer_client=auths["transfer"],
data_mdf_authorizer=auths["data_mdf"],
petrel_authorizer=auths["petrel"],
)
self.dlhub_client = DLHubClient(
dlh_authorizer=auths["dlhub"],
search_client=auths["search"],
fx_authorizer=auths[
"https://auth.globus.org/scopes/facd7ccc-c5f4-42aa-916b-a0e270e2c2a9/all"],
force_login=False,
)
self.xtract_tokens = {
'auth_token':
auths['petrel'].access_token,
'transfer_token':
auths['transfer'].authorizer.access_token,
'funx_token':
auths[
'https://auth.globus.org/scopes/facd7ccc-c5f4-42aa-916b-a0e270e2c2a9/all']
.access_token
}
def __init__(self,
http_timeout=None,
funcx_home=os.path.join('~', '.funcx'),
force_login=False,
fx_authorizer=None,
**kwargs):
""" Initialize the client
Parameters
----------
http_timeout: int
Timeout for any call to service in seconds.
Default is no timeout
force_login: bool
Whether to force a login to get new credentials.
fx_authorizer:class:`GlobusAuthorizer <globus_sdk.authorizers.base.GlobusAuthorizer>`:
A custom authorizer instance to communicate with funcX.
Default: ``None``, will be created.
Keyword arguments are the same as for BaseClient.
"""
self.ep_registration_path = 'register_endpoint_2'
self.funcx_home = os.path.expanduser(funcx_home)
if force_login or not fx_authorizer:
fx_scope = "https://auth.globus.org/scopes/facd7ccc-c5f4-42aa-916b-a0e270e2c2a9/all"
auth_res = login(services=[fx_scope],
app_name="funcX_Client",
client_id=self.CLIENT_ID,
clear_old_tokens=force_login,
token_dir=self.TOKEN_DIR)
dlh_authorizer = auth_res['funcx_service']
super(FuncXClient, self).__init__("funcX",
environment='funcx',
authorizer=dlh_authorizer,
http_timeout=http_timeout,
base_url=self.FUNCX_SERVICE_ADDRESS,
**kwargs)
self.fx_serializer = FuncXSerializer()
def ingest_to_search(name, path, idx):
if path == None:
return
dl = client.DLHub()
uuid = dl.get_id_by_name(name)
iden = "https://dlhub.org/api/v1/servables/{}".format(uuid)
index = mdf_toolbox.translate_index(idx)
with open(path, 'r') as f:
ingestable = json.load(f)
ingestable = mdf_toolbox.format_gmeta(ingestable,
acl="public",
identifier=iden)
ingestable = mdf_toolbox.format_gmeta(
[ingestable]) # Make it a GIngest list of GMetaEntry
ingest_client = mdf_toolbox.login(
services=["search_ingest"])["search_ingest"]
ingest_client.ingest(index, ingestable)
print("Ingestion of {} complete".format(name))
def __init__(self,
dlh_authorizer=None,
search_client=None,
http_timeout=None,
force_login=False,
**kwargs):
"""Initialize the client
Args:
dlh_authorizer (:class:`GlobusAuthorizer
<globus_sdk.authorizers.base.GlobusAuthorizer>`):
An authorizer instance used to communicate with DLHub.
If ``None``, will be created.
search_client (:class:`SearchClient <globus_sdk.SearchClient>`):
An authenticated SearchClient to communicate with Globus Search.
If ``None``, will be created.
http_timeout (int): Timeout for any call to service in seconds. (default is no timeout)
force_login (bool): Whether to force a login to get new credentials.
A login will always occur if ``dlh_authorizer`` or ``search_client``
are not provided.
Keyword arguments are the same as for BaseClient.
"""
if force_login or not dlh_authorizer or not search_client:
auth_res = login(services=["search", "dlhub"],
app_name="DLHub_Client",
client_id=CLIENT_ID,
clear_old_tokens=force_login,
token_dir=_token_dir)
dlh_authorizer = auth_res["dlhub"]
self._search_client = auth_res["search"]
super(DLHubClient, self).__init__("DLHub",
environment='dlhub',
authorizer=dlh_authorizer,
http_timeout=http_timeout,
base_url=DLHUB_SERVICE_ADDRESS,
**kwargs)
def search_ingest(task):
"""
Ingest the servable data into a Globus Search index.
Args:
task (dict): the task description.
"""
logging.debug("Ingesting servable into Search.")
# idx = "dlhub"
idx = '847c9105-18a0-4ffb-8a71-03dd76dfcc9d'
iden = "https://dlhub.org/servables/{}".format(task['dlhub']['id'])
index = mdf_toolbox.translate_index(idx)
ingestable = task
d = [convert_dict(ingestable, str)]
glist = []
visible_to = task['dlhub'].get('visible_to', ['public'])
# Add public so it isn't an empty list
if len(visible_to) == 0:
visible_to = ['public']
for document in d:
gmeta_entry = mdf_toolbox.format_gmeta(document, visible_to, iden)
glist.append(gmeta_entry)
gingest = mdf_toolbox.format_gmeta(glist)
ingest_client = mdf_toolbox.login(services=["search_ingest"],
no_local_server=True,
no_browser=True)["search_ingest"]
logging.info("ingesting to search")
logging.info(gingest)
ingest_client.ingest(idx, gingest)
logging.info("Ingestion of {} to DLHub servables complete".format(iden))
import pytest
import re
import mdf_toolbox
SEARCH_CLIENT = mdf_toolbox.login(services=["search"], app_name="SearchHelper",
client_id="878721f5-6b92-411e-beac-830672c0f69a")["search"]
INDEX = "mdf"
SCROLL_FIELD = "mdf.scroll_id"
class DummyClient(mdf_toolbox.AggregateHelper, mdf_toolbox.SearchHelper):
def __init__(self, *args, **kwargs):
super().__init__(*args, scroll_field=SCROLL_FIELD, **kwargs)
# Helper
# Return codes:
# -1: No match, the value was never found
# 0: Exclusive match, no values other than argument found
# 1: Inclusive match, some values other than argument found
# 2: Partial match, value is found in some but not all results
def check_field(res, field, regex):
dict_path = ""
for key in field.split("."):
if key == "[]":
dict_path += "[0]"
else:
dict_path += ".get('{}', {})".format(key, "{}")
# If no results, set matches to false
def __init__(self, dlh_authorizer: Optional[GlobusAuthorizer] = None,
search_authorizer: Optional[GlobusAuthorizer] = None,
fx_authorizer: Optional[GlobusAuthorizer] = None,
openid_authorizer: Optional[GlobusAuthorizer] = None,
http_timeout: Optional[int] = None,
force_login: bool = False, **kwargs):
"""Initialize the client
Args:
http_timeout (int): Timeout for any call to service in seconds. (default is no timeout)
force_login (bool): Whether to force a login to get new credentials.
A login will always occur if ``dlh_authorizer`` or ``search_client``
are not provided.
no_local_server (bool): Disable spinning up a local server to automatically
copy-paste the auth code. THIS IS REQUIRED if you are on a remote server.
When used locally with no_local_server=False, the domain is localhost with
a randomly chosen open port number.
**Default**: ``True``.
no_browser (bool): Do not automatically open the browser for the Globus Auth URL.
Display the URL instead and let the user navigate to that location manually.
**Default**: ``True``.
dlh_authorizer (:class:`GlobusAuthorizer <globus_sdk.authorizers.base.GlobusAuthorizer>`):
An authorizer instance used to communicate with DLHub.
If ``None``, will be created from your account's credentials.
search_authorizer (:class:`GlobusAuthorizer <globus_sdk.authorizers.base.GlobusAuthorizer>`):
An authenticated SearchClient to communicate with Globus Search.
If ``None``, will be created from your account's credentials.
fx_authorizer (:class:`GlobusAuthorizer
<globus_sdk.authorizers.base.GlobusAuthorizer>`):
An authorizer instance used to communicate with funcX.
If ``None``, will be created from your account's credentials.
openid_authorizer (:class:`GlobusAuthorizer
<globus_sdk.authorizers.base.GlobusAuthorizer>`):
An authorizer instance used to communicate with OpenID.
If ``None``, will be created from your account's credentials.
Keyword arguments are the same as for :class:`BaseClient <globus_sdk.base.BaseClient>`.
"""
authorizers = [dlh_authorizer, search_authorizer, openid_authorizer, fx_authorizer]
# Get authorizers through Globus login if any are not provided
if not all(a is not None for a in authorizers):
# If some but not all were provided, warn the user they could be making a mistake
if any(a is not None for a in authorizers):
logger.warning('You have defined some of the authorizers but not all. DLHub is falling back to login. '
'You must provide authorizers for DLHub, Search, OpenID, FuncX.')
fx_scope = "https://auth.globus.org/scopes/facd7ccc-c5f4-42aa-916b-a0e270e2c2a9/all"
auth_res = login(services=["search", "dlhub",
fx_scope, "openid"],
app_name="DLHub_Client",
make_clients=False,
client_id=CLIENT_ID,
clear_old_tokens=force_login,
token_dir=_token_dir,
no_local_server=kwargs.get("no_local_server", True),
no_browser=kwargs.get("no_browser", True))
# Unpack the authorizers
dlh_authorizer = auth_res["dlhub"]
fx_authorizer = auth_res[fx_scope]
openid_authorizer = auth_res['openid']
search_authorizer = auth_res['search']
# Define the subclients needed by the service
self._fx_client = FuncXClient(fx_authorizer=fx_authorizer,
search_authorizer=search_authorizer,
openid_authorizer=openid_authorizer,
no_local_server=kwargs.get("no_local_server", True),
no_browser=kwargs.get("no_browser", True))
self._search_client = globus_sdk.SearchClient(authorizer=search_authorizer,
http_timeout=5 * 60)
# funcX endpoint to use
self.fx_endpoint = '86a47061-f3d9-44f0-90dc-56ddc642c000'
self.fx_cache = {}
super(DLHubClient, self).__init__("DLHub", environment='dlhub',
authorizer=dlh_authorizer,
http_timeout=http_timeout,
base_url=DLHUB_SERVICE_ADDRESS,
**kwargs)
