def check_globus(src_uuid, dst_uuid, src_path, dst_path):
"""
Check that the globus endpoints are not only active but will return information
about the paths we're interested in.
Im assuming that the endpoints have already been activated
"""
try:
endpoints = [{
'type': 'source',
'id': src_uuid,
'path': src_path
}, {
'type': 'destination',
'id': dst_uuid,
'path': dst_path
}]
except Exception as e:
print_debug(e)
client = get_client()
try:
for endpoint in endpoints:
_ = get_ls(client, endpoint['path'], endpoint['id'], False, 0,
False)
hostname = client.endpoint_server_list(
endpoint)['DATA']['hostname']
print "Access confirmed for {}".format(hostname)
except Exception as e:
print_debug(e)
return False, endpoint
else:
return True, None
def server_add(
endpoint_id,
subject,
port,
scheme,
hostname,
incoming_data_ports,
outgoing_data_ports,
):
"""
Executor for `globus endpoint server add`
"""
client = get_client()
server_doc = assemble_generic_doc("server",
subject=subject,
port=port,
scheme=scheme,
hostname=hostname)
# n.b. must be done after assemble_generic_doc(), as that function filters
# out `None`s, which we need to be able to set for `'unspecified'`
if incoming_data_ports:
server_doc.update(
incoming_data_port_start=incoming_data_ports[0],
incoming_data_port_end=incoming_data_ports[1],
)
if outgoing_data_ports:
server_doc.update(
outgoing_data_port_start=outgoing_data_ports[0],
outgoing_data_port_end=outgoing_data_ports[1],
)
res = client.add_endpoint_server(endpoint_id, server_doc)
formatted_print(res, text_format=FORMAT_TEXT_RAW, response_key="message")
def role_list(endpoint_id):
"""
List the assigned roles on an endpoint.
You must have sufficient privileges to see the roles on the endpoint.
"""
client = get_client()
roles = client.endpoint_role_list(endpoint_id)
resolved_ids = IdentityMap(
get_auth_client(),
(x["principal"] for x in roles if x["principal_type"] == "identity"),
)
def principal_str(role):
principal = role["principal"]
if role["principal_type"] == "identity":
try:
return resolved_ids[principal]["username"]
except KeyError:
return principal
if role["principal_type"] == "group":
return (u"https://app.globus.org/groups/{}").format(principal)
return principal
formatted_print(
roles,
fields=[
("Principal Type", "principal_type"),
("Role ID", "id"),
("Principal", principal_str),
("Role", "role"),
],
)
def server_update(
endpoint_id,
server_id,
subject,
port,
scheme,
hostname,
incoming_data_ports,
outgoing_data_ports,
):
"""
Executor for `globus endpoint server update`
"""
client = get_client()
server_doc = assemble_generic_doc(
"server", subject=subject, port=port, scheme=scheme, hostname=hostname
)
# n.b. must be done after assemble_generic_doc(), as that function filters
# out `None`s, which we need to be able to set for `'unspecified'`
if incoming_data_ports:
server_doc.update(
incoming_data_port_start=incoming_data_ports[0],
incoming_data_port_end=incoming_data_ports[1],
)
if outgoing_data_ports:
server_doc.update(
outgoing_data_port_start=outgoing_data_ports[0],
outgoing_data_port_end=outgoing_data_ports[1],
)
res = client.update_endpoint_server(endpoint_id, server_id, server_doc)
formatted_print(res, text_format=FORMAT_TEXT_RAW, response_key="message")
def endpoint_search(filter_fulltext, filter_owner_id, filter_scope):
"""
Executor for `globus endpoint search`
"""
if filter_scope == "all" and not filter_fulltext:
raise click.UsageError(
"When searching all endpoints (--filter-scope=all, the default), "
"a full-text search filter is required. Other scopes (e.g. "
"--filter-scope=recently-used) may be used without specifying "
"an additional filter."
)
client = get_client()
owner_id = filter_owner_id
if owner_id:
owner_id = maybe_lookup_identity_id(owner_id)
search_iterator = client.endpoint_search(
filter_fulltext=filter_fulltext,
filter_scope=filter_scope,
filter_owner_id=owner_id,
)
formatted_print(
search_iterator,
fields=ENDPOINT_LIST_FIELDS,
json_converter=iterable_response_to_dict,
)
def bookmark_show(bookmark_id_or_name):
"""
Given a single bookmark ID or bookmark name, show the bookmark details. By default,
when the format is TEXT, this will display the endpoint ID and path in
'ENDPOINT_ID:PATH' notation.
The default output is suitable for use in a subshell in another command.
If *-v, --verbose* is given, several fields will be displayed.
"""
client = get_client()
res = resolve_id_or_name(client, bookmark_id_or_name)
formatted_print(
res,
text_format=FORMAT_TEXT_RECORD,
fields=(
("ID", "id"),
("Name", "name"),
("Endpoint ID", "endpoint_id"),
("Path", "path"),
),
simple_text=(
# standard output is endpoint:path format
"{}:{}".format(res["endpoint_id"], res["path"])
# verbose output includes all fields
if not is_verbose() else None),
)
def test_filemanager_update_remote_yes_sta(self):
sta = True
types = ['atm', 'ice', 'ocn', 'streams.cice', 'streams.ocean']
database = 'test.db'
simstart = 51
source_path = '/global/cscratch1/sd/golaz/ACME_simulations/20170915.beta2.A_WCYCL1850S.ne30_oECv3_ICG.edison'
simend = 60
experiment = '20170915.beta2.A_WCYCL1850S.ne30_oECv3_ICG.edison'
filemanager = FileManager(mutex=self.mutex,
sta=sta,
types=types,
database=database,
remote_endpoint=self.remote_endpoint,
remote_path=source_path,
local_endpoint=self.local_endpoint,
local_path=self.local_path)
filemanager.populate_file_list(simstart=simstart,
simend=simend,
experiment=experiment)
client = get_client()
filemanager.update_remote_status(client)
self.mutex.acquire()
for datafile in DataFile.select():
if datafile.remote_status != 0:
print datafile.name, datafile.remote_path
self.assertEqual(datafile.remote_status, 0)
if self.mutex.locked():
self.mutex.release()
self.assertTrue(filemanager.all_data_remote())
def bookmark_list():
"""
Executor for `globus bookmark list`
"""
client = get_client()
bookmark_iterator = client.bookmark_list()
def get_ep_name(item):
ep_id = item["endpoint_id"]
try:
ep_doc = client.get_endpoint(ep_id)
return display_name_or_cname(ep_doc)
except TransferAPIError as err:
if err.code == "EndpointDeleted":
return "[DELETED ENDPOINT]"
else:
raise err
formatted_print(
bookmark_iterator,
fields=[
("Name", "name"),
("Bookmark ID", "id"),
("Endpoint ID", "endpoint_id"),
("Endpoint Name", get_ep_name),
("Path", "path"),
],
response_key="DATA",
json_converter=iterable_response_to_dict,
)
def server_show(endpoint_id, server_id):
"""
Executor for `globus endpoint server show`
"""
client = get_client()
server_doc = client.get_endpoint_server(endpoint_id, server_id)
if not server_doc['uri']: # GCP endpoint server
fields = (('ID', 'id'), ('Is Connected', 'is_connected'),
('Is Paused (macOS only)', 'is_paused'))
else:
def advertised_port_summary(server):
def get_range_summary(start, end):
return ('unspecified' if not start and not end
else 'unrestricted' if start == 1024 and end == 65535
else '{}-{}'.format(start, end))
return "incoming {}, outgoing {}".format(
get_range_summary(server['incoming_data_port_start'],
server['incoming_data_port_end']),
get_range_summary(server['outgoing_data_port_start'],
server['outgoing_data_port_end']),
)
fields = (('ID', 'id'), ('URI', 'uri'), ('Subject', 'subject'),
('Data Ports', advertised_port_summary))
formatted_print(server_doc, text_format=FORMAT_TEXT_RECORD,
fields=fields)
def role_create(role, principal, endpoint_id):
"""
Executor for `globus endpoint role show`
"""
principal_type, principal_val = principal
client = get_client()
if principal_type == 'identity':
principal_val = maybe_lookup_identity_id(principal_val)
if not principal_val:
raise click.UsageError(
'Identity does not exist. '
'Use --provision-identity to auto-provision an identity.')
elif principal_type == 'provision-identity':
principal_val = maybe_lookup_identity_id(principal_val, provision=True)
principal_type = 'identity'
role_doc = assemble_generic_doc('role',
principal_type=principal_type,
principal=principal_val,
role=role)
res = client.add_endpoint_role(endpoint_id, role_doc)
formatted_print(res, simple_text='ID: {}'.format(res['id']))
def check_globus(**kwargs):
"""
Check that the globus endpoints are not only active but will return information
about the paths we're interested in.
Im assuming that the endpoints have already been activated
"""
try:
endpoints = [{
'type': 'source',
'id': kwargs['source_endpoint'],
'path': kwargs['source_path']
}, {
'type': 'destination',
'id': kwargs['destination_endpoint'],
'path': kwargs['destination_path']
}]
except Exception as e:
print_debug(e)
client = get_client()
try:
for endpoint in endpoints:
res = get_ls(
client,
endpoint['path'],
endpoint['id'],
False, 0, False)
except Exception as e:
print_debug(e)
return False, endpoint
else:
print "Access granted"
return True, None
def endpoint_delete(endpoint_id):
"""
Executor for `globus endpoint delete`
"""
client = get_client()
res = client.delete_endpoint(endpoint_id)
formatted_print(res, text_format=FORMAT_TEXT_RAW, response_key="message")
def role_delete(role_id, endpoint_id):
"""
Executor for `globus endpoint role delete`
"""
client = get_client()
res = client.delete_endpoint_role(endpoint_id, role_id)
formatted_print(res, text_format=FORMAT_TEXT_RAW, response_key='message')
def role_create(role, principal, endpoint_id):
"""
Create a role on an endpoint.
You must have sufficient privileges to modify the roles on the endpoint.
Either *--group* or *--identity* is required. You may not pass both.
Which one of these options you use will determine the 'Principal Type' on the
role, and the value given will be the 'Principal' of the resulting role.
The term "Principal" is used in the sense of "a security principal", an entity
which has some privileges associated with it.
"""
principal_type, principal_val = principal
client = get_client()
if principal_type == "identity":
principal_val = maybe_lookup_identity_id(principal_val)
if not principal_val:
raise click.UsageError(
"Identity does not exist. "
"Use --provision-identity to auto-provision an identity.")
elif principal_type == "provision-identity":
principal_val = maybe_lookup_identity_id(principal_val, provision=True)
principal_type = "identity"
role_doc = assemble_generic_doc("role",
principal_type=principal_type,
principal=principal_val,
role=role)
res = client.add_endpoint_role(endpoint_id, role_doc)
formatted_print(res, simple_text="ID: {}".format(res["id"]))
def list_command(endpoint_id):
"""List all rules in an endpoint's access control list."""
client = get_client()
rules = client.endpoint_acl_list(endpoint_id)
resolved_ids = IdentityMap(
get_auth_client(),
(x["principal"] for x in rules if x["principal_type"] == "identity"),
)
def principal_str(rule):
principal = rule["principal"]
if rule["principal_type"] == "identity":
try:
return resolved_ids[principal]["username"]
except KeyError:
return principal
if rule["principal_type"] == "group":
return (u"https://app.globus.org/groups/{}").format(principal)
return rule["principal_type"]
formatted_print(
rules,
fields=[
("Rule ID", "id"),
("Permissions", "permissions"),
("Shared With", principal_str),
("Path", "path"),
],
)
def clean_sharing():
"""
Cleans out any files in ~/.globus/sharing/ on go#ep1 older than an hour
TODO: remove this once deleting shared directories does full cleanup
"""
tc = get_client()
path = "~/.globus/sharing/"
hour_ago = datetime.utcnow() - timedelta(hours=1)
filter_string = "last_modified:," + hour_ago.strftime("%Y-%m-%d %H:%M:%S")
try:
old_files = tc.operation_ls(
GO_EP1_ID, path=path, filter=filter_string, num_results=None
)
except globus_sdk.TransferAPIError:
return
kwargs = {"notify_on_succeeded": False, "notify_on_fail": False}
ddata = globus_sdk.DeleteData(tc, GO_EP1_ID, **kwargs)
for item in old_files:
ddata.add_item(path + item["name"])
if len(ddata["DATA"]):
tc.submit_delete(ddata)
def setup_globus(endpoints, event_list):
"""
Check globus login status and login as nessisary, then
iterate over a list of endpoints and activate them all
Parameters:
endpoints: list of strings containing globus endpoint UUIDs
event_list: the event list to push user notifications into
return:
True if successful, False otherwise
"""
# First go through the globus login process
if not check_logged_in():
message = 'Globus login required. Please run {cmd}\n\n'.format(
cmd='"globus login"')
print_line(message, event_list)
print '================================================'
sys.exit(1)
if isinstance(endpoints, str):
endpoints = [endpoints]
activated = False
client = get_client()
while not activated:
activated = True
message = ''
for endpoint in endpoints:
msg = 'activating endpoint {}'.format(endpoint)
logging.info(msg)
try:
r = client.endpoint_autoactivate(endpoint, if_expires_in=3600)
logging.info(r['code'])
except Exception as e:
print_debug(e)
if e.code == 'ClientError.NotFound':
return False
else:
continue
if r["code"] == "AutoActivationFailed":
activated = False
logging.info('endpoint autoactivation failed')
server_document = client.endpoint_server_list(endpoint)
for server in server_document['DATA']:
hostname = server["hostname"]
break
message += """
Data transfer server {server} requires manual activation.
Please open the following URL in a browser to activate the endpoint:
https://www.globus.org/app/endpoints/{endpoint}/activate
""".format(endpoint=endpoint, server=server['hostname'])
if not activated:
print message
raw_input('Press ENTER once endpoints have been activated\n')
return True
def endpoint_update(**kwargs):
"""
Executor for `globus endpoint update`
"""
# validate params. Requires a get call to check the endpoint type
client = get_client()
endpoint_id = kwargs.pop("endpoint_id")
get_res = client.get_endpoint(endpoint_id)
if get_res["host_endpoint_id"]:
endpoint_type = "shared"
elif get_res["is_globus_connect"]:
endpoint_type = "personal"
elif get_res["s3_url"]:
endpoint_type = "s3"
else:
endpoint_type = "server"
validate_endpoint_create_and_update_params(
endpoint_type, get_res["subscription_id"], kwargs
)
# make the update
ep_doc = assemble_generic_doc("endpoint", **kwargs)
res = client.update_endpoint(endpoint_id, ep_doc)
formatted_print(res, text_format=FORMAT_TEXT_RAW, response_key="message")
def task_pause_info(task_id):
"""
Executor for `globus task pause-info`
"""
client = get_client()
res = client.task_pause_info(task_id)
def _custom_text_format(res):
explicit_pauses = [
field for field in EXPLICIT_PAUSE_MSG_FIELDS
# n.b. some keys are absent for completed tasks
if res.get(field[1])
]
effective_pause_rules = res['pause_rules']
if not explicit_pauses and not effective_pause_rules:
safeprint('Task {} is not paused.'.format(task_id))
click.get_current_context().exit(0)
if explicit_pauses:
formatted_print(
res, fields=explicit_pauses, text_format=FORMAT_TEXT_RECORD,
text_preamble='This task has been explicitly paused.\n',
text_epilog='\n' if effective_pause_rules else None)
if effective_pause_rules:
formatted_print(
effective_pause_rules, fields=PAUSE_RULE_DISPLAY_FIELDS,
text_preamble=(
'The following pause rules are effective on this task:\n'))
formatted_print(res, text_format=_custom_text_format)
def role_create(role, principal, endpoint_id):
"""
Executor for `globus endpoint role show`
"""
principal_type, principal_val = principal
client = get_client()
if principal_type == "identity":
principal_val = maybe_lookup_identity_id(principal_val)
if not principal_val:
raise click.UsageError(
"Identity does not exist. "
"Use --provision-identity to auto-provision an identity."
)
elif principal_type == "provision-identity":
principal_val = maybe_lookup_identity_id(principal_val, provision=True)
principal_type = "identity"
role_doc = assemble_generic_doc(
"role", principal_type=principal_type, principal=principal_val, role=role
)
res = client.add_endpoint_role(endpoint_id, role_doc)
formatted_print(res, simple_text="ID: {}".format(res["id"]))
def role_delete(role_id, endpoint_id):
"""
Executor for `globus endpoint role delete`
"""
client = get_client()
res = client.delete_endpoint_role(endpoint_id, role_id)
formatted_print(res, text_format=FORMAT_TEXT_RAW, response_key="message")
def server_show(endpoint_id, server_id):
"""
Executor for `globus endpoint server show`
"""
client = get_client()
server_doc = client.get_endpoint_server(endpoint_id, server_id)
if not server_doc['uri']: # GCP endpoint server
fields = (('ID', 'id'),)
text_epilog = dedent("""
This server is for a Globus Connect Personal installation.
For its connection status, try:
globus endpoint show {}
""".format(endpoint_id))
else:
def advertised_port_summary(server):
def get_range_summary(start, end):
return ('unspecified' if not start and not end
else 'unrestricted' if start == 1024 and end == 65535
else '{}-{}'.format(start, end))
return "incoming {}, outgoing {}".format(
get_range_summary(server['incoming_data_port_start'],
server['incoming_data_port_end']),
get_range_summary(server['outgoing_data_port_start'],
server['outgoing_data_port_end']),
)
fields = (('ID', 'id'), ('URI', 'uri'), ('Subject', 'subject'),
('Data Ports', advertised_port_summary))
text_epilog = None
formatted_print(server_doc, text_format=FORMAT_TEXT_RECORD,
fields=fields, text_epilog=text_epilog)
def bookmark_list():
"""
Executor for `globus bookmark list`
"""
client = get_client()
bookmark_iterator = client.bookmark_list()
def get_ep_name(item):
ep_id = item['endpoint_id']
try:
ep_doc = client.get_endpoint(ep_id)
return display_name_or_cname(ep_doc)
except TransferAPIError as err:
if err.code == "EndpointDeleted":
return "[DELETED ENDPOINT]"
else:
raise err
formatted_print(bookmark_iterator,
fields=[('Name', 'name'), ('Bookmark ID', 'id'),
('Endpoint ID', 'endpoint_id'),
('Endpoint Name', get_ep_name), ('Path', 'path')],
response_key='DATA',
json_converter=iterable_response_to_dict)
def list_command(endpoint_id):
"""
Executor for `globus endpoint permission list`
"""
client = get_client()
rules = client.endpoint_acl_list(endpoint_id)
resolved_ids = LazyIdentityMap(
x["principal"] for x in rules if x["principal_type"] == "identity"
)
def principal_str(rule):
principal = rule["principal"]
if rule["principal_type"] == "identity":
username = resolved_ids.get(principal)
return username or principal
elif rule["principal_type"] == "group":
return (u"https://app.globus.org/groups/{}").format(principal)
else:
principal = rule["principal_type"]
return principal
formatted_print(
rules,
fields=[
("Rule ID", "id"),
("Permissions", "permissions"),
("Shared With", principal_str),
("Path", "path"),
],
)
def role_list(endpoint_id):
"""
Executor for `globus access endpoint-role-list`
"""
client = get_client()
roles = client.endpoint_role_list(endpoint_id)
resolved_ids = LazyIdentityMap(
x["principal"] for x in roles if x["principal_type"] == "identity"
)
def principal_str(role):
principal = role["principal"]
if role["principal_type"] == "identity":
username = resolved_ids.get(principal)
return username or principal
elif role["principal_type"] == "group":
return (u"https://app.globus.org/groups/{}").format(principal)
else:
return principal
formatted_print(
roles,
fields=[
("Principal Type", "principal_type"),
("Role ID", "id"),
("Principal", principal_str),
("Role", "role"),
],
)
def list_command(endpoint_id):
"""
Executor for `globus endpoint permission list`
"""
client = get_client()
rules = client.endpoint_acl_list(endpoint_id)
resolved_ids = LazyIdentityMap(x["principal"] for x in rules
if x["principal_type"] == "identity")
def principal_str(rule):
principal = rule["principal"]
if rule["principal_type"] == "identity":
username = resolved_ids.get(principal)
return username or principal
elif rule["principal_type"] == "group":
return (u"https://app.globus.org/groups/{}").format(principal)
else:
principal = rule["principal_type"]
return principal
formatted_print(
rules,
fields=[
("Rule ID", "id"),
("Permissions", "permissions"),
("Shared With", principal_str),
("Path", "path"),
],
)
def bookmark_create(endpoint_plus_path, bookmark_name):
"""
Create a new bookmark. Given an endpoint plus a path, and a name for the bookmark,
the service will generate the bookmark's ID.
Bookmarks are aliases for locations on endpoints, and their names are unique
per account. You may not have multiple bookmarks with the same name. You can
use bookmarks in other commands by using *globus bookmark show*.
The new bookmark name may be up to 128 characters long.
Bookmarks are only visible and usable for the user who created them. If the
target endpoint is private or deleted, the bookmark is unusable.
'PATH' is assumed to be URL-encoded. 'PATH' must be a directory and end with "/".
"""
endpoint_id, path = endpoint_plus_path
client = get_client()
submit_data = {
"endpoint_id": str(endpoint_id),
"path": path,
"name": bookmark_name
}
res = client.create_bookmark(submit_data)
formatted_print(res, simple_text="Bookmark ID: {}".format(res["id"]))
def clean_sharing():
"""
Cleans out any files in ~/.globus/sharing/ on go#ep1 older than an hour
TODO: remove this once deleting shared directories does full cleanup
"""
tc = get_client()
path = "~/.globus/sharing/"
hour_ago = datetime.utcnow() - timedelta(hours=1)
filter_string = "last_modified:," + hour_ago.strftime("%Y-%m-%d %H:%M:%S")
try:
old_files = tc.operation_ls(GO_EP1_ID,
path=path,
filter=filter_string,
num_results=None)
except globus_sdk.TransferAPIError:
return
kwargs = {"notify_on_succeeded": False, "notify_on_fail": False}
ddata = globus_sdk.DeleteData(tc, GO_EP1_ID, **kwargs)
for item in old_files:
ddata.add_item(path + item["name"])
if len(ddata["DATA"]):
tc.submit_delete(ddata)
def endpoint_deactivate(endpoint_id):
"""
Executor for `globus endpoint deactivate`
"""
client = get_client()
res = client.endpoint_deactivate(endpoint_id)
formatted_print(res, text_format=FORMAT_TEXT_RAW, response_key="message")
def list_command(endpoint_id):
"""
Executor for `globus endpoint permission list`
"""
client = get_client()
rules = client.endpoint_acl_list(endpoint_id)
resolved_ids = LazyIdentityMap(x['principal'] for x in rules
if x['principal_type'] == 'identity')
def principal_str(rule):
principal = rule['principal']
if rule['principal_type'] == 'identity':
username = resolved_ids.get(principal)
return username or principal
elif rule['principal_type'] == 'group':
return (u'https://www.globus.org/app/groups/{}').format(principal)
else:
principal = rule['principal_type']
return principal
formatted_print(rules,
fields=[('Rule ID', 'id'), ('Permissions', 'permissions'),
('Shared With', principal_str), ('Path', 'path')])
def test_filemanager_update_remote_no_sta(self):
sta = False
types = ['atm', 'ice', 'ocn', 'rest', 'streams.cice', 'streams.ocean']
database = 'test.db'
simstart = 51
simend = 60
experiment = '20171011.beta2_FCT2-icedeep_branch.A_WCYCL1850S.ne30_oECv3_ICG.edison'
filemanager = FileManager(mutex=self.mutex,
sta=sta,
types=types,
database=database,
remote_endpoint=self.remote_endpoint,
remote_path=self.remote_path,
local_endpoint=self.local_endpoint,
local_path=self.local_path)
filemanager.populate_file_list(simstart=simstart,
simend=simend,
experiment=experiment)
client = get_client()
filemanager.update_remote_status(client)
self.mutex.acquire()
for datafile in DataFile.select():
self.assertEqual(datafile.remote_status, 0)
if self.mutex.locked():
self.mutex.release()
self.assertTrue(filemanager.all_data_remote())
def endpoint_update(**kwargs):
"""
Executor for `globus endpoint update`
"""
# validate params. Requires a get call to check the endpoint type
client = get_client()
endpoint_id = kwargs.pop("endpoint_id")
get_res = client.get_endpoint(endpoint_id)
if get_res["host_endpoint_id"]:
endpoint_type = "shared"
elif get_res["is_globus_connect"]:
endpoint_type = "personal"
elif get_res["s3_url"]:
endpoint_type = "s3"
else:
endpoint_type = "server"
validate_endpoint_create_and_update_params(endpoint_type,
get_res["subscription_id"],
kwargs)
# make the update
ep_doc = assemble_generic_doc('endpoint', **kwargs)
res = client.update_endpoint(endpoint_id, ep_doc)
formatted_print(res, text_format=FORMAT_TEXT_RAW, response_key='message')
def endpoint_show(endpoint_id):
"""
Executor for `globus endpoint show`
"""
client = get_client()
res = client.get_endpoint(endpoint_id)
def _managed_endpoint(x):
""" Helper for converting subscription_id into managed_endpoint """
return bool(x["subscription_id"])
formatted_print(
res,
text_format=FORMAT_TEXT_RECORD,
fields=(("Display Name", "display_name"), ("ID", "id"),
("Owner", "owner_string"), ("Activated", "activated"),
("Shareable", "shareable"), ("Department", "department"),
("Keywords", "keywords"), ("Endpoint Info Link",
"info_link"), ("Contact E-mail",
"contact_email"),
("Organization", "organization"), ("Department", "department"),
("Other Contact Info",
"contact_info"), ("Visibility",
"public"), ("Default Directory",
"default_directory"),
("Force Encryption", "force_encryption"), ("Managed Endpoint",
_managed_endpoint),
("Subscription ID", "subscription_id"), ("Legacy Name",
"canonical_name"),
("Local User Info Available", "local_user_info_available")))
def rename_command(source, destination):
"""Rename a file or directory on an endpoint.
The old path must be an existing file or directory. The new path must not yet
exist.
The new path does not have to be in the same directory as the old path, but
most endpoints will require it to stay on the same filesystem.
The endpoint must be entered twice for the sake of path syntax consistency.
"""
source_ep, source_path = source
dest_ep, dest_path = destination
if source_ep != dest_ep:
raise click.UsageError(
("rename requires that the source and dest "
"endpoints are the same, {} != {}").format(source_ep, dest_ep))
endpoint_id = source_ep
client = get_client()
autoactivate(client, endpoint_id, if_expires_in=60)
res = client.operation_rename(endpoint_id,
oldpath=source_path,
newpath=dest_path)
formatted_print(res, text_format=FORMAT_TEXT_RAW, response_key="message")
def rm_command(ignore_missing, star_silent, recursive, enable_globs,
endpoint_plus_path, label, submission_id, dry_run, deadline,
skip_activation_check, notify, meow, heartbeat,
polling_interval, timeout):
"""
Executor for `globus rm`
"""
endpoint_id, path = endpoint_plus_path
client = get_client()
# attempt to activate unless --skip-activation-check is given
if not skip_activation_check:
autoactivate(client, endpoint_id, if_expires_in=60)
delete_data = DeleteData(client,
endpoint_id,
label=label,
recursive=recursive,
ignore_missing=ignore_missing,
submission_id=submission_id,
deadline=deadline,
skip_activation_check=skip_activation_check,
interpret_globs=enable_globs,
**notify)
if not star_silent and enable_globs and path.endswith('*'):
# not intuitive, but `click.confirm(abort=True)` prints to stdout
# unnecessarily, which we don't really want...
# only do this check if stderr is a pty
if (err_is_terminal() and term_is_interactive() and not click.confirm(
'Are you sure you want to delete all files matching "{}"?'.
format(path),
err=True)):
safeprint('Aborted.', write_to_stderr=True)
click.get_current_context().exit(1)
delete_data.add_item(path)
if dry_run:
formatted_print(delete_data,
response_key='DATA',
fields=[('Path', 'path')])
# exit safely
return
# Print task submission to stderr so that `-Fjson` is still correctly
# respected, as it will be by `task wait`
res = client.submit_delete(delete_data)
task_id = res['task_id']
safeprint('Delete task submitted under ID "{}"'.format(task_id),
write_to_stderr=True)
# do a `task wait` equivalent, including printing and correct exit status
task_wait_with_io(meow,
heartbeat,
polling_interval,
timeout,
task_id,
client=client)
def setUpClass(self):
"""
Gets a TransferClient and AuthClient for direct sdk calls
Cleans any old sharing data created by previous test runs
"""
self.tc = get_client()
self.ac = get_auth_client()
clean_sharing()
def generate_submission_id():
"""
Executor for `globus task generate-submission-id`
"""
client = get_client()
res = client.get_submission_id()
formatted_print(res, text_format=FORMAT_TEXT_RAW, response_key="value")
def delete_command(endpoint_id, rule_id):
"""
Executor for `globus endpoint permission delete`
"""
client = get_client()
res = client.delete_endpoint_acl_rule(endpoint_id, rule_id)
formatted_print(res, text_format=FORMAT_TEXT_RAW, response_key="message")
def my_shared_endpoint_list(endpoint_id):
"""
Executor for `globus endpoint my-shared-endpoint-list`
"""
client = get_client()
ep_iterator = client.my_shared_endpoint_list(endpoint_id)
formatted_print(ep_iterator, fields=ENDPOINT_LIST_FIELDS)
def setUpClass(self):
"""
Gets a TransferClient and AuthClient for direct sdk calls
Cleans any old sharing data created by previous test runs
"""
self.tc = get_client()
self.ac = get_auth_client()
clean_sharing()
def delete_command(endpoint_id, rule_id):
"""
Executor for `globus endpoint permission delete`
"""
client = get_client()
res = client.delete_endpoint_acl_rule(endpoint_id, rule_id)
formatted_print(res, text_format=FORMAT_TEXT_RAW, response_key='message')
def generate_submission_id():
"""
Executor for `globus task generate-submission-id`
"""
client = get_client()
res = client.get_submission_id()
formatted_print(res, text_format=FORMAT_TEXT_RAW, response_key="value")
def update_command(permissions, rule_id, endpoint_id):
"""
Executor for `globus endpoint permission update`
"""
client = get_client()
rule_data = assemble_generic_doc("access", permissions=permissions)
res = client.update_endpoint_acl_rule(endpoint_id, rule_id, rule_data)
formatted_print(res, text_format=FORMAT_TEXT_RAW, response_key="message")
def bookmark_delete(bookmark_id_or_name):
"""
Executor for `globus bookmark delete`
"""
client = get_client()
bookmark_id = resolve_id_or_name(client, bookmark_id_or_name)["id"]
res = client.delete_bookmark(bookmark_id)
formatted_print(res, text_format=FORMAT_TEXT_RAW, response_key="message")
def show_task(successful_transfers, task_id):
"""
Executor for `globus task show`
"""
client = get_client()
if successful_transfers:
print_successful_transfers(client, task_id)
else:
print_task_detail(client, task_id)
def update_task(deadline, label, task_id):
"""
Executor for `globus task update`
"""
client = get_client()
task_doc = assemble_generic_doc("task", label=label, deadline=deadline)
res = client.update_task(task_id, task_doc)
formatted_print(res, simple_text="Success")
def server_show(endpoint_id, server_id):
"""
Executor for `globus endpoint server show`
"""
client = get_client()
server_doc = client.get_endpoint_server(endpoint_id, server_id)
if not server_doc["uri"]: # GCP endpoint server
fields = (("ID", "id"),)
text_epilog = dedent(
"""
This server is for a Globus Connect Personal installation.
For its connection status, try:
globus endpoint show {}
""".format(
endpoint_id
)
)
else:
def advertised_port_summary(server):
def get_range_summary(start, end):
return (
"unspecified"
if not start and not end
else "unrestricted"
if start == 1024 and end == 65535
else "{}-{}".format(start, end)
)
return "incoming {}, outgoing {}".format(
get_range_summary(
server["incoming_data_port_start"], server["incoming_data_port_end"]
),
get_range_summary(
server["outgoing_data_port_start"], server["outgoing_data_port_end"]
),
)
fields = (
("ID", "id"),
("URI", "uri"),
("Subject", "subject"),
("Data Ports", advertised_port_summary),
)
text_epilog = None
formatted_print(
server_doc,
text_format=FORMAT_TEXT_RECORD,
fields=fields,
text_epilog=text_epilog,
)
def bookmark_rename(bookmark_id_or_name, new_bookmark_name):
"""
Executor for `globus bookmark rename`
"""
client = get_client()
bookmark_id = resolve_id_or_name(client, bookmark_id_or_name)["id"]
submit_data = {"name": new_bookmark_name}
res = client.update_bookmark(bookmark_id, submit_data)
formatted_print(res, simple_text="Success")
def mkdir_command(endpoint_plus_path):
"""
Executor for `globus mkdir`
"""
endpoint_id, path = endpoint_plus_path
client = get_client()
autoactivate(client, endpoint_id, if_expires_in=60)
res = client.operation_mkdir(endpoint_id, path=path)
formatted_print(res, text_format=FORMAT_TEXT_RAW, response_key="message")
def bookmark_create(endpoint_plus_path, bookmark_name):
"""
Executor for `globus bookmark create`
"""
endpoint_id, path = endpoint_plus_path
client = get_client()
submit_data = {"endpoint_id": str(endpoint_id), "path": path, "name": bookmark_name}
res = client.create_bookmark(submit_data)
formatted_print(res, simple_text="Bookmark ID: {}".format(res["id"]))
def endpoint_show(endpoint_id):
"""
Executor for `globus endpoint show`
"""
client = get_client()
res = client.get_endpoint(endpoint_id)
formatted_print(
res,
text_format=FORMAT_TEXT_RECORD,
fields=GCP_FIELDS if res["is_globus_connect"] else STANDARD_FIELDS,
)
def role_show(endpoint_id, role_id):
"""
Executor for `globus endpoint role show`
"""
client = get_client()
role = client.get_endpoint_role(endpoint_id, role_id)
formatted_print(
role,
text_format=FORMAT_TEXT_RECORD,
fields=(
("Principal Type", "principal_type"),
("Principal", lookup_principal),
("Role", "role"),
),
)
def task_event_list(task_id, limit, filter_errors, filter_non_errors):
"""
Executor for `globus task-event-list`
"""
client = get_client()
# cannot filter by both errors and non errors
if filter_errors and filter_non_errors:
raise click.UsageError("Cannot filter by both errors and non errors")
elif filter_errors:
filter_string = "is_error:1"
elif filter_non_errors:
filter_string = "is_error:0"
else:
filter_string = ""
event_iterator = client.task_event_list(
task_id, num_results=limit, filter=filter_string
)
def squashed_json_details(x):
is_json = False
try:
loaded = json.loads(x["details"])
is_json = True
except ValueError:
loaded = x["details"]
if is_json:
return json.dumps(loaded, separators=(",", ":"), sort_keys=True)
else:
return loaded.replace("\n", "\\n")
formatted_print(
event_iterator,
fields=(
("Time", "time"),
("Code", "code"),
("Is Error", "is_error"),
("Details", squashed_json_details),
),
json_converter=iterable_response_to_dict,
)
def show_command(endpoint_id, rule_id):
"""
Executor for `globus endpoint permission show`
"""
client = get_client()
rule = client.get_endpoint_acl_rule(endpoint_id, rule_id)
formatted_print(
rule,
text_format=FORMAT_TEXT_RECORD,
fields=(
("Rule ID", "id"),
("Permissions", "permissions"),
("Shared With", _shared_with_keyfunc),
("Path", "path"),
),
)
def create_command(
principal, permissions, endpoint_plus_path, notify_email, notify_message
):
"""
Executor for `globus endpoint permission create`
"""
if not principal:
raise click.UsageError("A security principal is required for this command")
endpoint_id, path = endpoint_plus_path
principal_type, principal_val = principal
client = get_client()
if principal_type == "identity":
principal_val = maybe_lookup_identity_id(principal_val)
if not principal_val:
raise click.UsageError(
"Identity does not exist. "
"Use --provision-identity to auto-provision an identity."
)
elif principal_type == "provision-identity":
principal_val = maybe_lookup_identity_id(principal_val, provision=True)
principal_type = "identity"
if not notify_email:
notify_message = None
rule_data = assemble_generic_doc(
"access",
permissions=permissions,
principal=principal_val,
principal_type=principal_type,
path=path,
notify_email=notify_email,
notify_message=notify_message,
)
res = client.add_endpoint_acl_rule(endpoint_id, rule_data)
formatted_print(
res,
text_format=FORMAT_TEXT_RECORD,
fields=[("Message", "message"), ("Rule ID", "access_id")],
)
def endpoint_is_activated(endpoint_id, until, absolute_time):
"""
Executor for `globus endpoint is-activated`
"""
client = get_client()
res = client.endpoint_get_activation_requirements(endpoint_id)
def fail(deadline=None):
exp_string = ""
if deadline is not None:
exp_string = " or will expire within {} seconds".format(deadline)
message = "The endpoint is not activated{}.\n\n".format(
exp_string
) + activation_requirements_help_text(res, endpoint_id)
formatted_print(res, simple_text=message)
click.get_current_context().exit(1)
def success(msg, *format_params):
formatted_print(res, simple_text=(msg.format(endpoint_id, *format_params)))
click.get_current_context().exit(0)
# eternally active endpoints have a special expires_in value
if res["expires_in"] == -1:
success("{} does not require activation")
# autoactivation is not supported and --until was not passed
if until is None:
# and we are active right now (0s in the future)...
if res.active_until(0):
success("{} is activated")
# or we are not active
fail()
# autoactivation is not supported and --until was passed
if res.active_until(until, relative_time=not absolute_time):
success("{} will be active for at least {} seconds", until)
else:
fail(deadline=until)
def bookmark_show(bookmark_id_or_name):
"""
Executor for `globus bookmark show`
"""
client = get_client()
res = resolve_id_or_name(client, bookmark_id_or_name)
formatted_print(
res,
text_format=FORMAT_TEXT_RECORD,
fields=(
("ID", "id"),
("Name", "name"),
("Endpoint ID", "endpoint_id"),
("Path", "path"),
),
simple_text=(
# standard output is endpoint:path format
"{}:{}".format(res["endpoint_id"], res["path"])
# verbose output includes all fields
if not is_verbose()
else None
),
)
def rm_command(
ignore_missing,
star_silent,
recursive,
enable_globs,
endpoint_plus_path,
label,
submission_id,
dry_run,
deadline,
skip_activation_check,
notify,
meow,
heartbeat,
polling_interval,
timeout,
timeout_exit_code,
):
"""
Executor for `globus rm`
"""
endpoint_id, path = endpoint_plus_path
client = get_client()
# attempt to activate unless --skip-activation-check is given
if not skip_activation_check:
autoactivate(client, endpoint_id, if_expires_in=60)
delete_data = DeleteData(
client,
endpoint_id,
label=label,
recursive=recursive,
ignore_missing=ignore_missing,
submission_id=submission_id,
deadline=deadline,
skip_activation_check=skip_activation_check,
interpret_globs=enable_globs,
**notify
)
if not star_silent and enable_globs and path.endswith("*"):
# not intuitive, but `click.confirm(abort=True)` prints to stdout
# unnecessarily, which we don't really want...
# only do this check if stderr is a pty
if (
err_is_terminal()
and term_is_interactive()
and not click.confirm(
'Are you sure you want to delete all files matching "{}"?'.format(path),
err=True,
)
):
safeprint("Aborted.", write_to_stderr=True)
click.get_current_context().exit(1)
delete_data.add_item(path)
if dry_run:
formatted_print(delete_data, response_key="DATA", fields=[("Path", "path")])
# exit safely
return
# Print task submission to stderr so that `-Fjson` is still correctly
# respected, as it will be by `task wait`
res = client.submit_delete(delete_data)
task_id = res["task_id"]
safeprint(
'Delete task submitted under ID "{}"'.format(task_id), write_to_stderr=True
)
# do a `task wait` equivalent, including printing and correct exit status
task_wait_with_io(
meow,
heartbeat,
polling_interval,
timeout,
task_id,
timeout_exit_code,
client=client,
)
def endpoint_activate(
endpoint_id,
myproxy,
myproxy_username,
myproxy_password,
myproxy_lifetime,
web,
no_browser,
delegate_proxy,
proxy_lifetime,
no_autoactivate,
force,
):
"""
Executor for `globus endpoint activate`
"""
default_myproxy_username = lookup_option(MYPROXY_USERNAME_OPTNAME)
client = get_client()
# validate options
if web + myproxy + bool(delegate_proxy) > 1:
raise click.UsageError(
"--web, --myproxy, and --delegate-proxy are mutually exclusive."
)
if no_autoactivate and not (myproxy or web or delegate_proxy):
raise click.UsageError(
"--no-autoactivate requires another activation method be given."
)
if myproxy_username and not myproxy:
raise click.UsageError("--myproxy-username requires --myproxy.")
if myproxy_password and not myproxy:
raise click.UsageError("--myproxy-password requires --myproxy.")
# NOTE: "0" is a legitimate, though weird, value
# In the case where someone is setting this value programatically,
# respecting it behaves more consistently/predictably
if myproxy_lifetime is not None and not myproxy:
raise click.UsageError("--myproxy-lifetime requires --myproxy.")
if no_browser and not web:
raise click.UsageError("--no-browser requires --web.")
if proxy_lifetime and not delegate_proxy:
raise click.UsageError("--proxy-lifetime requires --delegate-proxy.")
# check if endpoint is already activated unless --force
if not force:
res = client.endpoint_autoactivate(endpoint_id, if_expires_in=60)
if "AlreadyActivated" == res["code"]:
formatted_print(
res,
simple_text=(
"Endpoint is already activated. Activation "
"expires at {}".format(res["expire_time"])
),
)
return
# attempt autoactivation unless --no-autoactivate
if not no_autoactivate:
res = client.endpoint_autoactivate(endpoint_id)
if "AutoActivated" in res["code"]:
formatted_print(
res,
simple_text=(
"Autoactivation succeeded with message: {}".format(res["message"])
),
)
return
# override potentially confusing autoactivation failure response
else:
message = (
"The endpoint could not be auto-activated.\n\n"
+ activation_requirements_help_text(res, endpoint_id)
)
res = {"message": message}
# myproxy activation
if myproxy:
# get username and password
if not (myproxy_username or default_myproxy_username):
myproxy_username = click.prompt("Myproxy username")
if not myproxy_password:
myproxy_password = click.prompt("Myproxy password", hide_input=True)
no_server_msg = (
"This endpoint has no myproxy server "
"and so cannot be activated through myproxy"
)
requirements_data = client.endpoint_get_activation_requirements(
endpoint_id
).data
if not len(requirements_data["DATA"]):
raise click.ClickException(no_server_msg)
for data in requirements_data["DATA"]:
# skip non-myproxy values
# although the API does not practice this today, in theory other
# activation types may have fields with the same names...
if data["type"] != "myproxy":
continue
if data["name"] == "passphrase":
data["value"] = myproxy_password
if data["name"] == "username":
data["value"] = myproxy_username or default_myproxy_username
if data["name"] == "hostname" and data["value"] is None:
raise click.ClickException(no_server_msg)
# NOTE: remember that "0" is a possible value
if data["name"] == "lifetime_in_hours" and myproxy_lifetime is not None:
data["value"] = str(myproxy_lifetime)
res = client.endpoint_activate(endpoint_id, requirements_data)
# web activation
elif web:
url = "https://app.globus.org/file-manager" "?origin_id={}".format(endpoint_id)
if no_browser or is_remote_session():
res = {"message": "Web activation url: {}".format(url), "url": url}
else:
webbrowser.open(url, new=1)
res = {"message": "Browser opened to web activation page", "url": url}
# delegate proxy activation
elif delegate_proxy:
requirements_data = client.endpoint_get_activation_requirements(
endpoint_id
).data
filled_requirements_data = fill_delegate_proxy_activation_requirements(
requirements_data, delegate_proxy, lifetime_hours=proxy_lifetime or 12
)
res = client.endpoint_activate(endpoint_id, filled_requirements_data)
# output
formatted_print(res, text_format=FORMAT_TEXT_RAW, response_key="message")