def check_globus(src_uuid, dst_uuid, src_path, dst_path): """ Check that the globus endpoints are not only active but will return information about the paths we're interested in. Im assuming that the endpoints have already been activated """ try: endpoints = [{ 'type': 'source', 'id': src_uuid, 'path': src_path }, { 'type': 'destination', 'id': dst_uuid, 'path': dst_path }] except Exception as e: print_debug(e) client = get_client() try: for endpoint in endpoints: _ = get_ls(client, endpoint['path'], endpoint['id'], False, 0, False) hostname = client.endpoint_server_list( endpoint)['DATA']['hostname'] print "Access confirmed for {}".format(hostname) except Exception as e: print_debug(e) return False, endpoint else: return True, None
def server_add( endpoint_id, subject, port, scheme, hostname, incoming_data_ports, outgoing_data_ports, ): """ Executor for `globus endpoint server add` """ client = get_client() server_doc = assemble_generic_doc("server", subject=subject, port=port, scheme=scheme, hostname=hostname) # n.b. must be done after assemble_generic_doc(), as that function filters # out `None`s, which we need to be able to set for `'unspecified'` if incoming_data_ports: server_doc.update( incoming_data_port_start=incoming_data_ports[0], incoming_data_port_end=incoming_data_ports[1], ) if outgoing_data_ports: server_doc.update( outgoing_data_port_start=outgoing_data_ports[0], outgoing_data_port_end=outgoing_data_ports[1], ) res = client.add_endpoint_server(endpoint_id, server_doc) formatted_print(res, text_format=FORMAT_TEXT_RAW, response_key="message")
def role_list(endpoint_id): """ List the assigned roles on an endpoint. You must have sufficient privileges to see the roles on the endpoint. """ client = get_client() roles = client.endpoint_role_list(endpoint_id) resolved_ids = IdentityMap( get_auth_client(), (x["principal"] for x in roles if x["principal_type"] == "identity"), ) def principal_str(role): principal = role["principal"] if role["principal_type"] == "identity": try: return resolved_ids[principal]["username"] except KeyError: return principal if role["principal_type"] == "group": return (u"https://app.globus.org/groups/{}").format(principal) return principal formatted_print( roles, fields=[ ("Principal Type", "principal_type"), ("Role ID", "id"), ("Principal", principal_str), ("Role", "role"), ], )
def server_update( endpoint_id, server_id, subject, port, scheme, hostname, incoming_data_ports, outgoing_data_ports, ): """ Executor for `globus endpoint server update` """ client = get_client() server_doc = assemble_generic_doc( "server", subject=subject, port=port, scheme=scheme, hostname=hostname ) # n.b. must be done after assemble_generic_doc(), as that function filters # out `None`s, which we need to be able to set for `'unspecified'` if incoming_data_ports: server_doc.update( incoming_data_port_start=incoming_data_ports[0], incoming_data_port_end=incoming_data_ports[1], ) if outgoing_data_ports: server_doc.update( outgoing_data_port_start=outgoing_data_ports[0], outgoing_data_port_end=outgoing_data_ports[1], ) res = client.update_endpoint_server(endpoint_id, server_id, server_doc) formatted_print(res, text_format=FORMAT_TEXT_RAW, response_key="message")
def endpoint_search(filter_fulltext, filter_owner_id, filter_scope): """ Executor for `globus endpoint search` """ if filter_scope == "all" and not filter_fulltext: raise click.UsageError( "When searching all endpoints (--filter-scope=all, the default), " "a full-text search filter is required. Other scopes (e.g. " "--filter-scope=recently-used) may be used without specifying " "an additional filter." ) client = get_client() owner_id = filter_owner_id if owner_id: owner_id = maybe_lookup_identity_id(owner_id) search_iterator = client.endpoint_search( filter_fulltext=filter_fulltext, filter_scope=filter_scope, filter_owner_id=owner_id, ) formatted_print( search_iterator, fields=ENDPOINT_LIST_FIELDS, json_converter=iterable_response_to_dict, )
def bookmark_show(bookmark_id_or_name): """ Given a single bookmark ID or bookmark name, show the bookmark details. By default, when the format is TEXT, this will display the endpoint ID and path in 'ENDPOINT_ID:PATH' notation. The default output is suitable for use in a subshell in another command. If *-v, --verbose* is given, several fields will be displayed. """ client = get_client() res = resolve_id_or_name(client, bookmark_id_or_name) formatted_print( res, text_format=FORMAT_TEXT_RECORD, fields=( ("ID", "id"), ("Name", "name"), ("Endpoint ID", "endpoint_id"), ("Path", "path"), ), simple_text=( # standard output is endpoint:path format "{}:{}".format(res["endpoint_id"], res["path"]) # verbose output includes all fields if not is_verbose() else None), )
def test_filemanager_update_remote_yes_sta(self): sta = True types = ['atm', 'ice', 'ocn', 'streams.cice', 'streams.ocean'] database = 'test.db' simstart = 51 source_path = '/global/cscratch1/sd/golaz/ACME_simulations/20170915.beta2.A_WCYCL1850S.ne30_oECv3_ICG.edison' simend = 60 experiment = '20170915.beta2.A_WCYCL1850S.ne30_oECv3_ICG.edison' filemanager = FileManager(mutex=self.mutex, sta=sta, types=types, database=database, remote_endpoint=self.remote_endpoint, remote_path=source_path, local_endpoint=self.local_endpoint, local_path=self.local_path) filemanager.populate_file_list(simstart=simstart, simend=simend, experiment=experiment) client = get_client() filemanager.update_remote_status(client) self.mutex.acquire() for datafile in DataFile.select(): if datafile.remote_status != 0: print datafile.name, datafile.remote_path self.assertEqual(datafile.remote_status, 0) if self.mutex.locked(): self.mutex.release() self.assertTrue(filemanager.all_data_remote())
def bookmark_list(): """ Executor for `globus bookmark list` """ client = get_client() bookmark_iterator = client.bookmark_list() def get_ep_name(item): ep_id = item["endpoint_id"] try: ep_doc = client.get_endpoint(ep_id) return display_name_or_cname(ep_doc) except TransferAPIError as err: if err.code == "EndpointDeleted": return "[DELETED ENDPOINT]" else: raise err formatted_print( bookmark_iterator, fields=[ ("Name", "name"), ("Bookmark ID", "id"), ("Endpoint ID", "endpoint_id"), ("Endpoint Name", get_ep_name), ("Path", "path"), ], response_key="DATA", json_converter=iterable_response_to_dict, )
def server_show(endpoint_id, server_id): """ Executor for `globus endpoint server show` """ client = get_client() server_doc = client.get_endpoint_server(endpoint_id, server_id) if not server_doc['uri']: # GCP endpoint server fields = (('ID', 'id'), ('Is Connected', 'is_connected'), ('Is Paused (macOS only)', 'is_paused')) else: def advertised_port_summary(server): def get_range_summary(start, end): return ('unspecified' if not start and not end else 'unrestricted' if start == 1024 and end == 65535 else '{}-{}'.format(start, end)) return "incoming {}, outgoing {}".format( get_range_summary(server['incoming_data_port_start'], server['incoming_data_port_end']), get_range_summary(server['outgoing_data_port_start'], server['outgoing_data_port_end']), ) fields = (('ID', 'id'), ('URI', 'uri'), ('Subject', 'subject'), ('Data Ports', advertised_port_summary)) formatted_print(server_doc, text_format=FORMAT_TEXT_RECORD, fields=fields)
def role_create(role, principal, endpoint_id): """ Executor for `globus endpoint role show` """ principal_type, principal_val = principal client = get_client() if principal_type == 'identity': principal_val = maybe_lookup_identity_id(principal_val) if not principal_val: raise click.UsageError( 'Identity does not exist. ' 'Use --provision-identity to auto-provision an identity.') elif principal_type == 'provision-identity': principal_val = maybe_lookup_identity_id(principal_val, provision=True) principal_type = 'identity' role_doc = assemble_generic_doc('role', principal_type=principal_type, principal=principal_val, role=role) res = client.add_endpoint_role(endpoint_id, role_doc) formatted_print(res, simple_text='ID: {}'.format(res['id']))
def check_globus(**kwargs): """ Check that the globus endpoints are not only active but will return information about the paths we're interested in. Im assuming that the endpoints have already been activated """ try: endpoints = [{ 'type': 'source', 'id': kwargs['source_endpoint'], 'path': kwargs['source_path'] }, { 'type': 'destination', 'id': kwargs['destination_endpoint'], 'path': kwargs['destination_path'] }] except Exception as e: print_debug(e) client = get_client() try: for endpoint in endpoints: res = get_ls( client, endpoint['path'], endpoint['id'], False, 0, False) except Exception as e: print_debug(e) return False, endpoint else: print "Access granted" return True, None
def endpoint_delete(endpoint_id): """ Executor for `globus endpoint delete` """ client = get_client() res = client.delete_endpoint(endpoint_id) formatted_print(res, text_format=FORMAT_TEXT_RAW, response_key="message")
def role_delete(role_id, endpoint_id): """ Executor for `globus endpoint role delete` """ client = get_client() res = client.delete_endpoint_role(endpoint_id, role_id) formatted_print(res, text_format=FORMAT_TEXT_RAW, response_key='message')
def role_create(role, principal, endpoint_id): """ Create a role on an endpoint. You must have sufficient privileges to modify the roles on the endpoint. Either *--group* or *--identity* is required. You may not pass both. Which one of these options you use will determine the 'Principal Type' on the role, and the value given will be the 'Principal' of the resulting role. The term "Principal" is used in the sense of "a security principal", an entity which has some privileges associated with it. """ principal_type, principal_val = principal client = get_client() if principal_type == "identity": principal_val = maybe_lookup_identity_id(principal_val) if not principal_val: raise click.UsageError( "Identity does not exist. " "Use --provision-identity to auto-provision an identity.") elif principal_type == "provision-identity": principal_val = maybe_lookup_identity_id(principal_val, provision=True) principal_type = "identity" role_doc = assemble_generic_doc("role", principal_type=principal_type, principal=principal_val, role=role) res = client.add_endpoint_role(endpoint_id, role_doc) formatted_print(res, simple_text="ID: {}".format(res["id"]))
def list_command(endpoint_id): """List all rules in an endpoint's access control list.""" client = get_client() rules = client.endpoint_acl_list(endpoint_id) resolved_ids = IdentityMap( get_auth_client(), (x["principal"] for x in rules if x["principal_type"] == "identity"), ) def principal_str(rule): principal = rule["principal"] if rule["principal_type"] == "identity": try: return resolved_ids[principal]["username"] except KeyError: return principal if rule["principal_type"] == "group": return (u"https://app.globus.org/groups/{}").format(principal) return rule["principal_type"] formatted_print( rules, fields=[ ("Rule ID", "id"), ("Permissions", "permissions"), ("Shared With", principal_str), ("Path", "path"), ], )
def clean_sharing(): """ Cleans out any files in ~/.globus/sharing/ on go#ep1 older than an hour TODO: remove this once deleting shared directories does full cleanup """ tc = get_client() path = "~/.globus/sharing/" hour_ago = datetime.utcnow() - timedelta(hours=1) filter_string = "last_modified:," + hour_ago.strftime("%Y-%m-%d %H:%M:%S") try: old_files = tc.operation_ls( GO_EP1_ID, path=path, filter=filter_string, num_results=None ) except globus_sdk.TransferAPIError: return kwargs = {"notify_on_succeeded": False, "notify_on_fail": False} ddata = globus_sdk.DeleteData(tc, GO_EP1_ID, **kwargs) for item in old_files: ddata.add_item(path + item["name"]) if len(ddata["DATA"]): tc.submit_delete(ddata)
def setup_globus(endpoints, event_list): """ Check globus login status and login as nessisary, then iterate over a list of endpoints and activate them all Parameters: endpoints: list of strings containing globus endpoint UUIDs event_list: the event list to push user notifications into return: True if successful, False otherwise """ # First go through the globus login process if not check_logged_in(): message = 'Globus login required. Please run {cmd}\n\n'.format( cmd='"globus login"') print_line(message, event_list) print '================================================' sys.exit(1) if isinstance(endpoints, str): endpoints = [endpoints] activated = False client = get_client() while not activated: activated = True message = '' for endpoint in endpoints: msg = 'activating endpoint {}'.format(endpoint) logging.info(msg) try: r = client.endpoint_autoactivate(endpoint, if_expires_in=3600) logging.info(r['code']) except Exception as e: print_debug(e) if e.code == 'ClientError.NotFound': return False else: continue if r["code"] == "AutoActivationFailed": activated = False logging.info('endpoint autoactivation failed') server_document = client.endpoint_server_list(endpoint) for server in server_document['DATA']: hostname = server["hostname"] break message += """ Data transfer server {server} requires manual activation. Please open the following URL in a browser to activate the endpoint: https://www.globus.org/app/endpoints/{endpoint}/activate """.format(endpoint=endpoint, server=server['hostname']) if not activated: print message raw_input('Press ENTER once endpoints have been activated\n') return True
def endpoint_update(**kwargs): """ Executor for `globus endpoint update` """ # validate params. Requires a get call to check the endpoint type client = get_client() endpoint_id = kwargs.pop("endpoint_id") get_res = client.get_endpoint(endpoint_id) if get_res["host_endpoint_id"]: endpoint_type = "shared" elif get_res["is_globus_connect"]: endpoint_type = "personal" elif get_res["s3_url"]: endpoint_type = "s3" else: endpoint_type = "server" validate_endpoint_create_and_update_params( endpoint_type, get_res["subscription_id"], kwargs ) # make the update ep_doc = assemble_generic_doc("endpoint", **kwargs) res = client.update_endpoint(endpoint_id, ep_doc) formatted_print(res, text_format=FORMAT_TEXT_RAW, response_key="message")
def task_pause_info(task_id): """ Executor for `globus task pause-info` """ client = get_client() res = client.task_pause_info(task_id) def _custom_text_format(res): explicit_pauses = [ field for field in EXPLICIT_PAUSE_MSG_FIELDS # n.b. some keys are absent for completed tasks if res.get(field[1]) ] effective_pause_rules = res['pause_rules'] if not explicit_pauses and not effective_pause_rules: safeprint('Task {} is not paused.'.format(task_id)) click.get_current_context().exit(0) if explicit_pauses: formatted_print( res, fields=explicit_pauses, text_format=FORMAT_TEXT_RECORD, text_preamble='This task has been explicitly paused.\n', text_epilog='\n' if effective_pause_rules else None) if effective_pause_rules: formatted_print( effective_pause_rules, fields=PAUSE_RULE_DISPLAY_FIELDS, text_preamble=( 'The following pause rules are effective on this task:\n')) formatted_print(res, text_format=_custom_text_format)
def role_create(role, principal, endpoint_id): """ Executor for `globus endpoint role show` """ principal_type, principal_val = principal client = get_client() if principal_type == "identity": principal_val = maybe_lookup_identity_id(principal_val) if not principal_val: raise click.UsageError( "Identity does not exist. " "Use --provision-identity to auto-provision an identity." ) elif principal_type == "provision-identity": principal_val = maybe_lookup_identity_id(principal_val, provision=True) principal_type = "identity" role_doc = assemble_generic_doc( "role", principal_type=principal_type, principal=principal_val, role=role ) res = client.add_endpoint_role(endpoint_id, role_doc) formatted_print(res, simple_text="ID: {}".format(res["id"]))
def role_delete(role_id, endpoint_id): """ Executor for `globus endpoint role delete` """ client = get_client() res = client.delete_endpoint_role(endpoint_id, role_id) formatted_print(res, text_format=FORMAT_TEXT_RAW, response_key="message")
def server_show(endpoint_id, server_id): """ Executor for `globus endpoint server show` """ client = get_client() server_doc = client.get_endpoint_server(endpoint_id, server_id) if not server_doc['uri']: # GCP endpoint server fields = (('ID', 'id'),) text_epilog = dedent(""" This server is for a Globus Connect Personal installation. For its connection status, try: globus endpoint show {} """.format(endpoint_id)) else: def advertised_port_summary(server): def get_range_summary(start, end): return ('unspecified' if not start and not end else 'unrestricted' if start == 1024 and end == 65535 else '{}-{}'.format(start, end)) return "incoming {}, outgoing {}".format( get_range_summary(server['incoming_data_port_start'], server['incoming_data_port_end']), get_range_summary(server['outgoing_data_port_start'], server['outgoing_data_port_end']), ) fields = (('ID', 'id'), ('URI', 'uri'), ('Subject', 'subject'), ('Data Ports', advertised_port_summary)) text_epilog = None formatted_print(server_doc, text_format=FORMAT_TEXT_RECORD, fields=fields, text_epilog=text_epilog)
def bookmark_list(): """ Executor for `globus bookmark list` """ client = get_client() bookmark_iterator = client.bookmark_list() def get_ep_name(item): ep_id = item['endpoint_id'] try: ep_doc = client.get_endpoint(ep_id) return display_name_or_cname(ep_doc) except TransferAPIError as err: if err.code == "EndpointDeleted": return "[DELETED ENDPOINT]" else: raise err formatted_print(bookmark_iterator, fields=[('Name', 'name'), ('Bookmark ID', 'id'), ('Endpoint ID', 'endpoint_id'), ('Endpoint Name', get_ep_name), ('Path', 'path')], response_key='DATA', json_converter=iterable_response_to_dict)
def list_command(endpoint_id): """ Executor for `globus endpoint permission list` """ client = get_client() rules = client.endpoint_acl_list(endpoint_id) resolved_ids = LazyIdentityMap( x["principal"] for x in rules if x["principal_type"] == "identity" ) def principal_str(rule): principal = rule["principal"] if rule["principal_type"] == "identity": username = resolved_ids.get(principal) return username or principal elif rule["principal_type"] == "group": return (u"https://app.globus.org/groups/{}").format(principal) else: principal = rule["principal_type"] return principal formatted_print( rules, fields=[ ("Rule ID", "id"), ("Permissions", "permissions"), ("Shared With", principal_str), ("Path", "path"), ], )
def role_list(endpoint_id): """ Executor for `globus access endpoint-role-list` """ client = get_client() roles = client.endpoint_role_list(endpoint_id) resolved_ids = LazyIdentityMap( x["principal"] for x in roles if x["principal_type"] == "identity" ) def principal_str(role): principal = role["principal"] if role["principal_type"] == "identity": username = resolved_ids.get(principal) return username or principal elif role["principal_type"] == "group": return (u"https://app.globus.org/groups/{}").format(principal) else: return principal formatted_print( roles, fields=[ ("Principal Type", "principal_type"), ("Role ID", "id"), ("Principal", principal_str), ("Role", "role"), ], )
def list_command(endpoint_id): """ Executor for `globus endpoint permission list` """ client = get_client() rules = client.endpoint_acl_list(endpoint_id) resolved_ids = LazyIdentityMap(x["principal"] for x in rules if x["principal_type"] == "identity") def principal_str(rule): principal = rule["principal"] if rule["principal_type"] == "identity": username = resolved_ids.get(principal) return username or principal elif rule["principal_type"] == "group": return (u"https://app.globus.org/groups/{}").format(principal) else: principal = rule["principal_type"] return principal formatted_print( rules, fields=[ ("Rule ID", "id"), ("Permissions", "permissions"), ("Shared With", principal_str), ("Path", "path"), ], )
def bookmark_create(endpoint_plus_path, bookmark_name): """ Create a new bookmark. Given an endpoint plus a path, and a name for the bookmark, the service will generate the bookmark's ID. Bookmarks are aliases for locations on endpoints, and their names are unique per account. You may not have multiple bookmarks with the same name. You can use bookmarks in other commands by using *globus bookmark show*. The new bookmark name may be up to 128 characters long. Bookmarks are only visible and usable for the user who created them. If the target endpoint is private or deleted, the bookmark is unusable. 'PATH' is assumed to be URL-encoded. 'PATH' must be a directory and end with "/". """ endpoint_id, path = endpoint_plus_path client = get_client() submit_data = { "endpoint_id": str(endpoint_id), "path": path, "name": bookmark_name } res = client.create_bookmark(submit_data) formatted_print(res, simple_text="Bookmark ID: {}".format(res["id"]))
def clean_sharing(): """ Cleans out any files in ~/.globus/sharing/ on go#ep1 older than an hour TODO: remove this once deleting shared directories does full cleanup """ tc = get_client() path = "~/.globus/sharing/" hour_ago = datetime.utcnow() - timedelta(hours=1) filter_string = "last_modified:," + hour_ago.strftime("%Y-%m-%d %H:%M:%S") try: old_files = tc.operation_ls(GO_EP1_ID, path=path, filter=filter_string, num_results=None) except globus_sdk.TransferAPIError: return kwargs = {"notify_on_succeeded": False, "notify_on_fail": False} ddata = globus_sdk.DeleteData(tc, GO_EP1_ID, **kwargs) for item in old_files: ddata.add_item(path + item["name"]) if len(ddata["DATA"]): tc.submit_delete(ddata)
def endpoint_deactivate(endpoint_id): """ Executor for `globus endpoint deactivate` """ client = get_client() res = client.endpoint_deactivate(endpoint_id) formatted_print(res, text_format=FORMAT_TEXT_RAW, response_key="message")
def list_command(endpoint_id): """ Executor for `globus endpoint permission list` """ client = get_client() rules = client.endpoint_acl_list(endpoint_id) resolved_ids = LazyIdentityMap(x['principal'] for x in rules if x['principal_type'] == 'identity') def principal_str(rule): principal = rule['principal'] if rule['principal_type'] == 'identity': username = resolved_ids.get(principal) return username or principal elif rule['principal_type'] == 'group': return (u'https://www.globus.org/app/groups/{}').format(principal) else: principal = rule['principal_type'] return principal formatted_print(rules, fields=[('Rule ID', 'id'), ('Permissions', 'permissions'), ('Shared With', principal_str), ('Path', 'path')])
def test_filemanager_update_remote_no_sta(self): sta = False types = ['atm', 'ice', 'ocn', 'rest', 'streams.cice', 'streams.ocean'] database = 'test.db' simstart = 51 simend = 60 experiment = '20171011.beta2_FCT2-icedeep_branch.A_WCYCL1850S.ne30_oECv3_ICG.edison' filemanager = FileManager(mutex=self.mutex, sta=sta, types=types, database=database, remote_endpoint=self.remote_endpoint, remote_path=self.remote_path, local_endpoint=self.local_endpoint, local_path=self.local_path) filemanager.populate_file_list(simstart=simstart, simend=simend, experiment=experiment) client = get_client() filemanager.update_remote_status(client) self.mutex.acquire() for datafile in DataFile.select(): self.assertEqual(datafile.remote_status, 0) if self.mutex.locked(): self.mutex.release() self.assertTrue(filemanager.all_data_remote())
def endpoint_update(**kwargs): """ Executor for `globus endpoint update` """ # validate params. Requires a get call to check the endpoint type client = get_client() endpoint_id = kwargs.pop("endpoint_id") get_res = client.get_endpoint(endpoint_id) if get_res["host_endpoint_id"]: endpoint_type = "shared" elif get_res["is_globus_connect"]: endpoint_type = "personal" elif get_res["s3_url"]: endpoint_type = "s3" else: endpoint_type = "server" validate_endpoint_create_and_update_params(endpoint_type, get_res["subscription_id"], kwargs) # make the update ep_doc = assemble_generic_doc('endpoint', **kwargs) res = client.update_endpoint(endpoint_id, ep_doc) formatted_print(res, text_format=FORMAT_TEXT_RAW, response_key='message')
def endpoint_show(endpoint_id): """ Executor for `globus endpoint show` """ client = get_client() res = client.get_endpoint(endpoint_id) def _managed_endpoint(x): """ Helper for converting subscription_id into managed_endpoint """ return bool(x["subscription_id"]) formatted_print( res, text_format=FORMAT_TEXT_RECORD, fields=(("Display Name", "display_name"), ("ID", "id"), ("Owner", "owner_string"), ("Activated", "activated"), ("Shareable", "shareable"), ("Department", "department"), ("Keywords", "keywords"), ("Endpoint Info Link", "info_link"), ("Contact E-mail", "contact_email"), ("Organization", "organization"), ("Department", "department"), ("Other Contact Info", "contact_info"), ("Visibility", "public"), ("Default Directory", "default_directory"), ("Force Encryption", "force_encryption"), ("Managed Endpoint", _managed_endpoint), ("Subscription ID", "subscription_id"), ("Legacy Name", "canonical_name"), ("Local User Info Available", "local_user_info_available")))
def rename_command(source, destination): """Rename a file or directory on an endpoint. The old path must be an existing file or directory. The new path must not yet exist. The new path does not have to be in the same directory as the old path, but most endpoints will require it to stay on the same filesystem. The endpoint must be entered twice for the sake of path syntax consistency. """ source_ep, source_path = source dest_ep, dest_path = destination if source_ep != dest_ep: raise click.UsageError( ("rename requires that the source and dest " "endpoints are the same, {} != {}").format(source_ep, dest_ep)) endpoint_id = source_ep client = get_client() autoactivate(client, endpoint_id, if_expires_in=60) res = client.operation_rename(endpoint_id, oldpath=source_path, newpath=dest_path) formatted_print(res, text_format=FORMAT_TEXT_RAW, response_key="message")
def rm_command(ignore_missing, star_silent, recursive, enable_globs, endpoint_plus_path, label, submission_id, dry_run, deadline, skip_activation_check, notify, meow, heartbeat, polling_interval, timeout): """ Executor for `globus rm` """ endpoint_id, path = endpoint_plus_path client = get_client() # attempt to activate unless --skip-activation-check is given if not skip_activation_check: autoactivate(client, endpoint_id, if_expires_in=60) delete_data = DeleteData(client, endpoint_id, label=label, recursive=recursive, ignore_missing=ignore_missing, submission_id=submission_id, deadline=deadline, skip_activation_check=skip_activation_check, interpret_globs=enable_globs, **notify) if not star_silent and enable_globs and path.endswith('*'): # not intuitive, but `click.confirm(abort=True)` prints to stdout # unnecessarily, which we don't really want... # only do this check if stderr is a pty if (err_is_terminal() and term_is_interactive() and not click.confirm( 'Are you sure you want to delete all files matching "{}"?'. format(path), err=True)): safeprint('Aborted.', write_to_stderr=True) click.get_current_context().exit(1) delete_data.add_item(path) if dry_run: formatted_print(delete_data, response_key='DATA', fields=[('Path', 'path')]) # exit safely return # Print task submission to stderr so that `-Fjson` is still correctly # respected, as it will be by `task wait` res = client.submit_delete(delete_data) task_id = res['task_id'] safeprint('Delete task submitted under ID "{}"'.format(task_id), write_to_stderr=True) # do a `task wait` equivalent, including printing and correct exit status task_wait_with_io(meow, heartbeat, polling_interval, timeout, task_id, client=client)
def setUpClass(self): """ Gets a TransferClient and AuthClient for direct sdk calls Cleans any old sharing data created by previous test runs """ self.tc = get_client() self.ac = get_auth_client() clean_sharing()
def generate_submission_id(): """ Executor for `globus task generate-submission-id` """ client = get_client() res = client.get_submission_id() formatted_print(res, text_format=FORMAT_TEXT_RAW, response_key="value")
def delete_command(endpoint_id, rule_id): """ Executor for `globus endpoint permission delete` """ client = get_client() res = client.delete_endpoint_acl_rule(endpoint_id, rule_id) formatted_print(res, text_format=FORMAT_TEXT_RAW, response_key="message")
def my_shared_endpoint_list(endpoint_id): """ Executor for `globus endpoint my-shared-endpoint-list` """ client = get_client() ep_iterator = client.my_shared_endpoint_list(endpoint_id) formatted_print(ep_iterator, fields=ENDPOINT_LIST_FIELDS)
def setUpClass(self): """ Gets a TransferClient and AuthClient for direct sdk calls Cleans any old sharing data created by previous test runs """ self.tc = get_client() self.ac = get_auth_client() clean_sharing()
def delete_command(endpoint_id, rule_id): """ Executor for `globus endpoint permission delete` """ client = get_client() res = client.delete_endpoint_acl_rule(endpoint_id, rule_id) formatted_print(res, text_format=FORMAT_TEXT_RAW, response_key='message')
def generate_submission_id(): """ Executor for `globus task generate-submission-id` """ client = get_client() res = client.get_submission_id() formatted_print(res, text_format=FORMAT_TEXT_RAW, response_key="value")
def update_command(permissions, rule_id, endpoint_id): """ Executor for `globus endpoint permission update` """ client = get_client() rule_data = assemble_generic_doc("access", permissions=permissions) res = client.update_endpoint_acl_rule(endpoint_id, rule_id, rule_data) formatted_print(res, text_format=FORMAT_TEXT_RAW, response_key="message")
def bookmark_delete(bookmark_id_or_name): """ Executor for `globus bookmark delete` """ client = get_client() bookmark_id = resolve_id_or_name(client, bookmark_id_or_name)["id"] res = client.delete_bookmark(bookmark_id) formatted_print(res, text_format=FORMAT_TEXT_RAW, response_key="message")
def show_task(successful_transfers, task_id): """ Executor for `globus task show` """ client = get_client() if successful_transfers: print_successful_transfers(client, task_id) else: print_task_detail(client, task_id)
def update_task(deadline, label, task_id): """ Executor for `globus task update` """ client = get_client() task_doc = assemble_generic_doc("task", label=label, deadline=deadline) res = client.update_task(task_id, task_doc) formatted_print(res, simple_text="Success")
def server_show(endpoint_id, server_id): """ Executor for `globus endpoint server show` """ client = get_client() server_doc = client.get_endpoint_server(endpoint_id, server_id) if not server_doc["uri"]: # GCP endpoint server fields = (("ID", "id"),) text_epilog = dedent( """ This server is for a Globus Connect Personal installation. For its connection status, try: globus endpoint show {} """.format( endpoint_id ) ) else: def advertised_port_summary(server): def get_range_summary(start, end): return ( "unspecified" if not start and not end else "unrestricted" if start == 1024 and end == 65535 else "{}-{}".format(start, end) ) return "incoming {}, outgoing {}".format( get_range_summary( server["incoming_data_port_start"], server["incoming_data_port_end"] ), get_range_summary( server["outgoing_data_port_start"], server["outgoing_data_port_end"] ), ) fields = ( ("ID", "id"), ("URI", "uri"), ("Subject", "subject"), ("Data Ports", advertised_port_summary), ) text_epilog = None formatted_print( server_doc, text_format=FORMAT_TEXT_RECORD, fields=fields, text_epilog=text_epilog, )
def bookmark_rename(bookmark_id_or_name, new_bookmark_name): """ Executor for `globus bookmark rename` """ client = get_client() bookmark_id = resolve_id_or_name(client, bookmark_id_or_name)["id"] submit_data = {"name": new_bookmark_name} res = client.update_bookmark(bookmark_id, submit_data) formatted_print(res, simple_text="Success")
def mkdir_command(endpoint_plus_path): """ Executor for `globus mkdir` """ endpoint_id, path = endpoint_plus_path client = get_client() autoactivate(client, endpoint_id, if_expires_in=60) res = client.operation_mkdir(endpoint_id, path=path) formatted_print(res, text_format=FORMAT_TEXT_RAW, response_key="message")
def bookmark_create(endpoint_plus_path, bookmark_name): """ Executor for `globus bookmark create` """ endpoint_id, path = endpoint_plus_path client = get_client() submit_data = {"endpoint_id": str(endpoint_id), "path": path, "name": bookmark_name} res = client.create_bookmark(submit_data) formatted_print(res, simple_text="Bookmark ID: {}".format(res["id"]))
def endpoint_show(endpoint_id): """ Executor for `globus endpoint show` """ client = get_client() res = client.get_endpoint(endpoint_id) formatted_print( res, text_format=FORMAT_TEXT_RECORD, fields=GCP_FIELDS if res["is_globus_connect"] else STANDARD_FIELDS, )
def role_show(endpoint_id, role_id): """ Executor for `globus endpoint role show` """ client = get_client() role = client.get_endpoint_role(endpoint_id, role_id) formatted_print( role, text_format=FORMAT_TEXT_RECORD, fields=( ("Principal Type", "principal_type"), ("Principal", lookup_principal), ("Role", "role"), ), )
def task_event_list(task_id, limit, filter_errors, filter_non_errors): """ Executor for `globus task-event-list` """ client = get_client() # cannot filter by both errors and non errors if filter_errors and filter_non_errors: raise click.UsageError("Cannot filter by both errors and non errors") elif filter_errors: filter_string = "is_error:1" elif filter_non_errors: filter_string = "is_error:0" else: filter_string = "" event_iterator = client.task_event_list( task_id, num_results=limit, filter=filter_string ) def squashed_json_details(x): is_json = False try: loaded = json.loads(x["details"]) is_json = True except ValueError: loaded = x["details"] if is_json: return json.dumps(loaded, separators=(",", ":"), sort_keys=True) else: return loaded.replace("\n", "\\n") formatted_print( event_iterator, fields=( ("Time", "time"), ("Code", "code"), ("Is Error", "is_error"), ("Details", squashed_json_details), ), json_converter=iterable_response_to_dict, )
def show_command(endpoint_id, rule_id): """ Executor for `globus endpoint permission show` """ client = get_client() rule = client.get_endpoint_acl_rule(endpoint_id, rule_id) formatted_print( rule, text_format=FORMAT_TEXT_RECORD, fields=( ("Rule ID", "id"), ("Permissions", "permissions"), ("Shared With", _shared_with_keyfunc), ("Path", "path"), ), )
def create_command( principal, permissions, endpoint_plus_path, notify_email, notify_message ): """ Executor for `globus endpoint permission create` """ if not principal: raise click.UsageError("A security principal is required for this command") endpoint_id, path = endpoint_plus_path principal_type, principal_val = principal client = get_client() if principal_type == "identity": principal_val = maybe_lookup_identity_id(principal_val) if not principal_val: raise click.UsageError( "Identity does not exist. " "Use --provision-identity to auto-provision an identity." ) elif principal_type == "provision-identity": principal_val = maybe_lookup_identity_id(principal_val, provision=True) principal_type = "identity" if not notify_email: notify_message = None rule_data = assemble_generic_doc( "access", permissions=permissions, principal=principal_val, principal_type=principal_type, path=path, notify_email=notify_email, notify_message=notify_message, ) res = client.add_endpoint_acl_rule(endpoint_id, rule_data) formatted_print( res, text_format=FORMAT_TEXT_RECORD, fields=[("Message", "message"), ("Rule ID", "access_id")], )
def endpoint_is_activated(endpoint_id, until, absolute_time): """ Executor for `globus endpoint is-activated` """ client = get_client() res = client.endpoint_get_activation_requirements(endpoint_id) def fail(deadline=None): exp_string = "" if deadline is not None: exp_string = " or will expire within {} seconds".format(deadline) message = "The endpoint is not activated{}.\n\n".format( exp_string ) + activation_requirements_help_text(res, endpoint_id) formatted_print(res, simple_text=message) click.get_current_context().exit(1) def success(msg, *format_params): formatted_print(res, simple_text=(msg.format(endpoint_id, *format_params))) click.get_current_context().exit(0) # eternally active endpoints have a special expires_in value if res["expires_in"] == -1: success("{} does not require activation") # autoactivation is not supported and --until was not passed if until is None: # and we are active right now (0s in the future)... if res.active_until(0): success("{} is activated") # or we are not active fail() # autoactivation is not supported and --until was passed if res.active_until(until, relative_time=not absolute_time): success("{} will be active for at least {} seconds", until) else: fail(deadline=until)
def bookmark_show(bookmark_id_or_name): """ Executor for `globus bookmark show` """ client = get_client() res = resolve_id_or_name(client, bookmark_id_or_name) formatted_print( res, text_format=FORMAT_TEXT_RECORD, fields=( ("ID", "id"), ("Name", "name"), ("Endpoint ID", "endpoint_id"), ("Path", "path"), ), simple_text=( # standard output is endpoint:path format "{}:{}".format(res["endpoint_id"], res["path"]) # verbose output includes all fields if not is_verbose() else None ), )
def rm_command( ignore_missing, star_silent, recursive, enable_globs, endpoint_plus_path, label, submission_id, dry_run, deadline, skip_activation_check, notify, meow, heartbeat, polling_interval, timeout, timeout_exit_code, ): """ Executor for `globus rm` """ endpoint_id, path = endpoint_plus_path client = get_client() # attempt to activate unless --skip-activation-check is given if not skip_activation_check: autoactivate(client, endpoint_id, if_expires_in=60) delete_data = DeleteData( client, endpoint_id, label=label, recursive=recursive, ignore_missing=ignore_missing, submission_id=submission_id, deadline=deadline, skip_activation_check=skip_activation_check, interpret_globs=enable_globs, **notify ) if not star_silent and enable_globs and path.endswith("*"): # not intuitive, but `click.confirm(abort=True)` prints to stdout # unnecessarily, which we don't really want... # only do this check if stderr is a pty if ( err_is_terminal() and term_is_interactive() and not click.confirm( 'Are you sure you want to delete all files matching "{}"?'.format(path), err=True, ) ): safeprint("Aborted.", write_to_stderr=True) click.get_current_context().exit(1) delete_data.add_item(path) if dry_run: formatted_print(delete_data, response_key="DATA", fields=[("Path", "path")]) # exit safely return # Print task submission to stderr so that `-Fjson` is still correctly # respected, as it will be by `task wait` res = client.submit_delete(delete_data) task_id = res["task_id"] safeprint( 'Delete task submitted under ID "{}"'.format(task_id), write_to_stderr=True ) # do a `task wait` equivalent, including printing and correct exit status task_wait_with_io( meow, heartbeat, polling_interval, timeout, task_id, timeout_exit_code, client=client, )
def endpoint_activate( endpoint_id, myproxy, myproxy_username, myproxy_password, myproxy_lifetime, web, no_browser, delegate_proxy, proxy_lifetime, no_autoactivate, force, ): """ Executor for `globus endpoint activate` """ default_myproxy_username = lookup_option(MYPROXY_USERNAME_OPTNAME) client = get_client() # validate options if web + myproxy + bool(delegate_proxy) > 1: raise click.UsageError( "--web, --myproxy, and --delegate-proxy are mutually exclusive." ) if no_autoactivate and not (myproxy or web or delegate_proxy): raise click.UsageError( "--no-autoactivate requires another activation method be given." ) if myproxy_username and not myproxy: raise click.UsageError("--myproxy-username requires --myproxy.") if myproxy_password and not myproxy: raise click.UsageError("--myproxy-password requires --myproxy.") # NOTE: "0" is a legitimate, though weird, value # In the case where someone is setting this value programatically, # respecting it behaves more consistently/predictably if myproxy_lifetime is not None and not myproxy: raise click.UsageError("--myproxy-lifetime requires --myproxy.") if no_browser and not web: raise click.UsageError("--no-browser requires --web.") if proxy_lifetime and not delegate_proxy: raise click.UsageError("--proxy-lifetime requires --delegate-proxy.") # check if endpoint is already activated unless --force if not force: res = client.endpoint_autoactivate(endpoint_id, if_expires_in=60) if "AlreadyActivated" == res["code"]: formatted_print( res, simple_text=( "Endpoint is already activated. Activation " "expires at {}".format(res["expire_time"]) ), ) return # attempt autoactivation unless --no-autoactivate if not no_autoactivate: res = client.endpoint_autoactivate(endpoint_id) if "AutoActivated" in res["code"]: formatted_print( res, simple_text=( "Autoactivation succeeded with message: {}".format(res["message"]) ), ) return # override potentially confusing autoactivation failure response else: message = ( "The endpoint could not be auto-activated.\n\n" + activation_requirements_help_text(res, endpoint_id) ) res = {"message": message} # myproxy activation if myproxy: # get username and password if not (myproxy_username or default_myproxy_username): myproxy_username = click.prompt("Myproxy username") if not myproxy_password: myproxy_password = click.prompt("Myproxy password", hide_input=True) no_server_msg = ( "This endpoint has no myproxy server " "and so cannot be activated through myproxy" ) requirements_data = client.endpoint_get_activation_requirements( endpoint_id ).data if not len(requirements_data["DATA"]): raise click.ClickException(no_server_msg) for data in requirements_data["DATA"]: # skip non-myproxy values # although the API does not practice this today, in theory other # activation types may have fields with the same names... if data["type"] != "myproxy": continue if data["name"] == "passphrase": data["value"] = myproxy_password if data["name"] == "username": data["value"] = myproxy_username or default_myproxy_username if data["name"] == "hostname" and data["value"] is None: raise click.ClickException(no_server_msg) # NOTE: remember that "0" is a possible value if data["name"] == "lifetime_in_hours" and myproxy_lifetime is not None: data["value"] = str(myproxy_lifetime) res = client.endpoint_activate(endpoint_id, requirements_data) # web activation elif web: url = "https://app.globus.org/file-manager" "?origin_id={}".format(endpoint_id) if no_browser or is_remote_session(): res = {"message": "Web activation url: {}".format(url), "url": url} else: webbrowser.open(url, new=1) res = {"message": "Browser opened to web activation page", "url": url} # delegate proxy activation elif delegate_proxy: requirements_data = client.endpoint_get_activation_requirements( endpoint_id ).data filled_requirements_data = fill_delegate_proxy_activation_requirements( requirements_data, delegate_proxy, lifetime_hours=proxy_lifetime or 12 ) res = client.endpoint_activate(endpoint_id, filled_requirements_data) # output formatted_print(res, text_format=FORMAT_TEXT_RAW, response_key="message")