def init(): tokens = None try: # if we already have tokens, load and use them tokens = load_tokens_from_db() except: pass if not tokens: # if we need to get tokens, start the Native App authentication process tokens = do_native_app_authentication(config.TRANSFER_CLIENT_ID, config.REDIRECT_URI, config.SCOPES) try: save_tokens_to_db(tokens) except: pass transfer_tokens = tokens['transfer.api.globus.org'] client = NativeAppAuthClient(client_id=config.TRANSFER_CLIENT_ID) authorizer = RefreshTokenAuthorizer( transfer_tokens['refresh_token'], client, access_token=transfer_tokens['access_token'], expires_at=transfer_tokens['expires_at_seconds'], on_refresh=update_tokens_file_on_refresh) transfer = TransferClient(authorizer=authorizer) prepare_call(transfer) return transfer
def refresh_tokens(self, tokens): if not self._refreshable(tokens): raise TokensExpired('No Refresh Token, cannot refresh tokens: ', resource_servers=tokens.keys()) for rs, token_dict in tokens.items(): authorizer = RefreshTokenAuthorizer( token_dict['refresh_token'], self.client, access_token=token_dict['access_token'], expires_at=token_dict['expires_at_seconds'], ) authorizer.check_expiration_time() token_dict['access_token'] = authorizer.access_token token_dict['expires_at_seconds'] = authorizer.expires_at return tokens
def identifiers_client(config, **kwargs): app_name = 'identifier_client' base_url = config.get('client', 'service_url') client_id = config.get('client', 'client_id') access_token = config.get('tokens', 'access_token') at_expires = int(config.get('tokens', 'access_token_expires')) refresh_token = config.get('tokens', 'refresh_token') if not (refresh_token and access_token): raise IdentifierNotLoggedIn("Missing tokens") def _on_refresh(tkn): extract_and_save_tokens(tkn, config) authorizer_client = NativeAppAuthClient(client_id, app_name=app_name) authorizer = RefreshTokenAuthorizer( refresh_token, authorizer_client, access_token, at_expires, on_refresh=_on_refresh, ) return IdentifierClient(base_url=base_url, app_name=app_name, authorizer=authorizer, **kwargs)
def login(self): ''' fetch refresh token, store in dj.config['globus.token'] ''' auth_client = self.auth_client print('Please login via: {}'.format( auth_client.oauth2_get_authorize_url())) code = input('and enter code:').strip() tokens = auth_client.oauth2_exchange_code_for_tokens(code) xfer_auth_cfg = tokens.by_resource_server['transfer.api.globus.org'] xfer_rt = xfer_auth_cfg['refresh_token'] xfer_at = xfer_auth_cfg['access_token'] xfer_exp = xfer_auth_cfg['expires_at_seconds'] xfer_auth = RefreshTokenAuthorizer(xfer_rt, auth_client, access_token=xfer_at, expires_at=xfer_exp) self.xfer_client = TransferClient(authorizer=xfer_auth) custom = dj.config.get('custom', {}) custom['globus.token'] = xfer_rt dj.config['custom'] = custom
def identifier_client(config, **kwargs): app_name = 'identifier_client' base_url = config['client']['service_url'] client_id = config['client']['client_id'] access_token = config['tokens']['access_token'] at_expires = int(config['tokens']['access_token_expires']) refresh_token = config['tokens']['refresh_token'] if not (refresh_token and access_token): raise IdentifierNotLoggedIn("Missing tokens") def _on_refresh(tkn): # extract_and_save_tokens(tkn, config) pass authorizer_client = NativeAppAuthClient(client_id, app_name=app_name) authorizer = RefreshTokenAuthorizer( refresh_token, authorizer_client, access_token, at_expires, on_refresh=_on_refresh, ) return IdentifierClient( "identifier", base_url=base_url, app_name=app_name, authorizer=authorizer, **kwargs )
def getTransferData(): cfg = load_config() client_id = cfg['globus']['apps']['SDK Tutorial App']['client_id'] auth_client = NativeAppAuthClient(client_id) refresh_token = cfg['globus']['apps']['SDK Tutorial App']['refresh_token'] source_endpoint_id = cfg['globus']['apps']['SDK Tutorial App'][ 'win10_endpoint_id'] destination_endpoint_id = cfg['globus']['apps']['SDK Tutorial App'][ 'sdccfed_endpoint_id'] authorizer = RefreshTokenAuthorizer(refresh_token=refresh_token, auth_client=auth_client) tc = TransferClient(authorizer=authorizer) # as both endpoints are expected to be Globus Server endpoints, send auto-activate commands for both globus endpoints auto_activate_endpoint(tc, source_endpoint_id) auto_activate_endpoint(tc, destination_endpoint_id) # make job_label for task a timestamp x = datetime.now() job_label = x.strftime('%Y%m%d%H%M%s') # from Globus... sync_level=checksum means that before files are transferred, Globus will compute checksums on the source and destination files, # and only transfer files that have different checksums are transferred. verify_checksum=True means that after a file is transferred, Globus will # compute checksums on the source and destination files to verify that the file was transferred correctly. If the checksums do not match, it will # redo the transfer of that file. tdata = TransferData(tc, source_endpoint_id, destination_endpoint_id, label=job_label, sync_level="checksum", verify_checksum=True) return tdata
def browse(dataset_id=None, endpoint_id=None, endpoint_path=None): """ - Get list of files for the selected dataset or endpoint ID/path - Return a list of files to a browse view The target template (browse.jinja2) expects an `endpoint_uri` (if available for the endpoint), `target` (either `"dataset"` or `"endpoint"`), and 'file_list' (list of dictionaries) containing the following information about each file in the result: {'name': 'file name', 'size': 'file size', 'id': 'file uri/path'} If you want to display additional information about each file, you must add those keys to the dictionary and modify the browse.jinja2 template accordingly. """ assert bool(dataset_id) != bool(endpoint_id and endpoint_path) if dataset_id: try: dataset = next(ds for ds in datasets if ds['id'] == dataset_id) except StopIteration: abort(404) endpoint_id = app.config['DATASET_ENDPOINT_ID'] endpoint_path = app.config['DATASET_ENDPOINT_BASE'] + dataset['path'] else: endpoint_path = '/' + endpoint_path transfer = TransferClient(authorizer=RefreshTokenAuthorizer( session['tokens']['transfer.api.globus.org']['refresh_token'], load_portal_client())) try: transfer.endpoint_autoactivate(endpoint_id) listing = transfer.operation_ls(endpoint_id, path=endpoint_path) except TransferAPIError as err: flash('Error [{}]: {}'.format(err.code, err.message)) return redirect(url_for('transfer')) file_list = [e for e in listing if e['type'] == 'file'] ep = transfer.get_endpoint(endpoint_id) https_server = ep['https_server'] endpoint_uri = https_server + endpoint_path if https_server else None webapp_xfer = 'https://www.globus.org/app/transfer?' + \ urlencode(dict(origin_id=endpoint_id, origin_path=endpoint_path)) return render_template( 'browse.jinja2', endpoint_uri=endpoint_uri, target="dataset" if dataset_id else "endpoint", description=(dataset['name'] if dataset_id else ep['display_name']), file_list=file_list, webapp_xfer=webapp_xfer)
def refresh(self): ''' use refresh token to refresh access token ''' auth_client = self.auth_client xfer_auth = RefreshTokenAuthorizer( dj.config['custom']['globus.token'], auth_client, access_token=None, expires_at=None) self.xfer_client = TransferClient(authorizer=xfer_auth)
def login(): tok_path = os.path.expanduser('~/.mdf_agent_tokens.json') def _read_tokfile(): tokens = {} if os.path.exists(tok_path): with open(tok_path) as f: tokens = json.load(f) return tokens def _write_tokfile(new_tokens): # We have multiple tokens in our tokens file, but on update we only # get the currently updated token, so read current and update with the # input tokens cur_tokens = _read_tokfile() for key in new_tokens: cur_tokens[key] = new_tokens[key] # deny rwx to Group and World -- don't bother storing the returned old # mask value, since we'll never restore it anyway # do this on every call to ensure that we're always consistent about it os.umask(0o077) with open(tok_path, 'w') as f: f.write(json.dumps(cur_tokens)) def _update_tokfile(tokens): _write_tokfile(tokens.by_resource_server['transfer.api.globus.org']) tokens = _read_tokfile() client_id = "1e162bfc-ad52-4014-8844-b82841145fc4" native_client = NativeAppAuthClient(client_id, app_name='MDF Agents') if not tokens: # and do the Native App Grant flow native_client.oauth2_start_flow( requested_scopes='urn:globus:auth:scope:transfer.api.globus.org:all', refresh_tokens=True) linkprompt = 'Please login to Globus here' print('{0}:\n{1}\n{2}\n{1}\n' .format(linkprompt, '-' * len(linkprompt), native_client.oauth2_get_authorize_url()), flush=True) auth_code = input( 'Enter the resulting Authorization Code here').strip() tkns = native_client.oauth2_exchange_code_for_tokens(auth_code) tokens = tkns.by_resource_server['transfer.api.globus.org'] _write_tokfile(tokens) transfer_tokens = tokens transfer_authorizer = RefreshTokenAuthorizer( transfer_tokens['refresh_token'], native_client, transfer_tokens['access_token'], transfer_tokens['expires_at_seconds'], on_refresh=_update_tokfile) transfer_client = TransferClient(authorizer=transfer_authorizer) return transfer_client
def get_transfer_client(): cfg = load_config() # cfg = yaml.safe_load(open("/opt/rucio/lib/rucio/transfertool/config.yml")) client_id = cfg['globus']['apps'][GLOBUS_AUTH_APP]['client_id'] auth_client = NativeAppAuthClient(client_id) refresh_token = cfg['globus']['apps'][GLOBUS_AUTH_APP]['refresh_token'] logging.info('authorizing token...') authorizer = RefreshTokenAuthorizer(refresh_token=refresh_token, auth_client=auth_client) logging.info('initializing TransferClient...') tc = TransferClient(authorizer=authorizer) return tc
def bulk_submit_xfer(submitjob, recursive=False): cfg = load_config() client_id = cfg['globus']['apps'][GLOBUS_AUTH_APP]['client_id'] auth_client = NativeAppAuthClient(client_id) refresh_token = cfg['globus']['apps'][GLOBUS_AUTH_APP]['refresh_token'] source_endpoint_id = submitjob[0].get('metadata').get( 'source_globus_endpoint_id') destination_endpoint_id = submitjob[0].get('metadata').get( 'dest_globus_endpoint_id') authorizer = RefreshTokenAuthorizer(refresh_token=refresh_token, auth_client=auth_client) tc = TransferClient(authorizer=authorizer) # as both endpoints are expected to be Globus Server endpoints, send auto-activate commands for both globus endpoints a = auto_activate_endpoint(tc, source_endpoint_id) logging.debug('a: %s' % a) if a != 'AlreadyActivated': return None b = auto_activate_endpoint(tc, destination_endpoint_id) logging.debug('b: %s' % b) if b != 'AlreadyActivated': return None # make job_label for task a timestamp x = datetime.now() job_label = x.strftime('%Y%m%d%H%M%s') # from Globus... sync_level=checksum means that before files are transferred, Globus will compute checksums on the source # and destination files, and only transfer files that have different checksums are transferred. verify_checksum=True means # that after a file is transferred, Globus will compute checksums on the source and destination files to verify that the # file was transferred correctly. If the checksums do not match, it will redo the transfer of that file. tdata = TransferData(tc, source_endpoint_id, destination_endpoint_id, label=job_label, sync_level="checksum") for file in submitjob: source_path = file.get('sources')[0] dest_path = file.get('destinations')[0] filesize = file['metadata']['filesize'] # TODO: support passing a recursive parameter to Globus # md5 = file['metadata']['md5'] # tdata.add_item(source_path, dest_path, recursive=False, external_checksum=md5) tdata.add_item(source_path, dest_path, recursive=False) record_counter( 'daemons.conveyor.transfer_submitter.globus.transfers.submit.filesize', filesize) # logging.info('submitting transfer...') transfer_result = tc.submit_transfer(tdata) # logging.info("task_id =", transfer_result["task_id"]) return transfer_result["task_id"]
def submit_transfer(): """ - Take the data returned by the Browse Endpoint helper page and make a Globus transfer request. - Send the user to the transfer status page with the task id from the transfer. """ browse_endpoint_form = request.form selected = session['form']['datasets'] filtered_datasets = [ds for ds in datasets if ds['id'] in selected] transfer_tokens = session['tokens']['transfer.api.globus.org'] authorizer = RefreshTokenAuthorizer( transfer_tokens['refresh_token'], load_portal_client(), access_token=transfer_tokens['access_token'], expires_at=transfer_tokens['expires_at_seconds']) transfer = TransferClient(authorizer=authorizer) source_endpoint_id = app.config['DATASET_ENDPOINT_ID'] source_endpoint_base = app.config['DATASET_ENDPOINT_BASE'] destination_endpoint_id = browse_endpoint_form['endpoint_id'] destination_folder = browse_endpoint_form.get('folder[0]') transfer_data = TransferData(transfer_client=transfer, source_endpoint=source_endpoint_id, destination_endpoint=destination_endpoint_id, label=browse_endpoint_form.get('label')) for ds in filtered_datasets: source_path = source_endpoint_base + ds['path'] dest_path = browse_endpoint_form['path'] if destination_folder: dest_path += destination_folder + '/' dest_path += ds['name'] + '/' transfer_data.add_item(source_path=source_path, destination_path=dest_path, recursive=True) transfer.endpoint_autoactivate(source_endpoint_id) transfer.endpoint_autoactivate(destination_endpoint_id) task_id = transfer.submit_transfer(transfer_data)['task_id'] flash('Transfer request submitted successfully. Task ID: ' + task_id) return (redirect(url_for('transfer_status', task_id=task_id)))
def bulk_submit_xfer(submitjob, recursive=False, logger=logging.log): cfg = load_config(logger=logger) client_id = cfg['globus']['apps'][GLOBUS_AUTH_APP]['client_id'] auth_client = NativeAppAuthClient(client_id) refresh_token = cfg['globus']['apps'][GLOBUS_AUTH_APP]['refresh_token'] source_endpoint_id = submitjob[0].get('metadata').get( 'source_globus_endpoint_id') destination_endpoint_id = submitjob[0].get('metadata').get( 'dest_globus_endpoint_id') authorizer = RefreshTokenAuthorizer(refresh_token=refresh_token, auth_client=auth_client) tc = TransferClient(authorizer=authorizer) # make job_label for task a timestamp now = datetime.datetime.now() job_label = now.strftime('%Y%m%d%H%M%s') # retrieve globus_task_deadline value to enforce time window to complete transfers # default is 2880 minutes or 48 hours globus_task_deadline = config_get_int('conveyor', 'globus_task_deadline', False, 2880) deadline = now + datetime.timedelta(minutes=globus_task_deadline) # from Globus... sync_level=checksum means that before files are transferred, Globus will compute checksums on the source # and destination files, and only transfer files that have different checksums are transferred. verify_checksum=True means # that after a file is transferred, Globus will compute checksums on the source and destination files to verify that the # file was transferred correctly. If the checksums do not match, it will redo the transfer of that file. tdata = TransferData(tc, source_endpoint_id, destination_endpoint_id, label=job_label, sync_level="checksum", deadline=str(deadline)) for file in submitjob: source_path = file.get('sources')[0] dest_path = file.get('destinations')[0] filesize = file['metadata']['filesize'] # TODO: support passing a recursive parameter to Globus # md5 = file['metadata']['md5'] # tdata.add_item(source_path, dest_path, recursive=False, external_checksum=md5) tdata.add_item(source_path, dest_path, recursive=False) record_counter( 'daemons.conveyor.transfer_submitter.globus.transfers.submit.filesize', filesize) # logging.info('submitting transfer...') transfer_result = tc.submit_transfer(tdata) logger(logging.INFO, "transfer_result: %s" % transfer_result) return transfer_result["task_id"]
def get_auth_client(): tokens = get_auth_tokens() authorizer = None # if there's a refresh token, use it to build the authorizer if tokens['refresh_token'] is not None: authorizer = RefreshTokenAuthorizer(tokens['refresh_token'], internal_auth_client(), tokens['access_token'], tokens['access_token_expires'], on_refresh=_update_access_tokens) client = AuthClient(authorizer=authorizer, app_name=version.app_name) return client
def get_client(): tokens = get_transfer_tokens() authorizer = None # if there's a refresh token, use it to build the authorizer if tokens['refresh_token'] is not None: authorizer = RefreshTokenAuthorizer(tokens['refresh_token'], internal_auth_client(), tokens['access_token'], tokens['access_token_expires'], on_refresh=_update_access_tokens) return RetryingTransferClient(tries=10, authorizer=authorizer, app_name=version.app_name)
def get_refresh_authorizer(tokens): """ Retrieve a refresh token authorizer from the provided tokens. """ authorizer = None if tokens['refresh_token'] is not None: authorizer = RefreshTokenAuthorizer(tokens['refresh_token'], internal_auth_client(), on_refresh=_update_access_tokens) else: raise RuntimeError('No refresh token found. Please login first.') return authorizer
def get_authorizers(self): authorizers = {} for resource_server, token_dict in self.load_tokens().items(): if token_dict.get('refresh_token') is not None: authorizers[resource_server] = RefreshTokenAuthorizer( token_dict['refresh_token'], self.client, access_token=token_dict['access_token'], expires_at=token_dict['expires_at_seconds'], on_refresh=self.on_refresh, ) else: authorizers[resource_server] = AccessTokenAuthorizer( token_dict['access_token']) return authorizers
def getAuthorizer(self, user): if 'otherTokens' not in user: raise Exception('No transfer token found') tokens = user['otherTokens'] for token in tokens: if token['scope'] == _TRANSFER_SCOPE: if 'refresh_token' in token and token[ 'refresh_token'] is not None: return RefreshTokenAuthorizer(token['refresh_token'], self.getAuthClient()) else: return AccessTokenAuthorizer(token['access_token']) raise Exception('No globus transfer token found')
def transfer_status(task_id): """ Call Globus to get status/details of transfer with task_id. The target template (tranfer_status.jinja2) expects a Transfer API 'task' object. 'task_id' is passed to the route in the URL as 'task_id'. """ transfer = TransferClient(authorizer=RefreshTokenAuthorizer( session['tokens']['transfer.api.globus.org']['refresh_token'], load_portal_client())) task = transfer.get_task(task_id) return render_template('transfer_status.jinja2', task=task)
def __init__(self): """Initiate an OAuth2() object. Initiate OAuth2 flow with Globus credentaials to obtain access tokens. Refresh the tokens automatically so another login is not required. Examples -------- Create an OAuth2 object: >>> from archeion.models import OAuth2 >>> authorizer = OAuth2() """ self.client = NativeAppAuthClient(CLIENT_ID) self.client.oauth2_start_flow(refresh_tokens=True) logger.info("Opening browser window for Globus Authentication") webbrowser.open_new(self.client.oauth2_get_authorize_url()) get_input = getattr(__builtins__, "raw_input", input) auth_code = get_input( "Please enter the code you get after login here: " ).strip() logger.debug("User has input authentication code") token_response = self.client.oauth2_exchange_code_for_tokens(auth_code) self.access_token = token_response.by_resource_server["auth.globus.org"][ "access_token" ] transfer_response = token_response.by_resource_server["transfer.api.globus.org"] self.transfer_token = transfer_response["access_token"] self.transfer_refresh_token = transfer_response["refresh_token"] self.transfer_expiry_seconds = transfer_response["expires_at_seconds"] authorizer = RefreshTokenAuthorizer( self.transfer_refresh_token, self.client, access_token=self.transfer_token, expires_at=self.transfer_expiry_seconds, ) self.transfer_client = TransferClient( AccessTokenAuthorizer(self.transfer_token) ) self.authorisation_client = AuthClient(authorizer=authorizer)
def create_globus_transfer_client(tmpLog, globus_client_id, globus_refresh_token): """ create Globus Transfer Client and return the transfer client """ # get logger tmpLog.info('Creating instance of GlobusTransferClient') # start the Native App authentication process # use the refresh token to get authorizer # create the Globus Transfer Client tc = None ErrStat = True try: client = NativeAppAuthClient(client_id=globus_client_id) authorizer = RefreshTokenAuthorizer(refresh_token=globus_refresh_token, auth_client=client) tc = TransferClient(authorizer=authorizer) except: errStat, errMsg = handle_globus_exception(tmpLog) return ErrStat, tc
def setup_transfer_client(transfer_tokens): authorizer = RefreshTokenAuthorizer( transfer_tokens['refresh_token'], NativeAppAuthClient(client_id=CLIENT_ID), access_token=transfer_tokens['access_token'], expires_at=transfer_tokens['expires_at_seconds']) transfer_client = TransferClient(authorizer=authorizer) try: transfer_client.endpoint_autoactivate(SOURCE_ENDPOINT) transfer_client.endpoint_autoactivate(DESTINATION_ENDPOINT) except GlobusAPIError as ex: if ex.http_status == 401: sys.exit('Refresh token has expired. ' 'Please delete the `tokens` object from ' '{} and try again.'.format(DATA_FILE)) else: raise ex return transfer_client
def setup_transfer_client(transfer_tokens, source_endpoint, destination_endpoint): authorizer = RefreshTokenAuthorizer( transfer_tokens['refresh_token'], NativeAppAuthClient(client_id=CLIENT_ID), access_token=transfer_tokens['access_token'], expires_at=transfer_tokens['expires_at_seconds'], on_refresh=update_tokens_file_on_refresh) transfer_client = TransferClient(authorizer=authorizer) try: transfer_client.endpoint_autoactivate(source_endpoint) transfer_client.endpoint_autoactivate(destination_endpoint) except GlobusAPIError as ex: if ex.http_status == 401: sys.exit('Refresh token has expired. ' 'Please delete the `tokens` object from ' '{} and try again.'.format(DATA_FILE)) else: raise ex return transfer_client
def make_authorizer(): """ Make a RefreshTokenAuthorizer given the tokens stored on disk Returns: (RefreshTokenAuthorizer): Tool to generate authorization credentials """ if not check_logged_in(): safeprint("No authorization credentials present. You must log in") do_login_flow() # Get the authorization client auth_client = internal_auth_client() # Get the tokens needed by the service rf_token = lookup_option(FUNCX_RT_OPTNAME) at_token = lookup_option(FUNCX_AT_OPTNAME) at_expires = int(lookup_option(FUNCX_AT_EXPIRES_OPTNAME)) authorizer = RefreshTokenAuthorizer(rf_token, auth_client, access_token=at_token, expires_at=at_expires) return authorizer
def __init__(self, src_endpoint_name, dst_endpoint_name, transfer_rt=None, log_lv=logging.INFO): log_format = '%(asctime)-15s %(levelname)s:\t class:%(name)s %(message)s' logging.basicConfig(format=log_format) self.logger = logging.getLogger(self.__class__.__name__) self.logger.setLevel(log_lv) self.logger.debug('CLIENT_ID: {0}'.format(CLIENT_ID)) self.client = NativeAppAuthClient(CLIENT_ID) self.client.oauth2_start_flow(refresh_tokens=True) if transfer_rt is not None: self.authorizer = RefreshTokenAuthorizer(transfer_rt, self.client) else: authorize_url = self.client.oauth2_get_authorize_url() print('Please go to this URL and login: {0}'.format(authorize_url)) get_input = getattr(__builtins__, 'raw_input', input) auth_code = get_input( 'Please enter the code you get after login here: ').strip() token_response = self.client.oauth2_exchange_code_for_tokens( auth_code) self.globus_auth_data = token_response.by_resource_server[ 'auth.globus.org'] self.globus_transfer_data = token_response.by_resource_server[ 'transfer.api.globus.org'] auth_token = self.globus_auth_data['access_token'] transfer_token = self.globus_transfer_data['access_token'] transfer_rt = self.globus_transfer_data['refresh_token'] transfer_at = self.globus_transfer_data['access_token'] expires_at_s = self.globus_transfer_data['expires_at_seconds'] self.authorizer = RefreshTokenAuthorizer(transfer_rt, self.client, access_token=transfer_at, expires_at=expires_at_s) self.transferClient = TransferClient(authorizer=self.authorizer) self.src_endpoint = None self.dst_endpoint = None for ep in self.transferClient.endpoint_search( filter_scope="shared-with-me"): if ep["display_name"] == src_endpoint_name: self.src_endpoint = ep self.logger.info('Source endpoint: [{0}] {1}'.format( self.src_endpoint['id'], self.src_endpoint['display_name'])) if self.src_endpoint is None: self.logger.error( 'No endpoint shared with you with name: {0}'.format( src_endpoint_name)) raise LookupError for ep in self.transferClient.endpoint_search( filter_scope="my-endpoints"): if ep['display_name'] == dst_endpoint_name: self.dst_endpoint = ep self.logger.info('Destination endpoint: [{0}] {1}'.format( self.dst_endpoint['id'], self.dst_endpoint['display_name'])) if self.dst_endpoint is None: self.logger.error('You don\'t have endpoint named: {0}'.format( dst_endpoint_name)) raise LookupError
def initiate_connection(self): """ Initiate the connection """ tokens = None try: # if we already have tokens, load and use them tokens = self.load_tokens_from_file() except: pass if not tokens: # if we need to get tokens, start the Native App authentication process client = NativeAppAuthClient( client_id=self.CLIENT_ID) #self.config['client_id']) # pass refresh_tokens=True to request refresh tokens client.oauth2_start_flow( requested_scopes=self. SCOPES, #self.config['requested_scopes'], redirect_uri=self.REDIRECT_URI, #self.config['redirect_uri'], refresh_tokens=True) url = client.oauth2_get_authorize_url() print 'Native App Authorization URL: \n{}'.format(url) auth_code = raw_input('Enter the auth code: ').strip() token_response = client.oauth2_exchange_code_for_tokens(auth_code) # return a set of tokens, organized by resource server name tokens = token_response.by_resource_server try: self.save_tokens_to_file(tokens) except: pass transfer_tokens = tokens['transfer.api.globus.org'] auth_client = NativeAppAuthClient(client_id=self.config['client_id']) authorizer = RefreshTokenAuthorizer( transfer_tokens['refresh_token'], auth_client, #access_token=transfer_tokens['access_token'], #expires_at=transfer_tokens['expires_at_seconds'], on_refresh=self.update_tokens_file_on_refresh) self.client = TransferClient(authorizer=authorizer) # print out a directory listing from an endpoint try: #print 'ACTIVATE' #print 'DEST',self.config['dest_ep'] self.client.endpoint_autoactivate(self.config['dest_ep']) ac = self.client.endpoint_get_activation_requirements( self.config['dest_ep']) #print ac self.client.endpoint_autoactivate(self.config['src_ep']) ac2 = self.client.endpoint_get_activation_requirements( self.config['src_ep']) #print ac2 except GlobusAPIError as ex: self.logger.error('Error in endpoint activation %s', str(ex)) if ex.http_status == 401: sys.exit('Refresh token has expired. ' 'Please delete refresh-tokens.json and try again.') else: raise ex
def transfer(sp,destination_endpoint_id,one_endpoint): tokens = None try: # if we already have tokens, load and use them tokens = load_tokens_from_file(TOKEN_FILE) except: pass if not tokens: # if we need to get tokens, start the Native App authentication process tokens = do_native_app_authentication(CLIENT_ID, REDIRECT_URI, SCOPES) try: save_tokens_to_file(TOKEN_FILE, tokens) except: pass transfer_tokens = tokens['transfer.api.globus.org'] auth_client = NativeAppAuthClient(client_id=CLIENT_ID,environment='sandbox') authorizer = RefreshTokenAuthorizer( transfer_tokens['refresh_token'], auth_client, access_token=transfer_tokens['access_token'], expires_at=transfer_tokens['expires_at_seconds'], on_refresh=update_tokens_file_on_refresh) #transfer = TransferClient(authorizer=authorizer,environment='sandbox') tc = TransferClient(authorizer=authorizer, environment="sandbox") ##################---ENDPOINTS---########################### source_endpoint_id = '5a2e5704-b028-11e7-bdad-22000bdb2406' #sb vmtb4 #source_endpoint_id = '55705028-aa15-11e7-bdad-22000bdb2406' #sb yulie7t #source_endpoint_id = 'b0b16296-88e7-11e7-a971-22000a92523b' #bare chameleon #source_endpoint_id = 'e5762bc2-8466-11e7-a8ed-22000a92523b' #large_chameleon #source_endpoint_id = '8b26cc0e-877b-11e7-a949-22000a92523b'#ubuntu-vm #source_endpoint_id = 'ad19b012-77cf-11e7-8b98-22000b9923ef'#chameleon # source_endpoint_id = raw_input('Input source endpoint UUID: ') #destination path ##############---SOURCE PATH---###################### #source_path = '/home/parallels/stream_transfer/test_files/' #source_path = '/home/parallels/stream_transfer/zero_globus/test_files/' source_path = sp #source_path ='/home/cc/streaming/zero_globus/test_files/test.txt' #source_path = '/home/parallels/stream_transfer/zero_globus/test_files/test.txt' #destination path destination_path = '/~/' #destination_path = '/~/'+ sp.split("/")[-1] #use for one file ##if one_endpoint: ## destination_path = '/projects/BrainImagingADSP/yzamora/' ##else: ## destination_path = '/projects/BrainImagingADSP/yzamora/'+ sp.split("/")[-1] #use for one file #Using my sample UUID from globus tutorial #destination_endpoint_id = 'ddb59aef-6d04-11e5-ba46-22000b92c6ec' #globus #destination_endpoint_id = '5d1da0fe-3c07-11e7-bcfc-22000b9a448b' #laptop #tc.endpoint_autoactivate(source_endpoint_id) #tc.endpoint_autoactive(destination_endpoint_id) ep1 = tc.get_endpoint(destination_endpoint_id) tc.endpoint_autoactivate(destination_endpoint_id) #ep1 is setting the activated endpoint to be a variable to work with tc.endpoint_autoactivate(source_endpoint_id) label = "medium data transfer" #tdata = globus_sdk.TransferData(tc, source_endpoint_id, destination_endpoint_id,label=label, sync_level='0') tdata = globus_sdk.TransferData(tc, source_endpoint_id, destination_endpoint_id,label=label, perf_cc=3, sync_level=None, verify_checksum=False) #tdata = globus_sdk.TransferData(tc, source_endpoint_id, destination_endpoint_id,label=label) if one_endpoint: tdata.add_item(source_path,destination_path,recursive=True) else: tdata.add_item(source_path,destination_path,recursive=False) submit_result = tc.submit_transfer(tdata) print("Task ID:", submit_result["task_id"]) """ Checking for time completion using globus calls """ #print("Completion time:", submit_result["completion_time"]) #setup of the transfer, submits as a https post request #transfer_data = TransferData(transfer_client=tc, # source_endpoint=source_endpoint_id, # destination_endpoint=destination_endpoint_id, # label='Transfer', # sync_level='checksum') #transfer_data.add_item(source_path=source_path,destination_path=destination_path, recursive=False) #task_id=transfer.submit_transfer(transfer_data)['task_id'] #waiting for file to transfer status = tc.get_task(submit_result["task_id"],fields="status")["status"] poll_interval = 2 max_wait = 90 wait_time = 0 while not (status in ["SUCCEEDED", "FAILED"]): if (wait_time >= max_wait): break print("Task not yet complete (status {}), sleeping for {} seconds..." \ .format(status, poll_interval)) time.sleep(poll_interval) wait_time += poll_interval status = tc.get_task(submit_result["task_id"], fields="status")["status"] if status == "FAILED": print("WARNING! File transfer FAILED!") #deleting file after transfer if status == "SUCCEEDED": end_time = datetime.datetime.utcnow() start_time = end_time - datetime.timedelta(minutes=200) #limit = response objects # data = tc.task_list(filter="type:TRANSFER,DELETE/request_time:%s,%s" # % (start_time, end_time), limit=5) #print("File transfer SUCCEEDED, will delete file from local directory now") """ r = tc.task_list(num_results=1, filter="type:TRANSFER,DELETE")
def __init__(self): self.log = logging.getLogger(self.__class__.__name__) self.log.info("init - started") config = configparser.ConfigParser() config.read(str(CONFIG_PATH)) # To set up a client_id, see https://auth.globus.org/v2/web/developers # Current client_id is in Project: MaterialsCommonsProject, App: MaterialsCommonsTest client_id = config['sdk']['id'] auth_tokens = None transfer_tokens = None tokens = None try: # if we already have tokens, load and use them tokens = self._load_tokens_from_file(TOKEN_FILE_PATH) except IOError: pass if not tokens: # if we need to get tokens, start the Native App authentication process tokens = self.do_native_app_authentication(client_id, REDIRECT_URI, SCOPES) try: self._save_tokens_to_file(TOKEN_FILE_PATH, tokens) except IOError: pass try: auth_tokens = tokens['auth.globus.org'] transfer_tokens = tokens['transfer.api.globus.org'] except KeyError as er: self.log.error( "KeyError on NativeApp tokens: {}\n delete {} and restart". format(er, TOKEN_FILE_PATH)) def refresh_tokens(token_response): context = self context._update_tokens_file_on_refresh(token_response) auth_client = NativeAppAuthClient(client_id=client_id) authorizer = RefreshTokenAuthorizer( auth_tokens['refresh_token'], auth_client, access_token=auth_tokens['access_token'], expires_at=auth_tokens['expires_at_seconds'], on_refresh=refresh_tokens) auth_client = AuthClient(client_id=client_id, authorizer=authorizer) authorizer = RefreshTokenAuthorizer( transfer_tokens['refresh_token'], auth_client, access_token=transfer_tokens['access_token'], expires_at=transfer_tokens['expires_at_seconds'], on_refresh=refresh_tokens) transfer_client = TransferClient(authorizer=authorizer) self._auth_client = auth_client self._transfer_client = transfer_client
.format(TOKEN_FILE)) sys.exit(-1) transfer_tokens = tokens['transfer.api.globus.org'] try: auth_client = NativeAppAuthClient(client_id=dcde_parsl_client_id) except: print( "ERROR: Globus NativeAppAuthClient() call failed! Unable to obtain a Globus authorizer!" ) sys.exit(-1) authorizer = RefreshTokenAuthorizer( transfer_tokens['refresh_token'], auth_client, access_token=transfer_tokens['access_token'], expires_at=transfer_tokens['expires_at_seconds'], on_refresh=update_tokens_file_on_refresh) try: tc = TransferClient(authorizer=authorizer) except: print( "ERROR: TransferClient() call failed! Unable to call the Globus transfer interface with the provided auth info!" ) sys.exit(-1) # print(transfer) # Now we should have auth, try setting up a transfer. tdata = TransferData(tc,
def browse(dataset_id=None, endpoint_id=None, endpoint_path=None): """ - Get list of files for the selected dataset or endpoint ID/path - Return a list of files to a browse view The target template (browse.jinja2) expects an `endpoint_uri` (if available for the endpoint), `target` (either `"dataset"` or `"endpoint"`), and 'file_list' (list of dictionaries) containing the following information about each file in the result: {'name': 'file name', 'size': 'file size', 'id': 'file uri/path'} If you want to display additional information about each file, you must add those keys to the dictionary and modify the browse.jinja2 template accordingly. """ if request.method == 'GET': assert bool(dataset_id) != bool(endpoint_id and endpoint_path) if dataset_id: try: dataset = next(ds for ds in datasets if ds['id'] == dataset_id) except StopIteration: abort(404) endpoint_id = app.config['DATASET_ENDPOINT_ID'] endpoint_path = app.config['DATASET_ENDPOINT_BASE'] + dataset['path'] else: endpoint_path = '/' + endpoint_path transfer_tokens = session['tokens']['transfer.api.globus.org'] authorizer = RefreshTokenAuthorizer( transfer_tokens['refresh_token'], load_portal_client(), access_token=transfer_tokens['access_token'], expires_at=transfer_tokens['expires_at_seconds']) transfer = TransferClient(authorizer=authorizer) try: transfer.endpoint_autoactivate(endpoint_id) listing = transfer.operation_ls(endpoint_id, path=endpoint_path) except TransferAPIError as err: flash('Error [{}]: {}'.format(err.code, err.message)) return redirect(url_for('browse')) file_list = [e for e in listing if e['type'] == 'file'] ep = transfer.get_endpoint(endpoint_id) https_server = ep['https_server'] endpoint_uri = https_server + endpoint_path if https_server else None webapp_xfer = 'https://app.globus.org/file-manager?' + \ urlencode(dict(origin_id=endpoint_id, origin_path=endpoint_path)) #print("endpintURL == " + endpoint_uri) return render_template('browse.jinja2', endpoint_uri=endpoint_uri, target="dataset" if dataset_id else "endpoint", description=(dataset['name'] if dataset_id else ep['display_name']), mypath=(dataset['path'] if dataset_id else None), myid=(dataset['id'] if dataset_id else None), file_list=file_list, webapp_xfer=webapp_xfer) if request.method == 'POST': if not request.form.get('file'): flash('Please select at least one file.') return redirect(url_for('browse')) params = { 'method': 'POST', 'action': url_for('submit_transfer', _external=True, _scheme='https'), 'filelimit': 0, 'folderlimit': 1 } browse_endpoint = 'https://app.globus.org/file-manager?{}' \ .format(urlencode(params)) session['form'] = { 'dirselect': False, 'datasets': request.form.getlist('file'), 'path': request.form.getlist('path'), 'id': request.form.getlist('id') } return redirect(browse_endpoint)