def _get_required_data_access_scopes( tc: TransferClient, collection_ids: Iterable[str], ) -> List[str]: data_access_scopes: List[str] = [] for collection_id in collection_ids: collection_id_info = tc.get_endpoint(collection_id) if collection_id_info["DATA_TYPE"] == "endpoint": gcs_version = collection_id_info.get("gcs_version") if gcs_version is None: continue gcs_version_parts = [int(x) for x in gcs_version.split(".")] requires_data_access = all( [ gcs_version_parts[0] > 5 or gcs_version_parts[0] == 5 and gcs_version_parts[1] >= 4, (collection_id_info.get("high_assurance", True) is False), collection_id_info.get("host_endpoint", True) is None, ] ) if requires_data_access: data_access_scopes.append( f"https://auth.globus.org/scopes/{collection_id}/data_access" ) return data_access_scopes
class GridFTPConnection: def __init__(self,endpoint_id,source_id): self.transfer_client = TransferClient() self.endpoint_id = endpoint_id self.source_id = source_id self.endpoint = self.transfer_client.get_endpoint(endpoint_id) print("Connection successful to:", self.endpoint["display_name"] or self.endpoint["canonical_name"])
def browse(dataset_id=None, endpoint_id=None, endpoint_path=None): """ - Get list of files for the selected dataset or endpoint ID/path - Return a list of files to a browse view The target template (browse.jinja2) expects an `endpoint_uri` (if available for the endpoint), `target` (either `"dataset"` or `"endpoint"`), and 'file_list' (list of dictionaries) containing the following information about each file in the result: {'name': 'file name', 'size': 'file size', 'id': 'file uri/path'} If you want to display additional information about each file, you must add those keys to the dictionary and modify the browse.jinja2 template accordingly. """ assert bool(dataset_id) != bool(endpoint_id and endpoint_path) if dataset_id: try: dataset = next(ds for ds in datasets if ds['id'] == dataset_id) except StopIteration: abort(404) endpoint_id = app.config['DATASET_ENDPOINT_ID'] endpoint_path = app.config['DATASET_ENDPOINT_BASE'] + dataset['path'] else: endpoint_path = '/' + endpoint_path transfer = TransferClient(authorizer=RefreshTokenAuthorizer( session['tokens']['transfer.api.globus.org']['refresh_token'], load_portal_client())) try: transfer.endpoint_autoactivate(endpoint_id) listing = transfer.operation_ls(endpoint_id, path=endpoint_path) except TransferAPIError as err: flash('Error [{}]: {}'.format(err.code, err.message)) return redirect(url_for('transfer')) file_list = [e for e in listing if e['type'] == 'file'] ep = transfer.get_endpoint(endpoint_id) https_server = ep['https_server'] endpoint_uri = https_server + endpoint_path if https_server else None webapp_xfer = 'https://www.globus.org/app/transfer?' + \ urlencode(dict(origin_id=endpoint_id, origin_path=endpoint_path)) return render_template( 'browse.jinja2', endpoint_uri=endpoint_uri, target="dataset" if dataset_id else "endpoint", description=(dataset['name'] if dataset_id else ep['display_name']), file_list=file_list, webapp_xfer=webapp_xfer)
def endpoints_not_activated(transfer_client: TransferClient, endpoints: List[str]) -> List[str]: """ Filter list of endpoint UUIDs, returning unactivated ones. Exit 1 if transfer responds with an error trying to look up endpoints. """ result = [] for endpoint in endpoints: try: if not transfer_client.get_endpoint(endpoint).get("activated"): result.append(endpoint) except GlobusError as e: click.echo( f"couldn't get information for endpoint {endpoint}:" f" {e.code}, {e.message}", err=True, ) sys.exit(1) return result
def authcallback(): def test_func(token_data): print('this is a test') """Handles the interaction with Globus Auth.""" # If we're coming back from Globus Auth in an error state, the error # will be in the "error" query string parameter. if 'error' in request.args: print('error') return redirect(url_for('task_server.transfer')) # Set up our Globus Auth/OAuth2 state redirect_uri = url_for('task_server.authcallback', _external=True) client = globus_sdk.ConfidentialAppAuthClient( app.config['TS_CLIENT_ID'], app.config['TS_CLIENT_SECRET']) client.oauth2_start_flow(redirect_uri, refresh_tokens=True) #let globus know where to go back to # If there's no "code" query string parameter, we're in this route # starting a Globus Auth login flow. if 'code' not in request.args: additional_authorize_params = ( {'signup': 1} if request.args.get('signup') else {}) auth_uri = client.oauth2_get_authorize_url( additional_params=additional_authorize_params) return redirect(auth_uri, code=307) else: # If we do have a "code" param, we're coming back from Globus Auth # and can start the process of exchanging an auth code for a token. #requests.post('http://localhost:8081/api/messenger', headers={'content-type': 'application/json'},data=json.dumps({'message':'User logged in....'})) code = request.args.get('code') tokens = client.oauth2_exchange_code_for_tokens(code) id_token = tokens.decode_id_token(client) session.update( tokens=tokens.by_resource_server, is_authenticated=True, name=id_token.get('name', ''), email=id_token.get('email', ''), institution=id_token.get('institution', ''), primary_username=id_token.get('preferred_username'), primary_identity=id_token.get('sub'), ) tokens = session['tokens'] stage_in_source = redis_store.get('stage_in_source').decode('utf-8') stage_in_destination = redis_store.get('stage_in_destination').decode('utf-8') stage_out_destination = redis_store.get('stage_out_destination').decode('utf-8') new_token=None authorizer = globus_sdk.RefreshTokenAuthorizer(tokens['transfer.api.globus.org']['refresh_token'],globus_sdk.ConfidentialAppAuthClient(app.config['TS_CLIENT_ID'], app.config['TS_CLIENT_SECRET']),tokens['transfer.api.globus.org']['access_token'],expires_at=1,on_refresh=test_func) #authorizer = globus_sdk.AccessTokenAuthorizer(tokens['transfer.api.globus.org']['access_token']) tc = TransferClient(authorizer=authorizer) a = tc.endpoint_autoactivate(stage_in_source, if_expires_in=3600) b = tc.endpoint_autoactivate(stage_in_destination, if_expires_in=3600) c = tc.endpoint_autoactivate(stage_out_destination, if_expires_in=3600) if a["code"] == "AutoActivationFailed" or b["code"] == "AutoActivationFailed" or c["code"] == "AutoActivationFailed": stage_in_source_response = tc.get_endpoint(stage_in_source) stage_in_destination_response = tc.get_endpoint(stage_in_destination) stage_out_destination_response = tc.get_endpoint(stage_out_destination) e1name = stage_in_source_response["display_name"] e2name = stage_in_destination_response["display_name"] e3name = stage_out_destination_response["display_name"] return redirect("http://localhost:8080/activate?e1id="+ (stage_in_source if a["code"] == "AutoActivationFailed" else "")+("&e1name="+e1name if a["code"] == "AutoActivationFailed" else "")+"&e2id="+ (stage_in_destination if b["code"] == "AutoActivationFailed" else "")+("&e2name="+e2name if a["code"] == "AutoActivationFailed" else "" )+"&e3id="+(stage_out_destination if c["code"] == "AutoActivationFailed" else "")+"e3id&e3name="+(e3name if c["code"] == "AutoActivationFailed" else "")) # read from db here sis=redis_store.get('stage_in_source').decode('utf-8') sid=redis_store.get('stage_in_destination').decode('utf-8') #... sod=redis_store.get('stage_out_destination').decode('utf-8')#those names........... siop= redis_store.get('stage_in_source_path').decode('utf-8') sidp = redis_store.get('stage_in_destination_path').decode('utf-8') sodp = redis_store.get('stage_out_destination_path').decode('utf-8') return redirect(url_for('task_server.transfer', stage_in_source=sis,stage_in_dest=sid,stage_out_dest=sod,stage_in_source_path=siop,stage_in_dest_path=sidp,stage_out_dest_path=sodp))
def doit(): """ - Call token introspect - Get dependent tokens """ dependent_tokens = get_dependent_tokens(g.req_token) # dependent_tokens is a token response object # create transfer_token and http_token variables containing # the correct token for each resource server transfer_token = dependent_tokens.by_resource_server[ 'transfer.api.globus.org']['access_token'] http_token = dependent_tokens.by_resource_server[ 'tutorial-https-endpoint.globus.org']['access_token'] selected_ids = request.form.getlist('datasets') selected_year = request.form.get('year') user_identity_id = request.form.get('user_identity_id') user_identity_name = request.form.get('user_identity_name') selected_datasets = [ dataset for dataset in datasets if dataset['id'] in selected_ids ] if not (selected_datasets and selected_year): raise BadRequestError() transfer = TransferClient(authorizer=AccessTokenAuthorizer(transfer_token)) source_ep = app.config['DATASET_ENDPOINT_ID'] source_info = transfer.get_endpoint(source_ep) source_https = source_info['https_server'] source_base = app.config['DATASET_ENDPOINT_BASE'] source_token = http_token dest_ep = app.config['GRAPH_ENDPOINT_ID'] dest_info = transfer.get_endpoint(dest_ep) dest_https = dest_info['https_server'] dest_base = app.config['GRAPH_ENDPOINT_BASE'] dest_path = '%sGraphs for %s/' % (dest_base, user_identity_name) dest_token = http_token if not (source_https and dest_https): raise InternalServerError(message='Endpoints must be HTTPS servers') svgs = {} for dataset in selected_datasets: source_path = dataset['path'] response = requests.get( '%s%s%s/%s.csv' % (source_https, source_base, source_path, selected_year), headers=dict(Authorization='Bearer ' + source_token), allow_redirects=False) response.raise_for_status() svgs.update( render_graphs( csv_data=response.iter_lines(decode_unicode=True), append_titles=' from %s for %s' % (dataset['name'], selected_year), )) transfer.endpoint_autoactivate(dest_ep) try: transfer.operation_mkdir(dest_ep, dest_path) except TransferAPIError as error: if 'MkdirFailed.Exists' not in error.code: raise try: transfer.add_endpoint_acl_rule( dest_ep, dict(principal=user_identity_id, principal_type='identity', path=dest_path, permissions='r'), ) except TransferAPIError as error: # PermissionDenied can happen if a new Portal client is swapped # in and it doesn't have endpoint manager on the dest_ep. # The /portal/processed directory has been set to to read/write # for all users so the subsequent operations will succeed. if error.code == 'PermissionDenied': pass elif error.code != 'Exists': raise for filename, svg in svgs.items(): requests.put('%s%s%s.svg' % (dest_https, dest_path, filename), data=svg, headers=dict(Authorization='Bearer ' + dest_token), allow_redirects=False).raise_for_status() results = { 'dest_ep': dest_ep, 'dest_path': dest_path, 'dest_name': dest_info['display_name'], 'graph_count': len(svgs) or 0 } return jsonify(results)
def browse(dataset_id=None, endpoint_id=None, endpoint_path=None): """ - Get list of files for the selected dataset or endpoint ID/path - Return a list of files to a browse view The target template (browse.jinja2) expects an `endpoint_uri` (if available for the endpoint), `target` (either `"dataset"` or `"endpoint"`), and 'file_list' (list of dictionaries) containing the following information about each file in the result: {'name': 'file name', 'size': 'file size', 'id': 'file uri/path'} If you want to display additional information about each file, you must add those keys to the dictionary and modify the browse.jinja2 template accordingly. """ if request.method == 'GET': assert bool(dataset_id) != bool(endpoint_id and endpoint_path) if dataset_id: try: dataset = next(ds for ds in datasets if ds['id'] == dataset_id) except StopIteration: abort(404) endpoint_id = app.config['DATASET_ENDPOINT_ID'] endpoint_path = app.config['DATASET_ENDPOINT_BASE'] + dataset['path'] else: endpoint_path = '/' + endpoint_path transfer_tokens = session['tokens']['transfer.api.globus.org'] authorizer = RefreshTokenAuthorizer( transfer_tokens['refresh_token'], load_portal_client(), access_token=transfer_tokens['access_token'], expires_at=transfer_tokens['expires_at_seconds']) transfer = TransferClient(authorizer=authorizer) try: transfer.endpoint_autoactivate(endpoint_id) listing = transfer.operation_ls(endpoint_id, path=endpoint_path) except TransferAPIError as err: flash('Error [{}]: {}'.format(err.code, err.message)) return redirect(url_for('browse')) file_list = [e for e in listing if e['type'] == 'file'] ep = transfer.get_endpoint(endpoint_id) https_server = ep['https_server'] endpoint_uri = https_server + endpoint_path if https_server else None webapp_xfer = 'https://app.globus.org/file-manager?' + \ urlencode(dict(origin_id=endpoint_id, origin_path=endpoint_path)) #print("endpintURL == " + endpoint_uri) return render_template('browse.jinja2', endpoint_uri=endpoint_uri, target="dataset" if dataset_id else "endpoint", description=(dataset['name'] if dataset_id else ep['display_name']), mypath=(dataset['path'] if dataset_id else None), myid=(dataset['id'] if dataset_id else None), file_list=file_list, webapp_xfer=webapp_xfer) if request.method == 'POST': if not request.form.get('file'): flash('Please select at least one file.') return redirect(url_for('browse')) params = { 'method': 'POST', 'action': url_for('submit_transfer', _external=True, _scheme='https'), 'filelimit': 0, 'folderlimit': 1 } browse_endpoint = 'https://app.globus.org/file-manager?{}' \ .format(urlencode(params)) session['form'] = { 'dirselect': False, 'datasets': request.form.getlist('file'), 'path': request.form.getlist('path'), 'id': request.form.getlist('id') } return redirect(browse_endpoint)
def transfer(sp,destination_endpoint_id,one_endpoint): tokens = None try: # if we already have tokens, load and use them tokens = load_tokens_from_file(TOKEN_FILE) except: pass if not tokens: # if we need to get tokens, start the Native App authentication process tokens = do_native_app_authentication(CLIENT_ID, REDIRECT_URI, SCOPES) try: save_tokens_to_file(TOKEN_FILE, tokens) except: pass transfer_tokens = tokens['transfer.api.globus.org'] auth_client = NativeAppAuthClient(client_id=CLIENT_ID,environment='sandbox') authorizer = RefreshTokenAuthorizer( transfer_tokens['refresh_token'], auth_client, access_token=transfer_tokens['access_token'], expires_at=transfer_tokens['expires_at_seconds'], on_refresh=update_tokens_file_on_refresh) #transfer = TransferClient(authorizer=authorizer,environment='sandbox') tc = TransferClient(authorizer=authorizer, environment="sandbox") ##################---ENDPOINTS---########################### source_endpoint_id = '5a2e5704-b028-11e7-bdad-22000bdb2406' #sb vmtb4 #source_endpoint_id = '55705028-aa15-11e7-bdad-22000bdb2406' #sb yulie7t #source_endpoint_id = 'b0b16296-88e7-11e7-a971-22000a92523b' #bare chameleon #source_endpoint_id = 'e5762bc2-8466-11e7-a8ed-22000a92523b' #large_chameleon #source_endpoint_id = '8b26cc0e-877b-11e7-a949-22000a92523b'#ubuntu-vm #source_endpoint_id = 'ad19b012-77cf-11e7-8b98-22000b9923ef'#chameleon # source_endpoint_id = raw_input('Input source endpoint UUID: ') #destination path ##############---SOURCE PATH---###################### #source_path = '/home/parallels/stream_transfer/test_files/' #source_path = '/home/parallels/stream_transfer/zero_globus/test_files/' source_path = sp #source_path ='/home/cc/streaming/zero_globus/test_files/test.txt' #source_path = '/home/parallels/stream_transfer/zero_globus/test_files/test.txt' #destination path destination_path = '/~/' #destination_path = '/~/'+ sp.split("/")[-1] #use for one file ##if one_endpoint: ## destination_path = '/projects/BrainImagingADSP/yzamora/' ##else: ## destination_path = '/projects/BrainImagingADSP/yzamora/'+ sp.split("/")[-1] #use for one file #Using my sample UUID from globus tutorial #destination_endpoint_id = 'ddb59aef-6d04-11e5-ba46-22000b92c6ec' #globus #destination_endpoint_id = '5d1da0fe-3c07-11e7-bcfc-22000b9a448b' #laptop #tc.endpoint_autoactivate(source_endpoint_id) #tc.endpoint_autoactive(destination_endpoint_id) ep1 = tc.get_endpoint(destination_endpoint_id) tc.endpoint_autoactivate(destination_endpoint_id) #ep1 is setting the activated endpoint to be a variable to work with tc.endpoint_autoactivate(source_endpoint_id) label = "medium data transfer" #tdata = globus_sdk.TransferData(tc, source_endpoint_id, destination_endpoint_id,label=label, sync_level='0') tdata = globus_sdk.TransferData(tc, source_endpoint_id, destination_endpoint_id,label=label, perf_cc=3, sync_level=None, verify_checksum=False) #tdata = globus_sdk.TransferData(tc, source_endpoint_id, destination_endpoint_id,label=label) if one_endpoint: tdata.add_item(source_path,destination_path,recursive=True) else: tdata.add_item(source_path,destination_path,recursive=False) submit_result = tc.submit_transfer(tdata) print("Task ID:", submit_result["task_id"]) """ Checking for time completion using globus calls """ #print("Completion time:", submit_result["completion_time"]) #setup of the transfer, submits as a https post request #transfer_data = TransferData(transfer_client=tc, # source_endpoint=source_endpoint_id, # destination_endpoint=destination_endpoint_id, # label='Transfer', # sync_level='checksum') #transfer_data.add_item(source_path=source_path,destination_path=destination_path, recursive=False) #task_id=transfer.submit_transfer(transfer_data)['task_id'] #waiting for file to transfer status = tc.get_task(submit_result["task_id"],fields="status")["status"] poll_interval = 2 max_wait = 90 wait_time = 0 while not (status in ["SUCCEEDED", "FAILED"]): if (wait_time >= max_wait): break print("Task not yet complete (status {}), sleeping for {} seconds..." \ .format(status, poll_interval)) time.sleep(poll_interval) wait_time += poll_interval status = tc.get_task(submit_result["task_id"], fields="status")["status"] if status == "FAILED": print("WARNING! File transfer FAILED!") #deleting file after transfer if status == "SUCCEEDED": end_time = datetime.datetime.utcnow() start_time = end_time - datetime.timedelta(minutes=200) #limit = response objects # data = tc.task_list(filter="type:TRANSFER,DELETE/request_time:%s,%s" # % (start_time, end_time), limit=5) #print("File transfer SUCCEEDED, will delete file from local directory now") """ r = tc.task_list(num_results=1, filter="type:TRANSFER,DELETE")
def doit(): """ - Call token introspect - Get dependent tokens """ dependent_tokens = get_dependent_tokens(g.req_token) # dependent_tokens is a token response object # create transfer_token and http_token variables containing # the correct token for each resource server transfer_token = dependent_tokens.by_resource_server[ 'transfer.api.globus.org']['access_token'] http_token = dependent_tokens.by_resource_server[ app.config['GRAPH_ENDPOINT_ID']]['access_token'] selected_ids = request.form.getlist('datasets') selected_year = request.form.get('year') user_identity_id = request.form.get('user_identity_id') user_identity_name = request.form.get('user_identity_name') selected_datasets = [dataset for dataset in datasets if dataset['id'] in selected_ids] if not (selected_datasets and selected_year): raise BadRequestError() transfer = TransferClient(authorizer=AccessTokenAuthorizer(transfer_token)) source_ep = app.config['DATASET_ENDPOINT_ID'] source_info = transfer.get_endpoint(source_ep) source_https = source_info['https_server'] source_base = app.config['DATASET_ENDPOINT_BASE'] source_token = http_token dest_ep = app.config['GRAPH_ENDPOINT_ID'] dest_info = transfer.get_endpoint(dest_ep) dest_https = dest_info['https_server'] dest_base = app.config['GRAPH_ENDPOINT_BASE'] dest_path = '%sGraphs for %s/' % (dest_base, user_identity_name) dest_token = http_token if not (source_https and dest_https): raise InternalServerError(message='Endpoints must be HTTPS servers') svgs = {} for dataset in selected_datasets: source_path = dataset['path'] response = requests.get('%s%s%s/%s.csv' % (source_https, source_base, source_path, selected_year), headers=dict( Authorization='Bearer ' + source_token), allow_redirects=False) response.raise_for_status() svgs.update(render_graphs( csv_data=response.iter_lines(decode_unicode=True), append_titles=' from %s for %s' % (dataset['name'], selected_year), )) transfer.endpoint_autoactivate(dest_ep) try: transfer.operation_mkdir(dest_ep, dest_path) except TransferAPIError as error: if 'MkdirFailed.Exists' not in error.code: raise try: transfer.add_endpoint_acl_rule( dest_ep, dict(principal=user_identity_id, principal_type='identity', path=dest_path, permissions='r'), ) except TransferAPIError as error: # PermissionDenied can happen if a new Portal client is swapped # in and it doesn't have endpoint manager on the dest_ep. # The /portal/processed directory has been set to to read/write # for all users so the subsequent operations will succeed. if error.code == 'PermissionDenied': pass elif error.code != 'Exists': raise for filename, svg in svgs.items(): requests.put('%s%s%s.svg' % (dest_https, dest_path, filename), data=svg, headers=dict(Authorization='Bearer ' + dest_token), allow_redirects=False).raise_for_status() results = { 'dest_ep': dest_ep, 'dest_path': dest_path, 'dest_name': dest_info['display_name'], 'graph_count': len(svgs) or 0 } return jsonify(results)