def _get_required_data_access_scopes(
tc: TransferClient,
collection_ids: Iterable[str],
) -> List[str]:
data_access_scopes: List[str] = []
for collection_id in collection_ids:
collection_id_info = tc.get_endpoint(collection_id)
if collection_id_info["DATA_TYPE"] == "endpoint":
gcs_version = collection_id_info.get("gcs_version")
if gcs_version is None:
continue
gcs_version_parts = [int(x) for x in gcs_version.split(".")]
requires_data_access = all(
[
gcs_version_parts[0] > 5
or gcs_version_parts[0] == 5
and gcs_version_parts[1] >= 4,
(collection_id_info.get("high_assurance", True) is False),
collection_id_info.get("host_endpoint", True) is None,
]
)
if requires_data_access:
data_access_scopes.append(
f"https://auth.globus.org/scopes/{collection_id}/data_access"
)
return data_access_scopes
class GridFTPConnection:
def __init__(self,endpoint_id,source_id):
self.transfer_client = TransferClient()
self.endpoint_id = endpoint_id
self.source_id = source_id
self.endpoint = self.transfer_client.get_endpoint(endpoint_id)
print("Connection successful to:", self.endpoint["display_name"] or self.endpoint["canonical_name"])
def browse(dataset_id=None, endpoint_id=None, endpoint_path=None):
"""
- Get list of files for the selected dataset or endpoint ID/path
- Return a list of files to a browse view
The target template (browse.jinja2) expects an `endpoint_uri` (if
available for the endpoint), `target` (either `"dataset"`
or `"endpoint"`), and 'file_list' (list of dictionaries) containing
the following information about each file in the result:
{'name': 'file name', 'size': 'file size', 'id': 'file uri/path'}
If you want to display additional information about each file, you
must add those keys to the dictionary and modify the browse.jinja2
template accordingly.
"""
assert bool(dataset_id) != bool(endpoint_id and endpoint_path)
if dataset_id:
try:
dataset = next(ds for ds in datasets if ds['id'] == dataset_id)
except StopIteration:
abort(404)
endpoint_id = app.config['DATASET_ENDPOINT_ID']
endpoint_path = app.config['DATASET_ENDPOINT_BASE'] + dataset['path']
else:
endpoint_path = '/' + endpoint_path
transfer = TransferClient(authorizer=RefreshTokenAuthorizer(
session['tokens']['transfer.api.globus.org']['refresh_token'],
load_portal_client()))
try:
transfer.endpoint_autoactivate(endpoint_id)
listing = transfer.operation_ls(endpoint_id, path=endpoint_path)
except TransferAPIError as err:
flash('Error [{}]: {}'.format(err.code, err.message))
return redirect(url_for('transfer'))
file_list = [e for e in listing if e['type'] == 'file']
ep = transfer.get_endpoint(endpoint_id)
https_server = ep['https_server']
endpoint_uri = https_server + endpoint_path if https_server else None
webapp_xfer = 'https://www.globus.org/app/transfer?' + \
urlencode(dict(origin_id=endpoint_id, origin_path=endpoint_path))
return render_template(
'browse.jinja2',
endpoint_uri=endpoint_uri,
target="dataset" if dataset_id else "endpoint",
description=(dataset['name'] if dataset_id else ep['display_name']),
file_list=file_list,
webapp_xfer=webapp_xfer)
def endpoints_not_activated(transfer_client: TransferClient,
endpoints: List[str]) -> List[str]:
"""
Filter list of endpoint UUIDs, returning unactivated ones.
Exit 1 if transfer responds with an error trying to look up endpoints.
"""
result = []
for endpoint in endpoints:
try:
if not transfer_client.get_endpoint(endpoint).get("activated"):
result.append(endpoint)
except GlobusError as e:
click.echo(
f"couldn't get information for endpoint {endpoint}:"
f" {e.code}, {e.message}",
err=True,
)
sys.exit(1)
return result
def authcallback():
def test_func(token_data):
print('this is a test')
"""Handles the interaction with Globus Auth."""
# If we're coming back from Globus Auth in an error state, the error
# will be in the "error" query string parameter.
if 'error' in request.args:
print('error')
return redirect(url_for('task_server.transfer'))
# Set up our Globus Auth/OAuth2 state
redirect_uri = url_for('task_server.authcallback', _external=True)
client = globus_sdk.ConfidentialAppAuthClient(
app.config['TS_CLIENT_ID'], app.config['TS_CLIENT_SECRET'])
client.oauth2_start_flow(redirect_uri, refresh_tokens=True) #let globus know where to go back to
# If there's no "code" query string parameter, we're in this route
# starting a Globus Auth login flow.
if 'code' not in request.args:
additional_authorize_params = (
{'signup': 1} if request.args.get('signup') else {})
auth_uri = client.oauth2_get_authorize_url(
additional_params=additional_authorize_params)
return redirect(auth_uri, code=307)
else:
# If we do have a "code" param, we're coming back from Globus Auth
# and can start the process of exchanging an auth code for a token.
#requests.post('http://localhost:8081/api/messenger', headers={'content-type': 'application/json'},data=json.dumps({'message':'User logged in....'}))
code = request.args.get('code')
tokens = client.oauth2_exchange_code_for_tokens(code)
id_token = tokens.decode_id_token(client)
session.update(
tokens=tokens.by_resource_server,
is_authenticated=True,
name=id_token.get('name', ''),
email=id_token.get('email', ''),
institution=id_token.get('institution', ''),
primary_username=id_token.get('preferred_username'),
primary_identity=id_token.get('sub'),
)
tokens = session['tokens']
stage_in_source = redis_store.get('stage_in_source').decode('utf-8')
stage_in_destination = redis_store.get('stage_in_destination').decode('utf-8')
stage_out_destination = redis_store.get('stage_out_destination').decode('utf-8')
new_token=None
authorizer = globus_sdk.RefreshTokenAuthorizer(tokens['transfer.api.globus.org']['refresh_token'],globus_sdk.ConfidentialAppAuthClient(app.config['TS_CLIENT_ID'], app.config['TS_CLIENT_SECRET']),tokens['transfer.api.globus.org']['access_token'],expires_at=1,on_refresh=test_func)
#authorizer = globus_sdk.AccessTokenAuthorizer(tokens['transfer.api.globus.org']['access_token'])
tc = TransferClient(authorizer=authorizer)
a = tc.endpoint_autoactivate(stage_in_source, if_expires_in=3600)
b = tc.endpoint_autoactivate(stage_in_destination, if_expires_in=3600)
c = tc.endpoint_autoactivate(stage_out_destination, if_expires_in=3600)
if a["code"] == "AutoActivationFailed" or b["code"] == "AutoActivationFailed" or c["code"] == "AutoActivationFailed":
stage_in_source_response = tc.get_endpoint(stage_in_source)
stage_in_destination_response = tc.get_endpoint(stage_in_destination)
stage_out_destination_response = tc.get_endpoint(stage_out_destination)
e1name = stage_in_source_response["display_name"]
e2name = stage_in_destination_response["display_name"]
e3name = stage_out_destination_response["display_name"]
return redirect("http://localhost:8080/activate?e1id="+ (stage_in_source if a["code"] == "AutoActivationFailed" else "")+("&e1name="+e1name if a["code"] == "AutoActivationFailed" else "")+"&e2id="+ (stage_in_destination if b["code"] == "AutoActivationFailed" else "")+("&e2name="+e2name if a["code"] == "AutoActivationFailed" else "" )+"&e3id="+(stage_out_destination if c["code"] == "AutoActivationFailed" else "")+"e3id&e3name="+(e3name if c["code"] == "AutoActivationFailed" else ""))
# read from db here
sis=redis_store.get('stage_in_source').decode('utf-8')
sid=redis_store.get('stage_in_destination').decode('utf-8') #...
sod=redis_store.get('stage_out_destination').decode('utf-8')#those names...........
siop= redis_store.get('stage_in_source_path').decode('utf-8')
sidp = redis_store.get('stage_in_destination_path').decode('utf-8')
sodp = redis_store.get('stage_out_destination_path').decode('utf-8')
return redirect(url_for('task_server.transfer', stage_in_source=sis,stage_in_dest=sid,stage_out_dest=sod,stage_in_source_path=siop,stage_in_dest_path=sidp,stage_out_dest_path=sodp))
def doit():
"""
- Call token introspect
- Get dependent tokens
"""
dependent_tokens = get_dependent_tokens(g.req_token)
# dependent_tokens is a token response object
# create transfer_token and http_token variables containing
# the correct token for each resource server
transfer_token = dependent_tokens.by_resource_server[
'transfer.api.globus.org']['access_token']
http_token = dependent_tokens.by_resource_server[
'tutorial-https-endpoint.globus.org']['access_token']
selected_ids = request.form.getlist('datasets')
selected_year = request.form.get('year')
user_identity_id = request.form.get('user_identity_id')
user_identity_name = request.form.get('user_identity_name')
selected_datasets = [
dataset for dataset in datasets if dataset['id'] in selected_ids
]
if not (selected_datasets and selected_year):
raise BadRequestError()
transfer = TransferClient(authorizer=AccessTokenAuthorizer(transfer_token))
source_ep = app.config['DATASET_ENDPOINT_ID']
source_info = transfer.get_endpoint(source_ep)
source_https = source_info['https_server']
source_base = app.config['DATASET_ENDPOINT_BASE']
source_token = http_token
dest_ep = app.config['GRAPH_ENDPOINT_ID']
dest_info = transfer.get_endpoint(dest_ep)
dest_https = dest_info['https_server']
dest_base = app.config['GRAPH_ENDPOINT_BASE']
dest_path = '%sGraphs for %s/' % (dest_base, user_identity_name)
dest_token = http_token
if not (source_https and dest_https):
raise InternalServerError(message='Endpoints must be HTTPS servers')
svgs = {}
for dataset in selected_datasets:
source_path = dataset['path']
response = requests.get(
'%s%s%s/%s.csv' %
(source_https, source_base, source_path, selected_year),
headers=dict(Authorization='Bearer ' + source_token),
allow_redirects=False)
response.raise_for_status()
svgs.update(
render_graphs(
csv_data=response.iter_lines(decode_unicode=True),
append_titles=' from %s for %s' %
(dataset['name'], selected_year),
))
transfer.endpoint_autoactivate(dest_ep)
try:
transfer.operation_mkdir(dest_ep, dest_path)
except TransferAPIError as error:
if 'MkdirFailed.Exists' not in error.code:
raise
try:
transfer.add_endpoint_acl_rule(
dest_ep,
dict(principal=user_identity_id,
principal_type='identity',
path=dest_path,
permissions='r'),
)
except TransferAPIError as error:
# PermissionDenied can happen if a new Portal client is swapped
# in and it doesn't have endpoint manager on the dest_ep.
# The /portal/processed directory has been set to to read/write
# for all users so the subsequent operations will succeed.
if error.code == 'PermissionDenied':
pass
elif error.code != 'Exists':
raise
for filename, svg in svgs.items():
requests.put('%s%s%s.svg' % (dest_https, dest_path, filename),
data=svg,
headers=dict(Authorization='Bearer ' + dest_token),
allow_redirects=False).raise_for_status()
results = {
'dest_ep': dest_ep,
'dest_path': dest_path,
'dest_name': dest_info['display_name'],
'graph_count': len(svgs) or 0
}
return jsonify(results)
def browse(dataset_id=None, endpoint_id=None, endpoint_path=None):
"""
- Get list of files for the selected dataset or endpoint ID/path
- Return a list of files to a browse view
The target template (browse.jinja2) expects an `endpoint_uri` (if
available for the endpoint), `target` (either `"dataset"`
or `"endpoint"`), and 'file_list' (list of dictionaries) containing
the following information about each file in the result:
{'name': 'file name', 'size': 'file size', 'id': 'file uri/path'}
If you want to display additional information about each file, you
must add those keys to the dictionary and modify the browse.jinja2
template accordingly.
"""
if request.method == 'GET':
assert bool(dataset_id) != bool(endpoint_id and endpoint_path)
if dataset_id:
try:
dataset = next(ds for ds in datasets if ds['id'] == dataset_id)
except StopIteration:
abort(404)
endpoint_id = app.config['DATASET_ENDPOINT_ID']
endpoint_path = app.config['DATASET_ENDPOINT_BASE'] + dataset['path']
else:
endpoint_path = '/' + endpoint_path
transfer_tokens = session['tokens']['transfer.api.globus.org']
authorizer = RefreshTokenAuthorizer(
transfer_tokens['refresh_token'],
load_portal_client(),
access_token=transfer_tokens['access_token'],
expires_at=transfer_tokens['expires_at_seconds'])
transfer = TransferClient(authorizer=authorizer)
try:
transfer.endpoint_autoactivate(endpoint_id)
listing = transfer.operation_ls(endpoint_id, path=endpoint_path)
except TransferAPIError as err:
flash('Error [{}]: {}'.format(err.code, err.message))
return redirect(url_for('browse'))
file_list = [e for e in listing if e['type'] == 'file']
ep = transfer.get_endpoint(endpoint_id)
https_server = ep['https_server']
endpoint_uri = https_server + endpoint_path if https_server else None
webapp_xfer = 'https://app.globus.org/file-manager?' + \
urlencode(dict(origin_id=endpoint_id, origin_path=endpoint_path))
#print("endpintURL == " + endpoint_uri)
return render_template('browse.jinja2', endpoint_uri=endpoint_uri,
target="dataset" if dataset_id else "endpoint",
description=(dataset['name'] if dataset_id
else ep['display_name']),
mypath=(dataset['path'] if dataset_id
else None),
myid=(dataset['id'] if dataset_id
else None),
file_list=file_list, webapp_xfer=webapp_xfer)
if request.method == 'POST':
if not request.form.get('file'):
flash('Please select at least one file.')
return redirect(url_for('browse'))
params = {
'method': 'POST',
'action': url_for('submit_transfer', _external=True,
_scheme='https'),
'filelimit': 0,
'folderlimit': 1
}
browse_endpoint = 'https://app.globus.org/file-manager?{}' \
.format(urlencode(params))
session['form'] = {
'dirselect': False,
'datasets': request.form.getlist('file'),
'path': request.form.getlist('path'),
'id': request.form.getlist('id')
}
return redirect(browse_endpoint)
def transfer(sp,destination_endpoint_id,one_endpoint):
tokens = None
try:
# if we already have tokens, load and use them
tokens = load_tokens_from_file(TOKEN_FILE)
except:
pass
if not tokens:
# if we need to get tokens, start the Native App authentication process
tokens = do_native_app_authentication(CLIENT_ID, REDIRECT_URI, SCOPES)
try:
save_tokens_to_file(TOKEN_FILE, tokens)
except:
pass
transfer_tokens = tokens['transfer.api.globus.org']
auth_client = NativeAppAuthClient(client_id=CLIENT_ID,environment='sandbox')
authorizer = RefreshTokenAuthorizer(
transfer_tokens['refresh_token'],
auth_client,
access_token=transfer_tokens['access_token'],
expires_at=transfer_tokens['expires_at_seconds'],
on_refresh=update_tokens_file_on_refresh)
#transfer = TransferClient(authorizer=authorizer,environment='sandbox')
tc = TransferClient(authorizer=authorizer, environment="sandbox")
##################---ENDPOINTS---###########################
source_endpoint_id = '5a2e5704-b028-11e7-bdad-22000bdb2406' #sb vmtb4
#source_endpoint_id = '55705028-aa15-11e7-bdad-22000bdb2406' #sb yulie7t
#source_endpoint_id = 'b0b16296-88e7-11e7-a971-22000a92523b' #bare chameleon
#source_endpoint_id = 'e5762bc2-8466-11e7-a8ed-22000a92523b' #large_chameleon
#source_endpoint_id = '8b26cc0e-877b-11e7-a949-22000a92523b'#ubuntu-vm
#source_endpoint_id = 'ad19b012-77cf-11e7-8b98-22000b9923ef'#chameleon
# source_endpoint_id = raw_input('Input source endpoint UUID: ')
#destination path
##############---SOURCE PATH---######################
#source_path = '/home/parallels/stream_transfer/test_files/'
#source_path = '/home/parallels/stream_transfer/zero_globus/test_files/'
source_path = sp
#source_path ='/home/cc/streaming/zero_globus/test_files/test.txt'
#source_path = '/home/parallels/stream_transfer/zero_globus/test_files/test.txt'
#destination path
destination_path = '/~/'
#destination_path = '/~/'+ sp.split("/")[-1] #use for one file
##if one_endpoint:
## destination_path = '/projects/BrainImagingADSP/yzamora/'
##else:
## destination_path = '/projects/BrainImagingADSP/yzamora/'+ sp.split("/")[-1] #use for one file
#Using my sample UUID from globus tutorial
#destination_endpoint_id = 'ddb59aef-6d04-11e5-ba46-22000b92c6ec' #globus
#destination_endpoint_id = '5d1da0fe-3c07-11e7-bcfc-22000b9a448b' #laptop
#tc.endpoint_autoactivate(source_endpoint_id)
#tc.endpoint_autoactive(destination_endpoint_id)
ep1 = tc.get_endpoint(destination_endpoint_id)
tc.endpoint_autoactivate(destination_endpoint_id)
#ep1 is setting the activated endpoint to be a variable to work with
tc.endpoint_autoactivate(source_endpoint_id)
label = "medium data transfer"
#tdata = globus_sdk.TransferData(tc, source_endpoint_id, destination_endpoint_id,label=label, sync_level='0')
tdata = globus_sdk.TransferData(tc, source_endpoint_id, destination_endpoint_id,label=label, perf_cc=3, sync_level=None, verify_checksum=False)
#tdata = globus_sdk.TransferData(tc, source_endpoint_id, destination_endpoint_id,label=label)
if one_endpoint:
tdata.add_item(source_path,destination_path,recursive=True)
else:
tdata.add_item(source_path,destination_path,recursive=False)
submit_result = tc.submit_transfer(tdata)
print("Task ID:", submit_result["task_id"])
"""
Checking for time completion using globus calls
"""
#print("Completion time:", submit_result["completion_time"])
#setup of the transfer, submits as a https post request
#transfer_data = TransferData(transfer_client=tc,
# source_endpoint=source_endpoint_id,
# destination_endpoint=destination_endpoint_id,
# label='Transfer',
# sync_level='checksum')
#transfer_data.add_item(source_path=source_path,destination_path=destination_path, recursive=False)
#task_id=transfer.submit_transfer(transfer_data)['task_id']
#waiting for file to transfer
status = tc.get_task(submit_result["task_id"],fields="status")["status"]
poll_interval = 2
max_wait = 90
wait_time = 0
while not (status in ["SUCCEEDED", "FAILED"]):
if (wait_time >= max_wait): break
print("Task not yet complete (status {}), sleeping for {} seconds..." \
.format(status, poll_interval))
time.sleep(poll_interval)
wait_time += poll_interval
status = tc.get_task(submit_result["task_id"], fields="status")["status"]
if status == "FAILED":
print("WARNING! File transfer FAILED!")
#deleting file after transfer
if status == "SUCCEEDED":
end_time = datetime.datetime.utcnow()
start_time = end_time - datetime.timedelta(minutes=200)
#limit = response objects
# data = tc.task_list(filter="type:TRANSFER,DELETE/request_time:%s,%s"
# % (start_time, end_time), limit=5)
#print("File transfer SUCCEEDED, will delete file from local directory now")
""" r = tc.task_list(num_results=1, filter="type:TRANSFER,DELETE")
def doit():
"""
- Call token introspect
- Get dependent tokens
"""
dependent_tokens = get_dependent_tokens(g.req_token)
# dependent_tokens is a token response object
# create transfer_token and http_token variables containing
# the correct token for each resource server
transfer_token = dependent_tokens.by_resource_server[
'transfer.api.globus.org']['access_token']
http_token = dependent_tokens.by_resource_server[
app.config['GRAPH_ENDPOINT_ID']]['access_token']
selected_ids = request.form.getlist('datasets')
selected_year = request.form.get('year')
user_identity_id = request.form.get('user_identity_id')
user_identity_name = request.form.get('user_identity_name')
selected_datasets = [dataset for dataset in datasets
if dataset['id'] in selected_ids]
if not (selected_datasets and selected_year):
raise BadRequestError()
transfer = TransferClient(authorizer=AccessTokenAuthorizer(transfer_token))
source_ep = app.config['DATASET_ENDPOINT_ID']
source_info = transfer.get_endpoint(source_ep)
source_https = source_info['https_server']
source_base = app.config['DATASET_ENDPOINT_BASE']
source_token = http_token
dest_ep = app.config['GRAPH_ENDPOINT_ID']
dest_info = transfer.get_endpoint(dest_ep)
dest_https = dest_info['https_server']
dest_base = app.config['GRAPH_ENDPOINT_BASE']
dest_path = '%sGraphs for %s/' % (dest_base, user_identity_name)
dest_token = http_token
if not (source_https and dest_https):
raise InternalServerError(message='Endpoints must be HTTPS servers')
svgs = {}
for dataset in selected_datasets:
source_path = dataset['path']
response = requests.get('%s%s%s/%s.csv' % (source_https, source_base,
source_path, selected_year),
headers=dict(
Authorization='Bearer ' + source_token),
allow_redirects=False)
response.raise_for_status()
svgs.update(render_graphs(
csv_data=response.iter_lines(decode_unicode=True),
append_titles=' from %s for %s' % (dataset['name'], selected_year),
))
transfer.endpoint_autoactivate(dest_ep)
try:
transfer.operation_mkdir(dest_ep, dest_path)
except TransferAPIError as error:
if 'MkdirFailed.Exists' not in error.code:
raise
try:
transfer.add_endpoint_acl_rule(
dest_ep,
dict(principal=user_identity_id,
principal_type='identity', path=dest_path, permissions='r'),
)
except TransferAPIError as error:
# PermissionDenied can happen if a new Portal client is swapped
# in and it doesn't have endpoint manager on the dest_ep.
# The /portal/processed directory has been set to to read/write
# for all users so the subsequent operations will succeed.
if error.code == 'PermissionDenied':
pass
elif error.code != 'Exists':
raise
for filename, svg in svgs.items():
requests.put('%s%s%s.svg' % (dest_https, dest_path, filename),
data=svg,
headers=dict(Authorization='Bearer ' + dest_token),
allow_redirects=False).raise_for_status()
results = {
'dest_ep': dest_ep,
'dest_path': dest_path,
'dest_name': dest_info['display_name'],
'graph_count': len(svgs) or 0
}
return jsonify(results)
