def create_shared_endpoint(globus_dict,
host_endpoint,
host_path,
display_name='Globus endpoint',
description='description'):
globus_transfer_token = globus_dict['transfer_token']
scopes = "urn:globus:auth:scopes:transfer.api.globus.org:all"
authorizer = globus_sdk.AccessTokenAuthorizer(globus_transfer_token)
tc = TransferClient(authorizer=authorizer)
# high level interface; provides iterators for list responses
shared_ep_data = {
"DATA_TYPE": "shared_endpoint",
"host_endpoint": host_endpoint,
"host_path": host_path,
"display_name": display_name,
# optionally specify additional endpoint fields
"description": description
}
#r = tc.operation_mkdir(host_id, path=share_path) #TODO create the directory directly from here instead of at local level?
tc.endpoint_autoactivate(host_endpoint,
if_expires_in=3600) #necessary for real use?
create_result = tc.create_shared_endpoint(
shared_ep_data) #not the app's end point, so should fail
endpoint_id = create_result['id']
globus_dict['endpoint_id'] = endpoint_id
globus_dict['transfer_client'] = tc
return globus_dict
def browse(dataset_id=None, endpoint_id=None, endpoint_path=None):
"""
- Get list of files for the selected dataset or endpoint ID/path
- Return a list of files to a browse view
The target template (browse.jinja2) expects an `endpoint_uri` (if
available for the endpoint), `target` (either `"dataset"`
or `"endpoint"`), and 'file_list' (list of dictionaries) containing
the following information about each file in the result:
{'name': 'file name', 'size': 'file size', 'id': 'file uri/path'}
If you want to display additional information about each file, you
must add those keys to the dictionary and modify the browse.jinja2
template accordingly.
"""
assert bool(dataset_id) != bool(endpoint_id and endpoint_path)
if dataset_id:
try:
dataset = next(ds for ds in datasets if ds['id'] == dataset_id)
except StopIteration:
abort(404)
endpoint_id = app.config['DATASET_ENDPOINT_ID']
endpoint_path = app.config['DATASET_ENDPOINT_BASE'] + dataset['path']
else:
endpoint_path = '/' + endpoint_path
transfer = TransferClient(authorizer=RefreshTokenAuthorizer(
session['tokens']['transfer.api.globus.org']['refresh_token'],
load_portal_client()))
try:
transfer.endpoint_autoactivate(endpoint_id)
listing = transfer.operation_ls(endpoint_id, path=endpoint_path)
except TransferAPIError as err:
flash('Error [{}]: {}'.format(err.code, err.message))
return redirect(url_for('transfer'))
file_list = [e for e in listing if e['type'] == 'file']
ep = transfer.get_endpoint(endpoint_id)
https_server = ep['https_server']
endpoint_uri = https_server + endpoint_path if https_server else None
webapp_xfer = 'https://www.globus.org/app/transfer?' + \
urlencode(dict(origin_id=endpoint_id, origin_path=endpoint_path))
return render_template(
'browse.jinja2',
endpoint_uri=endpoint_uri,
target="dataset" if dataset_id else "endpoint",
description=(dataset['name'] if dataset_id else ep['display_name']),
file_list=file_list,
webapp_xfer=webapp_xfer)
def main():
current_time = datetime.utcnow().replace(microsecond=0).isoformat()
last_cleanup_time = datetime.utcnow().replace(microsecond=0)\
- timedelta(hours=24)
last_cleanup = last_cleanup_time.isoformat()
completion_range = last_cleanup + "," + current_time
print("Cleaning up source endpoint {} \nfor outbound transfers completed "
"in range {}\n ".format(SOURCE_ENDPOINT_ID, completion_range))
transfer_token = do_client_authentication(CLIENT_ID, CLIENT_SECRET)
authorizer = AccessTokenAuthorizer(access_token=transfer_token)
tc = TransferClient(authorizer=authorizer)
# print out a directory listing from an endpoint
tc.endpoint_autoactivate(SOURCE_ENDPOINT_ID)
try:
task_fields = "task_id,source_endpoint,destination_endpoint," \
"source_host_path,owner_string,source_endpoint_id,type"
tasks = tc.endpoint_manager_task_list(
filter_status="SUCCEEDED",
filter_endpoint=SOURCE_ENDPOINT_ID,
filter_completion_time=completion_range,
fields=task_fields)
except TransferAPIError as tapie:
if tapie.code == 'PermissionDenied':
print('Permission denied! Give your app permission by going to '
'"globus.org/app/endpoints/{}/roles", and under '
'"Identity/E-mail adding "{}@clients.auth.globus.org" as '
'an "AccessManager" and "Activity Manager"'.format(
SOURCE_ENDPOINT_ID, CLIENT_ID))
sys.exit(1)
# Nothing weird *should* happen here, but if so re-raise so the user
# can deal with it.
raise
tasklist = tasks.data
if not tasklist:
print("No transfers from {} found in the last 24 hours, "
"nothing to clean up".format(SOURCE_ENDPOINT_ID))
else:
print("{} total transfers found from {} in the last 24 hours, "
"some may not be of type TRANSFER".format(
len(tasklist), SOURCE_ENDPOINT_ID))
delete_tasks = [
task.data for task in tasklist
if task_delete_conditions_satisfied(task)
]
for task in delete_tasks:
files_list, common_dir = select_dir_to_delete(tc, task)
delete_dir_and_acls(tc, task, files_list, common_dir)
def submit_transfer():
"""
- Take the data returned by the Browse Endpoint helper page
and make a Globus transfer request.
- Send the user to the transfer status page with the task id
from the transfer.
"""
browse_endpoint_form = request.form
selected = session['form']['datasets']
filtered_datasets = [ds for ds in datasets if ds['id'] in selected]
transfer_tokens = session['tokens']['transfer.api.globus.org']
authorizer = RefreshTokenAuthorizer(
transfer_tokens['refresh_token'],
load_portal_client(),
access_token=transfer_tokens['access_token'],
expires_at=transfer_tokens['expires_at_seconds'])
transfer = TransferClient(authorizer=authorizer)
source_endpoint_id = app.config['DATASET_ENDPOINT_ID']
source_endpoint_base = app.config['DATASET_ENDPOINT_BASE']
destination_endpoint_id = browse_endpoint_form['endpoint_id']
destination_folder = browse_endpoint_form.get('folder[0]')
transfer_data = TransferData(transfer_client=transfer,
source_endpoint=source_endpoint_id,
destination_endpoint=destination_endpoint_id,
label=browse_endpoint_form.get('label'))
for ds in filtered_datasets:
source_path = source_endpoint_base + ds['path']
dest_path = browse_endpoint_form['path']
if destination_folder:
dest_path += destination_folder + '/'
dest_path += ds['name'] + '/'
transfer_data.add_item(source_path=source_path,
destination_path=dest_path,
recursive=True)
transfer.endpoint_autoactivate(source_endpoint_id)
transfer.endpoint_autoactivate(destination_endpoint_id)
task_id = transfer.submit_transfer(transfer_data)['task_id']
flash('Transfer request submitted successfully. Task ID: ' + task_id)
return (redirect(url_for('transfer_status', task_id=task_id)))
def cleanup():
user_identity_name = request.form.get('user_identity_name')
dependent_tokens = get_dependent_tokens(g.req_token)
transfer_token = dependent_tokens.by_resource_server[
'transfer.api.globus.org']['access_token']
dest_ep = app.config['GRAPH_ENDPOINT_ID']
dest_base = app.config['GRAPH_ENDPOINT_BASE']
dest_path = '%sGraphs for %s/' % (dest_base, user_identity_name)
transfer = TransferClient(authorizer=AccessTokenAuthorizer(transfer_token))
transfer.endpoint_autoactivate(dest_ep)
try:
acl = next(acl for acl in transfer.endpoint_acl_list(dest_ep)
if dest_path == acl['path'])
except StopIteration:
pass
except TransferAPIError as ex:
# PermissionDenied can happen if a new Portal client is swapped
# in and it doesn't have endpoint manager on the dest_ep.
# The /portal/processed directory has been set to to writeable
# for all users so the delete task will succeed even if an ACL
# can't be set.
if ex.code == 'PermissionDenied':
pass
else:
transfer.delete_endpoint_acl_rule(dest_ep, acl['id'])
delete_request = DeleteData(transfer_client=transfer,
endpoint=dest_ep,
label="Delete Graphs from the Service Demo",
recursive=True)
delete_request.add_item(dest_path)
try:
task = transfer.submit_delete(delete_request)
except TransferAPIError as ex:
raise InternalServerError(message=ex.message)
else:
return jsonify(dict(task_id=task['task_id']))
def cleanup():
user_identity_name = request.form.get('user_identity_name')
dependent_tokens = get_dependent_tokens(g.req_token)
transfer_token = dependent_tokens.by_resource_server[
'transfer.api.globus.org']['access_token']
dest_ep = app.config['GRAPH_ENDPOINT_ID']
dest_base = app.config['GRAPH_ENDPOINT_BASE']
dest_path = '%sGraphs for %s/' % (dest_base, user_identity_name)
transfer = TransferClient(authorizer=AccessTokenAuthorizer(transfer_token))
transfer.endpoint_autoactivate(dest_ep)
try:
acl = next(acl for acl in transfer.endpoint_acl_list(dest_ep)
if dest_path == acl['path'])
except StopIteration:
pass
except TransferAPIError as ex:
# PermissionDenied can happen if a new Portal client is swapped
# in and it doesn't have endpoint manager on the dest_ep.
# The /portal/processed directory has been set to to writeable
# for all users so the delete task will succeed even if an ACL
# can't be set.
if ex.code == 'PermissionDenied':
pass
else:
transfer.delete_endpoint_acl_rule(dest_ep, acl['id'])
delete_request = DeleteData(transfer_client=transfer, endpoint=dest_ep,
label="Delete Graphs from the Service Demo",
recursive=True)
delete_request.add_item(dest_path)
try:
task = transfer.submit_delete(delete_request)
except TransferAPIError as ex:
raise InternalServerError(message=ex.message)
else:
return jsonify(dict(task_id=task['task_id']))
def setup_transfer_client(transfer_tokens):
authorizer = RefreshTokenAuthorizer(
transfer_tokens['refresh_token'],
NativeAppAuthClient(client_id=CLIENT_ID),
access_token=transfer_tokens['access_token'],
expires_at=transfer_tokens['expires_at_seconds'])
transfer_client = TransferClient(authorizer=authorizer)
try:
transfer_client.endpoint_autoactivate(SOURCE_ENDPOINT)
transfer_client.endpoint_autoactivate(DESTINATION_ENDPOINT)
except GlobusAPIError as ex:
if ex.http_status == 401:
sys.exit('Refresh token has expired. '
'Please delete the `tokens` object from '
'{} and try again.'.format(DATA_FILE))
else:
raise ex
return transfer_client
def setup_transfer_client(transfer_tokens, source_endpoint,
destination_endpoint):
authorizer = RefreshTokenAuthorizer(
transfer_tokens['refresh_token'],
NativeAppAuthClient(client_id=CLIENT_ID),
access_token=transfer_tokens['access_token'],
expires_at=transfer_tokens['expires_at_seconds'],
on_refresh=update_tokens_file_on_refresh)
transfer_client = TransferClient(authorizer=authorizer)
try:
transfer_client.endpoint_autoactivate(source_endpoint)
transfer_client.endpoint_autoactivate(destination_endpoint)
except GlobusAPIError as ex:
if ex.http_status == 401:
sys.exit('Refresh token has expired. '
'Please delete the `tokens` object from '
'{} and try again.'.format(DATA_FILE))
else:
raise ex
return transfer_client
def error_if_not_activated(
transfer_client: TransferClient,
endpoints: List[str],
reactivate_if_expires_in=86400,
):
not_activated = endpoints_not_activated(transfer_client, endpoints)
still_not_activated = []
for endpoint in not_activated:
response = transfer_client.endpoint_autoactivate(
endpoint, if_expires_in=reactivate_if_expires_in)
if response.get("code") == "AutoActivationFailed":
still_not_activated.append(endpoint)
if still_not_activated:
click.echo(
f"Error: requested endpoint is not activated: {', '.join(not_activated)}\n"
"Open in the web app to activate:",
err=True,
)
for endpoint in not_activated:
click.echo(
f" https://app.globus.org/file-manager?origin_id={endpoint}",
err=True,
)
sys.exit(1)
def browse(dataset_id=None, endpoint_id=None, endpoint_path=None):
"""
- Get list of files for the selected dataset or endpoint ID/path
- Return a list of files to a browse view
The target template (browse.jinja2) expects an `endpoint_uri` (if
available for the endpoint), `target` (either `"dataset"`
or `"endpoint"`), and 'file_list' (list of dictionaries) containing
the following information about each file in the result:
{'name': 'file name', 'size': 'file size', 'id': 'file uri/path'}
If you want to display additional information about each file, you
must add those keys to the dictionary and modify the browse.jinja2
template accordingly.
"""
if request.method == 'GET':
assert bool(dataset_id) != bool(endpoint_id and endpoint_path)
if dataset_id:
try:
dataset = next(ds for ds in datasets if ds['id'] == dataset_id)
except StopIteration:
abort(404)
endpoint_id = app.config['DATASET_ENDPOINT_ID']
endpoint_path = app.config['DATASET_ENDPOINT_BASE'] + dataset['path']
else:
endpoint_path = '/' + endpoint_path
transfer_tokens = session['tokens']['transfer.api.globus.org']
authorizer = RefreshTokenAuthorizer(
transfer_tokens['refresh_token'],
load_portal_client(),
access_token=transfer_tokens['access_token'],
expires_at=transfer_tokens['expires_at_seconds'])
transfer = TransferClient(authorizer=authorizer)
try:
transfer.endpoint_autoactivate(endpoint_id)
listing = transfer.operation_ls(endpoint_id, path=endpoint_path)
except TransferAPIError as err:
flash('Error [{}]: {}'.format(err.code, err.message))
return redirect(url_for('browse'))
file_list = [e for e in listing if e['type'] == 'file']
ep = transfer.get_endpoint(endpoint_id)
https_server = ep['https_server']
endpoint_uri = https_server + endpoint_path if https_server else None
webapp_xfer = 'https://app.globus.org/file-manager?' + \
urlencode(dict(origin_id=endpoint_id, origin_path=endpoint_path))
#print("endpintURL == " + endpoint_uri)
return render_template('browse.jinja2', endpoint_uri=endpoint_uri,
target="dataset" if dataset_id else "endpoint",
description=(dataset['name'] if dataset_id
else ep['display_name']),
mypath=(dataset['path'] if dataset_id
else None),
myid=(dataset['id'] if dataset_id
else None),
file_list=file_list, webapp_xfer=webapp_xfer)
if request.method == 'POST':
if not request.form.get('file'):
flash('Please select at least one file.')
return redirect(url_for('browse'))
params = {
'method': 'POST',
'action': url_for('submit_transfer', _external=True,
_scheme='https'),
'filelimit': 0,
'folderlimit': 1
}
browse_endpoint = 'https://app.globus.org/file-manager?{}' \
.format(urlencode(params))
session['form'] = {
'dirselect': False,
'datasets': request.form.getlist('file'),
'path': request.form.getlist('path'),
'id': request.form.getlist('id')
}
return redirect(browse_endpoint)
def authcallback():
def test_func(token_data):
print('this is a test')
"""Handles the interaction with Globus Auth."""
# If we're coming back from Globus Auth in an error state, the error
# will be in the "error" query string parameter.
if 'error' in request.args:
print('error')
return redirect(url_for('task_server.transfer'))
# Set up our Globus Auth/OAuth2 state
redirect_uri = url_for('task_server.authcallback', _external=True)
client = globus_sdk.ConfidentialAppAuthClient(
app.config['TS_CLIENT_ID'], app.config['TS_CLIENT_SECRET'])
client.oauth2_start_flow(redirect_uri, refresh_tokens=True) #let globus know where to go back to
# If there's no "code" query string parameter, we're in this route
# starting a Globus Auth login flow.
if 'code' not in request.args:
additional_authorize_params = (
{'signup': 1} if request.args.get('signup') else {})
auth_uri = client.oauth2_get_authorize_url(
additional_params=additional_authorize_params)
return redirect(auth_uri, code=307)
else:
# If we do have a "code" param, we're coming back from Globus Auth
# and can start the process of exchanging an auth code for a token.
#requests.post('http://localhost:8081/api/messenger', headers={'content-type': 'application/json'},data=json.dumps({'message':'User logged in....'}))
code = request.args.get('code')
tokens = client.oauth2_exchange_code_for_tokens(code)
id_token = tokens.decode_id_token(client)
session.update(
tokens=tokens.by_resource_server,
is_authenticated=True,
name=id_token.get('name', ''),
email=id_token.get('email', ''),
institution=id_token.get('institution', ''),
primary_username=id_token.get('preferred_username'),
primary_identity=id_token.get('sub'),
)
tokens = session['tokens']
stage_in_source = redis_store.get('stage_in_source').decode('utf-8')
stage_in_destination = redis_store.get('stage_in_destination').decode('utf-8')
stage_out_destination = redis_store.get('stage_out_destination').decode('utf-8')
new_token=None
authorizer = globus_sdk.RefreshTokenAuthorizer(tokens['transfer.api.globus.org']['refresh_token'],globus_sdk.ConfidentialAppAuthClient(app.config['TS_CLIENT_ID'], app.config['TS_CLIENT_SECRET']),tokens['transfer.api.globus.org']['access_token'],expires_at=1,on_refresh=test_func)
#authorizer = globus_sdk.AccessTokenAuthorizer(tokens['transfer.api.globus.org']['access_token'])
tc = TransferClient(authorizer=authorizer)
a = tc.endpoint_autoactivate(stage_in_source, if_expires_in=3600)
b = tc.endpoint_autoactivate(stage_in_destination, if_expires_in=3600)
c = tc.endpoint_autoactivate(stage_out_destination, if_expires_in=3600)
if a["code"] == "AutoActivationFailed" or b["code"] == "AutoActivationFailed" or c["code"] == "AutoActivationFailed":
stage_in_source_response = tc.get_endpoint(stage_in_source)
stage_in_destination_response = tc.get_endpoint(stage_in_destination)
stage_out_destination_response = tc.get_endpoint(stage_out_destination)
e1name = stage_in_source_response["display_name"]
e2name = stage_in_destination_response["display_name"]
e3name = stage_out_destination_response["display_name"]
return redirect("http://localhost:8080/activate?e1id="+ (stage_in_source if a["code"] == "AutoActivationFailed" else "")+("&e1name="+e1name if a["code"] == "AutoActivationFailed" else "")+"&e2id="+ (stage_in_destination if b["code"] == "AutoActivationFailed" else "")+("&e2name="+e2name if a["code"] == "AutoActivationFailed" else "" )+"&e3id="+(stage_out_destination if c["code"] == "AutoActivationFailed" else "")+"e3id&e3name="+(e3name if c["code"] == "AutoActivationFailed" else ""))
# read from db here
sis=redis_store.get('stage_in_source').decode('utf-8')
sid=redis_store.get('stage_in_destination').decode('utf-8') #...
sod=redis_store.get('stage_out_destination').decode('utf-8')#those names...........
siop= redis_store.get('stage_in_source_path').decode('utf-8')
sidp = redis_store.get('stage_in_destination_path').decode('utf-8')
sodp = redis_store.get('stage_out_destination_path').decode('utf-8')
return redirect(url_for('task_server.transfer', stage_in_source=sis,stage_in_dest=sid,stage_out_dest=sod,stage_in_source_path=siop,stage_in_dest_path=sidp,stage_out_dest_path=sodp))
def doit():
"""
- Call token introspect
- Get dependent tokens
"""
dependent_tokens = get_dependent_tokens(g.req_token)
# dependent_tokens is a token response object
# create transfer_token and http_token variables containing
# the correct token for each resource server
transfer_token = dependent_tokens.by_resource_server[
'transfer.api.globus.org']['access_token']
http_token = dependent_tokens.by_resource_server[
'tutorial-https-endpoint.globus.org']['access_token']
selected_ids = request.form.getlist('datasets')
selected_year = request.form.get('year')
user_identity_id = request.form.get('user_identity_id')
user_identity_name = request.form.get('user_identity_name')
selected_datasets = [
dataset for dataset in datasets if dataset['id'] in selected_ids
]
if not (selected_datasets and selected_year):
raise BadRequestError()
transfer = TransferClient(authorizer=AccessTokenAuthorizer(transfer_token))
source_ep = app.config['DATASET_ENDPOINT_ID']
source_info = transfer.get_endpoint(source_ep)
source_https = source_info['https_server']
source_base = app.config['DATASET_ENDPOINT_BASE']
source_token = http_token
dest_ep = app.config['GRAPH_ENDPOINT_ID']
dest_info = transfer.get_endpoint(dest_ep)
dest_https = dest_info['https_server']
dest_base = app.config['GRAPH_ENDPOINT_BASE']
dest_path = '%sGraphs for %s/' % (dest_base, user_identity_name)
dest_token = http_token
if not (source_https and dest_https):
raise InternalServerError(message='Endpoints must be HTTPS servers')
svgs = {}
for dataset in selected_datasets:
source_path = dataset['path']
response = requests.get(
'%s%s%s/%s.csv' %
(source_https, source_base, source_path, selected_year),
headers=dict(Authorization='Bearer ' + source_token),
allow_redirects=False)
response.raise_for_status()
svgs.update(
render_graphs(
csv_data=response.iter_lines(decode_unicode=True),
append_titles=' from %s for %s' %
(dataset['name'], selected_year),
))
transfer.endpoint_autoactivate(dest_ep)
try:
transfer.operation_mkdir(dest_ep, dest_path)
except TransferAPIError as error:
if 'MkdirFailed.Exists' not in error.code:
raise
try:
transfer.add_endpoint_acl_rule(
dest_ep,
dict(principal=user_identity_id,
principal_type='identity',
path=dest_path,
permissions='r'),
)
except TransferAPIError as error:
# PermissionDenied can happen if a new Portal client is swapped
# in and it doesn't have endpoint manager on the dest_ep.
# The /portal/processed directory has been set to to read/write
# for all users so the subsequent operations will succeed.
if error.code == 'PermissionDenied':
pass
elif error.code != 'Exists':
raise
for filename, svg in svgs.items():
requests.put('%s%s%s.svg' % (dest_https, dest_path, filename),
data=svg,
headers=dict(Authorization='Bearer ' + dest_token),
allow_redirects=False).raise_for_status()
results = {
'dest_ep': dest_ep,
'dest_path': dest_path,
'dest_name': dest_info['display_name'],
'graph_count': len(svgs) or 0
}
return jsonify(results)
def globus_transfer( # noqa: C901
remote_endpoint,
remote_path,
name,
transfer_type,
non_blocking=False):
"""
Read the local globus endpoint UUID from ~/.zstash.ini.
If the ini file does not exist, create an ini file with empty values,
and try to find the local endpoint UUID based on the FQDN
"""
ini_path = os.path.expanduser("~/.zstash.ini")
ini = configparser.ConfigParser()
local_endpoint = None
if ini.read(ini_path):
if "local" in ini.sections():
local_endpoint = ini["local"].get("globus_endpoint_uuid")
else:
ini["local"] = {"globus_endpoint_uuid": ""}
try:
with open(ini_path, "w") as f:
ini.write(f)
except Exception as e:
logger.error(e)
sys.exit(1)
if not local_endpoint:
fqdn = socket.getfqdn()
for pattern in regex_endpoint_map.keys():
if re.fullmatch(pattern, fqdn):
local_endpoint = regex_endpoint_map.get(pattern)
break
if not local_endpoint:
logger.error(
"{} does not have the local Globus endpoint set".format(ini_path))
sys.exit(1)
if remote_endpoint.upper() in hpss_endpoint_map.keys():
remote_endpoint = hpss_endpoint_map.get(remote_endpoint.upper())
if transfer_type == "get":
src_ep = remote_endpoint
src_path = os.path.join(remote_path, name)
dst_ep = local_endpoint
dst_path = os.path.join(os.getcwd(), name)
else:
src_ep = local_endpoint
src_path = os.path.join(os.getcwd(), name)
dst_ep = remote_endpoint
dst_path = os.path.join(remote_path, name)
subdir = os.path.basename(os.path.normpath(remote_path))
subdir_label = re.sub("[^A-Za-z0-9_ -]", "", subdir)
filename = name.split(".")[0]
label = subdir_label + " " + filename
native_client = NativeClient(
client_id="6c1629cf-446c-49e7-af95-323c6412397f",
app_name="Zstash",
default_scopes=
"openid urn:globus:auth:scope:transfer.api.globus.org:all",
)
native_client.login(no_local_server=True, refresh_tokens=True)
transfer_authorizer = native_client.get_authorizers().get(
"transfer.api.globus.org")
tc = TransferClient(transfer_authorizer)
for ep_id in [src_ep, dst_ep]:
r = tc.endpoint_autoactivate(ep_id, if_expires_in=600)
if r.get("code") == "AutoActivationFailed":
logger.error(
"The {} endpoint is not activated or the current activation expires soon. Please go to https://app.globus.org/file-manager/collections/{} and (re)activate the endpoint."
.format(ep_id, ep_id))
sys.exit(1)
td = TransferData(
tc,
src_ep,
dst_ep,
label=label,
sync_level="checksum",
verify_checksum=True,
preserve_timestamp=True,
fail_on_quota_errors=True,
)
td.add_item(src_path, dst_path)
try:
task = tc.submit_transfer(td)
except TransferAPIError as e:
if e.code == "NoCredException":
logger.error(
"{}. Please go to https://app.globus.org/endpoints and activate the endpoint."
.format(e.message))
else:
logger.error(e)
sys.exit(1)
except Exception as e:
logger.error("Exception: {}".format(e))
sys.exit(1)
if non_blocking:
return
try:
task_id = task.get("task_id")
"""
A Globus transfer job (task) can be in one of the three states:
ACTIVE, SUCCEEDED, FAILED. The script every 20 seconds polls a
status of the transfer job (task) from the Globus Transfer service,
with 20 second timeout limit. If the task is ACTIVE after time runs
out 'task_wait' returns False, and True otherwise.
"""
while not tc.task_wait(task_id, 20, 20):
pass
"""
The Globus transfer job (task) has been finished (SUCCEEDED or FAILED).
Check if the transfer SUCCEEDED or FAILED.
"""
task = tc.get_task(task_id)
if task["status"] == "SUCCEEDED":
logger.info(
"Globus transfer {}, from {}{} to {}{} succeeded".format(
task_id, src_ep, src_path, dst_ep, dst_path))
else:
logger.error("Transfer FAILED")
except TransferAPIError as e:
if e.code == "NoCredException":
logger.error(
"{}. Please go to https://app.globus.org/endpoints and activate the endpoint."
.format(e.message))
else:
logger.error(e)
sys.exit(1)
except Exception as e:
logger.error("Exception: {}".format(e))
sys.exit(1)
def transfer(sp,destination_endpoint_id,one_endpoint):
tokens = None
try:
# if we already have tokens, load and use them
tokens = load_tokens_from_file(TOKEN_FILE)
except:
pass
if not tokens:
# if we need to get tokens, start the Native App authentication process
tokens = do_native_app_authentication(CLIENT_ID, REDIRECT_URI, SCOPES)
try:
save_tokens_to_file(TOKEN_FILE, tokens)
except:
pass
transfer_tokens = tokens['transfer.api.globus.org']
auth_client = NativeAppAuthClient(client_id=CLIENT_ID,environment='sandbox')
authorizer = RefreshTokenAuthorizer(
transfer_tokens['refresh_token'],
auth_client,
access_token=transfer_tokens['access_token'],
expires_at=transfer_tokens['expires_at_seconds'],
on_refresh=update_tokens_file_on_refresh)
#transfer = TransferClient(authorizer=authorizer,environment='sandbox')
tc = TransferClient(authorizer=authorizer, environment="sandbox")
##################---ENDPOINTS---###########################
source_endpoint_id = '5a2e5704-b028-11e7-bdad-22000bdb2406' #sb vmtb4
#source_endpoint_id = '55705028-aa15-11e7-bdad-22000bdb2406' #sb yulie7t
#source_endpoint_id = 'b0b16296-88e7-11e7-a971-22000a92523b' #bare chameleon
#source_endpoint_id = 'e5762bc2-8466-11e7-a8ed-22000a92523b' #large_chameleon
#source_endpoint_id = '8b26cc0e-877b-11e7-a949-22000a92523b'#ubuntu-vm
#source_endpoint_id = 'ad19b012-77cf-11e7-8b98-22000b9923ef'#chameleon
# source_endpoint_id = raw_input('Input source endpoint UUID: ')
#destination path
##############---SOURCE PATH---######################
#source_path = '/home/parallels/stream_transfer/test_files/'
#source_path = '/home/parallels/stream_transfer/zero_globus/test_files/'
source_path = sp
#source_path ='/home/cc/streaming/zero_globus/test_files/test.txt'
#source_path = '/home/parallels/stream_transfer/zero_globus/test_files/test.txt'
#destination path
destination_path = '/~/'
#destination_path = '/~/'+ sp.split("/")[-1] #use for one file
##if one_endpoint:
## destination_path = '/projects/BrainImagingADSP/yzamora/'
##else:
## destination_path = '/projects/BrainImagingADSP/yzamora/'+ sp.split("/")[-1] #use for one file
#Using my sample UUID from globus tutorial
#destination_endpoint_id = 'ddb59aef-6d04-11e5-ba46-22000b92c6ec' #globus
#destination_endpoint_id = '5d1da0fe-3c07-11e7-bcfc-22000b9a448b' #laptop
#tc.endpoint_autoactivate(source_endpoint_id)
#tc.endpoint_autoactive(destination_endpoint_id)
ep1 = tc.get_endpoint(destination_endpoint_id)
tc.endpoint_autoactivate(destination_endpoint_id)
#ep1 is setting the activated endpoint to be a variable to work with
tc.endpoint_autoactivate(source_endpoint_id)
label = "medium data transfer"
#tdata = globus_sdk.TransferData(tc, source_endpoint_id, destination_endpoint_id,label=label, sync_level='0')
tdata = globus_sdk.TransferData(tc, source_endpoint_id, destination_endpoint_id,label=label, perf_cc=3, sync_level=None, verify_checksum=False)
#tdata = globus_sdk.TransferData(tc, source_endpoint_id, destination_endpoint_id,label=label)
if one_endpoint:
tdata.add_item(source_path,destination_path,recursive=True)
else:
tdata.add_item(source_path,destination_path,recursive=False)
submit_result = tc.submit_transfer(tdata)
print("Task ID:", submit_result["task_id"])
"""
Checking for time completion using globus calls
"""
#print("Completion time:", submit_result["completion_time"])
#setup of the transfer, submits as a https post request
#transfer_data = TransferData(transfer_client=tc,
# source_endpoint=source_endpoint_id,
# destination_endpoint=destination_endpoint_id,
# label='Transfer',
# sync_level='checksum')
#transfer_data.add_item(source_path=source_path,destination_path=destination_path, recursive=False)
#task_id=transfer.submit_transfer(transfer_data)['task_id']
#waiting for file to transfer
status = tc.get_task(submit_result["task_id"],fields="status")["status"]
poll_interval = 2
max_wait = 90
wait_time = 0
while not (status in ["SUCCEEDED", "FAILED"]):
if (wait_time >= max_wait): break
print("Task not yet complete (status {}), sleeping for {} seconds..." \
.format(status, poll_interval))
time.sleep(poll_interval)
wait_time += poll_interval
status = tc.get_task(submit_result["task_id"], fields="status")["status"]
if status == "FAILED":
print("WARNING! File transfer FAILED!")
#deleting file after transfer
if status == "SUCCEEDED":
end_time = datetime.datetime.utcnow()
start_time = end_time - datetime.timedelta(minutes=200)
#limit = response objects
# data = tc.task_list(filter="type:TRANSFER,DELETE/request_time:%s,%s"
# % (start_time, end_time), limit=5)
#print("File transfer SUCCEEDED, will delete file from local directory now")
""" r = tc.task_list(num_results=1, filter="type:TRANSFER,DELETE")
class TestGlobus(TestZstash):
def preactivate_globus(self):
"""
Read the local globus endpoint UUID from ~/.zstash.ini.
If the ini file does not exist, create an ini file with empty values,
and try to find the local endpoint UUID based on the FQDN
"""
local_endpoint = None
ini_path = os.path.expanduser("~/.zstash.ini")
ini = configparser.ConfigParser()
if ini.read(ini_path):
if "local" in ini.sections():
local_endpoint = ini["local"].get("globus_endpoint_uuid")
else:
ini["local"] = {"globus_endpoint_uuid": ""}
try:
with open(ini_path, "w") as f:
ini.write(f)
except Exception as e:
self.fail(e)
if not local_endpoint:
fqdn = socket.getfqdn()
for pattern in regex_endpoint_map.keys():
if re.fullmatch(pattern, fqdn):
local_endpoint = regex_endpoint_map.get(pattern)
break
if not local_endpoint:
# self.fail("{} does not have the local Globus endpoint set".format(ini_path))
self.skipTest(
"{} does not have the local Globus endpoint set".format(
ini_path))
native_client = NativeClient(
client_id="6c1629cf-446c-49e7-af95-323c6412397f",
app_name="Zstash",
default_scopes=
"openid urn:globus:auth:scope:transfer.api.globus.org:all",
)
native_client.login(no_local_server=True, refresh_tokens=True)
transfer_authorizer = native_client.get_authorizers().get(
"transfer.api.globus.org")
self.transfer_client = TransferClient(transfer_authorizer)
for ep_id in [hpss_globus_endpoint, local_endpoint]:
r = self.transfer_client.endpoint_autoactivate(ep_id,
if_expires_in=600)
if r.get("code") == "AutoActivationFailed":
self.fail(
"The {} endpoint is not activated or the current activation expires soon. Please go to https://app.globus.org/file-manager/collections/{} and (re)-activate the endpoint."
.format(ep_id, ep_id))
def delete_files_globus(self):
ep_id = hpss_globus_endpoint
r = self.transfer_client.endpoint_autoactivate(ep_id, if_expires_in=60)
if r.get("code") == "AutoActivationFailed":
self.fail(
"The {} endpoint is not activated. Please go to https://app.globus.org/file-manager/collections/{} and activate the endpoint."
.format(ep_id, ep_id))
ddata = DeleteData(self.transfer_client,
hpss_globus_endpoint,
recursive=True)
ddata.add_item("/~/zstash_test/")
try:
task = self.transfer_client.submit_delete(ddata)
task_id = task.get("task_id")
"""
A Globus transfer job (task) can be in one of the three states:
ACTIVE, SUCCEEDED, FAILED. The script every 5 seconds polls a
status of the transfer job (task) from the Globus Transfer service,
with 5 second timeout limit. If the task is ACTIVE after time runs
out 'task_wait' returns False, and True otherwise.
"""
while not self.transfer_client.task_wait(task_id, 5, 5):
task = self.transfer_client.get_task(task_id)
if task.get("is_paused"):
break
"""
The Globus transfer job (task) has been finished (SUCCEEDED or FAILED),
or is still active (ACTIVE). Check if the transfer SUCCEEDED or FAILED.
"""
task = self.transfer_client.get_task(task_id)
if task["status"] == "SUCCEEDED":
pass
elif task.get("status") == "ACTIVE":
if task.get("is_paused"):
pause_info = self.transfer_client.task_pause_info(task_id)
paused_rules = pause_info.get("pause_rules")
reason = paused_rules[0].get("message")
message = "The task was paused. Reason: {}".format(reason)
print(message)
else:
message = "The task reached a {} second deadline\n".format(
24 * 3600)
print(message)
self.transfer_client.cancel_task(task_id)
else:
print("Globus delete FAILED")
except TransferAPIError as e:
if e.code == "NoCredException":
self.fail(
"{}. Please go to https://app.globus.org/endpoints and activate the endpoint."
.format(e.message))
else:
self.fail(e)
except Exception as e:
self.fail("{} - exception: {}".format(self, e))
def tearDown(self):
"""
Tear down a test. This is run after every test method.
After the script has failed or completed, remove all created files, even those on the HPSS repo.
"""
os.chdir(TOP_LEVEL)
print("Removing test files, both locally and at the HPSS repo")
# self.cache may appear in any of these directories,
# but should not appear at the same level as these.
# Therefore, there is no need to explicitly remove it.
for d in [self.test_dir, self.backup_dir]:
if os.path.exists(d):
shutil.rmtree(d)
if self.hpss_path and self.hpss_path.lower().startswith("globus:"):
self.delete_files_globus()
def helperLsGlobus(self,
test_name,
hpss_path,
cache=None,
zstash_path=ZSTASH_PATH):
"""
Test `zstash ls --hpss=globus://...`.
"""
self.preactivate_globus()
self.hpss_path = hpss_path
if cache:
# Override default cache
self.cache = cache
cache_option = " --cache={}".format(self.cache)
else:
cache_option = ""
use_hpss = self.setupDirs(test_name)
self.create(use_hpss, zstash_path, cache=self.cache)
self.assertWorkspace()
os.chdir(self.test_dir)
for option in ["", "-v", "-l"]:
print_starred("Testing zstash ls {}".format(option))
cmd = "{}zstash ls{} {} --hpss={}".format(zstash_path,
cache_option, option,
self.hpss_path)
output, err = run_cmd(cmd)
self.check_strings(cmd, output + err, ["file0.txt"], ["ERROR"])
os.chdir(TOP_LEVEL)
def testLs(self):
self.helperLsGlobus("testLsGlobus",
f"globus://{hpss_globus_endpoint}/~/zstash_test/")
class GlobusConnection(DecadeFileBase):
REDIRECT_URI = 'https://auth.globus.org/v2/web/auth-code'
SCOPES = ('openid email profile '
'urn:globus:auth:scope:transfer.api.globus.org:all')
TRANSLIMIT = 25
def __init__(self, config, tag):
DecadeFileBase.__init__(self, config, tag)
self.client = None
self.active_transfer_count = 0
self.noao_dirs = []
self.scantime = 0.
self.starttime = datetime.datetime.now()
self.active_transfer_count = 0
self.number_started = 0
self.number_successful = 0
self.number_failed = 0
self.number_waiting = 0
def __del__(self):
self.close()
logging.shutdown()
def load_tokens_from_file(self):
"""Load a set of saved tokens."""
with open(self.config['token_file'], 'r') as _file:
tokens = json.load(_file)
return tokens
def save_tokens_to_file(self, tokens):
"""Save a set of tokens for later use."""
with open(self.config['token_file'], 'w') as _file:
json.dump(tokens, _file)
def update_tokens_file_on_refresh(self, token_response):
"""
Callback function passed into the RefreshTokenAuthorizer.
Will be invoked any time a new access token is fetched.
"""
self.save_tokens_to_file(token_response.by_resource_server)
def initiate_connection(self):
""" Initiate the connection
"""
tokens = None
try:
# if we already have tokens, load and use them
tokens = self.load_tokens_from_file()
except:
pass
if not tokens:
# if we need to get tokens, start the Native App authentication process
client = NativeAppAuthClient(
client_id=self.CLIENT_ID) #self.config['client_id'])
# pass refresh_tokens=True to request refresh tokens
client.oauth2_start_flow(
requested_scopes=self.
SCOPES, #self.config['requested_scopes'],
redirect_uri=self.REDIRECT_URI, #self.config['redirect_uri'],
refresh_tokens=True)
url = client.oauth2_get_authorize_url()
print 'Native App Authorization URL: \n{}'.format(url)
auth_code = raw_input('Enter the auth code: ').strip()
token_response = client.oauth2_exchange_code_for_tokens(auth_code)
# return a set of tokens, organized by resource server name
tokens = token_response.by_resource_server
try:
self.save_tokens_to_file(tokens)
except:
pass
transfer_tokens = tokens['transfer.api.globus.org']
auth_client = NativeAppAuthClient(client_id=self.config['client_id'])
authorizer = RefreshTokenAuthorizer(
transfer_tokens['refresh_token'],
auth_client,
#access_token=transfer_tokens['access_token'],
#expires_at=transfer_tokens['expires_at_seconds'],
on_refresh=self.update_tokens_file_on_refresh)
self.client = TransferClient(authorizer=authorizer)
# print out a directory listing from an endpoint
try:
#print 'ACTIVATE'
#print 'DEST',self.config['dest_ep']
self.client.endpoint_autoactivate(self.config['dest_ep'])
ac = self.client.endpoint_get_activation_requirements(
self.config['dest_ep'])
#print ac
self.client.endpoint_autoactivate(self.config['src_ep'])
ac2 = self.client.endpoint_get_activation_requirements(
self.config['src_ep'])
#print ac2
except GlobusAPIError as ex:
self.logger.error('Error in endpoint activation %s', str(ex))
if ex.http_status == 401:
sys.exit('Refresh token has expired. '
'Please delete refresh-tokens.json and try again.')
else:
raise ex
def get_dirs(self):
""" function
"""
now = time.time()
full_dirs = self.client.operation_ls(self.config['src_ep'],
path=self.config['noao_root'])
self.scantime = time.time() - now
for entry in full_dirs:
if entry['type'].lower() == 'dir':
self.noao_dirs.append(str(entry['name']))
#!/usr/bin/env python2.7
from __future__ import print_function # for python 2
from globus_sdk import TransferClient
import globus_sdk
import traceback
import argparse
parser = argparse.ArgumentParser(description='kbase share creator')
parser.add_argument('--share-dir', dest='sharedDir',
help='Directory to create a share on')
parser.add_argument('--share-name', dest='shareName',
help='name for the share (must be unique among all globus shares)')
args = parser.parse_args()
tc = TransferClient() # uses transfer_token from the config file
shared_ep_data = {
"DATA_TYPE": "shared_endpoint",
"host_endpoint": 'e25a4bda-0636-11e6-a732-22000bf2d559',
"host_path": args.sharedDir,
"display_name": args.shareName,
# optionally specify additional endpoint fields
"description": "autocreated by kbase tool support"
}
tc.endpoint_autoactivate('e25a4bda-0636-11e6-a732-22000bf2d559', if_expires_in=3600)
create_result = tc.create_shared_endpoint(shared_ep_data)
endpoint_id = create_result["id"]
print("new endpoint id: ", endpoint_id)
def doit():
"""
- Call token introspect
- Get dependent tokens
"""
dependent_tokens = get_dependent_tokens(g.req_token)
# dependent_tokens is a token response object
# create transfer_token and http_token variables containing
# the correct token for each resource server
transfer_token = dependent_tokens.by_resource_server[
'transfer.api.globus.org']['access_token']
http_token = dependent_tokens.by_resource_server[
app.config['GRAPH_ENDPOINT_ID']]['access_token']
selected_ids = request.form.getlist('datasets')
selected_year = request.form.get('year')
user_identity_id = request.form.get('user_identity_id')
user_identity_name = request.form.get('user_identity_name')
selected_datasets = [dataset for dataset in datasets
if dataset['id'] in selected_ids]
if not (selected_datasets and selected_year):
raise BadRequestError()
transfer = TransferClient(authorizer=AccessTokenAuthorizer(transfer_token))
source_ep = app.config['DATASET_ENDPOINT_ID']
source_info = transfer.get_endpoint(source_ep)
source_https = source_info['https_server']
source_base = app.config['DATASET_ENDPOINT_BASE']
source_token = http_token
dest_ep = app.config['GRAPH_ENDPOINT_ID']
dest_info = transfer.get_endpoint(dest_ep)
dest_https = dest_info['https_server']
dest_base = app.config['GRAPH_ENDPOINT_BASE']
dest_path = '%sGraphs for %s/' % (dest_base, user_identity_name)
dest_token = http_token
if not (source_https and dest_https):
raise InternalServerError(message='Endpoints must be HTTPS servers')
svgs = {}
for dataset in selected_datasets:
source_path = dataset['path']
response = requests.get('%s%s%s/%s.csv' % (source_https, source_base,
source_path, selected_year),
headers=dict(
Authorization='Bearer ' + source_token),
allow_redirects=False)
response.raise_for_status()
svgs.update(render_graphs(
csv_data=response.iter_lines(decode_unicode=True),
append_titles=' from %s for %s' % (dataset['name'], selected_year),
))
transfer.endpoint_autoactivate(dest_ep)
try:
transfer.operation_mkdir(dest_ep, dest_path)
except TransferAPIError as error:
if 'MkdirFailed.Exists' not in error.code:
raise
try:
transfer.add_endpoint_acl_rule(
dest_ep,
dict(principal=user_identity_id,
principal_type='identity', path=dest_path, permissions='r'),
)
except TransferAPIError as error:
# PermissionDenied can happen if a new Portal client is swapped
# in and it doesn't have endpoint manager on the dest_ep.
# The /portal/processed directory has been set to to read/write
# for all users so the subsequent operations will succeed.
if error.code == 'PermissionDenied':
pass
elif error.code != 'Exists':
raise
for filename, svg in svgs.items():
requests.put('%s%s%s.svg' % (dest_https, dest_path, filename),
data=svg,
headers=dict(Authorization='Bearer ' + dest_token),
allow_redirects=False).raise_for_status()
results = {
'dest_ep': dest_ep,
'dest_path': dest_path,
'dest_name': dest_info['display_name'],
'graph_count': len(svgs) or 0
}
return jsonify(results)
def validate():
params = request.json
crawl_id = params["crawl_id"]
globus_eid = params["globus_eid"]
transfer_token = params["transfer_token"]
source_destination = params["source_destination"]
dataset_info = params["dataset_info"] # To be implemented later
client = boto3.client('sqs',
aws_access_key_id=os.environ["aws_access"],
aws_secret_access_key=os.environ["aws_secret"],
region_name='us-east-1')
try:
response = client.get_queue_url(
QueueName=f'validate_{crawl_id}',
QueueOwnerAWSAccountId=os.environ["aws_account_id"])
except: # Add SQS.Client.exceptions.QueueDoesNotExist error
abort(400, "Invalid crawl ID")
try:
authorizer = AccessTokenAuthorizer(transfer_token)
tc = TransferClient(authorizer=authorizer)
except: # Add exception
abort(400, "Invalid transfer token")
crawl_queue = response["QueueUrl"]
date = datetime.datetime.now()
file_name = date.strftime("%m_%d_%Y-%H_%M_%S") + ".txt"
try:
with open(file_name, "w") as f:
while True:
sqs_response = client.receive_message(
QueueUrl=crawl_queue,
MaxNumberOfMessages=1, # To be toggled
WaitTimeSeconds=1)
if "Messages" not in sqs_response:
# xtract_status = requests.get(f"{eb_url}/get_extract_status", json={"crawl_id": crawl_id})
# print("HERE")
# print(xtract_status.content)
# xtract_content = json.loads(xtract_status.content)
# # print(xtract_content)
#
# if xtract_content["IDLE"] == 0 and xtract_content["PENDING"] == 0:
break
del_list = []
for message in sqs_response["Messages"]:
message_body = message["Body"]
# PROCESS MESSAGE_BODY
f.write(message_body)
# print(message_body)
del_list.append({
'ReceiptHandle': message["ReceiptHandle"],
'Id': message["MessageId"]
})
if len(del_list) > 0:
client.delete_message_batch(QueueUrl=crawl_queue,
Entries=del_list)
tdata = TransferData(
tc,
"5ecf6444-affc-11e9-98d4-0a63aa6b37da", #TODO: Add source endpoint
globus_eid,
label=f"{crawl_id}")
tdata.add_item(os.path.abspath(file_name),
os.path.join(source_destination, file_name))
tc.endpoint_autoactivate(
"5ecf6444-affc-11e9-98d4-0a63aa6b37da") #TODO: Add source endpoint
tc.endpoint_autoactivate(globus_eid)
submit_result = tc.submit_transfer(tdata)
while True:
result = tc.get_task(submit_result['task_id'])
if result.data["status"] == "SUCCEEDED":
break
elif result.data["status"] == "FAILED":
raise RuntimeError # TODO: Change this
else:
time.sleep(0.5)
except Exception as e:
print(e)
abort(400, "Failed to validate")
finally:
os.remove(file_name)
return "[200] Submitted"
