def doit():
"""
- Call token introspect
- Get dependent tokens
"""
dependent_tokens = get_dependent_tokens(g.req_token)
# dependent_tokens is a token response object
# create transfer_token and http_token variables containing
# the correct token for each resource server
transfer_token = dependent_tokens.by_resource_server[
'transfer.api.globus.org']['access_token']
http_token = dependent_tokens.by_resource_server[
'tutorial-https-endpoint.globus.org']['access_token']
selected_ids = request.form.getlist('datasets')
selected_year = request.form.get('year')
user_identity_id = request.form.get('user_identity_id')
user_identity_name = request.form.get('user_identity_name')
selected_datasets = [
dataset for dataset in datasets if dataset['id'] in selected_ids
]
if not (selected_datasets and selected_year):
raise BadRequestError()
transfer = TransferClient(authorizer=AccessTokenAuthorizer(transfer_token))
source_ep = app.config['DATASET_ENDPOINT_ID']
source_info = transfer.get_endpoint(source_ep)
source_https = source_info['https_server']
source_base = app.config['DATASET_ENDPOINT_BASE']
source_token = http_token
dest_ep = app.config['GRAPH_ENDPOINT_ID']
dest_info = transfer.get_endpoint(dest_ep)
dest_https = dest_info['https_server']
dest_base = app.config['GRAPH_ENDPOINT_BASE']
dest_path = '%sGraphs for %s/' % (dest_base, user_identity_name)
dest_token = http_token
if not (source_https and dest_https):
raise InternalServerError(message='Endpoints must be HTTPS servers')
svgs = {}
for dataset in selected_datasets:
source_path = dataset['path']
response = requests.get(
'%s%s%s/%s.csv' %
(source_https, source_base, source_path, selected_year),
headers=dict(Authorization='Bearer ' + source_token),
allow_redirects=False)
response.raise_for_status()
svgs.update(
render_graphs(
csv_data=response.iter_lines(decode_unicode=True),
append_titles=' from %s for %s' %
(dataset['name'], selected_year),
))
transfer.endpoint_autoactivate(dest_ep)
try:
transfer.operation_mkdir(dest_ep, dest_path)
except TransferAPIError as error:
if 'MkdirFailed.Exists' not in error.code:
raise
try:
transfer.add_endpoint_acl_rule(
dest_ep,
dict(principal=user_identity_id,
principal_type='identity',
path=dest_path,
permissions='r'),
)
except TransferAPIError as error:
# PermissionDenied can happen if a new Portal client is swapped
# in and it doesn't have endpoint manager on the dest_ep.
# The /portal/processed directory has been set to to read/write
# for all users so the subsequent operations will succeed.
if error.code == 'PermissionDenied':
pass
elif error.code != 'Exists':
raise
for filename, svg in svgs.items():
requests.put('%s%s%s.svg' % (dest_https, dest_path, filename),
data=svg,
headers=dict(Authorization='Bearer ' + dest_token),
allow_redirects=False).raise_for_status()
results = {
'dest_ep': dest_ep,
'dest_path': dest_path,
'dest_name': dest_info['display_name'],
'graph_count': len(svgs) or 0
}
return jsonify(results)
class GlobusStorageManager:
# https://globus-sdk-python.readthedocs.io/en/stable/clients/transfer/
app_id = 'b2fe5703-edb0-4f7f-80a6-2147c8ae35f0' # map transfer app id
class GlobusQueue:
''' placeholder for globus async helpers '''
pass
def __init__(self):
self.auth_client = NativeAppAuthClient(self.app_id)
self.auth_client.oauth2_start_flow(refresh_tokens=True)
self.xfer_client = None
custom = dj.config.get('custom', None)
if custom and 'globus.token' in custom:
self.refresh()
else:
self.login()
# authentication methods
def login(self):
''' fetch refresh token, store in dj.config['globus.token'] '''
auth_client = self.auth_client
print('Please login via: {}'.format(
auth_client.oauth2_get_authorize_url()))
code = input('and enter code:').strip()
tokens = auth_client.oauth2_exchange_code_for_tokens(code)
xfer_auth_cfg = tokens.by_resource_server['transfer.api.globus.org']
xfer_rt = xfer_auth_cfg['refresh_token']
xfer_at = xfer_auth_cfg['access_token']
xfer_exp = xfer_auth_cfg['expires_at_seconds']
xfer_auth = RefreshTokenAuthorizer(
xfer_rt, auth_client, access_token=xfer_at, expires_at=xfer_exp)
self.xfer_client = TransferClient(authorizer=xfer_auth)
custom = dj.config.get('custom', {})
custom['globus.token'] = xfer_rt
dj.config['custom'] = custom
def refresh(self):
''' use refresh token to refresh access token '''
auth_client = self.auth_client
xfer_auth = RefreshTokenAuthorizer(
dj.config['custom']['globus.token'], auth_client,
access_token=None, expires_at=None)
self.xfer_client = TransferClient(authorizer=xfer_auth)
# endpoint managment / utility methods
@classmethod
def ep_parts(cls, endpoint_path):
# split endpoint:/path to endpoint, path
epsplit = endpoint_path.split(':')
return epsplit[0], ':'.join(epsplit[1:])
def activate_endpoint(self, endpoint):
''' activate an endpoint '''
tc = self.xfer_client
r = tc.endpoint_autoactivate(endpoint, if_expires_in=3600)
log.debug('activate_endpoint() code: {}'.format(r['code']))
if r['code'] == 'AutoActivationFailed':
print('Endpoint({}) Not Active! Error! Source message: {}'
.format(endpoint, r['message']))
raise Exception('globus endpoint activation failure')
knownok = any(('AutoActivated' in r['code'],
'AlreadyActivated' in r['code']))
if not knownok:
log.debug('activate_endpoint(): not knownok response')
def _wait(self, task, timeout=10, polling_interval=10):
''' tranfer client common wait wrapper '''
return self.xfer_client.task_wait(task, timeout, polling_interval)
def _tasks(self):
'''
>>> tl = tc.task_list(num_results=25, filter="type:TRANSFER,DELETE")
>>> _ = [print(t["task_id"], t["type"], t["status"]) for t in tl]
'''
pass
def _task_info(self):
'''
>>> for event in tc.task_event_list(task_id):
>>> print("Event on Task({}) at {}:\n{}".format(
>>> task_id, event["time"], event["description"])
or
get_task
'''
pass
# transfer methods
def ls(self, endpoint_path):
'''
returns:
{
"DATA": [
{
"DATA_TYPE": "file",
"group": "staff",
"last_modified": "2018-05-22 18:49:19+00:00",
"link_group": null,
"link_last_modified": null,
"link_size": null,
"link_target": null,
"link_user": null,
"name": "map",
"permissions": "0755",
"size": 102,
"type": "dir",
"user": "******"
},
],
"DATA_TYPE": "file_list",
"absolute_path": null,
"endpoint": "aa4e5f9c-05f3-11e8-a6ad-0a448319c2f8",
"length": 2,
"path": "/~/Globus/",
"rename_supported": true,
"symlink_supported": false,
"total": 2
}
'''
ep, path = self.ep_parts(endpoint_path)
return self.xfer_client.operation_ls(ep, path=path)
def mkdir(self, ep_path):
''' create a directory at ep_path '''
ep, path = self.ep_parts(ep_path)
return self.xfer_client.operation_mkdir(ep, path=path)
def rmdir(self, ep_path, recursive=False):
''' remove a directory at ep_path '''
tc = self.xfer_client
ep, path = self.ep_parts(ep_path)
ddata = DeleteData(tc, ep, recursive=recursive)
ddata.add_item(path)
task_id = tc.submit_delete(ddata)['task_id']
return self._wait(task_id)
def cp(self, src_ep_path, dst_ep_path, recursive=False):
'''
copy file/path
todo: support label, sync_level, etc?
sync_level: ["exists", "size", "mtime", "checksum"]
'''
tc = self.xfer_client
sep, spath = self.ep_parts(src_ep_path)
dep, dpath = self.ep_parts(dst_ep_path)
td = TransferData(tc, sep, dep)
td.add_item(spath, dpath, recursive=recursive)
task_id = tc.submit_transfer(td)['task_id']
return self._wait(task_id)
def rename(self, src_ep_path, dst_ep_path):
''' rename a file/path '''
tc = self.xfer_client
sep, spath = self.ep_parts(src_ep_path)
dep, dpath = self.ep_parts(dst_ep_path)
if sep != dep:
raise Exception('rename between two different endpoints')
return tc.operation_rename(sep, spath, dpath)
def doit():
"""
- Call token introspect
- Get dependent tokens
"""
dependent_tokens = get_dependent_tokens(g.req_token)
# dependent_tokens is a token response object
# create transfer_token and http_token variables containing
# the correct token for each resource server
transfer_token = dependent_tokens.by_resource_server[
'transfer.api.globus.org']['access_token']
http_token = dependent_tokens.by_resource_server[
app.config['GRAPH_ENDPOINT_ID']]['access_token']
selected_ids = request.form.getlist('datasets')
selected_year = request.form.get('year')
user_identity_id = request.form.get('user_identity_id')
user_identity_name = request.form.get('user_identity_name')
selected_datasets = [dataset for dataset in datasets
if dataset['id'] in selected_ids]
if not (selected_datasets and selected_year):
raise BadRequestError()
transfer = TransferClient(authorizer=AccessTokenAuthorizer(transfer_token))
source_ep = app.config['DATASET_ENDPOINT_ID']
source_info = transfer.get_endpoint(source_ep)
source_https = source_info['https_server']
source_base = app.config['DATASET_ENDPOINT_BASE']
source_token = http_token
dest_ep = app.config['GRAPH_ENDPOINT_ID']
dest_info = transfer.get_endpoint(dest_ep)
dest_https = dest_info['https_server']
dest_base = app.config['GRAPH_ENDPOINT_BASE']
dest_path = '%sGraphs for %s/' % (dest_base, user_identity_name)
dest_token = http_token
if not (source_https and dest_https):
raise InternalServerError(message='Endpoints must be HTTPS servers')
svgs = {}
for dataset in selected_datasets:
source_path = dataset['path']
response = requests.get('%s%s%s/%s.csv' % (source_https, source_base,
source_path, selected_year),
headers=dict(
Authorization='Bearer ' + source_token),
allow_redirects=False)
response.raise_for_status()
svgs.update(render_graphs(
csv_data=response.iter_lines(decode_unicode=True),
append_titles=' from %s for %s' % (dataset['name'], selected_year),
))
transfer.endpoint_autoactivate(dest_ep)
try:
transfer.operation_mkdir(dest_ep, dest_path)
except TransferAPIError as error:
if 'MkdirFailed.Exists' not in error.code:
raise
try:
transfer.add_endpoint_acl_rule(
dest_ep,
dict(principal=user_identity_id,
principal_type='identity', path=dest_path, permissions='r'),
)
except TransferAPIError as error:
# PermissionDenied can happen if a new Portal client is swapped
# in and it doesn't have endpoint manager on the dest_ep.
# The /portal/processed directory has been set to to read/write
# for all users so the subsequent operations will succeed.
if error.code == 'PermissionDenied':
pass
elif error.code != 'Exists':
raise
for filename, svg in svgs.items():
requests.put('%s%s%s.svg' % (dest_https, dest_path, filename),
data=svg,
headers=dict(Authorization='Bearer ' + dest_token),
allow_redirects=False).raise_for_status()
results = {
'dest_ep': dest_ep,
'dest_path': dest_path,
'dest_name': dest_info['display_name'],
'graph_count': len(svgs) or 0
}
return jsonify(results)
