def init():
tokens = None
try:
# if we already have tokens, load and use them
tokens = load_tokens_from_db()
except:
pass
if not tokens:
# if we need to get tokens, start the Native App authentication process
tokens = do_native_app_authentication(config.TRANSFER_CLIENT_ID,
config.REDIRECT_URI,
config.SCOPES)
try:
save_tokens_to_db(tokens)
except:
pass
transfer_tokens = tokens['transfer.api.globus.org']
client = NativeAppAuthClient(client_id=config.TRANSFER_CLIENT_ID)
authorizer = RefreshTokenAuthorizer(
transfer_tokens['refresh_token'],
client,
access_token=transfer_tokens['access_token'],
expires_at=transfer_tokens['expires_at_seconds'],
on_refresh=update_tokens_file_on_refresh)
transfer = TransferClient(authorizer=authorizer)
prepare_call(transfer)
return transfer
def refresh_tokens(self, tokens):
if not self._refreshable(tokens):
raise TokensExpired('No Refresh Token, cannot refresh tokens: ',
resource_servers=tokens.keys())
for rs, token_dict in tokens.items():
authorizer = RefreshTokenAuthorizer(
token_dict['refresh_token'],
self.client,
access_token=token_dict['access_token'],
expires_at=token_dict['expires_at_seconds'],
)
authorizer.check_expiration_time()
token_dict['access_token'] = authorizer.access_token
token_dict['expires_at_seconds'] = authorizer.expires_at
return tokens
def identifiers_client(config, **kwargs):
app_name = 'identifier_client'
base_url = config.get('client', 'service_url')
client_id = config.get('client', 'client_id')
access_token = config.get('tokens', 'access_token')
at_expires = int(config.get('tokens', 'access_token_expires'))
refresh_token = config.get('tokens', 'refresh_token')
if not (refresh_token and access_token):
raise IdentifierNotLoggedIn("Missing tokens")
def _on_refresh(tkn):
extract_and_save_tokens(tkn, config)
authorizer_client = NativeAppAuthClient(client_id, app_name=app_name)
authorizer = RefreshTokenAuthorizer(
refresh_token,
authorizer_client,
access_token,
at_expires,
on_refresh=_on_refresh,
)
return IdentifierClient(base_url=base_url,
app_name=app_name,
authorizer=authorizer,
**kwargs)
def login(self):
''' fetch refresh token, store in dj.config['globus.token'] '''
auth_client = self.auth_client
print('Please login via: {}'.format(
auth_client.oauth2_get_authorize_url()))
code = input('and enter code:').strip()
tokens = auth_client.oauth2_exchange_code_for_tokens(code)
xfer_auth_cfg = tokens.by_resource_server['transfer.api.globus.org']
xfer_rt = xfer_auth_cfg['refresh_token']
xfer_at = xfer_auth_cfg['access_token']
xfer_exp = xfer_auth_cfg['expires_at_seconds']
xfer_auth = RefreshTokenAuthorizer(xfer_rt,
auth_client,
access_token=xfer_at,
expires_at=xfer_exp)
self.xfer_client = TransferClient(authorizer=xfer_auth)
custom = dj.config.get('custom', {})
custom['globus.token'] = xfer_rt
dj.config['custom'] = custom
def identifier_client(config, **kwargs):
app_name = 'identifier_client'
base_url = config['client']['service_url']
client_id = config['client']['client_id']
access_token = config['tokens']['access_token']
at_expires = int(config['tokens']['access_token_expires'])
refresh_token = config['tokens']['refresh_token']
if not (refresh_token and access_token):
raise IdentifierNotLoggedIn("Missing tokens")
def _on_refresh(tkn):
# extract_and_save_tokens(tkn, config)
pass
authorizer_client = NativeAppAuthClient(client_id, app_name=app_name)
authorizer = RefreshTokenAuthorizer(
refresh_token,
authorizer_client,
access_token,
at_expires,
on_refresh=_on_refresh,
)
return IdentifierClient(
"identifier",
base_url=base_url,
app_name=app_name,
authorizer=authorizer,
**kwargs
)
def getTransferData():
cfg = load_config()
client_id = cfg['globus']['apps']['SDK Tutorial App']['client_id']
auth_client = NativeAppAuthClient(client_id)
refresh_token = cfg['globus']['apps']['SDK Tutorial App']['refresh_token']
source_endpoint_id = cfg['globus']['apps']['SDK Tutorial App'][
'win10_endpoint_id']
destination_endpoint_id = cfg['globus']['apps']['SDK Tutorial App'][
'sdccfed_endpoint_id']
authorizer = RefreshTokenAuthorizer(refresh_token=refresh_token,
auth_client=auth_client)
tc = TransferClient(authorizer=authorizer)
# as both endpoints are expected to be Globus Server endpoints, send auto-activate commands for both globus endpoints
auto_activate_endpoint(tc, source_endpoint_id)
auto_activate_endpoint(tc, destination_endpoint_id)
# make job_label for task a timestamp
x = datetime.now()
job_label = x.strftime('%Y%m%d%H%M%s')
# from Globus... sync_level=checksum means that before files are transferred, Globus will compute checksums on the source and destination files,
# and only transfer files that have different checksums are transferred. verify_checksum=True means that after a file is transferred, Globus will
# compute checksums on the source and destination files to verify that the file was transferred correctly. If the checksums do not match, it will
# redo the transfer of that file.
tdata = TransferData(tc,
source_endpoint_id,
destination_endpoint_id,
label=job_label,
sync_level="checksum",
verify_checksum=True)
return tdata
def browse(dataset_id=None, endpoint_id=None, endpoint_path=None):
"""
- Get list of files for the selected dataset or endpoint ID/path
- Return a list of files to a browse view
The target template (browse.jinja2) expects an `endpoint_uri` (if
available for the endpoint), `target` (either `"dataset"`
or `"endpoint"`), and 'file_list' (list of dictionaries) containing
the following information about each file in the result:
{'name': 'file name', 'size': 'file size', 'id': 'file uri/path'}
If you want to display additional information about each file, you
must add those keys to the dictionary and modify the browse.jinja2
template accordingly.
"""
assert bool(dataset_id) != bool(endpoint_id and endpoint_path)
if dataset_id:
try:
dataset = next(ds for ds in datasets if ds['id'] == dataset_id)
except StopIteration:
abort(404)
endpoint_id = app.config['DATASET_ENDPOINT_ID']
endpoint_path = app.config['DATASET_ENDPOINT_BASE'] + dataset['path']
else:
endpoint_path = '/' + endpoint_path
transfer = TransferClient(authorizer=RefreshTokenAuthorizer(
session['tokens']['transfer.api.globus.org']['refresh_token'],
load_portal_client()))
try:
transfer.endpoint_autoactivate(endpoint_id)
listing = transfer.operation_ls(endpoint_id, path=endpoint_path)
except TransferAPIError as err:
flash('Error [{}]: {}'.format(err.code, err.message))
return redirect(url_for('transfer'))
file_list = [e for e in listing if e['type'] == 'file']
ep = transfer.get_endpoint(endpoint_id)
https_server = ep['https_server']
endpoint_uri = https_server + endpoint_path if https_server else None
webapp_xfer = 'https://www.globus.org/app/transfer?' + \
urlencode(dict(origin_id=endpoint_id, origin_path=endpoint_path))
return render_template(
'browse.jinja2',
endpoint_uri=endpoint_uri,
target="dataset" if dataset_id else "endpoint",
description=(dataset['name'] if dataset_id else ep['display_name']),
file_list=file_list,
webapp_xfer=webapp_xfer)
def refresh(self):
''' use refresh token to refresh access token '''
auth_client = self.auth_client
xfer_auth = RefreshTokenAuthorizer(
dj.config['custom']['globus.token'], auth_client,
access_token=None, expires_at=None)
self.xfer_client = TransferClient(authorizer=xfer_auth)
def login():
tok_path = os.path.expanduser('~/.mdf_agent_tokens.json')
def _read_tokfile():
tokens = {}
if os.path.exists(tok_path):
with open(tok_path) as f:
tokens = json.load(f)
return tokens
def _write_tokfile(new_tokens):
# We have multiple tokens in our tokens file, but on update we only
# get the currently updated token, so read current and update with the
# input tokens
cur_tokens = _read_tokfile()
for key in new_tokens:
cur_tokens[key] = new_tokens[key]
# deny rwx to Group and World -- don't bother storing the returned old
# mask value, since we'll never restore it anyway
# do this on every call to ensure that we're always consistent about it
os.umask(0o077)
with open(tok_path, 'w') as f:
f.write(json.dumps(cur_tokens))
def _update_tokfile(tokens):
_write_tokfile(tokens.by_resource_server['transfer.api.globus.org'])
tokens = _read_tokfile()
client_id = "1e162bfc-ad52-4014-8844-b82841145fc4"
native_client = NativeAppAuthClient(client_id, app_name='MDF Agents')
if not tokens:
# and do the Native App Grant flow
native_client.oauth2_start_flow(
requested_scopes='urn:globus:auth:scope:transfer.api.globus.org:all',
refresh_tokens=True)
linkprompt = 'Please login to Globus here'
print('{0}:\n{1}\n{2}\n{1}\n'
.format(linkprompt, '-' * len(linkprompt),
native_client.oauth2_get_authorize_url()), flush=True)
auth_code = input(
'Enter the resulting Authorization Code here').strip()
tkns = native_client.oauth2_exchange_code_for_tokens(auth_code)
tokens = tkns.by_resource_server['transfer.api.globus.org']
_write_tokfile(tokens)
transfer_tokens = tokens
transfer_authorizer = RefreshTokenAuthorizer(
transfer_tokens['refresh_token'], native_client,
transfer_tokens['access_token'], transfer_tokens['expires_at_seconds'],
on_refresh=_update_tokfile)
transfer_client = TransferClient(authorizer=transfer_authorizer)
return transfer_client
def get_transfer_client():
cfg = load_config()
# cfg = yaml.safe_load(open("/opt/rucio/lib/rucio/transfertool/config.yml"))
client_id = cfg['globus']['apps'][GLOBUS_AUTH_APP]['client_id']
auth_client = NativeAppAuthClient(client_id)
refresh_token = cfg['globus']['apps'][GLOBUS_AUTH_APP]['refresh_token']
logging.info('authorizing token...')
authorizer = RefreshTokenAuthorizer(refresh_token=refresh_token, auth_client=auth_client)
logging.info('initializing TransferClient...')
tc = TransferClient(authorizer=authorizer)
return tc
def bulk_submit_xfer(submitjob, recursive=False):
cfg = load_config()
client_id = cfg['globus']['apps'][GLOBUS_AUTH_APP]['client_id']
auth_client = NativeAppAuthClient(client_id)
refresh_token = cfg['globus']['apps'][GLOBUS_AUTH_APP]['refresh_token']
source_endpoint_id = submitjob[0].get('metadata').get(
'source_globus_endpoint_id')
destination_endpoint_id = submitjob[0].get('metadata').get(
'dest_globus_endpoint_id')
authorizer = RefreshTokenAuthorizer(refresh_token=refresh_token,
auth_client=auth_client)
tc = TransferClient(authorizer=authorizer)
# as both endpoints are expected to be Globus Server endpoints, send auto-activate commands for both globus endpoints
a = auto_activate_endpoint(tc, source_endpoint_id)
logging.debug('a: %s' % a)
if a != 'AlreadyActivated':
return None
b = auto_activate_endpoint(tc, destination_endpoint_id)
logging.debug('b: %s' % b)
if b != 'AlreadyActivated':
return None
# make job_label for task a timestamp
x = datetime.now()
job_label = x.strftime('%Y%m%d%H%M%s')
# from Globus... sync_level=checksum means that before files are transferred, Globus will compute checksums on the source
# and destination files, and only transfer files that have different checksums are transferred. verify_checksum=True means
# that after a file is transferred, Globus will compute checksums on the source and destination files to verify that the
# file was transferred correctly. If the checksums do not match, it will redo the transfer of that file.
tdata = TransferData(tc,
source_endpoint_id,
destination_endpoint_id,
label=job_label,
sync_level="checksum")
for file in submitjob:
source_path = file.get('sources')[0]
dest_path = file.get('destinations')[0]
filesize = file['metadata']['filesize']
# TODO: support passing a recursive parameter to Globus
# md5 = file['metadata']['md5']
# tdata.add_item(source_path, dest_path, recursive=False, external_checksum=md5)
tdata.add_item(source_path, dest_path, recursive=False)
record_counter(
'daemons.conveyor.transfer_submitter.globus.transfers.submit.filesize',
filesize)
# logging.info('submitting transfer...')
transfer_result = tc.submit_transfer(tdata)
# logging.info("task_id =", transfer_result["task_id"])
return transfer_result["task_id"]
def submit_transfer():
"""
- Take the data returned by the Browse Endpoint helper page
and make a Globus transfer request.
- Send the user to the transfer status page with the task id
from the transfer.
"""
browse_endpoint_form = request.form
selected = session['form']['datasets']
filtered_datasets = [ds for ds in datasets if ds['id'] in selected]
transfer_tokens = session['tokens']['transfer.api.globus.org']
authorizer = RefreshTokenAuthorizer(
transfer_tokens['refresh_token'],
load_portal_client(),
access_token=transfer_tokens['access_token'],
expires_at=transfer_tokens['expires_at_seconds'])
transfer = TransferClient(authorizer=authorizer)
source_endpoint_id = app.config['DATASET_ENDPOINT_ID']
source_endpoint_base = app.config['DATASET_ENDPOINT_BASE']
destination_endpoint_id = browse_endpoint_form['endpoint_id']
destination_folder = browse_endpoint_form.get('folder[0]')
transfer_data = TransferData(transfer_client=transfer,
source_endpoint=source_endpoint_id,
destination_endpoint=destination_endpoint_id,
label=browse_endpoint_form.get('label'))
for ds in filtered_datasets:
source_path = source_endpoint_base + ds['path']
dest_path = browse_endpoint_form['path']
if destination_folder:
dest_path += destination_folder + '/'
dest_path += ds['name'] + '/'
transfer_data.add_item(source_path=source_path,
destination_path=dest_path,
recursive=True)
transfer.endpoint_autoactivate(source_endpoint_id)
transfer.endpoint_autoactivate(destination_endpoint_id)
task_id = transfer.submit_transfer(transfer_data)['task_id']
flash('Transfer request submitted successfully. Task ID: ' + task_id)
return (redirect(url_for('transfer_status', task_id=task_id)))
def bulk_submit_xfer(submitjob, recursive=False, logger=logging.log):
cfg = load_config(logger=logger)
client_id = cfg['globus']['apps'][GLOBUS_AUTH_APP]['client_id']
auth_client = NativeAppAuthClient(client_id)
refresh_token = cfg['globus']['apps'][GLOBUS_AUTH_APP]['refresh_token']
source_endpoint_id = submitjob[0].get('metadata').get(
'source_globus_endpoint_id')
destination_endpoint_id = submitjob[0].get('metadata').get(
'dest_globus_endpoint_id')
authorizer = RefreshTokenAuthorizer(refresh_token=refresh_token,
auth_client=auth_client)
tc = TransferClient(authorizer=authorizer)
# make job_label for task a timestamp
now = datetime.datetime.now()
job_label = now.strftime('%Y%m%d%H%M%s')
# retrieve globus_task_deadline value to enforce time window to complete transfers
# default is 2880 minutes or 48 hours
globus_task_deadline = config_get_int('conveyor', 'globus_task_deadline',
False, 2880)
deadline = now + datetime.timedelta(minutes=globus_task_deadline)
# from Globus... sync_level=checksum means that before files are transferred, Globus will compute checksums on the source
# and destination files, and only transfer files that have different checksums are transferred. verify_checksum=True means
# that after a file is transferred, Globus will compute checksums on the source and destination files to verify that the
# file was transferred correctly. If the checksums do not match, it will redo the transfer of that file.
tdata = TransferData(tc,
source_endpoint_id,
destination_endpoint_id,
label=job_label,
sync_level="checksum",
deadline=str(deadline))
for file in submitjob:
source_path = file.get('sources')[0]
dest_path = file.get('destinations')[0]
filesize = file['metadata']['filesize']
# TODO: support passing a recursive parameter to Globus
# md5 = file['metadata']['md5']
# tdata.add_item(source_path, dest_path, recursive=False, external_checksum=md5)
tdata.add_item(source_path, dest_path, recursive=False)
record_counter(
'daemons.conveyor.transfer_submitter.globus.transfers.submit.filesize',
filesize)
# logging.info('submitting transfer...')
transfer_result = tc.submit_transfer(tdata)
logger(logging.INFO, "transfer_result: %s" % transfer_result)
return transfer_result["task_id"]
def get_auth_client():
tokens = get_auth_tokens()
authorizer = None
# if there's a refresh token, use it to build the authorizer
if tokens['refresh_token'] is not None:
authorizer = RefreshTokenAuthorizer(tokens['refresh_token'],
internal_auth_client(),
tokens['access_token'],
tokens['access_token_expires'],
on_refresh=_update_access_tokens)
client = AuthClient(authorizer=authorizer, app_name=version.app_name)
return client
def get_client():
tokens = get_transfer_tokens()
authorizer = None
# if there's a refresh token, use it to build the authorizer
if tokens['refresh_token'] is not None:
authorizer = RefreshTokenAuthorizer(tokens['refresh_token'],
internal_auth_client(),
tokens['access_token'],
tokens['access_token_expires'],
on_refresh=_update_access_tokens)
return RetryingTransferClient(tries=10,
authorizer=authorizer,
app_name=version.app_name)
def get_refresh_authorizer(tokens):
"""
Retrieve a refresh token authorizer from the provided tokens.
"""
authorizer = None
if tokens['refresh_token'] is not None:
authorizer = RefreshTokenAuthorizer(tokens['refresh_token'],
internal_auth_client(),
on_refresh=_update_access_tokens)
else:
raise RuntimeError('No refresh token found. Please login first.')
return authorizer
def get_authorizers(self):
authorizers = {}
for resource_server, token_dict in self.load_tokens().items():
if token_dict.get('refresh_token') is not None:
authorizers[resource_server] = RefreshTokenAuthorizer(
token_dict['refresh_token'],
self.client,
access_token=token_dict['access_token'],
expires_at=token_dict['expires_at_seconds'],
on_refresh=self.on_refresh,
)
else:
authorizers[resource_server] = AccessTokenAuthorizer(
token_dict['access_token'])
return authorizers
def getAuthorizer(self, user):
if 'otherTokens' not in user:
raise Exception('No transfer token found')
tokens = user['otherTokens']
for token in tokens:
if token['scope'] == _TRANSFER_SCOPE:
if 'refresh_token' in token and token[
'refresh_token'] is not None:
return RefreshTokenAuthorizer(token['refresh_token'],
self.getAuthClient())
else:
return AccessTokenAuthorizer(token['access_token'])
raise Exception('No globus transfer token found')
def transfer_status(task_id):
"""
Call Globus to get status/details of transfer with
task_id.
The target template (tranfer_status.jinja2) expects a Transfer API
'task' object.
'task_id' is passed to the route in the URL as 'task_id'.
"""
transfer = TransferClient(authorizer=RefreshTokenAuthorizer(
session['tokens']['transfer.api.globus.org']['refresh_token'],
load_portal_client()))
task = transfer.get_task(task_id)
return render_template('transfer_status.jinja2', task=task)
def __init__(self):
"""Initiate an OAuth2() object.
Initiate OAuth2 flow with Globus credentaials to obtain access tokens.
Refresh the tokens automatically so another login is not required.
Examples
--------
Create an OAuth2 object:
>>> from archeion.models import OAuth2
>>> authorizer = OAuth2()
"""
self.client = NativeAppAuthClient(CLIENT_ID)
self.client.oauth2_start_flow(refresh_tokens=True)
logger.info("Opening browser window for Globus Authentication")
webbrowser.open_new(self.client.oauth2_get_authorize_url())
get_input = getattr(__builtins__, "raw_input", input)
auth_code = get_input(
"Please enter the code you get after login here: "
).strip()
logger.debug("User has input authentication code")
token_response = self.client.oauth2_exchange_code_for_tokens(auth_code)
self.access_token = token_response.by_resource_server["auth.globus.org"][
"access_token"
]
transfer_response = token_response.by_resource_server["transfer.api.globus.org"]
self.transfer_token = transfer_response["access_token"]
self.transfer_refresh_token = transfer_response["refresh_token"]
self.transfer_expiry_seconds = transfer_response["expires_at_seconds"]
authorizer = RefreshTokenAuthorizer(
self.transfer_refresh_token,
self.client,
access_token=self.transfer_token,
expires_at=self.transfer_expiry_seconds,
)
self.transfer_client = TransferClient(
AccessTokenAuthorizer(self.transfer_token)
)
self.authorisation_client = AuthClient(authorizer=authorizer)
def create_globus_transfer_client(tmpLog, globus_client_id,
globus_refresh_token):
"""
create Globus Transfer Client and return the transfer client
"""
# get logger
tmpLog.info('Creating instance of GlobusTransferClient')
# start the Native App authentication process
# use the refresh token to get authorizer
# create the Globus Transfer Client
tc = None
ErrStat = True
try:
client = NativeAppAuthClient(client_id=globus_client_id)
authorizer = RefreshTokenAuthorizer(refresh_token=globus_refresh_token,
auth_client=client)
tc = TransferClient(authorizer=authorizer)
except:
errStat, errMsg = handle_globus_exception(tmpLog)
return ErrStat, tc
def setup_transfer_client(transfer_tokens):
authorizer = RefreshTokenAuthorizer(
transfer_tokens['refresh_token'],
NativeAppAuthClient(client_id=CLIENT_ID),
access_token=transfer_tokens['access_token'],
expires_at=transfer_tokens['expires_at_seconds'])
transfer_client = TransferClient(authorizer=authorizer)
try:
transfer_client.endpoint_autoactivate(SOURCE_ENDPOINT)
transfer_client.endpoint_autoactivate(DESTINATION_ENDPOINT)
except GlobusAPIError as ex:
if ex.http_status == 401:
sys.exit('Refresh token has expired. '
'Please delete the `tokens` object from '
'{} and try again.'.format(DATA_FILE))
else:
raise ex
return transfer_client
def setup_transfer_client(transfer_tokens, source_endpoint,
destination_endpoint):
authorizer = RefreshTokenAuthorizer(
transfer_tokens['refresh_token'],
NativeAppAuthClient(client_id=CLIENT_ID),
access_token=transfer_tokens['access_token'],
expires_at=transfer_tokens['expires_at_seconds'],
on_refresh=update_tokens_file_on_refresh)
transfer_client = TransferClient(authorizer=authorizer)
try:
transfer_client.endpoint_autoactivate(source_endpoint)
transfer_client.endpoint_autoactivate(destination_endpoint)
except GlobusAPIError as ex:
if ex.http_status == 401:
sys.exit('Refresh token has expired. '
'Please delete the `tokens` object from '
'{} and try again.'.format(DATA_FILE))
else:
raise ex
return transfer_client
def make_authorizer():
"""
Make a RefreshTokenAuthorizer given the tokens stored on disk
Returns:
(RefreshTokenAuthorizer): Tool to generate authorization credentials
"""
if not check_logged_in():
safeprint("No authorization credentials present. You must log in")
do_login_flow()
# Get the authorization client
auth_client = internal_auth_client()
# Get the tokens needed by the service
rf_token = lookup_option(FUNCX_RT_OPTNAME)
at_token = lookup_option(FUNCX_AT_OPTNAME)
at_expires = int(lookup_option(FUNCX_AT_EXPIRES_OPTNAME))
authorizer = RefreshTokenAuthorizer(rf_token, auth_client, access_token=at_token,
expires_at=at_expires)
return authorizer
def __init__(self,
src_endpoint_name,
dst_endpoint_name,
transfer_rt=None,
log_lv=logging.INFO):
log_format = '%(asctime)-15s %(levelname)s:\t class:%(name)s %(message)s'
logging.basicConfig(format=log_format)
self.logger = logging.getLogger(self.__class__.__name__)
self.logger.setLevel(log_lv)
self.logger.debug('CLIENT_ID: {0}'.format(CLIENT_ID))
self.client = NativeAppAuthClient(CLIENT_ID)
self.client.oauth2_start_flow(refresh_tokens=True)
if transfer_rt is not None:
self.authorizer = RefreshTokenAuthorizer(transfer_rt, self.client)
else:
authorize_url = self.client.oauth2_get_authorize_url()
print('Please go to this URL and login: {0}'.format(authorize_url))
get_input = getattr(__builtins__, 'raw_input', input)
auth_code = get_input(
'Please enter the code you get after login here: ').strip()
token_response = self.client.oauth2_exchange_code_for_tokens(
auth_code)
self.globus_auth_data = token_response.by_resource_server[
'auth.globus.org']
self.globus_transfer_data = token_response.by_resource_server[
'transfer.api.globus.org']
auth_token = self.globus_auth_data['access_token']
transfer_token = self.globus_transfer_data['access_token']
transfer_rt = self.globus_transfer_data['refresh_token']
transfer_at = self.globus_transfer_data['access_token']
expires_at_s = self.globus_transfer_data['expires_at_seconds']
self.authorizer = RefreshTokenAuthorizer(transfer_rt,
self.client,
access_token=transfer_at,
expires_at=expires_at_s)
self.transferClient = TransferClient(authorizer=self.authorizer)
self.src_endpoint = None
self.dst_endpoint = None
for ep in self.transferClient.endpoint_search(
filter_scope="shared-with-me"):
if ep["display_name"] == src_endpoint_name:
self.src_endpoint = ep
self.logger.info('Source endpoint: [{0}] {1}'.format(
self.src_endpoint['id'],
self.src_endpoint['display_name']))
if self.src_endpoint is None:
self.logger.error(
'No endpoint shared with you with name: {0}'.format(
src_endpoint_name))
raise LookupError
for ep in self.transferClient.endpoint_search(
filter_scope="my-endpoints"):
if ep['display_name'] == dst_endpoint_name:
self.dst_endpoint = ep
self.logger.info('Destination endpoint: [{0}] {1}'.format(
self.dst_endpoint['id'],
self.dst_endpoint['display_name']))
if self.dst_endpoint is None:
self.logger.error('You don\'t have endpoint named: {0}'.format(
dst_endpoint_name))
raise LookupError
def initiate_connection(self):
""" Initiate the connection
"""
tokens = None
try:
# if we already have tokens, load and use them
tokens = self.load_tokens_from_file()
except:
pass
if not tokens:
# if we need to get tokens, start the Native App authentication process
client = NativeAppAuthClient(
client_id=self.CLIENT_ID) #self.config['client_id'])
# pass refresh_tokens=True to request refresh tokens
client.oauth2_start_flow(
requested_scopes=self.
SCOPES, #self.config['requested_scopes'],
redirect_uri=self.REDIRECT_URI, #self.config['redirect_uri'],
refresh_tokens=True)
url = client.oauth2_get_authorize_url()
print 'Native App Authorization URL: \n{}'.format(url)
auth_code = raw_input('Enter the auth code: ').strip()
token_response = client.oauth2_exchange_code_for_tokens(auth_code)
# return a set of tokens, organized by resource server name
tokens = token_response.by_resource_server
try:
self.save_tokens_to_file(tokens)
except:
pass
transfer_tokens = tokens['transfer.api.globus.org']
auth_client = NativeAppAuthClient(client_id=self.config['client_id'])
authorizer = RefreshTokenAuthorizer(
transfer_tokens['refresh_token'],
auth_client,
#access_token=transfer_tokens['access_token'],
#expires_at=transfer_tokens['expires_at_seconds'],
on_refresh=self.update_tokens_file_on_refresh)
self.client = TransferClient(authorizer=authorizer)
# print out a directory listing from an endpoint
try:
#print 'ACTIVATE'
#print 'DEST',self.config['dest_ep']
self.client.endpoint_autoactivate(self.config['dest_ep'])
ac = self.client.endpoint_get_activation_requirements(
self.config['dest_ep'])
#print ac
self.client.endpoint_autoactivate(self.config['src_ep'])
ac2 = self.client.endpoint_get_activation_requirements(
self.config['src_ep'])
#print ac2
except GlobusAPIError as ex:
self.logger.error('Error in endpoint activation %s', str(ex))
if ex.http_status == 401:
sys.exit('Refresh token has expired. '
'Please delete refresh-tokens.json and try again.')
else:
raise ex
def transfer(sp,destination_endpoint_id,one_endpoint):
tokens = None
try:
# if we already have tokens, load and use them
tokens = load_tokens_from_file(TOKEN_FILE)
except:
pass
if not tokens:
# if we need to get tokens, start the Native App authentication process
tokens = do_native_app_authentication(CLIENT_ID, REDIRECT_URI, SCOPES)
try:
save_tokens_to_file(TOKEN_FILE, tokens)
except:
pass
transfer_tokens = tokens['transfer.api.globus.org']
auth_client = NativeAppAuthClient(client_id=CLIENT_ID,environment='sandbox')
authorizer = RefreshTokenAuthorizer(
transfer_tokens['refresh_token'],
auth_client,
access_token=transfer_tokens['access_token'],
expires_at=transfer_tokens['expires_at_seconds'],
on_refresh=update_tokens_file_on_refresh)
#transfer = TransferClient(authorizer=authorizer,environment='sandbox')
tc = TransferClient(authorizer=authorizer, environment="sandbox")
##################---ENDPOINTS---###########################
source_endpoint_id = '5a2e5704-b028-11e7-bdad-22000bdb2406' #sb vmtb4
#source_endpoint_id = '55705028-aa15-11e7-bdad-22000bdb2406' #sb yulie7t
#source_endpoint_id = 'b0b16296-88e7-11e7-a971-22000a92523b' #bare chameleon
#source_endpoint_id = 'e5762bc2-8466-11e7-a8ed-22000a92523b' #large_chameleon
#source_endpoint_id = '8b26cc0e-877b-11e7-a949-22000a92523b'#ubuntu-vm
#source_endpoint_id = 'ad19b012-77cf-11e7-8b98-22000b9923ef'#chameleon
# source_endpoint_id = raw_input('Input source endpoint UUID: ')
#destination path
##############---SOURCE PATH---######################
#source_path = '/home/parallels/stream_transfer/test_files/'
#source_path = '/home/parallels/stream_transfer/zero_globus/test_files/'
source_path = sp
#source_path ='/home/cc/streaming/zero_globus/test_files/test.txt'
#source_path = '/home/parallels/stream_transfer/zero_globus/test_files/test.txt'
#destination path
destination_path = '/~/'
#destination_path = '/~/'+ sp.split("/")[-1] #use for one file
##if one_endpoint:
## destination_path = '/projects/BrainImagingADSP/yzamora/'
##else:
## destination_path = '/projects/BrainImagingADSP/yzamora/'+ sp.split("/")[-1] #use for one file
#Using my sample UUID from globus tutorial
#destination_endpoint_id = 'ddb59aef-6d04-11e5-ba46-22000b92c6ec' #globus
#destination_endpoint_id = '5d1da0fe-3c07-11e7-bcfc-22000b9a448b' #laptop
#tc.endpoint_autoactivate(source_endpoint_id)
#tc.endpoint_autoactive(destination_endpoint_id)
ep1 = tc.get_endpoint(destination_endpoint_id)
tc.endpoint_autoactivate(destination_endpoint_id)
#ep1 is setting the activated endpoint to be a variable to work with
tc.endpoint_autoactivate(source_endpoint_id)
label = "medium data transfer"
#tdata = globus_sdk.TransferData(tc, source_endpoint_id, destination_endpoint_id,label=label, sync_level='0')
tdata = globus_sdk.TransferData(tc, source_endpoint_id, destination_endpoint_id,label=label, perf_cc=3, sync_level=None, verify_checksum=False)
#tdata = globus_sdk.TransferData(tc, source_endpoint_id, destination_endpoint_id,label=label)
if one_endpoint:
tdata.add_item(source_path,destination_path,recursive=True)
else:
tdata.add_item(source_path,destination_path,recursive=False)
submit_result = tc.submit_transfer(tdata)
print("Task ID:", submit_result["task_id"])
"""
Checking for time completion using globus calls
"""
#print("Completion time:", submit_result["completion_time"])
#setup of the transfer, submits as a https post request
#transfer_data = TransferData(transfer_client=tc,
# source_endpoint=source_endpoint_id,
# destination_endpoint=destination_endpoint_id,
# label='Transfer',
# sync_level='checksum')
#transfer_data.add_item(source_path=source_path,destination_path=destination_path, recursive=False)
#task_id=transfer.submit_transfer(transfer_data)['task_id']
#waiting for file to transfer
status = tc.get_task(submit_result["task_id"],fields="status")["status"]
poll_interval = 2
max_wait = 90
wait_time = 0
while not (status in ["SUCCEEDED", "FAILED"]):
if (wait_time >= max_wait): break
print("Task not yet complete (status {}), sleeping for {} seconds..." \
.format(status, poll_interval))
time.sleep(poll_interval)
wait_time += poll_interval
status = tc.get_task(submit_result["task_id"], fields="status")["status"]
if status == "FAILED":
print("WARNING! File transfer FAILED!")
#deleting file after transfer
if status == "SUCCEEDED":
end_time = datetime.datetime.utcnow()
start_time = end_time - datetime.timedelta(minutes=200)
#limit = response objects
# data = tc.task_list(filter="type:TRANSFER,DELETE/request_time:%s,%s"
# % (start_time, end_time), limit=5)
#print("File transfer SUCCEEDED, will delete file from local directory now")
""" r = tc.task_list(num_results=1, filter="type:TRANSFER,DELETE")
def __init__(self):
self.log = logging.getLogger(self.__class__.__name__)
self.log.info("init - started")
config = configparser.ConfigParser()
config.read(str(CONFIG_PATH))
# To set up a client_id, see https://auth.globus.org/v2/web/developers
# Current client_id is in Project: MaterialsCommonsProject, App: MaterialsCommonsTest
client_id = config['sdk']['id']
auth_tokens = None
transfer_tokens = None
tokens = None
try:
# if we already have tokens, load and use them
tokens = self._load_tokens_from_file(TOKEN_FILE_PATH)
except IOError:
pass
if not tokens:
# if we need to get tokens, start the Native App authentication process
tokens = self.do_native_app_authentication(client_id, REDIRECT_URI,
SCOPES)
try:
self._save_tokens_to_file(TOKEN_FILE_PATH, tokens)
except IOError:
pass
try:
auth_tokens = tokens['auth.globus.org']
transfer_tokens = tokens['transfer.api.globus.org']
except KeyError as er:
self.log.error(
"KeyError on NativeApp tokens: {}\n delete {} and restart".
format(er, TOKEN_FILE_PATH))
def refresh_tokens(token_response):
context = self
context._update_tokens_file_on_refresh(token_response)
auth_client = NativeAppAuthClient(client_id=client_id)
authorizer = RefreshTokenAuthorizer(
auth_tokens['refresh_token'],
auth_client,
access_token=auth_tokens['access_token'],
expires_at=auth_tokens['expires_at_seconds'],
on_refresh=refresh_tokens)
auth_client = AuthClient(client_id=client_id, authorizer=authorizer)
authorizer = RefreshTokenAuthorizer(
transfer_tokens['refresh_token'],
auth_client,
access_token=transfer_tokens['access_token'],
expires_at=transfer_tokens['expires_at_seconds'],
on_refresh=refresh_tokens)
transfer_client = TransferClient(authorizer=authorizer)
self._auth_client = auth_client
self._transfer_client = transfer_client
.format(TOKEN_FILE))
sys.exit(-1)
transfer_tokens = tokens['transfer.api.globus.org']
try:
auth_client = NativeAppAuthClient(client_id=dcde_parsl_client_id)
except:
print(
"ERROR: Globus NativeAppAuthClient() call failed! Unable to obtain a Globus authorizer!"
)
sys.exit(-1)
authorizer = RefreshTokenAuthorizer(
transfer_tokens['refresh_token'],
auth_client,
access_token=transfer_tokens['access_token'],
expires_at=transfer_tokens['expires_at_seconds'],
on_refresh=update_tokens_file_on_refresh)
try:
tc = TransferClient(authorizer=authorizer)
except:
print(
"ERROR: TransferClient() call failed! Unable to call the Globus transfer interface with the provided auth info!"
)
sys.exit(-1)
# print(transfer)
# Now we should have auth, try setting up a transfer.
tdata = TransferData(tc,
def browse(dataset_id=None, endpoint_id=None, endpoint_path=None):
"""
- Get list of files for the selected dataset or endpoint ID/path
- Return a list of files to a browse view
The target template (browse.jinja2) expects an `endpoint_uri` (if
available for the endpoint), `target` (either `"dataset"`
or `"endpoint"`), and 'file_list' (list of dictionaries) containing
the following information about each file in the result:
{'name': 'file name', 'size': 'file size', 'id': 'file uri/path'}
If you want to display additional information about each file, you
must add those keys to the dictionary and modify the browse.jinja2
template accordingly.
"""
if request.method == 'GET':
assert bool(dataset_id) != bool(endpoint_id and endpoint_path)
if dataset_id:
try:
dataset = next(ds for ds in datasets if ds['id'] == dataset_id)
except StopIteration:
abort(404)
endpoint_id = app.config['DATASET_ENDPOINT_ID']
endpoint_path = app.config['DATASET_ENDPOINT_BASE'] + dataset['path']
else:
endpoint_path = '/' + endpoint_path
transfer_tokens = session['tokens']['transfer.api.globus.org']
authorizer = RefreshTokenAuthorizer(
transfer_tokens['refresh_token'],
load_portal_client(),
access_token=transfer_tokens['access_token'],
expires_at=transfer_tokens['expires_at_seconds'])
transfer = TransferClient(authorizer=authorizer)
try:
transfer.endpoint_autoactivate(endpoint_id)
listing = transfer.operation_ls(endpoint_id, path=endpoint_path)
except TransferAPIError as err:
flash('Error [{}]: {}'.format(err.code, err.message))
return redirect(url_for('browse'))
file_list = [e for e in listing if e['type'] == 'file']
ep = transfer.get_endpoint(endpoint_id)
https_server = ep['https_server']
endpoint_uri = https_server + endpoint_path if https_server else None
webapp_xfer = 'https://app.globus.org/file-manager?' + \
urlencode(dict(origin_id=endpoint_id, origin_path=endpoint_path))
#print("endpintURL == " + endpoint_uri)
return render_template('browse.jinja2', endpoint_uri=endpoint_uri,
target="dataset" if dataset_id else "endpoint",
description=(dataset['name'] if dataset_id
else ep['display_name']),
mypath=(dataset['path'] if dataset_id
else None),
myid=(dataset['id'] if dataset_id
else None),
file_list=file_list, webapp_xfer=webapp_xfer)
if request.method == 'POST':
if not request.form.get('file'):
flash('Please select at least one file.')
return redirect(url_for('browse'))
params = {
'method': 'POST',
'action': url_for('submit_transfer', _external=True,
_scheme='https'),
'filelimit': 0,
'folderlimit': 1
}
browse_endpoint = 'https://app.globus.org/file-manager?{}' \
.format(urlencode(params))
session['form'] = {
'dirselect': False,
'datasets': request.form.getlist('file'),
'path': request.form.getlist('path'),
'id': request.form.getlist('id')
}
return redirect(browse_endpoint)
