def __init__(self):
self.auth_client = NativeAppAuthClient(self.app_id)
self.auth_client.oauth2_start_flow(refresh_tokens=True)
self.xfer_client = None
if 'globus.token' not in dj.config:
self.login()
else:
self.refresh()
def __init__(self):
self.auth_client = NativeAppAuthClient(self.app_id)
self.auth_client.oauth2_start_flow(refresh_tokens=True)
self.xfer_client = None
custom = dj.config.get('custom', None)
if custom and 'globus.token' in custom:
self.refresh()
else:
self.login()
class OAuth2(object):
"""Base class for OAuth2 model
"""
def __init__(self):
"""Initiate an OAuth2() object.
Initiate OAuth2 flow with Globus credentaials to obtain access tokens.
Refresh the tokens automatically so another login is not required.
Examples
--------
Create an OAuth2 object:
>>> from archeion.models import OAuth2
>>> authorizer = OAuth2()
"""
self.client = NativeAppAuthClient(CLIENT_ID)
self.client.oauth2_start_flow(refresh_tokens=True)
logger.info("Opening browser window for Globus Authentication")
webbrowser.open_new(self.client.oauth2_get_authorize_url())
get_input = getattr(__builtins__, "raw_input", input)
auth_code = get_input(
"Please enter the code you get after login here: "
).strip()
logger.debug("User has input authentication code")
token_response = self.client.oauth2_exchange_code_for_tokens(auth_code)
self.access_token = token_response.by_resource_server["auth.globus.org"][
"access_token"
]
transfer_response = token_response.by_resource_server["transfer.api.globus.org"]
self.transfer_token = transfer_response["access_token"]
self.transfer_refresh_token = transfer_response["refresh_token"]
self.transfer_expiry_seconds = transfer_response["expires_at_seconds"]
authorizer = RefreshTokenAuthorizer(
self.transfer_refresh_token,
self.client,
access_token=self.transfer_token,
expires_at=self.transfer_expiry_seconds,
)
self.transfer_client = TransferClient(
AccessTokenAuthorizer(self.transfer_token)
)
self.authorisation_client = AuthClient(authorizer=authorizer)
def identifier_client(config, **kwargs):
app_name = 'identifier_client'
base_url = config['client']['service_url']
client_id = config['client']['client_id']
access_token = config['tokens']['access_token']
at_expires = int(config['tokens']['access_token_expires'])
refresh_token = config['tokens']['refresh_token']
if not (refresh_token and access_token):
raise IdentifierNotLoggedIn("Missing tokens")
def _on_refresh(tkn):
# extract_and_save_tokens(tkn, config)
pass
authorizer_client = NativeAppAuthClient(client_id, app_name=app_name)
authorizer = RefreshTokenAuthorizer(
refresh_token,
authorizer_client,
access_token,
at_expires,
on_refresh=_on_refresh,
)
return IdentifierClient(
"identifier",
base_url=base_url,
app_name=app_name,
authorizer=authorizer,
**kwargs
)
def init():
tokens = None
try:
# if we already have tokens, load and use them
tokens = load_tokens_from_db()
except:
pass
if not tokens:
# if we need to get tokens, start the Native App authentication process
tokens = do_native_app_authentication(config.TRANSFER_CLIENT_ID,
config.REDIRECT_URI,
config.SCOPES)
try:
save_tokens_to_db(tokens)
except:
pass
transfer_tokens = tokens['transfer.api.globus.org']
client = NativeAppAuthClient(client_id=config.TRANSFER_CLIENT_ID)
authorizer = RefreshTokenAuthorizer(
transfer_tokens['refresh_token'],
client,
access_token=transfer_tokens['access_token'],
expires_at=transfer_tokens['expires_at_seconds'],
on_refresh=update_tokens_file_on_refresh)
transfer = TransferClient(authorizer=authorizer)
prepare_call(transfer)
return transfer
def getTransferData():
cfg = load_config()
client_id = cfg['globus']['apps']['SDK Tutorial App']['client_id']
auth_client = NativeAppAuthClient(client_id)
refresh_token = cfg['globus']['apps']['SDK Tutorial App']['refresh_token']
source_endpoint_id = cfg['globus']['apps']['SDK Tutorial App'][
'win10_endpoint_id']
destination_endpoint_id = cfg['globus']['apps']['SDK Tutorial App'][
'sdccfed_endpoint_id']
authorizer = RefreshTokenAuthorizer(refresh_token=refresh_token,
auth_client=auth_client)
tc = TransferClient(authorizer=authorizer)
# as both endpoints are expected to be Globus Server endpoints, send auto-activate commands for both globus endpoints
auto_activate_endpoint(tc, source_endpoint_id)
auto_activate_endpoint(tc, destination_endpoint_id)
# make job_label for task a timestamp
x = datetime.now()
job_label = x.strftime('%Y%m%d%H%M%s')
# from Globus... sync_level=checksum means that before files are transferred, Globus will compute checksums on the source and destination files,
# and only transfer files that have different checksums are transferred. verify_checksum=True means that after a file is transferred, Globus will
# compute checksums on the source and destination files to verify that the file was transferred correctly. If the checksums do not match, it will
# redo the transfer of that file.
tdata = TransferData(tc,
source_endpoint_id,
destination_endpoint_id,
label=job_label,
sync_level="checksum",
verify_checksum=True)
return tdata
def identifiers_client(config, **kwargs):
app_name = 'identifier_client'
base_url = config.get('client', 'service_url')
client_id = config.get('client', 'client_id')
access_token = config.get('tokens', 'access_token')
at_expires = int(config.get('tokens', 'access_token_expires'))
refresh_token = config.get('tokens', 'refresh_token')
if not (refresh_token and access_token):
raise IdentifierNotLoggedIn("Missing tokens")
def _on_refresh(tkn):
extract_and_save_tokens(tkn, config)
authorizer_client = NativeAppAuthClient(client_id, app_name=app_name)
authorizer = RefreshTokenAuthorizer(
refresh_token,
authorizer_client,
access_token,
at_expires,
on_refresh=_on_refresh,
)
return IdentifierClient(base_url=base_url,
app_name=app_name,
authorizer=authorizer,
**kwargs)
def __init__(self,
token_storage=MultiClientTokenStorage(),
local_server_code_handler=LocalServerCodeHandler(),
secondary_code_handler=InputCodeHandler(),
default_scopes=None,
*args,
**kwargs):
self.client = NativeAppAuthClient(*args, **kwargs)
self.token_storage = token_storage
if token_storage is not None:
self.verify_token_storage(self.token_storage)
self.app_name = kwargs.get('app_name') or 'My App'
self.local_server_code_handler = local_server_code_handler
self.local_server_code_handler.set_app_name(self.app_name)
self.secondary_code_handler = secondary_code_handler
if isinstance(self.token_storage, MultiClientTokenStorage):
self.token_storage.set_client_id(kwargs.get('client_id'))
self.default_scopes = default_scopes
def _do_login_for_scopes(native_client: NativeAppAuthClient,
scopes: List[str]) -> OAuthTokenResponse:
label = CLIENT_NAME
host = platform.node()
if host:
label = label + f" on {host}"
native_client.oauth2_start_flow(
requested_scopes=scopes,
refresh_tokens=True,
prefill_named_grant=label,
)
linkprompt = "Please log into Globus here"
safeprint("{0}:\n{1}\n{2}\n{1}\n".format(
linkprompt, "-" * len(linkprompt),
native_client.oauth2_get_authorize_url()))
auth_code = click.prompt(
"Enter the resulting Authorization Code here").strip()
return native_client.oauth2_exchange_code_for_tokens(auth_code)
def get_transfer_client():
cfg = load_config()
# cfg = yaml.safe_load(open("/opt/rucio/lib/rucio/transfertool/config.yml"))
client_id = cfg['globus']['apps'][GLOBUS_AUTH_APP]['client_id']
auth_client = NativeAppAuthClient(client_id)
refresh_token = cfg['globus']['apps'][GLOBUS_AUTH_APP]['refresh_token']
logging.info('authorizing token...')
authorizer = RefreshTokenAuthorizer(refresh_token=refresh_token, auth_client=auth_client)
logging.info('initializing TransferClient...')
tc = TransferClient(authorizer=authorizer)
return tc
def bulk_submit_xfer(submitjob, recursive=False):
cfg = load_config()
client_id = cfg['globus']['apps'][GLOBUS_AUTH_APP]['client_id']
auth_client = NativeAppAuthClient(client_id)
refresh_token = cfg['globus']['apps'][GLOBUS_AUTH_APP]['refresh_token']
source_endpoint_id = submitjob[0].get('metadata').get(
'source_globus_endpoint_id')
destination_endpoint_id = submitjob[0].get('metadata').get(
'dest_globus_endpoint_id')
authorizer = RefreshTokenAuthorizer(refresh_token=refresh_token,
auth_client=auth_client)
tc = TransferClient(authorizer=authorizer)
# as both endpoints are expected to be Globus Server endpoints, send auto-activate commands for both globus endpoints
a = auto_activate_endpoint(tc, source_endpoint_id)
logging.debug('a: %s' % a)
if a != 'AlreadyActivated':
return None
b = auto_activate_endpoint(tc, destination_endpoint_id)
logging.debug('b: %s' % b)
if b != 'AlreadyActivated':
return None
# make job_label for task a timestamp
x = datetime.now()
job_label = x.strftime('%Y%m%d%H%M%s')
# from Globus... sync_level=checksum means that before files are transferred, Globus will compute checksums on the source
# and destination files, and only transfer files that have different checksums are transferred. verify_checksum=True means
# that after a file is transferred, Globus will compute checksums on the source and destination files to verify that the
# file was transferred correctly. If the checksums do not match, it will redo the transfer of that file.
tdata = TransferData(tc,
source_endpoint_id,
destination_endpoint_id,
label=job_label,
sync_level="checksum")
for file in submitjob:
source_path = file.get('sources')[0]
dest_path = file.get('destinations')[0]
filesize = file['metadata']['filesize']
# TODO: support passing a recursive parameter to Globus
# md5 = file['metadata']['md5']
# tdata.add_item(source_path, dest_path, recursive=False, external_checksum=md5)
tdata.add_item(source_path, dest_path, recursive=False)
record_counter(
'daemons.conveyor.transfer_submitter.globus.transfers.submit.filesize',
filesize)
# logging.info('submitting transfer...')
transfer_result = tc.submit_transfer(tdata)
# logging.info("task_id =", transfer_result["task_id"])
return transfer_result["task_id"]
def do_native_app_authentication(client_id,
redirect_uri,
requested_scopes=None):
"""
Does a Native App authentication flow and returns a
dict of tokens keyed by service name.
"""
client = NativeAppAuthClient(client_id=client_id)
# pass refresh_tokens=True to request refresh tokens
client.oauth2_start_flow(requested_scopes=requested_scopes,
redirect_uri=redirect_uri,
refresh_tokens=True)
url = client.oauth2_get_authorize_url()
print('Native App Authorization URL:\n{}'.format(url))
if not is_remote_session():
# There was a bug in webbrowser recently that this fixes:
# https://bugs.python.org/issue30392
if sys.platform == 'darwin':
webbrowser.get('safari').open(url, new=1)
else:
webbrowser.open(url, new=1)
auth_code = get_input('Enter the auth code: ').strip()
token_response = client.oauth2_exchange_code_for_tokens(auth_code)
# return a set of tokens, organized by resource server name
return token_response.by_resource_server
def do_native_app_authentication(client_id,
redirect_uri,
requested_scopes=None):
"""
Does a Native App authentication flow and returns a
dict of tokens keyed by service name.
"""
client = NativeAppAuthClient(client_id=client_id)
# pass refresh_tokens=True to request refresh tokens
client.oauth2_start_flow(requested_scopes=requested_scopes,
redirect_uri=redirect_uri,
refresh_tokens=True)
url = client.oauth2_get_authorize_url()
print('Native App Authorization URL: \n{}'.format(url))
if not is_remote_session():
webbrowser.open(url, new=1)
auth_code = input('Enter the auth code: ').strip()
token_response = client.oauth2_exchange_code_for_tokens(auth_code)
# return a set of tokens, organized by resource server name
return token_response.by_resource_server
def bulk_submit_xfer(submitjob, recursive=False, logger=logging.log):
cfg = load_config(logger=logger)
client_id = cfg['globus']['apps'][GLOBUS_AUTH_APP]['client_id']
auth_client = NativeAppAuthClient(client_id)
refresh_token = cfg['globus']['apps'][GLOBUS_AUTH_APP]['refresh_token']
source_endpoint_id = submitjob[0].get('metadata').get(
'source_globus_endpoint_id')
destination_endpoint_id = submitjob[0].get('metadata').get(
'dest_globus_endpoint_id')
authorizer = RefreshTokenAuthorizer(refresh_token=refresh_token,
auth_client=auth_client)
tc = TransferClient(authorizer=authorizer)
# make job_label for task a timestamp
now = datetime.datetime.now()
job_label = now.strftime('%Y%m%d%H%M%s')
# retrieve globus_task_deadline value to enforce time window to complete transfers
# default is 2880 minutes or 48 hours
globus_task_deadline = config_get_int('conveyor', 'globus_task_deadline',
False, 2880)
deadline = now + datetime.timedelta(minutes=globus_task_deadline)
# from Globus... sync_level=checksum means that before files are transferred, Globus will compute checksums on the source
# and destination files, and only transfer files that have different checksums are transferred. verify_checksum=True means
# that after a file is transferred, Globus will compute checksums on the source and destination files to verify that the
# file was transferred correctly. If the checksums do not match, it will redo the transfer of that file.
tdata = TransferData(tc,
source_endpoint_id,
destination_endpoint_id,
label=job_label,
sync_level="checksum",
deadline=str(deadline))
for file in submitjob:
source_path = file.get('sources')[0]
dest_path = file.get('destinations')[0]
filesize = file['metadata']['filesize']
# TODO: support passing a recursive parameter to Globus
# md5 = file['metadata']['md5']
# tdata.add_item(source_path, dest_path, recursive=False, external_checksum=md5)
tdata.add_item(source_path, dest_path, recursive=False)
record_counter(
'daemons.conveyor.transfer_submitter.globus.transfers.submit.filesize',
filesize)
# logging.info('submitting transfer...')
transfer_result = tc.submit_transfer(tdata)
logger(logging.INFO, "transfer_result: %s" % transfer_result)
return transfer_result["task_id"]
def login():
tok_path = os.path.expanduser('~/.mdf_agent_tokens.json')
def _read_tokfile():
tokens = {}
if os.path.exists(tok_path):
with open(tok_path) as f:
tokens = json.load(f)
return tokens
def _write_tokfile(new_tokens):
# We have multiple tokens in our tokens file, but on update we only
# get the currently updated token, so read current and update with the
# input tokens
cur_tokens = _read_tokfile()
for key in new_tokens:
cur_tokens[key] = new_tokens[key]
# deny rwx to Group and World -- don't bother storing the returned old
# mask value, since we'll never restore it anyway
# do this on every call to ensure that we're always consistent about it
os.umask(0o077)
with open(tok_path, 'w') as f:
f.write(json.dumps(cur_tokens))
def _update_tokfile(tokens):
_write_tokfile(tokens.by_resource_server['transfer.api.globus.org'])
tokens = _read_tokfile()
client_id = "1e162bfc-ad52-4014-8844-b82841145fc4"
native_client = NativeAppAuthClient(client_id, app_name='MDF Agents')
if not tokens:
# and do the Native App Grant flow
native_client.oauth2_start_flow(
requested_scopes='urn:globus:auth:scope:transfer.api.globus.org:all',
refresh_tokens=True)
linkprompt = 'Please login to Globus here'
print('{0}:\n{1}\n{2}\n{1}\n'
.format(linkprompt, '-' * len(linkprompt),
native_client.oauth2_get_authorize_url()), flush=True)
auth_code = input(
'Enter the resulting Authorization Code here').strip()
tkns = native_client.oauth2_exchange_code_for_tokens(auth_code)
tokens = tkns.by_resource_server['transfer.api.globus.org']
_write_tokfile(tokens)
transfer_tokens = tokens
transfer_authorizer = RefreshTokenAuthorizer(
transfer_tokens['refresh_token'], native_client,
transfer_tokens['access_token'], transfer_tokens['expires_at_seconds'],
on_refresh=_update_tokfile)
transfer_client = TransferClient(authorizer=transfer_authorizer)
return transfer_client
def register_agent(self):
"""
Manage the login with Globus and register
this agent with the cloud service under this
user's name. If it is a new user they will have
account created for them by the service.
"""
scopes = ("urn:globus:auth:scope:transfer.api.globus.org:all "
"urn:globus:auth:scope:auth.globus.org:view_identities "
"openid email profile")
redirect_uri = 'https://auth.globus.org/v2/web/auth-code'
client = NativeAppAuthClient(client_id=self.client_id)
client.oauth2_start_flow(requested_scopes=scopes,
redirect_uri=redirect_uri,
refresh_tokens=True)
url = client.oauth2_get_authorize_url()
print('Globus Authorization URL: \n{}'.format(url))
auth_code = input('Enter the auth code: ').strip()
token_response = client.oauth2_exchange_code_for_tokens(auth_code)
id_data = token_response.decode_id_token(client)
user_email = id_data.get('preferred_username', '')
transfer_tokens = token_response.by_resource_server[
'transfer.api.globus.org']
print('Authenticated as: %s' % user_email)
# Get a user friendly name for this agent
agent_name = input(
'Enter a name for this agent (e.g. laptop): ').strip()
payload = {
"access_token": transfer_tokens['access_token'],
"username": user_email,
"agent_id": self.endpoint_id,
"refresh_token": transfer_tokens['refresh_token'],
"expires_at_seconds": transfer_tokens['expires_at_seconds'],
"agent_name": agent_name
}
# register this data with the service
r = requests.post(self.agent_path, json=payload)
logger.info("Response: %s %s" % (r.status_code, r.reason))
# Check it was successfully registered, otherwise try again
# Note: this returns 500 at the moment as I haven't configured
# a response integration with SNS.
if r.status_code == requests.codes.ok or r.status_code == 500:
logger.info("Successfully registered agent.")
def create_globus_transfer_client(tmpLog, globus_client_id,
globus_refresh_token):
"""
create Globus Transfer Client and return the transfer client
"""
# get logger
tmpLog.info('Creating instance of GlobusTransferClient')
# start the Native App authentication process
# use the refresh token to get authorizer
# create the Globus Transfer Client
tc = None
ErrStat = True
try:
client = NativeAppAuthClient(client_id=globus_client_id)
authorizer = RefreshTokenAuthorizer(refresh_token=globus_refresh_token,
auth_client=client)
tc = TransferClient(authorizer=authorizer)
except:
errStat, errMsg = handle_globus_exception(tmpLog)
return ErrStat, tc
def setup_transfer_client(transfer_tokens):
authorizer = RefreshTokenAuthorizer(
transfer_tokens['refresh_token'],
NativeAppAuthClient(client_id=CLIENT_ID),
access_token=transfer_tokens['access_token'],
expires_at=transfer_tokens['expires_at_seconds'])
transfer_client = TransferClient(authorizer=authorizer)
try:
transfer_client.endpoint_autoactivate(SOURCE_ENDPOINT)
transfer_client.endpoint_autoactivate(DESTINATION_ENDPOINT)
except GlobusAPIError as ex:
if ex.http_status == 401:
sys.exit('Refresh token has expired. '
'Please delete the `tokens` object from '
'{} and try again.'.format(DATA_FILE))
else:
raise ex
return transfer_client
def setup_transfer_client(transfer_tokens, source_endpoint,
destination_endpoint):
authorizer = RefreshTokenAuthorizer(
transfer_tokens['refresh_token'],
NativeAppAuthClient(client_id=CLIENT_ID),
access_token=transfer_tokens['access_token'],
expires_at=transfer_tokens['expires_at_seconds'],
on_refresh=update_tokens_file_on_refresh)
transfer_client = TransferClient(authorizer=authorizer)
try:
transfer_client.endpoint_autoactivate(source_endpoint)
transfer_client.endpoint_autoactivate(destination_endpoint)
except GlobusAPIError as ex:
if ex.http_status == 401:
sys.exit('Refresh token has expired. '
'Please delete the `tokens` object from '
'{} and try again.'.format(DATA_FILE))
else:
raise ex
return transfer_client
def do_native_app_authentication(client_id=config['CLIENT_ID'],
redirect_uri=config['REDIRECT_URI'],
requested_scopes=config['SCOPES']):
"""
Does a Native App authentication flow and returns a
dict of tokens keyed by service name.
"""
client = NativeAppAuthClient(client_id=client_id)
client.oauth2_start_flow(requested_scopes=config['SCOPES'],
redirect_uri=redirect_uri)
url = client.oauth2_get_authorize_url()
server = start_local_server(listen=config['SERVER_ADDRESS'])
if not is_remote_session():
webbrowser.open(url, new=1)
auth_code = server.wait_for_code()
token_response = client.oauth2_exchange_code_for_tokens(auth_code)
server.shutdown()
# return a set of tokens, organized by resource server name
return token_response.by_resource_server
class Transfer:
'''
Modified Transfer, add an option to pass refresh token to avoid the web login
'''
def __init__(self,
src_endpoint_name,
dst_endpoint_name,
transfer_rt=None,
log_lv=logging.INFO):
log_format = '%(asctime)-15s %(levelname)s:\t class:%(name)s %(message)s'
logging.basicConfig(format=log_format)
self.logger = logging.getLogger(self.__class__.__name__)
self.logger.setLevel(log_lv)
self.logger.debug('CLIENT_ID: {0}'.format(CLIENT_ID))
self.client = NativeAppAuthClient(CLIENT_ID)
self.client.oauth2_start_flow(refresh_tokens=True)
if transfer_rt is not None:
self.authorizer = RefreshTokenAuthorizer(transfer_rt, self.client)
else:
authorize_url = self.client.oauth2_get_authorize_url()
print('Please go to this URL and login: {0}'.format(authorize_url))
get_input = getattr(__builtins__, 'raw_input', input)
auth_code = get_input(
'Please enter the code you get after login here: ').strip()
token_response = self.client.oauth2_exchange_code_for_tokens(
auth_code)
self.globus_auth_data = token_response.by_resource_server[
'auth.globus.org']
self.globus_transfer_data = token_response.by_resource_server[
'transfer.api.globus.org']
auth_token = self.globus_auth_data['access_token']
transfer_token = self.globus_transfer_data['access_token']
transfer_rt = self.globus_transfer_data['refresh_token']
transfer_at = self.globus_transfer_data['access_token']
expires_at_s = self.globus_transfer_data['expires_at_seconds']
self.authorizer = RefreshTokenAuthorizer(transfer_rt,
self.client,
access_token=transfer_at,
expires_at=expires_at_s)
self.transferClient = TransferClient(authorizer=self.authorizer)
self.src_endpoint = None
self.dst_endpoint = None
for ep in self.transferClient.endpoint_search(
filter_scope="shared-with-me"):
if ep["display_name"] == src_endpoint_name:
self.src_endpoint = ep
self.logger.info('Source endpoint: [{0}] {1}'.format(
self.src_endpoint['id'],
self.src_endpoint['display_name']))
if self.src_endpoint is None:
self.logger.error(
'No endpoint shared with you with name: {0}'.format(
src_endpoint_name))
raise LookupError
for ep in self.transferClient.endpoint_search(
filter_scope="my-endpoints"):
if ep['display_name'] == dst_endpoint_name:
self.dst_endpoint = ep
self.logger.info('Destination endpoint: [{0}] {1}'.format(
self.dst_endpoint['id'],
self.dst_endpoint['display_name']))
if self.dst_endpoint is None:
self.logger.error('You don\'t have endpoint named: {0}'.format(
dst_endpoint_name))
raise LookupError
def transfer_dir(self, src_dir, dst_dir):
transfer_data = TransferData(self.transferClient,
self.src_endpoint['id'],
self.dst_endpoint['id'])
transfer_data.add_item(src_dir, dst_dir, recursive=True)
result = self.transferClient.submit_transfer(transfer_data)
self.logger.info('task [{0}] {1}'.format(result['task_id'],
result['code']))
return result
def transfer_file(self, src_file, dst_file):
transfer_data = TransferData(self.transferClient,
self.src_endpoint['id'],
self.dst_endpoint['id'])
transfer_data.add_item(src_file, dst_file)
result = self.transferClient.submit_transfer(transfer_data)
self.logger.info('task_id [{0}] {1}'.format(result['task_id'],
result['code']))
return result
def ls_src_dir(self, path, ls_filter=''):
# using iteration to get every entry from result
# an entry contain two keys: 'name' and 'type'
# type define the entry is a file or folder
result = self.transferClient.operation_ls(self.src_endpoint['id'],
path=path,
filter=ls_filter)
for entry in result:
self.logger.debug('name: {0}\ttype: {1}'.format(
entry["name"], entry["type"]))
return result
def task_list(self, num_results=10):
result = self.transferClient.task_list(num_results=num_results)
for task in result:
self.logger.debug('task_id: [{0}]\t status: {1}'.format(
task['task_id'], task['status']))
result = self.transferClient.task_list(num_results=num_results)
return result
def get_task(self, task_id):
return self.transferClient.get_task(task_id)
def _get_globus_sdk_native_client(
client_id: str = CLIENT_ID,
client_name: str = CLIENT_NAME,
):
return NativeAppAuthClient(client_id, app_name=client_name)
class GlobusStorageManager:
# https://globus-sdk-python.readthedocs.io/en/stable/clients/transfer/
app_id = 'b2fe5703-edb0-4f7f-80a6-2147c8ae35f0' # map transfer app id
class GlobusQueue:
''' placeholder for globus async helpers '''
pass
def __init__(self):
self.auth_client = NativeAppAuthClient(self.app_id)
self.auth_client.oauth2_start_flow(refresh_tokens=True)
self.xfer_client = None
custom = dj.config.get('custom', None)
if custom and 'globus.token' in custom:
self.refresh()
else:
self.login()
# authentication methods
def login(self):
''' fetch refresh token, store in dj.config['globus.token'] '''
auth_client = self.auth_client
print('Please login via: {}'.format(
auth_client.oauth2_get_authorize_url()))
code = input('and enter code:').strip()
tokens = auth_client.oauth2_exchange_code_for_tokens(code)
xfer_auth_cfg = tokens.by_resource_server['transfer.api.globus.org']
xfer_rt = xfer_auth_cfg['refresh_token']
xfer_at = xfer_auth_cfg['access_token']
xfer_exp = xfer_auth_cfg['expires_at_seconds']
xfer_auth = RefreshTokenAuthorizer(
xfer_rt, auth_client, access_token=xfer_at, expires_at=xfer_exp)
self.xfer_client = TransferClient(authorizer=xfer_auth)
custom = dj.config.get('custom', {})
custom['globus.token'] = xfer_rt
dj.config['custom'] = custom
def refresh(self):
''' use refresh token to refresh access token '''
auth_client = self.auth_client
xfer_auth = RefreshTokenAuthorizer(
dj.config['custom']['globus.token'], auth_client,
access_token=None, expires_at=None)
self.xfer_client = TransferClient(authorizer=xfer_auth)
# endpoint managment / utility methods
@classmethod
def ep_parts(cls, endpoint_path):
# split endpoint:/path to endpoint, path
epsplit = endpoint_path.split(':')
return epsplit[0], ':'.join(epsplit[1:])
def activate_endpoint(self, endpoint):
''' activate an endpoint '''
tc = self.xfer_client
r = tc.endpoint_autoactivate(endpoint, if_expires_in=3600)
log.debug('activate_endpoint() code: {}'.format(r['code']))
if r['code'] == 'AutoActivationFailed':
print('Endpoint({}) Not Active! Error! Source message: {}'
.format(endpoint, r['message']))
raise Exception('globus endpoint activation failure')
knownok = any(('AutoActivated' in r['code'],
'AlreadyActivated' in r['code']))
if not knownok:
log.debug('activate_endpoint(): not knownok response')
def _wait(self, task, timeout=10, polling_interval=10):
''' tranfer client common wait wrapper '''
return self.xfer_client.task_wait(task, timeout, polling_interval)
def _tasks(self):
'''
>>> tl = tc.task_list(num_results=25, filter="type:TRANSFER,DELETE")
>>> _ = [print(t["task_id"], t["type"], t["status"]) for t in tl]
'''
pass
def _task_info(self):
'''
>>> for event in tc.task_event_list(task_id):
>>> print("Event on Task({}) at {}:\n{}".format(
>>> task_id, event["time"], event["description"])
or
get_task
'''
pass
# transfer methods
def ls(self, endpoint_path):
'''
returns:
{
"DATA": [
{
"DATA_TYPE": "file",
"group": "staff",
"last_modified": "2018-05-22 18:49:19+00:00",
"link_group": null,
"link_last_modified": null,
"link_size": null,
"link_target": null,
"link_user": null,
"name": "map",
"permissions": "0755",
"size": 102,
"type": "dir",
"user": "******"
},
],
"DATA_TYPE": "file_list",
"absolute_path": null,
"endpoint": "aa4e5f9c-05f3-11e8-a6ad-0a448319c2f8",
"length": 2,
"path": "/~/Globus/",
"rename_supported": true,
"symlink_supported": false,
"total": 2
}
'''
ep, path = self.ep_parts(endpoint_path)
return self.xfer_client.operation_ls(ep, path=path)
def mkdir(self, ep_path):
''' create a directory at ep_path '''
ep, path = self.ep_parts(ep_path)
return self.xfer_client.operation_mkdir(ep, path=path)
def rmdir(self, ep_path, recursive=False):
''' remove a directory at ep_path '''
tc = self.xfer_client
ep, path = self.ep_parts(ep_path)
ddata = DeleteData(tc, ep, recursive=recursive)
ddata.add_item(path)
task_id = tc.submit_delete(ddata)['task_id']
return self._wait(task_id)
def cp(self, src_ep_path, dst_ep_path, recursive=False):
'''
copy file/path
todo: support label, sync_level, etc?
sync_level: ["exists", "size", "mtime", "checksum"]
'''
tc = self.xfer_client
sep, spath = self.ep_parts(src_ep_path)
dep, dpath = self.ep_parts(dst_ep_path)
td = TransferData(tc, sep, dep)
td.add_item(spath, dpath, recursive=recursive)
task_id = tc.submit_transfer(td)['task_id']
return self._wait(task_id)
def rename(self, src_ep_path, dst_ep_path):
''' rename a file/path '''
tc = self.xfer_client
sep, spath = self.ep_parts(src_ep_path)
dep, dpath = self.ep_parts(dst_ep_path)
if sep != dep:
raise Exception('rename between two different endpoints')
return tc.operation_rename(sep, spath, dpath)
#!/usr/bin/env python
from globus_sdk import NativeAppAuthClient
CLIENT_ID = "d0f1d9b0-bd81-4108-be74-ea981664453a"
SCOPES = ("openid profile email "
"urn:globus:auth:scope:transfer.api.globus.org:all")
get_input = getattr(__builtins__, 'raw_input', input)
client = NativeAppAuthClient(client_id=CLIENT_ID)
client.oauth2_start_flow(requested_scopes=SCOPES, refresh_tokens=True)
url = client.oauth2_get_authorize_url()
print("Native App Authorization URL: \n{}".format(url))
auth_code = get_input("Enter the auth code: ").strip()
token_res = client.oauth2_exchange_code_for_tokens(auth_code)
tokens = token_res.by_resource_server
transfer_token = tokens["transfer.api.globus.org"]["refresh_token"]
auth_token = tokens["auth.globus.org"]["refresh_token"]
id_token = token_res["id_token"]
access_token = token_res["access_token"]
print("Transfer Refresh Token: {}".format(transfer_token))
print("Auth Refresh Token : {}".format(auth_token))
print("Openid id_token : {}".format(id_token))
print("Access Token : {}".format(access_token))
# First authorize using those refresh tokens:
try:
tokens = load_tokens_from_file(TOKEN_FILE)
except:
print(
"Valid refresh tokens not found in {}. Unable to authorize to Globus. Exiting!"
.format(TOKEN_FILE))
sys.exit(-1)
transfer_tokens = tokens['transfer.api.globus.org']
try:
auth_client = NativeAppAuthClient(client_id=dcde_parsl_client_id)
except:
print(
"ERROR: Globus NativeAppAuthClient() call failed! Unable to obtain a Globus authorizer!"
)
sys.exit(-1)
authorizer = RefreshTokenAuthorizer(
transfer_tokens['refresh_token'],
auth_client,
access_token=transfer_tokens['access_token'],
expires_at=transfer_tokens['expires_at_seconds'],
on_refresh=update_tokens_file_on_refresh)
try:
tc = TransferClient(authorizer=authorizer)
def transfer(sp,destination_endpoint_id,one_endpoint):
tokens = None
try:
# if we already have tokens, load and use them
tokens = load_tokens_from_file(TOKEN_FILE)
except:
pass
if not tokens:
# if we need to get tokens, start the Native App authentication process
tokens = do_native_app_authentication(CLIENT_ID, REDIRECT_URI, SCOPES)
try:
save_tokens_to_file(TOKEN_FILE, tokens)
except:
pass
transfer_tokens = tokens['transfer.api.globus.org']
auth_client = NativeAppAuthClient(client_id=CLIENT_ID,environment='sandbox')
authorizer = RefreshTokenAuthorizer(
transfer_tokens['refresh_token'],
auth_client,
access_token=transfer_tokens['access_token'],
expires_at=transfer_tokens['expires_at_seconds'],
on_refresh=update_tokens_file_on_refresh)
#transfer = TransferClient(authorizer=authorizer,environment='sandbox')
tc = TransferClient(authorizer=authorizer, environment="sandbox")
##################---ENDPOINTS---###########################
source_endpoint_id = '5a2e5704-b028-11e7-bdad-22000bdb2406' #sb vmtb4
#source_endpoint_id = '55705028-aa15-11e7-bdad-22000bdb2406' #sb yulie7t
#source_endpoint_id = 'b0b16296-88e7-11e7-a971-22000a92523b' #bare chameleon
#source_endpoint_id = 'e5762bc2-8466-11e7-a8ed-22000a92523b' #large_chameleon
#source_endpoint_id = '8b26cc0e-877b-11e7-a949-22000a92523b'#ubuntu-vm
#source_endpoint_id = 'ad19b012-77cf-11e7-8b98-22000b9923ef'#chameleon
# source_endpoint_id = raw_input('Input source endpoint UUID: ')
#destination path
##############---SOURCE PATH---######################
#source_path = '/home/parallels/stream_transfer/test_files/'
#source_path = '/home/parallels/stream_transfer/zero_globus/test_files/'
source_path = sp
#source_path ='/home/cc/streaming/zero_globus/test_files/test.txt'
#source_path = '/home/parallels/stream_transfer/zero_globus/test_files/test.txt'
#destination path
destination_path = '/~/'
#destination_path = '/~/'+ sp.split("/")[-1] #use for one file
##if one_endpoint:
## destination_path = '/projects/BrainImagingADSP/yzamora/'
##else:
## destination_path = '/projects/BrainImagingADSP/yzamora/'+ sp.split("/")[-1] #use for one file
#Using my sample UUID from globus tutorial
#destination_endpoint_id = 'ddb59aef-6d04-11e5-ba46-22000b92c6ec' #globus
#destination_endpoint_id = '5d1da0fe-3c07-11e7-bcfc-22000b9a448b' #laptop
#tc.endpoint_autoactivate(source_endpoint_id)
#tc.endpoint_autoactive(destination_endpoint_id)
ep1 = tc.get_endpoint(destination_endpoint_id)
tc.endpoint_autoactivate(destination_endpoint_id)
#ep1 is setting the activated endpoint to be a variable to work with
tc.endpoint_autoactivate(source_endpoint_id)
label = "medium data transfer"
#tdata = globus_sdk.TransferData(tc, source_endpoint_id, destination_endpoint_id,label=label, sync_level='0')
tdata = globus_sdk.TransferData(tc, source_endpoint_id, destination_endpoint_id,label=label, perf_cc=3, sync_level=None, verify_checksum=False)
#tdata = globus_sdk.TransferData(tc, source_endpoint_id, destination_endpoint_id,label=label)
if one_endpoint:
tdata.add_item(source_path,destination_path,recursive=True)
else:
tdata.add_item(source_path,destination_path,recursive=False)
submit_result = tc.submit_transfer(tdata)
print("Task ID:", submit_result["task_id"])
"""
Checking for time completion using globus calls
"""
#print("Completion time:", submit_result["completion_time"])
#setup of the transfer, submits as a https post request
#transfer_data = TransferData(transfer_client=tc,
# source_endpoint=source_endpoint_id,
# destination_endpoint=destination_endpoint_id,
# label='Transfer',
# sync_level='checksum')
#transfer_data.add_item(source_path=source_path,destination_path=destination_path, recursive=False)
#task_id=transfer.submit_transfer(transfer_data)['task_id']
#waiting for file to transfer
status = tc.get_task(submit_result["task_id"],fields="status")["status"]
poll_interval = 2
max_wait = 90
wait_time = 0
while not (status in ["SUCCEEDED", "FAILED"]):
if (wait_time >= max_wait): break
print("Task not yet complete (status {}), sleeping for {} seconds..." \
.format(status, poll_interval))
time.sleep(poll_interval)
wait_time += poll_interval
status = tc.get_task(submit_result["task_id"], fields="status")["status"]
if status == "FAILED":
print("WARNING! File transfer FAILED!")
#deleting file after transfer
if status == "SUCCEEDED":
end_time = datetime.datetime.utcnow()
start_time = end_time - datetime.timedelta(minutes=200)
#limit = response objects
# data = tc.task_list(filter="type:TRANSFER,DELETE/request_time:%s,%s"
# % (start_time, end_time), limit=5)
#print("File transfer SUCCEEDED, will delete file from local directory now")
""" r = tc.task_list(num_results=1, filter="type:TRANSFER,DELETE")
def __init__(self):
self.log = logging.getLogger(self.__class__.__name__)
self.log.info("init - started")
config = configparser.ConfigParser()
config.read(str(CONFIG_PATH))
# To set up a client_id, see https://auth.globus.org/v2/web/developers
# Current client_id is in Project: MaterialsCommonsProject, App: MaterialsCommonsTest
client_id = config['sdk']['id']
auth_tokens = None
transfer_tokens = None
tokens = None
try:
# if we already have tokens, load and use them
tokens = self._load_tokens_from_file(TOKEN_FILE_PATH)
except IOError:
pass
if not tokens:
# if we need to get tokens, start the Native App authentication process
tokens = self.do_native_app_authentication(client_id, REDIRECT_URI,
SCOPES)
try:
self._save_tokens_to_file(TOKEN_FILE_PATH, tokens)
except IOError:
pass
try:
auth_tokens = tokens['auth.globus.org']
transfer_tokens = tokens['transfer.api.globus.org']
except KeyError as er:
self.log.error(
"KeyError on NativeApp tokens: {}\n delete {} and restart".
format(er, TOKEN_FILE_PATH))
def refresh_tokens(token_response):
context = self
context._update_tokens_file_on_refresh(token_response)
auth_client = NativeAppAuthClient(client_id=client_id)
authorizer = RefreshTokenAuthorizer(
auth_tokens['refresh_token'],
auth_client,
access_token=auth_tokens['access_token'],
expires_at=auth_tokens['expires_at_seconds'],
on_refresh=refresh_tokens)
auth_client = AuthClient(client_id=client_id, authorizer=authorizer)
authorizer = RefreshTokenAuthorizer(
transfer_tokens['refresh_token'],
auth_client,
access_token=transfer_tokens['access_token'],
expires_at=transfer_tokens['expires_at_seconds'],
on_refresh=refresh_tokens)
transfer_client = TransferClient(authorizer=authorizer)
self._auth_client = auth_client
self._transfer_client = transfer_client
def _login_client(config):
client_id = config.get('client', 'client_id')
return NativeAppAuthClient(client_id, app_name='identifier_client_dev')
def initiate_connection(self):
""" Initiate the connection
"""
tokens = None
try:
# if we already have tokens, load and use them
tokens = self.load_tokens_from_file()
except:
pass
if not tokens:
# if we need to get tokens, start the Native App authentication process
client = NativeAppAuthClient(
client_id=self.CLIENT_ID) #self.config['client_id'])
# pass refresh_tokens=True to request refresh tokens
client.oauth2_start_flow(
requested_scopes=self.
SCOPES, #self.config['requested_scopes'],
redirect_uri=self.REDIRECT_URI, #self.config['redirect_uri'],
refresh_tokens=True)
url = client.oauth2_get_authorize_url()
print 'Native App Authorization URL: \n{}'.format(url)
auth_code = raw_input('Enter the auth code: ').strip()
token_response = client.oauth2_exchange_code_for_tokens(auth_code)
# return a set of tokens, organized by resource server name
tokens = token_response.by_resource_server
try:
self.save_tokens_to_file(tokens)
except:
pass
transfer_tokens = tokens['transfer.api.globus.org']
auth_client = NativeAppAuthClient(client_id=self.config['client_id'])
authorizer = RefreshTokenAuthorizer(
transfer_tokens['refresh_token'],
auth_client,
#access_token=transfer_tokens['access_token'],
#expires_at=transfer_tokens['expires_at_seconds'],
on_refresh=self.update_tokens_file_on_refresh)
self.client = TransferClient(authorizer=authorizer)
# print out a directory listing from an endpoint
try:
#print 'ACTIVATE'
#print 'DEST',self.config['dest_ep']
self.client.endpoint_autoactivate(self.config['dest_ep'])
ac = self.client.endpoint_get_activation_requirements(
self.config['dest_ep'])
#print ac
self.client.endpoint_autoactivate(self.config['src_ep'])
ac2 = self.client.endpoint_get_activation_requirements(
self.config['src_ep'])
#print ac2
except GlobusAPIError as ex:
self.logger.error('Error in endpoint activation %s', str(ex))
if ex.http_status == 401:
sys.exit('Refresh token has expired. '
'Please delete refresh-tokens.json and try again.')
else:
raise ex
class NativeClient(object):
"""
The Native Client serves as another small layer on top of the Globus SDK
to automatically handle token storage and provide a customizable
Local Server. It can be used both by simple scripts to simplify the
auth flow, or by full command line clients that may extend various pieces
and tailor them to its own needs.
"""
TOKEN_STORAGE_ATTRS = {'write_tokens', 'read_tokens', 'clear_tokens'}
def __init__(self,
token_storage=MultiClientTokenStorage(),
local_server_code_handler=LocalServerCodeHandler(),
secondary_code_handler=InputCodeHandler(),
default_scopes=None,
*args,
**kwargs):
self.client = NativeAppAuthClient(*args, **kwargs)
self.token_storage = token_storage
if token_storage is not None:
self.verify_token_storage(self.token_storage)
self.app_name = kwargs.get('app_name') or 'My App'
self.local_server_code_handler = local_server_code_handler
self.local_server_code_handler.set_app_name(self.app_name)
self.secondary_code_handler = secondary_code_handler
if isinstance(self.token_storage, MultiClientTokenStorage):
self.token_storage.set_client_id(kwargs.get('client_id'))
self.default_scopes = default_scopes
def login(self,
no_local_server=False,
no_browser=False,
requested_scopes=(),
refresh_tokens=None,
prefill_named_grant=None,
additional_params=None,
force=False):
"""
Do a Native App Auth Flow to get tokens for requested scopes. This
first attempts to load tokens and will simply return those if they are
valid, and will automatically attempt to save tokens on login success
(token_storage must be set for automatic load/save functionality).
:param no_local_server: Don't use the local server. This may be because
of permissions issues of standing up a server on the clients machine.
:param no_browser: Don't automatically open a browser, and instead
instruct the user to manually follow the URL to login. This is useful
on remote servers which don't have native browsers for clients to use.
:param requested_scopes: Globus Scopes to request on the users behalf.
:param refresh_tokens: Use refresh tokens to extend login time
:param prefill_named_grant: Named Grant to use on Consent Page
:param additional_params: Additional Params to supply, such as for
using Globus Sessions
:param force: Force a login flow, even if loaded tokens are valid.
:return:
"""
if force is False:
try:
return self.load_tokens(requested_scopes=requested_scopes)
except (LoadError, Exception):
pass
grant_name = prefill_named_grant or '{} Login'.format(self.app_name)
code_handler = (self.secondary_code_handler
if no_local_server else self.local_server_code_handler)
with code_handler.start():
self.client.oauth2_start_flow(
requested_scopes=requested_scopes or self.default_scopes,
refresh_tokens=refresh_tokens,
prefill_named_grant=grant_name,
redirect_uri=code_handler.get_redirect_uri())
auth_url = self.client.oauth2_get_authorize_url(
additional_params=additional_params)
auth_code = code_handler.authenticate(url=auth_url,
no_browser=no_browser)
token_response = self.client.oauth2_exchange_code_for_tokens(auth_code)
try:
self.save_tokens(token_response.by_resource_server)
except LoadError:
pass
return token_response.by_resource_server
def verify_token_storage(self, obj):
for attr in self.TOKEN_STORAGE_ATTRS:
if getattr(obj, attr, None) is None:
raise AttributeError('token_storage requires object "{}" to '
'have the {} attribute'.format(obj, attr))
def save_tokens(self, tokens):
"""
Save tokens if token_storage is set. Typically this is called
automatically in a successful login().
:param tokens: globus_sdk.auth.token_response.OAuthTokenResponse.
:return: None
"""
if self.token_storage is not None:
return self.token_storage.write_tokens(tokens)
raise LoadError('No token_storage set on client.')
def _load_raw_tokens(self):
"""
Loads tokens without checking them
:return: tokens by resource server, or an exception if that fails
"""
if self.token_storage is not None:
return self.token_storage.read_tokens()
raise LoadError('No token_storage set on client.')
def load_tokens(self, requested_scopes=None):
"""
Load tokens from the set token_storage object if one exists.
:param requested_scopes: Check that the loaded scopes match these
requested scopes. Raises ScopesMismatch if there is a discrepancy.
:return: Loaded tokens, or a LoadError if loading fails.
"""
tokens = self._load_raw_tokens()
if not tokens:
raise LoadError('No Tokens loaded')
if requested_scopes not in [None, ()]:
check_scopes(tokens, requested_scopes)
try:
check_expired(tokens)
except TokensExpired as te:
expired = {rs: tokens[rs] for rs in te.resource_servers}
if not self._refreshable(expired):
raise
tokens.update(self.refresh_tokens(expired))
self.save_tokens(tokens)
return tokens
def _refreshable(self, tokens):
return all([bool(ts['refresh_token']) for ts in tokens.values()])
def refresh_tokens(self, tokens):
if not self._refreshable(tokens):
raise TokensExpired('No Refresh Token, cannot refresh tokens: ',
resource_servers=tokens.keys())
for rs, token_dict in tokens.items():
authorizer = RefreshTokenAuthorizer(
token_dict['refresh_token'],
self.client,
access_token=token_dict['access_token'],
expires_at=token_dict['expires_at_seconds'],
)
authorizer.check_expiration_time()
token_dict['access_token'] = authorizer.access_token
token_dict['expires_at_seconds'] = authorizer.expires_at
return tokens
def get_authorizers(self):
authorizers = {}
for resource_server, token_dict in self.load_tokens().items():
if token_dict.get('refresh_token') is not None:
authorizers[resource_server] = RefreshTokenAuthorizer(
token_dict['refresh_token'],
self.client,
access_token=token_dict['access_token'],
expires_at=token_dict['expires_at_seconds'],
on_refresh=self.on_refresh,
)
else:
authorizers[resource_server] = AccessTokenAuthorizer(
token_dict['access_token'])
return authorizers
def on_refresh(self, token_response):
loaded_tokens = self._load_raw_tokens()
loaded_tokens.update(token_response.by_resource_server)
self.save_tokens(loaded_tokens)
def logout(self):
"""
Revoke saved tokens and clear them from storage
"""
self.revoke_token_set(self._load_raw_tokens())
self.token_storage.clear_tokens()
def revoke_token_set(self, tokens):
for rs, tok_set in tokens.items():
self.client.oauth2_revoke_token(tok_set.get('access_token'))
self.client.oauth2_revoke_token(tok_set.get('refresh_token'))
