def add_nonce(self, permission='READ', maxuses=1, actorId=None):
"""
Add a new nonce.
Positional arguments:
None
Keyword arguments:
username: str - a valid TACC.cloud username or role account
permission: str - a valid Abaco permission level
maxuses: int (-1,inf) - maximum number of uses for a given nonce
actorId: str - an Abaco actor ID. Defaults to self.uid if not set.
"""
assert permission in ('READ', 'EXECUTE', 'UPDATE'), \
'Invalid permission: (READ, EXECUTE, UPDATE)'
assert isinstance(maxuses, int), 'Invalid max_uses: (-1,-inf)'
assert maxuses >= -1, 'Invalid max_uses: (-1,-inf)'
if actorId:
_actorId = actorId
else:
_actorId = self.uid
try:
body = {'level': permission,
'maxUses': maxuses}
resp = self.client.actors.addNonce(actorId=_actorId,
body=json.dumps(body))
return resp
except HTTPError as h:
http_err_resp = agaveutils.process_agave_httperror(h)
raise AgaveError(http_err_resp)
except Exception as e:
raise AgaveError(
"Unknown error: {}".format(e))
def create_webhook(self, permission='EXECUTE', maxuses=-1, actorId=None):
"""
Create a .actor.messages URI suitable for use in integrations
Default is to grant EXECUTE with unlimited uses.
"""
if actorId is not None:
_actorId = actorId
else:
_actorId = self.uid
try:
api_server = agaveutils.utils.get_api_server(self.client)
nonce = self.add_nonce(permission,
maxuses, actorId=_actorId)
nonce_id = nonce.get('id')
uri = '{}/actors/v2/{}/messages?x-nonce={}'.format(
api_server, _actorId, nonce_id)
if validators.url(uri):
return uri
else:
raise ValueError("Webhook URI {} is not valid".format(uri))
except HTTPError as h:
http_err_resp = agaveutils.process_agave_httperror(h)
raise AgaveError(http_err_resp)
except Exception as e:
raise AgaveError(
"Unknown error: {}".format(e))
def delete_nonce(self, nonceId, actorId=None):
"""
Delete an specific nonce by its ID
Positional arguments:
nonceId: str - a valid TACC.cloud username or role account
Keyword arguments:
actorId: str - an Abaco actor ID. Defaults to self.uid if not set.
"""
if actorId:
_actorId = actorId
else:
_actorId = self.uid
try:
resp = self.client.actors.deleteNonce(
actorId=_actorId, nonceId=nonceId)
return resp
except HTTPError as h:
http_err_resp = agaveutils.process_agave_httperror(h)
raise AgaveError(http_err_resp)
except Exception as e:
raise AgaveError(
"Unknown error: {}".format(e))
def list_nonces(self, actorId=None):
"""
List all nonces
Positional arguments:
None
Keyword arguments:
actorId: str - an Abaco actor ID. Defaults to self.uid if not set.
"""
if actorId:
_actorId = actorId
else:
_actorId = self.uid
try:
resp = self.client.actors.listNonces(
actorId=_actorId)
return resp
except HTTPError as h:
http_err_resp = agaveutils.process_agave_httperror(h)
raise AgaveError(http_err_resp)
except Exception as e:
raise AgaveError(
"Unknown error: {}".format(e))
def take_action(self, parsed_args):
parsed_args = CreateTokenFormatOne.before_take_action(
self, parsed_args)
self.requests_client.setup(API_NAME, SERVICE_VERSION)
self.take_action_defaults(parsed_args)
# Allow prompt for password when not specified
passwd = parsed_args.tapis_password
if passwd is None:
passwd = prompt('Password', passwd, secret=True)
headers = SearchableCommand.headers(self, Token, parsed_args)
try:
self.tapis_client.token.password = passwd
result = self.tapis_client.token.create()
self.tapis_client.token.password = None
except HTTPError as h:
if str(h).startswith('400'):
raise AgaveError(
'Failed to create a token pair: {0}'.format(h))
else:
raise AgaveError(str(h))
result = list()
for h in headers:
result.append(self.tapis_client.token.token_info.get(h))
return (tuple(headers), tuple(result))
def delete_webhook(self, webhook, actorId=None):
"""
'Delete' an actor-specific webhook by deleting its nonce
A key assumption is that webhook was constructed by create_webhook or
its equivalent, as this method sensitive to case and url structure
"""
if actorId is not None:
_actorId = actorId
else:
_actorId = self.uid
# webhook must be plausibly associated with the specified actor
if not re.search('/actors/v2/{}'.format(_actorId), webhook):
raise ValueError("URI doesn't map to actor {}".format(_actorId))
try:
m = re.search('x-nonce=([A-Z0-9a-z\\.]+_[A-Z0-9a-z]+)', webhook)
nonce_id = m.groups(0)[0]
self.delete_nonce(nonceId=nonce_id, actorId=_actorId)
except HTTPError as h:
http_err_resp = agaveutils.process_agave_httperror(h)
raise AgaveError(http_err_resp)
except Exception as e:
raise AgaveError(
"Unknown error: {}".format(e))
def init_clients(self, parsed_args):
"""Override CommandBase to set up client with passed token
"""
# client = Agave.restore()
if parsed_args.api_server is not None and parsed_args.access_token is not None:
self.tapis_client = Agave(api_server=parsed_args.api_server,
token=parsed_args.access_token)
elif parsed_args.access_token is not None:
try:
client = Agave._read_current(agave_kwargs=True)
except Exception:
raise AgaveError(
'Tapis API server was not discoverable. Exiting.')
self.tapis_client = Agave(api_server=client['api_server'],
token=parsed_args.access_token)
else:
try:
client = Agave.restore()
self.tapis_client = client
self.tapis_client.refresh()
# self.requests_client = self._get_direct(self.tapis_client)
except Exception:
raise AgaveError(constants.TAPIS_AUTH_FAIL)
try:
self.requests_client = self._get_direct(self.tapis_client)
except Exception:
raise AgaveError(constants.TAPIS_AUTH_FAIL)
return self
def take_action(self, parsed_args):
parsed_args = super(TokenCreate, self).preprocess_args(parsed_args)
self.update_payload(parsed_args)
# Allow prompt for password when not specified
passwd = parsed_args.tapis_password
if passwd is None:
passwd = prompt('Password', passwd, secret=True)
headers = super(TokenCreate, self).render_headers(Token, parsed_args)
self.tapis_client.token.password = passwd
result = list()
try:
if parsed_args.token_username is None:
resp = self.tapis_client.token.create()
self.tapis_client.token.password = None
for h in headers:
# DERP
result.append(self.tapis_client.token.token_info.get(h))
else:
self.requests_client.setup(API_NAME, None)
data = {
'token_username': parsed_args.token_username,
'username': self.tapis_client.username,
'password': passwd,
'scope': 'PRODUCTION',
'grant_type': 'admin_password'
}
resp = self.requests_client.post_data_basic(data)
# Not returned by service
headers.remove('expires_at')
# Do not return - we want impersonation tokens to expire
headers.remove('refresh_token')
for h in headers:
# DERP
result.append(resp.get(h))
# Manually insert token username into response
headers.append('username')
# Nice feature - highlight the username
result.append(parsed_args.token_username)
except HTTPError as h:
if str(h).startswith('400'):
raise AgaveError(
'Failed to create a token pair: {0}'.format(h))
else:
raise AgaveError(str(h))
et.phone_home()
return (tuple(headers), tuple(result))
def init_clients(self, parsed_args):
"""Override CommandBase to set up client with passed token
"""
api_server = getattr(parsed_args, 'api_server', None)
token = getattr(parsed_args, 'access_token', None)
nonce = getattr(parsed_args, 'nonce', None)
verify_ssl = getattr(parsed_args, 'verify_ssl', True)
if (token is not None or nonce is not None) and api_server is None:
try:
client = Agave._read_current(agave_kwargs=True)
api_server = client['api_server']
except Exception:
raise AgaveError('Unable to discover Tapis API server URL.')
# Initialize the AgavePy client
try:
if api_server is not None and token is not None:
self.tapis_client = Agave(api_server=api_server,
token=token,
verify=verify_ssl)
elif api_server is not None and nonce is not None:
self.tapis_client = Agave(api_server=api_server,
use_nonce=True,
verify=verify_ssl)
self.client_extra_args['nonce'] = nonce
else:
# Load from disk cache
# client = Agave.restore()
clients = Agave._read_clients()
client0 = clients[0]
# Override SSL verification from stored client
client0['verify'] = verify_ssl
client = Agave(**client0)
self.tapis_client = client
self.tapis_client.refresh()
except Exception:
raise AgaveError(constants.TAPIS_AUTH_FAIL)
# Initialize the direct requests client
try:
# Direct client will inherit SSL check behavior from Tapis client
self.requests_client = self._get_direct(self.tapis_client)
except Exception:
raise AgaveError(constants.TAPIS_AUTH_FAIL)
return self
def upload_launch(self, key_event_id):
"""Launch instance of Abaco upload manager
Retries failed attempts using exponential backoff
Arguments:
key_event_id (tuple): (key:str, x-amz-request-id:str)
Returns:
tuple: (actor_id:str, execution_id:str, event_id:str)
"""
# See https://stackoverflow.com/questions/23857005/get-the-name-of-celery-worker-from-inside-a-celery-task
# for better ways of doing this
worker = self.request.hostname.split('@')[0]
# Probably init this on worker setup as well...
ag = Agave(api_server=settings['api']['server'],
username=settings['api']['username'],
password=settings['api']['password'],
api_key=settings['api']['clients'][worker]['api_key'],
api_secret=settings['api']['clients'][worker]['api_secret'])
msg = {
'uri': 's3://{0}'.format(key_event_id[0]),
'event_id': key_event_id[1]
}
try:
resp = ag.actors.sendMessage(actorId=settings['actor_id'],
body={'message': msg})
except Exception as exc:
raise AgaveError('Failed to launch task: {0}'.format(exc))
return (settings['actor_id'], resp['executionId'], key_event_id[1])
def upload_monitor(self, actor_exec_event, *args, **kwargs):
"""Monitor a running instance of Abaco uploads manager
Implements polling via Celery retry
Arguments:
actor_exec_event (tuple): (actor_id:str, execution_id:str, event_id:str)
Returns:
tuple: (actor_id:str, execution_id:str, status:str, event_id:str)
"""
actor_id = actor_exec_event[0]
execution_id = actor_exec_event[1]
event_id = actor_exec_event[2]
worker = self.request.hostname.split('@')[0]
# Probably init this on worker setup as well...
ag = Agave(api_server=settings['api']['server'],
username=settings['api']['username'],
password=settings['api']['password'],
api_key=settings['api']['clients'][worker]['api_key'],
api_secret=settings['api']['clients'][worker]['api_secret'])
try:
status = ag.actors.getExecution(actorId=actor_id,
executionId=execution_id).get(
'status', None)
except Exception as exc:
raise AgaveError('Failed to poll task: {0}'.format(exc))
fsevent_update.s((event_id, status)).apply_async()
if status not in ['COMPLETE', 'ERROR']:
raise ActorExecutionInProgress
else:
return (actor_id, execution_id, status, event_id)
def _get(self, key):
'''Get value by key name.'''
shares = False
key_name = self._namespace(key)
username = self._username()
# An Agave metadata object, not yet the value
if shares:
_regex = "^{}/{}#".format(self.prefix, key)
query = json.dumps({'name': {'$regex': _regex, '$options': 'i'}})
else:
query = json.dumps({'name': key_name})
try:
key_objs = self.client.meta.listMetadata(q=query)
assert isinstance(key_objs, list)
except Exception as e:
self.logging.debug("Failed to listMetadata")
raise AgaveError("Failed at meta.listMetadata: {}".format(e))
key_objs_owner = []
key_objs_other = []
for key_obj in key_objs:
if key_obj['owner'] == username:
key_objs_owner.append(key_obj)
else:
key_objs_other.append(key_obj)
key_objs_merged = key_objs_owner + key_objs_other
if len(key_objs_merged) > 0:
return key_objs_merged[0]
else:
raise KeyError("No such key: {}".format(key))
def get_tapis_user(self, username=None, permissive=False):
"""Retrieve a username record from the Tapis profile service
"""
if username is None:
try:
uname = self.current_tapis_user()
return uname
except Exception:
raise ValueError('Username must be resolvable from environment or provided')
# Agave/APIM specialty accounts
if username in tacc.username.ROLE_USERNAMES:
return {'first_name': None, 'last_name': None, 'full_name': None,
'email': None, 'phone': None, 'mobile_phone': None,
'nonce': None, 'status': None,
'create_time': '20140515180317Z', 'uid': None,
'username': username}
try:
if self.client is None:
raise AgaveError('TACC API client not initialized before use')
else:
return self.client.profiles.listByUsername(username=username)
except Exception:
if permissive:
return None
else:
raise
def validate_acl(cls, acl, permissive=False):
"""
Validate an ACL object as a dict
Failure raises Exception unless permissive is True
* Does not validate that username exists
"""
err = 'Invalid ACL: {}'
try:
assert isinstance(acl, dict), "Not a dict"
assert 'username' in acl and 'permission' in acl, \
"Both username and permission are required"
assert isinstance(acl['permission'], dict), \
"Permission must be a dict"
assert isinstance(acl['username'], basestring), \
"Username must be string or unicode"
assert set(acl['permission'].keys()) == set(VALID_PEMS) or \
set(acl['permission'].keys()) <= set(VALID_PEMS), \
"Valid permission types are {} not {}".format(
VALID_PEMS, list(acl['permission'].keys()))
for p in acl['permission']:
assert isinstance(acl['permission'][p], bool), \
"Only Boolean values allowed for permission values"
return True
except Exception as exc:
if permissive is True:
return False
else:
raise AgaveError(err.format(exc))
def _getall(self, namespace=False, sort_aliases=True, uuids=False):
'''Fetch and return all keys visible to the user'''
all_keys = []
_regex = "^{}/*".format(self.prefix)
query = json.dumps({'name': {'$regex': _regex, '$options': 'i'}})
# collection of Agave metadata objects
try:
key_objs = self.client.meta.listMetadata(q=query)
assert isinstance(key_objs, list)
except Exception as e:
self.logging.debug("Failed to listMetadata")
raise AgaveError("Failed at meta.listMetadata: {}".format(e))
for key_obj in key_objs:
if uuids:
all_keys.append(key_obj['uuid'])
elif namespace:
all_keys.append(key_obj['name'])
else:
all_keys.append(self._rev_namespace(key_obj['name']))
if sort_aliases:
all_keys.sort()
return all_keys
def upload(agave_client,
file_to_upload,
destination_path,
system_id='data-sd2e-community',
autogrant=False):
try:
direct_put(file_to_upload,
destination_path,
system_id='data-sd2e-community')
except DirectOperationFailed as exc:
pprint(exc)
try:
agave_client.files.importData(systemId=system_id,
filePath=destination_path,
fileToUpload=open(
file_to_upload, 'rb'))
except HTTPError as h:
http_err_resp = agaveutils.process_agave_httperror(h)
raise Exception(http_err_resp)
except Exception as e:
raise AgaveError("Error uploading {}: {}".format(
file_to_upload, e))
if autogrant:
return grant(agave_client, destination_path)
else:
return True
def take_action(self, parsed_args):
parsed_args = self.preprocess_args(parsed_args)
actor_id = ActorIdentifier().get_identifier(parsed_args)
msg = self.prepare_message(parsed_args)
resp = None
try:
API_PATH = '{0}/messages'.format(actor_id)
self.requests_client.setup(API_NAME, SERVICE_VERSION, API_PATH)
# Expecting a binary response
resp = self.requests_client.post(params=msg[1], json=msg[0])
except Exception as err:
raise AgaveError('Message failed or timed out: {0}'.format(err))
try:
of = None
if parsed_args.output == sys.stdout:
print(resp.decode('utf-8'))
elif parsed_args.binary:
of = open(parsed_args.output, 'wb')
of.write(resp)
of.close()
else:
of = open(parsed_args.output, 'w')
of.write(resp.decode('utf-8'))
of.close()
except Exception:
raise
sys.exit(0)
def _getacls(self, key, user=None):
'''List ACLs on a given key'''
key_uuid = None
key_uuid_obj = {}
acls = []
try:
key_uuid_obj = self._get(key)
key_uuid = key_uuid_obj['uuid']
except KeyError:
self.logging.debug("Key {} not found".format(key))
raise KeyError("Key {} not found".format(key))
try:
resp = self.client.meta.listMetadataPermissions(uuid=key_uuid)
for acl in resp:
formatted_acl = {
'username': acl.get('username'),
'permission': acl.get('permission')
}
if user is None:
acls.append(formatted_acl)
else:
if user == acl.get('username'):
acls.append(formatted_acl)
# show the user is inheriting world acl
elif 'world' == acl.get('username'):
acls.append(formatted_acl)
return acls
except Exception as e:
self.logging.debug("Failed getting ACLs for for {}: {}".format(
key, e))
raise AgaveError("Failed to get ACL for {}: {}".format(key, e))
def __get_api_username(self):
'''Determine username'''
if os.environ.get('_abaco_username'):
return os.environ.get('_abaco_username')
elif self.client.username is not None:
return self.client.username
else:
raise AgaveError("No username could be determined")
def __get_api_token(self):
'''Determine API access_token'''
if os.environ.get('_abaco_access_token'):
return os.environ.get('_abaco_access_token')
elif self.client.token.token_info.get('access_token') is not None:
return self.client.token.token_info.get('access_token')
else:
raise AgaveError("Failed to retrieve API access_token")
def _rem_by_uuid(self, key_uuid):
'''Delete key by its UUID'''
try:
self.client.meta.deleteMetadata(uuid=key_uuid)
return True
except Exception as e:
raise AgaveError("Failed to delete UUID {}: {}}".format(
key_uuid, e))
def take_action(self, parsed_args):
parsed_args = TokenFormatOne.before_take_action(self, parsed_args)
self.requests_client.setup(API_NAME, SERVICE_VERSION)
self.take_action_defaults(parsed_args)
headers = SearchableCommand.headers(self, Token, parsed_args)
try:
result = self.tapis_client.token.refresh()
except HTTPError as h:
if str(h).startswith('400'):
raise AgaveError(
'Failed to refresh token. Try "tapis sessions token create"'
)
else:
raise AgaveError(str(h))
result = list()
for h in headers:
result.append(self.tapis_client.token.token_info.get(h))
return (tuple(headers), tuple(result))
def get_client_with_mock_support():
'''
Get the current Actor API client
Returns the Abaco actor's client if running deployed. Attempts to
bootstrap a client from supplied credentials if running in local or
debug mode.
'''
client = None
if '_abaco_access_token' not in os.environ:
try:
client = Agave.restore()
except TypeError as err:
raise AgaveError('Unable to restore Agave client: {}'.format(err))
else:
try:
client = get_client()
except Exception as err:
raise AgaveError('Unable to get Agave client from context: {}'.format(err))
return client
def take_action(self, parsed_args):
parsed_args = super(TokenRefresh, self).preprocess_args(parsed_args)
self.requests_client.setup(API_NAME, SERVICE_VERSION)
self.update_payload(parsed_args)
headers = super(TokenRefresh, self).render_headers(Token, parsed_args)
try:
result = self.tapis_client.refresh()
except HTTPError as h:
if str(h).startswith('400'):
raise AgaveError(
'Failed to refresh token. Try "tapis sessions token create"'
)
else:
raise AgaveError(str(h))
result = list()
for h in headers:
result.append(self.tapis_client.token.token_info.get(h))
et.phone_home()
return (tuple(headers), tuple(result))
def download(agave_client,
file_to_download,
local_filename,
system_id='data-sd2e-community'):
try:
direct_get(file_to_download,
local_filename,
system_id='data-sd2e-community')
except DirectOperationFailed as exc:
pprint(exc)
# Download using Agave API call
try:
downloadFileName = os.path.join(PWD, local_filename)
# Implements atomic download
f = tempfile.NamedTemporaryFile('wb', delete=False, dir=PWD)
# with open(downloadFileName, 'wb') as f:
rsp = agave_client.files.download(systemId=system_id,
filePath=file_to_download)
if isinstance(rsp, dict):
raise AgaveError(
"Failed to download {}".format(file_to_download))
for block in rsp.iter_content(2048):
if not block:
break
f.write(block)
try:
os.rename(f.name, downloadFileName)
except Exception as rexc:
raise OSError('Atomic rename failed after download', rexc)
return downloadFileName
except (HTTPError, AgaveError) as http_err:
try:
os.unlink(f.name)
except Exception:
pass
if re.compile('404 Client Error').search(str(http_err)):
raise HTTPError('404 Not Found') from http_err
else:
http_err_resp = agaveutils.process_agave_httperror(http_err)
raise AgaveError(http_err_resp) from http_err
def delete_all_nonces(self, actorId=None):
"""
Delete all nonces from an actor
Keyword arguments:
actorId: str - an Abaco actor ID. Defaults to self.uid if not set.
"""
if actorId:
_actorId = actorId
else:
_actorId = self.uid
try:
nonces = self.list_nonces(actorId=_actorId)
assert isinstance(nonces, list)
for nonce in nonces:
self.delete_nonce(nonce.get('id'), actorId=_actorId)
except HTTPError as h:
http_err_resp = agaveutils.process_agave_httperror(h)
raise AgaveError(http_err_resp)
except Exception as e:
raise AgaveError(
"Unknown error: {}".format(e))
def from_agave_uri(uri=None, validate=False):
"""Partition an Agave storage URI into its components
Args:
uri (str): An agave-canonical files URI
validate (bool, optional): Whether to validate the URL using an API call
Raises:
AgaveError: Occurs when invalid URI is passed
Returns:
tuple: Three strings are returned: storageSystem, directoryPath, and fileName
"""
systemId = None
dirPath = None
fileName = None
proto = re.compile(r'agave:\/\/(.*)$')
if uri is None:
raise AgaveError("URI cannot be empty")
resourcepath = proto.search(uri)
if resourcepath is None:
raise AgaveError("Unable identify protocol: {}".format(uri))
resourcepath = resourcepath.group(1)
firstSlash = resourcepath.find('/')
if firstSlash is -1:
raise AgaveError("Unable to resolve systemId: {}".format(uri))
try:
systemId = resourcepath[0:firstSlash]
origDirPath = resourcepath[firstSlash + 1:]
dirPath = '/' + os.path.dirname(origDirPath)
dirPath = normpath(dirPath)
fileName = os.path.basename(origDirPath)
return (systemId, dirPath, fileName)
except Exception as e:
raise AgaveError(
"Error resolving directory path or file name: {}".format(e))
def take_action(self, parsed_args):
parsed_args = self.preprocess_args(parsed_args)
self.requests_client.setup(API_NAME, SERVICE_VERSION)
self.update_payload(parsed_args)
# List roles on the System to show the new role
headers = self.render_headers(SystemRole, parsed_args)
rec = self.tapis_client.systems.deleteRoleForUser(
systemId=parsed_args.identifier, username=parsed_args.username)
if rec is None:
data = [parsed_args.username, None]
else:
raise AgaveError('Failed to revoke role from {0}'.format(
parsed_args.identifier))
return (tuple(headers), tuple(data))
def take_action(self, parsed_args):
parsed_args = SystemsFormatOne.before_take_action(self, parsed_args)
self.requests_client.setup(API_NAME, SERVICE_VERSION)
self.take_action_defaults(parsed_args)
# List roles on the System to show the new role
headers = SearchableCommand.headers(self, SystemRole, parsed_args)
rec = self.tapis_client.systems.deleteRoleForUser(
systemId=parsed_args.identifier, username=parsed_args.username)
if rec is None:
data = [parsed_args.username, None]
else:
raise AgaveError('Failed to revoke role from {0}'.format(
parsed_args.identifier))
return (tuple(headers), tuple(data))
def _setacl(self, key, acl):
'''Add or update an ACL to a key'''
key_uuid = None
key_uuid_obj = {}
try:
key_uuid_obj = self._get(key)
key_uuid = key_uuid_obj['uuid']
except KeyError:
self.logging.debug("Key {} not found".format(key))
raise KeyError("Key {} not found".format(key))
pem = self.to_text_pem(acl)
meta = json.dumps(pem, indent=0)
try:
self.client.meta.updateMetadataPermissions(uuid=key_uuid,
body=meta)
return True
except Exception as e:
self.logging.debug("Error setting ACL for {}: {}".format(key, e))
raise AgaveError("Failed to set ACL on {}: {}".format(key, e))