def __init__(self,
api_server=TACC_API_SERVER,
nonce=None,
access_token=None,
refresh_token=None,
sync=REACTOR_MESSAGE_SYNC,
verbose=Verbosity.DEFAULT):
self.log.debug('Initialize Abaco API client')
try:
if nonce is not None:
self.log.debug('Using Abaco authorization nonce')
ag = bacanora.agaveutils.agave.AgaveNonceOnly(
api_server=api_server, nonce=nonce)
elif access_token is not None:
self.log.debug('Using supplied Tapis credentials')
ag = Agave(api_server=api_server, token=access_token)
else:
# Use local credential cache
self.log.debug('Using local Tapis credential cache')
ag = Agave.restore()
except Exception:
self.log.exception('Unable to initialize Tapis client')
raise
self.log.debug('Done initializing Abaco API client')
self.client = ag
self.sync = sync
self.verbosity = verbose
def connect(self):
agave_connection_type = self._config.get('connection_type',
'impersonate')
if agave_connection_type == 'impersonate':
token_username = '******'.format(
self._config['domain'], '/' if self._config['domain'] else '',
self._config['token_username'])
Log.some().debug('user impersonation: %s', token_username)
self._agave = Agave(api_server=self._config['server'],
username=self._config['username'],
password=self._config['password'],
token_username=token_username,
client_name=self._config['client'],
api_key=self._config['key'],
api_secret=self._config['secret'],
verify=False)
elif agave_connection_type == 'agave-cli':
# get credentials from ~/.agave/current
agave_clients = Agave._read_clients()
agave_clients[0]['verify'] = False # don't verify ssl
self._agave = Agave(**agave_clients[0])
# when using agave-cli, token_username must be the same as the
# stored creds in user's home directory, this can be different
# from job username
self._config['token_username'] \
= agave_clients[0]['username']
else:
Log.an().error('invalid agave connection type: %s',
agave_connection_type)
return False
return True
def create_client():
"""
Create a session using credentials in secrets.json.
:return: new Agave client
"""
secrets = get_secrets()
credentials = secrets['agave']
if not (credentials.get('api_key') and credentials.get('api_secret')):
print(
"Generating new api_key and api_secret and saving to secrets.json")
ag = Agave(api_server=credentials['api_server'],
username=credentials['username'],
password=credentials['password'])
new_secrets = ag.clients_create()
credentials.update(new_secrets)
save_secrets(secrets)
ag = Agave(api_server=credentials['api_server'],
username=credentials['username'],
password=credentials['password'],
api_key=credentials['api_key'],
api_secret=credentials['api_secret'],
client_name=credentials['client_name'],
tenant_id="sd2e")
me = ag.profiles.get()
print('Client created as {} {}'.format(me['first_name'], me['last_name']))
return ag
def check_or_create_agave_home_dir(username):
try:
# TODO should use OS calls to create directory.
user = get_user_model().objects.get(username=username)
logger.info(
"Checking home directory for user=%s on "
"default storage systemId=%s",
username,
settings.AGAVE_STORAGE_SYSTEM
)
ag = Agave(api_server=settings.AGAVE_TENANT_BASEURL,
token=settings.AGAVE_SUPER_TOKEN)
try:
ag.files.list(
systemId=settings.AGAVE_STORAGE_SYSTEM,
filePath=username)
except HTTPError as e:
if e.status_code == 404:
body = {'action': 'mkdir', 'path': username}
ag.files.manage(systemId=settings.AGAVE_STORAGE_SYSTEM,
filePath='',
body=body)
ds_admin_client = Agave(
api_server=getattr(
settings,
'AGAVE_TENANT_BASEURL'
),
token=getattr(
settings,
'AGAVE_SUPER_TOKEN'
),
)
job_body = {
'inputs': {
'username': username,
'directory': 'shared/{}'.format(username)
},
'name': 'setfacl',
'appId': 'setfacl_corral3-0.1'
}
ds_admin_client.jobs.submit(body=job_body)
# logger.debug('setfacl response: {}'.format(response))
# add dir to index
fm = FileManager(user)
fm.indexer.index(
settings.AGAVE_STORAGE_SYSTEM,
username, username,
levels=1
)
except (HTTPError, AgaveException):
logger.exception('Failed to create home directory.',
extra={'user': username,
'systemId': settings.AGAVE_STORAGE_SYSTEM})
def check_or_create_agave_home_dir(username):
try:
# TODO should use OS calls to create directory.
logger.info(
"Checking home directory for user=%s on "
"default storage systemId=%s", username,
settings.AGAVE_STORAGE_SYSTEM)
ag = Agave(api_server=settings.AGAVE_TENANT_BASEURL,
token=settings.AGAVE_SUPER_TOKEN)
try:
listing_response = ag.files.list(
systemId=settings.AGAVE_STORAGE_SYSTEM, filePath=username)
logger.info('check home dir response: {}'.format(listing_response))
except HTTPError as e:
if e.response.status_code == 404:
logger.info(
"Creating the home directory for user=%s then going to run setfacl",
username)
body = {'action': 'mkdir', 'path': username}
fm_response = ag.files.manage(
systemId=settings.AGAVE_STORAGE_SYSTEM,
filePath='',
body=body)
logger.info('mkdir response: {}'.format(fm_response))
ds_admin_client = Agave(
api_server=getattr(settings, 'AGAVE_TENANT_BASEURL'),
token=getattr(settings, 'AGAVE_SUPER_TOKEN'),
)
job_body = {
'parameters': {
'username': username,
'directory': 'shared/{}'.format(username)
},
'name': 'setfacl',
'appId': 'setfacl_corral3-0.1'
}
jobs_response = ds_admin_client.jobs.submit(body=job_body)
logger.info('setfacl response: {}'.format(jobs_response))
# add dir to index
logger.info("Indexing the home directory for user=%s",
username)
agave_indexer.apply_async(kwargs={
'username': username,
'systemId': settings.AGAVE_STORAGE_SYSTEM,
'filePath': username
},
queue='indexing')
except (AgaveException):
#except (HTTPError, AgaveException):
logger.exception('Failed to create home directory.',
extra={
'user': username,
'systemId': settings.AGAVE_STORAGE_SYSTEM
})
class _TaccApisBearer(TaccApisCommandBase):
"""Base class for Tapis API commands that accept only an access token
"""
def add_common_parser_arguments(self, parser):
parser = super(TaccApisBearer,
self).add_common_parser_arguments(parser)
g = parser.add_argument_group('auth override')
g.add_argument('-H',
'--api-server',
dest='api_server',
metavar='URL',
help="{0} {1}".format(constants.PLATFORM,
constants.API_SERVER))
g.add_argument('-z',
'--token',
dest='access_token',
metavar='TOKEN',
help="{0} {1}".format(constants.PLATFORM,
constants.ACCESS_TOKEN))
g.add_argument('--no-verify',
dest='verify_ssl',
action='store_false',
help="Allow insecure SSL connections")
return parser
def init_clients(self, parsed_args):
"""Override CommandBase to set up client with passed token
"""
# client = Agave.restore()
if parsed_args.api_server is not None and parsed_args.access_token is not None:
self.tapis_client = Agave(api_server=parsed_args.api_server,
token=parsed_args.access_token)
elif parsed_args.access_token is not None:
try:
client = Agave._read_current(agave_kwargs=True)
except Exception:
raise AgaveError(
'Tapis API server was not discoverable. Exiting.')
self.tapis_client = Agave(api_server=client['api_server'],
token=parsed_args.access_token)
else:
try:
client = Agave.restore()
self.tapis_client = client
self.tapis_client.refresh()
# self.requests_client = self._get_direct(self.tapis_client)
except Exception:
raise AgaveError(constants.TAPIS_AUTH_FAIL)
try:
self.requests_client = self._get_direct(self.tapis_client)
except Exception:
raise AgaveError(constants.TAPIS_AUTH_FAIL)
return self
def init_clients(self, parsed_args):
"""Override CommandBase to set up client with passed token
"""
client = Agave.restore()
if parsed_args.access_token is not None:
self.tapis_client = Agave(api_server=client.api_server,
token=parsed_args.access_token)
else:
self.tapis_client = client
self.tapis_client.token.refresh()
self.requests_client = self._get_direct(self.tapis_client)
return self
def test_Agave_init(self, mock_get_tenants, mock_input):
""" Test Agave initialization
"""
# Patch list_tenants function and input.
mock_input.return_value = sample_tenants_list_response
mock_get_tenants.return_value = "sd2e"
# Instantiate Agave object making reference to local mock server.
ag = Agave()
ag.init()
assert ag.tenant_id == "sd2e"
assert ag.api_server == "https://api.sd2e.org"
def upload_monitor(self, actor_exec_event, *args, **kwargs):
"""Monitor a running instance of Abaco uploads manager
Implements polling via Celery retry
Arguments:
actor_exec_event (tuple): (actor_id:str, execution_id:str, event_id:str)
Returns:
tuple: (actor_id:str, execution_id:str, status:str, event_id:str)
"""
actor_id = actor_exec_event[0]
execution_id = actor_exec_event[1]
event_id = actor_exec_event[2]
worker = self.request.hostname.split('@')[0]
# Probably init this on worker setup as well...
ag = Agave(api_server=settings['api']['server'],
username=settings['api']['username'],
password=settings['api']['password'],
api_key=settings['api']['clients'][worker]['api_key'],
api_secret=settings['api']['clients'][worker]['api_secret'])
try:
status = ag.actors.getExecution(actorId=actor_id,
executionId=execution_id).get(
'status', None)
except Exception as exc:
raise AgaveError('Failed to poll task: {0}'.format(exc))
fsevent_update.s((event_id, status)).apply_async()
if status not in ['COMPLETE', 'ERROR']:
raise ActorExecutionInProgress
else:
return (actor_id, execution_id, status, event_id)
def upload_launch(self, key_event_id):
"""Launch instance of Abaco upload manager
Retries failed attempts using exponential backoff
Arguments:
key_event_id (tuple): (key:str, x-amz-request-id:str)
Returns:
tuple: (actor_id:str, execution_id:str, event_id:str)
"""
# See https://stackoverflow.com/questions/23857005/get-the-name-of-celery-worker-from-inside-a-celery-task
# for better ways of doing this
worker = self.request.hostname.split('@')[0]
# Probably init this on worker setup as well...
ag = Agave(api_server=settings['api']['server'],
username=settings['api']['username'],
password=settings['api']['password'],
api_key=settings['api']['clients'][worker]['api_key'],
api_secret=settings['api']['clients'][worker]['api_secret'])
msg = {
'uri': 's3://{0}'.format(key_event_id[0]),
'event_id': key_event_id[1]
}
try:
resp = ag.actors.sendMessage(actorId=settings['actor_id'],
body={'message': msg})
except Exception as exc:
raise AgaveError('Failed to launch task: {0}'.format(exc))
return (settings['actor_id'], resp['executionId'], key_event_id[1])
def init_clients(self, parsed_args=dict()):
# To pick up client properties set using parameters set up from
# inherited parsers, this must be called at the end of the
# inheritance chain, immediately before getting to work
# making API calls
# 1. Read from local JSON cache
# 2. Process overrides from parser args
# 3. Init the client(s)
# TODO - use Agavepy public API for getting this
# current = json.load(open(os.path.expanduser('~/.agave/current'), 'r'))
# config = dict()
# for e, k, p in self.PROPS:
# parsed_arg = getattr(parsed_args, p, None)
# current_arg = current.get(k, None)
# if parsed_arg is not None:
# config[e] = parsed_arg
# elif current_arg is not None:
# config[e] = current_arg
# ag = Agave(**config)
# self.tapis_client = ag
# self.tapis_client.token.refresh()
# self.tapis_client = Agave.restore()
# for requests made directly via requests module
# TODO - only populate this if there is an access_token
# TODO - Come back and re-enable overrides via CLI option
self.tapis_client = Agave.restore()
self.tapis_client.token.refresh()
self.requests_client = TaccApiDirectClient(self.tapis_client)
return self
def take_action(self, parsed_args):
# Load what we can from credentials cache.
try:
ag_context = Agave._read_current(agave_kwargs=True)
except FileNotFoundError:
# A very common case (all new tapis_cli installs, for instance), will
# find NO cache file on disk. This must be recoverable, so set the
# context to an empty dict
raise FileNotFoundError(
'Auth configuration was not loaded. Try running "tapis auth init".'
)
# Formulate a table view of key values for current session
headers = [
'tenant_id', 'username', 'api_key', 'access_token', 'expires_at',
'refresh_token'
]
data = [
# Coerce to string to avoid failures where a deepcopy
# operation in Python's implementation of tuple() is
# unable to accomodate copying properties of an Agave object
ag_context['tenant_id'],
ag_context['username'],
ag_context['api_key'],
ag_context['token'],
ag_context['expires_at'],
ag_context['refresh_token']
]
return (tuple(headers), tuple(data))
def get_tapis_jobs_for_experiment(db_uri, ex_id, agave=None):
print(f"checking ex_id: {ex_id}")
client = pymongo.MongoClient(db_uri)
if agave is None:
agave = Agave.restore()
# The view contains only RUNNING pipeline jobs
etjv = client.catalog_staging.experiment_tapis_jobs_view
experiments = etjv.find({"_id.experiment_id": ex_id})
jobs_map = {}
for e in experiments:
jobs = e["jobs"]
for j in jobs:
#print(f"j: {j}")
for tj in j["agave_jobs"]:
#print(f"tj: {tj}")
try:
tjob = agave.jobs.get(jobId=tj)
if tjob.status not in ["FINISHED", "FAILED", "STOPPED"]:
if j["pipeline_name"] not in jobs_map:
jobs_map[j["pipeline_name"]] = []
analysis = j["analysis"] if "analysis" in j else None
jobs_map[j["pipeline_name"]].append([analysis, tj])
except Exception as exc:
print(f"j: {j} exc: {exc}")
return jobs_map
def test_username_from_client():
'''Can we pick up username from Agave.client'''
from agavepy.agave import Agave
ag = Agave.restore()
local_uname = ag.profiles.get()['username']
r = Reactor()
assert local_uname == r.username
def list_directory(self, path, user):
token_query = user.profile.tokens.filter(app__provider='agave')
if not token_query.exists():
raise PermissionDenied('No agave token found.')
token = token_query.get().token
systemProvider = path.split('/')[1]
systemId = path.split('/')[2]
directoryPath = '/'.join([''] + path.split('/')[3:])
ag = Agave(api_server=settings.API_BASE_URL_AGAVE, token=token)
response = ag.files.list(filePath=directoryPath,
systemId=systemId,
limit=1000,
offset=0)
# Filter out the current directory
listing = list(filter(lambda i: i.name != '.', response))
return {
'directories': [
'/'.join(path.split('/')[0:3]) + d.path + '/'
for d in filter(lambda x: x.type == 'dir', listing)
],
'files': [
'/'.join(path.split('/')[0:3]) + f.path
for f in filter(lambda x: x.type == 'file', listing)
]
}
def download(self, localPath, remotePath, user, uniqueBatchId):
token_query = user.profile.tokens.filter(app__provider='agave')
if not token_query.exists():
raise PermissionDenied('No agave token found.')
token = token_query.get().token
systemProvider = remotePath.split('/')[1]
systemId = remotePath.split('/')[2]
remoteFilePath = '/'.join([''] + remotePath.split('/')[3:])
downloadDirectory = '/transient/%s/' % uniqueBatchId
fullLocalPath = downloadDirectory + localPath
if not os.path.exists(downloadDirectory):
os.makedirs(downloadDirectory)
fullLocalDir = '/'.join(fullLocalPath.split('/')[0:-1] + [''])
if not os.path.exists(fullLocalDir):
os.makedirs(fullLocalDir, mode=0o755, exist_ok=True)
ag = Agave(api_server=settings.API_BASE_URL_AGAVE, token=token)
response = ag.files.download(filePath=remoteFilePath,
systemId=systemId)
f = open(fullLocalPath, 'wb')
f.write(response.content)
f.close()
def main(args):
logger.debug('Project config: ' + PARENT + '/config.yml')
project_settings = config.read_config(places_list=[PARENT])
logger.debug('Local config:' + THIS + '/config.yml')
bootstrap_settings = config.read_config(places_list=[THIS])
settings = datacatalog.dicthelpers.data_merge(project_settings,
bootstrap_settings)
env = args.environment
if env is None:
env = 'production'
if args.verbose is True:
settings['verbose'] = True
mongodb = settings.get(env).get('mongodb')
agave_client = Agave.restore()
mgr = datacatalog.managers.common.Manager(mongodb, agave_client)
resp = mgr.stores[args.collection].query(query={}, attr_dict=True)
json.dump(resp,
args.output,
sort_keys=True,
indent=2,
separators=(',', ':'),
default=json_datetime_serializer)
def impersonate_service_account(username):
"""Return agave client as username.
:param str username: Username to impersonate.
"""
url = '/'.join([settings.AGAVE_TENANT_BASEURL, 'token'])
cred = (settings.AGAVE_CLIENT_KEY, settings.AGAVE_CLIENT_SECRET)
if getattr(settings, 'AGAVE_USE_SANDBOX', False):
cred = (settings.AGAVE_SANDBOX_CLIENT_KEY,
settings.AGAVE_SANDBOX_CLIENT_SECRET)
body = {
'grant_type': 'admin_password',
'username': settings.DS_ADMIN_USERNAME,
'password': settings.DS_ADMIN_PASSWORD,
'token_username': '******'.join([settings.AGAVE_USER_STORE_ID, username]),
'scope': 'PRODUCTION',
}
response = requests.post(url, data=body, auth=cred)
response.raise_for_status()
token_data = response.json()
return Agave(api_server=settings.AGAVE_TENANT_BASEURL,
api_key=cred[0],
api_secret=cred[1],
token=token_data['access_token'],
resources=AGAVE_RESOURCES,
refresh_token=token_data['access_token'])
def _agave_connect(self):
"""
Connect to Agave.
Args:
self: class instance.
Returns:
On success: True.
On failure: False.
"""
agave_connection_type = self._config['agave'].get(
'connection_type', 'impersonate')
if agave_connection_type == 'impersonate':
if not self._agave_username:
Log.an().error('agave username required if impersonating')
return False
self._agave = Agave(api_server=self._config['agave']['server'],
username=self._config['agave']['username'],
password=self._config['agave']['password'],
token_username=self._agave_username,
client_name=self._config['agave']['client'],
api_key=self._config['agave']['key'],
api_secret=self._config['agave']['secret'],
verify=False)
elif agave_connection_type == 'agave-cli':
# get credentials from ~/.agave/current
agave_clients = Agave._read_clients()
agave_clients[0]['verify'] = False # don't verify ssl
self._agave = Agave(**agave_clients[0])
# when using agave-cli, token_username must be the same as the
# stored creds in user's home directory, this can be different
# from job username
self._agave_username = agave_clients[0]['username']
else:
Log.an().error('invalid agave connection type: %s',
agave_connection_type)
return False
return True
def main():
uri = get_uri()
access_token = os.environ.get('access_token')
refresh_token = os.environ.get('refresh_token')
if not access_token:
raise Error("access_token required.")
if not refresh_token:
raise Error("refresh_token required.")
print("using access_token: {}".format(access_token))
print("using refresh_token: {}".format(refresh_token))
api_server = os.environ.get('api_server')
if not api_server:
raise Error("api_server required.")
print("using api_server: {}".format(api_server))
verify = os.environ.get('verify')
if verify == 'False':
verify = False
else:
verify = True
print("using verify: {}".format(verify))
api_key = os.environ.get('api_key')
if not api_key:
raise Error("api_key required.")
print("using api_key: {}".format(api_key))
api_secret = os.environ.get('api_secret')
if not api_secret:
raise Error("api_secret required.")
print("using api_secret: {}".format(api_secret))
ag = Agave(api_server=api_server,
api_key=api_key,
api_secret=api_secret,
token=access_token,
refresh_token=refresh_token,
verify=verify)
try:
rsp = ag.download_uri(uri, '/agave/outputs/1')
except Exception as e:
print("Error trying to download file: {}. Exception: {}".format(uri, e))
raise e
print("Download complete.")
def get(self, request, *args, **kwargs):
target = request.GET.get('target')
logger.info(request.GET)
admin_client = Agave(api_server=getattr(settings,
'AGAVE_TENANT_BASEURL'),
token=getattr(settings, 'AGAVE_SUPER_TOKEN'))
data = admin_client.monitors.list(target=target)
return JsonResponse({"response": data})
def client(self):
return Agave(api_server=getattr(settings, 'AGAVE_TENANT_BASEURL'),
api_key=getattr(settings, 'AGAVE_CLIENT_KEY'),
api_secret=getattr(settings, 'AGAVE_CLIENT_SECRET'),
token=self.access_token,
resources=AGAVE_RESOURCES,
refresh_token=self.refresh_token,
token_callback=self.update)
def test_files_list(self, capfd):
""" Test files listings
"""
local_uri = "http://localhost:{port}/".format(port=self.mock_server_port)
agave = Agave(api_server=local_uri)
agave.token = "mock-access-token"
# List files.
agave.files_list("tacc-globalfs-username/tests", long_format=False)
# Stdout should contain the putput from the command.
# Stderr will contain logs from the mock http server.
out, err = capfd.readouterr()
assert "copy.txt" in out
assert "sp1-copy.fq" in out
assert "upload.txt" in out
assert "\"GET //files/v2/listings/system/tacc-globalfs-username/tests?pretty=true HTTP/1.1\" 200" in err
def test_files_download(self):
""" Test file copying from a remote to the local system
"""
# Create a "remote" dummy file.
tmp_file = self.create_remote_dummy_file()
# Destination of downloaded file.
local_file = "testfile.txt"
try:
local_uri = "http://localhost:{port}/".format(
port=self.mock_server_port)
agave = Agave(api_server=local_uri)
agave.token = "mock-access-token"
agave.created_at = str(int(time.time()))
agave.expires_in = str(14400)
agave.files_download("tacc-globalfs-user/" + tmp_file, local_file)
finally:
assert filecmp.cmp(tmp_file, local_file)
# rm local file.
if os.path.exists(local_file):
os.remove(local_file)
# rm dummy file in current working directory.
if os.path.exists(tmp_file):
os.remove(tmp_file)
def get_user_configs(username):
ag = Agave(api_server=base_url, token=service_token)
q = {
'value.user': username,
'value.tenant': TENANT,
'value.instance': INSTANCE
}
print('user query: {}'.format(q))
return ag.meta.listMetadata(q=str(q))
def __init__(self, api_server, system_id, token, root):
self.root = root
self.system_id = system_id
self.ag = Agave(api_server=api_server, token=token)
self.json_resp = self.ag.files.list(filePath=self.root,
systemId=self.system_id,
limit=250,
offset=0)
self.current_dir = root
self.current_files = {}
def test_clients_subscribe(self, mock_input, mock_pass, capfd):
""" Test clients subscribe
"""
# Patch username and password.
mock_input.return_value = "user"
mock_pass.return_value = "pass"
# Instantiate Agave object making reference to local mock server.
local_uri = "http://localhost:{port}".format(
port=self.mock_server_port)
ag = Agave(api_server=local_uri)
# Subscribe a client.
ag.clients_subscribe("PublicKeys", "v2", "admin", client_name="client")
# Stdout should contain the putput from the command.
# Stderr will contain logs from the mock http server.
out, err = capfd.readouterr()
assert "200" in err
def get_service_client(tenant):
"""Returns the service client for a specific tenant."""
service_token = os.environ.get('_abaco_{}_service_token'.format(tenant))
if not service_token:
raise ClientException(
"No service token configured for tenant: {}".format(tenant))
api_server = get_api_server(tenant)
verify = get_tenant_verify(tenant)
# generate an Agave client with the service token
logger.info("Attempting to generate an agave client.")
return Agave(api_server=api_server, token=service_token, verify=verify)
def test_get_access_token(self, mock_pass):
""" Test access token creation
Patch username and password from user to send a client create request
to mock server.
"""
# Patch password prompt.
mock_pass.return_value = "pass"
# Instantiate Agave object making reference to local mock server.
local_uri = "http://localhost:{port}/".format(port=self.mock_server_port)
ag = Agave(api_server=local_uri, username="******")
ag.api_key = "somekey"
ag.api_secret = "somesecret"
# Get access token.
ag.get_access_token()
assert ag.token == "access token"
assert ag.refresh_token == "refresh token"
def get_service_client():
"""Returns an agave client representing the service account. This client can be used to access
the authorized endpoints such as the abaco endpoint."""
if not settings.CALL_ACTOR:
logger.debug(
"Skipping call to actor since settings.CALL_ACTOR was False.")
service_token = os.environ.get('AGAVE_SERVICE_TOKEN')
if not service_token:
raise Exception("Missing SERVICE_TOKEN configuration.")
base_url = os.environ.get('AGAVE_BASE_URL', "https://api.tacc.utexas.edu")
return Agave(api_server=base_url, token=service_token)
def get_agave_client(username, password):
if not client_key or not client_secret:
raise Exception("Missing OAuth client credentials.")
return Agave(api_server=tenant_base_url,
username=username,
password=password,
client_name="tacc-stats",
api_key=client_key,
api_secret=client_secret,
token_callback=update_session_tokens)
def test_load_configs(self, mock_get_tenants):
""" Test load_configs function
"""
try:
# create a tmp dir and use it as a cache dir.
cache_dir = tempfile.mkdtemp()
# Save sample configurations to cache dir.
with open("{}/config.json".format(cache_dir), "w") as f:
json.dump(sample_config, f, indent=4)
ag = Agave()
ag.load_configs(cache_dir=cache_dir)
sample_client = list(sample_config["current"].keys())[0]
assert ag.client_name == sample_client
assert ag.tenant_id == sample_config["current"][sample_client]["tenantid"]
assert ag.username == sample_config["current"][sample_client]["username"]
finally:
shutil.rmtree(cache_dir)
def test_client_create(self, mock_input, mock_pass):
""" Test client create op
Patch username and password from user to send a client create request
to mock server.
"""
# Patch username and password.
mock_input.return_value = "user"
mock_pass.return_value = "pass"
# Instantiate Agave object making reference to local mock server.
local_uri = "http://localhost:{port}/".format(
port=self.mock_server_port)
ag = Agave(api_server=local_uri)
# Create client.
ag.clients_create("client-name", "some description")
assert ag.api_key == "some api key"
assert ag.api_secret == "some secret"
def test_files_download(self):
""" Test file copying from a remote to the local system
"""
# Create a "remote" dummy file.
tmp_file = self.create_remote_dummy_file()
# Destination of downloaded file.
local_file = "testfile.txt"
try:
local_uri = "http://localhost:{port}/".format(port=self.mock_server_port)
agave = Agave(api_server=local_uri)
agave.token = "mock-access-token"
agave.created_at = str(int(time.time()))
agave.expires_in = str(14400)
agave.files_download("tacc-globalfs-user/"+tmp_file, local_file)
finally:
assert filecmp.cmp(tmp_file, local_file)
# rm local file.
if os.path.exists(local_file):
os.remove(local_file)
# rm dummy file in current working directory.
if os.path.exists(tmp_file):
os.remove(tmp_file)
def test_load_configs_specify_session(self, mock_get_tenants):
""" Test load_configs function
Load a specific session from a configurations file.
"""
try:
# create a tmp dir and use it as a cache dir.
cache_dir = tempfile.mkdtemp()
# Save sample configurations to cache dir.
with open("{}/config.json".format(cache_dir), "w") as f:
json.dump(sample_config, f, indent=4)
ag = Agave()
ag.load_configs(cache_dir=cache_dir, tenant_id="tacc.prod",
username="******", client_name="client-2")
assert ag.client_name == "client-2"
assert ag.username == "user-2"
assert ag.tenant_id == "tacc.prod"
finally:
shutil.rmtree(cache_dir)
def test_refresh_tokens(self):
""" Test refresh token operation
"""
local_uri = "http://localhost:{port}/".format(port=self.mock_server_port)
ag = Agave(api_server=local_uri)
ag.api_key = "xxx"
ag.api_secret = "xxx"
ag.refresh_token = "xxx"
# See agavepy/agave.py:refresh_tokens() for more info.
ag.created_at = str( int(time.time()) - 100 )
ag.expires_in = str(0)
# Refresh access token
ag.refresh_tokens()
assert ag.token == "access token"
assert ag.refresh_token == "refresh token"
def test_files_import(self, capfd):
""" Test files import
"""
local_uri = "http://localhost:{port}/".format(port=self.mock_server_port)
agave = Agave(api_server=local_uri)
agave.token = "mock-access-token"
agave.created_at = str(int(time.time()))
agave.expires_in = str(14400)
# Import file.
agave.files_import("agave://data-sd2e-community/test.txt", "tacc-globalfs-username/")
out, err = capfd.readouterr()
assert "200" in err
def test_files_mkdir(self):
""" test files mkdir method
The call to files_mkdir has no side effects on the host so the function
call should simply be able to return successfully.
"""
local_uri = "http://localhost:{port}/".format(port=self.mock_server_port)
agave = Agave(api_server=local_uri)
agave.token = "mock-access-token"
agave.created_at = str(int(time.time()))
agave.expires_in = str(14400)
agave.files_mkdir("tacc-globalfs/new/path")
def test_files_pems_delete(self, capfd):
""" Test permissions deletion
"""
local_uri = "http://localhost:{port}/".format(port=self.mock_server_port)
agave = Agave(api_server=local_uri)
agave.token = "mock-access-token"
agave.created_at = str(int(time.time()))
agave.expires_in = str(14400)
# Delete permissions.
agave.files_pems_delete("tacc-globalfs-username/")
out, err = capfd.readouterr()
assert "200" in err
def test_files_copy(self):
""" test files copying from remote to remote
The call to files_copy has no side effects on the host so the function
call should simply be able to return successfully.
"""
local_uri = "http://localhost:{port}/".format(port=self.mock_server_port)
agave = Agave(api_server=local_uri)
agave.token = "mock-access-token"
agave.created_at = str(int(time.time()))
agave.expires_in = str(14400)
agave.files_copy("tacc-globalfs/file", "tacc-globalfs/file-copy")
def test_files_pems_list(self, capfd):
""" Test permissions listings
"""
local_uri = "http://localhost:{port}/".format(port=self.mock_server_port)
agave = Agave(api_server=local_uri)
agave.token = "mock-access-token"
agave.created_at = str(int(time.time()))
agave.expires_in = str(14400)
# List permissions.
agave.files_pems_list("tacc-globalfs-username/")
# Stdout should contain the putput from the command.
# Stderr will contain logs from the mock http server.
out, err = capfd.readouterr()
assert "username" in out
assert "yes" in out
assert "200" in err
def test_files_upload(self):
""" Test file uload to remote systems
"""
# Create a local dummy file.
tmp_file = self.create_local_dummy_file()
try:
local_uri = "http://localhost:{port}/".format(port=self.mock_server_port)
agave = Agave(api_server=local_uri)
agave.token = "mock-access-token"
agave.created_at = str(int(time.time()))
agave.expires_in = str(14400)
agave.files_upload(tmp_file, "tacc-globalfs-user/")
finally:
_, tmp_filename = os.path.split(tmp_file)
assert filecmp.cmp(tmp_file, tmp_filename)
# rm dummy file.
os.remove(tmp_file)
# rm dummy file in current working directory.
if os.path.exists(tmp_filename):
os.remove(tmp_filename)
def test_save_configs(self, mock_get_tenants):
""" Test Agave initialization
"""
# Patch list_tenants function and input.
mock_get_tenants.return_value = sample_tenants_list_response
# Instantiate Agave objects and save configurations to cache dir.
try:
# create a tmp dir and use it as a cache dir.
cache_dir = tempfile.mkdtemp()
a = Agave(tenant_id="sd2e")
a.init()
a.client_name = "client-1"
a.username = "******"
a.save_configs(cache_dir)
b = Agave(tenant_id="tacc.prod")
b.init()
b.client_name = "client-2"
b.username = "******"
b.save_configs(cache_dir)
c = Agave(tenant_id="sd2e")
c.init()
c.client_name = "client-3"
c.username = "******"
c.save_configs(cache_dir)
with open("{}/config.json".format(cache_dir), "r") as f:
config = json.load(f)
print(config)
print()
print(sample_config)
assert config == sample_config
finally:
shutil.rmtree(cache_dir)
