def __init__(self, api_server=TACC_API_SERVER, nonce=None, access_token=None, refresh_token=None, sync=REACTOR_MESSAGE_SYNC, verbose=Verbosity.DEFAULT): self.log.debug('Initialize Abaco API client') try: if nonce is not None: self.log.debug('Using Abaco authorization nonce') ag = bacanora.agaveutils.agave.AgaveNonceOnly( api_server=api_server, nonce=nonce) elif access_token is not None: self.log.debug('Using supplied Tapis credentials') ag = Agave(api_server=api_server, token=access_token) else: # Use local credential cache self.log.debug('Using local Tapis credential cache') ag = Agave.restore() except Exception: self.log.exception('Unable to initialize Tapis client') raise self.log.debug('Done initializing Abaco API client') self.client = ag self.sync = sync self.verbosity = verbose
def connect(self): agave_connection_type = self._config.get('connection_type', 'impersonate') if agave_connection_type == 'impersonate': token_username = '******'.format( self._config['domain'], '/' if self._config['domain'] else '', self._config['token_username']) Log.some().debug('user impersonation: %s', token_username) self._agave = Agave(api_server=self._config['server'], username=self._config['username'], password=self._config['password'], token_username=token_username, client_name=self._config['client'], api_key=self._config['key'], api_secret=self._config['secret'], verify=False) elif agave_connection_type == 'agave-cli': # get credentials from ~/.agave/current agave_clients = Agave._read_clients() agave_clients[0]['verify'] = False # don't verify ssl self._agave = Agave(**agave_clients[0]) # when using agave-cli, token_username must be the same as the # stored creds in user's home directory, this can be different # from job username self._config['token_username'] \ = agave_clients[0]['username'] else: Log.an().error('invalid agave connection type: %s', agave_connection_type) return False return True
def create_client(): """ Create a session using credentials in secrets.json. :return: new Agave client """ secrets = get_secrets() credentials = secrets['agave'] if not (credentials.get('api_key') and credentials.get('api_secret')): print( "Generating new api_key and api_secret and saving to secrets.json") ag = Agave(api_server=credentials['api_server'], username=credentials['username'], password=credentials['password']) new_secrets = ag.clients_create() credentials.update(new_secrets) save_secrets(secrets) ag = Agave(api_server=credentials['api_server'], username=credentials['username'], password=credentials['password'], api_key=credentials['api_key'], api_secret=credentials['api_secret'], client_name=credentials['client_name'], tenant_id="sd2e") me = ag.profiles.get() print('Client created as {} {}'.format(me['first_name'], me['last_name'])) return ag
def check_or_create_agave_home_dir(username): try: # TODO should use OS calls to create directory. user = get_user_model().objects.get(username=username) logger.info( "Checking home directory for user=%s on " "default storage systemId=%s", username, settings.AGAVE_STORAGE_SYSTEM ) ag = Agave(api_server=settings.AGAVE_TENANT_BASEURL, token=settings.AGAVE_SUPER_TOKEN) try: ag.files.list( systemId=settings.AGAVE_STORAGE_SYSTEM, filePath=username) except HTTPError as e: if e.status_code == 404: body = {'action': 'mkdir', 'path': username} ag.files.manage(systemId=settings.AGAVE_STORAGE_SYSTEM, filePath='', body=body) ds_admin_client = Agave( api_server=getattr( settings, 'AGAVE_TENANT_BASEURL' ), token=getattr( settings, 'AGAVE_SUPER_TOKEN' ), ) job_body = { 'inputs': { 'username': username, 'directory': 'shared/{}'.format(username) }, 'name': 'setfacl', 'appId': 'setfacl_corral3-0.1' } ds_admin_client.jobs.submit(body=job_body) # logger.debug('setfacl response: {}'.format(response)) # add dir to index fm = FileManager(user) fm.indexer.index( settings.AGAVE_STORAGE_SYSTEM, username, username, levels=1 ) except (HTTPError, AgaveException): logger.exception('Failed to create home directory.', extra={'user': username, 'systemId': settings.AGAVE_STORAGE_SYSTEM})
def check_or_create_agave_home_dir(username): try: # TODO should use OS calls to create directory. logger.info( "Checking home directory for user=%s on " "default storage systemId=%s", username, settings.AGAVE_STORAGE_SYSTEM) ag = Agave(api_server=settings.AGAVE_TENANT_BASEURL, token=settings.AGAVE_SUPER_TOKEN) try: listing_response = ag.files.list( systemId=settings.AGAVE_STORAGE_SYSTEM, filePath=username) logger.info('check home dir response: {}'.format(listing_response)) except HTTPError as e: if e.response.status_code == 404: logger.info( "Creating the home directory for user=%s then going to run setfacl", username) body = {'action': 'mkdir', 'path': username} fm_response = ag.files.manage( systemId=settings.AGAVE_STORAGE_SYSTEM, filePath='', body=body) logger.info('mkdir response: {}'.format(fm_response)) ds_admin_client = Agave( api_server=getattr(settings, 'AGAVE_TENANT_BASEURL'), token=getattr(settings, 'AGAVE_SUPER_TOKEN'), ) job_body = { 'parameters': { 'username': username, 'directory': 'shared/{}'.format(username) }, 'name': 'setfacl', 'appId': 'setfacl_corral3-0.1' } jobs_response = ds_admin_client.jobs.submit(body=job_body) logger.info('setfacl response: {}'.format(jobs_response)) # add dir to index logger.info("Indexing the home directory for user=%s", username) agave_indexer.apply_async(kwargs={ 'username': username, 'systemId': settings.AGAVE_STORAGE_SYSTEM, 'filePath': username }, queue='indexing') except (AgaveException): #except (HTTPError, AgaveException): logger.exception('Failed to create home directory.', extra={ 'user': username, 'systemId': settings.AGAVE_STORAGE_SYSTEM })
class _TaccApisBearer(TaccApisCommandBase): """Base class for Tapis API commands that accept only an access token """ def add_common_parser_arguments(self, parser): parser = super(TaccApisBearer, self).add_common_parser_arguments(parser) g = parser.add_argument_group('auth override') g.add_argument('-H', '--api-server', dest='api_server', metavar='URL', help="{0} {1}".format(constants.PLATFORM, constants.API_SERVER)) g.add_argument('-z', '--token', dest='access_token', metavar='TOKEN', help="{0} {1}".format(constants.PLATFORM, constants.ACCESS_TOKEN)) g.add_argument('--no-verify', dest='verify_ssl', action='store_false', help="Allow insecure SSL connections") return parser def init_clients(self, parsed_args): """Override CommandBase to set up client with passed token """ # client = Agave.restore() if parsed_args.api_server is not None and parsed_args.access_token is not None: self.tapis_client = Agave(api_server=parsed_args.api_server, token=parsed_args.access_token) elif parsed_args.access_token is not None: try: client = Agave._read_current(agave_kwargs=True) except Exception: raise AgaveError( 'Tapis API server was not discoverable. Exiting.') self.tapis_client = Agave(api_server=client['api_server'], token=parsed_args.access_token) else: try: client = Agave.restore() self.tapis_client = client self.tapis_client.refresh() # self.requests_client = self._get_direct(self.tapis_client) except Exception: raise AgaveError(constants.TAPIS_AUTH_FAIL) try: self.requests_client = self._get_direct(self.tapis_client) except Exception: raise AgaveError(constants.TAPIS_AUTH_FAIL) return self
def init_clients(self, parsed_args): """Override CommandBase to set up client with passed token """ client = Agave.restore() if parsed_args.access_token is not None: self.tapis_client = Agave(api_server=client.api_server, token=parsed_args.access_token) else: self.tapis_client = client self.tapis_client.token.refresh() self.requests_client = self._get_direct(self.tapis_client) return self
def test_Agave_init(self, mock_get_tenants, mock_input): """ Test Agave initialization """ # Patch list_tenants function and input. mock_input.return_value = sample_tenants_list_response mock_get_tenants.return_value = "sd2e" # Instantiate Agave object making reference to local mock server. ag = Agave() ag.init() assert ag.tenant_id == "sd2e" assert ag.api_server == "https://api.sd2e.org"
def upload_monitor(self, actor_exec_event, *args, **kwargs): """Monitor a running instance of Abaco uploads manager Implements polling via Celery retry Arguments: actor_exec_event (tuple): (actor_id:str, execution_id:str, event_id:str) Returns: tuple: (actor_id:str, execution_id:str, status:str, event_id:str) """ actor_id = actor_exec_event[0] execution_id = actor_exec_event[1] event_id = actor_exec_event[2] worker = self.request.hostname.split('@')[0] # Probably init this on worker setup as well... ag = Agave(api_server=settings['api']['server'], username=settings['api']['username'], password=settings['api']['password'], api_key=settings['api']['clients'][worker]['api_key'], api_secret=settings['api']['clients'][worker]['api_secret']) try: status = ag.actors.getExecution(actorId=actor_id, executionId=execution_id).get( 'status', None) except Exception as exc: raise AgaveError('Failed to poll task: {0}'.format(exc)) fsevent_update.s((event_id, status)).apply_async() if status not in ['COMPLETE', 'ERROR']: raise ActorExecutionInProgress else: return (actor_id, execution_id, status, event_id)
def upload_launch(self, key_event_id): """Launch instance of Abaco upload manager Retries failed attempts using exponential backoff Arguments: key_event_id (tuple): (key:str, x-amz-request-id:str) Returns: tuple: (actor_id:str, execution_id:str, event_id:str) """ # See https://stackoverflow.com/questions/23857005/get-the-name-of-celery-worker-from-inside-a-celery-task # for better ways of doing this worker = self.request.hostname.split('@')[0] # Probably init this on worker setup as well... ag = Agave(api_server=settings['api']['server'], username=settings['api']['username'], password=settings['api']['password'], api_key=settings['api']['clients'][worker]['api_key'], api_secret=settings['api']['clients'][worker]['api_secret']) msg = { 'uri': 's3://{0}'.format(key_event_id[0]), 'event_id': key_event_id[1] } try: resp = ag.actors.sendMessage(actorId=settings['actor_id'], body={'message': msg}) except Exception as exc: raise AgaveError('Failed to launch task: {0}'.format(exc)) return (settings['actor_id'], resp['executionId'], key_event_id[1])
def init_clients(self, parsed_args=dict()): # To pick up client properties set using parameters set up from # inherited parsers, this must be called at the end of the # inheritance chain, immediately before getting to work # making API calls # 1. Read from local JSON cache # 2. Process overrides from parser args # 3. Init the client(s) # TODO - use Agavepy public API for getting this # current = json.load(open(os.path.expanduser('~/.agave/current'), 'r')) # config = dict() # for e, k, p in self.PROPS: # parsed_arg = getattr(parsed_args, p, None) # current_arg = current.get(k, None) # if parsed_arg is not None: # config[e] = parsed_arg # elif current_arg is not None: # config[e] = current_arg # ag = Agave(**config) # self.tapis_client = ag # self.tapis_client.token.refresh() # self.tapis_client = Agave.restore() # for requests made directly via requests module # TODO - only populate this if there is an access_token # TODO - Come back and re-enable overrides via CLI option self.tapis_client = Agave.restore() self.tapis_client.token.refresh() self.requests_client = TaccApiDirectClient(self.tapis_client) return self
def take_action(self, parsed_args): # Load what we can from credentials cache. try: ag_context = Agave._read_current(agave_kwargs=True) except FileNotFoundError: # A very common case (all new tapis_cli installs, for instance), will # find NO cache file on disk. This must be recoverable, so set the # context to an empty dict raise FileNotFoundError( 'Auth configuration was not loaded. Try running "tapis auth init".' ) # Formulate a table view of key values for current session headers = [ 'tenant_id', 'username', 'api_key', 'access_token', 'expires_at', 'refresh_token' ] data = [ # Coerce to string to avoid failures where a deepcopy # operation in Python's implementation of tuple() is # unable to accomodate copying properties of an Agave object ag_context['tenant_id'], ag_context['username'], ag_context['api_key'], ag_context['token'], ag_context['expires_at'], ag_context['refresh_token'] ] return (tuple(headers), tuple(data))
def get_tapis_jobs_for_experiment(db_uri, ex_id, agave=None): print(f"checking ex_id: {ex_id}") client = pymongo.MongoClient(db_uri) if agave is None: agave = Agave.restore() # The view contains only RUNNING pipeline jobs etjv = client.catalog_staging.experiment_tapis_jobs_view experiments = etjv.find({"_id.experiment_id": ex_id}) jobs_map = {} for e in experiments: jobs = e["jobs"] for j in jobs: #print(f"j: {j}") for tj in j["agave_jobs"]: #print(f"tj: {tj}") try: tjob = agave.jobs.get(jobId=tj) if tjob.status not in ["FINISHED", "FAILED", "STOPPED"]: if j["pipeline_name"] not in jobs_map: jobs_map[j["pipeline_name"]] = [] analysis = j["analysis"] if "analysis" in j else None jobs_map[j["pipeline_name"]].append([analysis, tj]) except Exception as exc: print(f"j: {j} exc: {exc}") return jobs_map
def test_username_from_client(): '''Can we pick up username from Agave.client''' from agavepy.agave import Agave ag = Agave.restore() local_uname = ag.profiles.get()['username'] r = Reactor() assert local_uname == r.username
def list_directory(self, path, user): token_query = user.profile.tokens.filter(app__provider='agave') if not token_query.exists(): raise PermissionDenied('No agave token found.') token = token_query.get().token systemProvider = path.split('/')[1] systemId = path.split('/')[2] directoryPath = '/'.join([''] + path.split('/')[3:]) ag = Agave(api_server=settings.API_BASE_URL_AGAVE, token=token) response = ag.files.list(filePath=directoryPath, systemId=systemId, limit=1000, offset=0) # Filter out the current directory listing = list(filter(lambda i: i.name != '.', response)) return { 'directories': [ '/'.join(path.split('/')[0:3]) + d.path + '/' for d in filter(lambda x: x.type == 'dir', listing) ], 'files': [ '/'.join(path.split('/')[0:3]) + f.path for f in filter(lambda x: x.type == 'file', listing) ] }
def download(self, localPath, remotePath, user, uniqueBatchId): token_query = user.profile.tokens.filter(app__provider='agave') if not token_query.exists(): raise PermissionDenied('No agave token found.') token = token_query.get().token systemProvider = remotePath.split('/')[1] systemId = remotePath.split('/')[2] remoteFilePath = '/'.join([''] + remotePath.split('/')[3:]) downloadDirectory = '/transient/%s/' % uniqueBatchId fullLocalPath = downloadDirectory + localPath if not os.path.exists(downloadDirectory): os.makedirs(downloadDirectory) fullLocalDir = '/'.join(fullLocalPath.split('/')[0:-1] + ['']) if not os.path.exists(fullLocalDir): os.makedirs(fullLocalDir, mode=0o755, exist_ok=True) ag = Agave(api_server=settings.API_BASE_URL_AGAVE, token=token) response = ag.files.download(filePath=remoteFilePath, systemId=systemId) f = open(fullLocalPath, 'wb') f.write(response.content) f.close()
def main(args): logger.debug('Project config: ' + PARENT + '/config.yml') project_settings = config.read_config(places_list=[PARENT]) logger.debug('Local config:' + THIS + '/config.yml') bootstrap_settings = config.read_config(places_list=[THIS]) settings = datacatalog.dicthelpers.data_merge(project_settings, bootstrap_settings) env = args.environment if env is None: env = 'production' if args.verbose is True: settings['verbose'] = True mongodb = settings.get(env).get('mongodb') agave_client = Agave.restore() mgr = datacatalog.managers.common.Manager(mongodb, agave_client) resp = mgr.stores[args.collection].query(query={}, attr_dict=True) json.dump(resp, args.output, sort_keys=True, indent=2, separators=(',', ':'), default=json_datetime_serializer)
def impersonate_service_account(username): """Return agave client as username. :param str username: Username to impersonate. """ url = '/'.join([settings.AGAVE_TENANT_BASEURL, 'token']) cred = (settings.AGAVE_CLIENT_KEY, settings.AGAVE_CLIENT_SECRET) if getattr(settings, 'AGAVE_USE_SANDBOX', False): cred = (settings.AGAVE_SANDBOX_CLIENT_KEY, settings.AGAVE_SANDBOX_CLIENT_SECRET) body = { 'grant_type': 'admin_password', 'username': settings.DS_ADMIN_USERNAME, 'password': settings.DS_ADMIN_PASSWORD, 'token_username': '******'.join([settings.AGAVE_USER_STORE_ID, username]), 'scope': 'PRODUCTION', } response = requests.post(url, data=body, auth=cred) response.raise_for_status() token_data = response.json() return Agave(api_server=settings.AGAVE_TENANT_BASEURL, api_key=cred[0], api_secret=cred[1], token=token_data['access_token'], resources=AGAVE_RESOURCES, refresh_token=token_data['access_token'])
def _agave_connect(self): """ Connect to Agave. Args: self: class instance. Returns: On success: True. On failure: False. """ agave_connection_type = self._config['agave'].get( 'connection_type', 'impersonate') if agave_connection_type == 'impersonate': if not self._agave_username: Log.an().error('agave username required if impersonating') return False self._agave = Agave(api_server=self._config['agave']['server'], username=self._config['agave']['username'], password=self._config['agave']['password'], token_username=self._agave_username, client_name=self._config['agave']['client'], api_key=self._config['agave']['key'], api_secret=self._config['agave']['secret'], verify=False) elif agave_connection_type == 'agave-cli': # get credentials from ~/.agave/current agave_clients = Agave._read_clients() agave_clients[0]['verify'] = False # don't verify ssl self._agave = Agave(**agave_clients[0]) # when using agave-cli, token_username must be the same as the # stored creds in user's home directory, this can be different # from job username self._agave_username = agave_clients[0]['username'] else: Log.an().error('invalid agave connection type: %s', agave_connection_type) return False return True
def main(): uri = get_uri() access_token = os.environ.get('access_token') refresh_token = os.environ.get('refresh_token') if not access_token: raise Error("access_token required.") if not refresh_token: raise Error("refresh_token required.") print("using access_token: {}".format(access_token)) print("using refresh_token: {}".format(refresh_token)) api_server = os.environ.get('api_server') if not api_server: raise Error("api_server required.") print("using api_server: {}".format(api_server)) verify = os.environ.get('verify') if verify == 'False': verify = False else: verify = True print("using verify: {}".format(verify)) api_key = os.environ.get('api_key') if not api_key: raise Error("api_key required.") print("using api_key: {}".format(api_key)) api_secret = os.environ.get('api_secret') if not api_secret: raise Error("api_secret required.") print("using api_secret: {}".format(api_secret)) ag = Agave(api_server=api_server, api_key=api_key, api_secret=api_secret, token=access_token, refresh_token=refresh_token, verify=verify) try: rsp = ag.download_uri(uri, '/agave/outputs/1') except Exception as e: print("Error trying to download file: {}. Exception: {}".format(uri, e)) raise e print("Download complete.")
def get(self, request, *args, **kwargs): target = request.GET.get('target') logger.info(request.GET) admin_client = Agave(api_server=getattr(settings, 'AGAVE_TENANT_BASEURL'), token=getattr(settings, 'AGAVE_SUPER_TOKEN')) data = admin_client.monitors.list(target=target) return JsonResponse({"response": data})
def client(self): return Agave(api_server=getattr(settings, 'AGAVE_TENANT_BASEURL'), api_key=getattr(settings, 'AGAVE_CLIENT_KEY'), api_secret=getattr(settings, 'AGAVE_CLIENT_SECRET'), token=self.access_token, resources=AGAVE_RESOURCES, refresh_token=self.refresh_token, token_callback=self.update)
def test_files_list(self, capfd): """ Test files listings """ local_uri = "http://localhost:{port}/".format(port=self.mock_server_port) agave = Agave(api_server=local_uri) agave.token = "mock-access-token" # List files. agave.files_list("tacc-globalfs-username/tests", long_format=False) # Stdout should contain the putput from the command. # Stderr will contain logs from the mock http server. out, err = capfd.readouterr() assert "copy.txt" in out assert "sp1-copy.fq" in out assert "upload.txt" in out assert "\"GET //files/v2/listings/system/tacc-globalfs-username/tests?pretty=true HTTP/1.1\" 200" in err
def test_files_download(self): """ Test file copying from a remote to the local system """ # Create a "remote" dummy file. tmp_file = self.create_remote_dummy_file() # Destination of downloaded file. local_file = "testfile.txt" try: local_uri = "http://localhost:{port}/".format( port=self.mock_server_port) agave = Agave(api_server=local_uri) agave.token = "mock-access-token" agave.created_at = str(int(time.time())) agave.expires_in = str(14400) agave.files_download("tacc-globalfs-user/" + tmp_file, local_file) finally: assert filecmp.cmp(tmp_file, local_file) # rm local file. if os.path.exists(local_file): os.remove(local_file) # rm dummy file in current working directory. if os.path.exists(tmp_file): os.remove(tmp_file)
def get_user_configs(username): ag = Agave(api_server=base_url, token=service_token) q = { 'value.user': username, 'value.tenant': TENANT, 'value.instance': INSTANCE } print('user query: {}'.format(q)) return ag.meta.listMetadata(q=str(q))
def __init__(self, api_server, system_id, token, root): self.root = root self.system_id = system_id self.ag = Agave(api_server=api_server, token=token) self.json_resp = self.ag.files.list(filePath=self.root, systemId=self.system_id, limit=250, offset=0) self.current_dir = root self.current_files = {}
def test_clients_subscribe(self, mock_input, mock_pass, capfd): """ Test clients subscribe """ # Patch username and password. mock_input.return_value = "user" mock_pass.return_value = "pass" # Instantiate Agave object making reference to local mock server. local_uri = "http://localhost:{port}".format( port=self.mock_server_port) ag = Agave(api_server=local_uri) # Subscribe a client. ag.clients_subscribe("PublicKeys", "v2", "admin", client_name="client") # Stdout should contain the putput from the command. # Stderr will contain logs from the mock http server. out, err = capfd.readouterr() assert "200" in err
def get_service_client(tenant): """Returns the service client for a specific tenant.""" service_token = os.environ.get('_abaco_{}_service_token'.format(tenant)) if not service_token: raise ClientException( "No service token configured for tenant: {}".format(tenant)) api_server = get_api_server(tenant) verify = get_tenant_verify(tenant) # generate an Agave client with the service token logger.info("Attempting to generate an agave client.") return Agave(api_server=api_server, token=service_token, verify=verify)
def test_get_access_token(self, mock_pass): """ Test access token creation Patch username and password from user to send a client create request to mock server. """ # Patch password prompt. mock_pass.return_value = "pass" # Instantiate Agave object making reference to local mock server. local_uri = "http://localhost:{port}/".format(port=self.mock_server_port) ag = Agave(api_server=local_uri, username="******") ag.api_key = "somekey" ag.api_secret = "somesecret" # Get access token. ag.get_access_token() assert ag.token == "access token" assert ag.refresh_token == "refresh token"
def get_service_client(): """Returns an agave client representing the service account. This client can be used to access the authorized endpoints such as the abaco endpoint.""" if not settings.CALL_ACTOR: logger.debug( "Skipping call to actor since settings.CALL_ACTOR was False.") service_token = os.environ.get('AGAVE_SERVICE_TOKEN') if not service_token: raise Exception("Missing SERVICE_TOKEN configuration.") base_url = os.environ.get('AGAVE_BASE_URL', "https://api.tacc.utexas.edu") return Agave(api_server=base_url, token=service_token)
def get_agave_client(username, password): if not client_key or not client_secret: raise Exception("Missing OAuth client credentials.") return Agave(api_server=tenant_base_url, username=username, password=password, client_name="tacc-stats", api_key=client_key, api_secret=client_secret, token_callback=update_session_tokens)
def test_load_configs(self, mock_get_tenants): """ Test load_configs function """ try: # create a tmp dir and use it as a cache dir. cache_dir = tempfile.mkdtemp() # Save sample configurations to cache dir. with open("{}/config.json".format(cache_dir), "w") as f: json.dump(sample_config, f, indent=4) ag = Agave() ag.load_configs(cache_dir=cache_dir) sample_client = list(sample_config["current"].keys())[0] assert ag.client_name == sample_client assert ag.tenant_id == sample_config["current"][sample_client]["tenantid"] assert ag.username == sample_config["current"][sample_client]["username"] finally: shutil.rmtree(cache_dir)
def test_client_create(self, mock_input, mock_pass): """ Test client create op Patch username and password from user to send a client create request to mock server. """ # Patch username and password. mock_input.return_value = "user" mock_pass.return_value = "pass" # Instantiate Agave object making reference to local mock server. local_uri = "http://localhost:{port}/".format( port=self.mock_server_port) ag = Agave(api_server=local_uri) # Create client. ag.clients_create("client-name", "some description") assert ag.api_key == "some api key" assert ag.api_secret == "some secret"
def test_files_download(self): """ Test file copying from a remote to the local system """ # Create a "remote" dummy file. tmp_file = self.create_remote_dummy_file() # Destination of downloaded file. local_file = "testfile.txt" try: local_uri = "http://localhost:{port}/".format(port=self.mock_server_port) agave = Agave(api_server=local_uri) agave.token = "mock-access-token" agave.created_at = str(int(time.time())) agave.expires_in = str(14400) agave.files_download("tacc-globalfs-user/"+tmp_file, local_file) finally: assert filecmp.cmp(tmp_file, local_file) # rm local file. if os.path.exists(local_file): os.remove(local_file) # rm dummy file in current working directory. if os.path.exists(tmp_file): os.remove(tmp_file)
def test_load_configs_specify_session(self, mock_get_tenants): """ Test load_configs function Load a specific session from a configurations file. """ try: # create a tmp dir and use it as a cache dir. cache_dir = tempfile.mkdtemp() # Save sample configurations to cache dir. with open("{}/config.json".format(cache_dir), "w") as f: json.dump(sample_config, f, indent=4) ag = Agave() ag.load_configs(cache_dir=cache_dir, tenant_id="tacc.prod", username="******", client_name="client-2") assert ag.client_name == "client-2" assert ag.username == "user-2" assert ag.tenant_id == "tacc.prod" finally: shutil.rmtree(cache_dir)
def test_refresh_tokens(self): """ Test refresh token operation """ local_uri = "http://localhost:{port}/".format(port=self.mock_server_port) ag = Agave(api_server=local_uri) ag.api_key = "xxx" ag.api_secret = "xxx" ag.refresh_token = "xxx" # See agavepy/agave.py:refresh_tokens() for more info. ag.created_at = str( int(time.time()) - 100 ) ag.expires_in = str(0) # Refresh access token ag.refresh_tokens() assert ag.token == "access token" assert ag.refresh_token == "refresh token"
def test_files_import(self, capfd): """ Test files import """ local_uri = "http://localhost:{port}/".format(port=self.mock_server_port) agave = Agave(api_server=local_uri) agave.token = "mock-access-token" agave.created_at = str(int(time.time())) agave.expires_in = str(14400) # Import file. agave.files_import("agave://data-sd2e-community/test.txt", "tacc-globalfs-username/") out, err = capfd.readouterr() assert "200" in err
def test_files_mkdir(self): """ test files mkdir method The call to files_mkdir has no side effects on the host so the function call should simply be able to return successfully. """ local_uri = "http://localhost:{port}/".format(port=self.mock_server_port) agave = Agave(api_server=local_uri) agave.token = "mock-access-token" agave.created_at = str(int(time.time())) agave.expires_in = str(14400) agave.files_mkdir("tacc-globalfs/new/path")
def test_files_pems_delete(self, capfd): """ Test permissions deletion """ local_uri = "http://localhost:{port}/".format(port=self.mock_server_port) agave = Agave(api_server=local_uri) agave.token = "mock-access-token" agave.created_at = str(int(time.time())) agave.expires_in = str(14400) # Delete permissions. agave.files_pems_delete("tacc-globalfs-username/") out, err = capfd.readouterr() assert "200" in err
def test_files_copy(self): """ test files copying from remote to remote The call to files_copy has no side effects on the host so the function call should simply be able to return successfully. """ local_uri = "http://localhost:{port}/".format(port=self.mock_server_port) agave = Agave(api_server=local_uri) agave.token = "mock-access-token" agave.created_at = str(int(time.time())) agave.expires_in = str(14400) agave.files_copy("tacc-globalfs/file", "tacc-globalfs/file-copy")
def test_files_pems_list(self, capfd): """ Test permissions listings """ local_uri = "http://localhost:{port}/".format(port=self.mock_server_port) agave = Agave(api_server=local_uri) agave.token = "mock-access-token" agave.created_at = str(int(time.time())) agave.expires_in = str(14400) # List permissions. agave.files_pems_list("tacc-globalfs-username/") # Stdout should contain the putput from the command. # Stderr will contain logs from the mock http server. out, err = capfd.readouterr() assert "username" in out assert "yes" in out assert "200" in err
def test_files_upload(self): """ Test file uload to remote systems """ # Create a local dummy file. tmp_file = self.create_local_dummy_file() try: local_uri = "http://localhost:{port}/".format(port=self.mock_server_port) agave = Agave(api_server=local_uri) agave.token = "mock-access-token" agave.created_at = str(int(time.time())) agave.expires_in = str(14400) agave.files_upload(tmp_file, "tacc-globalfs-user/") finally: _, tmp_filename = os.path.split(tmp_file) assert filecmp.cmp(tmp_file, tmp_filename) # rm dummy file. os.remove(tmp_file) # rm dummy file in current working directory. if os.path.exists(tmp_filename): os.remove(tmp_filename)
def test_save_configs(self, mock_get_tenants): """ Test Agave initialization """ # Patch list_tenants function and input. mock_get_tenants.return_value = sample_tenants_list_response # Instantiate Agave objects and save configurations to cache dir. try: # create a tmp dir and use it as a cache dir. cache_dir = tempfile.mkdtemp() a = Agave(tenant_id="sd2e") a.init() a.client_name = "client-1" a.username = "******" a.save_configs(cache_dir) b = Agave(tenant_id="tacc.prod") b.init() b.client_name = "client-2" b.username = "******" b.save_configs(cache_dir) c = Agave(tenant_id="sd2e") c.init() c.client_name = "client-3" c.username = "******" c.save_configs(cache_dir) with open("{}/config.json".format(cache_dir), "r") as f: config = json.load(f) print(config) print() print(sample_config) assert config == sample_config finally: shutil.rmtree(cache_dir)