def __init__(self,
api_server=TACC_API_SERVER,
nonce=None,
access_token=None,
refresh_token=None,
sync=REACTOR_MESSAGE_SYNC,
verbose=Verbosity.DEFAULT):
self.log.debug('Initialize Abaco API client')
try:
if nonce is not None:
self.log.debug('Using Abaco authorization nonce')
ag = bacanora.agaveutils.agave.AgaveNonceOnly(
api_server=api_server, nonce=nonce)
elif access_token is not None:
self.log.debug('Using supplied Tapis credentials')
ag = Agave(api_server=api_server, token=access_token)
else:
# Use local credential cache
self.log.debug('Using local Tapis credential cache')
ag = Agave.restore()
except Exception:
self.log.exception('Unable to initialize Tapis client')
raise
self.log.debug('Done initializing Abaco API client')
self.client = ag
self.sync = sync
self.verbosity = verbose
def init_clients(self, parsed_args=dict()):
# To pick up client properties set using parameters set up from
# inherited parsers, this must be called at the end of the
# inheritance chain, immediately before getting to work
# making API calls
# 1. Read from local JSON cache
# 2. Process overrides from parser args
# 3. Init the client(s)
# TODO - use Agavepy public API for getting this
# current = json.load(open(os.path.expanduser('~/.agave/current'), 'r'))
# config = dict()
# for e, k, p in self.PROPS:
# parsed_arg = getattr(parsed_args, p, None)
# current_arg = current.get(k, None)
# if parsed_arg is not None:
# config[e] = parsed_arg
# elif current_arg is not None:
# config[e] = current_arg
# ag = Agave(**config)
# self.tapis_client = ag
# self.tapis_client.token.refresh()
# self.tapis_client = Agave.restore()
# for requests made directly via requests module
# TODO - only populate this if there is an access_token
# TODO - Come back and re-enable overrides via CLI option
self.tapis_client = Agave.restore()
self.tapis_client.token.refresh()
self.requests_client = TaccApiDirectClient(self.tapis_client)
return self
def test_username_from_client():
'''Can we pick up username from Agave.client'''
from agavepy.agave import Agave
ag = Agave.restore()
local_uname = ag.profiles.get()['username']
r = Reactor()
assert local_uname == r.username
def init_clients(self, parsed_args):
"""Override CommandBase to set up client with passed token
"""
# client = Agave.restore()
if parsed_args.api_server is not None and parsed_args.access_token is not None:
self.tapis_client = Agave(api_server=parsed_args.api_server,
token=parsed_args.access_token)
elif parsed_args.access_token is not None:
try:
client = Agave._read_current(agave_kwargs=True)
except Exception:
raise AgaveError(
'Tapis API server was not discoverable. Exiting.')
self.tapis_client = Agave(api_server=client['api_server'],
token=parsed_args.access_token)
else:
try:
client = Agave.restore()
self.tapis_client = client
self.tapis_client.refresh()
# self.requests_client = self._get_direct(self.tapis_client)
except Exception:
raise AgaveError(constants.TAPIS_AUTH_FAIL)
try:
self.requests_client = self._get_direct(self.tapis_client)
except Exception:
raise AgaveError(constants.TAPIS_AUTH_FAIL)
return self
def get_tapis_jobs_for_experiment(db_uri, ex_id, agave=None):
print(f"checking ex_id: {ex_id}")
client = pymongo.MongoClient(db_uri)
if agave is None:
agave = Agave.restore()
# The view contains only RUNNING pipeline jobs
etjv = client.catalog_staging.experiment_tapis_jobs_view
experiments = etjv.find({"_id.experiment_id": ex_id})
jobs_map = {}
for e in experiments:
jobs = e["jobs"]
for j in jobs:
#print(f"j: {j}")
for tj in j["agave_jobs"]:
#print(f"tj: {tj}")
try:
tjob = agave.jobs.get(jobId=tj)
if tjob.status not in ["FINISHED", "FAILED", "STOPPED"]:
if j["pipeline_name"] not in jobs_map:
jobs_map[j["pipeline_name"]] = []
analysis = j["analysis"] if "analysis" in j else None
jobs_map[j["pipeline_name"]].append([analysis, tj])
except Exception as exc:
print(f"j: {j} exc: {exc}")
return jobs_map
def main(args):
logger.debug('Project config: ' + PARENT + '/config.yml')
project_settings = config.read_config(places_list=[PARENT])
logger.debug('Local config:' + THIS + '/config.yml')
bootstrap_settings = config.read_config(places_list=[THIS])
settings = datacatalog.dicthelpers.data_merge(project_settings,
bootstrap_settings)
env = args.environment
if env is None:
env = 'production'
if args.verbose is True:
settings['verbose'] = True
mongodb = settings.get(env).get('mongodb')
agave_client = Agave.restore()
mgr = datacatalog.managers.common.Manager(mongodb, agave_client)
resp = mgr.stores[args.collection].query(query={}, attr_dict=True)
json.dump(resp,
args.output,
sort_keys=True,
indent=2,
separators=(',', ':'),
default=json_datetime_serializer)
def init_clients(self, parsed_args):
"""Override CommandBase to set up client with passed token
"""
client = Agave.restore()
if parsed_args.access_token is not None:
self.tapis_client = Agave(api_server=client.api_server,
token=parsed_args.access_token)
else:
self.tapis_client = client
self.tapis_client.token.refresh()
self.requests_client = self._get_direct(self.tapis_client)
return self
def main():
if len(sys.argv) < 2:
print('python tapis_job_cleanup.py db_uri')
sys.exit(2)
print(sys.argv[1])
agave = Agave.restore()
jobs_map = get_tapis_jobs_for_experiment_reference(
sys.argv[1], "NovelChassis-Endogenous-Promoter", agave=agave)
#jobs_map = get_tapis_jobs_for_experiment(sys.argv[1], ex_id = "experiment.transcriptic.r1f7aux4qxty6b", agave=agave)
print(f"jobs_map: {jobs_map}")
stop_tapis_jobs(jobs_map, ["Precomputed data table"], agave=agave)
def stop_tapis_jobs(jobs_map, pipelines, analysis=None, agave=None):
if agave is None:
agave = Agave.restore()
for p in pipelines:
tapis_jobs = jobs_map[p] if p in jobs_map else []
for tj in tapis_jobs:
if analysis is None or p == "Precomputed data table" and analysis == tj[
0]:
try:
print(f"stopping job {tj[1]}")
agave.jobs.manage(jobId=tj[1], body={"action": "stop"})
except Exception as exc:
print(f"exc: {exc}")
# avoid race condition by waiting for 1 minute to give Tapis jobs ample time to react
time.sleep(60)
def get_client_with_mock_support():
"""
Return the Abaco actor's API client if
running as a function. Try to return
user's local Agave client for debug
and test purposes.
"""
_client = None
if os.environ.get('_abaco_access_token') is None:
from agavepy.agave import Agave
try:
_client = Agave.restore()
except TypeError:
_client = None
else:
_client = get_client()
return _client
def get_client_with_mock_support():
'''
Get the current Actor API client
Returns the Abaco actor's client if running deployed. Attempts to
bootstrap a client from supplied credentials if running in local or
debug mode.
'''
_client = None
if os.environ.get('_abaco_access_token') is None:
from agavepy.agave import Agave
try:
_client = Agave.restore()
except TypeError:
_client = None
else:
_client = get_client()
return _client
def __init__(self,
api_server=TACC_API_SERVER,
access_token=None,
refresh_token=None,
verbose=Verbosity.DEFAULT):
self.log.debug('Initialize Agave API client')
# TODO - Implement refresh_token
try:
if access_token is not None:
self.log.debug('Using supplied Tapis credentials')
ag = Agave(api_server=api_server, token=access_token)
else:
# Use local credential cache
self.log.debug('Loading from local Tapis credential cache')
ag = Agave.restore()
except Exception:
self.log.exception('Unable to initialize Tapis client')
raise
self.client = ag
self.verbosity = verbose
def get_client_with_mock_support():
'''
Get the current Actor API client
Returns the Abaco actor's client if running deployed. Attempts to
bootstrap a client from supplied credentials if running in local or
debug mode.
'''
client = None
if '_abaco_access_token' not in os.environ:
try:
client = Agave.restore()
except TypeError as err:
raise AgaveError('Unable to restore Agave client: {}'.format(err))
else:
try:
client = get_client()
except Exception as err:
raise AgaveError('Unable to get Agave client from context: {}'.format(err))
return client
def main():
global RESTART_FILE
logger.setLevel(logging.INFO)
handler = logging.StreamHandler()
handler.setFormatter(logging.Formatter(FORMAT, datefmt=DATEFORMAT))
logger.addHandler(handler)
ts1 = time()
ag = Agave.restore()
system_id = sys.argv[1]
fpath = sys.argv[2]
RESTART_FILE = get_restart_filename(system_id, fpath)
load_restart_file(RESTART_FILE)
tree(ag, system_id, fpath)
# flush
update_completed(force=True)
elapsed = time() - ts1
logger.info("Elapsed: {} sec".format(elapsed))
events = '*'
assoc_ids = None
dest_uri = None
def get_requestbin():
"""Returns a temportary requestbin for API testing"""
r = requests.post('https://requestbin.agaveapi.co/api/v1/bins')
r.raise_for_status()
body = r.json()
name = body.get('name')
return 'https://requestbin.agaveapi.co/' + name
try:
ag = Agave.restore()
except HTTPError as h:
raise h
except Exception as e:
print("Unexpected error occurred: {}".format(e))
try:
listing = ag.files.list(systemId=system_id, filePath=file_path, limit=1)[0]
meta = listing['_links'].get('metadata', {})
if 'href' in meta:
muri = meta['href']
parsed = urlparse.urlparse(muri)
assoc_ids = urlparse.parse_qs(parsed.query)['q'][0].split(':')[1]
assoc_ids = re.sub(r'[^.a-zA-Z0-9-]', '', assoc_ids)
print('assocationIds = {}'.format(assoc_ids))
#!/usr/bin/env python
import sys
from agavepy.agave import Agave, AgaveException
try:
Agave.restore()
except Exception as a:
raise AgaveException("Unable to authenticate: {}".format(a))
sys.exit(1)
def main():
global debug
context = {}
args = parse_args()
debug = args.debug
template = args.input.read()
# extract variables so we can implement --no-empty or
# --blank-empty behavior
varpattern = re.compile(r"\{\{\s+?([a-zA-Z0-9\.\-\_]+)\s+?\}\}")
template_varnames = re.findall(varpattern, template)
if args.blankempty:
for varname in template_varnames:
context.update({varname: None})
# populate date_time
context.update(flatten(get_date_time_facts()))
# populate agave variables based on logged in user
ag = Agave.restore()
agave_vars = expand(get_agave_vars(ag), 'agave')
context.update(agave_vars)
# read in values from optional config file
if args.variables is not None:
config = configparser.ConfigParser()
config_dict = {}
config.read(args.variables)
for s in config.sections():
for k in config.options(s):
if s != 'DEFAULT':
template_var_key = u'.'.join([s, k])
else:
template_var_key = k
config_dict[template_var_key] = config.get(s, k)
context.update(config_dict)
# Since the intent of namespaced variables to to override
# the variables file, we read it and do the update on the
# context after reading from that file
env_dict = get_env_vars_from_args(args)
context.update(env_dict)
# implement our docker variable override
docker_vars = get_docker_vars(context)
context.update(docker_vars)
# should it be optional to inherit from os.environ?
context.update(os.environ)
# Do the subsitution dance
for var in list(context.keys()):
try:
pattern = re.compile('{{\s{0,}' + var + '\s{0,}}}')
replacement = context[var]
if replacement is None:
replacement = ''
template = re.sub(pattern, replacement, template)
except Exception:
pass
# handle all three paths for dealing with undefined or empty values
empty_vars = []
for varname in template_varnames:
if context.get(varname) is None:
empty_vars.append(varname)
if args.noempty:
if len(empty_vars) > 0:
raise ValueError("No assigned value for: {}".format(
', '.join(empty_vars)))
if args.blankempty:
if debug:
if len(empty_vars) > 0:
print("Variables evaluated as empty include: {}".format(
', '.join(empty_vars)))
args.output.write(template)
args.output.close()
def init_clients(self, passed_args=None):
self.tapis_client = Agave.restore()
self.tapis_client.refresh()
self.requests_client = self._get_direct(self.tapis_client)
return self
def __init__(self, options, *args, **kwargs):
self.options = options
self.args = args
self.kwargs = kwargs
self.store = alias.AliasStore(Agave.restore())