def register_nest_endpoints(flask_app, project_env, authenticator):
db_engine = nest_db.get_global_sqlalchemy_engine()
sqla_md = nest_db.get_global_sqlalchemy_metadata()
if ProjectEnv.hello_world_instance() == project_env:
import nest_py.hello_world.flask.hw_flask as hw_flask
nest_endpoints = hw_flask.get_nest_endpoints(db_engine, sqla_md,
authenticator)
elif ProjectEnv.mmbdb_instance() == project_env:
import nest_py.omix.flask.omix_flask as omix_flask
nest_endpoints = omix_flask.get_nest_endpoints(db_engine, sqla_md,
authenticator)
elif ProjectEnv.knoweng_instance() == project_env:
import nest_py.knoweng.flask.knoweng_flask as knoweng_flask
nest_endpoints = knoweng_flask.get_nest_endpoints(
db_engine, sqla_md, authenticator)
else:
raise Exception("Unknown project when registering endpoints")
for flask_ep in nest_endpoints.get_flask_endpoints():
nest_ep = nest_endpoints.get_endpoint(flask_ep)
relative_flask_rule = nest_ep.get_flask_rule()
rule = API_PREFIX + relative_flask_rule
print('registering flask rule: ' + str(rule))
flask_ep = nest_ep.get_flask_endpoint()
renderer = nest_ep.handle_request
flask_app.add_url_rule(rule,
flask_ep,
view_func=renderer,
methods=['GET', 'POST', 'PATCH', 'DELETE'])
return
def setup_db():
#these tests are detructive of the nest_users table in
#the database, so point nest_users at a different table
#for the duration of these tests
nest_users.COLLECTION_NAME = 'test_users'
md = nest_db.get_global_sqlalchemy_metadata()
sqla_maker = core_db.get_nest_users_sqla_maker()
users_tbl = sqla_maker.get_sqla_table(md)
# initialize the test_projects db table
projects.COLLECTION_NAME = 'test_projects'
sp = SecurityPolicy(anyone_can_write=True, anyone_can_read_all=False)
proj_sqla_maker = TablelikeSqlaMaker(projects.generate_schema(), security_policy=sp)
proj_tbl = proj_sqla_maker.get_sqla_table(md)
engine = nest_db.get_global_sqlalchemy_engine()
if users_tbl.exists(engine):
print('dropping existing table test_users')
users_tbl.drop(engine)
users_tbl.create(engine)
print('created table test_users')
# This is a side-effect of KNOW-516
# TODO: Update this when fixing/moving db_ops_utils.ensure_default_project()
if proj_tbl.exists(engine):
print('dropping existing table test_projects')
proj_tbl.drop(engine)
proj_tbl.create(engine)
print('created table test_projects')
return
def init_crud_clients():
config = nest_config.generate_config(ProjectEnv.knoweng_instance(),
RunLevel.development_instance())
init_token_maker(config['JWT_SECRET'], config['JWT_ISSUER'],
config['JWT_AUDIENCES'])
global SCHEMA_REGISTRY
SCHEMA_REGISTRY = knoweng_schemas.get_schemas()
global CLIENT_REGISTRY
CLIENT_REGISTRY = dict()
#make db clients
#TODO: knoweng should declare the db it's using, but for now
#the default postgres container on localhost is all there is,
#which is what the global db engine defaults to.
engine = nest_db.get_global_sqlalchemy_engine()
sqla_md = nest_db.get_global_sqlalchemy_metadata()
db_client_makers = knoweng_db.get_sqla_makers()
for name in DB_COLLECTIONS:
cm = db_client_makers[name]
client = cm.get_db_client(engine, sqla_md)
CLIENT_REGISTRY[name] = client
#make api clients
http_client = get_http_client()
api_client_makers = knoweng_api_clients.get_api_client_makers()
for name in API_COLLECTIONS:
cm = api_client_makers[name]
crud_client = cm.get_crud_client(http_client)
CLIENT_REGISTRY[name] = crud_client
return
def setup_db():
#FIXME: use an isolated database 'nest_test'.
#for now, these tests will create and delete
#data in the hello_world table(s)
sqla_md = nest_db.get_global_sqlalchemy_metadata()
hw_db.register_sqla_bindings(sqla_md)
db_ops_utils.ensure_tables_in_db()
return
def make_users_db_client():
md = nest_db.get_global_sqlalchemy_metadata()
sqla_maker = core_db.get_nest_users_sqla_maker()
engine = nest_db.get_global_sqlalchemy_engine()
sys_user = core_db.get_system_user()
db_client = sqla_maker.get_db_client(engine, md)
db_client.set_requesting_user(sys_user)
return db_client
def ensure_tables_in_db():
engine = nest_db.get_global_sqlalchemy_engine()
md = nest_db.get_global_sqlalchemy_metadata()
for tbl in md.sorted_tables:
if tbl.exists(engine):
print(' exists : ' + str(tbl.name))
else:
tbl.create(engine)
print(' CREATED: ' + str(tbl.name))
return True
def ensure_default_project(nest_user):
# initialize the projects db client
schema = projects.generate_schema()
sp = SecurityPolicy(anyone_can_write=True, anyone_can_read_all=False)
sqla_maker = TablelikeSqlaMaker(schema, security_policy=sp)
db_engine = nest_db.get_global_sqlalchemy_engine()
md = nest_db.get_global_sqlalchemy_metadata()
inspector = Inspector.from_engine(db_engine)
# Noop for non-knoweng projects
if not 'projects' in inspector.get_table_names():
print 'No projects table defined... skipping'
return None
projects_client = sqla_maker.get_db_client(db_engine, md)
projects_client.set_requesting_user(nest_user)
uname = nest_user.get_username()
owner_id = nest_user.get_nest_id().get_value()
default_project_name = "Default Project"
fltr = {'name': default_project_name, 'owner_id': owner_id}
existings = projects_client.simple_filter_query(fltr)
if len(existings) == 0:
#user hasn't accessed Nest before, create new default project
new_default_project = ProjectDTO(default_project_name)
raw_tle = new_default_project.to_tablelike_entry()
default_project_tle = projects_client.create_entry(raw_tle)
print(" Created: Default Project for user '" + uname + "'")
elif len(existings) == 1:
#found a valid record for this project in the DB
default_project_tle = existings[0]
print(" Exists: Default Project for user '" + uname + "'")
else:
print(" WARNING: Multiple Default Projects detected for username '" +
uname + "'")
default_project_tle = existings[0]
if default_project_tle is None:
default_project = None
print('FAILURE ensuring default project: ' + uname)
else:
default_project = ProjectDTO.from_tablelike_entry(default_project_tle)
print('ensured default project: ' + uname)
return default_project
def seed_users(project_env, runlevel):
"""
adds users declared in nest_config to the nest_users table
if they don't already exist
"""
db_client_maker = core_db.get_nest_users_sqla_maker()
md = nest_db.get_global_sqlalchemy_metadata()
engine = nest_db.get_global_sqlalchemy_engine()
#note this is a tablelike client, not a NestUser client
db_client = db_client_maker.get_db_client(engine, md)
#needs a unique *instance* of system_user to act as 'owner'
#as we will alter the instance that we add to the table
db_client.set_requesting_user(core_db.get_system_user())
user_configs = nest_config.generate_seed_users(project_env, runlevel)
success = _add_users_from_configs(db_client, user_configs)
return success
def check_db_connection():
engine = nest_db.get_global_sqlalchemy_engine()
md = nest_db.get_global_sqlalchemy_metadata()
try:
connection = engine.connect()
qs = 'select 1'
q = engine.execute(qs)
obs = q.first()[0]
if obs == 1:
print("'Select 1' returned 1")
connectable = True
else:
print("'Select 1' returned " + str(obs))
connectable = False
except Exception as e:
print('Connecting to DB Failed. Exception: ' + str(e))
connectable = False
if not connectable:
print('config was' + str(nest_db.GLOBAL_SQLA_RESOURCES.config))
return connectable
def init_db_client_registry(jcx):
"""
create a dict of database clients for hello_world project's datatypes.
put the dictionary in the jcx 'runtime' so that they are accessible as:
jcx.runtime()['db_clients'][<datatype>]
"""
#get the 'maker' objects for database tables and clients in
#the hello_world project
hw_sqla_makers = hw_db.get_sqla_makers()
sqla_md = nest_db.get_global_sqlalchemy_metadata()
engine = nest_db.get_global_sqlalchemy_engine()
registry = dict()
for maker in hw_sqla_makers.values():
nm = maker.get_table_name()
client = maker.get_db_client(engine, sqla_md)
#need a user to 'own' any data we write. this gives us
#the default user for jobs
jobs_auth.set_db_user(client)
registry[nm] = client
jcx.runtime()['db_clients'] = registry
return
def set_db_user(db_client):
"""
similar to 'logging in', sets the requesting_user for a
database client to the standard user that jobs run under.
Note this is a NestUser for Nest's data ownership model, this
is not the postgres user that connects to postgres.
"""
#FIXME: this is a user that is also hardcoded in flask_config,
#but we are keeping the flask packages out of the jobs packages
#so we cut and paste
users_sqla_maker = core_db.get_nest_users_sqla_maker()
db_engine = nest_db.get_global_sqlalchemy_engine()
md = nest_db.get_global_sqlalchemy_metadata()
users_client = users_sqla_maker.get_db_client(db_engine, md)
users_client.set_requesting_user(core_db.get_system_user())
username = '******'
user_tle = users_client.simple_filter_query({'username':username})[0]
jobs_user = NestUser.from_tablelike_entry(user_tle)
db_client.set_requesting_user(jobs_user)
return
def add_user(username, password):
"""
leaves most fields blank except the username and password needed
to login to the webapp. adds the user to the localhost instance.
"""
db_client_maker = core_db.get_nest_users_sqla_maker()
md = nest_db.get_global_sqlalchemy_metadata()
engine = nest_db.get_global_sqlalchemy_engine()
#note this is a tablelike client, not a NestUser client
db_client = db_client_maker.get_db_client(engine, md)
system_user = core_db.get_system_user()
db_client.set_requesting_user(system_user)
schema = nest_users.generate_schema()
passlib_hash = password_hash.compute_passlib_hash(password)
nu = NestUser(None,
username,
None,
None,
is_superuser=False,
passlib_hash=passlib_hash,
origin='nest')
tle = nu.to_tablelike_entry()
tle = db_client.create_entry(tle)
if tle is None:
print('FAILURE ensuring user: '******'ensured user: ' + str(username))
success = True
ensure_default_project(NestUser.from_tablelike_entry(tle))
return success
def build_authenticator(flask_app, project_env, runlevel):
users_sqla_maker = core_db.get_nest_users_sqla_maker()
db_engine = nest_db.get_global_sqlalchemy_engine()
md = nest_db.get_global_sqlalchemy_metadata()
users_client = users_sqla_maker.get_db_client(db_engine, md)
#the authenticator will interact with the local db as the master system_user
auth_user = core_db.get_system_user()
users_client.set_requesting_user(auth_user)
#knoweng uses hubzero to look up user accounts in production
#all other situations will use user accounts stored in the local db
use_hubzero = (ProjectEnv.knoweng_instance() == project_env
and RunLevel.production_instance() == runlevel)
if use_hubzero:
print('registering Hubzero authenticator')
from nest_py.knoweng.flask.accounts.knoweng_authentication import HubzeroAuthenticationStrategy
authenticator = HubzeroAuthenticationStrategy(flask_app, users_client)
else:
from nest_py.core.flask.accounts.authentication import NativeAuthenticationStrategy
authenticator = NativeAuthenticationStrategy(flask_app, users_client)
return authenticator
def run(http_client, db_engine, data_dir, subsample, data_flavor_key):
"""
db_engine, sqla_md (sqlalchemy.Engine): a postgres hook to the
db we will use. Tables must already exist in the db.
data_dir (string): location to write data files
subsample (bool) if true, only load a 100 samples from the biom_table
and process. results will not be valid but all api endpoints will
be populated with data
"""
timer = CheckpointTimer('mmbdb_seed')
timer.checkpoint("mmbdb_seed_job: Begin")
exit_code = 0
###############
##Connect CRUD clients
################
if DB_VS_API:
sqla_md = nest_db.get_global_sqlalchemy_metadata()
clients = client_registry.make_db_client_registry(db_engine, sqla_md)
for client in clients.values():
jobs_auth.set_db_user(client)
else:
jobs_auth.login_jobs_user(http_client)
clients = client_registry.make_api_client_registry(http_client)
###################
##Download Raw Data
###################
timer.checkpoint("Downloading biom data if necessary")
host_user = container_users.make_host_user_container_user()
seed_data = get_data_flavor(data_flavor_key, data_dir, host_user,
subsample)
biom_table = seed_data.get_biom_table()
timer.checkpoint("Download complete.")
timer.checkpoint("Downloaded/Loaded All Patient Metadata")
####################
##Upload Primitive Data
####################
timer.checkpoint('uploading tornado_run: Begin')
tornado_run_tle = biom_etl.upload_tornado_run(clients, biom_table)
tornado_run_nest_id = tornado_run_tle.get_nest_id()
tornado_run_id = tornado_run_nest_id.get_value()
timer.checkpoint('uploading tornado_run: End')
timer.checkpoint('uploading otu_defs: Begin')
otu_defs = biom_etl.upload_otu_defs(clients, biom_table, tornado_run_id)
timer.checkpoint('uploading otu_defs: End')
timer.checkpoint('uploading geno_samples: Begin')
geno_samples = biom_etl.upload_geno_samples(clients, biom_table,
tornado_run_id, otu_defs)
timer.checkpoint('uploading geno_samples: End')
####################
#Define Cohorts
####################
all_cohort_tles = dict()
cohort_configs = seed_data.get_cohort_configs()
for cohort_config in cohort_configs:
cohort_key = cohort_config['display_name_short']
timer.checkpoint('uploading cohort: ' + str(cohort_key))
tornado_sample_keys = seed_data.get_tornado_sample_keys(cohort_key)
sample_ids = cohort_etl.tornado_sample_keys_to_nest_ids(
tornado_sample_keys, geno_samples)
cohort_tle = cohort_etl.upload_cohort(clients, cohort_config,
sample_ids, tornado_run_id)
all_cohort_tles[cohort_key] = cohort_tle
####################
##Define Comparisons
####################
all_comparisons = list()
comparison_configs = seed_data.get_comparison_configs()
for comparison_config in comparison_configs:
comparison_key = comparison_config['comparison_key']
baseline_key = comparison_config['baseline_cohort_key']
baseline_cohort_tle = all_cohort_tles[baseline_key]
variant_key = comparison_config['variant_cohort_key']
variant_cohort_tle = all_cohort_tles[variant_key]
patient_key = comparison_config['patient_cohort_key']
patient_cohort_tle = all_cohort_tles[patient_key]
timer.checkpoint('fst begin for: ' + comparison_key)
fst_results = fst_etl.get_fst_of_comparison(comparison_key, seed_data,
baseline_cohort_tle,
variant_cohort_tle,
otu_defs, geno_samples,
data_dir, host_user)
timer.checkpoint('api upload begin: ' + comparison_key)
comparison_tle = comparison_etl.upload_comparison(
clients, comparison_config, baseline_cohort_tle,
variant_cohort_tle, patient_cohort_tle, fst_results)
timer.checkpoint('api upload done')
all_comparisons.append(comparison_tle)
if DO_TREE_ANALYTICS:
###############
###Cohort Node Analytics
###############
#this also does the upload asap to reduce memory footprint
cohort_analysis.compute_all_for_cohorts(clients,
all_cohort_tles.values(),
geno_samples,
otu_defs,
num_quantiles=NUM_QUANTILES,
num_bins=NUM_BINS,
timer=timer)
###############
###Comparison Node Analytics
###############
taxonomy_empty_tree = otu_analysis.compute_taxonomy_tree(otu_defs)
for comp in all_comparisons:
tree = taxonomy_empty_tree.copy()
log('begin analytics for comparison: ' +
comp.get_value('display_name'))
timer.checkpoint('computing comparison analytics:begin')
comparison_analysis.compute_all(comp, otu_defs, tree)
timer.checkpoint('computing comparison analytics:end')
timer.checkpoint('uploading comparison analytics nodes:begin')
comparison_tree_etl.upload_nodes(clients, comp, tree)
timer.checkpoint('uploading comparison analytics nodes:end')
timer.checkpoint("mmbdb_seed_job: Done")
return exit_code
def list_metadata_tables():
md = nest_db.get_global_sqlalchemy_metadata()
print("sqlalchemy table bindings: ")
for tbl in md.sorted_tables:
print(' ' + str(tbl.name))
return True
def test_sqla_tablelike_lifecycle():
"""
walks through making a table, making it in the database,
making an entry, updating, etc, using the nest config
way of setting things up and crud_db_client
"""
setup_db()
sqla_md = nest_db.get_global_sqlalchemy_metadata()
db_engine = nest_db.get_global_sqlalchemy_engine()
db_registry = hw_db.get_sqla_makers()
for tbl_name in db_registry:
print('db registry: ' + str(tbl_name))
db_client = db_registry['hello_tablelike'].get_db_client(db_engine, sqla_md)
sys_user = core_db.get_system_user()
db_client.set_requesting_user(sys_user)
db_client.delete_all_entries()
print('testing create_entry')
tle_orig = HelloTablelikeDTO(1.1, 2.2, 'x', ['a','a'], [0.0, 0.0], NestId(1), [NestId(4), NestId(2)], {'x':'innerx'}, 5, [6, 7]).to_tablelike_entry()
print(str(tle_orig))
tle_updated = db_client.create_entry(tle_orig)
assert(not tle_updated is None)
nid = tle_updated.get_nest_id()
print('generated nid: ' + str(nid))
assert(not nid is None)
print('testing read_entry')
tle_rt = db_client.read_entry(nid)
print('tle_rt: ' + str(tle_rt))
print('tle_updated: ' + str(tle_updated))
assert(tle_rt == tle_updated)
print('testing update_entry')
tle_unupdated = tle_rt
tle_unupdated.set_value('flt_val_0', 22.0)
tle_updated = db_client.update_entry(tle_unupdated)
tle_updated_rt = db_client.read_entry(nid)
assert(tle_updated_rt == tle_updated)
print('testing delete_entry')
deleted_nid = db_client.delete_entry(nid)
assert(deleted_nid == nid)
fetched_deleted = db_client.read_entry(nid)
assert(fetched_deleted is None)
print('testing bulk_create_entries')
t1 = HelloTablelikeDTO(3.0, 3.3, 'y', ['a','a'], [0.0, 0.0], NestId(5), [NestId(1), NestId(2)], {'x':'innerx'}, 5, [6, 7]).to_tablelike_entry()
t2 = HelloTablelikeDTO(4.0, 4.4, 'y', ['a','b'], [0.0, 0.0], NestId(5), [NestId(1), NestId(2)], {'x':'innerx'}, 5, [6, 7]).to_tablelike_entry()
t3 = HelloTablelikeDTO(5.0, 5.5, 'x', ['b','b'], [0.0, 0.0], NestId(5), [NestId(1), NestId(2)], {'x':'innerx'}, 5, [6, 7]).to_tablelike_entry()
tles = [t1, t2, t3]
up_tles = db_client.bulk_create_entries(tles, batch_size=2)
assert(not up_tles is None)
assert(len(up_tles) == len(tles))
for (orig_tle, up_tle) in zip(tles, up_tles):
nid = up_tle.get_nest_id()
assert(not nid is None)
rt_tle = db_client.read_entry(nid)
assert(rt_tle == up_tle)
for att in ['flt_val_0', 'flt_val_1', 'string_val']:
assert(orig_tle.get_value(att) == rt_tle.get_value(att))
print('testing bulk_create_entries_async and simple_filter_query')
db_client.delete_all_entries()
t1 = HelloTablelikeDTO(6.0, 6.6, 'x', ['b','b'], [0.0, 0.0], NestId(5), [NestId(1), NestId(2)], {'x':'innerx'}, 5, [6, 7]).to_tablelike_entry()
t2 = HelloTablelikeDTO(7.0, 7.7, 'z', ['a','b'], [2.0, 0.0], NestId(5), [NestId(1), NestId(2)], {'x':'innerx'}, 5, [6, 7]).to_tablelike_entry()
t3 = HelloTablelikeDTO(8.0, 8.8, 'z', ['b','a'], [0.0, 3.0], NestId(5), [NestId(1), NestId(2)], {'x':'innerx'}, 5, [6, 7]).to_tablelike_entry()
tles = [t1, t2, t3]
print('begin async upload')
upload_count = db_client.bulk_create_entries_async(tles, batch_size=2)
print('upload count returned: ' + str(upload_count))
z_tles = db_client.simple_filter_query({'string_val':'z'})
for z_tle in z_tles:
nid = z_tle.get_nest_id()
assert(not nid is None)
z_tle.set_nest_id(None) #so we can test equivalence easier
assert(len(z_tles) == 2)
assert(t2 in z_tles)
assert(t3 in z_tles)
x_tles = db_client.simple_filter_query({'string_val':'x', 'flt_val_0':6.0})
for x_tle in x_tles:
nid = x_tle.get_nest_id()
assert(not nid is None)
x_tle.set_nest_id(None) #so we can test equivalence easier
assert(len(x_tles) == 1)
assert(t1 in x_tles)
print("testing filter by json attribute. should work b/c it's string matching in the db")
json_tles = db_client.simple_filter_query({'json_val':json.dumps({'x':'innerx'})})
assert(len(json_tles) == 3)
#
# print('testing ensure_entry')
# tK = HelloTablelikeDTO(9.0, 9.9, 'z', ['b','a'], [0.0, 3.0], NestId(5), [NestId(1), NestId(2)], {u'x':u'innerx'}).to_tablelike_entry()
# tk2 = db_client.ensure_entry(tK)
# tk3 = db_client.ensure_entry(tK)
# assert(tk2.get_nest_id() == tk3.get_nest_id())
#
db_client.get_sqla_connection().close()
return