def register_nest_endpoints(flask_app, project_env, authenticator): db_engine = nest_db.get_global_sqlalchemy_engine() sqla_md = nest_db.get_global_sqlalchemy_metadata() if ProjectEnv.hello_world_instance() == project_env: import nest_py.hello_world.flask.hw_flask as hw_flask nest_endpoints = hw_flask.get_nest_endpoints(db_engine, sqla_md, authenticator) elif ProjectEnv.mmbdb_instance() == project_env: import nest_py.omix.flask.omix_flask as omix_flask nest_endpoints = omix_flask.get_nest_endpoints(db_engine, sqla_md, authenticator) elif ProjectEnv.knoweng_instance() == project_env: import nest_py.knoweng.flask.knoweng_flask as knoweng_flask nest_endpoints = knoweng_flask.get_nest_endpoints( db_engine, sqla_md, authenticator) else: raise Exception("Unknown project when registering endpoints") for flask_ep in nest_endpoints.get_flask_endpoints(): nest_ep = nest_endpoints.get_endpoint(flask_ep) relative_flask_rule = nest_ep.get_flask_rule() rule = API_PREFIX + relative_flask_rule print('registering flask rule: ' + str(rule)) flask_ep = nest_ep.get_flask_endpoint() renderer = nest_ep.handle_request flask_app.add_url_rule(rule, flask_ep, view_func=renderer, methods=['GET', 'POST', 'PATCH', 'DELETE']) return
def setup_db(): #these tests are detructive of the nest_users table in #the database, so point nest_users at a different table #for the duration of these tests nest_users.COLLECTION_NAME = 'test_users' md = nest_db.get_global_sqlalchemy_metadata() sqla_maker = core_db.get_nest_users_sqla_maker() users_tbl = sqla_maker.get_sqla_table(md) # initialize the test_projects db table projects.COLLECTION_NAME = 'test_projects' sp = SecurityPolicy(anyone_can_write=True, anyone_can_read_all=False) proj_sqla_maker = TablelikeSqlaMaker(projects.generate_schema(), security_policy=sp) proj_tbl = proj_sqla_maker.get_sqla_table(md) engine = nest_db.get_global_sqlalchemy_engine() if users_tbl.exists(engine): print('dropping existing table test_users') users_tbl.drop(engine) users_tbl.create(engine) print('created table test_users') # This is a side-effect of KNOW-516 # TODO: Update this when fixing/moving db_ops_utils.ensure_default_project() if proj_tbl.exists(engine): print('dropping existing table test_projects') proj_tbl.drop(engine) proj_tbl.create(engine) print('created table test_projects') return
def init_crud_clients(): config = nest_config.generate_config(ProjectEnv.knoweng_instance(), RunLevel.development_instance()) init_token_maker(config['JWT_SECRET'], config['JWT_ISSUER'], config['JWT_AUDIENCES']) global SCHEMA_REGISTRY SCHEMA_REGISTRY = knoweng_schemas.get_schemas() global CLIENT_REGISTRY CLIENT_REGISTRY = dict() #make db clients #TODO: knoweng should declare the db it's using, but for now #the default postgres container on localhost is all there is, #which is what the global db engine defaults to. engine = nest_db.get_global_sqlalchemy_engine() sqla_md = nest_db.get_global_sqlalchemy_metadata() db_client_makers = knoweng_db.get_sqla_makers() for name in DB_COLLECTIONS: cm = db_client_makers[name] client = cm.get_db_client(engine, sqla_md) CLIENT_REGISTRY[name] = client #make api clients http_client = get_http_client() api_client_makers = knoweng_api_clients.get_api_client_makers() for name in API_COLLECTIONS: cm = api_client_makers[name] crud_client = cm.get_crud_client(http_client) CLIENT_REGISTRY[name] = crud_client return
def setup_db(): #FIXME: use an isolated database 'nest_test'. #for now, these tests will create and delete #data in the hello_world table(s) sqla_md = nest_db.get_global_sqlalchemy_metadata() hw_db.register_sqla_bindings(sqla_md) db_ops_utils.ensure_tables_in_db() return
def make_users_db_client(): md = nest_db.get_global_sqlalchemy_metadata() sqla_maker = core_db.get_nest_users_sqla_maker() engine = nest_db.get_global_sqlalchemy_engine() sys_user = core_db.get_system_user() db_client = sqla_maker.get_db_client(engine, md) db_client.set_requesting_user(sys_user) return db_client
def ensure_tables_in_db(): engine = nest_db.get_global_sqlalchemy_engine() md = nest_db.get_global_sqlalchemy_metadata() for tbl in md.sorted_tables: if tbl.exists(engine): print(' exists : ' + str(tbl.name)) else: tbl.create(engine) print(' CREATED: ' + str(tbl.name)) return True
def ensure_default_project(nest_user): # initialize the projects db client schema = projects.generate_schema() sp = SecurityPolicy(anyone_can_write=True, anyone_can_read_all=False) sqla_maker = TablelikeSqlaMaker(schema, security_policy=sp) db_engine = nest_db.get_global_sqlalchemy_engine() md = nest_db.get_global_sqlalchemy_metadata() inspector = Inspector.from_engine(db_engine) # Noop for non-knoweng projects if not 'projects' in inspector.get_table_names(): print 'No projects table defined... skipping' return None projects_client = sqla_maker.get_db_client(db_engine, md) projects_client.set_requesting_user(nest_user) uname = nest_user.get_username() owner_id = nest_user.get_nest_id().get_value() default_project_name = "Default Project" fltr = {'name': default_project_name, 'owner_id': owner_id} existings = projects_client.simple_filter_query(fltr) if len(existings) == 0: #user hasn't accessed Nest before, create new default project new_default_project = ProjectDTO(default_project_name) raw_tle = new_default_project.to_tablelike_entry() default_project_tle = projects_client.create_entry(raw_tle) print(" Created: Default Project for user '" + uname + "'") elif len(existings) == 1: #found a valid record for this project in the DB default_project_tle = existings[0] print(" Exists: Default Project for user '" + uname + "'") else: print(" WARNING: Multiple Default Projects detected for username '" + uname + "'") default_project_tle = existings[0] if default_project_tle is None: default_project = None print('FAILURE ensuring default project: ' + uname) else: default_project = ProjectDTO.from_tablelike_entry(default_project_tle) print('ensured default project: ' + uname) return default_project
def seed_users(project_env, runlevel): """ adds users declared in nest_config to the nest_users table if they don't already exist """ db_client_maker = core_db.get_nest_users_sqla_maker() md = nest_db.get_global_sqlalchemy_metadata() engine = nest_db.get_global_sqlalchemy_engine() #note this is a tablelike client, not a NestUser client db_client = db_client_maker.get_db_client(engine, md) #needs a unique *instance* of system_user to act as 'owner' #as we will alter the instance that we add to the table db_client.set_requesting_user(core_db.get_system_user()) user_configs = nest_config.generate_seed_users(project_env, runlevel) success = _add_users_from_configs(db_client, user_configs) return success
def check_db_connection(): engine = nest_db.get_global_sqlalchemy_engine() md = nest_db.get_global_sqlalchemy_metadata() try: connection = engine.connect() qs = 'select 1' q = engine.execute(qs) obs = q.first()[0] if obs == 1: print("'Select 1' returned 1") connectable = True else: print("'Select 1' returned " + str(obs)) connectable = False except Exception as e: print('Connecting to DB Failed. Exception: ' + str(e)) connectable = False if not connectable: print('config was' + str(nest_db.GLOBAL_SQLA_RESOURCES.config)) return connectable
def init_db_client_registry(jcx): """ create a dict of database clients for hello_world project's datatypes. put the dictionary in the jcx 'runtime' so that they are accessible as: jcx.runtime()['db_clients'][<datatype>] """ #get the 'maker' objects for database tables and clients in #the hello_world project hw_sqla_makers = hw_db.get_sqla_makers() sqla_md = nest_db.get_global_sqlalchemy_metadata() engine = nest_db.get_global_sqlalchemy_engine() registry = dict() for maker in hw_sqla_makers.values(): nm = maker.get_table_name() client = maker.get_db_client(engine, sqla_md) #need a user to 'own' any data we write. this gives us #the default user for jobs jobs_auth.set_db_user(client) registry[nm] = client jcx.runtime()['db_clients'] = registry return
def set_db_user(db_client): """ similar to 'logging in', sets the requesting_user for a database client to the standard user that jobs run under. Note this is a NestUser for Nest's data ownership model, this is not the postgres user that connects to postgres. """ #FIXME: this is a user that is also hardcoded in flask_config, #but we are keeping the flask packages out of the jobs packages #so we cut and paste users_sqla_maker = core_db.get_nest_users_sqla_maker() db_engine = nest_db.get_global_sqlalchemy_engine() md = nest_db.get_global_sqlalchemy_metadata() users_client = users_sqla_maker.get_db_client(db_engine, md) users_client.set_requesting_user(core_db.get_system_user()) username = '******' user_tle = users_client.simple_filter_query({'username':username})[0] jobs_user = NestUser.from_tablelike_entry(user_tle) db_client.set_requesting_user(jobs_user) return
def add_user(username, password): """ leaves most fields blank except the username and password needed to login to the webapp. adds the user to the localhost instance. """ db_client_maker = core_db.get_nest_users_sqla_maker() md = nest_db.get_global_sqlalchemy_metadata() engine = nest_db.get_global_sqlalchemy_engine() #note this is a tablelike client, not a NestUser client db_client = db_client_maker.get_db_client(engine, md) system_user = core_db.get_system_user() db_client.set_requesting_user(system_user) schema = nest_users.generate_schema() passlib_hash = password_hash.compute_passlib_hash(password) nu = NestUser(None, username, None, None, is_superuser=False, passlib_hash=passlib_hash, origin='nest') tle = nu.to_tablelike_entry() tle = db_client.create_entry(tle) if tle is None: print('FAILURE ensuring user: '******'ensured user: ' + str(username)) success = True ensure_default_project(NestUser.from_tablelike_entry(tle)) return success
def build_authenticator(flask_app, project_env, runlevel): users_sqla_maker = core_db.get_nest_users_sqla_maker() db_engine = nest_db.get_global_sqlalchemy_engine() md = nest_db.get_global_sqlalchemy_metadata() users_client = users_sqla_maker.get_db_client(db_engine, md) #the authenticator will interact with the local db as the master system_user auth_user = core_db.get_system_user() users_client.set_requesting_user(auth_user) #knoweng uses hubzero to look up user accounts in production #all other situations will use user accounts stored in the local db use_hubzero = (ProjectEnv.knoweng_instance() == project_env and RunLevel.production_instance() == runlevel) if use_hubzero: print('registering Hubzero authenticator') from nest_py.knoweng.flask.accounts.knoweng_authentication import HubzeroAuthenticationStrategy authenticator = HubzeroAuthenticationStrategy(flask_app, users_client) else: from nest_py.core.flask.accounts.authentication import NativeAuthenticationStrategy authenticator = NativeAuthenticationStrategy(flask_app, users_client) return authenticator
def run(http_client, db_engine, data_dir, subsample, data_flavor_key): """ db_engine, sqla_md (sqlalchemy.Engine): a postgres hook to the db we will use. Tables must already exist in the db. data_dir (string): location to write data files subsample (bool) if true, only load a 100 samples from the biom_table and process. results will not be valid but all api endpoints will be populated with data """ timer = CheckpointTimer('mmbdb_seed') timer.checkpoint("mmbdb_seed_job: Begin") exit_code = 0 ############### ##Connect CRUD clients ################ if DB_VS_API: sqla_md = nest_db.get_global_sqlalchemy_metadata() clients = client_registry.make_db_client_registry(db_engine, sqla_md) for client in clients.values(): jobs_auth.set_db_user(client) else: jobs_auth.login_jobs_user(http_client) clients = client_registry.make_api_client_registry(http_client) ################### ##Download Raw Data ################### timer.checkpoint("Downloading biom data if necessary") host_user = container_users.make_host_user_container_user() seed_data = get_data_flavor(data_flavor_key, data_dir, host_user, subsample) biom_table = seed_data.get_biom_table() timer.checkpoint("Download complete.") timer.checkpoint("Downloaded/Loaded All Patient Metadata") #################### ##Upload Primitive Data #################### timer.checkpoint('uploading tornado_run: Begin') tornado_run_tle = biom_etl.upload_tornado_run(clients, biom_table) tornado_run_nest_id = tornado_run_tle.get_nest_id() tornado_run_id = tornado_run_nest_id.get_value() timer.checkpoint('uploading tornado_run: End') timer.checkpoint('uploading otu_defs: Begin') otu_defs = biom_etl.upload_otu_defs(clients, biom_table, tornado_run_id) timer.checkpoint('uploading otu_defs: End') timer.checkpoint('uploading geno_samples: Begin') geno_samples = biom_etl.upload_geno_samples(clients, biom_table, tornado_run_id, otu_defs) timer.checkpoint('uploading geno_samples: End') #################### #Define Cohorts #################### all_cohort_tles = dict() cohort_configs = seed_data.get_cohort_configs() for cohort_config in cohort_configs: cohort_key = cohort_config['display_name_short'] timer.checkpoint('uploading cohort: ' + str(cohort_key)) tornado_sample_keys = seed_data.get_tornado_sample_keys(cohort_key) sample_ids = cohort_etl.tornado_sample_keys_to_nest_ids( tornado_sample_keys, geno_samples) cohort_tle = cohort_etl.upload_cohort(clients, cohort_config, sample_ids, tornado_run_id) all_cohort_tles[cohort_key] = cohort_tle #################### ##Define Comparisons #################### all_comparisons = list() comparison_configs = seed_data.get_comparison_configs() for comparison_config in comparison_configs: comparison_key = comparison_config['comparison_key'] baseline_key = comparison_config['baseline_cohort_key'] baseline_cohort_tle = all_cohort_tles[baseline_key] variant_key = comparison_config['variant_cohort_key'] variant_cohort_tle = all_cohort_tles[variant_key] patient_key = comparison_config['patient_cohort_key'] patient_cohort_tle = all_cohort_tles[patient_key] timer.checkpoint('fst begin for: ' + comparison_key) fst_results = fst_etl.get_fst_of_comparison(comparison_key, seed_data, baseline_cohort_tle, variant_cohort_tle, otu_defs, geno_samples, data_dir, host_user) timer.checkpoint('api upload begin: ' + comparison_key) comparison_tle = comparison_etl.upload_comparison( clients, comparison_config, baseline_cohort_tle, variant_cohort_tle, patient_cohort_tle, fst_results) timer.checkpoint('api upload done') all_comparisons.append(comparison_tle) if DO_TREE_ANALYTICS: ############### ###Cohort Node Analytics ############### #this also does the upload asap to reduce memory footprint cohort_analysis.compute_all_for_cohorts(clients, all_cohort_tles.values(), geno_samples, otu_defs, num_quantiles=NUM_QUANTILES, num_bins=NUM_BINS, timer=timer) ############### ###Comparison Node Analytics ############### taxonomy_empty_tree = otu_analysis.compute_taxonomy_tree(otu_defs) for comp in all_comparisons: tree = taxonomy_empty_tree.copy() log('begin analytics for comparison: ' + comp.get_value('display_name')) timer.checkpoint('computing comparison analytics:begin') comparison_analysis.compute_all(comp, otu_defs, tree) timer.checkpoint('computing comparison analytics:end') timer.checkpoint('uploading comparison analytics nodes:begin') comparison_tree_etl.upload_nodes(clients, comp, tree) timer.checkpoint('uploading comparison analytics nodes:end') timer.checkpoint("mmbdb_seed_job: Done") return exit_code
def list_metadata_tables(): md = nest_db.get_global_sqlalchemy_metadata() print("sqlalchemy table bindings: ") for tbl in md.sorted_tables: print(' ' + str(tbl.name)) return True
def test_sqla_tablelike_lifecycle(): """ walks through making a table, making it in the database, making an entry, updating, etc, using the nest config way of setting things up and crud_db_client """ setup_db() sqla_md = nest_db.get_global_sqlalchemy_metadata() db_engine = nest_db.get_global_sqlalchemy_engine() db_registry = hw_db.get_sqla_makers() for tbl_name in db_registry: print('db registry: ' + str(tbl_name)) db_client = db_registry['hello_tablelike'].get_db_client(db_engine, sqla_md) sys_user = core_db.get_system_user() db_client.set_requesting_user(sys_user) db_client.delete_all_entries() print('testing create_entry') tle_orig = HelloTablelikeDTO(1.1, 2.2, 'x', ['a','a'], [0.0, 0.0], NestId(1), [NestId(4), NestId(2)], {'x':'innerx'}, 5, [6, 7]).to_tablelike_entry() print(str(tle_orig)) tle_updated = db_client.create_entry(tle_orig) assert(not tle_updated is None) nid = tle_updated.get_nest_id() print('generated nid: ' + str(nid)) assert(not nid is None) print('testing read_entry') tle_rt = db_client.read_entry(nid) print('tle_rt: ' + str(tle_rt)) print('tle_updated: ' + str(tle_updated)) assert(tle_rt == tle_updated) print('testing update_entry') tle_unupdated = tle_rt tle_unupdated.set_value('flt_val_0', 22.0) tle_updated = db_client.update_entry(tle_unupdated) tle_updated_rt = db_client.read_entry(nid) assert(tle_updated_rt == tle_updated) print('testing delete_entry') deleted_nid = db_client.delete_entry(nid) assert(deleted_nid == nid) fetched_deleted = db_client.read_entry(nid) assert(fetched_deleted is None) print('testing bulk_create_entries') t1 = HelloTablelikeDTO(3.0, 3.3, 'y', ['a','a'], [0.0, 0.0], NestId(5), [NestId(1), NestId(2)], {'x':'innerx'}, 5, [6, 7]).to_tablelike_entry() t2 = HelloTablelikeDTO(4.0, 4.4, 'y', ['a','b'], [0.0, 0.0], NestId(5), [NestId(1), NestId(2)], {'x':'innerx'}, 5, [6, 7]).to_tablelike_entry() t3 = HelloTablelikeDTO(5.0, 5.5, 'x', ['b','b'], [0.0, 0.0], NestId(5), [NestId(1), NestId(2)], {'x':'innerx'}, 5, [6, 7]).to_tablelike_entry() tles = [t1, t2, t3] up_tles = db_client.bulk_create_entries(tles, batch_size=2) assert(not up_tles is None) assert(len(up_tles) == len(tles)) for (orig_tle, up_tle) in zip(tles, up_tles): nid = up_tle.get_nest_id() assert(not nid is None) rt_tle = db_client.read_entry(nid) assert(rt_tle == up_tle) for att in ['flt_val_0', 'flt_val_1', 'string_val']: assert(orig_tle.get_value(att) == rt_tle.get_value(att)) print('testing bulk_create_entries_async and simple_filter_query') db_client.delete_all_entries() t1 = HelloTablelikeDTO(6.0, 6.6, 'x', ['b','b'], [0.0, 0.0], NestId(5), [NestId(1), NestId(2)], {'x':'innerx'}, 5, [6, 7]).to_tablelike_entry() t2 = HelloTablelikeDTO(7.0, 7.7, 'z', ['a','b'], [2.0, 0.0], NestId(5), [NestId(1), NestId(2)], {'x':'innerx'}, 5, [6, 7]).to_tablelike_entry() t3 = HelloTablelikeDTO(8.0, 8.8, 'z', ['b','a'], [0.0, 3.0], NestId(5), [NestId(1), NestId(2)], {'x':'innerx'}, 5, [6, 7]).to_tablelike_entry() tles = [t1, t2, t3] print('begin async upload') upload_count = db_client.bulk_create_entries_async(tles, batch_size=2) print('upload count returned: ' + str(upload_count)) z_tles = db_client.simple_filter_query({'string_val':'z'}) for z_tle in z_tles: nid = z_tle.get_nest_id() assert(not nid is None) z_tle.set_nest_id(None) #so we can test equivalence easier assert(len(z_tles) == 2) assert(t2 in z_tles) assert(t3 in z_tles) x_tles = db_client.simple_filter_query({'string_val':'x', 'flt_val_0':6.0}) for x_tle in x_tles: nid = x_tle.get_nest_id() assert(not nid is None) x_tle.set_nest_id(None) #so we can test equivalence easier assert(len(x_tles) == 1) assert(t1 in x_tles) print("testing filter by json attribute. should work b/c it's string matching in the db") json_tles = db_client.simple_filter_query({'json_val':json.dumps({'x':'innerx'})}) assert(len(json_tles) == 3) # # print('testing ensure_entry') # tK = HelloTablelikeDTO(9.0, 9.9, 'z', ['b','a'], [0.0, 3.0], NestId(5), [NestId(1), NestId(2)], {u'x':u'innerx'}).to_tablelike_entry() # tk2 = db_client.ensure_entry(tK) # tk3 = db_client.ensure_entry(tK) # assert(tk2.get_nest_id() == tk3.get_nest_id()) # db_client.get_sqla_connection().close() return