def register_nest_endpoints(flask_app, project_env, authenticator):
db_engine = nest_db.get_global_sqlalchemy_engine()
sqla_md = nest_db.get_global_sqlalchemy_metadata()
if ProjectEnv.hello_world_instance() == project_env:
import nest_py.hello_world.flask.hw_flask as hw_flask
nest_endpoints = hw_flask.get_nest_endpoints(db_engine, sqla_md,
authenticator)
elif ProjectEnv.mmbdb_instance() == project_env:
import nest_py.omix.flask.omix_flask as omix_flask
nest_endpoints = omix_flask.get_nest_endpoints(db_engine, sqla_md,
authenticator)
elif ProjectEnv.knoweng_instance() == project_env:
import nest_py.knoweng.flask.knoweng_flask as knoweng_flask
nest_endpoints = knoweng_flask.get_nest_endpoints(
db_engine, sqla_md, authenticator)
else:
raise Exception("Unknown project when registering endpoints")
for flask_ep in nest_endpoints.get_flask_endpoints():
nest_ep = nest_endpoints.get_endpoint(flask_ep)
relative_flask_rule = nest_ep.get_flask_rule()
rule = API_PREFIX + relative_flask_rule
print('registering flask rule: ' + str(rule))
flask_ep = nest_ep.get_flask_endpoint()
renderer = nest_ep.handle_request
flask_app.add_url_rule(rule,
flask_ep,
view_func=renderer,
methods=['GET', 'POST', 'PATCH', 'DELETE'])
return
def setup_db():
#these tests are detructive of the nest_users table in
#the database, so point nest_users at a different table
#for the duration of these tests
nest_users.COLLECTION_NAME = 'test_users'
md = nest_db.get_global_sqlalchemy_metadata()
sqla_maker = core_db.get_nest_users_sqla_maker()
users_tbl = sqla_maker.get_sqla_table(md)
# initialize the test_projects db table
projects.COLLECTION_NAME = 'test_projects'
sp = SecurityPolicy(anyone_can_write=True, anyone_can_read_all=False)
proj_sqla_maker = TablelikeSqlaMaker(projects.generate_schema(), security_policy=sp)
proj_tbl = proj_sqla_maker.get_sqla_table(md)
engine = nest_db.get_global_sqlalchemy_engine()
if users_tbl.exists(engine):
print('dropping existing table test_users')
users_tbl.drop(engine)
users_tbl.create(engine)
print('created table test_users')
# This is a side-effect of KNOW-516
# TODO: Update this when fixing/moving db_ops_utils.ensure_default_project()
if proj_tbl.exists(engine):
print('dropping existing table test_projects')
proj_tbl.drop(engine)
proj_tbl.create(engine)
print('created table test_projects')
return
def init_crud_clients():
config = nest_config.generate_config(ProjectEnv.knoweng_instance(),
RunLevel.development_instance())
init_token_maker(config['JWT_SECRET'], config['JWT_ISSUER'],
config['JWT_AUDIENCES'])
global SCHEMA_REGISTRY
SCHEMA_REGISTRY = knoweng_schemas.get_schemas()
global CLIENT_REGISTRY
CLIENT_REGISTRY = dict()
#make db clients
#TODO: knoweng should declare the db it's using, but for now
#the default postgres container on localhost is all there is,
#which is what the global db engine defaults to.
engine = nest_db.get_global_sqlalchemy_engine()
sqla_md = nest_db.get_global_sqlalchemy_metadata()
db_client_makers = knoweng_db.get_sqla_makers()
for name in DB_COLLECTIONS:
cm = db_client_makers[name]
client = cm.get_db_client(engine, sqla_md)
CLIENT_REGISTRY[name] = client
#make api clients
http_client = get_http_client()
api_client_makers = knoweng_api_clients.get_api_client_makers()
for name in API_COLLECTIONS:
cm = api_client_makers[name]
crud_client = cm.get_crud_client(http_client)
CLIENT_REGISTRY[name] = crud_client
return
def _make_db_engine(project_env):
db_config = nest_db.generate_db_config(project_env=project_env)
db_config['verbose_logging'] = False
sqla_res = nest_db.GLOBAL_SQLA_RESOURCES
sqla_res.set_config(db_config)
nest_db.set_global_sqla_resources(sqla_res)
db_engine = nest_db.get_global_sqlalchemy_engine().connect()
return db_engine
def make_users_db_client():
md = nest_db.get_global_sqlalchemy_metadata()
sqla_maker = core_db.get_nest_users_sqla_maker()
engine = nest_db.get_global_sqlalchemy_engine()
sys_user = core_db.get_system_user()
db_client = sqla_maker.get_db_client(engine, md)
db_client.set_requesting_user(sys_user)
return db_client
def ensure_tables_in_db():
engine = nest_db.get_global_sqlalchemy_engine()
md = nest_db.get_global_sqlalchemy_metadata()
for tbl in md.sorted_tables:
if tbl.exists(engine):
print(' exists : ' + str(tbl.name))
else:
tbl.create(engine)
print(' CREATED: ' + str(tbl.name))
return True
def ensure_default_project(nest_user):
# initialize the projects db client
schema = projects.generate_schema()
sp = SecurityPolicy(anyone_can_write=True, anyone_can_read_all=False)
sqla_maker = TablelikeSqlaMaker(schema, security_policy=sp)
db_engine = nest_db.get_global_sqlalchemy_engine()
md = nest_db.get_global_sqlalchemy_metadata()
inspector = Inspector.from_engine(db_engine)
# Noop for non-knoweng projects
if not 'projects' in inspector.get_table_names():
print 'No projects table defined... skipping'
return None
projects_client = sqla_maker.get_db_client(db_engine, md)
projects_client.set_requesting_user(nest_user)
uname = nest_user.get_username()
owner_id = nest_user.get_nest_id().get_value()
default_project_name = "Default Project"
fltr = {'name': default_project_name, 'owner_id': owner_id}
existings = projects_client.simple_filter_query(fltr)
if len(existings) == 0:
#user hasn't accessed Nest before, create new default project
new_default_project = ProjectDTO(default_project_name)
raw_tle = new_default_project.to_tablelike_entry()
default_project_tle = projects_client.create_entry(raw_tle)
print(" Created: Default Project for user '" + uname + "'")
elif len(existings) == 1:
#found a valid record for this project in the DB
default_project_tle = existings[0]
print(" Exists: Default Project for user '" + uname + "'")
else:
print(" WARNING: Multiple Default Projects detected for username '" +
uname + "'")
default_project_tle = existings[0]
if default_project_tle is None:
default_project = None
print('FAILURE ensuring default project: ' + uname)
else:
default_project = ProjectDTO.from_tablelike_entry(default_project_tle)
print('ensured default project: ' + uname)
return default_project
def seed_users(project_env, runlevel):
"""
adds users declared in nest_config to the nest_users table
if they don't already exist
"""
db_client_maker = core_db.get_nest_users_sqla_maker()
md = nest_db.get_global_sqlalchemy_metadata()
engine = nest_db.get_global_sqlalchemy_engine()
#note this is a tablelike client, not a NestUser client
db_client = db_client_maker.get_db_client(engine, md)
#needs a unique *instance* of system_user to act as 'owner'
#as we will alter the instance that we add to the table
db_client.set_requesting_user(core_db.get_system_user())
user_configs = nest_config.generate_seed_users(project_env, runlevel)
success = _add_users_from_configs(db_client, user_configs)
return success
def list_tables_in_db():
engine = nest_db.get_global_sqlalchemy_engine()
qs = """
select table_catalog, table_schema, table_name, table_type
from information_schema.tables
where table_schema not in ('pg_catalog', 'information_schema');
"""
q = engine.execute(qs)
tables = q.fetchall()
print("tables from db:")
if len(tables) == 0:
print(' <None>')
for i, tbl in enumerate(tables):
(c, s, n, t) = tbl
print(str(i) + '.')
print(' name :' + str(n))
print(' catalog :' + str(c))
print(' type :' + str(t))
print(' schema :' + str(s))
return True
def check_db_connection():
engine = nest_db.get_global_sqlalchemy_engine()
md = nest_db.get_global_sqlalchemy_metadata()
try:
connection = engine.connect()
qs = 'select 1'
q = engine.execute(qs)
obs = q.first()[0]
if obs == 1:
print("'Select 1' returned 1")
connectable = True
else:
print("'Select 1' returned " + str(obs))
connectable = False
except Exception as e:
print('Connecting to DB Failed. Exception: ' + str(e))
connectable = False
if not connectable:
print('config was' + str(nest_db.GLOBAL_SQLA_RESOURCES.config))
return connectable
def init_db_client_registry(jcx):
"""
create a dict of database clients for hello_world project's datatypes.
put the dictionary in the jcx 'runtime' so that they are accessible as:
jcx.runtime()['db_clients'][<datatype>]
"""
#get the 'maker' objects for database tables and clients in
#the hello_world project
hw_sqla_makers = hw_db.get_sqla_makers()
sqla_md = nest_db.get_global_sqlalchemy_metadata()
engine = nest_db.get_global_sqlalchemy_engine()
registry = dict()
for maker in hw_sqla_makers.values():
nm = maker.get_table_name()
client = maker.get_db_client(engine, sqla_md)
#need a user to 'own' any data we write. this gives us
#the default user for jobs
jobs_auth.set_db_user(client)
registry[nm] = client
jcx.runtime()['db_clients'] = registry
return
def set_db_user(db_client):
"""
similar to 'logging in', sets the requesting_user for a
database client to the standard user that jobs run under.
Note this is a NestUser for Nest's data ownership model, this
is not the postgres user that connects to postgres.
"""
#FIXME: this is a user that is also hardcoded in flask_config,
#but we are keeping the flask packages out of the jobs packages
#so we cut and paste
users_sqla_maker = core_db.get_nest_users_sqla_maker()
db_engine = nest_db.get_global_sqlalchemy_engine()
md = nest_db.get_global_sqlalchemy_metadata()
users_client = users_sqla_maker.get_db_client(db_engine, md)
users_client.set_requesting_user(core_db.get_system_user())
username = '******'
user_tle = users_client.simple_filter_query({'username':username})[0]
jobs_user = NestUser.from_tablelike_entry(user_tle)
db_client.set_requesting_user(jobs_user)
return
def add_user(username, password):
"""
leaves most fields blank except the username and password needed
to login to the webapp. adds the user to the localhost instance.
"""
db_client_maker = core_db.get_nest_users_sqla_maker()
md = nest_db.get_global_sqlalchemy_metadata()
engine = nest_db.get_global_sqlalchemy_engine()
#note this is a tablelike client, not a NestUser client
db_client = db_client_maker.get_db_client(engine, md)
system_user = core_db.get_system_user()
db_client.set_requesting_user(system_user)
schema = nest_users.generate_schema()
passlib_hash = password_hash.compute_passlib_hash(password)
nu = NestUser(None,
username,
None,
None,
is_superuser=False,
passlib_hash=passlib_hash,
origin='nest')
tle = nu.to_tablelike_entry()
tle = db_client.create_entry(tle)
if tle is None:
print('FAILURE ensuring user: '******'ensured user: ' + str(username))
success = True
ensure_default_project(NestUser.from_tablelike_entry(tle))
return success
def build_authenticator(flask_app, project_env, runlevel):
users_sqla_maker = core_db.get_nest_users_sqla_maker()
db_engine = nest_db.get_global_sqlalchemy_engine()
md = nest_db.get_global_sqlalchemy_metadata()
users_client = users_sqla_maker.get_db_client(db_engine, md)
#the authenticator will interact with the local db as the master system_user
auth_user = core_db.get_system_user()
users_client.set_requesting_user(auth_user)
#knoweng uses hubzero to look up user accounts in production
#all other situations will use user accounts stored in the local db
use_hubzero = (ProjectEnv.knoweng_instance() == project_env
and RunLevel.production_instance() == runlevel)
if use_hubzero:
print('registering Hubzero authenticator')
from nest_py.knoweng.flask.accounts.knoweng_authentication import HubzeroAuthenticationStrategy
authenticator = HubzeroAuthenticationStrategy(flask_app, users_client)
else:
from nest_py.core.flask.accounts.authentication import NativeAuthenticationStrategy
authenticator = NativeAuthenticationStrategy(flask_app, users_client)
return authenticator
def drop_tables_in_db():
engine = nest_db.get_global_sqlalchemy_engine()
nest_db.get_global_sqlalchemy_base().metadata.drop_all(engine)
return True
def test_sqla_tablelike_lifecycle():
"""
walks through making a table, making it in the database,
making an entry, updating, etc, using the nest config
way of setting things up and crud_db_client
"""
setup_db()
sqla_md = nest_db.get_global_sqlalchemy_metadata()
db_engine = nest_db.get_global_sqlalchemy_engine()
db_registry = hw_db.get_sqla_makers()
for tbl_name in db_registry:
print('db registry: ' + str(tbl_name))
db_client = db_registry['hello_tablelike'].get_db_client(db_engine, sqla_md)
sys_user = core_db.get_system_user()
db_client.set_requesting_user(sys_user)
db_client.delete_all_entries()
print('testing create_entry')
tle_orig = HelloTablelikeDTO(1.1, 2.2, 'x', ['a','a'], [0.0, 0.0], NestId(1), [NestId(4), NestId(2)], {'x':'innerx'}, 5, [6, 7]).to_tablelike_entry()
print(str(tle_orig))
tle_updated = db_client.create_entry(tle_orig)
assert(not tle_updated is None)
nid = tle_updated.get_nest_id()
print('generated nid: ' + str(nid))
assert(not nid is None)
print('testing read_entry')
tle_rt = db_client.read_entry(nid)
print('tle_rt: ' + str(tle_rt))
print('tle_updated: ' + str(tle_updated))
assert(tle_rt == tle_updated)
print('testing update_entry')
tle_unupdated = tle_rt
tle_unupdated.set_value('flt_val_0', 22.0)
tle_updated = db_client.update_entry(tle_unupdated)
tle_updated_rt = db_client.read_entry(nid)
assert(tle_updated_rt == tle_updated)
print('testing delete_entry')
deleted_nid = db_client.delete_entry(nid)
assert(deleted_nid == nid)
fetched_deleted = db_client.read_entry(nid)
assert(fetched_deleted is None)
print('testing bulk_create_entries')
t1 = HelloTablelikeDTO(3.0, 3.3, 'y', ['a','a'], [0.0, 0.0], NestId(5), [NestId(1), NestId(2)], {'x':'innerx'}, 5, [6, 7]).to_tablelike_entry()
t2 = HelloTablelikeDTO(4.0, 4.4, 'y', ['a','b'], [0.0, 0.0], NestId(5), [NestId(1), NestId(2)], {'x':'innerx'}, 5, [6, 7]).to_tablelike_entry()
t3 = HelloTablelikeDTO(5.0, 5.5, 'x', ['b','b'], [0.0, 0.0], NestId(5), [NestId(1), NestId(2)], {'x':'innerx'}, 5, [6, 7]).to_tablelike_entry()
tles = [t1, t2, t3]
up_tles = db_client.bulk_create_entries(tles, batch_size=2)
assert(not up_tles is None)
assert(len(up_tles) == len(tles))
for (orig_tle, up_tle) in zip(tles, up_tles):
nid = up_tle.get_nest_id()
assert(not nid is None)
rt_tle = db_client.read_entry(nid)
assert(rt_tle == up_tle)
for att in ['flt_val_0', 'flt_val_1', 'string_val']:
assert(orig_tle.get_value(att) == rt_tle.get_value(att))
print('testing bulk_create_entries_async and simple_filter_query')
db_client.delete_all_entries()
t1 = HelloTablelikeDTO(6.0, 6.6, 'x', ['b','b'], [0.0, 0.0], NestId(5), [NestId(1), NestId(2)], {'x':'innerx'}, 5, [6, 7]).to_tablelike_entry()
t2 = HelloTablelikeDTO(7.0, 7.7, 'z', ['a','b'], [2.0, 0.0], NestId(5), [NestId(1), NestId(2)], {'x':'innerx'}, 5, [6, 7]).to_tablelike_entry()
t3 = HelloTablelikeDTO(8.0, 8.8, 'z', ['b','a'], [0.0, 3.0], NestId(5), [NestId(1), NestId(2)], {'x':'innerx'}, 5, [6, 7]).to_tablelike_entry()
tles = [t1, t2, t3]
print('begin async upload')
upload_count = db_client.bulk_create_entries_async(tles, batch_size=2)
print('upload count returned: ' + str(upload_count))
z_tles = db_client.simple_filter_query({'string_val':'z'})
for z_tle in z_tles:
nid = z_tle.get_nest_id()
assert(not nid is None)
z_tle.set_nest_id(None) #so we can test equivalence easier
assert(len(z_tles) == 2)
assert(t2 in z_tles)
assert(t3 in z_tles)
x_tles = db_client.simple_filter_query({'string_val':'x', 'flt_val_0':6.0})
for x_tle in x_tles:
nid = x_tle.get_nest_id()
assert(not nid is None)
x_tle.set_nest_id(None) #so we can test equivalence easier
assert(len(x_tles) == 1)
assert(t1 in x_tles)
print("testing filter by json attribute. should work b/c it's string matching in the db")
json_tles = db_client.simple_filter_query({'json_val':json.dumps({'x':'innerx'})})
assert(len(json_tles) == 3)
#
# print('testing ensure_entry')
# tK = HelloTablelikeDTO(9.0, 9.9, 'z', ['b','a'], [0.0, 3.0], NestId(5), [NestId(1), NestId(2)], {u'x':u'innerx'}).to_tablelike_entry()
# tk2 = db_client.ensure_entry(tK)
# tk3 = db_client.ensure_entry(tK)
# assert(tk2.get_nest_id() == tk3.get_nest_id())
#
db_client.get_sqla_connection().close()
return
