def _generate_deploy_params(runlevel):
if runlevel == RunLevel.development_instance():
supplemental_config = _generate_development_params()
elif runlevel == RunLevel.production_instance():
supplemental_config = _generate_production_params()
else:
raise Exception("Unsupported runlevel: " + str(runlevel))
return supplemental_config
def _perform_project_stages(project_env, project_root_dir):
"""
performs the stages of an individual project: docker startup,
seed_users, smoke_tests, docker teardown
takes a ProjectEnv
Returns a list of stage_results (the objects returned
by _perform_stage)
"""
stage_results = list()
project_name = project_env.get_project_name()
stage_name = project_name + ' web_assets:dist'
client_dist_stage = _perform_stage(stage_name, compile_ops._compile,
project_root_dir, 'web_assets:dist',
ProjectEnv.hello_world_instance(),
RunLevel.development_instance())
stage_results.append(client_dist_stage)
stage_name = project_name + ' startup containers'
startup_containers_stage = _perform_stage(stage_name,
docker_ops._docker_action,
'startup', 'all', project_env,
RunLevel.development_instance(),
NestSite.localhost_instance(),
project_root_dir)
stage_results.append(startup_containers_stage)
stage_name = project_name + ' ensure db tables'
db_tables_project_stage = _perform_stage(stage_name, db_ops._run_db_action,
'ensure_tables', project_env,
NestSite.localhost_instance())
stage_results.append(db_tables_project_stage)
stage_name = project_name + ' seed_users'
seed_users_project_stage = _perform_stage(
stage_name, seed_users_ops._run_seed_users_script, project_env,
RunLevel.development_instance())
stage_results.append(seed_users_project_stage)
stage_name = project_name + ' smoke_test'
smoke_project_stage = _perform_stage(stage_name,
smoke_test_ops._run_smoke_test,
project_env,
NestSite.localhost_instance())
stage_results.append(smoke_project_stage)
stage_name = project_name + ' teardown containers'
teardown_containers_stage = _perform_stage(stage_name,
docker_ops._docker_action,
'teardown', 'all', project_env,
RunLevel.development_instance(),
NestSite.localhost_instance(),
project_root_dir)
stage_results.append(teardown_containers_stage)
return stage_results
def make_flask_app():
project_env = ProjectEnv.detect_from_os(fallback_to_default=True)
runlevel = RunLevel.detect_from_os(fallback_to_default=True)
config = nest_config.generate_config(project_env, runlevel)
print('make flask app: ' + str(project_env) + ' ' + str(runlevel))
app = create_app(config, project_env, runlevel)
return app
def test_deploy_os_roundtrip():
#delete Deploy_ENV if it's currently set
os.environ.pop("DEPLOY_ENV", None)
env_obs = RunLevel.detect_from_os()
assert env_obs == None
env_obs = RunLevel.detect_from_os(fallback_to_default=True)
assert env_obs == RunLevel.default_instance()
env1 = RunLevel.default_instance()
assert not env1 == None
env1.write_to_os()
env_obs = RunLevel.detect_from_os()
assert env1 == env_obs
return
def init_crud_clients():
config = nest_config.generate_config(ProjectEnv.knoweng_instance(),
RunLevel.development_instance())
init_token_maker(config['JWT_SECRET'], config['JWT_ISSUER'],
config['JWT_AUDIENCES'])
global SCHEMA_REGISTRY
SCHEMA_REGISTRY = knoweng_schemas.get_schemas()
global CLIENT_REGISTRY
CLIENT_REGISTRY = dict()
#make db clients
#TODO: knoweng should declare the db it's using, but for now
#the default postgres container on localhost is all there is,
#which is what the global db engine defaults to.
engine = nest_db.get_global_sqlalchemy_engine()
sqla_md = nest_db.get_global_sqlalchemy_metadata()
db_client_makers = knoweng_db.get_sqla_makers()
for name in DB_COLLECTIONS:
cm = db_client_makers[name]
client = cm.get_db_client(engine, sqla_md)
CLIENT_REGISTRY[name] = client
#make api clients
http_client = get_http_client()
api_client_makers = knoweng_api_clients.get_api_client_makers()
for name in API_COLLECTIONS:
cm = api_client_makers[name]
crud_client = cm.get_crud_client(http_client)
CLIENT_REGISTRY[name] = crud_client
return
def test_deploy_instances():
development_env = RunLevel.development_instance()
production_env = RunLevel.production_instance()
development_rt = RunLevel(development_env.get_runlevel_name())
production_rt = RunLevel(production_env.get_runlevel_name())
assert development_env == development_rt
assert development_env == development_env
assert not development_env == production_env
assert production_env == production_rt
s = (str(development_env))
print(s)
assert s.endswith('development')
assert development_env == RunLevel('development')
return
def test_dev_config():
run_level = RunLevel.development_instance()
project_env = ProjectEnv.knoweng_instance()
config = nest_config.generate_config(project_env, run_level)
app = create_app(config, project_env, run_level)
if app is None:
assert False, "create_app returned None"
assert app.config['ENV'] == 'dev'
assert app.config['DEBUG'] is True
return
def generate_config_from_os():
"""
Determines runlevel and project_env from environment variables, then
returns generate_eve_config(project_env, runlevel).
"""
project_env = ProjectEnv.detect_from_os(fallback_to_default=True)
print "Detected PROJECT_ENV: " + str(project_env)
runlevel = RunLevel.detect_from_os(fallback_to_default=True)
print "Detected RUN_LEVEL: " + str(runlevel)
return generate_config(project_env, runlevel)
def prep_db():
"""
test_users has methods to point the nest_users clients to a
different table so that any existing user entries aren't disturbed
on the local postgres instance. We also seed the hello_world project
users into the testing table.
"""
test_users.setup_db()
runlevel = RunLevel.development_instance()
project_env = ProjectEnv.hello_world_instance()
db_ops_utils.seed_users(project_env, runlevel)
return
def test_production_config():
run_level = RunLevel.production_instance()
project_env = ProjectEnv.knoweng_instance()
config = nest_config.generate_config(project_env, run_level)
app = create_app(config, project_env, run_level)
if app is None:
assert False, "create_app returned None"
assert app.config['ENV'] == 'prod'
assert app.config['DEBUG'] is False
assert app.config['DEBUG_TB_ENABLED'] is False
return
def test_all_config_combos():
"""
just making sure they can all be generated
"""
all_project_names = nest_envs.VALID_PROJECT_NAMES
all_run_levels = nest_envs.VALID_RUNLEVEL_NAMES
for project_name in all_project_names:
project_env = ProjectEnv(project_name)
for run_level in all_run_levels:
run_env = RunLevel(run_level)
config = nest_config.generate_config(project_env, run_env)
return
def _run_compile_cmd(arg_map):
"""
translates arguments from commandline to calls
to python methods. Input is the output from
argparse.parse_args(), output is an exit code
indicating if the compilation succeeded.
"""
code_type = arg_map['code_type']
project_root_dir = arg_map['project_root_dir']
project_env_name = arg_map['project']
runlevel_name = arg_map['runlevel']
project_env = ProjectEnv.from_string(project_env_name)
runlevel = RunLevel.from_string(runlevel_name)
exit_code = _compile(project_root_dir, code_type, project_env, runlevel)
return exit_code
def app():
from nest_py.core.flask.app2 import create_app
run_level = RunLevel.development_instance()
project_env = ProjectEnv.knoweng_instance()
config = nest_config.generate_config(run_level, project_env)
#force localhost as the db server
host = NestSite.localhost_instance().get_server_ip_address()
config['host'] = host
_app = create_app(config, project_env, run_level)
ctx = _app.test_request_context()
ctx.push()
yield _app
ctx.pop()
def _run_seed_users_cmd(arg_map):
"""
translates arguments from commandline to calls to python methods. Input is
the output from argparse.parse_args(), output is an exit code indicating if
the job succeeded.
"""
project_root_dir = arg_map['project_root_dir']
project_env_name = arg_map['project']
project_env = ProjectEnv.from_string(project_env_name)
runlevel_name = arg_map['runlevel']
runlevel = RunLevel.from_string(runlevel_name)
#target_site_name = arg_map['site']
#target_site = NestSite.from_string(target_site_name)
exit_code = _run_seed_users_script(project_env, runlevel)
return exit_code
def generate_db_config(project_env=None, runlevel=None):
if project_env is None:
project_env = ProjectEnv.hello_world_instance()
if runlevel is None:
runlevel = RunLevel.development_instance()
config = {
"user":os.getenv('POSTGRES_USERNAME', "nest"),
"port": os.getenv('POSTGRES_PORT', 5432), #exported in docker startup
"password":os.getenv('POSTGRES_PASSWORD', "GARBAGESECRET"),
"db_name":os.getenv('POSTGRES_DATABASE', "nest"),
#"verbose_logging":True
"verbose_logging":False
}
host = os.getenv('POSTGRES_HOST', NestSite.localhost_instance().get_server_ip_address())
config['host'] = host
return config
def _run_docker_cmd(arg_map):
"""
translates arguments from commandline to calls to python methods. Input is
the output from argparse.parse_args(), output is an exit code indicating if
the compilation succeeded.
"""
project_root_dir = arg_map['project_root_dir']
action = arg_map['action']
project_env_name = arg_map['project']
project_env = ProjectEnv.from_string(project_env_name)
target_site_name = arg_map['site']
target_site = NestSite.from_string(target_site_name)
runlevel_name = arg_map['runlevel']
runlevel = RunLevel.from_string(runlevel_name)
service = arg_map['service']
exit_code = _docker_action(action, service, project_env, \
runlevel, target_site, project_root_dir)
return exit_code
def _test_seed_users_for_project(project_env):
test_users.setup_db()
runlevel = RunLevel.development_instance()
db_ops_utils.seed_users(project_env, runlevel)
user_configs = nest_config.generate_seed_users(project_env, runlevel)
assert (len(user_configs) > 0)
users_client = test_users.make_users_db_client()
print("DB CLIENT USER: " + str(users_client.requesting_user))
#an empty filter dict should return all entries in the db
all_user_tles = users_client.simple_filter_query({})
assert (not all_user_tles is None)
#the +1 is for the system_user which is hardcoded, not part of configs
assert (len(all_user_tles) == (len(user_configs) + 1))
#running again, it should detect them and not do anything
assert (db_ops_utils.seed_users(project_env, runlevel))
test_users.finish_up()
return
def build_authenticator(flask_app, project_env, runlevel):
users_sqla_maker = core_db.get_nest_users_sqla_maker()
db_engine = nest_db.get_global_sqlalchemy_engine()
md = nest_db.get_global_sqlalchemy_metadata()
users_client = users_sqla_maker.get_db_client(db_engine, md)
#the authenticator will interact with the local db as the master system_user
auth_user = core_db.get_system_user()
users_client.set_requesting_user(auth_user)
#knoweng uses hubzero to look up user accounts in production
#all other situations will use user accounts stored in the local db
use_hubzero = (ProjectEnv.knoweng_instance() == project_env
and RunLevel.production_instance() == runlevel)
if use_hubzero:
print('registering Hubzero authenticator')
from nest_py.knoweng.flask.accounts.knoweng_authentication import HubzeroAuthenticationStrategy
authenticator = HubzeroAuthenticationStrategy(flask_app, users_client)
else:
from nest_py.core.flask.accounts.authentication import NativeAuthenticationStrategy
authenticator = NativeAuthenticationStrategy(flask_app, users_client)
return authenticator
def test_deploy_bad_name():
with pytest.raises(Exception):
env = RunLevel("not a real Deploy name")
return
def get_nest_endpoints(db_engine, sqla_metadata, authenticator):
all_eps = NestEndpointSet()
schema_registry = knoweng_schemas.get_schemas()
sqla_registry = knoweng_db.get_sqla_makers()
##STANDARD CRUD ENDPOINTS
standard_eps = _make_standard_crud_endpoints(schema_registry,
sqla_registry, authenticator,
db_engine, sqla_metadata)
all_eps.add_endpoint_set(standard_eps)
##CLIENT LOGGING
logging_endpoint = LoggingEndpoint(authenticator)
all_eps.add_endpoint(logging_endpoint)
####FILES
files_db_client = sqla_registry[files.COLLECTION_NAME].get_db_client(
db_engine, sqla_metadata)
files_endpoints_set = files_endpoints.get_endpoint_set(
files_db_client, authenticator)
all_eps.add_endpoint_set(files_endpoints_set)
####FILE DOWNLOADS
files_download_endpoint = FileDownloadsEndpoint(files_db_client,
authenticator)
all_eps.add_endpoint(files_download_endpoint)
####PROJECTS
projects_db_client = sqla_registry[projects.COLLECTION_NAME].get_db_client(
db_engine, sqla_metadata)
projects_endpoints_set = projects_endpoints.get_endpoint_set(
projects_db_client, authenticator)
all_eps.add_endpoint_set(projects_endpoints_set)
###JOBS
jobs_db_client = sqla_registry[jobs.COLLECTION_NAME].get_db_client(
db_engine, sqla_metadata)
jobs_endpoints_set = jobs_endpoints.get_endpoint_set(
jobs_db_client, authenticator)
all_eps.add_endpoint_set(jobs_endpoints_set)
###JOB DOWNLOADS
jobs_download_endpoint = JobDownloadsEndpoint(jobs_db_client,
authenticator)
all_eps.add_endpoint(jobs_download_endpoint)
##LOGIN
sessions_endpoint = SessionsEndpoint(authenticator)
all_eps.add_endpoint(sessions_endpoint)
project_env_str = os.getenv('PROJECT_ENV', 'knoweng')
runlevel_str = os.getenv('NEST_RUNLEVEL', 'development')
project_env = ProjectEnv(project_env_str)
runlevel = RunLevel(runlevel_str)
#FIXME: lock DB to prevent duplication errors
db_ops_utils.ensure_tables_in_db()
db_ops_utils.seed_users(project_env, runlevel)
exit_code = _run_seed_cmd({'project': project_env_str})
if exit_code != 0:
print('WARNING: non-zero exit code from seed command:' + exit_code)
return all_eps
def build_and_test(project_root_dir):
"""
run all compilation and unit tests
"""
stage_results = list()
log("START ci_ops.BUILD_AND_TEST")
py_compile_stage = _perform_stage('python compilation ',
compile_ops._compile, project_root_dir,
'python', ProjectEnv.default_instance(),
RunLevel.development_instance())
stage_results.append(py_compile_stage)
js_compile_stage = _perform_stage('web_assets compilation',
compile_ops._compile, project_root_dir,
'web_assets',
ProjectEnv.hello_world_instance(),
RunLevel.development_instance())
stage_results.append(js_compile_stage)
build_containers_stage = _perform_stage('build containers ',
docker_ops._docker_action, 'build',
'all',
ProjectEnv.hello_world_instance(),
RunLevel.development_instance(),
NestSite.localhost_instance(),
project_root_dir)
stage_results.append(build_containers_stage)
#run pytests and clienttests against the hello_world_app containers
startup_containers_stage = _perform_stage(
'startup containers ', docker_ops._docker_action, 'startup', 'all',
ProjectEnv.hello_world_instance(), RunLevel.development_instance(),
NestSite.localhost_instance(), project_root_dir)
stage_results.append(startup_containers_stage)
clienttest_stage = _perform_stage('clienttest ',
clienttest_ops._run_unit_test,
project_root_dir)
stage_results.append(clienttest_stage)
pytest_stage = _perform_stage('pytest tests/unit/ ',
pytest_ops._run_unit_test, project_root_dir,
True)
stage_results.append(pytest_stage)
teardown_containers_stage = _perform_stage(
'teardown containers ', docker_ops._docker_action, 'teardown', 'all',
ProjectEnv.hello_world_instance(), RunLevel.development_instance(),
NestSite.localhost_instance(), project_root_dir)
stage_results.append(teardown_containers_stage)
#test the lifecycle and smoke scripts of all projects
project_names = nest_envs.VALID_PROJECT_NAMES
for project_name in project_names:
project_env = ProjectEnv(project_name)
project_stage_results = _perform_project_stages(
project_env, project_root_dir)
stage_results += project_stage_results
exit_code = _finalize_build(stage_results)
log("BUILD_AND_TESTS returning exit_code: " + str(exit_code))
return exit_code