def _run_docker_shell_script(script_name,
docker_dir,
trailing_args=None,
env_variables=dict()):
"""
script_name (String) filename of a script in the nest/docker/ directory
docker_dir (String) directory of docker scripts in the nest repo. expected
to be /code_live/docker, but may be different is run outside
of the nest_ops container
env_variables (dict of string->string): will be set as commandline env
variables in the shell that the script is run
"""
#sh cannot be broken into its own list element or you will be dropped
#into a shell
cmd_ary = [('sh ' + script_name)]
if trailing_args is not None:
cmd_ary.extend(trailing_args)
cmd = " ".join(cmd_ary)
log("Executing command: " + str(cmd))
cr = container_users.make_host_user_command_runner()
cr.set_working_dir(docker_dir)
cr.add_env_variables(env_variables)
result = cr.run(cmd, stream_log=True)
exit_code = result.get_exit_code()
return exit_code
def _make_user_inside_container(host_user_name, host_user_id):
cr = make_root_command_runner()
#the group ids of the users in the containers aren't the same
#as the uid, which is how they are on host machines.
#don't know why. This was causing
#problems on jenkins where the gid was already in use. we don't
#ever use the user's gid in the container (we always use the docker
#group's gid), so it is safe to make it a much larger number as
#we'll never use it
user_gid = 500 + int(host_user_id)
#http://askubuntu.com/a/94067
cmd = 'adduser --disabled-password --gecos \"\"'
cmd += ' --uid ' + str(host_user_id)
cmd += ' --ingroup docker ' #+ str(user_gid)
cmd += ' ' + host_user_name
res = cr.run(cmd)
if not res.succeeded():
log(str(res))
raise Exception("Failed to add user '" + host_user_name +
"' inside the container")
#ssh complains a lot if it can't make a known_hosts file
ssh_dir = '/home/' + host_user_name + '/.ssh'
res = cr.run('mkdir ' + ssh_dir + '; touch ' + ssh_dir + '/known_hosts')
#log(str(res))
res = cr.run('chown -R ' + host_user_name + ':' + 'docker ' + ssh_dir)
#log(str(res))
return
def _compile_web_assets_ts(project_root_dir, runlevel):
"""Build the client assets."""
clientdir = os.path.join(project_root_dir, 'client')
gulppath = os.path.join(clientdir, 'node_modules', '.bin', 'gulp')
log("running gulp from " + str(clientdir))
exit_sum = 0
# clean and build
cr = container_users.make_host_user_command_runner()
cr.set_working_dir(clientdir)
# Pass runlevel down into gulp
runlevel.write_to_os()
# Pass HUBZERO_APPLICATION_HOST and MAX_CONTENT_LENGTH
# down into gulp (knoweng only)
project_params = knoweng_config.generate_project_params(runlevel)
hz_host = project_params['HUBZERO_APPLICATION_HOST']
if hz_host != 'FIXME':
os.environ['HUBZERO_APPLICATION_HOST'] = hz_host
os.environ['MAX_CONTENT_LENGTH'] = str(
project_params['MAX_CONTENT_LENGTH'])
cmd = gulppath + " clean"
res = cr.run(cmd, stream_log=True)
exit_sum += res.get_exit_code()
cmd = gulppath + " dist"
res = cr.run(cmd, stream_log=True)
exit_sum += res.get_exit_code()
return exit_sum
def _run_remote_maintenance_commands(command_runner):
"""
command_runner(CommandRunner): The command_runner should
be configured to run against the machine and user desired.
remote_maintenance shuts down the nest app on the remote
machine and runs the script at ci_scripts/nightly_maintenance.sh
This command will run in the directory called ~/nest_releases/current,
which must already exist and have a valid deployment installed.
"""
cr = command_runner
try:
_run_step(cr, 'whoami')
current_deploy_dir = '~/nest_releases/current/'
cr.set_remote_working_dir(current_deploy_dir)
_run_step(cr,'./nest_ops docker teardown --service=all')
#this seems to always 'fail' if it can't delete 100% of the docker cache,
#but it is still reclaiming the vast majority of the disk space used
#by docker, so don't require it to succeed
_run_step(cr,'sh ./ci_scripts/nightly_maintenance.sh', must_succeed=False)
exit_code = 0
except Exception as e:
log("remote_maintenance Failed")
log(str(e))
exit_code = 1
return exit_code
def _finalize_build(stage_results):
"""
stage_results (list of (stage_exit_code, stage_name, stage_status)) as returned by
_perform_stage()
logs a final report and returns an overall exit code for whether the
build succeeded.
"""
overall_success = True
final_report = ""
for stage_result in stage_results:
stage_success, x, y = stage_result
overall_success = overall_success and stage_success
final_report += _format_stages_summary(stage_results)
final_report += '\n'
if overall_success:
final_report += "BUILD SUCCEEDED"
exit_code = 0
else:
final_report += "BUILD FAILED"
exit_code = 1
log(final_report)
return exit_code
def _run_step(command_runner, remote_command, must_succeed=True):
res = command_runner.run(remote_command)
if must_succeed and not res.succeeded():
raise Exception("Failed Deploy Step: " + str(res))
else:
log(str(res))
return
def _run_seed_users_script(project_env, runlevel):
"""
"""
db_engine = _make_db_engine(project_env)
try:
db_ops_utils.seed_users(project_env, runlevel)
exit_code = 0
except Exception as e:
log('ERROR: ' + str(e))
exit_code = 1
return exit_code
def _perform_stage(stage_name, stage_fun, *args):
"""
takes a function that performs a build stage and returns an exit_code
(where the exit code is 0 on success and >0 otherwise)
Also takes any arguments that need to be fed into that function.
Returns a tuple of (success:boolean, stage_name:string, status_msg:string).
This provides a common way of doing hardened error checking so that the
ci job will always finish and print Failure, regardless of how
bad the errors are. Also makes consistent reporting.
"""
log(str(80 * '#'))
log("CI STAGE BEGIN: " + stage_name)
try:
exit_code = stage_fun(*args)
if exit_code == 0:
success = True
status_msg = 'SUCCEEDED'
else:
success = False
status_msg = 'FAILED with exit_code: ' + str(exit_code)
except Exception as e:
status_msg = "FAILED with exception: " + str(e)
success = False
log("CI STAGE END : " + stage_name + ' :: ' + str(status_msg))
log(str(80 * '#'))
ret_pair = (success, stage_name, status_msg)
return ret_pair
def _run_smoke_test(project_env, target_site):
"""
runs the smoke_test subystem. logs the final result and returns
a 0/1 exit code based on whether all walkthroughs worked.
"""
http_client = target_site.build_http_client()
smoke_res = run_project_scripts(project_env, http_client)
if smoke_res.did_succeed():
exit_code = 0
else:
exit_code = 1
rpt = smoke_res.get_full_report()
log(rpt)
return exit_code
def run(self, remote_bash_command_str):
"""
run a bash command from the current python process as
the remote user on the remote machine, in the remote working_dir
This will be accomplished by running an ssh command from
the local machine (probably nest_ops container), as
the local user, in the local working_dir
"""
#log("Executing command: " + str(remote_bash_command_str))
ssh_cmd = self._build_full_ssh_cmd(remote_bash_command_str)
log("Executing as remote command: " + str(ssh_cmd))
res = self.local_runner.run(ssh_cmd)
#log('Last command exit code: ' + str(res.get_exit_code()))
return res
def _compile_python(project_root_dir):
try:
from pylint.lint import Run
rcfile = os.path.join(project_root_dir, '.pylintrc')
cmd = 'pylint '
cmd += '--errors-only '
cmd += '--rcfile ' + rcfile
cmd += ' nest_py'
cr = container_users.make_host_user_command_runner()
res = cr.run(cmd, stream_log=True)
exit_code = res.get_exit_code()
except Exception as e:
log("linting crashed: " + str(e))
traceback.print_exc()
exit_code = 1
return exit_code
def _docker_check_startup(service_names, nest_site, timeout_secs=180):
"""
Confirms the services in service_names are running.
Arguments:
service_names (list): List of names of services to check.
nest_site (string): The deployment site name.
timeout_secs (float): The maximum time in seconds to wait; actual wait
time may slightly exceed timeout_secs.
Returns:
int: the number of services whose statuses are failure or still unknown
after the timeout expires
"""
exit_sum = 0
# to_check contains the names of services whose statuses we haven't yet
# determined; we'll poll them, removing items as each is determined, until
# the list is empty or we reach the timeout
to_check = list(service_names)
start_time = time.time()
# fyi, some ways we can exceed timeout_secs:
# - while condition checks remaining time, which might be < 1s, but sleep
# always sleeps 1s
# - sleep may take longer than requested
# - checks on individual services may take significant time, especially if
# they have their own timeout/retry behavior
def elapsed_time():
"""Returns time elapsed since start_time, measured in seconds."""
return time.time() - start_time
while len(to_check) > 0 and (elapsed_time() < timeout_secs):
time.sleep(1) # seconds
# iterating over dict of service/status pairs makes it easy to log
# changes and remove from to_check
statuses = {service: _docker_check_status(\
service, nest_site, timeout_secs - elapsed_time()) \
for service in to_check}
current_time = time.ctime()
for service, status in statuses.iteritems():
if status == 'success':
log(service + " ready at " + current_time + \
" (" + str(elapsed_time()) + " sec)")
to_check.remove(service)
elif status == 'failure':
log(service + " failed at " + current_time)
exit_sum += 1
to_check.remove(service)
elif status == 'unknown':
pass
else:
raise ValueError("Bad status " + status + " for " + service)
if len(to_check) > 0:
log("Timeout expired at " + time.ctime() + \
" before startup tests complete: " + str(to_check))
exit_sum += len(to_check)
return exit_sum
def _exec(self, bash_cmd, stream_log, interactive):
"""
Actually runs a bash cmd as is. Always captures stdout and stderr
and saving it in the returned BashResult. Optionally also prints
the output as it occurs (stream_log=True)
This was a huge pain to get working. Be careful if you touch this.
"""
if interactive:
#TODO: this works but always returns an error code
#I don't think it matters that interactive commands
#can't have their stdout captured
text_output = ''
exit_code = os.system(bash_cmd)
else:
p = subprocess.Popen(bash_cmd,
shell=True,
bufsize=1,
stderr=subprocess.STDOUT,
stdout=subprocess.PIPE)
text_buffer = StringIO()
for line in iter(p.stdout.readline, b''):
if stream_log:
isolated_line = line.strip() #get rid of trailing newline
log(isolated_line)
text_buffer.write(line)
p.stdout.close()
text_buffer.flush()
p.wait()
exit_code = p.returncode
text_output = text_buffer.getvalue()
text_buffer.close()
result = BashResult(bash_cmd, exit_code, text_output)
#log(str(result))
return result
def wipe_by_api_clients(api_client_makers, http_client):
"""
api_client_makers(list of ApiClientMaker)
http_client(NestHttpClient)
"""
exit_code = 0
for cm in api_client_makers.values():
log("wiping collection endpoint: " + cm.get_collection_name())
api_client = cm.get_crud_client(http_client)
resp = api_client.response_of_delete_all_entries()
if resp.did_succeed():
num_deleted = resp.get_data_payload_as_jdata()['num_deleted']
log(" .. deleted: " + str(num_deleted))
else:
log(" ... FAILED")
log(resp.get_error_message())
exit_code = 1
return exit_code
def _compile_web_assets_npm(project_root_dir):
"""(Re)install the node modules."""
clientdir = os.path.join(project_root_dir, 'client')
modulesdir = os.path.join(clientdir, 'node_modules')
if os.path.isdir(modulesdir):
log("removing " + str(modulesdir))
try:
rmtree(modulesdir)
except OSError as exception:
log(exception.strerror + ": " + exception.filename)
return 1
log("installing node modules under " + str(clientdir))
cmd = 'npm i'
cr = container_users.make_host_user_command_runner()
cr.set_working_dir(clientdir)
res = cr.run(cmd, stream_log=True)
return res.get_exit_code()
def _docker_action(action, service, project_env, runlevel, \
nest_site, project_root_dir):
"""
action (String)
service (String)
project_env (ProjectEnv)
runlevel (RunLevel)
nest_site (NestSite)
docker_dir (String)
"""
docker_dir = os.path.join(project_root_dir, 'docker')
log('Performing Docker action:')
log(' action=' + str(action))
log(' service=' + str(service))
log(' ' + str(project_env))
log(' ' + str(runlevel))
log(' ' + str(nest_site))
log(' ' + str(docker_dir))
if 'all' == service:
service_names = list(VALID_DOCKER_SERVICES)
service_names.remove('all')
else:
service_names = [service]
if 'build' == action:
exit_code = _docker_build(service_names, project_env, runlevel, \
docker_dir)
elif 'startup' == action:
# we could check status inside _docker_startup, but for the case where
# we're starting multiple services, it saves us a little time to call
# start on them all first, then go back and check their statuses
exit_code = _docker_startup(service_names, project_env, runlevel, \
nest_site, docker_dir, project_root_dir)
exit_code += _docker_check_startup(service_names, nest_site)
elif 'teardown' == action:
exit_code = _docker_teardown(service_names, project_env, runlevel, \
nest_site, docker_dir)
else:
log("Unknown docker action: '" + str(action) + "'")
exit_code = 1
return exit_code
def build_and_test(project_root_dir):
"""
run all compilation and unit tests
"""
stage_results = list()
log("START ci_ops.BUILD_AND_TEST")
py_compile_stage = _perform_stage('python compilation ',
compile_ops._compile, project_root_dir,
'python', ProjectEnv.default_instance(),
RunLevel.development_instance())
stage_results.append(py_compile_stage)
js_compile_stage = _perform_stage('web_assets compilation',
compile_ops._compile, project_root_dir,
'web_assets',
ProjectEnv.hello_world_instance(),
RunLevel.development_instance())
stage_results.append(js_compile_stage)
build_containers_stage = _perform_stage('build containers ',
docker_ops._docker_action, 'build',
'all',
ProjectEnv.hello_world_instance(),
RunLevel.development_instance(),
NestSite.localhost_instance(),
project_root_dir)
stage_results.append(build_containers_stage)
#run pytests and clienttests against the hello_world_app containers
startup_containers_stage = _perform_stage(
'startup containers ', docker_ops._docker_action, 'startup', 'all',
ProjectEnv.hello_world_instance(), RunLevel.development_instance(),
NestSite.localhost_instance(), project_root_dir)
stage_results.append(startup_containers_stage)
clienttest_stage = _perform_stage('clienttest ',
clienttest_ops._run_unit_test,
project_root_dir)
stage_results.append(clienttest_stage)
pytest_stage = _perform_stage('pytest tests/unit/ ',
pytest_ops._run_unit_test, project_root_dir,
True)
stage_results.append(pytest_stage)
teardown_containers_stage = _perform_stage(
'teardown containers ', docker_ops._docker_action, 'teardown', 'all',
ProjectEnv.hello_world_instance(), RunLevel.development_instance(),
NestSite.localhost_instance(), project_root_dir)
stage_results.append(teardown_containers_stage)
#test the lifecycle and smoke scripts of all projects
project_names = nest_envs.VALID_PROJECT_NAMES
for project_name in project_names:
project_env = ProjectEnv(project_name)
project_stage_results = _perform_project_stages(
project_env, project_root_dir)
stage_results += project_stage_results
exit_code = _finalize_build(stage_results)
log("BUILD_AND_TESTS returning exit_code: " + str(exit_code))
return exit_code
def _run_deploy_commands(project_env, git_tag_or_branch, command_runner):
"""
command_runner(CommandRunner): The command_runner should
be configured to run against the machine and user desired.
deploys the code of project project_env, at git revision
git_tag_or_branch, using the given command_runner to run
the individual bash commands.
This command will create a directory called
~/nest_releases/nest_<git_sha>/ and launch the
requested projected from there.
"""
cr = command_runner
try:
_run_step(cr, 'whoami')
# the -p means it won't error if it's already there
_run_step(cr, 'mkdir -p ~/nest_releases')
#TODO: maintaining these data directories through .gitkeep is starting to
#get creaky. this is to ensure they exist in a 'shared' space
#which will live across releases
shared_data_dir = '~/nest_releases/persistent_data/'
_run_step(cr, 'mkdir -p ' + shared_data_dir + 'userfiles')
_run_step(cr, 'mkdir -p ' + shared_data_dir + 'db')
_run_step(cr, 'mkdir -p ' + shared_data_dir + 'knoweng')
install_dir = '~/nest_releases/nest_' + git_tag_or_branch
path_to_current_dir = '~/nest_releases/current'
project_name = project_env.get_project_name()
#always delete the install_dir if it exists. it means
#we're redeploying something that was deployed before
_run_step(cr, 'rm -rf ' + install_dir)
repo = '[email protected]:arivisualanalytics/nest.git'
_run_step(
cr, 'git clone -b ' + git_tag_or_branch + ' --single-branch ' +
repo + ' ' + install_dir)
#replace the soft link to ~/nest_releases/current with a link to what we
#just installed
_run_step(cr, 'rm ' + path_to_current_dir, must_succeed=False)
_run_step(cr, 'ln -s ' + install_dir + ' ' + path_to_current_dir)
#replace the data/ dir with a link to persistent_data/
cr.set_remote_working_dir(install_dir)
_run_step(cr, 'rm -rf data && ln -s ' + shared_data_dir + ' data')
#now do standard build and launch
docker_dir = install_dir + '/docker/'
cr.set_remote_working_dir(docker_dir)
_run_step(cr, 'sh build_nest_ops.sh')
cr.set_remote_working_dir(install_dir)
_run_step(cr, './nest_ops compile --project=' + project_name)
_run_step(
cr,
'./nest_ops docker build --service=all --project=' + project_name)
#stop the previous installation if it's running
_run_step(cr, './nest_ops docker teardown --service=all')
_run_step(
cr, './nest_ops docker startup --service=all --project=' +
project_name)
_run_step(cr, './nest_ops db ensure_tables --project=' + project_name)
_run_step(cr, './nest_ops seed_users --project=' + project_name)
#Instruct the target machine to run it's smoke tests against itself.
#Do it this way in case we are deploying code with different smoke tests
_run_step(
cr,
'./nest_ops smoke_test --site=localhost --project=' + project_name)
exit_code = 0
except Exception as e:
log("Deploy Failed")
log(str(e))
exit_code = 1
return exit_code