def test_s3utils_creation_ff_prd():
# TODO: I'm not sure what this is testing, so it's hard to rewrite
# But I fear this use of env 'data' implies the GA test environment has overbroad privilege.
# We should make this work without access to 'data'.
# -kmp 13-Jan-2021
print("In test_s3Utils_creation_ff_prd. It is now", str(datetime.datetime.now()))
def test_prd(ff_production_envname):
util = s3Utils(env=ff_production_envname)
actual_props = {
'sys_bucket': util.sys_bucket,
'outfile_bucket': util.outfile_bucket,
'raw_file_bucket': util.raw_file_bucket,
'url': util.url,
}
assert actual_props == {
'sys_bucket': 'elasticbeanstalk-fourfront-webprod-system',
'outfile_bucket': 'elasticbeanstalk-fourfront-webprod-wfoutput',
'raw_file_bucket': 'elasticbeanstalk-fourfront-webprod-files',
'url': FF_PUBLIC_URL_PRD,
}
test_prd('data')
# NOTE: These values should not be parameters because we don't know how long PyTest caches the
# parameter values before using them. By doing the test this way, we hold the value for as
# little time as possible, making it least risk of being stale. -kmp 10-Jul-2020
prd_beanstalk_env = compute_ff_prd_env()
test_prd(prd_beanstalk_env)
def _deploy_application_to_beanstalk(connection, **kwargs):
""" Creates and returns a CheckResult for 'deploy_application_to_beanstalk'.
NOTE: this does NOT have the @check_function decorator because I'd like
to "call this check" in other checks. The decorator will effectively cast
the resulting check to a dictionary, breaking nested checks. This could be
fixed by refactoring "store_formatted_result". - Will 05/28/2020
"""
check = CheckResult(connection, 'deploy_application_to_beanstalk')
env = kwargs.get('env', 'fourfront-' + DEV_ENV) # by default
branch = kwargs.get('branch', 'master') # by default deploy master
application_version_name = kwargs.get('application_version_name', None)
repo = kwargs.get('repo', None)
# error if we try to deploy prod
if env == compute_ff_prd_env():
check.status = 'ERROR'
check.summary = 'Tried to deploy production env %s from Foursight, aborting.' % env
return check
if application_version_name is None: # if not specified, use branch+timestamp
application_version_name = FOURSIGHT_PREFIX + '-package-%s-%s' % (
branch, datetime.datetime.utcnow())
if repo is not None: # NOTE: if you specify this, assume a CGAP deployment
repo_location = clone_repo_to_temporary_dir(repo, name='cgap-portal')
else:
repo_location = clone_repo_to_temporary_dir()
try:
packaging_was_successful = EBDeployer.build_application_version(
repo_location, application_version_name, branch=branch)
time.sleep(10) # give EB some time to index the new template
if packaging_was_successful:
try:
EBDeployer.deploy_new_version(env, repo_location,
application_version_name)
check.status = 'PASS'
check.summary = 'Successfully deployed version %s to env %s' % (
application_version_name, env)
except Exception as e:
check.status = 'ERROR'
check.summary = 'Exception thrown while deploying: %s' % str(e)
else:
check.status = 'ERROR'
check.summary = 'Could not package repository: %s' % packaging_was_successful
except Exception as e:
check.status = 'ERROR'
check.summary = 'Exception thrown while building application version: %s' % str(
e)
finally:
cleanup_tempdir(repo_location)
return check
def handler(event, context):
'''
Kick off the ff_deploy_staging step function, which will trigger a staging
deployment. The ElasticBeanstalk environments for data and staging are
determined using beanstalk_utils.compute_ff_prd_env, which requires ElasticBeanstalk
IAM permissions
'''
# determine if we are deploying to a production env
event['source_env'] = bs.compute_ff_prd_env()
source_env = event['source_env']
dest_env = None
if is_fourfront_env(source_env) and is_stg_or_prd_env(source_env):
# TODO: Making this cgap-compatible means making an adjustment indicated later in this file. -kmp 10-Apr-2020
dest_env = guess_mirror_env(source_env)
if not dest_env:
# e.g., can get here if:
# - source_env is a cgap environment (downstream isn't yet ready for that)
# - source_env is not a production environment
# - source_env is a production environment with no mirror
return {
"type": "trigger_staging_build",
"torb_message": "invalid event.source_env",
"event": event
}
event['dest_env'] = dest_env
start_time = datetime.now().strftime("%a_%b_%d_%Y__%H%M%S")
run_name = "%s_deploy_for_%s" % (dest_env, start_time)
event['start_time'] = start_time
if event.get('run_name'):
run_name = event.get('run_name') # used for testing
try:
input = make_input(event)
except Exception as e:
return {
"type": "trigger_staging_build",
"torb_message": str(e),
}
# trigger the step function to run
response = client.start_execution(
stateMachineArn=STEP_FUNCTION_ARN,
name=run_name,
input=input,
)
# pop non-JSON serializable stuff...
response.pop('startDate')
return response
def test_compute_ff_prd_env_by_alternate_computation():
assert compute_ff_prd_env() == _ff_production_env_for_testing()
def handler(event, context):
"""
Create a new ElasticBeanstalk environment or update it if it already exists.
Takes a bunch of options from the input JSON and pass them onto
beanstalk_utils.create_bs, which sets environment variables for the EB env
and determines what configuration template to use. Check out the docs on
that function for more specifics.
Also creates a new Foursight environment with PUT to /api/environments.
The PUT body contains Fourfront and Elasticsearch urls.
Adds `prev_bs_version` to the output event, which is the pre-existing
application version of the ElasticBeanstalk environment, if applicable.
"""
logger.info("Before processing overrides: %s" % event)
event = process_overrides(event)
logger.info("After processing overrides: %s" % event)
source_env = get_default(event, 'source_env')
dest_env = get_default(event, 'dest_env')
db_endpoint = get_default(event, 'db_endpoint')
es_url = get_default(event, 'es_url')
dry_run = get_default(event, 'dry_run')
load_prod = get_default(event, 'load_prod')
large_instance_beanstalk = get_default(event, 'large_bs')
fs_url = dest_env
# overwrite db_endpoint potentially if working with staging/data
# specifically, if using the ff_deploy_staging workflow, this code is hit
if is_stg_or_prd_env(source_env) and is_stg_or_prd_env(dest_env):
if not db_endpoint:
db_endpoint = dcicutils.base._FF_GOLDEN_DB
# determine fs_url
fs_url = 'staging'
if dest_env == bs.compute_ff_prd_env():
fs_url = 'data'
retval = {
"type": "create_bs",
"id": dest_env,
"dry_run": dry_run,
"source_env": source_env,
"dest_env": dest_env,
}
if not dry_run:
# Make sure we have the required s3 buckets
try:
bs.create_s3_buckets(dest_env)
except Exception as exc:
logger.info('Error on create_s3_buckets: %s' % exc)
pass
# add the previous beanstalk version to event, which is used in the
# waitfor function down the line
try:
prev_bs_info = bs.beanstalk_info(dest_env)
except Exception as exc:
logger.warning(
'create_beanstalk: could not add prev_bs_version to event. %s'
% exc)
else:
retval['prev_bs_version'] = prev_bs_info['VersionLabel']
# Create or update ElasticBeanstalk environment
res = bs.create_bs(dest_env, load_prod, db_endpoint, es_url,
large_instance_beanstalk)
bs_url = res.get('CNAME')
logger.info("got bs_url as %s" % bs_url)
# we won't always have bs_url
if bs_url:
retval['cname'] = bs_url
fs_res = bs.create_foursight(dest_env, bs_url, es_url, fs_url)
logger.info("Created foursightenv with result: %s " % fs_res)
else:
logger.info(
"No beanstalk endpoint yet, can't create foursight env")
logger.info(res)
return retval