def prepare_resources_json_from_terraform_outs():
# Add all module resources to root module resources json
def terraform_json_to_m2a_resources_json(terraform_json):
return {
output_key: terraform_json[output_key]['value']
for output_key in list(terraform_json.keys())
}
(root_module_resources_json,
_) = run.cmd('terraform output -json',
cwd=TERRAFORM_SRC_DIR,
env_vars=os.environ,
realtime_stdout_pipe=sys.stdout.write)
root_module_resources = terraform_json_to_m2a_resources_json(
json.loads(root_module_resources_json))
module_file = '{}/terraform/{}/modules.tf'.format(CURR_WORK_DIR,
environment)
terraform_module_paths = get_module_paths(module_file)
terraform_modules = get_terraform_modules(module_file)
print('{}Retrieving terraform outputs for modules {}'.format(
bcolors.OKBLUE, terraform_modules))
terraform_module_resources = utils_terraform.terraform_outputs(
terraform_modules, TERRAFORM_SRC_DIR)
module_resources = terraform_json_to_m2a_resources_json(
terraform_module_resources)
root_module_resources.update(module_resources)
resources_json = root_module_resources
print('{}resources.json of all modules:\n{}'.format(
bcolors.OKGREEN, json.dumps(resources_json, indent=2)))
return (terraform_module_paths, resources_json)
def sync_files(boto3_session, component_path, component_svc_info,
resources_json, env):
s3rsc = boto3_session.resource('s3')
deployment_bucket = resources_json[component_svc_info['terraformBucket']]
resources_key = component_svc_info.get('terraformResourcesKey')
bucket = s3rsc.Bucket(deployment_bucket)
if resources_key:
print('Syncing resources.json to S3 key: {}'.format(resources_key))
bucket.put_object(Key=resources_key,
Body=io.BytesIO((json.dumps(resources_json,
indent=4,
sort_keys=True))))
real_component_path = os.path.realpath(component_path)
print('Syncing {} to S3 bucket: {}'.format(real_component_path,
deployment_bucket))
run.cmd("aws --profile {} s3 sync {} s3://{}/".format(
env, real_component_path, deployment_bucket),
realtime_stdout_pipe=sys.stdout.write)
def generate_and_sync_env_report_file(env, region, report_path=os.getcwd()):
try:
if os.path.exists("{}/env_report/".format(report_path)):
print("Deleting previous report files...")
run.cmd("rm -rf {}/env_report/".format(report_path))
print("Creating report file...")
print("Running command: Scout2 --profile {} --regions {} --report-dir {}/env_report/ " \
"--no-browser".format(env, region, report_path))
run.cmd(
"Scout2 --profile {} --regions {} --report-dir {}/env_report/ --no-browser"
.format(env, region, report_path))
except:
print(("Error while producing report for {}".format(env)))
pass
try:
print("Renaming report file...")
run.cmd(
"mv {}/env_report/report-{}.html {}/env_report/report.html".format(
report_path, env, report_path))
print(
"Syncing report to s3://m2a-dashboard-{}-ui/public/report".format(
env))
run.cmd(
"aws --profile {} s3 sync {}/env_report/ s3://m2a-dashboard-{}-ui/public/report/"
.format(env, report_path, env))
print("Deleting local report files...")
run.cmd("rm -rf {}/env_report/".format(report_path))
except:
print("Error uploading report file {}".format(report_path))
pass
def init_remote_state():
run.cmd('terraform init',
cwd=TERRAFORM_SRC_DIR,
realtime_stdout_pipe=sys.stdout.write)
current_modules_dir = '{}/.terraform/modules'.format(CURR_WORK_DIR)
if os.path.exists(current_modules_dir):
print("Deleting existing modules...\nCalling " \
"command: {}".format('rm -rf {}'.format(current_modules_dir)))
run.cmd('rm -rf {}'.format(current_modules_dir))
if os.path.exists('{}/.terraform/terraform.tfstate'.format(CURR_WORK_DIR)):
print("Deleting existing terraform state file...\nCalling " \
"command: {}".format('rm {}/.terraform/terraform.tfstate'.format(CURR_WORK_DIR)))
run.cmd('rm {}/.terraform/terraform.tfstate'.format(CURR_WORK_DIR))
print("Copying modules and terraform state file from" \
" {}/terraform/{}/.terraform/ to {}/.terraform/'".format(CURR_WORK_DIR, environment, CURR_WORK_DIR))
run.cmd('cp -R {}/terraform/{}/.terraform {}/'.format(
CURR_WORK_DIR, environment, CURR_WORK_DIR))
def main():
tflog_file = None
try:
profile, region, tf_environment = setup_tf_env_vars()
boto3_session = boto3.session.Session(profile_name=profile,
region_name=region)
init_remote_state()
if args.show_resources_only:
prepare_resources_json_from_terraform_outs()
return 0
ssh_key_create.create_ssh_key(tf_environment, boto3_session)
os.environ["TF_VAR_dev_api_key"] = api_gateway_helper.create_api_key(
tf_environment, boto3_session)
# Did not work: "Active stages pointing to this deployment must be moved or deleted"
# print '{}Tainting API GW deployments - so it can be redeployed'.format(bcolors.OKBLUE)
# terraform_state_resources = utils_terraform.terraform_list_resources(os.environ)
# api_gw_deployments = [rsc for rsc in terraform_state_resources.split('\n') if 'aws_api_gateway_deployment' in rsc]
# for api_gw_deployment in api_gw_deployments:
# utils_terraform.terraform_taint_resource(api_gw_deployment, os.environ)
# if m2a_path provided recursively zip all modules to single dir (terraform/lambda_files)
if m2a_path:
zip_modules.zip_modules(m2a_path, environment, zip_dir=zip_cache)
else:
utils_terraform.zip_files(zip_files_location=zip_cache)
terraform_src_path = "terraform/{}".format(environment)
module_param = ''
if module_name:
print('{}Deploying module {}'.format(bcolors.OKBLUE, module_name))
module_param = "-target=module.{}".format(module_name)
# Always log to a file at WARN level, and output after plan/apply operations as it might contain essential info for resolving issues:
tflog_file = tempfile.NamedTemporaryFile(prefix='terraform-tflog-')
os.environ['TF_LOG'] = 'WARN'
os.environ['TF_LOG_PATH'] = tflog_file.name
if destroy:
print('{}Destroying terraform deployment!'.format(bcolors.WARNING))
run.cmd('terraform destroy {} {}'.format(module_param,
terraform_src_path),
env_vars=os.environ,
realtime_stdout_pipe=sys.stdout.write)
elif plan:
print('{}Running plan only, will not deploy'.format(
bcolors.WARNING))
run.cmd('terraform plan {} {}'.format(module_param,
terraform_src_path),
env_vars=os.environ,
realtime_stdout_pipe=sys.stdout.write)
else:
plan_file = tempfile.NamedTemporaryFile(prefix='terraform-plan-')
print('{}Planning'.format(bcolors.OKBLUE))
run.cmd('terraform plan -parallelism=20 {} -out {} {}'.format(
module_param, plan_file.name, terraform_src_path),
env_vars=os.environ,
realtime_stdout_pipe=sys.stdout.write)
print('{}Creating/Updating resources in AWS'.format(
bcolors.OKBLUE))
run.cmd('terraform apply {} -parallelism=20 {} {}'.format(
'-auto-approve' if non_interactive else '', module_param,
plan_file.name),
env_vars=os.environ,
realtime_stdout_pipe=sys.stdout.write)
print('{}Successfully applied changes'.format(bcolors.OKBLUE))
plan_file.close()
(terraform_module_paths,
resources_json) = prepare_resources_json_from_terraform_outs()
deploy_s3_sync_modules(boto3_session, terraform_module_paths,
resources_json)
print('{}Updating API GW Stages...'.format(bcolors.ENDC))
update_apigateway_stages.update_stages(boto3_session, environment)
print('{}Generating report...'.format(bcolors.ENDC))
environment_report.generate_and_sync_env_report_file(
environment, region)
if m2a_path and upload_resources:
print('{}Creating usage plan...'.format(bcolors.ENDC))
api_gateway_helper.create_usage_plan(environment,
boto3_session,
create_usage)
print('{}Creating modules resources...'.format(bcolors.ENDC))
create_resources_per_module(boto3_session,
terraform_module_paths,
resources_json)
### WIP: api gw automatic deployment
# print '{}Retrieving API GW deployments for redeployment'.format(bcolors.OKBLUE)
# terraform_outputs = utils_terraform.terraform_outputs(modules, os.environ)
else:
print('{}NOT creating/updating changes in AWS'.format(
bcolors.WARNING))
print('{}Complete.'.format(bcolors.OKGREEN))
return 0
except BaseException as ex:
print(bcolors.FAIL)
traceback.print_exc()
if tflog_file:
tflog = open(tflog_file.name).read()
print('===TFLOG===\n')
PPRINT(re.findall('\[WARN.*', tflog))
PPRINT(re.findall('\[ERROR.*', tflog))
return 1
finally:
if tflog_file:
tflog_file.close()
def get_tf_output(module):
(stdout,
_) = run.cmd('terraform output -json -module={}'.format(module),
cwd=terraform_src_dir)
outputs = json.loads(stdout)
return outputs