def prepare_resources_json_from_terraform_outs(): # Add all module resources to root module resources json def terraform_json_to_m2a_resources_json(terraform_json): return { output_key: terraform_json[output_key]['value'] for output_key in list(terraform_json.keys()) } (root_module_resources_json, _) = run.cmd('terraform output -json', cwd=TERRAFORM_SRC_DIR, env_vars=os.environ, realtime_stdout_pipe=sys.stdout.write) root_module_resources = terraform_json_to_m2a_resources_json( json.loads(root_module_resources_json)) module_file = '{}/terraform/{}/modules.tf'.format(CURR_WORK_DIR, environment) terraform_module_paths = get_module_paths(module_file) terraform_modules = get_terraform_modules(module_file) print('{}Retrieving terraform outputs for modules {}'.format( bcolors.OKBLUE, terraform_modules)) terraform_module_resources = utils_terraform.terraform_outputs( terraform_modules, TERRAFORM_SRC_DIR) module_resources = terraform_json_to_m2a_resources_json( terraform_module_resources) root_module_resources.update(module_resources) resources_json = root_module_resources print('{}resources.json of all modules:\n{}'.format( bcolors.OKGREEN, json.dumps(resources_json, indent=2))) return (terraform_module_paths, resources_json)
def sync_files(boto3_session, component_path, component_svc_info, resources_json, env): s3rsc = boto3_session.resource('s3') deployment_bucket = resources_json[component_svc_info['terraformBucket']] resources_key = component_svc_info.get('terraformResourcesKey') bucket = s3rsc.Bucket(deployment_bucket) if resources_key: print('Syncing resources.json to S3 key: {}'.format(resources_key)) bucket.put_object(Key=resources_key, Body=io.BytesIO((json.dumps(resources_json, indent=4, sort_keys=True)))) real_component_path = os.path.realpath(component_path) print('Syncing {} to S3 bucket: {}'.format(real_component_path, deployment_bucket)) run.cmd("aws --profile {} s3 sync {} s3://{}/".format( env, real_component_path, deployment_bucket), realtime_stdout_pipe=sys.stdout.write)
def generate_and_sync_env_report_file(env, region, report_path=os.getcwd()): try: if os.path.exists("{}/env_report/".format(report_path)): print("Deleting previous report files...") run.cmd("rm -rf {}/env_report/".format(report_path)) print("Creating report file...") print("Running command: Scout2 --profile {} --regions {} --report-dir {}/env_report/ " \ "--no-browser".format(env, region, report_path)) run.cmd( "Scout2 --profile {} --regions {} --report-dir {}/env_report/ --no-browser" .format(env, region, report_path)) except: print(("Error while producing report for {}".format(env))) pass try: print("Renaming report file...") run.cmd( "mv {}/env_report/report-{}.html {}/env_report/report.html".format( report_path, env, report_path)) print( "Syncing report to s3://m2a-dashboard-{}-ui/public/report".format( env)) run.cmd( "aws --profile {} s3 sync {}/env_report/ s3://m2a-dashboard-{}-ui/public/report/" .format(env, report_path, env)) print("Deleting local report files...") run.cmd("rm -rf {}/env_report/".format(report_path)) except: print("Error uploading report file {}".format(report_path)) pass
def init_remote_state(): run.cmd('terraform init', cwd=TERRAFORM_SRC_DIR, realtime_stdout_pipe=sys.stdout.write) current_modules_dir = '{}/.terraform/modules'.format(CURR_WORK_DIR) if os.path.exists(current_modules_dir): print("Deleting existing modules...\nCalling " \ "command: {}".format('rm -rf {}'.format(current_modules_dir))) run.cmd('rm -rf {}'.format(current_modules_dir)) if os.path.exists('{}/.terraform/terraform.tfstate'.format(CURR_WORK_DIR)): print("Deleting existing terraform state file...\nCalling " \ "command: {}".format('rm {}/.terraform/terraform.tfstate'.format(CURR_WORK_DIR))) run.cmd('rm {}/.terraform/terraform.tfstate'.format(CURR_WORK_DIR)) print("Copying modules and terraform state file from" \ " {}/terraform/{}/.terraform/ to {}/.terraform/'".format(CURR_WORK_DIR, environment, CURR_WORK_DIR)) run.cmd('cp -R {}/terraform/{}/.terraform {}/'.format( CURR_WORK_DIR, environment, CURR_WORK_DIR))
def main(): tflog_file = None try: profile, region, tf_environment = setup_tf_env_vars() boto3_session = boto3.session.Session(profile_name=profile, region_name=region) init_remote_state() if args.show_resources_only: prepare_resources_json_from_terraform_outs() return 0 ssh_key_create.create_ssh_key(tf_environment, boto3_session) os.environ["TF_VAR_dev_api_key"] = api_gateway_helper.create_api_key( tf_environment, boto3_session) # Did not work: "Active stages pointing to this deployment must be moved or deleted" # print '{}Tainting API GW deployments - so it can be redeployed'.format(bcolors.OKBLUE) # terraform_state_resources = utils_terraform.terraform_list_resources(os.environ) # api_gw_deployments = [rsc for rsc in terraform_state_resources.split('\n') if 'aws_api_gateway_deployment' in rsc] # for api_gw_deployment in api_gw_deployments: # utils_terraform.terraform_taint_resource(api_gw_deployment, os.environ) # if m2a_path provided recursively zip all modules to single dir (terraform/lambda_files) if m2a_path: zip_modules.zip_modules(m2a_path, environment, zip_dir=zip_cache) else: utils_terraform.zip_files(zip_files_location=zip_cache) terraform_src_path = "terraform/{}".format(environment) module_param = '' if module_name: print('{}Deploying module {}'.format(bcolors.OKBLUE, module_name)) module_param = "-target=module.{}".format(module_name) # Always log to a file at WARN level, and output after plan/apply operations as it might contain essential info for resolving issues: tflog_file = tempfile.NamedTemporaryFile(prefix='terraform-tflog-') os.environ['TF_LOG'] = 'WARN' os.environ['TF_LOG_PATH'] = tflog_file.name if destroy: print('{}Destroying terraform deployment!'.format(bcolors.WARNING)) run.cmd('terraform destroy {} {}'.format(module_param, terraform_src_path), env_vars=os.environ, realtime_stdout_pipe=sys.stdout.write) elif plan: print('{}Running plan only, will not deploy'.format( bcolors.WARNING)) run.cmd('terraform plan {} {}'.format(module_param, terraform_src_path), env_vars=os.environ, realtime_stdout_pipe=sys.stdout.write) else: plan_file = tempfile.NamedTemporaryFile(prefix='terraform-plan-') print('{}Planning'.format(bcolors.OKBLUE)) run.cmd('terraform plan -parallelism=20 {} -out {} {}'.format( module_param, plan_file.name, terraform_src_path), env_vars=os.environ, realtime_stdout_pipe=sys.stdout.write) print('{}Creating/Updating resources in AWS'.format( bcolors.OKBLUE)) run.cmd('terraform apply {} -parallelism=20 {} {}'.format( '-auto-approve' if non_interactive else '', module_param, plan_file.name), env_vars=os.environ, realtime_stdout_pipe=sys.stdout.write) print('{}Successfully applied changes'.format(bcolors.OKBLUE)) plan_file.close() (terraform_module_paths, resources_json) = prepare_resources_json_from_terraform_outs() deploy_s3_sync_modules(boto3_session, terraform_module_paths, resources_json) print('{}Updating API GW Stages...'.format(bcolors.ENDC)) update_apigateway_stages.update_stages(boto3_session, environment) print('{}Generating report...'.format(bcolors.ENDC)) environment_report.generate_and_sync_env_report_file( environment, region) if m2a_path and upload_resources: print('{}Creating usage plan...'.format(bcolors.ENDC)) api_gateway_helper.create_usage_plan(environment, boto3_session, create_usage) print('{}Creating modules resources...'.format(bcolors.ENDC)) create_resources_per_module(boto3_session, terraform_module_paths, resources_json) ### WIP: api gw automatic deployment # print '{}Retrieving API GW deployments for redeployment'.format(bcolors.OKBLUE) # terraform_outputs = utils_terraform.terraform_outputs(modules, os.environ) else: print('{}NOT creating/updating changes in AWS'.format( bcolors.WARNING)) print('{}Complete.'.format(bcolors.OKGREEN)) return 0 except BaseException as ex: print(bcolors.FAIL) traceback.print_exc() if tflog_file: tflog = open(tflog_file.name).read() print('===TFLOG===\n') PPRINT(re.findall('\[WARN.*', tflog)) PPRINT(re.findall('\[ERROR.*', tflog)) return 1 finally: if tflog_file: tflog_file.close()
def get_tf_output(module): (stdout, _) = run.cmd('terraform output -json -module={}'.format(module), cwd=terraform_src_dir) outputs = json.loads(stdout) return outputs