def create_cron_event(aws_cli, name, project_arn, schedule_expression, git_branch): print_message('create cron event') cmd = ['events', 'put-rule'] rule_name = f'{name}CronRuleSourceBy{git_branch.title()}' cmd += ['--name', rule_name] cmd += ['--schedule-expression', schedule_expression] aws_cli.run(cmd) print_message('link event and codebuild project') role_arn = aws_cli.get_role_arn('aws-events-rule-codebuild-role') target_input = { "sourceVersion": git_branch, "timeoutInMinutesOverride": 60 } target_input = json.dumps(target_input) target = { "Id": "1", "Arn": project_arn, "RoleArn": role_arn, "Input": target_input } target = json.dumps(target) cmd = ['events', 'put-targets'] cmd += ['--rule', rule_name] cmd += ['--targets', target] aws_cli.run(cmd)
def run_terminate_environment(name): print_message('terminate %s' % name) elapsed_time = 0 while True: cmd = ['elasticbeanstalk', 'describe-environments'] cmd += ['--application-name', eb_application_name] result = aws_cli.run(cmd) count = 0 for r in result['Environments']: if not r['EnvironmentName'].startswith(name): continue if r['Status'] not in ('Ready', 'Terminating', 'Terminated'): count += 1 continue if r['Status'] == 'Ready': cmd = ['elasticbeanstalk', 'terminate-environment'] cmd += ['--environment-name', r['EnvironmentName']] aws_cli.run(cmd, ignore_error=True) if count == 0: break print('deleting the environment... (elapsed time: \'%d\' seconds)' % elapsed_time) time.sleep(5) elapsed_time += 5
def _manual_backup(): aws_cli = AWSCli() ################################################################################ print_session('dump mysql data') engine = env['rds']['ENGINE'] if engine not in ('mysql', 'aurora'): print('not supported:', engine) raise Exception() ################################################################################ print_message('get database address') if env['common']['PHASE'] != 'dv': host = aws_cli.get_rds_address(read_replica=True) else: while True: answer = input('Do you use a database of Vagrant VM? (yes/no): ') if answer.lower() == 'no': host = aws_cli.get_rds_address(read_replica=True) break if answer.lower() == 'yes': host = 'dv-database.hbsmith.io' break database = env['rds']['DATABASE'] password = env['rds']['USER_PASSWORD'] user = env['rds']['USER_NAME'] template_name = env['template']['NAME'] filename_path = 'template/%s/rds/mysql_data.sql' % template_name _mysql_dump(host, user, password, database, filename_path)
def run_create_cloudwatch_alarm_rds(name, settings): phase = env['common']['PHASE'] alarm_region = settings['AWS_DEFAULT_REGION'] aws_cli = AWSCli(alarm_region) alarm_name = '%s-%s_%s_%s' % (phase, name, alarm_region, settings['METRIC_NAME']) print_message('create or update cloudwatch alarm: %s' % alarm_name) topic_arn = aws_cli.get_topic_arn(settings['SNS_TOPIC_NAME']) if not topic_arn: print('sns topic: "%s" is not exists in %s' % (settings['SNS_TOPIC_NAME'], alarm_region)) raise Exception() dimension_list = list() if settings['DIMENSIONS'] == 'DBClusterIdentifier': db_cluster_id = env['rds']['DB_CLUSTER_ID'] dimension = 'Name=DBClusterIdentifier,Value=%s' % db_cluster_id dimension_list.append(dimension) cmd = ['cloudwatch', 'put-metric-alarm'] cmd += ['--alarm-actions', topic_arn] cmd += ['--alarm-description', settings['DESCRIPTION']] cmd += ['--alarm-name', alarm_name] cmd += ['--comparison-operator', settings['COMPARISON_OPERATOR']] cmd += ['--datapoints-to-alarm', settings['DATAPOINTS_TO_ALARM']] cmd += ['--dimensions', ' '.join(dimension_list)] cmd += ['--evaluation-periods', settings['EVALUATION_PERIODS']] cmd += ['--metric-name', settings['METRIC_NAME']] cmd += ['--namespace', settings['NAMESPACE']] cmd += ['--period', settings['PERIOD']] cmd += ['--statistic', settings['STATISTIC']] cmd += ['--threshold', settings['THRESHOLD']] aws_cli.run(cmd)
def run_create_cloudwatch_alarm_sns(name, settings): phase = env['common']['PHASE'] alarm_region = settings['AWS_DEFAULT_REGION'] aws_cli = AWSCli(alarm_region) alarm_name = '%s-%s_%s_%s' % (phase, name, alarm_region, settings['METRIC_NAME']) print_message('create or update cloudwatch alarm: %s' % alarm_name) topic_arn = aws_cli.get_topic_arn(settings['SNS_TOPIC_NAME']) if not topic_arn: print('sns topic: "%s" is not exists in %s' % (settings['SNS_TOPIC_NAME'], alarm_region)) raise Exception() dimensions = list() for ii in settings['DIMENSIONS']: dd = dict() dd['Name'] = ii['name'] dd['Value'] = ii['value'] dimensions.append(dd) cmd = ['cloudwatch', 'put-metric-alarm'] cmd += ['--alarm-actions', topic_arn] cmd += ['--alarm-description', settings['DESCRIPTION']] cmd += ['--alarm-name', alarm_name] cmd += ['--comparison-operator', settings['COMPARISON_OPERATOR']] cmd += ['--datapoints-to-alarm', settings['DATAPOINTS_TO_ALARM']] cmd += ['--dimensions', json.dumps(dimensions)] cmd += ['--evaluation-periods', settings['EVALUATION_PERIODS']] cmd += ['--metric-name', settings['METRIC_NAME']] cmd += ['--namespace', settings['NAMESPACE']] cmd += ['--period', settings['PERIOD']] cmd += ['--statistic', settings['STATISTIC']] cmd += ['--threshold', settings['THRESHOLD']] aws_cli.run(cmd)
def create_iam_for_rds(): sleep_required = False role_name = 'rds-monitoring-role' if not aws_cli.get_iam_role(role_name): print_message('create iam role') cc = ['iam', 'create-role'] cc += ['--role-name', role_name] cc += [ '--assume-role-policy-document', 'file://aws_iam/rds-monitoring-role.json' ] aws_cli.run(cc) cc = ['iam', 'attach-role-policy'] cc += ['--role-name', role_name] cc += [ '--policy-arn', 'arn:aws:iam::aws:policy/service-role/AmazonRDSEnhancedMonitoringRole' ] aws_cli.run(cc) sleep_required = True if sleep_required: print_message( 'wait 120 seconds to let iam role and policy propagated to all regions...' ) time.sleep(120)
def delete_role_for_sms(): aws_cli = AWSCli('ap-northeast-1') role_name = 'aws-sns-sms-log-role' policy_name = 'aws-sns-sms-log-policy' cmd = ['iam', 'delete-role-policy'] cmd += ['--role-name', role_name] cmd += ['--policy-nam', policy_name] aws_cli.run(cmd, ignore_error=True) cmd = ['iam', 'delete-role'] cmd += ['--role-name', role_name] aws_cli.run(cmd, ignore_error=True) print_message('stop sms log') dd = { 'attributes': { 'DeliveryStatusSuccessSamplingRate': '', 'DeliveryStatusIAMRole': '' } } cmd = ['sns', 'set-sms-attributes'] cmd += ['--cli-input-json', json.dumps(dd)] aws_cli.run(cmd, ignore_error=True)
def create_iam_for_lambda(lambda_type): sleep_required = False role_name = 'aws-lambda-%s-role' % lambda_type if not aws_cli.get_iam_role(role_name): print_message('create iam role') role_file_path = 'file://aws_iam/%s.json' % role_name cmd = ['iam', 'create-role'] cmd += ['--role-name', role_name] cmd += ['--assume-role-policy-document', role_file_path] aws_cli.run(cmd) sleep_required = True policy_name = 'aws-lambda-%s-policy' % lambda_type if not aws_cli.get_iam_role_policy(role_name, policy_name): print_message('put iam role policy') policy_file_path = 'file://aws_iam/%s.json' % policy_name cmd = ['iam', 'put-role-policy'] cmd += ['--role-name', role_name] cmd += ['--policy-name', policy_name] cmd += ['--policy-document', policy_file_path] aws_cli.run(cmd) sleep_required = True return sleep_required
def create_iam_for_events(): sleep_required = False role_name = 'aws-events-rule-codebuild-role' if not aws_cli.get_iam_role(role_name): print_message('create iam role: %s' % role_name) cmd = ['iam', 'create-role'] cmd += ['--role-name', role_name] cmd += [ '--assume-role-policy-document', 'file://aws_iam/%s.json' % role_name ] aws_cli.run(cmd) sleep_required = True policy_name = 'aws-events-rule-codebuild-policy' if not aws_cli.get_iam_role_policy(role_name, policy_name): print_message('create iam role policy: %s' % policy_name) cmd = ['iam', 'put-role-policy'] cmd += ['--role-name', role_name] cmd += ['--policy-name', policy_name] cmd += ['--policy-document', 'file://aws_iam/%s.json' % policy_name] aws_cli.run(cmd) sleep_required = True return sleep_required
def run_terminate_sns_tpoic(name, settings): region = settings['AWS_DEFAULT_REGION'] aws_cli = AWSCli(region) ################################################################################ print_message('terminate sns topic subscriptions') topic_arn = aws_cli.get_topic_arn(name) if not topic_arn: return cmd = ['sns', 'list-subscriptions-by-topic'] cmd += ['--topic-arn', topic_arn] rr = aws_cli.run(cmd) for ss in rr['Subscriptions']: cmd = ['sns', 'unsubscribe'] cmd += ['--subscription-arn', ss['SubscriptionArn']] aws_cli.run(cmd, ignore_error=True) print_message('terminate sns topic: "%s" in %s' % (name, region)) cmd = ['sns', 'delete-topic'] cmd += ['--topic-arn', topic_arn] aws_cli.run(cmd)
def create_iam_for_codebuild(codebuild_type): sleep_required = False role_name = f'aws-codebuild-{codebuild_type}-role' if not aws_cli.get_iam_role(role_name): print_message(f'create iam role: {role_name}') cmd = ['iam', 'create-role'] cmd += ['--role-name', role_name] cmd += ['--assume-role-policy-document', 'file://aws_iam/%s.json' % role_name] aws_cli.run(cmd) sleep_required = True policy_name = 'aws-codebuild-%s-policy' % codebuild_type if not aws_cli.get_iam_role_policy(role_name, policy_name): print_message('create iam role policy: %s' % policy_name) cmd = ['iam', 'put-role-policy'] cmd += ['--role-name', role_name] cmd += ['--policy-name', policy_name] cmd += ['--policy-document', 'file://aws_iam/%s.json' % policy_name] aws_cli.run(cmd) sleep_required = True return sleep_required
def run_create_cloudwatch_alarm_sqs(name, settings): phase = env['common']['PHASE'] alarm_region = settings['AWS_DEFAULT_REGION'] sqs_name = settings['QUEUE_NAME'] aws_cli = AWSCli(alarm_region) alarm_name = '%s-%s_%s_%s_%s' % (phase, name, alarm_region, sqs_name, settings['METRIC_NAME']) print_message('create or update cloudwatch alarm: %s' % alarm_name) topic_arn = aws_cli.get_topic_arn(settings['SNS_TOPIC_NAME']) if not topic_arn: print('sns topic: "%s" is not exists in %s' % (settings['SNS_TOPIC_NAME'], alarm_region)) raise Exception() dimension_list = list() if settings['DIMENSIONS'] == 'QueueName': dimension = 'Name=QueueName,Value=%s' % sqs_name dimension_list.append(dimension) cmd = ['cloudwatch', 'put-metric-alarm'] cmd += ['--alarm-actions', topic_arn] cmd += ['--alarm-description', settings['DESCRIPTION']] cmd += ['--alarm-name', alarm_name] cmd += ['--comparison-operator', settings['COMPARISON_OPERATOR']] cmd += ['--datapoints-to-alarm', settings['DATAPOINTS_TO_ALARM']] cmd += ['--dimensions', ' '.join(dimension_list)] cmd += ['--evaluation-periods', settings['EVALUATION_PERIODS']] cmd += ['--metric-name', settings['METRIC_NAME']] cmd += ['--namespace', settings['NAMESPACE']] cmd += ['--period', settings['PERIOD']] cmd += ['--statistic', settings['STATISTIC']] cmd += ['--threshold', settings['THRESHOLD']] aws_cli.run(cmd)
def run_create_s3_script_file_lifecycle(args): aws_cli = AWSCli() bucket_name = args.bucket_name expire_days = int(args.expire_days) print_message('set life cycle rule') cc = { "Rules": [{ "Expiration": { "Days": expire_days }, "ID": "script_file_manage_rule", "Filter": { "Prefix": "_trash/" }, "Status": "Enabled", "NoncurrentVersionExpiration": { "NoncurrentDays": expire_days }, "AbortIncompleteMultipartUpload": { "DaysAfterInitiation": 7 } }] } cmd = [ 's3api', 'put-bucket-lifecycle-configuration', '--bucket', bucket_name ] cmd += ['--lifecycle-configuration', json.dumps(cc)] aws_cli.run(cmd)
def _create(aws_cli, deploy_folder, description, function_name, build_info, queue_arn): print_session('create lambda: %s' % function_name) role_arn = aws_cli.get_role_arn('aws-lambda-sqs-role') cmd = ['lambda', 'create-function', '--function-name', function_name, '--description', description, '--zip-file', 'fileb://deploy.zip', '--role', role_arn, '--handler', 'lambda.handler', '--runtime', 'python3.6', '--tags', ','.join(build_info), '--timeout', '120'] aws_cli.run(cmd, cwd=deploy_folder) print_message('give event permission') cmd = ['lambda', 'add-permission', '--function-name', function_name, '--statement-id', function_name + 'StatementId', '--action', 'lambda:InvokeFunction', '--principal', 'events.amazonaws.com', '--source-arn', queue_arn] aws_cli.run(cmd) print_message('create sqs event source for %s' % function_name) cmd = ['lambda', 'create-event-source-mapping', '--event-source-arn', queue_arn, '--function-name', function_name] aws_cli.run(cmd)
def terminate_iam_for_appstream(): aws_cli = AWSCli() role_name = 'AmazonAppStreamServiceAccess' print_message('delete iam role') # noinspection PyShadowingNames cc = ['iam', 'detach-role-policy'] cc += ['--role-name', role_name] cc += [ '--policy-arn', 'arn:aws:iam::aws:policy/service-role/AmazonAppStreamServiceAccess' ] aws_cli.run(cc, ignore_error=True) # noinspection PyShadowingNames cc = ['iam', 'delete-role'] cc += ['--role-name', role_name] aws_cli.run(cc, ignore_error=True) role_name = 'ApplicationAutoScalingForAmazonAppStreamAccess' # noinspection PyShadowingNames cc = ['iam', 'detach-role-policy'] cc += ['--role-name', role_name] cc += [ '--policy-arn', 'arn:aws:iam::aws:policy/service-role/ApplicationAutoScalingForAmazonAppStreamAccess' ] aws_cli.run(cc, ignore_error=True) # noinspection PyShadowingNames cc = ['iam', 'delete-role'] cc += ['--role-name', role_name] aws_cli.run(cc, ignore_error=True)
def create_sms_log(): aws_cli = AWSCli('ap-northeast-1') role_name = 'aws-sns-sms-log-role' policy_name = 'aws-sns-sms-log-policy' print_message(f'create role: {role_name}') role = aws_cli.get_iam_role(role_name) if not role: cmd = ['iam', 'create-role'] cmd += ['--role-name', role_name] cmd += ['--assume-role-policy-document', f'file://aws_iam/{role_name}.json'] role = aws_cli.run(cmd) cmd = ['iam', 'put-role-policy'] cmd += ['--role-name', role_name] cmd += ['--policy-name', policy_name] cmd += ['--policy-document', f'file://aws_iam/{policy_name}.json'] aws_cli.run(cmd) role_arn = role['Role']['Arn'] print_message('start sms log') dd = {'attributes': {'DeliveryStatusSuccessSamplingRate': '100', 'DeliveryStatusIAMRole': role_arn}} cmd = ['sns', 'set-sms-attributes'] cmd += ['--cli-input-json', json.dumps(dd)] aws_cli.run(cmd)
def create_iam_role(role_name, role_file_path): if not aws_cli.get_iam_role(role_name) and role_file_path: print_message('create iam role %s' % role_name) cmd = ['iam', 'create-role'] cmd += ['--role-name', role_name] cmd += ['--assume-role-policy-document', role_file_path] return aws_cli.run(cmd)
def run_export_cloudwatch_dashboard(name, settings): region = settings['AWS_DEFAULT_REGION'] aws_cli = AWSCli(region) dashboard_name = '%s_%s' % (name, region) print_message('export cloudwatch dashboard: %s' % dashboard_name) cmd = ['cloudwatch', 'get-dashboard'] cmd += ['--dashboard-name', dashboard_name] result = aws_cli.run(cmd) service_type = settings['TYPE'] dashboard_body = json.loads(result['DashboardBody']) for dw in dashboard_body['widgets']: pm = dw['properties'].get('metrics') if not pm: return pm = pm[:1] prev = '' current_index = 0 if len(pm) < 1: return for dimension in pm[0]: if service_type == 'elasticbeanstalk': if prev == 'AutoScalingGroupName': pm[0][current_index] = 'AUTO_SCALING_GROUP_NAME' if prev == 'EnvironmentName': pm[0][current_index] = 'ENVIRONMENT_NAME' if prev == 'InstanceId': pm[0][current_index] = 'INSTANCE_ID' if prev == 'LoadBalancerName': pm[0][current_index] = 'LOAD_BALANCER_NAME' if prev == 'LoadBalancer': pm[0][current_index] = 'LOAD_BALANCER' if prev == 'TargetGroup': pm[0][current_index] = 'TARGET_GROUP' if type(dimension) == dict and 'label' in dimension \ and re.match(r'^%s-[0-9]{10}$' % name, dimension['label']): dimension['label'] = 'ENVIRONMENT_NAME' if service_type == 'rds/aurora': if prev == 'Role': pm[0][current_index] = 'ROLE' if prev == 'DBClusterIdentifier': pm[0][current_index] = 'DB_CLUSTER_IDENTIFIER' if prev == 'DbClusterIdentifier': pm[0][current_index] = 'DB_CLUSTER_IDENTIFIER' prev = dimension current_index += 1 dw['properties']['metrics'] = pm template_name = env['template']['NAME'] filename_path = 'template/%s/cloudwatch/%s.json' % (template_name, dashboard_name) with open(filename_path, 'w') as ff: json.dump(dashboard_body, ff, sort_keys=True, indent=2)
def run_terminate_default_lambda(function_name, settings): aws_cli = AWSCli(settings['AWS_DEFAULT_REGION']) ################################################################################ print_session('terminate lambda: %s' % function_name) print_message('delete lambda function') cmd = ['lambda', 'delete-function', '--function-name', function_name] aws_cli.run(cmd, ignore_error=True)
def run_terminate_cw_dashboard(name, settings): region = settings['AWS_DEFAULT_REGION'] aws_cli = AWSCli(region) dashboard_name = f'{name}_{region}' print_message(f'terminate cloudwatch dashboard: {dashboard_name}') cmd = ['cloudwatch', 'delete-dashboards'] cmd += ['--dashboard-names', dashboard_name] aws_cli.run(cmd)
def run_terminate_cw_dashboard(name, settings): region = settings['AWS_DEFAULT_REGION'] aws_cli = AWSCli(region) dashboard_name = '%s_%s' % (name, region) print_message('terminate cloudwatch dashboard: %s' % dashboard_name) cmd = ['cloudwatch', 'delete-dashboards'] cmd += ['--dashboard-names', dashboard_name] aws_cli.run(cmd)
def _check_need_update(aws_cli, function_name): print_message('check previous version') need_update = False cmd = ['lambda', 'list-functions'] result = aws_cli.run(cmd) for ff in result['Functions']: if function_name == ff['FunctionName']: need_update = True break return need_update
def run_terminate_cloudwatch_dashboard(name, settings): region = settings['AWS_DEFAULT_REGION'] aws_cli = AWSCli(region) dashboard_name = '%s_%s' % (name, region) print_message('terminate cloudwatch dashboard: %s' % dashboard_name) cmd = ['cloudwatch', 'delete-dashboards'] cmd += ['--dashboard-names', dashboard_name] aws_cli.run(cmd)
def terminate_iam_for_rds(): print_message('delete iam role') cc = ['iam', 'detach-role-policy'] cc += ['--role-name', 'rds-monitoring-role'] cc += ['--policy-arn', 'arn:aws:iam::aws:policy/service-role/AmazonRDSEnhancedMonitoringRole'] aws_cli.run(cc, ignore_error=True) cc = ['iam', 'delete-role'] cc += ['--role-name', 'rds-monitoring-role'] aws_cli.run(cc, ignore_error=True)
def terminate_cron_event(_aws_cli, rule_name): print_message('delete events rule %s' % rule_name) cmd = ['events', 'remove-targets'] cmd += ['--rule', rule_name] cmd += ['--ids', '1'] _aws_cli.run(cmd, ignore_error=True) cmd = ['events', 'delete-rule'] cmd += ['--name', rule_name] _aws_cli.run(cmd, ignore_error=True)
def create_iam_role_policy(role_name, policy_name, policy_file_path): if not aws_cli.get_iam_role_policy(role_name, policy_name) and policy_file_path: print_message('create iam role policy %s of %s' % (policy_name, role_name)) cmd = ['iam', 'put-role-policy'] cmd += ['--role-name', role_name] cmd += ['--policy-name', policy_name] cmd += ['--policy-document', policy_file_path] return aws_cli.run(cmd)
def run_create_sns_topic(name, settings): region = settings['AWS_DEFAULT_REGION'] aws_cli = AWSCli(region) ################################################################################ print_message('create sns topic: %s' % name) cmd = ['sns', 'create-topic'] cmd += ['--name', name] result = aws_cli.run(cmd) print('created:', result['TopicArn'])
def run_terminate_sns_tpoic(name): ################################################################################ print_message('terminate sns topic: %s' % name) topic_arn = aws_cli.get_topic_arn(name) if not topic_arn: return cmd = ['sns', 'delete-topic'] cmd += ['--topic-arn', topic_arn] aws_cli.run(cmd)
def run_terminate_cloudwatch_alarm(name, settings): phase = env['common']['PHASE'] region = settings['AWS_DEFAULT_REGION'] aws_cli = AWSCli(region) ################################################################################ alarm_name = '%s-%s_%s_%s' % (phase, name, region, settings['METRIC_NAME']) print_message('terminate cloudwatch alarm: %s' % alarm_name) cmd = ['cloudwatch', 'delete-alarms'] cmd += ['--alarm-names', alarm_name] aws_cli.run(cmd, ignore_error=True)
def terminate_events(): for codebuild in env['codebuilds_cron']: for cron in codebuild['CRONS']: rule_name = '%s_build_%s_cron' % (codebuild['NAME'], cron['BRANCH']) print_message('delete events rule %s' % rule_name) cmd = ['events', 'remove-targets'] cmd += ['--rule', rule_name] cmd += ['--ids', rule_name] aws_cli.run(cmd, ignore_error=True) cmd = ['events', 'delete-rule'] cmd += ['--name', rule_name] aws_cli.run(cmd, ignore_error=True)
def is_exist_cname(cname_list): aws_cli = AWSCli() cmd = ['cloudfront', 'list-distributions'] rr = aws_cli.run(cmd) for vv in rr['DistributionList']['Items']: for cc in cname_list: if 'Items' in vv['Aliases'] and cc in vv['Aliases']['Items']: print_message('exist cname(%s)' % cc) return True return False
def terminate_iam_for_lambda(): print_message('delete iam role policy') cmd = ['iam', 'delete-role-policy'] cmd += ['--role-name', 'aws-lambda-default-role'] cmd += ['--policy-name', 'aws-lambda-default-policy'] aws_cli.run(cmd, ignore_error=True) print_message('delete iam role') cmd = ['iam', 'delete-role'] cmd += ['--role-name', 'aws-lambda-default-role'] aws_cli.run(cmd, ignore_error=True)
def terminate_iam_for_lambda(lambda_type): print_message('delete iam role policy') cmd = ['iam', 'delete-role-policy'] cmd += ['--role-name', 'aws-lambda-%s-role' % lambda_type] cmd += ['--policy-name', 'aws-lambda-%s-policy' % lambda_type] aws_cli.run(cmd, ignore_error=True) print_message('delete iam role') cmd = ['iam', 'delete-role'] cmd += ['--role-name', 'aws-lambda-%s-role' % lambda_type] aws_cli.run(cmd, ignore_error=True)
def run_terminate_s3_webapp(name, settings): deploy_bucket_name = settings['BUCKET_NAME'] bucket_prefix = settings.get('BUCKET_PREFIX', '') deploy_bucket_prefix = os.path.normpath('%s/%s' % (deploy_bucket_name, bucket_prefix)) ################################################################################ print_message('terminate %s' % name) ################################################################################ print_message('cleanup deploy bucket') cmd = ['s3', 'rm', 's3://%s' % deploy_bucket_prefix, '--recursive'] delete_excluded_files = list(settings.get('DELETE_EXCLUDED_FILES', '')) for ff in delete_excluded_files: cmd += ['--exclude', '%s' % ff] delete_result = aws_cli.run(cmd) for ll in delete_result.split('\n'): print(ll) ################################################################################ print_message('remove tag from deploy bucket') cmd = ['s3api', 'delete-bucket-tagging', '--bucket', deploy_bucket_name] aws_cli.run(cmd) ################################################################################ print_message('invalidate cache from cloudfront') cf_dist_id = settings.get('CLOUDFRONT_DIST_ID', '') if len(cf_dist_id) > 0: path_list = list(settings['INVALIDATE_PATHS']) cmd = ['cloudfront', 'create-invalidation', '--distribution-id', cf_dist_id, '--paths', ' '.join(path_list)] invalidate_result = aws_cli.run(cmd) print(invalidate_result)
def run_terminate_sns_lambda(name, settings): function_name = settings['NAME'] template_name = env['template']['NAME'] template_path = 'template/%s' % template_name deploy_folder = '%s/lambda/%s' % (template_path, name) ################################################################################ print_session('terminate lambda: %s' % function_name) cmd = ['lambda', 'get-policy', '--function-name', function_name] result = aws_cli.run(cmd, ignore_error=True) if result: policy = result['Policy'] policy = json.loads(policy) statement_list = policy['Statement'] for statement in statement_list: print_message('remove subscription') arn_like = statement['Condition']['ArnLike'] source_arn = arn_like['AWS:SourceArn'] sns_region = source_arn.split(':')[3] cmd = ['sns', 'list-subscriptions-by-topic', '--topic-arn', source_arn] result = AWSCli(sns_region).run(cmd, ignore_error=True) if not result: continue subscription_list = result['Subscriptions'] for subscription in subscription_list: if subscription['Protocol'] != 'lambda': continue subscription_arn = subscription['SubscriptionArn'] cmd = ['sns', 'unsubscribe', '--subscription-arn', subscription_arn] AWSCli(sns_region).run(cmd, ignore_error=True) print_message('delete lambda function') cmd = ['lambda', 'delete-function', '--function-name', function_name] aws_cli.run(cmd, cwd=deploy_folder, ignore_error=True)
def run_terminate_default_lambda(name, settings): function_name = settings['NAME'] template_name = env['template']['NAME'] template_path = 'template/%s' % template_name deploy_folder = '%s/lambda/%s' % (template_path, name) ################################################################################ print_session('terminate lambda: %s' % function_name) print_message('delete lambda function') cmd = ['lambda', 'delete-function', '--function-name', function_name] aws_cli.run(cmd, cwd=deploy_folder, ignore_error=True)
def run_terminate_sns_tpoic(name, settings): region = settings['AWS_DEFAULT_REGION'] aws_cli = AWSCli(region) ################################################################################ print_message('terminate sns topic: "%s" in %s' % (name, region)) topic_arn = aws_cli.get_topic_arn(name) if not topic_arn: return cmd = ['sns', 'delete-topic'] cmd += ['--topic-arn', topic_arn] aws_cli.run(cmd)
def run_export_cloudwatch_dashboard_sqs_lambda(name, settings): region = settings['AWS_DEFAULT_REGION'] aws_cli = AWSCli(region) dashboard_name = '%s_%s' % (name, region) print_message('export cloudwatch dashboard: %s' % dashboard_name) cmd = ['cloudwatch', 'get-dashboard'] cmd += ['--dashboard-name', dashboard_name] result = aws_cli.run(cmd) dashboard_body = json.loads(result['DashboardBody']) for dw in dashboard_body['widgets']: pm = dw['properties']['metrics'] first_pm = pm[:1] prev = '' current_index = 0 if len(first_pm) < 1: return for dimension in first_pm[0]: if prev == 'QueueName': queue_name = first_pm[0][current_index] if queue_name.startswith('dv-'): queue_name = queue_name.replace('dv-', 'PHASE-') if queue_name.startswith('qa-'): queue_name = queue_name.replace('qa-', 'PHASE-') if queue_name.startswith('op-'): queue_name = queue_name.replace('op-', 'PHASE-') first_pm[0][current_index] = queue_name prev = dimension current_index += 1 dw['properties']['metrics'] = pm title = dw['properties']['title'] if title.startswith('SQS: dv-'): title = title.replace('SQS: dv-', 'SQS: PHASE-') if title.startswith('SQS: qa-'): title = title.replace('SQS: qa-', 'SQS: PHASE-') if title.startswith('SQS: op-'): title = title.replace('SQS: op-', 'SQS: PHASE-') dw['properties']['title'] = title template_name = env['template']['NAME'] filename_path = 'template/%s/cloudwatch/%s.json' % (template_name, dashboard_name) with open(filename_path, 'w') as ff: json.dump(dashboard_body, ff, sort_keys=True, indent=2)
def run_terminate_cw_alarm(name, settings): phase = env['common']['PHASE'] region = settings['AWS_DEFAULT_REGION'] aws_cli = AWSCli(region) ################################################################################ alarm_name = '%s-%s_%s_%s' % (phase, name, region, settings['METRIC_NAME']) if settings['TYPE'] == 'sqs': sqs_name = settings['QUEUE_NAME'] alarm_name = '%s-%s_%s_%s_%s' % (phase, name, region, sqs_name, settings['METRIC_NAME']) print_message('terminate cloudwatch alarm: %s' % alarm_name) cmd = ['cloudwatch', 'delete-alarms'] cmd += ['--alarm-names', alarm_name] aws_cli.run(cmd, ignore_error=True)
def run_export_cloudwatch_dashboard(name, settings): region = settings['AWS_DEFAULT_REGION'] aws_cli = AWSCli(region) dashboard_name = '%s_%s' % (name, region) print_message('export cloudwatch dashboard: %s' % dashboard_name) cmd = ['cloudwatch', 'get-dashboard'] cmd += ['--dashboard-name', dashboard_name] result = aws_cli.run(cmd) service_type = settings['TYPE'] dashboard_body = json.loads(result['DashboardBody']) for dw in dashboard_body['widgets']: pm = dw['properties']['metrics'] pm = pm[:1] prev = '' current_index = 0 if len(pm) < 1: return for dimension in pm[0]: if prev == 'InstanceId' and service_type == 'elasticbeanstalk': pm[0][current_index] = 'INSTANCE_ID' if prev == 'AutoScalingGroupName' and service_type == 'elasticbeanstalk': pm[0][current_index] = 'AUTO_SCALING_GROUP_NAME' if prev == 'LoadBalancerName' and service_type == 'elasticbeanstalk': pm[0][current_index] = 'LOAD_BALANCER_NAME' if prev == 'EnvironmentName' and service_type == 'elasticbeanstalk': pm[0][current_index] = 'ENVIRONMENT_NAME' if prev == 'Role' and service_type == 'rds/aurora': pm[0][current_index] = 'ROLE' if prev == 'DBClusterIdentifier' and service_type == 'rds/aurora': pm[0][current_index] = 'DB_CLUSTER_IDENTIFIER' if prev == 'DbClusterIdentifier' and service_type == 'rds/aurora': pm[0][current_index] = 'DB_CLUSTER_IDENTIFIER' prev = dimension current_index += 1 dw['properties']['metrics'] = pm template_name = env['template']['NAME'] filename_path = 'template/%s/cloudwatch/%s.json' % (template_name, dashboard_name) with open(filename_path, 'w') as ff: json.dump(dashboard_body, ff, sort_keys=True, indent=2)
def _mysql_dump(host, user, password, database, filename_path): ################################################################################ print_message('dump data') cmd = ['mysqldump'] cmd += ['-h' + host] cmd += ['-u' + user] cmd += ['-p' + password] cmd += ['--comments'] cmd += ['--databases', database] cmd += ['--hex-blob'] cmd += ['--ignore-table=%s.auth_permission' % database] cmd += ['--ignore-table=%s.django_content_type' % database] cmd += ['--ignore-table=%s.django_migrations' % database] cmd += ['--ignore-table=%s.django_session' % database] cmd += ['--no-create-info'] cmd += ['--single-transaction'] cmd += ['--skip-extended-insert'] print('\n>>> ' + ' '.join(cmd) + '\n') filename_path_raw = filename_path + '.raw' with open(filename_path_raw, 'w') as ff: subprocess.Popen(cmd, stdout=ff).communicate() with open(filename_path_raw, 'r') as ff_raw, open(filename_path, 'w') as ff: while True: line = ff_raw.readline() if not line: break if line.startswith('-- MySQL dump') or \ line.startswith('-- Host') or \ line.startswith('-- Server version') or \ line.startswith('-- Dump completed on'): ff.write('\n') continue ff.write(line) cmd = ['rm', filename_path_raw] subprocess.Popen(cmd, stdout=PIPE).communicate()
def create_iam_for_rds(): sleep_required = False role_name = 'rds-monitoring-role' if not aws_cli.get_iam_role(role_name): print_message('create iam role') cc = ['iam', 'create-role'] cc += ['--role-name', role_name] cc += ['--assume-role-policy-document', 'file://aws_iam/rds-monitoring-role.json'] aws_cli.run(cc) cc = ['iam', 'attach-role-policy'] cc += ['--role-name', role_name] cc += ['--policy-arn', 'arn:aws:iam::aws:policy/service-role/AmazonRDSEnhancedMonitoringRole'] aws_cli.run(cc) sleep_required = True if sleep_required: print_message('wait 30 seconds to let iam role and policy propagated to all regions...') time.sleep(30)
def create_iam_for_lambda(): sleep_required = False role_name = 'aws-lambda-default-role' if not aws_cli.get_iam_role(role_name): print_message('create iam role') cmd = ['iam', 'create-role'] cmd += ['--role-name', role_name] cmd += ['--assume-role-policy-document', 'file://aws_iam/aws-lambda-default-role.json'] aws_cli.run(cmd) sleep_required = True policy_name = 'aws-lambda-default-policy' if not aws_cli.get_iam_role_policy(role_name, policy_name): print_message('put iam role policy') cmd = ['iam', 'put-role-policy'] cmd += ['--role-name', role_name] cmd += ['--policy-name', policy_name] cmd += ['--policy-document', 'file://aws_iam/aws-lambda-default-policy.json'] aws_cli.run(cmd) sleep_required = True if sleep_required: print_message('wait two minutes to let iam role and policy propagated to all regions...') time.sleep(120)
def _mysql_dump(host, user, password, database, filename_path): ################################################################################ print_message('dump schema') cmd = ['mysqldump'] cmd += ['-h' + host] cmd += ['-u' + user] cmd += ['-p' + password] cmd += ['--comments'] cmd += ['--databases', database] cmd += ['--ignore-table=%s.django_migrations' % database] cmd += ['--no-data'] print('\n>>> ' + ' '.join(cmd) + '\n') filename_path_raw = filename_path + '.raw' with open(filename_path_raw, 'w') as ff: subprocess.Popen(cmd, stdout=ff).communicate() with open(filename_path_raw, 'r') as ff_raw, open(filename_path, 'w') as ff: while True: line = ff_raw.readline() if not line: break if line.startswith('-- MySQL dump') or \ line.startswith('-- Host') or \ line.startswith('-- Server version') or \ line.startswith('-- Dump completed on'): ff.write('\n') continue line = re.sub(' AUTO_INCREMENT=[0-9]*', '', line) ff.write(line) cmd = ['rm', filename_path_raw] subprocess.Popen(cmd, stdout=PIPE).communicate()
def run_terminate_cron_lambda(name, settings): function_name = settings['NAME'] template_name = env['template']['NAME'] template_path = 'template/%s' % template_name deploy_folder = '%s/lambda/%s' % (template_path, name) ################################################################################ print_session('terminate lambda: %s' % function_name) print_message('unlink event and lambda') cmd = ['events', 'list-targets-by-rule', '--rule', function_name + 'CronRule'] result = aws_cli.run(cmd, ignore_error=True) if type(result) is dict: target_list = result['Targets'] else: target_list = list() ids_list = [] for target in target_list: target_id = '"%s"' % target['Id'] ids_list.append(target_id) ids_list = '[%s]' % ','.join(ids_list) cmd = ['events', 'remove-targets', '--rule', function_name + 'CronRule', '--ids', ids_list] aws_cli.run(cmd, ignore_error=True) print_message('remove event permission') cmd = ['lambda', 'remove-permission', '--function-name', function_name, '--statement-id', function_name + 'StatementId'] aws_cli.run(cmd, ignore_error=True) print_message('delete cron event') cmd = ['events', 'delete-rule', '--name', function_name + 'CronRule'] aws_cli.run(cmd, ignore_error=True) print_message('delete lambda function') cmd = ['lambda', 'delete-function', '--function-name', function_name] aws_cli.run(cmd, cwd=deploy_folder, ignore_error=True)
def run_create_cw_dashboard_sqs_lambda(name, settings): phase = env['common']['PHASE'] dashboard_region = settings['AWS_DEFAULT_REGION'] aws_cli = AWSCli(dashboard_region) dashboard_name = '%s_%s' % (name, dashboard_region) print_message('create or update cloudwatch dashboard: %s' % dashboard_name) template_name = env['template']['NAME'] filename_path = 'template/%s/cloudwatch/%s.json' % (template_name, dashboard_name) with open(filename_path, 'r') as ff: dashboard_body = json.load(ff) for dw in dashboard_body['widgets']: pm = dw['properties']['metrics'] current_index = 0 for pp in pm: template = json.dumps(pp) template = template.replace('PHASE-', '%s-' % phase) pm[current_index] = json.loads(template) current_index += 1 dw['properties']['metrics'] = pm title = dw['properties']['title'] if title.startswith('SQS: PHASE-'): title = title.replace('SQS: PHASE-', 'SQS: %s-' % phase) dw['properties']['title'] = title dashboard_body = json.dumps(dashboard_body) cmd = ['cloudwatch', 'put-dashboard'] cmd += ['--dashboard-name', dashboard_name] cmd += ['--dashboard-body', dashboard_body] aws_cli.run(cmd)
def run_create_cw_dashboard_rds_aurora(name, settings): if not env.get('rds'): print_message('No RDS settings in config.json') return if env['rds'].get('ENGINE') != 'aurora': print_message('Only RDS Aurora supported') dashboard_region = settings['AWS_DEFAULT_REGION'] aws_cli = AWSCli(dashboard_region) cluster_id = env['rds']['DB_CLUSTER_ID'] instance_role_list = list() instance_role_list.append('WRITER') instance_role_list.append('READER') dashboard_name = '%s_%s' % (name, dashboard_region) print_message('create or update cloudwatch dashboard: %s' % dashboard_name) template_name = env['template']['NAME'] filename_path = 'template/%s/cloudwatch/%s.json' % (template_name, dashboard_name) with open(filename_path, 'r') as ff: dashboard_body = json.load(ff) for dw in dashboard_body['widgets']: pm = dw['properties']['metrics'] cluster_id_only = True for dimension in pm[0]: if dimension == 'Role': cluster_id_only = False template = json.dumps(pm[0]) new_metrics_list = list() if cluster_id_only: new_metric = template.replace('DB_CLUSTER_IDENTIFIER', cluster_id) new_metric = json.loads(new_metric) new_metrics_list.append(new_metric) else: for ir in instance_role_list: new_metric = template.replace('DB_CLUSTER_IDENTIFIER', cluster_id) new_metric = new_metric.replace('ROLE', ir) new_metric = json.loads(new_metric) new_metrics_list.append(new_metric) dw['properties']['metrics'] = new_metrics_list dashboard_body = json.dumps(dashboard_body) cmd = ['cloudwatch', 'put-dashboard'] cmd += ['--dashboard-name', dashboard_name] cmd += ['--dashboard-body', dashboard_body] aws_cli.run(cmd)
def run_create_lambda_sns(name, settings): aws_cli = AWSCli() description = settings['DESCRIPTION'] function_name = settings['NAME'] phase = env['common']['PHASE'] template_name = env['template']['NAME'] template_path = 'template/%s' % template_name deploy_folder = '%s/lambda/%s' % (template_path, name) git_rev = ['git', 'rev-parse', 'HEAD'] git_hash_johanna = subprocess.Popen(git_rev, stdout=subprocess.PIPE).communicate()[0] git_hash_template = subprocess.Popen(git_rev, stdout=subprocess.PIPE, cwd=template_path).communicate()[0] ################################################################################ topic_arn_list = list() for sns_topic_name in settings['SNS_TOPICS_NAMES']: print_message('check topic exists: %s' % sns_topic_name) region, topic_name = sns_topic_name.split('/') topic_arn = AWSCli(region).get_topic_arn(topic_name) if not topic_arn: print('sns topic: "%s" is not exists in %s' % (settings['SNS_TOPIC_NAME'], region)) raise Exception() topic_arn_list.append(topic_arn) ################################################################################ print_session('packaging lambda: %s' % function_name) print_message('cleanup generated files') subprocess.Popen(['git', 'clean', '-d', '-f', '-x'], cwd=deploy_folder).communicate() requirements_path = '%s/requirements.txt' % deploy_folder if os.path.exists(requirements_path): print_message('install dependencies') cmd = ['pip3', 'install', '-r', requirements_path, '-t', deploy_folder] subprocess.Popen(cmd).communicate() settings_path = '%s/settings_local_sample.py' % deploy_folder if os.path.exists(settings_path): print_message('create environment values') lines = read_file(settings_path) option_list = list() option_list.append(['PHASE', phase]) for key in settings: value = settings[key] option_list.append([key, value]) for oo in option_list: lines = re_sub_lines(lines, '^(%s) .*' % oo[0], '\\1 = \'%s\'' % oo[1]) write_file('%s/settings_local.py' % deploy_folder, lines) print_message('zip files') cmd = ['zip', '-r', 'deploy.zip', '.'] subprocess.Popen(cmd, cwd=deploy_folder).communicate() print_message('create lambda function') role_arn = aws_cli.get_role_arn('aws-lambda-default-role') tags = list() # noinspection PyUnresolvedReferences tags.append('git_hash_johanna=%s' % git_hash_johanna.decode('utf-8').strip()) # noinspection PyUnresolvedReferences tags.append('git_hash_%s=%s' % (template_name, git_hash_template.decode('utf-8').strip())) ################################################################################ print_message('check previous version') need_update = False cmd = ['lambda', 'list-functions'] result = aws_cli.run(cmd) for ff in result['Functions']: if function_name == ff['FunctionName']: need_update = True break ################################################################################ if need_update: print_session('update lambda: %s' % function_name) cmd = ['lambda', 'update-function-code', '--function-name', function_name, '--zip-file', 'fileb://deploy.zip'] result = aws_cli.run(cmd, cwd=deploy_folder) function_arn = result['FunctionArn'] print_message('update lambda tags') cmd = ['lambda', 'tag-resource', '--resource', function_arn, '--tags', ','.join(tags)] aws_cli.run(cmd, cwd=deploy_folder) return ################################################################################ print_session('create lambda: %s' % function_name) cmd = ['lambda', 'create-function', '--function-name', function_name, '--description', description, '--zip-file', 'fileb://deploy.zip', '--role', role_arn, '--handler', 'lambda.handler', '--runtime', 'python3.6', '--tags', ','.join(tags), '--timeout', '120'] result = aws_cli.run(cmd, cwd=deploy_folder) function_arn = result['FunctionArn'] for topic_arn in topic_arn_list: print_message('create subscription') topic_region = topic_arn.split(':')[3] cmd = ['sns', 'subscribe', '--topic-arn', topic_arn, '--protocol', 'lambda', '--notification-endpoint', function_arn] AWSCli(topic_region).run(cmd) print_message('Add permission to lambda') statement_id = '%s_%s_Permission' % (function_name, topic_region) cmd = ['lambda', 'add-permission', '--function-name', function_name, '--statement-id', statement_id, '--action', 'lambda:InvokeFunction', '--principal', 'sns.amazonaws.com', '--source-arn', topic_arn] aws_cli.run(cmd) print_message('update tag with subscription info') cmd = ['lambda', 'tag-resource', '--resource', function_arn, '--tags', ','.join(tags)] aws_cli.run(cmd, cwd=deploy_folder)
def main(settings): aws_availability_zone_1 = settings['AWS_AVAILABILITY_ZONE_1'] aws_availability_zone_2 = settings['AWS_AVAILABILITY_ZONE_2'] aws_cli = AWSCli(settings['AWS_DEFAULT_REGION']) rds_engine = env['rds']['ENGINE'] rds_subnet_name = env['rds']['DB_SUBNET_NAME'] service_name = env['common'].get('SERVICE_NAME', '') name_prefix = '%s_' % service_name if service_name else '' cidr_vpc = aws_cli.cidr_vpc cidr_subnet = aws_cli.cidr_subnet print_message('get vpc id') rds_vpc_id, eb_vpc_id = aws_cli.get_vpc_id() if rds_vpc_id or eb_vpc_id: print_message('VPC already exists') print('RDS: %s \n' % rds_vpc_id) print('EB: %s \n' % eb_vpc_id) print_session('finish python code') sys.exit(0) ################################################################################ # # EB Application # ################################################################################ print_session('create eb application') ################################################################################ print_message('import key pair') cmd = ['ec2', 'import-key-pair'] cmd += ['--key-name', env['common']['AWS_KEY_PAIR_NAME']] cmd += ['--public-key-material', env['common']['AWS_KEY_PAIR_MATERIAL']] aws_cli.run(cmd) ################################################################################ print_message('create application') eb_service_role_arn = aws_cli.get_iam_role('aws-elasticbeanstalk-service-role')['Role']['Arn'] config_format = '%s=%s' eb_max_count_rule = list() eb_max_count_rule.append(config_format % ('DeleteSourceFromS3', 'true')) eb_max_count_rule.append(config_format % ('Enabled', 'true')) eb_max_count_rule.append(config_format % ('MaxCount', 100)) cmd = ['elasticbeanstalk', 'create-application'] cmd += ['--application-name', env['elasticbeanstalk']['APPLICATION_NAME']] cmd += ['--resource-lifecycle-config', 'ServiceRole=%s,VersionLifecycleConfig={MaxCountRule={%s}}' % ( eb_service_role_arn, ','.join(eb_max_count_rule))] aws_cli.run(cmd) ################################################################################ # # RDS # ################################################################################ print_session('rds') ################################################################################ print_message('create vpc') cmd = ['ec2', 'create-vpc'] cmd += ['--cidr-block', cidr_vpc['rds']] result = aws_cli.run(cmd) rds_vpc_id = result['Vpc']['VpcId'] aws_cli.set_name_tag(rds_vpc_id, '%srds' % name_prefix) ################################################################################ print_message('create subnet') rds_subnet_id = dict() cmd = ['ec2', 'create-subnet'] cmd += ['--vpc-id', rds_vpc_id] cmd += ['--cidr-block', cidr_subnet['rds']['private_1']] cmd += ['--availability-zone', aws_availability_zone_1] result = aws_cli.run(cmd) rds_subnet_id['private_1'] = result['Subnet']['SubnetId'] aws_cli.set_name_tag(rds_subnet_id['private_1'], '%srds_private_1' % name_prefix) cmd = ['ec2', 'create-subnet'] cmd += ['--vpc-id', rds_vpc_id] cmd += ['--cidr-block', cidr_subnet['rds']['private_2']] cmd += ['--availability-zone', aws_availability_zone_2] result = aws_cli.run(cmd) rds_subnet_id['private_2'] = result['Subnet']['SubnetId'] aws_cli.set_name_tag(rds_subnet_id['private_2'], '%srds_private_2' % name_prefix) ################################################################################ print_message('create db subnet group') cmd = ['rds', 'create-db-subnet-group'] cmd += ['--db-subnet-group-name', rds_subnet_name] cmd += ['--db-subnet-group-description', rds_subnet_name] cmd += ['--subnet-ids', rds_subnet_id['private_1'], rds_subnet_id['private_2']] aws_cli.run(cmd) ################################################################################ print_message('create ' + 'route table') # [FYI] PyCharm inspects 'create route table' as SQL query. rds_route_table_id = dict() cmd = ['ec2', 'create-route-table'] cmd += ['--vpc-id', rds_vpc_id] result = aws_cli.run(cmd) rds_route_table_id['private'] = result['RouteTable']['RouteTableId'] aws_cli.set_name_tag(rds_route_table_id['private'], '%srds_private' % name_prefix) ################################################################################ print_message('associate route table') cmd = ['ec2', 'associate-route-table'] cmd += ['--subnet-id', rds_subnet_id['private_1']] cmd += ['--route-table-id', rds_route_table_id['private']] aws_cli.run(cmd) cmd = ['ec2', 'associate-route-table'] cmd += ['--subnet-id', rds_subnet_id['private_2']] cmd += ['--route-table-id', rds_route_table_id['private']] aws_cli.run(cmd) ################################################################################ print_message('create security group') rds_security_group_id = dict() cmd = ['ec2', 'create-security-group'] cmd += ['--group-name', '%srds' % name_prefix] cmd += ['--description', '%srds' % name_prefix] cmd += ['--vpc-id', rds_vpc_id] result = aws_cli.run(cmd) rds_security_group_id['private'] = result['GroupId'] ################################################################################ print_message('authorize security group ingress') cmd = ['ec2', 'authorize-security-group-ingress'] cmd += ['--group-id', rds_security_group_id['private']] cmd += ['--protocol', 'all'] cmd += ['--source-group', rds_security_group_id['private']] aws_cli.run(cmd) cmd = ['ec2', 'authorize-security-group-ingress'] cmd += ['--group-id', rds_security_group_id['private']] cmd += ['--protocol', 'tcp'] if rds_engine == 'aurora-postgresql': cmd += ['--port', '5432'] else: cmd += ['--port', '3306'] cmd += ['--cidr', cidr_vpc['eb']] aws_cli.run(cmd) ################################################################################ # # EB # ################################################################################ print_session('eb') ################################################################################ print_message('create vpc') cmd = ['ec2', 'create-vpc'] cmd += ['--cidr-block', cidr_vpc['eb']] result = aws_cli.run(cmd) eb_vpc_id = result['Vpc']['VpcId'] aws_cli.set_name_tag(eb_vpc_id, '%seb' % name_prefix) ################################################################################ print_message('create subnet') eb_subnet_id = dict() cmd = ['ec2', 'create-subnet'] cmd += ['--vpc-id', eb_vpc_id] cmd += ['--cidr-block', cidr_subnet['eb']['private_1']] cmd += ['--availability-zone', aws_availability_zone_1] result = aws_cli.run(cmd) eb_subnet_id['private_1'] = result['Subnet']['SubnetId'] aws_cli.set_name_tag(eb_subnet_id['private_1'], '%seb_private_1' % name_prefix) cmd = ['ec2', 'create-subnet'] cmd += ['--vpc-id', eb_vpc_id] cmd += ['--cidr-block', cidr_subnet['eb']['private_2']] cmd += ['--availability-zone', aws_availability_zone_2] result = aws_cli.run(cmd) eb_subnet_id['private_2'] = result['Subnet']['SubnetId'] aws_cli.set_name_tag(eb_subnet_id['private_2'], '%seb_private_2' % name_prefix) cmd = ['ec2', 'create-subnet'] cmd += ['--vpc-id', eb_vpc_id] cmd += ['--cidr-block', cidr_subnet['eb']['public_1']] cmd += ['--availability-zone', aws_availability_zone_1] result = aws_cli.run(cmd) eb_subnet_id['public_1'] = result['Subnet']['SubnetId'] aws_cli.set_name_tag(eb_subnet_id['public_1'], '%seb_public_1' % name_prefix) cmd = ['ec2', 'create-subnet'] cmd += ['--vpc-id', eb_vpc_id] cmd += ['--cidr-block', cidr_subnet['eb']['public_2']] cmd += ['--availability-zone', aws_availability_zone_2] result = aws_cli.run(cmd) eb_subnet_id['public_2'] = result['Subnet']['SubnetId'] aws_cli.set_name_tag(eb_subnet_id['public_2'], '%seb_public_2' % name_prefix) ################################################################################ print_message('create internet gateway') cmd = ['ec2', 'create-internet-gateway'] result = aws_cli.run(cmd) internet_gateway_id = result['InternetGateway']['InternetGatewayId'] aws_cli.set_name_tag(internet_gateway_id, '%seb' % name_prefix) ################################################################################ print_message('attach internet gateway') cmd = ['ec2', 'attach-internet-gateway'] cmd += ['--internet-gateway-id', internet_gateway_id] cmd += ['--vpc-id', eb_vpc_id] aws_cli.run(cmd) ################################################################################ print_message('create eip') # We use only one NAT gateway at subnet 'public_1' cmd = ['ec2', 'allocate-address'] cmd += ['--domain', 'vpc'] result = aws_cli.run(cmd) eb_eip_id = result['AllocationId'] aws_cli.set_name_tag(eb_eip_id, '%snat' % name_prefix) ################################################################################ print_message('create nat gateway') # We use only one NAT gateway at subnet 'public_1' cmd = ['ec2', 'create-nat-gateway'] cmd += ['--subnet-id', eb_subnet_id['public_1']] cmd += ['--allocation-id', eb_eip_id] result = aws_cli.run(cmd) eb_nat_gateway_id = result['NatGateway']['NatGatewayId'] aws_cli.set_name_tag(eb_nat_gateway_id, '%seb' % name_prefix) ################################################################################ print_message('wait create nat gateway') aws_cli.wait_create_nat_gateway(eb_vpc_id) ################################################################################ print_message('create ' + 'route table') # [FYI] PyCharm inspects 'create route table' as SQL query. eb_route_table_id = dict() cmd = ['ec2', 'create-route-table'] cmd += ['--vpc-id', eb_vpc_id] result = aws_cli.run(cmd) eb_route_table_id['private'] = result['RouteTable']['RouteTableId'] aws_cli.set_name_tag(eb_route_table_id['private'], '%seb_private' % name_prefix) cmd = ['ec2', 'create-route-table'] cmd += ['--vpc-id', eb_vpc_id] result = aws_cli.run(cmd) eb_route_table_id['public'] = result['RouteTable']['RouteTableId'] aws_cli.set_name_tag(eb_route_table_id['public'], '%seb_public' % name_prefix) ################################################################################ print_message('associate route table') cmd = ['ec2', 'associate-route-table'] cmd += ['--subnet-id', eb_subnet_id['private_1']] cmd += ['--route-table-id', eb_route_table_id['private']] aws_cli.run(cmd) cmd = ['ec2', 'associate-route-table'] cmd += ['--subnet-id', eb_subnet_id['private_2']] cmd += ['--route-table-id', eb_route_table_id['private']] aws_cli.run(cmd) cmd = ['ec2', 'associate-route-table'] cmd += ['--subnet-id', eb_subnet_id['public_1']] cmd += ['--route-table-id', eb_route_table_id['public']] aws_cli.run(cmd) cmd = ['ec2', 'associate-route-table'] cmd += ['--subnet-id', eb_subnet_id['public_2']] cmd += ['--route-table-id', eb_route_table_id['public']] aws_cli.run(cmd) ################################################################################ print_message('create route') cmd = ['ec2', 'create-route'] cmd += ['--route-table-id', eb_route_table_id['public']] cmd += ['--destination-cidr-block', '0.0.0.0/0'] cmd += ['--gateway-id', internet_gateway_id] aws_cli.run(cmd) cmd = ['ec2', 'create-route'] cmd += ['--route-table-id', eb_route_table_id['private']] cmd += ['--destination-cidr-block', '0.0.0.0/0'] cmd += ['--nat-gateway-id', eb_nat_gateway_id] aws_cli.run(cmd) ################################################################################ print_message('create security group') eb_security_group_id = dict() cmd = ['ec2', 'create-security-group'] cmd += ['--group-name', '%seb_private' % name_prefix] cmd += ['--description', '%seb_private' % name_prefix] cmd += ['--vpc-id', eb_vpc_id] result = aws_cli.run(cmd) eb_security_group_id['private'] = result['GroupId'] cmd = ['ec2', 'create-security-group'] cmd += ['--group-name', '%seb_public' % name_prefix] cmd += ['--description', '%seb_public' % name_prefix] cmd += ['--vpc-id', eb_vpc_id] result = aws_cli.run(cmd) eb_security_group_id['public'] = result['GroupId'] ################################################################################ print_message('authorize security group ingress') cmd = ['ec2', 'authorize-security-group-ingress'] cmd += ['--group-id', eb_security_group_id['private']] cmd += ['--protocol', 'all'] cmd += ['--source-group', eb_security_group_id['private']] aws_cli.run(cmd) cmd = ['ec2', 'authorize-security-group-ingress'] cmd += ['--group-id', eb_security_group_id['private']] cmd += ['--protocol', 'all'] cmd += ['--source-group', eb_security_group_id['public']] aws_cli.run(cmd) cmd = ['ec2', 'authorize-security-group-ingress'] cmd += ['--group-id', eb_security_group_id['public']] cmd += ['--protocol', 'all'] cmd += ['--source-group', eb_security_group_id['private']] aws_cli.run(cmd) cmd = ['ec2', 'authorize-security-group-ingress'] cmd += ['--group-id', eb_security_group_id['public']] cmd += ['--protocol', 'all'] cmd += ['--source-group', eb_security_group_id['public']] aws_cli.run(cmd) cmd = ['ec2', 'authorize-security-group-ingress'] cmd += ['--group-id', eb_security_group_id['public']] cmd += ['--protocol', 'tcp'] cmd += ['--port', '22'] cmd += ['--cidr', cidr_vpc['eb']] aws_cli.run(cmd) cmd = ['ec2', 'authorize-security-group-ingress'] cmd += ['--group-id', eb_security_group_id['public']] cmd += ['--protocol', 'tcp'] cmd += ['--port', '80'] cmd += ['--cidr', '0.0.0.0/0'] aws_cli.run(cmd) ################################################################################ # # ElastiCache # ################################################################################ print_session('elasticache') ################################################################################ if env.get('elasticache'): elasticache_subnet_name = env['elasticache']['CACHE_SUBNET_NAME'] print_message('create cache subnet group') cmd = ['elasticache', 'create-cache-subnet-group'] cmd += ['--cache-subnet-group-name', elasticache_subnet_name] cmd += ['--cache-subnet-group-description', elasticache_subnet_name] cmd += ['--subnet-ids', eb_subnet_id['private_1'], eb_subnet_id['private_2']] aws_cli.run(cmd) ################################################################################ # # vpc peering connection # ################################################################################ print_session('vpc peering connection') ################################################################################ print_message('create vpc peering connection') cmd = ['ec2', 'create-vpc-peering-connection'] cmd += ['--vpc-id', rds_vpc_id] cmd += ['--peer-vpc-id', eb_vpc_id] result = aws_cli.run(cmd) peering_connection_id = result['VpcPeeringConnection']['VpcPeeringConnectionId'] aws_cli.set_name_tag(peering_connection_id, '%s' % service_name) cmd = ['ec2', 'accept-vpc-peering-connection'] cmd += ['--vpc-peering-connection-id', peering_connection_id] aws_cli.run(cmd) ################################################################################ print_message('create route: rds -> eb') cmd = ['ec2', 'create-route'] cmd += ['--route-table-id', rds_route_table_id['private']] cmd += ['--destination-cidr-block', cidr_subnet['eb']['private_1']] cmd += ['--vpc-peering-connection-id', peering_connection_id] aws_cli.run(cmd) cmd = ['ec2', 'create-route'] cmd += ['--route-table-id', rds_route_table_id['private']] cmd += ['--destination-cidr-block', cidr_subnet['eb']['private_2']] cmd += ['--vpc-peering-connection-id', peering_connection_id] aws_cli.run(cmd) cmd = ['ec2', 'create-route'] cmd += ['--route-table-id', rds_route_table_id['private']] cmd += ['--destination-cidr-block', cidr_subnet['eb']['public_1']] cmd += ['--vpc-peering-connection-id', peering_connection_id] aws_cli.run(cmd) cmd = ['ec2', 'create-route'] cmd += ['--route-table-id', rds_route_table_id['private']] cmd += ['--destination-cidr-block', cidr_subnet['eb']['public_2']] cmd += ['--vpc-peering-connection-id', peering_connection_id] aws_cli.run(cmd) ################################################################################ print_message('create route: eb -> rds') cmd = ['ec2', 'create-route'] cmd += ['--route-table-id', eb_route_table_id['private']] cmd += ['--destination-cidr-block', cidr_subnet['rds']['private_1']] cmd += ['--vpc-peering-connection-id', peering_connection_id] aws_cli.run(cmd) cmd = ['ec2', 'create-route'] cmd += ['--route-table-id', eb_route_table_id['private']] cmd += ['--destination-cidr-block', cidr_subnet['rds']['private_2']] cmd += ['--vpc-peering-connection-id', peering_connection_id] aws_cli.run(cmd) cmd = ['ec2', 'create-route'] cmd += ['--route-table-id', eb_route_table_id['public']] cmd += ['--destination-cidr-block', cidr_subnet['rds']['private_1']] cmd += ['--vpc-peering-connection-id', peering_connection_id] aws_cli.run(cmd) cmd = ['ec2', 'create-route'] cmd += ['--route-table-id', eb_route_table_id['public']] cmd += ['--destination-cidr-block', cidr_subnet['rds']['private_2']] cmd += ['--vpc-peering-connection-id', peering_connection_id] aws_cli.run(cmd) ################################################################################ # # Network Interface # ################################################################################ print_session('network interface') ################################################################################ environment_list = env['elasticbeanstalk']['ENVIRONMENTS'] for environment in environment_list: cname = environment['CNAME'] private_ip = environment.get('PRIVATE_IP') if cname and private_ip: print_message('create network interface for %s' % cname) cmd = ['ec2', 'create-network-interface'] cmd += ['--subnet-id', eb_subnet_id['private_1']] cmd += ['--description', cname] cmd += ['--private-ip-address', private_ip] cmd += ['--groups', eb_security_group_id['private']] result = aws_cli.run(cmd) network_interface_id = result['NetworkInterface']['NetworkInterfaceId'] aws_cli.set_name_tag(network_interface_id, '%snat' % name_prefix)
def run_create_eb_spring(name, settings): aws_cli = AWSCli() aws_asg_max_value = settings['AWS_ASG_MAX_VALUE'] aws_asg_min_value = settings['AWS_ASG_MIN_VALUE'] aws_default_region = env['aws']['AWS_DEFAULT_REGION'] cname = settings['CNAME'] db_conn_str_suffix = settings.get('DB_CONNECTION_STR_SUFFIX', '') eb_application_name = env['elasticbeanstalk']['APPLICATION_NAME'] git_url = settings['GIT_URL'] instance_type = settings.get('INSTANCE_TYPE', 't2.medium') key_pair_name = env['common']['AWS_KEY_PAIR_NAME'] phase = env['common']['PHASE'] service_name = env['common'].get('SERVICE_NAME', '') subnet_type = settings['SUBNET_TYPE'] name_prefix = '%s_' % service_name if service_name else '' cidr_subnet = aws_cli.cidr_subnet str_timestamp = str(int(time.time())) war_filename = '%s-%s.war' % (name, str_timestamp) eb_environment_name = '%s-%s' % (name, str_timestamp) eb_environment_name_old = None template_folder = 'template/%s' % name target_folder = 'template/%s/target' % name ebextensions_folder = 'template/%s/_provisioning/.ebextensions' % name configuration_folder = 'template/%s/_provisioning/configuration' % name properties_file = 'template/%s/%s' % (name, settings['PROPERTIES_FILE']) git_rev = ['git', 'rev-parse', 'HEAD'] git_hash_johanna = subprocess.Popen(git_rev, stdout=subprocess.PIPE).communicate()[0] ################################################################################ print_session('create %s' % name) ################################################################################ print_message('get vpc id') rds_vpc_id, eb_vpc_id = aws_cli.get_vpc_id() if not eb_vpc_id: print('ERROR!!! No VPC found') raise Exception() ################################################################################ print_message('get subnet id') elb_subnet_id_1 = None elb_subnet_id_2 = None ec2_subnet_id_1 = None ec2_subnet_id_2 = None cmd = ['ec2', 'describe-subnets'] result = aws_cli.run(cmd) for r in result['Subnets']: if r['VpcId'] != eb_vpc_id: continue if 'public' == subnet_type: if r['CidrBlock'] == cidr_subnet['eb']['public_1']: elb_subnet_id_1 = r['SubnetId'] if r['CidrBlock'] == cidr_subnet['eb']['public_2']: elb_subnet_id_2 = r['SubnetId'] if r['CidrBlock'] == cidr_subnet['eb']['private_1']: ec2_subnet_id_1 = r['SubnetId'] if r['CidrBlock'] == cidr_subnet['eb']['private_2']: ec2_subnet_id_2 = r['SubnetId'] elif 'private' == subnet_type: if r['CidrBlock'] == cidr_subnet['eb']['private_1']: elb_subnet_id_1 = ec2_subnet_id_1 = r['SubnetId'] if r['CidrBlock'] == cidr_subnet['eb']['private_2']: elb_subnet_id_2 = ec2_subnet_id_2 = r['SubnetId'] else: print('ERROR!!! Unknown subnet type:', subnet_type) raise Exception() ################################################################################ print_message('get security group id') security_group_id = None cmd = ['ec2', 'describe-security-groups'] result = aws_cli.run(cmd) for r in result['SecurityGroups']: if r['VpcId'] != eb_vpc_id: continue if 'public' == subnet_type: if r['GroupName'] == '%seb_private' % name_prefix: security_group_id = r['GroupId'] break elif 'private' == subnet_type: if r['GroupName'] == '%seb_private' % name_prefix: security_group_id = r['GroupId'] break else: print('ERROR!!! Unknown subnet type:', subnet_type) raise Exception() ################################################################################ print_message('get database address') db_address = aws_cli.get_rds_address() ################################################################################ print_message('get cache address') cache_address = aws_cli.get_elasticache_address() ################################################################################ print_message('git clone') subprocess.Popen(['mkdir', '-p', 'template']).communicate() subprocess.Popen(['rm', '-rf', '%s/' % name], cwd='template').communicate() branch = aws_cli.env.get('GIT_BRANCH_APP', phase) git_command = ['git', 'clone', '--depth=1'] if branch != 'dv': git_command += ['-b', branch] git_command += [git_url] subprocess.Popen(git_command, cwd='template').communicate() if not os.path.exists('%s' % template_folder): raise Exception() git_hash_app = subprocess.Popen(git_rev, stdout=subprocess.PIPE, cwd=template_folder).communicate()[0] subprocess.Popen(['rm', '-rf', '.git'], cwd=template_folder).communicate() subprocess.Popen(['rm', '-rf', '.gitignore'], cwd=template_folder).communicate() ################################################################################ print_message('configuration %s' % name) with open('%s/phase' % configuration_folder, 'w') as f: f.write(phase) f.close() lines = read_file('%s/etc/logstash/conf.d/logstash_sample.conf' % configuration_folder) write_file('%s/etc/logstash/conf.d/logstash.conf' % configuration_folder, lines) lines = read_file('%s/%s.config.sample' % (ebextensions_folder, name)) lines = re_sub_lines(lines, 'AWS_ASG_MAX_VALUE', aws_asg_max_value) lines = re_sub_lines(lines, 'AWS_ASG_MIN_VALUE', aws_asg_min_value) write_file('%s/%s.config' % (ebextensions_folder, name), lines) sample_file = properties_file.replace('.properties', '-sample.properties') lines = read_file(sample_file) option_list = list() option_list.append(['jdbc.url', 'jdbc:mysql://%s%s' % (db_address, db_conn_str_suffix)]) option_list.append(['jdbc.username', env['rds']['USER_NAME']]) option_list.append(['jdbc.password', env['rds']['USER_PASSWORD']]) option_list.append(['redis.host', cache_address]) for key in settings: value = settings[key] option_list.append([key, value]) for oo in option_list: lines = re_sub_lines(lines, '^(%s)=.*' % oo[0], '\\1=%s' % oo[1]) write_file(properties_file, lines) ################################################################################ print_message('check previous version') cmd = ['elasticbeanstalk', 'describe-environments'] cmd += ['--application-name', eb_application_name] result = aws_cli.run(cmd) for r in result['Environments']: if 'CNAME' not in r: continue if r['CNAME'] == '%s.%s.elasticbeanstalk.com' % (cname, aws_default_region): if r['Status'] == 'Terminated': continue elif r['Status'] != 'Ready': print('previous version is not ready.') raise Exception() eb_environment_name_old = r['EnvironmentName'] cname += '-%s' % str_timestamp break ################################################################################ print_message('build artifact') build_command = ['mvn'] if phase != 'dv': build_command += ['exec:exec'] build_command += ['package'] print_message('build %s: %s' % (name, ' '.join(build_command))) subprocess.Popen(build_command, cwd=template_folder).communicate() ################################################################################ print_message('create storage location') cmd = ['elasticbeanstalk', 'create-storage-location'] result = aws_cli.run(cmd) s3_bucket = result['S3Bucket'] s3_war_filename = '/'.join(['s3://' + s3_bucket, eb_application_name, war_filename]) ################################################################################ print_message('create application version') cmd = ['mv', 'ROOT.war', war_filename] subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=target_folder).communicate() cmd = ['s3', 'cp', war_filename, s3_war_filename] aws_cli.run(cmd, cwd=target_folder) cmd = ['rm', '-rf', war_filename] subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=target_folder).communicate() cmd = ['elasticbeanstalk', 'create-application-version'] cmd += ['--application-name', eb_application_name] cmd += ['--source-bundle', 'S3Bucket="%s",S3Key="%s/%s"' % (s3_bucket, eb_application_name, war_filename)] cmd += ['--version-label', eb_environment_name] aws_cli.run(cmd, cwd=template_folder) ################################################################################ print_message('create environment %s' % name) option_settings = list() oo = dict() oo['Namespace'] = 'aws:autoscaling:launchconfiguration' oo['OptionName'] = 'EC2KeyName' oo['Value'] = key_pair_name option_settings.append(oo) oo = dict() oo['Namespace'] = 'aws:autoscaling:launchconfiguration' oo['OptionName'] = 'InstanceType' oo['Value'] = instance_type option_settings.append(oo) oo = dict() oo['Namespace'] = 'aws:autoscaling:launchconfiguration' oo['OptionName'] = 'IamInstanceProfile' oo['Value'] = 'aws-elasticbeanstalk-ec2-role' option_settings.append(oo) oo = dict() oo['Namespace'] = 'aws:autoscaling:launchconfiguration' oo['OptionName'] = 'SecurityGroups' oo['Value'] = security_group_id option_settings.append(oo) oo = dict() oo['Namespace'] = 'aws:ec2:vpc' oo['OptionName'] = 'AssociatePublicIpAddress' oo['Value'] = 'false' option_settings.append(oo) oo = dict() oo['Namespace'] = 'aws:ec2:vpc' oo['OptionName'] = 'ELBScheme' oo['Value'] = 'public' if 'private' == subnet_type: oo['Value'] = 'internal' option_settings.append(oo) oo = dict() oo['Namespace'] = 'aws:ec2:vpc' oo['OptionName'] = 'ELBSubnets' oo['Value'] = ','.join([elb_subnet_id_1, elb_subnet_id_2]) option_settings.append(oo) oo = dict() oo['Namespace'] = 'aws:ec2:vpc' oo['OptionName'] = 'Subnets' oo['Value'] = ','.join([ec2_subnet_id_1, ec2_subnet_id_2]) option_settings.append(oo) oo = dict() oo['Namespace'] = 'aws:ec2:vpc' oo['OptionName'] = 'VPCId' oo['Value'] = eb_vpc_id option_settings.append(oo) oo = dict() oo['Namespace'] = 'aws:elasticbeanstalk:environment' oo['OptionName'] = 'EnvironmentType' oo['Value'] = 'LoadBalanced' option_settings.append(oo) oo = dict() oo['Namespace'] = 'aws:elasticbeanstalk:environment' oo['OptionName'] = 'ServiceRole' oo['Value'] = 'aws-elasticbeanstalk-service-role' option_settings.append(oo) oo = dict() oo['Namespace'] = 'aws:elasticbeanstalk:healthreporting:system' oo['OptionName'] = 'SystemType' oo['Value'] = 'enhanced' option_settings.append(oo) oo = dict() oo['Namespace'] = 'aws:elasticbeanstalk:healthreporting:system' oo['OptionName'] = 'ConfigDocument' cw_env = dict() cw_env['ApplicationRequestsTotal'] = 60 cw_env['ApplicationRequests2xx'] = 60 cw_env['ApplicationRequests3xx'] = 60 cw_env['ApplicationRequests4xx'] = 60 cw_env['ApplicationRequests5xx'] = 60 cw_instance = dict() cw_instance['RootFilesystemUtil'] = 60 cw_instance['InstanceHealth'] = 60 cw_instance['CPUIdle'] = 60 cw = dict() cw['Environment'] = cw_env cw['Instance'] = cw_instance cfg_doc = dict() cfg_doc['CloudWatchMetrics'] = cw cfg_doc['Version'] = 1 oo['Value'] = json.dumps(cfg_doc) option_settings.append(oo) option_settings = json.dumps(option_settings) tag0 = 'Key=git_hash_johanna,Value=%s' % git_hash_johanna.decode('utf-8').strip() tag2 = 'Key=git_hash_%s,Value=%s' % (name, git_hash_app.decode('utf-8').strip()) cmd = ['elasticbeanstalk', 'create-environment'] cmd += ['--application-name', eb_application_name] cmd += ['--cname-prefix', cname] cmd += ['--environment-name', eb_environment_name] cmd += ['--option-settings', option_settings] cmd += ['--solution-stack-name', '64bit Amazon Linux 2018.03 v3.0.1 running Tomcat 8.5 Java 8'] cmd += ['--tags', tag0, tag2] cmd += ['--version-label', eb_environment_name] aws_cli.run(cmd, cwd=template_folder) elapsed_time = 0 while True: cmd = ['elasticbeanstalk', 'describe-environments'] cmd += ['--application-name', eb_application_name] cmd += ['--environment-name', eb_environment_name] result = aws_cli.run(cmd) ee = result['Environments'][0] print(json.dumps(ee, sort_keys=True, indent=4)) if ee.get('Health', '') == 'Green' and ee.get('Status', '') == 'Ready': break print('creating... (elapsed time: \'%d\' seconds)' % elapsed_time) time.sleep(5) elapsed_time += 5 if elapsed_time > 60 * 30: raise Exception() subprocess.Popen(['rm', '-rf', '%s/' % name], cwd='template').communicate() ################################################################################ print_message('revoke security group ingress') cmd = ['ec2', 'describe-security-groups'] cmd += ['--filters', 'Name=tag-key,Values=Name,Name=tag-value,Values=%s' % eb_environment_name] result = aws_cli.run(cmd) for ss in result['SecurityGroups']: cmd = ['ec2', 'revoke-security-group-ingress'] cmd += ['--group-id', ss['GroupId']] cmd += ['--protocol', 'tcp'] cmd += ['--port', '22'] cmd += ['--cidr', '0.0.0.0/0'] aws_cli.run(cmd, ignore_error=True) ################################################################################ print_message('swap CNAME if the previous version exists') if eb_environment_name_old: cmd = ['elasticbeanstalk', 'swap-environment-cnames'] cmd += ['--source-environment-name', eb_environment_name_old] cmd += ['--destination-environment-name', eb_environment_name] aws_cli.run(cmd)
return True ################################################################################ # # start # ################################################################################ print_session('terminate old environment version') timestamp = int(time.time()) max_age_seconds = 60 * 60 * 24 * 3 ################################################################################ print_message('terminate old environment version (current timestamp: %d)' % timestamp) eb_application_name = env['elasticbeanstalk']['APPLICATION_NAME'] for vpc_env in env['vpc']: aws_cli = AWSCli(vpc_env['AWS_DEFAULT_REGION']) aws_default_region = vpc_env['AWS_DEFAULT_REGION'] cmd = ['elasticbeanstalk', 'describe-application-versions'] cmd += ['--application-name', eb_application_name] result = aws_cli.run(cmd) for r in result['ApplicationVersions']: if r['Status'] in ('PROCESSING', 'BUILDING'): continue
from run_common import check_template_availability from run_common import print_message from run_common import print_session aws_cli = AWSCli() print_session('alter database') check_template_availability() engine = env['rds']['ENGINE'] if engine not in ('mysql', 'aurora'): print('not supported:', engine) raise Exception() print_message('get database address') if env['common']['PHASE'] != 'dv': db_host = aws_cli.get_rds_address() else: while True: answer = input('Do you use a database of Vagrant VM? (yes/no): ') if answer.lower() == 'no': db_host = aws_cli.get_rds_address() break if answer.lower() == 'yes': db_host = 'dv-database.hbsmith.io' break db_password = env['rds']['USER_PASSWORD'] db_user = env['rds']['USER_NAME']
def run_create_queue(name, settings): delay_seconds = settings['DELAY_SECONDS'] receive_count = settings['RECEIVE_COUNT'] receive_message_wait_time_seconds = settings['RECEIVE_MESSAGE_WAIT_TIME_SECONDS'] retention = settings['RETENTION'] timeout = settings['TIMEOUT'] use_redrive_policy = settings['USE_REDRIVE_POLICY'] dead_letter_queue_arn = None if use_redrive_policy == "True": ################################################################################ print_message('create queue (dead letter)') attributes = dict() attributes['MessageRetentionPeriod'] = retention cmd = ['sqs', 'create-queue'] cmd += ['--queue-name', '%s-dead-letter' % name] cmd += ['--attributes', json.dumps(attributes)] result = aws_cli.run(cmd) print('create :', result['QueueUrl']) ################################################################################ print_message('get queue url (dead letter)') elapsed_time = 0 while True: cmd = ['sqs', 'get-queue-url', '--queue-name', '%s-dead-letter' % name] result = aws_cli.run(cmd) if type(result) == dict: if result.get('QueueUrl', None): break print('get url... (elapsed time: \'%d\' seconds)' % elapsed_time) time.sleep(5) elapsed_time += 5 ################################################################################ print_message('get queue arn (dead letter)') cmd = ['sqs', 'get-queue-attributes'] cmd += ['--queue-url', result['QueueUrl']] cmd += ['--attribute-names', 'QueueArn'] result = aws_cli.run(cmd) dead_letter_queue_arn = result['Attributes']['QueueArn'] redrive_policy = dict() redrive_policy['deadLetterTargetArn'] = dead_letter_queue_arn redrive_policy['maxReceiveCount'] = receive_count attr = dict() if dead_letter_queue_arn is not None: attr['RedrivePolicy'] = json.dumps(redrive_policy) attr['DelaySeconds'] = delay_seconds attr['MessageRetentionPeriod'] = retention attr['ReceiveMessageWaitTimeSeconds'] = receive_message_wait_time_seconds attr['VisibilityTimeout'] = timeout cmd = ['sqs', 'create-queue'] cmd += ['--queue-name', name] cmd += ['--attributes', json.dumps(attr)] result = aws_cli.run(cmd) print('create :', result['QueueUrl'])
def run_create_cloudwatch_alarm_elasticbeanstalk(name, settings): phase = env['common']['PHASE'] alarm_region = settings['AWS_DEFAULT_REGION'] aws_cli = AWSCli(alarm_region) print_message('get elasticbeanstalk environment info: %s' % name) cmd = ['elasticbeanstalk', 'describe-environments'] cmd += ['--no-include-deleted'] result = aws_cli.run(cmd) env_list = list() for ee in result['Environments']: cname = ee['CNAME'] if not cname.endswith('%s.elasticbeanstalk.com' % alarm_region): continue if '%s.' % name not in cname and '%s2.' % name not in cname: continue ename = ee['EnvironmentName'] if ename.startswith(name): env_list.append(ee) env_instances_list = list() env_asg_list = list() env_elb_list = list() for ee in env_list: cmd = ['elasticbeanstalk', 'describe-environment-resources'] cmd += ['--environment-id', ee['EnvironmentId']] result = aws_cli.run(cmd) ee_res = result['EnvironmentResources'] for instance in ee_res['Instances']: ii = dict() ii['Id'] = instance['Id'] ii['EnvironmentName'] = ee_res['EnvironmentName'] env_instances_list.append(ii) for asg in ee_res['AutoScalingGroups']: env_asg_list.append(asg) for elb in ee_res['LoadBalancers']: env_elb_list.append(elb) ################################################################################ alarm_name = '%s-%s_%s_%s' % (phase, name, alarm_region, settings['METRIC_NAME']) print_message('create or update cloudwatch alarm: %s' % alarm_name) topic_arn = aws_cli.get_topic_arn(settings['SNS_TOPIC_NAME']) if not topic_arn: print('sns topic: "%s" is not exists in %s' % (settings['SNS_TOPIC_NAME'], alarm_region)) raise Exception() dimension_list = list() for ei in env_instances_list: if settings['DIMENSIONS'] == 'InstanceId': dimension = 'Name=InstanceId,Value=%s' % ei['Id'] dimension_list.append(dimension) if settings['DIMENSIONS'] == 'EnvironmentName': dimension = 'Name=EnvironmentName,Value=%s' % ei['EnvironmentName'] dimension_list.append(dimension) break for ei in env_asg_list: if settings['DIMENSIONS'] == 'AutoScalingGroupName': dimension = 'Name=AutoScalingGroupName,Value=%s' % ei['Name'] dimension_list.append(dimension) for ei in env_elb_list: if settings['DIMENSIONS'] == 'LoadBalancerName': dimension = 'Name=LoadBalancerName,Value=%s' % ei['Name'] dimension_list.append(dimension) cmd = ['cloudwatch', 'put-metric-alarm'] cmd += ['--alarm-actions', topic_arn] cmd += ['--alarm-description', settings['DESCRIPTION']] cmd += ['--alarm-name', alarm_name] cmd += ['--comparison-operator', settings['COMPARISON_OPERATOR']] cmd += ['--datapoints-to-alarm', settings['DATAPOINTS_TO_ALARM']] cmd += ['--dimensions', ' '.join(dimension_list)] cmd += ['--evaluation-periods', settings['EVALUATION_PERIODS']] cmd += ['--metric-name', settings['METRIC_NAME']] cmd += ['--namespace', settings['NAMESPACE']] cmd += ['--period', settings['PERIOD']] cmd += ['--statistic', settings['STATISTIC']] cmd += ['--threshold', settings['THRESHOLD']] aws_cli.run(cmd)
def create_iam(): ################################################################################ # # IAM # ################################################################################ print_session('create iam') aws_cli = AWSCli() ################################################################################ print_message('create iam: aws-elasticbeanstalk-ec2-role') cmd = ['iam', 'create-instance-profile'] cmd += ['--instance-profile-name', 'aws-elasticbeanstalk-ec2-role'] aws_cli.run(cmd) cmd = ['iam', 'create-role'] cmd += ['--role-name', 'aws-elasticbeanstalk-ec2-role'] cmd += ['--assume-role-policy-document', 'file://aws_iam/aws-elasticbeanstalk-ec2-role.json'] aws_cli.run(cmd) cmd = ['iam', 'add-role-to-instance-profile'] cmd += ['--instance-profile-name', 'aws-elasticbeanstalk-ec2-role'] cmd += ['--role-name', 'aws-elasticbeanstalk-ec2-role'] aws_cli.run(cmd) cmd = ['iam', 'attach-role-policy'] cmd += ['--role-name', 'aws-elasticbeanstalk-ec2-role'] cmd += ['--policy-arn', 'arn:aws:iam::aws:policy/AWSElasticBeanstalkWebTier'] aws_cli.run(cmd) cmd = ['iam', 'attach-role-policy'] cmd += ['--role-name', 'aws-elasticbeanstalk-ec2-role'] cmd += ['--policy-arn', 'arn:aws:iam::aws:policy/AWSElasticBeanstalkMulticontainerDocker'] aws_cli.run(cmd) cmd = ['iam', 'attach-role-policy'] cmd += ['--role-name', 'aws-elasticbeanstalk-ec2-role'] cmd += ['--policy-arn', 'arn:aws:iam::aws:policy/AWSElasticBeanstalkWorkerTier'] aws_cli.run(cmd) cmd = ['iam', 'put-role-policy'] cmd += ['--role-name', 'aws-elasticbeanstalk-ec2-role'] cmd += ['--policy-name', 'aws-elasticbeanstalk-ec2-policy'] cmd += ['--policy-document', 'file://aws_iam/aws-elasticbeanstalk-ec2-policy.json'] aws_cli.run(cmd) ################################################################################ print_message('create iam: aws-elasticbeanstalk-service-role') cmd = ['iam', 'create-role'] cmd += ['--role-name', 'aws-elasticbeanstalk-service-role'] cmd += ['--assume-role-policy-document', 'file://aws_iam/aws-elasticbeanstalk-service-role.json'] aws_cli.run(cmd) cmd = ['iam', 'attach-role-policy'] cmd += ['--role-name', 'aws-elasticbeanstalk-service-role'] cmd += ['--policy-arn', 'arn:aws:iam::aws:policy/service-role/AWSElasticBeanstalkEnhancedHealth'] aws_cli.run(cmd) cmd = ['iam', 'attach-role-policy'] cmd += ['--role-name', 'aws-elasticbeanstalk-service-role'] cmd += ['--policy-arn', 'arn:aws:iam::aws:policy/service-role/AWSElasticBeanstalkService'] aws_cli.run(cmd)
def run_create_eb_cron_job(name, settings): aws_cli = AWSCli(settings['AWS_DEFAULT_REGION']) aws_asg_max_value = settings['AWS_ASG_MAX_VALUE'] aws_asg_min_value = settings['AWS_ASG_MIN_VALUE'] aws_default_region = settings['AWS_DEFAULT_REGION'] aws_eb_notification_email = settings['AWS_EB_NOTIFICATION_EMAIL'] cname = settings['CNAME'] debug = env['common']['DEBUG'] eb_application_name = env['elasticbeanstalk']['APPLICATION_NAME'] git_url = settings['GIT_URL'] key_pair_name = env['common']['AWS_KEY_PAIR_NAME'] phase = env['common']['PHASE'] subnet_type = settings['SUBNET_TYPE'] template_name = env['template']['NAME'] service_name = env['common'].get('SERVICE_NAME', '') name_prefix = '%s_' % service_name if service_name else '' if hasattr(settings, 'PRIVATE_IP'): private_ip = settings['PRIVATE_IP'] else: private_ip = None cidr_subnet = aws_cli.cidr_subnet str_timestamp = str(int(time.time())) zip_filename = '%s-%s.zip' % (name, str_timestamp) eb_environment_name = '%s-%s' % (name, str_timestamp) eb_environment_name_old = None template_path = 'template/%s' % template_name environment_path = '%s/elasticbeanstalk/%s' % (template_path, name) git_rev = ['git', 'rev-parse', 'HEAD'] git_hash_johanna = subprocess.Popen(git_rev, stdout=subprocess.PIPE).communicate()[0] git_hash_template = subprocess.Popen(git_rev, stdout=subprocess.PIPE, cwd=template_path).communicate()[0] ################################################################################ print_session('create %s' % name) ################################################################################ print_message('get vpc id') rds_vpc_id, eb_vpc_id = aws_cli.get_vpc_id() if not eb_vpc_id: print('ERROR!!! No VPC found') raise Exception() ################################################################################ print_message('get subnet id') subnet_id_1 = None subnet_id_2 = None cmd = ['ec2', 'describe-subnets'] result = aws_cli.run(cmd) for r in result['Subnets']: if r['VpcId'] != eb_vpc_id: continue if 'public' == subnet_type: if r['CidrBlock'] == cidr_subnet['eb']['public_1']: subnet_id_1 = r['SubnetId'] if r['CidrBlock'] == cidr_subnet['eb']['public_2']: subnet_id_2 = r['SubnetId'] elif 'private' == subnet_type: if r['CidrBlock'] == cidr_subnet['eb']['private_1']: subnet_id_1 = r['SubnetId'] if r['CidrBlock'] == cidr_subnet['eb']['private_2']: subnet_id_2 = r['SubnetId'] else: print('ERROR!!! Unknown subnet type:', subnet_type) raise Exception() ################################################################################ print_message('get security group id') security_group_id = None cmd = ['ec2', 'describe-security-groups'] result = aws_cli.run(cmd) for r in result['SecurityGroups']: if r['VpcId'] != eb_vpc_id: continue if 'public' == subnet_type: if r['GroupName'] == '%seb_public' % name_prefix: security_group_id = r['GroupId'] break elif 'private' == subnet_type: if r['GroupName'] == '%seb_private' % name_prefix: security_group_id = r['GroupId'] break else: print('ERROR!!! Unknown subnet type:', subnet_type) raise Exception() ################################################################################ print_message('configuration %s' % name) with open('%s/configuration/phase' % environment_path, 'w') as f: f.write(phase) f.close() lines = read_file('%s/.ebextensions/%s.config.sample' % (environment_path, name)) lines = re_sub_lines(lines, 'AWS_ASG_MIN_VALUE', aws_asg_min_value) lines = re_sub_lines(lines, 'AWS_ASG_MAX_VALUE', aws_asg_max_value) lines = re_sub_lines(lines, 'AWS_EB_NOTIFICATION_EMAIL', aws_eb_notification_email) write_file('%s/.ebextensions/%s.config' % (environment_path, name), lines) ################################################################################ print_message('git clone') subprocess.Popen(['rm', '-rf', './%s' % name], cwd=environment_path).communicate() if phase == 'dv': git_command = ['git', 'clone', '--depth=1', git_url] else: git_command = ['git', 'clone', '--depth=1', '-b', phase, git_url] subprocess.Popen(git_command, cwd=environment_path).communicate() if not os.path.exists('%s/%s' % (environment_path, name)): raise Exception() git_hash_app = subprocess.Popen(git_rev, stdout=subprocess.PIPE, cwd='%s/%s' % (environment_path, name)).communicate()[0] subprocess.Popen(['rm', '-rf', './%s/.git' % name], cwd=environment_path).communicate() subprocess.Popen(['rm', '-rf', './%s/.gitignore' % name], cwd=environment_path).communicate() ################################################################################ for ss in settings['SETTINGS_LOCAL_PATH']: lines = read_file('%s/%s/settings_local_sample.py' % (environment_path, ss)) lines = re_sub_lines(lines, '^(DEBUG).*', '\\1 = %s' % debug) option_list = list() option_list.append(['PHASE', phase]) for key in settings: value = settings[key] option_list.append([key, value]) for oo in option_list: lines = re_sub_lines(lines, '^(%s) .*' % oo[0], '\\1 = \'%s\'' % oo[1]) write_file('%s/%s/settings_local.py' % (environment_path, ss), lines) ################################################################################ print_message('check previous version') cmd = ['elasticbeanstalk', 'describe-environments'] cmd += ['--application-name', eb_application_name] result = aws_cli.run(cmd) for r in result['Environments']: if 'CNAME' not in r: continue if r['CNAME'] == '%s.%s.elasticbeanstalk.com' % (cname, aws_default_region): if r['Status'] == 'Terminated': continue elif r['Status'] != 'Ready': print('previous version is not ready.') raise Exception() eb_environment_name_old = r['EnvironmentName'] cname += '-%s' % str_timestamp break ################################################################################ print_message('create storage location') cmd = ['elasticbeanstalk', 'create-storage-location'] result = aws_cli.run(cmd) s3_bucket = result['S3Bucket'] s3_zip_filename = '/'.join(['s3://' + s3_bucket, eb_application_name, zip_filename]) ################################################################################ print_message('create application version') cmd = ['zip', '-r', zip_filename, '.', '.ebextensions'] subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=environment_path).communicate() cmd = ['s3', 'cp', zip_filename, s3_zip_filename] aws_cli.run(cmd, cwd=environment_path) cmd = ['rm', '-rf', zip_filename] subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=environment_path).communicate() cmd = ['elasticbeanstalk', 'create-application-version'] cmd += ['--application-name', eb_application_name] cmd += ['--source-bundle', 'S3Bucket="%s",S3Key="%s/%s"' % (s3_bucket, eb_application_name, zip_filename)] cmd += ['--version-label', eb_environment_name] aws_cli.run(cmd, cwd=environment_path) ################################################################################ print_message('create environment %s' % name) option_settings = list() oo = dict() oo['Namespace'] = 'aws:autoscaling:launchconfiguration' oo['OptionName'] = 'EC2KeyName' oo['Value'] = key_pair_name option_settings.append(oo) oo = dict() oo['Namespace'] = 'aws:autoscaling:launchconfiguration' oo['OptionName'] = 'InstanceType' oo['Value'] = 't2.micro' option_settings.append(oo) oo = dict() oo['Namespace'] = 'aws:autoscaling:launchconfiguration' oo['OptionName'] = 'IamInstanceProfile' oo['Value'] = 'aws-elasticbeanstalk-ec2-role' option_settings.append(oo) oo = dict() oo['Namespace'] = 'aws:autoscaling:launchconfiguration' oo['OptionName'] = 'SecurityGroups' oo['Value'] = security_group_id option_settings.append(oo) oo = dict() oo['Namespace'] = 'aws:ec2:vpc' oo['OptionName'] = 'AssociatePublicIpAddress' oo['Value'] = 'true' if 'private' == subnet_type: oo['Value'] = 'false' option_settings.append(oo) oo = dict() oo['Namespace'] = 'aws:ec2:vpc' oo['OptionName'] = 'ELBScheme' oo['Value'] = '...' if 'private' == subnet_type: oo['Value'] = 'internal' option_settings.append(oo) oo = dict() oo['Namespace'] = 'aws:ec2:vpc' oo['OptionName'] = 'ELBSubnets' oo['Value'] = ','.join([subnet_id_1, subnet_id_2]) option_settings.append(oo) oo = dict() oo['Namespace'] = 'aws:ec2:vpc' oo['OptionName'] = 'Subnets' oo['Value'] = ','.join([subnet_id_1, subnet_id_2]) option_settings.append(oo) oo = dict() oo['Namespace'] = 'aws:ec2:vpc' oo['OptionName'] = 'VPCId' oo['Value'] = eb_vpc_id option_settings.append(oo) oo = dict() oo['Namespace'] = 'aws:elasticbeanstalk:environment' oo['OptionName'] = 'EnvironmentType' oo['Value'] = 'LoadBalanced' option_settings.append(oo) oo = dict() oo['Namespace'] = 'aws:elasticbeanstalk:environment' oo['OptionName'] = 'ServiceRole' oo['Value'] = 'aws-elasticbeanstalk-service-role' option_settings.append(oo) oo = dict() oo['Namespace'] = 'aws:elasticbeanstalk:healthreporting:system' oo['OptionName'] = 'SystemType' oo['Value'] = 'enhanced' option_settings.append(oo) oo = dict() oo['Namespace'] = 'aws:elasticbeanstalk:healthreporting:system' oo['OptionName'] = 'ConfigDocument' cw_instance = dict() cw_instance['RootFilesystemUtil'] = 60 cw_instance['InstanceHealth'] = 60 cw_instance['CPUIdle'] = 60 cw = dict() cw['Instance'] = cw_instance cfg_doc = dict() cfg_doc['CloudWatchMetrics'] = cw cfg_doc['Version'] = 1 oo['Value'] = json.dumps(cfg_doc) option_settings.append(oo) oo = dict() oo['Namespace'] = 'aws:elasticbeanstalk:cloudwatch:logs' oo['OptionName'] = 'StreamLogs' oo['Value'] = 'true' option_settings.append(oo) oo = dict() oo['Namespace'] = 'aws:elasticbeanstalk:cloudwatch:logs' oo['OptionName'] = 'DeleteOnTerminate' oo['Value'] = 'true' option_settings.append(oo) oo = dict() oo['Namespace'] = 'aws:elasticbeanstalk:cloudwatch:logs' oo['OptionName'] = 'RetentionInDays' oo['Value'] = '3' option_settings.append(oo) option_settings = json.dumps(option_settings) tag0 = 'Key=git_hash_johanna,Value=%s' % git_hash_johanna.decode('utf-8').strip() tag1 = 'Key=git_hash_%s,Value=%s' % (template_name, git_hash_template.decode('utf-8').strip()) tag2 = 'Key=git_hash_%s,Value=%s' % (name, git_hash_app.decode('utf-8').strip()) cmd = ['elasticbeanstalk', 'create-environment'] cmd += ['--application-name', eb_application_name] cmd += ['--cname-prefix', cname] cmd += ['--environment-name', eb_environment_name] cmd += ['--option-settings', option_settings] cmd += ['--solution-stack-name', '64bit Amazon Linux 2018.03 v2.7.1 running Python 3.6'] cmd += ['--tags', tag0, tag1, tag2] cmd += ['--version-label', eb_environment_name] aws_cli.run(cmd, cwd=environment_path) elapsed_time = 0 while True: cmd = ['elasticbeanstalk', 'describe-environments'] cmd += ['--application-name', eb_application_name] cmd += ['--environment-name', eb_environment_name] result = aws_cli.run(cmd) ee = result['Environments'][0] print(json.dumps(ee, sort_keys=True, indent=4)) if ee.get('Health', '') == 'Green' and ee.get('Status', '') == 'Ready': break print('creating... (elapsed time: \'%d\' seconds)' % elapsed_time) time.sleep(5) elapsed_time += 5 if elapsed_time > 60 * 30: raise Exception() subprocess.Popen(['rm', '-rf', './%s' % name], cwd=environment_path).communicate() ################################################################################ print_message('revoke security group ingress') cmd = ['ec2', 'describe-security-groups'] cmd += ['--filters', 'Name=tag-key,Values=Name,Name=tag-value,Values=%s' % eb_environment_name] result = aws_cli.run(cmd) for ss in result['SecurityGroups']: cmd = ['ec2', 'revoke-security-group-ingress'] cmd += ['--group-id', ss['GroupId']] cmd += ['--protocol', 'tcp'] cmd += ['--port', '22'] cmd += ['--cidr', '0.0.0.0/0'] aws_cli.run(cmd, ignore_error=True) ################################################################################ if private_ip is not None: print_message('attach network interface') elapsed_time = 0 while True: cmd = ['ec2', 'describe-network-interfaces'] cmd += ['--filters', 'Name=private-ip-address,Values=%s' % private_ip] result = aws_cli.run(cmd) network_interface_id = result['NetworkInterfaces'][0]['NetworkInterfaceId'] if 'Attachment' not in result['NetworkInterfaces'][0]: cmd = ['ec2', 'describe-instances'] cmd += ['--filters', 'Name=tag-key,Values=Name,Name=tag-value,Values=%s' % eb_environment_name] result = aws_cli.run(cmd) instance_id = result['Reservations'][0]['Instances'][0]['InstanceId'] cmd = ['ec2', 'attach-network-interface'] cmd += ['--network-interface-id', network_interface_id] cmd += ['--instance-id', instance_id] cmd += ['--device-index', '1'] aws_cli.run(cmd) break attachment_id = result['NetworkInterfaces'][0]['Attachment']['AttachmentId'] cmd = ['ec2', 'detach-network-interface'] cmd += ['--attachment-id', attachment_id] aws_cli.run(cmd, ignore_error=True) print('detaching network interface... (elapsed time: \'%d\' seconds)' % elapsed_time) time.sleep(5) elapsed_time += 5 ################################################################################ print_message('swap CNAME if the previous version exists') if eb_environment_name_old: cmd = ['elasticbeanstalk', 'swap-environment-cnames'] cmd += ['--source-environment-name', eb_environment_name_old] cmd += ['--destination-environment-name', eb_environment_name] aws_cli.run(cmd)