def __init__(self, kms_key_arn, secrets_s3_path, vpc_subnet, security_group_ids, region, availability_zones, publicly_accessible, cluster_tags, dart_host, dart_port, dart_api_version=1): self.dart = Dart(dart_host, dart_port, dart_api_version) self._action_handlers = { RedshiftActionTypes.start_datastore.name: start_datastore, RedshiftActionTypes.stop_datastore.name: stop_datastore, RedshiftActionTypes.execute_sql.name: execute_sql, RedshiftActionTypes.load_dataset.name: load_dataset, RedshiftActionTypes.consume_subscription.name: consume_subscription, RedshiftActionTypes.copy_to_s3.name: copy_to_s3, RedshiftActionTypes.create_snapshot.name: create_snapshot, RedshiftActionTypes.data_check.name: data_check, RedshiftActionTypes.cluster_maintenance.name: cluster_maintenance, } self.vpc_subnet = vpc_subnet self.availability_zones = availability_zones self.publicly_accessible = publicly_accessible self.security_group_ids = security_group_ids self.cluster_tags = cluster_tags self.region = region self.secrets = Secrets(kms_key_arn, secrets_s3_path)
def __init__(self, kms_key_arn, secrets_s3_path, dart_host, dart_port, dart_api_version=1): self.dart = Dart(dart_host, dart_port, dart_api_version) self._action_handlers = { ElasticsearchActionTypes.data_check.name: data_check, ElasticsearchActionTypes.create_index.name: create_index, ElasticsearchActionTypes.create_template.name: create_template, ElasticsearchActionTypes.create_mapping.name: create_mapping, ElasticsearchActionTypes.delete_index.name: delete_index, ElasticsearchActionTypes.delete_template.name: delete_template, ElasticsearchActionTypes.force_merge_index.name: force_merge_index, } self.secrets = Secrets(kms_key_arn, secrets_s3_path)
def configuration(config_path, suppress_decryption=False): def env(loader, node): pattern = re.compile(r'\$\{(.+?)\}') raw_value = loader.construct_scalar(node) result = str(raw_value) try: for env_var in re.findall(pattern, raw_value): value = os.environ.get(env_var) if not value: raise Exception( 'empty environment variable found (%s) in config file: %s' % (env_var, config_path)) result = result.replace('${%s}' % env_var, os.environ[env_var]) return str(result) except: raise Exception( 'could not interpolate !env value (%s) in config file: %s' % (raw_value, config_path)) yaml.add_constructor('!env', env) raw_file_data = _get_raw_config_data(config_path) # this first load has a pass-through decryption function, which allows us to find the secrets_s3_path def decrypt(loader, node): return '!decrypt %s' % loader.construct_scalar(node) yaml.add_constructor('!decrypt', decrypt) config = yaml.load(raw_file_data) if not suppress_decryption: secrets_config = get_secrets_config(config) secrets_service = Secrets(secrets_config['kms_key_arn'], secrets_config['secrets_s3_path']) # now we can instantiate the secrets service to perform any necessary decryption def decrypt(loader, node): return secrets_service.get(loader.construct_scalar(node)) yaml.add_constructor('!decrypt', decrypt) return yaml.load(raw_file_data)
def create_partial(self, output_config): _logger.info('updating configuration with trigger queue urls/arns') trigger_queue_arn, trigger_queue_url = self._ensure_queue_exists( output_config, 'trigger_queue') events_params = output_config['cloudformation_stacks']['events'][ 'boto_args']['Parameters'] _get_element(events_params, 'ParameterKey', 'TriggerQueueUrl')['ParameterValue'] = trigger_queue_url _get_element(events_params, 'ParameterKey', 'TriggerQueueArn')['ParameterValue'] = trigger_queue_arn _logger.info('creating initial stacks') events_stack_name = self._create_stack('events', self.mode, output_config) rds_stack_name = self._create_stack('rds', self.mode, output_config) elb_stack_name = self._create_stack('elb', self.mode, output_config) elb_int_stack_name = self._create_stack('elb-internal', self.mode, output_config) engine_taskrunner_stack_name = self._create_stack( 'engine-taskrunner', self.mode, output_config) _logger.info('waiting for stack completion') events_outputs = self._wait_for_stack_completion_and_get_outputs( events_stack_name, 1) rds_outputs = self._wait_for_stack_completion_and_get_outputs( rds_stack_name, 1) elb_outputs = self._wait_for_stack_completion_and_get_outputs( elb_stack_name, 1) elb_int_outputs = self._wait_for_stack_completion_and_get_outputs( elb_int_stack_name, 1) engine_taskrunner_outputs = self._wait_for_stack_completion_and_get_outputs( engine_taskrunner_stack_name, 1) _logger.info( 'updating configuration with new cloudwatch scheduled events sns topic name' ) sns_arn = events_outputs[0]['OutputValue'] output_config['triggers']['scheduled'][ 'cloudwatch_scheduled_events_sns_arn'] = sns_arn _logger.info( 'updating configuration with new rds endpoint and password') db_uri_secret_key = 'database-uri-%s' % self.environment_name output_config['flask'][ 'SQLALCHEMY_DATABASE_URI'] = '!decrypt %s' % db_uri_secret_key secrets_config = get_secrets_config(output_config) secrets_service = Secrets(secrets_config['kms_key_arn'], secrets_config['secrets_s3_path']) rds_pwd = os.environ['DART_RDS_PASSWORD'] rds_host = rds_outputs[0]['OutputValue'] secrets_service.put( db_uri_secret_key, 'postgresql://*****:*****@%s:5432/dart' % (rds_pwd, rds_host)) _logger.info('updating configuration with new elb name') web_params = output_config['cloudformation_stacks']['web'][ 'boto_args']['Parameters'] elb_name_param = _get_element(web_params, 'ParameterKey', 'WebEcsServiceLoadBalancerName') elb_name = elb_outputs[0]['OutputValue'] elb_name_param['ParameterValue'] = elb_name _logger.info('updating configuration with new internal elb name') web_int_params = output_config['cloudformation_stacks'][ 'web-internal']['boto_args']['Parameters'] elb_int_name_param = _get_element(web_int_params, 'ParameterKey', 'WebEcsServiceLoadBalancerName') elb_int_name = elb_int_outputs[0]['OutputValue'] elb_int_name_param['ParameterValue'] = elb_int_name _logger.info( 'updating configuration with new engine taskrunner ecs cluster name' ) output_config['dart'][ 'engine_taskrunner_ecs_cluster'] = engine_taskrunner_outputs[0][ 'OutputValue'] _logger.info( 'updating configuration with encrypted dart email username/password' ) mailer_options = output_config['email']['mailer'] mailer_options['usr'] = '******' mailer_options['pwd'] = '!decrypt email-password' secrets_service.put('email-username', self.dart_email_username) secrets_service.put('email-password', self.dart_email_password) _logger.info('uploading the output configuration to s3') body = yaml.dump(output_config, default_flow_style=False) body = re.sub(r"'!decrypt (.+?)'", r"!decrypt \1", body) body = re.sub(r"'!env (.+?)'", r"!env \1", body) body = re.sub(r"__DARTBANG__", r"!", body) body = re.sub(r"__DARTQUOTE__", r"'", body) body = re.sub(r"__DARTDOLLAR__", r"$", body) boto3.client('s3').put_object( Bucket=get_bucket_name(self.output_config_s3_path), Key=get_key_name(self.output_config_s3_path), Body=body) _logger.info('creating and waiting for web stacks') web_stack_name = self._create_stack('web', self.mode, output_config) web_internal_stack_name = self._create_stack('web-internal', self.mode, output_config) web_outputs = self._wait_for_stack_completion_and_get_outputs( web_stack_name, 2) self._wait_for_stack_completion_and_get_outputs( web_internal_stack_name) _logger.info('waiting for web ecs service to stabilize') cluster_name = _get_element(web_outputs, 'OutputKey', 'EcsClusterResourceName')['OutputValue'] service_name = _get_element(web_outputs, 'OutputKey', 'WebEcsServiceResourceName')['OutputValue'] boto3.client('ecs').get_waiter('services_stable').wait( cluster=cluster_name, services=[service_name]) _logger.info('done') _logger.info('waiting for web app to attach to load balancer') self._wait_for_web_app(elb_name) time.sleep(5) _logger.info('initializing database schema') dart_host = _get_dart_host(output_config) response = requests.post('http://%s/admin/create_all' % dart_host) response.raise_for_status() time.sleep(5) _logger.info('creating database triggers') with open(dart_root_relative_path('src', 'database', 'triggers.sql')) as f: engine = sqlalchemy.create_engine( 'postgresql://*****:*****@%s:5432/dart' % (rds_pwd, rds_host)) engine.execute(f.read()) _logger.info('done') time.sleep(5) _logger.info('adding engines') self._with_retries(add_no_op_engine, output_config) self._with_retries(add_no_op_engine_sub_graphs, output_config) self._with_retries(add_emr_engine, output_config) self._with_retries(add_emr_engine_sub_graphs, output_config) self._with_retries(add_dynamodb_engine, output_config) self._with_retries(add_redshift_engine, output_config) self._with_retries(add_s3_engine, output_config) _logger.info('creating and waiting for remaining stacks') engine_worker_stack_name = self._create_stack('engine-worker', self.mode, output_config) trigger_worker_stack_name = self._create_stack('trigger-worker', self.mode, output_config) subscription_worker_stack_name = self._create_stack( 'subscription-worker', self.mode, output_config) self._wait_for_stack_completion_and_get_outputs( engine_worker_stack_name) self._wait_for_stack_completion_and_get_outputs( trigger_worker_stack_name) self._wait_for_stack_completion_and_get_outputs( subscription_worker_stack_name)
def create_partial(self, output_config): _logger.info('updating configuration with trigger queue urls/arns') trigger_queue_arn, trigger_queue_url = self._ensure_queue_exists(output_config, 'trigger_queue') events_params = output_config['cloudformation_stacks']['events']['boto_args']['Parameters'] self._get_element(events_params, 'ParameterKey', 'TriggerQueueUrl')['ParameterValue'] = trigger_queue_url self._get_element(events_params, 'ParameterKey', 'TriggerQueueArn')['ParameterValue'] = trigger_queue_arn _logger.info('creating initial stacks') events_stack_name = self._create_stack('events', output_config) rds_stack_name = self._create_stack('rds', output_config) elb_stack_name = self._create_stack('elb', output_config) elb_int_stack_name = self._create_stack('elb-internal', output_config) engine_taskrunner_stack_name = self._create_stack('engine-taskrunner', output_config) _logger.info('waiting for stack completion') events_outputs = self._wait_for_stack_completion_and_get_outputs(events_stack_name, 1) rds_outputs = self._wait_for_stack_completion_and_get_outputs(rds_stack_name, 1) elb_outputs = self._wait_for_stack_completion_and_get_outputs(elb_stack_name, 1) elb_int_outputs = self._wait_for_stack_completion_and_get_outputs(elb_int_stack_name, 1) engine_taskrunner_outputs = self._wait_for_stack_completion_and_get_outputs(engine_taskrunner_stack_name, 1) _logger.info('updating configuration with new cloudwatch scheduled events sns topic name') sns_arn = events_outputs[0]['OutputValue'] output_config['triggers']['scheduled']['cloudwatch_scheduled_events_sns_arn'] = sns_arn _logger.info('updating configuration with new rds endpoint and password') db_uri_secret_key = 'database-uri-%s' % self.environment_name output_config['flask']['SQLALCHEMY_DATABASE_URI'] = '!decrypt %s' % db_uri_secret_key secrets_config = get_secrets_config(output_config) secrets_service = Secrets(secrets_config['kms_key_arn'], secrets_config['secrets_s3_path']) rds_pwd = os.environ['DART_RDS_PASSWORD'] rds_host = rds_outputs[0]['OutputValue'] secrets_service.put(db_uri_secret_key, 'postgresql://*****:*****@%s:5432/dart' % (rds_pwd, rds_host)) _logger.info('updating configuration with new elb name') web_params = output_config['cloudformation_stacks']['web']['boto_args']['Parameters'] elb_name_param = self._get_element(web_params, 'ParameterKey', 'WebEcsServiceLoadBalancerName') elb_name = elb_outputs[0]['OutputValue'] elb_name_param['ParameterValue'] = elb_name _logger.info('updating configuration with new internal elb name') web_int_params = output_config['cloudformation_stacks']['web-internal']['boto_args']['Parameters'] elb_int_name_param = self._get_element(web_int_params, 'ParameterKey', 'WebEcsServiceLoadBalancerName') elb_int_name = elb_int_outputs[0]['OutputValue'] elb_int_name_param['ParameterValue'] = elb_int_name _logger.info('updating configuration with new engine taskrunner ecs cluster name') output_config['dart']['engine_taskrunner_ecs_cluster'] = engine_taskrunner_outputs[0]['OutputValue'] _logger.info('updating configuration with encrypted dart email username/password') mailer_options = output_config['email']['mailer'] mailer_options['usr'] = '******' mailer_options['pwd'] = '!decrypt email-password' secrets_service.put('email-username', self.dart_email_username) secrets_service.put('email-password', self.dart_email_password) _logger.info('uploading the output configuration to s3') body = yaml.dump(output_config, default_flow_style=False) body = re.sub(r"'!decrypt (.+?)'", r"!decrypt \1", body) body = re.sub(r"'!env (.+?)'", r"!env \1", body) body = re.sub(r"__DARTBANG__", r"!", body) body = re.sub(r"__DARTQUOTE__", r"'", body) body = re.sub(r"__DARTDOLLAR__", r"$", body) boto3.client('s3').put_object( Bucket=get_bucket_name(self.output_config_s3_path), Key=get_key_name(self.output_config_s3_path), Body=body ) _logger.info('creating and waiting for web stacks') web_stack_name = self._create_stack('web', output_config) web_internal_stack_name = self._create_stack('web-internal', output_config) web_outputs = self._wait_for_stack_completion_and_get_outputs(web_stack_name, 2) self._wait_for_stack_completion_and_get_outputs(web_internal_stack_name) _logger.info('waiting for web ecs service to stabilize') cluster_name = self._get_element(web_outputs, 'OutputKey', 'EcsClusterResourceName')['OutputValue'] service_name = self._get_element(web_outputs, 'OutputKey', 'WebEcsServiceResourceName')['OutputValue'] boto3.client('ecs').get_waiter('services_stable').wait(cluster=cluster_name, services=[service_name]) _logger.info('done') _logger.info('waiting for web app to attach to load balancer') self._wait_for_web_app(elb_name) time.sleep(5) _logger.info('initializing database schema') dart_host = self._get_dart_host(output_config) response = requests.post('http://%s/admin/create_all' % dart_host) response.raise_for_status() time.sleep(5) _logger.info('creating database triggers') with open(dart_root_relative_path('src', 'database', 'triggers.sql')) as f: engine = sqlalchemy.create_engine('postgresql://*****:*****@%s:5432/dart' % (rds_pwd, rds_host)) engine.execute(f.read()) _logger.info('done') time.sleep(5) _logger.info('adding engines') add_no_op_engine(output_config) add_no_op_engine_sub_graphs(output_config) add_emr_engine(output_config) add_emr_engine_sub_graphs(output_config) add_redshift_engine(output_config) _logger.info('creating and waiting for remaining stacks') engine_worker_stack_name = self._create_stack('engine-worker', output_config) trigger_worker_stack_name = self._create_stack('trigger-worker', output_config) subscription_worker_stack_name = self._create_stack('subscription-worker', output_config) self._wait_for_stack_completion_and_get_outputs(engine_worker_stack_name) self._wait_for_stack_completion_and_get_outputs(trigger_worker_stack_name) self._wait_for_stack_completion_and_get_outputs(subscription_worker_stack_name)