def __init__(self,
kms_key_arn,
secrets_s3_path,
vpc_subnet,
security_group_ids,
region,
availability_zones,
publicly_accessible,
cluster_tags,
dart_host,
dart_port,
dart_api_version=1):
self.dart = Dart(dart_host, dart_port, dart_api_version)
self._action_handlers = {
RedshiftActionTypes.start_datastore.name: start_datastore,
RedshiftActionTypes.stop_datastore.name: stop_datastore,
RedshiftActionTypes.execute_sql.name: execute_sql,
RedshiftActionTypes.load_dataset.name: load_dataset,
RedshiftActionTypes.consume_subscription.name:
consume_subscription,
RedshiftActionTypes.copy_to_s3.name: copy_to_s3,
RedshiftActionTypes.create_snapshot.name: create_snapshot,
RedshiftActionTypes.data_check.name: data_check,
RedshiftActionTypes.cluster_maintenance.name: cluster_maintenance,
}
self.vpc_subnet = vpc_subnet
self.availability_zones = availability_zones
self.publicly_accessible = publicly_accessible
self.security_group_ids = security_group_ids
self.cluster_tags = cluster_tags
self.region = region
self.secrets = Secrets(kms_key_arn, secrets_s3_path)
def __init__(self,
kms_key_arn,
secrets_s3_path,
dart_host,
dart_port,
dart_api_version=1):
self.dart = Dart(dart_host, dart_port, dart_api_version)
self._action_handlers = {
ElasticsearchActionTypes.data_check.name: data_check,
ElasticsearchActionTypes.create_index.name: create_index,
ElasticsearchActionTypes.create_template.name: create_template,
ElasticsearchActionTypes.create_mapping.name: create_mapping,
ElasticsearchActionTypes.delete_index.name: delete_index,
ElasticsearchActionTypes.delete_template.name: delete_template,
ElasticsearchActionTypes.force_merge_index.name: force_merge_index,
}
self.secrets = Secrets(kms_key_arn, secrets_s3_path)
def configuration(config_path, suppress_decryption=False):
def env(loader, node):
pattern = re.compile(r'\$\{(.+?)\}')
raw_value = loader.construct_scalar(node)
result = str(raw_value)
try:
for env_var in re.findall(pattern, raw_value):
value = os.environ.get(env_var)
if not value:
raise Exception(
'empty environment variable found (%s) in config file: %s'
% (env_var, config_path))
result = result.replace('${%s}' % env_var, os.environ[env_var])
return str(result)
except:
raise Exception(
'could not interpolate !env value (%s) in config file: %s' %
(raw_value, config_path))
yaml.add_constructor('!env', env)
raw_file_data = _get_raw_config_data(config_path)
# this first load has a pass-through decryption function, which allows us to find the secrets_s3_path
def decrypt(loader, node):
return '!decrypt %s' % loader.construct_scalar(node)
yaml.add_constructor('!decrypt', decrypt)
config = yaml.load(raw_file_data)
if not suppress_decryption:
secrets_config = get_secrets_config(config)
secrets_service = Secrets(secrets_config['kms_key_arn'],
secrets_config['secrets_s3_path'])
# now we can instantiate the secrets service to perform any necessary decryption
def decrypt(loader, node):
return secrets_service.get(loader.construct_scalar(node))
yaml.add_constructor('!decrypt', decrypt)
return yaml.load(raw_file_data)
def create_partial(self, output_config):
_logger.info('updating configuration with trigger queue urls/arns')
trigger_queue_arn, trigger_queue_url = self._ensure_queue_exists(
output_config, 'trigger_queue')
events_params = output_config['cloudformation_stacks']['events'][
'boto_args']['Parameters']
_get_element(events_params, 'ParameterKey',
'TriggerQueueUrl')['ParameterValue'] = trigger_queue_url
_get_element(events_params, 'ParameterKey',
'TriggerQueueArn')['ParameterValue'] = trigger_queue_arn
_logger.info('creating initial stacks')
events_stack_name = self._create_stack('events', self.mode,
output_config)
rds_stack_name = self._create_stack('rds', self.mode, output_config)
elb_stack_name = self._create_stack('elb', self.mode, output_config)
elb_int_stack_name = self._create_stack('elb-internal', self.mode,
output_config)
engine_taskrunner_stack_name = self._create_stack(
'engine-taskrunner', self.mode, output_config)
_logger.info('waiting for stack completion')
events_outputs = self._wait_for_stack_completion_and_get_outputs(
events_stack_name, 1)
rds_outputs = self._wait_for_stack_completion_and_get_outputs(
rds_stack_name, 1)
elb_outputs = self._wait_for_stack_completion_and_get_outputs(
elb_stack_name, 1)
elb_int_outputs = self._wait_for_stack_completion_and_get_outputs(
elb_int_stack_name, 1)
engine_taskrunner_outputs = self._wait_for_stack_completion_and_get_outputs(
engine_taskrunner_stack_name, 1)
_logger.info(
'updating configuration with new cloudwatch scheduled events sns topic name'
)
sns_arn = events_outputs[0]['OutputValue']
output_config['triggers']['scheduled'][
'cloudwatch_scheduled_events_sns_arn'] = sns_arn
_logger.info(
'updating configuration with new rds endpoint and password')
db_uri_secret_key = 'database-uri-%s' % self.environment_name
output_config['flask'][
'SQLALCHEMY_DATABASE_URI'] = '!decrypt %s' % db_uri_secret_key
secrets_config = get_secrets_config(output_config)
secrets_service = Secrets(secrets_config['kms_key_arn'],
secrets_config['secrets_s3_path'])
rds_pwd = os.environ['DART_RDS_PASSWORD']
rds_host = rds_outputs[0]['OutputValue']
secrets_service.put(
db_uri_secret_key,
'postgresql://*****:*****@%s:5432/dart' % (rds_pwd, rds_host))
_logger.info('updating configuration with new elb name')
web_params = output_config['cloudformation_stacks']['web'][
'boto_args']['Parameters']
elb_name_param = _get_element(web_params, 'ParameterKey',
'WebEcsServiceLoadBalancerName')
elb_name = elb_outputs[0]['OutputValue']
elb_name_param['ParameterValue'] = elb_name
_logger.info('updating configuration with new internal elb name')
web_int_params = output_config['cloudformation_stacks'][
'web-internal']['boto_args']['Parameters']
elb_int_name_param = _get_element(web_int_params, 'ParameterKey',
'WebEcsServiceLoadBalancerName')
elb_int_name = elb_int_outputs[0]['OutputValue']
elb_int_name_param['ParameterValue'] = elb_int_name
_logger.info(
'updating configuration with new engine taskrunner ecs cluster name'
)
output_config['dart'][
'engine_taskrunner_ecs_cluster'] = engine_taskrunner_outputs[0][
'OutputValue']
_logger.info(
'updating configuration with encrypted dart email username/password'
)
mailer_options = output_config['email']['mailer']
mailer_options['usr'] = '******'
mailer_options['pwd'] = '!decrypt email-password'
secrets_service.put('email-username', self.dart_email_username)
secrets_service.put('email-password', self.dart_email_password)
_logger.info('uploading the output configuration to s3')
body = yaml.dump(output_config, default_flow_style=False)
body = re.sub(r"'!decrypt (.+?)'", r"!decrypt \1", body)
body = re.sub(r"'!env (.+?)'", r"!env \1", body)
body = re.sub(r"__DARTBANG__", r"!", body)
body = re.sub(r"__DARTQUOTE__", r"'", body)
body = re.sub(r"__DARTDOLLAR__", r"$", body)
boto3.client('s3').put_object(
Bucket=get_bucket_name(self.output_config_s3_path),
Key=get_key_name(self.output_config_s3_path),
Body=body)
_logger.info('creating and waiting for web stacks')
web_stack_name = self._create_stack('web', self.mode, output_config)
web_internal_stack_name = self._create_stack('web-internal', self.mode,
output_config)
web_outputs = self._wait_for_stack_completion_and_get_outputs(
web_stack_name, 2)
self._wait_for_stack_completion_and_get_outputs(
web_internal_stack_name)
_logger.info('waiting for web ecs service to stabilize')
cluster_name = _get_element(web_outputs, 'OutputKey',
'EcsClusterResourceName')['OutputValue']
service_name = _get_element(web_outputs, 'OutputKey',
'WebEcsServiceResourceName')['OutputValue']
boto3.client('ecs').get_waiter('services_stable').wait(
cluster=cluster_name, services=[service_name])
_logger.info('done')
_logger.info('waiting for web app to attach to load balancer')
self._wait_for_web_app(elb_name)
time.sleep(5)
_logger.info('initializing database schema')
dart_host = _get_dart_host(output_config)
response = requests.post('http://%s/admin/create_all' % dart_host)
response.raise_for_status()
time.sleep(5)
_logger.info('creating database triggers')
with open(dart_root_relative_path('src', 'database',
'triggers.sql')) as f:
engine = sqlalchemy.create_engine(
'postgresql://*****:*****@%s:5432/dart' % (rds_pwd, rds_host))
engine.execute(f.read())
_logger.info('done')
time.sleep(5)
_logger.info('adding engines')
self._with_retries(add_no_op_engine, output_config)
self._with_retries(add_no_op_engine_sub_graphs, output_config)
self._with_retries(add_emr_engine, output_config)
self._with_retries(add_emr_engine_sub_graphs, output_config)
self._with_retries(add_dynamodb_engine, output_config)
self._with_retries(add_redshift_engine, output_config)
self._with_retries(add_s3_engine, output_config)
_logger.info('creating and waiting for remaining stacks')
engine_worker_stack_name = self._create_stack('engine-worker',
self.mode, output_config)
trigger_worker_stack_name = self._create_stack('trigger-worker',
self.mode,
output_config)
subscription_worker_stack_name = self._create_stack(
'subscription-worker', self.mode, output_config)
self._wait_for_stack_completion_and_get_outputs(
engine_worker_stack_name)
self._wait_for_stack_completion_and_get_outputs(
trigger_worker_stack_name)
self._wait_for_stack_completion_and_get_outputs(
subscription_worker_stack_name)