def test_firehose_enabled_log(self):
"""CLI - Terraform Generate Kinesis Firehose, Enabled Log"""
cluster_dict = common.infinitedict()
# Add an enabled log, with no alarm configuration (aka: alarms disabled)
self.config['global']['infrastructure']['firehose']['enabled_logs'] = {
'json:embedded': {}
}
firehose.generate_firehose(self._logging_bucket_name, cluster_dict,
self.config)
expected_result = {
'module': {
'kinesis_firehose_setup': self._default_firehose_config(),
'kinesis_firehose_json_embedded': {
'source': './modules/tf_kinesis_firehose_delivery_stream',
'buffer_size': 128,
'buffer_interval': 900,
'file_format': 'parquet',
'stream_name': 'unit_test_streamalert_json_embedded',
'role_arn':
'${module.kinesis_firehose_setup.firehose_role_arn}',
's3_bucket_name': 'unit-test-streamalert-data',
'kms_key_arn': '${aws_kms_key.server_side_encryption.arn}',
'glue_catalog_db_name': 'unit-test_streamalert',
'glue_catalog_table_name': 'json_embedded',
'schema': self._get_expected_schema()
}
}
}
assert_equal(cluster_dict, expected_result)
def test_firehose_enabled_log_alarm_custom(self):
"""CLI - Terraform Generate Kinesis Firehose, Enabled Alarm - Custom Settings"""
cluster_dict = common.infinitedict()
# Add an enabled log, with alarms on with custom settings
self.config['global']['infrastructure']['firehose']['enabled_logs'] = {
'json:embedded': {
'enable_alarm': True,
'evaluation_periods': 10,
'period_seconds': 3600,
'log_min_count_threshold': 100000
}
}
firehose.generate_firehose(self._logging_bucket_name, cluster_dict,
self.config)
expected_result = {
'module': {
'kinesis_firehose_setup': self._default_firehose_config(),
'kinesis_firehose_json_embedded': {
'source':
'./modules/tf_kinesis_firehose_delivery_stream',
'buffer_size':
128,
'buffer_interval':
900,
'file_format':
'parquet',
'stream_name':
'unit_test_streamalert_json_embedded',
'role_arn':
'${module.kinesis_firehose_setup.firehose_role_arn}',
's3_bucket_name':
'unit-test-streamalert-data',
'kms_key_arn':
'${aws_kms_key.server_side_encryption.arn}',
'enable_alarm':
True,
'evaluation_periods':
10,
'period_seconds':
3600,
'alarm_threshold':
100000,
'alarm_actions': [
'arn:aws:sns:us-west-1:12345678910:unit-test_streamalert_monitoring'
],
'glue_catalog_db_name':
'unit-test_streamalert',
'glue_catalog_table_name':
'json_embedded',
'schema':
self._get_expected_schema()
}
}
}
assert_equal(cluster_dict, expected_result)
def test_firehose_defaults(self):
"""CLI - Terraform Generate Kinesis Firehose, Defaults"""
cluster_dict = common.infinitedict()
firehose.generate_firehose(self._logging_bucket_name, cluster_dict,
self.config)
expected_result = {
'module': {
'kinesis_firehose_setup': self._default_firehose_config(),
}
}
assert_equal(cluster_dict, expected_result)
def test_firehose_enabled_log_alarm_defaults(self):
"""CLI - Terraform Generate Kinesis Firehose, Enabled Alarm - Default Settings"""
cluster_dict = common.infinitedict()
# Add an enabled log, with alarms on (will use terraform default settings)
self.config['global']['infrastructure']['firehose']['enabled_logs'] = {
'json:embedded': {
'enable_alarm': True
}
}
firehose.generate_firehose(self._logging_bucket_name, cluster_dict,
self.config)
expected_result = {
'module': {
'kinesis_firehose_setup': self._default_firehose_config(),
'kinesis_firehose_json_embedded': {
'source':
'./modules/tf_kinesis_firehose_delivery_stream',
'buffer_size':
128,
'buffer_interval':
900,
'compression_format':
'GZIP',
'use_prefix':
True,
'prefix':
'unit-test',
'log_name':
'json_embedded',
'role_arn':
'${module.kinesis_firehose_setup.firehose_role_arn}',
's3_bucket_name':
'unit-test-streamalert-data',
'kms_key_arn':
'${aws_kms_key.server_side_encryption.arn}',
'enable_alarm':
True,
'alarm_actions': [
'arn:aws:sns:us-west-1:12345678910:unit-test_streamalert_monitoring'
]
}
}
}
assert_equal(cluster_dict, expected_result)
def generate_main(config, init=False):
"""Generate the main.tf.json Terraform dict
Args:
config (CLIConfig): The loaded CLI config
init (bool): Terraform is running in the init phase or not (optional)
Returns:
dict: main.tf.json Terraform dict
"""
write_vars(config, region=config['global']['account']['region'])
main_dict = infinitedict()
logging_bucket, create_logging_bucket = s3_access_logging_bucket(config)
state_lock_table_name = '{}_streamalert_terraform_state_lock'.format(
config['global']['account']['prefix']
)
# Setup the Backend depending on the deployment phase.
# When first setting up StreamAlert, the Terraform statefile
# is stored locally. After the first dependencies are created,
# this moves to S3.
if init:
main_dict['terraform']['backend']['local'] = {
'path': 'terraform.tfstate',
}
else:
terraform_bucket_name, _ = terraform_state_bucket(config)
main_dict['terraform']['backend']['s3'] = {
'bucket': terraform_bucket_name,
'key': config['global'].get('terraform', {}).get(
'state_key_name',
'streamalert_state/terraform.tfstate'
),
'region': config['global']['account']['region'],
'encrypt': True,
'dynamodb_table': state_lock_table_name,
'acl': 'private',
'kms_key_id': 'alias/{}'.format(
config['global']['account'].get(
'kms_key_alias',
'{}_streamalert_secrets'.format(config['global']['account']['prefix'])
)
),
}
# Configure initial S3 buckets
main_dict['resource']['aws_s3_bucket'] = {
'streamalerts': generate_s3_bucket(
bucket=firehose_alerts_bucket(config),
logging=logging_bucket
)
}
# Configure remote state locking table
main_dict['resource']['aws_dynamodb_table'] = {
'terraform_remote_state_lock': {
'name': state_lock_table_name,
'billing_mode': 'PAY_PER_REQUEST',
'hash_key': 'LockID',
'attribute': {
'name': 'LockID',
'type': 'S'
},
'tags': {
'Name': 'StreamAlert'
}
}
}
# Create bucket for S3 access logs (if applicable)
if create_logging_bucket:
main_dict['resource']['aws_s3_bucket']['logging_bucket'] = generate_s3_bucket(
bucket=logging_bucket,
logging=logging_bucket,
acl='log-delivery-write',
lifecycle_rule={
'prefix': '/',
'enabled': True,
'transition': {
'days': 365,
'storage_class': 'GLACIER'
}
},
sse_algorithm='AES256' # SSE-KMS doesn't seem to work with access logs
)
terraform_bucket_name, create_state_bucket = terraform_state_bucket(config)
# Create bucket for Terraform state (if applicable)
if create_state_bucket:
main_dict['resource']['aws_s3_bucket']['terraform_remote_state'] = generate_s3_bucket(
bucket=terraform_bucket_name,
logging=logging_bucket
)
# Setup Firehose Delivery Streams
generate_firehose(logging_bucket, main_dict, config)
# Configure global resources like Firehose alert delivery and alerts table
main_dict['module']['globals'] = _generate_global_module(config)
# KMS Key and Alias creation
main_dict['resource']['aws_kms_key']['server_side_encryption'] = {
'enable_key_rotation': True,
'description': 'StreamAlert S3 Server-Side Encryption',
'policy': json.dumps({
'Version': '2012-10-17',
'Statement': [
{
'Sid': 'Enable IAM User Permissions',
'Effect': 'Allow',
'Principal': {
'AWS': 'arn:aws:iam::{}:root'.format(
config['global']['account']['aws_account_id']
)
},
'Action': 'kms:*',
'Resource': '*'
},
{
'Sid': 'Allow principals in the account to use the key',
'Effect': 'Allow',
'Principal': '*',
'Action': ['kms:Decrypt', 'kms:GenerateDataKey*', 'kms:Encrypt'],
'Resource': '*',
'Condition': {
'StringEquals': {
'kms:CallerAccount': config['global']['account']['aws_account_id']
}
}
}
]
})
}
main_dict['resource']['aws_kms_alias']['server_side_encryption'] = {
'name': 'alias/{}_server-side-encryption'.format(config['global']['account']['prefix']),
'target_key_id': '${aws_kms_key.server_side_encryption.key_id}'
}
main_dict['resource']['aws_kms_key']['streamalert_secrets'] = {
'enable_key_rotation': True,
'description': 'StreamAlert secret management'
}
main_dict['resource']['aws_kms_alias']['streamalert_secrets'] = {
'name': 'alias/{}'.format(
config['global']['account'].get(
'kms_key_alias',
'{}_streamalert_secrets'.format(config['global']['account']['prefix'])
)
),
'target_key_id': '${aws_kms_key.streamalert_secrets.key_id}'
}
# Global infrastructure settings
topic_name, create_topic = monitoring_topic_name(config)
if create_topic:
main_dict['resource']['aws_sns_topic']['monitoring'] = {
'name': topic_name
}
return main_dict
