def generate_alert_merger(config):
"""Generate Terraform for the Alert Merger
Args:
config (dict): The loaded config from the 'conf/' directory
Returns:
dict: Alert Merger Terraform definition to be marshaled to JSON
"""
prefix = config['global']['account']['prefix']
result = infinitedict()
# Set variables for the alert merger's IAM permissions
result['module']['alert_merger_iam'] = {
'source': 'modules/tf_alert_merger_iam',
'account_id': config['global']['account']['aws_account_id'],
'region': config['global']['account']['region'],
'prefix': config['global']['account']['prefix'],
'role_id': '${module.alert_merger_lambda.role_id}'
}
# Set variables for the Lambda module
result['module']['alert_merger_lambda'] = generate_lambda(
'{}_streamalert_{}'.format(config['global']['account']['prefix'], ALERT_MERGER_NAME),
config['lambda']['alert_merger_config'],
config,
environment={
'ALERTS_TABLE': '{}_streamalert_alerts'.format(prefix),
'ALERT_PROCESSOR': '{}_streamalert_alert_processor'.format(prefix),
'ALERT_PROCESSOR_TIMEOUT_SEC': config['lambda']['alert_processor_config']['timeout'],
}
)
return result
def generate_aggregate_cloudwatch_metric_alarms(config):
"""Return any CloudWatch Metric Alarms for aggregate metrics
Args:
config (dict): The loaded config from the 'conf/' directory
"""
result = infinitedict()
sns_topic_arn = monitoring_topic_arn(config)
for func, func_config in config['lambda'].iteritems():
metric_alarms = func_config.get('custom_metric_alarms')
if not metric_alarms:
continue
func = func.replace('_config', '')
for idx, name in enumerate(metric_alarms):
alarm_settings = metric_alarms[name]
alarm_settings['source'] = 'modules/tf_metric_alarms',
alarm_settings['sns_topic_arn'] = sns_topic_arn
alarm_settings['alarm_name'] = name
result['module']['metric_alarm_{}_{}'.format(func, idx)] = alarm_settings
return result
def test_firehose_enabled_log(self):
"""CLI - Terraform Generate Kinesis Firehose, Enabled Log"""
cluster_dict = common.infinitedict()
# Add an enabled log, with no alarm configuration (aka: alarms disabled)
self.config['global']['infrastructure']['firehose']['enabled_logs'] = {
'json:embedded': {}
}
firehose.generate_firehose(self._logging_bucket_name, cluster_dict, self.config)
expected_result = {
'module': {
'kinesis_firehose_setup': self._default_firehose_config(),
'kinesis_firehose_json_embedded': {
'source': 'modules/tf_stream_alert_kinesis_firehose_delivery_stream',
'buffer_size': 128,
'buffer_interval': 900,
'compression_format': 'GZIP',
'log_name': 'json_embedded',
'role_arn': '${module.kinesis_firehose_setup.firehose_role_arn}',
's3_bucket_name': 'unit-testing.streamalert.data',
'kms_key_arn': '${aws_kms_key.server_side_encryption.arn}'
}
}
}
assert_equal(cluster_dict, expected_result)
def test_kinesis_events():
"""CLI - Terraform Generate Kinesis Events"""
cluster_dict = common.infinitedict()
result = kinesis_events.generate_kinesis_events('advanced', cluster_dict,
CONFIG)
expected_result = {
'module': {
'kinesis_events_advanced': {
'source':
'modules/tf_stream_alert_kinesis_events',
'batch_size':
100,
'lambda_production_enabled':
True,
'lambda_role_id':
'${module.classifier_advanced_lambda.role_id}',
'lambda_function_alias_arn':
('${module.classifier_advanced_lambda.function_alias_arn}'),
'kinesis_stream_arn':
'${module.kinesis_advanced.arn}',
'role_policy_prefix':
'advanced'
}
}
}
assert_true(result)
assert_equal(cluster_dict, expected_result)
def test_generate_cloudwatch_monitoring_custom_sns():
"""CLI - Terraform Generate Cloudwatch Monitoring with Existing SNS Topic"""
# Test a custom SNS topic name
CONFIG['clusters']['test']['modules']['cloudwatch_monitoring'] = {
'enabled': True
}
CONFIG['global']['infrastructure']['monitoring'][
'create_sns_topic'] = False
CONFIG['global']['infrastructure']['monitoring'][
'sns_topic_name'] = 'unit_test_monitoring'
cluster_dict = common.infinitedict()
result = monitoring.generate_monitoring('test', cluster_dict, CONFIG)
expected_cloudwatch_tf_custom = {
'source': 'modules/tf_stream_alert_monitoring',
'sns_topic_arn':
'arn:aws:sns:us-west-1:12345678910:unit_test_monitoring',
'lambda_functions': ['unit-testing_streamalert_classifier_test'],
'kinesis_stream': 'unit-testing_test_stream_alert_kinesis',
'lambda_alarms_enabled': True,
'kinesis_alarms_enabled': True
}
assert_true(result)
assert_equal(cluster_dict['module']['cloudwatch_monitoring_test'],
expected_cloudwatch_tf_custom)
def generate_aggregate_cloudwatch_metric_filters(config):
"""Return the CloudWatch Metric Filters information for aggregate metrics
Args:
config (dict): The loaded config from the 'conf/' directory
"""
functions = {
cluster: [
func.replace('_config', '')
for func in CLUSTERED_FUNCTIONS
if cluster_config['modules']['stream_alert']['{}_config'.format(func)].get(
'enable_custom_metrics'
)
] for cluster, cluster_config in config['clusters'].iteritems()
}
functions['global'] = {
func.replace('_config', '') for func, func_config in config['lambda'].iteritems()
if func_config.get('enable_custom_metrics')
}
if not any(funcs for funcs in functions.values()):
return # Nothing to add if no funcs have metrics enabled
result = infinitedict()
current_metrics = metrics.MetricLogger.get_available_metrics()
for cluster, functions in functions.iteritems():
is_global = cluster == 'global'
for function in functions:
# This function may not actually support any custom metrics
if function not in current_metrics:
continue
metric_prefix = metrics.FUNC_PREFIXES.get(function)
if not metric_prefix:
continue
log_group_name = (
'${{module.{}_{}_lambda.log_group_name}}'.format(function, cluster)
if not is_global else '${{module.{}_lambda.log_group_name}}'.format(function)
)
# Add filters for the cluster and aggregate
for metric, filter_settings in current_metrics[function].iteritems():
module_name = (
'metric_filters_{}_{}_{}'.format(metric_prefix, metric, cluster)
if is_global else 'metric_filters_{}_{}'.format(metric_prefix, metric)
)
result['module'][module_name] = {
'source': 'modules/tf_metric_filters',
'log_group_name': log_group_name,
'metric_name': '{}-{}'.format(metric_prefix, metric),
'metric_pattern': filter_settings[0],
'metric_value': filter_settings[1],
}
return result
def test_kinesis_streams_with_trusted_account():
"""CLI - Terraform Generate Kinesis Streams with trusted account"""
cluster_dict = common.infinitedict()
result = kinesis_streams.generate_kinesis_streams('trusted', cluster_dict,
CONFIG)
expected_result = {
'module': {
'kinesis_trusted': {
'source': 'modules/tf_stream_alert_kinesis_streams',
'account_id': '12345678910',
'shard_level_metrics': [],
'region': 'us-west-1',
'prefix': 'unit-testing',
'cluster_name': 'trusted',
'stream_name': 'unit-testing_trusted_stream_alert_kinesis',
'shards': 1,
'retention': 24,
'create_user': True,
'trusted_accounts': ['98765432100']
}
}
}
assert_true(result)
assert_equal(cluster_dict, expected_result)
def generate_alert_processor(config):
"""Generate Terraform for the Alert Processor
Args:
config (dict): The loaded config from the 'conf/' directory
Returns:
dict: Alert Processor dict to be marshaled to JSON
"""
prefix = config['global']['account']['prefix']
result = infinitedict()
# Set variables for the IAM permissions module
result['module']['alert_processor_iam'] = {
'source': 'modules/tf_alert_processor_iam',
'account_id': config['global']['account']['aws_account_id'],
'region': config['global']['account']['region'],
'prefix': prefix,
'role_id': '${module.alert_processor_lambda.role_id}',
'kms_key_arn': '${aws_kms_key.stream_alert_secrets.arn}',
'output_lambda_functions': [
# Strip qualifiers: only the function name is needed for the IAM permissions
func.split(':')[0] for func in config['outputs'].get('aws-lambda', {}).values()
],
'output_s3_buckets': config['outputs'].get('aws-s3', {}).values(),
'output_sns_topics': config['outputs'].get('aws-sns', {}).values(),
'output_sqs_queues': config['outputs'].get('aws-sqs', {}).values()
}
# Set variables for the Lambda module
result['module']['alert_processor_lambda'] = generate_lambda(
ALERT_PROCESSOR_NAME, config, {'ALERTS_TABLE': '{}_streamalert_alerts'.format(prefix)})
return result
def test_firehose_enabled_log_alarm_defaults(self):
"""CLI - Terraform Generate Kinesis Firehose, Enabled Alarm - Default Settings"""
cluster_dict = common.infinitedict()
# Add an enabled log, with alarms on (will use terraform default settings)
self.config['global']['infrastructure']['firehose']['enabled_logs'] = {
'json:embedded': {
'enable_alarm': True
}
}
firehose.generate_firehose(self._logging_bucket_name, cluster_dict, self.config)
expected_result = {
'module': {
'kinesis_firehose_setup': self._default_firehose_config(),
'kinesis_firehose_json_embedded': {
'source': 'modules/tf_stream_alert_kinesis_firehose_delivery_stream',
'buffer_size': 128,
'buffer_interval': 900,
'compression_format': 'GZIP',
'log_name': 'json_embedded',
'role_arn': '${module.kinesis_firehose_setup.firehose_role_arn}',
's3_bucket_name': 'unit-testing.streamalert.data',
'kms_key_arn': '${aws_kms_key.server_side_encryption.arn}',
'enable_alarm': True,
'alarm_actions': ['arn:aws:sns:us-west-1:12345678910:stream_alert_monitoring']
}
}
}
assert_equal(cluster_dict, expected_result)
def test_generate_s3_events():
"""CLI - Terraform - S3 Events with Valid Buckets"""
cluster_dict = common.infinitedict()
result = s3_events.generate_s3_events('advanced', cluster_dict, CONFIG)
expected_config = {
'module': {
's3_events_unit-testing_advanced_0': {
'source': 'modules/tf_stream_alert_s3_events',
'lambda_function_arn': '${module.stream_alert_advanced.lambda_arn}',
'bucket_id': 'unit-test-bucket.data',
'notification_id': 'advanced_0',
'enable_events': True,
'lambda_role_id': '${module.stream_alert_advanced.lambda_role_id}',
'filter_suffix': '.log',
'filter_prefix': 'AWSLogs/123456789/CloudTrail/us-east-1/'
},
's3_events_unit-testing_advanced_1': {
'source': 'modules/tf_stream_alert_s3_events',
'lambda_function_arn': '${module.stream_alert_advanced.lambda_arn}',
'bucket_id': 'unit-test.cloudtrail.data',
'enable_events': False,
'notification_id': 'advanced_1',
'lambda_role_id': '${module.stream_alert_advanced.lambda_role_id}',
'filter_suffix': '',
'filter_prefix': ''
}
}
}
assert_true(result)
assert_equal(cluster_dict, expected_config)
def test_generate_s3_events_invalid_bucket(mock_logging):
"""CLI - Terraform - S3 Events with Missing Bucket Key"""
cluster_dict = common.infinitedict()
CONFIG['clusters']['advanced']['modules']['s3_events'] = [{'wrong_key': 'my-bucket!!!'}]
result = s3_events.generate_s3_events('advanced', cluster_dict, CONFIG)
assert_true(mock_logging.error.called)
assert_false(result)
def test_generate_cloudwatch_monitoring_disabled():
"""CLI - Terraform Generate Cloudwatch Monitoring Disabled"""
cluster_dict = common.infinitedict()
cluster = 'trusted'
result = monitoring.generate_monitoring(cluster, cluster_dict, CONFIG)
assert_true(result)
assert_true('cloudwatch_monitoring_{}'.format(cluster) not in cluster_dict['module'])
def generate_cluster(**kwargs):
"""Generate a StreamAlert cluster file.
Keyword Args:
cluster_name (str): The name of the currently generating cluster
config (dict): The loaded config from the 'conf/' directory
Returns:
dict: generated Terraform cluster dictionary
"""
config = kwargs.get('config')
cluster_name = kwargs.get('cluster_name')
modules = config['clusters'][cluster_name]['modules']
cluster_dict = infinitedict()
if not generate_stream_alert(cluster_name, cluster_dict, config):
return
generate_cloudwatch_metric_filters(cluster_name, cluster_dict, config)
generate_cloudwatch_metric_alarms(cluster_name, cluster_dict, config)
if modules.get('cloudwatch_monitoring', {}).get('enabled'):
if not generate_monitoring(cluster_name, cluster_dict, config):
return
if modules.get('kinesis'):
if not generate_kinesis_streams(cluster_name, cluster_dict, config):
return
outputs = config['clusters'][cluster_name].get('outputs')
if outputs:
if not generate_outputs(cluster_name, cluster_dict, config):
return
if modules.get('kinesis_events'):
if not generate_kinesis_events(cluster_name, cluster_dict, config):
return
cloudtrail_info = modules.get('cloudtrail')
if cloudtrail_info:
if not generate_cloudtrail(cluster_name, cluster_dict, config):
return
flow_log_info = modules.get('flow_logs')
if flow_log_info:
if not generate_flow_logs(cluster_name, cluster_dict, config):
return
s3_events_info = modules.get('s3_events')
if s3_events_info:
if not generate_s3_events(cluster_name, cluster_dict, config):
return
generate_app_integrations(cluster_name, cluster_dict, config)
return cluster_dict
def generate_cluster(config, cluster_name):
"""Generate a StreamAlert cluster file.
Args:
config (dict): The loaded config from the 'conf/' directory
cluster_name (str): The name of the currently generating cluster
Returns:
dict: generated Terraform cluster dictionary
"""
modules = config['clusters'][cluster_name]['modules']
cluster_dict = infinitedict()
generate_classifier(cluster_name, cluster_dict, config)
generate_cluster_cloudwatch_metric_filters(cluster_name, cluster_dict,
config)
generate_cluster_cloudwatch_metric_alarms(cluster_name, cluster_dict,
config)
if modules.get('cloudwatch_monitoring', {}).get('enabled'):
if not generate_monitoring(cluster_name, cluster_dict, config):
return
if modules.get('kinesis'):
if not generate_kinesis_streams(cluster_name, cluster_dict, config):
return
outputs = config['clusters'][cluster_name].get('outputs')
if outputs:
if not generate_outputs(cluster_name, cluster_dict, config):
return
if modules.get('kinesis_events'):
if not generate_kinesis_events(cluster_name, cluster_dict, config):
return
if modules.get('cloudtrail'):
if not generate_cloudtrail(cluster_name, cluster_dict, config):
return
if modules.get('cloudwatch'):
if not generate_cloudwatch(cluster_name, cluster_dict, config):
return
if modules.get('flow_logs'):
if not generate_flow_logs(cluster_name, cluster_dict, config):
return
if modules.get('s3_events'):
if not generate_s3_events(cluster_name, cluster_dict, config):
return
generate_apps(cluster_name, cluster_dict, config)
return cluster_dict
def test_generate_cloudwatch_monitoring_invalid_config(mock_logging):
"""CLI - Terraform Generate Cloudwatch Monitoring with Invalid Config"""
CONFIG['global']['infrastructure'] = {}
cluster_dict = common.infinitedict()
result = monitoring.generate_monitoring('test', cluster_dict, CONFIG)
assert_true(mock_logging.error.called)
assert_false(result)
def generate_rules_engine(config):
"""Generate Terraform for the Rules Engine
Args:
config (dict): The loaded config from the 'conf/' directory
Returns:
dict: Rules Engine Terraform definition to be marshaled to JSON
"""
prefix = config['global']['account']['prefix']
result = infinitedict()
# Set variables for the rules engine IAM permissions
result['module']['rules_engine_iam'] = {
'source':
'modules/tf_rules_engine',
'account_id':
config['global']['account']['aws_account_id'],
'region':
config['global']['account']['region'],
'prefix':
prefix,
'function_role_id':
'${module.rules_engine_lambda.role_id}',
'function_alias_arn':
'${module.rules_engine_lambda.function_alias_arn}',
'function_name':
'${module.rules_engine_lambda.function_name}',
'threat_intel_enabled':
config.get('threat_intel', {}).get('enabled'),
'dynamodb_table_name':
config.get('threat_intel', {}).get('dynamodb_table_name'),
'rules_table_arn':
'${module.globals.rules_table_arn}',
'classifier_sqs_queue_arn':
'${module.globals.classifier_sqs_queue_arn}',
'classifier_sqs_sse_kms_key_arn':
'${module.globals.classifier_sqs_sse_kms_key_arn}',
'sqs_record_batch_size':
min(config.get('sqs_record_batch_size', 10), 10)
}
# Set variables for the Lambda module
result['module']['rules_engine_lambda'] = generate_lambda(
'{}_streamalert_{}'.format(prefix, RULES_ENGINE_FUNCTION_NAME),
RulesEnginePackage.package_name + '.zip',
RulesEnginePackage.lambda_handler,
config['lambda']['rules_engine_config'],
config,
environment={
'ALERTS_TABLE': '{}_streamalert_alerts'.format(prefix),
'STREAMALERT_PREFIX': prefix
})
return result
def test_firehose_defaults(self):
"""CLI - Terraform Generate Kinesis Firehose, Defaults"""
cluster_dict = common.infinitedict()
firehose.generate_firehose(self._logging_bucket_name, cluster_dict, self.config)
expected_result = {
'module': {
'kinesis_firehose_setup': self._default_firehose_config(),
}
}
assert_equal(cluster_dict, expected_result)
def test_generate_s3_events_legacy():
"""CLI - Terraform - S3 Events - Legacy"""
cluster_dict = common.infinitedict()
CONFIG['clusters']['test']['modules']['s3_events'] = {
's3_bucket_id': 'unit-test-bucket.legacy.data'
}
result = s3_events.generate_s3_events('test', cluster_dict, CONFIG)
assert_true(result)
assert_equal(CONFIG['clusters']['test']['modules']['s3_events'],
[{
'bucket_id': 'unit-test-bucket.legacy.data'
}])
def generate_rule_promotion(config):
"""Generate Terraform for the Rule Promotion function
Args:
config (dict): The loaded config from the 'conf/' directory
Returns:
dict: Rule Promotion dict to be marshaled to JSON
"""
result = infinitedict()
state_param = json.dumps(
{
'send_digest_hour_utc':
int(config['lambda']['rule_promotion_config']
['send_digest_hour_utc']),
'sent_daily_digest':
False
},
sort_keys=True)
# Set variables for the IAM permissions, etc module
result['module']['rule_promotion_iam'] = {
'source':
'modules/tf_rule_promotion_iam',
'stats_publisher_state_name':
StatsPublisher.SSM_STATE_NAME,
'stats_publisher_state_value':
state_param,
'digest_sns_topic':
StatsPublisher.formatted_sns_topic_arn(config).split(':')[-1],
'role_id':
'${module.rule_promotion_lambda.role_id}',
'rules_table_arn':
'${module.globals.rules_table_arn}',
'athena_results_bucket_arn':
'${module.stream_alert_athena.results_bucket_arn}'
}
# Set variables for the Lambda module
result['module']['rule_promotion_lambda'] = generate_lambda(
'{}_streamalert_{}'.format(config['global']['account']['prefix'],
RULE_PROMOTION_NAME),
RulePromotionPackage.package_name + '.zip',
RulePromotionPackage.lambda_handler,
config['lambda']['rule_promotion_config'], config)
return result
def generate_threat_intel_downloader(config):
"""Generate Threat Intel Downloader Terrafrom
Args:
config (dict): The loaded config from the 'conf/' directory
Returns:
dict: Athena dict to be marshalled to JSON
"""
# Use the monitoring topic as a dead letter queue
infrastructure_config = config['global'].get('infrastructure')
dlq_topic = (DEFAULT_SNS_MONITORING_TOPIC
if infrastructure_config.get('monitoring', {}).get('create_sns_topic')
else infrastructure_config.get('monitoring', {}).get('sns_topic_name',
DEFAULT_SNS_MONITORING_TOPIC))
# Threat Intel Downloader module
ti_downloader_config = config['lambda']['threat_intel_downloader_config']
ti_downloader_dict = infinitedict()
ti_downloader_dict['module']['threat_intel_downloader'] = {
'account_id': config['global']['account']['aws_account_id'],
'region': config['global']['account']['region'],
'source': 'modules/tf_threat_intel_downloader',
'lambda_function_arn': '${module.threat_intel_downloader.lambda_arn}',
'lambda_handler': ti_downloader_config['handler'],
'lambda_memory': ti_downloader_config.get('memory', '128'),
'lambda_timeout': ti_downloader_config.get('timeout', '60'),
'lambda_s3_bucket': ti_downloader_config['source_bucket'],
'lambda_s3_key': ti_downloader_config['source_object_key'],
'lambda_log_level': ti_downloader_config.get('log_level', 'info'),
'interval': ti_downloader_config.get('interval', 'rate(1 day)'),
'current_version': ti_downloader_config['current_version'],
'prefix': config['global']['account']['prefix'],
'monitoring_sns_topic': dlq_topic,
'table_rcu': ti_downloader_config.get('table_rcu', '10'),
'table_wcu': ti_downloader_config.get('table_wcu', '10'),
'ioc_keys': ti_downloader_config.get('ioc_keys',
['expiration_ts', 'itype', 'source', 'type', 'value']),
'ioc_filters': ti_downloader_config.get('ioc_filters', ['crowdstrike', '@airbnb.com']),
'ioc_types': ti_downloader_config.get('ioc_types', ['domain', 'ip', 'md5']),
'excluded_sub_types': ti_downloader_config.get('excluded_sub_types'),
'max_read_capacity': ti_downloader_config.get('max_read_capacity', '5'),
'min_read_capacity': ti_downloader_config.get('min_read_capacity', '5'),
'target_utilization': ti_downloader_config.get('target_utilization', '70')
}
return ti_downloader_dict
def generate_rule_promotion(config):
"""Generate Terraform for the Rule Promotion function
Args:
config (dict): The loaded config from the 'conf/' directory
Returns:
dict: Rule Promotion dict to be marshaled to JSON
"""
# The Rule Promotion Lambda function is dependent on the rule staging feature being
# enabled, so do not generate the code for this Lambda function if it not enabled
if not config['global']['infrastructure']['rule_staging'].get('enabled', False):
return False
result = infinitedict()
athena_config = config['lambda']['athena_partition_refresh_config']
data_buckets = athena_config['buckets'].keys()
# Set variables for the IAM permissions, etc module
result['module']['rule_promotion_iam'] = {
'source': 'modules/tf_rule_promotion_iam',
'send_digest_schedule_expression':
config['lambda']['rule_promotion_config']['send_digest_schedule_expression'],
'digest_sns_topic': StatsPublisher.formatted_sns_topic_arn(config).split(':')[-1],
'role_id': '${module.rule_promotion_lambda.role_id}',
'rules_table_arn': '${module.globals.rules_table_arn}',
'function_alias_arn': '${module.rule_promotion_lambda.function_alias_arn}',
'function_name': '${module.rule_promotion_lambda.function_name}',
'athena_results_bucket_arn': '${module.stream_alert_athena.results_bucket_arn}',
'athena_data_buckets': data_buckets,
's3_kms_key_arn': '${aws_kms_key.server_side_encryption.arn}'
}
# Set variables for the Lambda module
result['module']['rule_promotion_lambda'] = generate_lambda(
'{}_streamalert_{}'.format(config['global']['account']['prefix'], RULE_PROMOTION_NAME),
RulePromotionPackage.package_name + '.zip',
RulePromotionPackage.lambda_handler,
config['lambda']['rule_promotion_config'],
config
)
return result
def test_generate_cloudwatch_monitoring():
"""CLI - Terraform Generate Cloudwatch Monitoring"""
cluster_dict = common.infinitedict()
result = monitoring.generate_monitoring('test', cluster_dict, CONFIG)
# Test the default SNS topic option
expected_cloudwatch_tf = {
'source': 'modules/tf_stream_alert_monitoring',
'sns_topic_arn': 'arn:aws:sns:us-west-1:12345678910:stream_alert_monitoring',
'lambda_functions': ['unit-testing_test_streamalert_classifier'],
'kinesis_stream': 'unit-testing_test_stream_alert_kinesis',
'lambda_alarms_enabled': True,
'kinesis_alarms_enabled': True
}
assert_true(result)
assert_equal(
cluster_dict['module']['cloudwatch_monitoring_test'],
expected_cloudwatch_tf)
def test_generate_cloudwatch_monitoring_with_settings():
"""CLI - Terraform Generate Cloudwatch Monitoring with Custom Settings"""
cluster_dict = common.infinitedict()
result = monitoring.generate_monitoring('advanced', cluster_dict, CONFIG)
# Test the default SNS topic option
expected_cloudwatch_tf = {
'source': 'modules/tf_stream_alert_monitoring',
'sns_topic_arn': 'arn:aws:sns:us-west-1:12345678910:stream_alert_monitoring',
'lambda_functions': ['unit-testing_advanced_streamalert_rule_processor'],
'kinesis_stream': 'unit-testing_advanced_stream_alert_kinesis',
'lambda_alarms_enabled': True,
'kinesis_alarms_enabled': True,
'kinesis_iterator_age_error_threshold': '3000000'
}
assert_true(result)
assert_equal(
cluster_dict['module']['cloudwatch_monitoring_advanced'],
expected_cloudwatch_tf)
def generate_athena(config):
"""Generate Athena Terraform.
Args:
config (dict): The loaded config from the 'conf/' directory
Returns:
dict: Athena dict to be marshalled to JSON
"""
athena_dict = infinitedict()
athena_config = config['lambda']['athena_partition_refresh_config']
data_buckets = athena_config['buckets'].keys()
prefix = config['global']['account']['prefix']
database = athena_config.get('database_name',
'{}_streamalert'.format(prefix))
results_bucket_name = athena_config.get(
'results_bucket',
'{}.streamalert.athena-results'.format(prefix)).strip()
queue_name = athena_config.get(
'queue_name',
'{}_streamalert_athena_s3_notifications'.format(prefix)).strip()
athena_dict['module']['stream_alert_athena'] = {
's3_logging_bucket':
'{}.streamalert.s3-logging'.format(prefix),
'source':
'modules/tf_stream_alert_athena',
'database_name':
database,
'queue_name':
queue_name,
'results_bucket':
results_bucket_name,
'lambda_handler':
AthenaPackage.lambda_handler,
'lambda_memory':
athena_config.get('memory', '128'),
'lambda_timeout':
athena_config.get('timeout', '60'),
'lambda_log_level':
athena_config.get('log_level', 'info'),
'athena_data_buckets':
data_buckets,
'schedule_expression':
athena_config.get('schedule_expression', 'rate(10 minutes)'),
'enable_metrics':
athena_config.get('enable_metrics', False),
'account_id':
config['global']['account']['aws_account_id'],
'prefix':
prefix
}
# Cloudwatch monitoring setup
monitoring_config = config['global'].get('infrastructure',
{}).get('monitoring', {})
sns_topic_name = DEFAULT_SNS_MONITORING_TOPIC if monitoring_config.get(
'create_sns_topic') else monitoring_config.get('sns_topic_name')
athena_dict['module']['athena_monitoring'] = {
'source':
'modules/tf_stream_alert_monitoring',
'sns_topic_arn':
'arn:aws:sns:{region}:{account_id}:{topic}'.format(
region=config['global']['account']['region'],
account_id=config['global']['account']['aws_account_id'],
topic=sns_topic_name),
'lambda_functions':
['{}_streamalert_athena_partition_refresh'.format(prefix)],
'kinesis_alarms_enabled':
False
}
# Metrics setup
if not athena_config.get('enable_metrics', False):
return athena_dict
# Check to see if there are any metrics configured for the athena function
current_metrics = metrics.MetricLogger.get_available_metrics()
if metrics.ATHENA_PARTITION_REFRESH_NAME not in current_metrics:
return athena_dict
metric_prefix = 'AthenaRefresh'
filter_pattern_idx, filter_value_idx = 0, 1
# Add filters for the cluster and aggregate
# Use a list of strings that represnt the following comma separated values:
# <filter_name>,<filter_pattern>,<value>
filters = [
'{},{},{}'.format('{}-{}'.format(metric_prefix,
metric), settings[filter_pattern_idx],
settings[filter_value_idx]) for metric, settings in
current_metrics[metrics.ATHENA_PARTITION_REFRESH_NAME].iteritems()
]
athena_dict['module']['stream_alert_athena'][
'athena_metric_filters'] = filters
return athena_dict
def generate_main(config, init=False):
"""Generate the main.tf.json Terraform dict
Args:
config (CLIConfig): The loaded CLI config
init (bool): Terraform is running in the init phase or not (optional)
Returns:
dict: main.tf.json Terraform dict
"""
main_dict = infinitedict()
# Configure provider along with the minimum version
main_dict['provider']['aws'] = {
'version': TERRAFORM_VERSIONS['provider']['aws'],
'region': config['global']['account']['region']
}
# Configure Terraform version requirement
main_dict['terraform']['required_version'] = TERRAFORM_VERSIONS['application']
# Setup the Backend depending on the deployment phase.
# When first setting up StreamAlert, the Terraform statefile
# is stored locally. After the first dependencies are created,
# this moves to S3.
if init:
main_dict['terraform']['backend']['local'] = {
'path': 'terraform.tfstate'}
else:
main_dict['terraform']['backend']['s3'] = {
'bucket': config['global']['terraform']['tfstate_bucket'],
'key': config['global']['terraform']['tfstate_s3_key'],
'region': config['global']['account']['region'],
'encrypt': True,
'acl': 'private',
'kms_key_id': 'alias/{}'.format(config['global']['account']['kms_key_alias'])}
logging_bucket = config['global']['s3_access_logging']['logging_bucket']
# Configure initial S3 buckets
main_dict['resource']['aws_s3_bucket'] = {
'stream_alert_secrets': generate_s3_bucket(
# FIXME (derek.wang) DRY out by using OutputCredentialsProvider?
bucket='{}.streamalert.secrets'.format(config['global']['account']['prefix']),
logging=logging_bucket
),
'streamalerts': generate_s3_bucket(
bucket='{}.streamalerts'.format(config['global']['account']['prefix']),
logging=logging_bucket
)
}
# Create bucket for S3 access logs (if applicable)
if config['global']['s3_access_logging'].get('create_bucket', True):
main_dict['resource']['aws_s3_bucket']['logging_bucket'] = generate_s3_bucket(
bucket=logging_bucket,
logging=logging_bucket,
acl='log-delivery-write',
lifecycle_rule={
'prefix': '/',
'enabled': True,
'transition': {
'days': 365,
'storage_class': 'GLACIER'
}
},
sse_algorithm='AES256' # SSE-KMS doesn't seem to work with access logs
)
# Create bucket for Terraform state (if applicable)
if config['global']['terraform'].get('create_bucket', True):
main_dict['resource']['aws_s3_bucket']['terraform_remote_state'] = generate_s3_bucket(
bucket=config['global']['terraform']['tfstate_bucket'],
logging=logging_bucket
)
# Setup Firehose Delivery Streams
generate_firehose(logging_bucket, main_dict, config)
# Configure global resources like Firehose alert delivery and alerts table
global_module = {
'source': 'modules/tf_stream_alert_globals',
'account_id': config['global']['account']['aws_account_id'],
'region': config['global']['account']['region'],
'prefix': config['global']['account']['prefix'],
'kms_key_arn': '${aws_kms_key.server_side_encryption.arn}',
'alerts_table_read_capacity': (
config['global']['infrastructure']['alerts_table']['read_capacity']),
'alerts_table_write_capacity': (
config['global']['infrastructure']['alerts_table']['write_capacity']),
'rules_engine_timeout': config['lambda']['rules_engine_config']['timeout']
}
if config['global']['infrastructure']['rule_staging'].get('enabled'):
global_module['enable_rule_staging'] = True
global_module['rules_table_read_capacity'] = (
config['global']['infrastructure']['rule_staging']['table']['read_capacity'])
global_module['rules_table_write_capacity'] = (
config['global']['infrastructure']['rule_staging']['table']['write_capacity'])
main_dict['module']['globals'] = global_module
# KMS Key and Alias creation
main_dict['resource']['aws_kms_key']['server_side_encryption'] = {
'enable_key_rotation': True,
'description': 'StreamAlert S3 Server-Side Encryption',
'policy': json.dumps({
'Version': '2012-10-17',
'Statement': [
{
'Sid': 'Enable IAM User Permissions',
'Effect': 'Allow',
'Principal': {
'AWS': 'arn:aws:iam::{}:root'.format(
config['global']['account']['aws_account_id']
)
},
'Action': 'kms:*',
'Resource': '*'
},
{
'Sid': 'Allow principals in the account to use the key',
'Effect': 'Allow',
'Principal': '*',
'Action': ['kms:Decrypt', 'kms:GenerateDataKey*', 'kms:Encrypt'],
'Resource': '*',
'Condition': {
'StringEquals': {
'kms:CallerAccount': config['global']['account']['aws_account_id']
}
}
}
]
})
}
main_dict['resource']['aws_kms_alias']['server_side_encryption'] = {
'name': 'alias/{}_server-side-encryption'.format(config['global']['account']['prefix']),
'target_key_id': '${aws_kms_key.server_side_encryption.key_id}'
}
main_dict['resource']['aws_kms_key']['stream_alert_secrets'] = {
'enable_key_rotation': True,
'description': 'StreamAlert secret management'
}
main_dict['resource']['aws_kms_alias']['stream_alert_secrets'] = {
'name': 'alias/{}'.format(config['global']['account']['kms_key_alias']),
'target_key_id': '${aws_kms_key.stream_alert_secrets.key_id}'
}
# Global infrastructure settings
infrastructure_config = config['global'].get('infrastructure')
if infrastructure_config and 'monitoring' in infrastructure_config:
if infrastructure_config['monitoring'].get('create_sns_topic'):
main_dict['resource']['aws_sns_topic']['stream_alert_monitoring'] = {
'name': DEFAULT_SNS_MONITORING_TOPIC
}
return main_dict
def generate_main(config, init=False):
"""Generate the main.tf.json Terraform dict
Args:
config (CLIConfig): The loaded CLI config
init (bool): Terraform is running in the init phase or not (optional)
Returns:
dict: main.tf.json Terraform dict
"""
main_dict = infinitedict()
# Configure provider along with the minimum version
main_dict['provider']['aws'] = {
'version': TERRAFORM_VERSIONS['provider']['aws']
}
# Configure Terraform version requirement
main_dict['terraform']['required_version'] = TERRAFORM_VERSIONS[
'application']
# Setup the Backend depending on the deployment phase.
# When first setting up StreamAlert, the Terraform statefile
# is stored locally. After the first dependencies are created,
# this moves to S3.
if init:
main_dict['terraform']['backend']['local'] = {
'path': 'terraform.tfstate'
}
else:
main_dict['terraform']['backend']['s3'] = {
'bucket':
config['global']['terraform']['tfstate_bucket'],
'key':
config['global']['terraform']['tfstate_s3_key'],
'region':
config['global']['account']['region'],
'encrypt':
True,
'acl':
'private',
'kms_key_id':
'alias/{}'.format(config['global']['account']['kms_key_alias'])
}
logging_bucket = config['global']['s3_access_logging']['logging_bucket']
# Configure initial S3 buckets
main_dict['resource']['aws_s3_bucket'] = {
'stream_alert_secrets':
generate_s3_bucket(bucket='{}.streamalert.secrets'.format(
config['global']['account']['prefix']),
logging=logging_bucket),
'streamalerts':
generate_s3_bucket(bucket='{}.streamalerts'.format(
config['global']['account']['prefix']),
logging=logging_bucket)
}
# Create bucket for S3 access logs (if applicable)
if config['global']['s3_access_logging'].get('create_bucket', True):
main_dict['resource']['aws_s3_bucket'][
'logging_bucket'] = generate_s3_bucket(
bucket=logging_bucket,
logging=logging_bucket,
acl='log-delivery-write',
lifecycle_rule={
'prefix': '/',
'enabled': True,
'transition': {
'days': 365,
'storage_class': 'GLACIER'
}
},
sse_algorithm=
'AES256' # SSE-KMS doesn't seem to work with access logs
)
# Create bucket for Terraform state (if applicable)
if config['global']['terraform'].get('create_bucket', True):
main_dict['resource']['aws_s3_bucket'][
'terraform_remote_state'] = generate_s3_bucket(
bucket=config['global']['terraform']['tfstate_bucket'],
logging=logging_bucket)
# Setup Firehose Delivery Streams
generate_firehose(config, main_dict, logging_bucket)
# Configure global resources like Firehose alert delivery and alerts table
global_module = {
'source':
'modules/tf_stream_alert_globals',
'account_id':
config['global']['account']['aws_account_id'],
'region':
config['global']['account']['region'],
'prefix':
config['global']['account']['prefix'],
'kms_key_arn':
'${aws_kms_key.server_side_encryption.arn}',
'alerts_table_read_capacity':
(config['global']['infrastructure']['alerts_table']['read_capacity']),
'alerts_table_write_capacity':
(config['global']['infrastructure']['alerts_table']['write_capacity'])
}
if config['global']['infrastructure']['rule_staging'].get('enabled'):
global_module['enable_rule_staging'] = True
global_module['rules_table_read_capacity'] = (
config['global']['infrastructure']['rule_staging']['table']
['read_capacity'])
global_module['rules_table_write_capacity'] = (
config['global']['infrastructure']['rule_staging']['table']
['write_capacity'])
main_dict['module']['globals'] = global_module
# KMS Key and Alias creation
main_dict['resource']['aws_kms_key']['server_side_encryption'] = {
'enable_key_rotation':
True,
'description':
'StreamAlert S3 Server-Side Encryption',
'policy':
json.dumps({
'Version':
'2012-10-17',
'Statement': [{
'Sid': 'Enable IAM User Permissions',
'Effect': 'Allow',
'Principal': {
'AWS':
'arn:aws:iam::{}:root'.format(
config['global']['account']['aws_account_id'])
},
'Action': 'kms:*',
'Resource': '*'
}, {
'Sid':
'Allow principals in the account to use the key',
'Effect':
'Allow',
'Principal':
'*',
'Action':
['kms:Decrypt', 'kms:GenerateDataKey*', 'kms:Encrypt'],
'Resource':
'*',
'Condition': {
'StringEquals': {
'kms:CallerAccount':
config['global']['account']['aws_account_id']
}
}
}]
})
}
main_dict['resource']['aws_kms_alias']['server_side_encryption'] = {
'name':
'alias/{}_server-side-encryption'.format(
config['global']['account']['prefix']),
'target_key_id':
'${aws_kms_key.server_side_encryption.key_id}'
}
main_dict['resource']['aws_kms_key']['stream_alert_secrets'] = {
'enable_key_rotation': True,
'description': 'StreamAlert secret management'
}
main_dict['resource']['aws_kms_alias']['stream_alert_secrets'] = {
'name':
'alias/{}'.format(config['global']['account']['kms_key_alias']),
'target_key_id': '${aws_kms_key.stream_alert_secrets.key_id}'
}
# Global infrastructure settings
infrastructure_config = config['global'].get('infrastructure')
if infrastructure_config and 'monitoring' in infrastructure_config:
if infrastructure_config['monitoring'].get('create_sns_topic'):
main_dict['resource']['aws_sns_topic'][
'stream_alert_monitoring'] = {
'name': DEFAULT_SNS_MONITORING_TOPIC
}
# Add any global cloudwatch alarms to the main.tf
monitoring_config = config['global']['infrastructure'].get('monitoring')
if not monitoring_config:
return main_dict
global_metrics = monitoring_config.get('metric_alarms')
if not global_metrics:
return main_dict
sns_topic_arn = monitoring_topic_arn(config)
formatted_alarms = {}
# Add global metric alarms for the rule and alert processors
for func in FUNC_PREFIXES:
if func not in global_metrics:
continue
for name, settings in global_metrics[func].iteritems():
alarm_info = settings.copy()
alarm_info['alarm_name'] = name
alarm_info['namespace'] = 'StreamAlert'
alarm_info['alarm_actions'] = [sns_topic_arn]
# Terraform only allows certain characters in resource names
acceptable_chars = ''.join([string.digits, string.letters, '_-'])
name = filter(acceptable_chars.__contains__, name)
formatted_alarms['metric_alarm_{}'.format(name)] = alarm_info
if formatted_alarms:
main_dict['resource']['aws_cloudwatch_metric_alarm'] = formatted_alarms
return main_dict
def generate_main(config, init=False):
"""Generate the main.tf.json Terraform dict
Args:
config (CLIConfig): The loaded CLI config
init (bool): Terraform is running in the init phase or not (optional)
Returns:
dict: main.tf.json Terraform dict
"""
main_dict = infinitedict()
# Configure provider along with the minimum version
main_dict['provider']['aws'] = {'version': TERRAFORM_VERSIONS['provider']['aws']}
# Configure Terraform version requirement
main_dict['terraform']['required_version'] = TERRAFORM_VERSIONS['application']
# Setup the Backend depending on the deployment phase.
# When first setting up StreamAlert, the Terraform statefile
# is stored locally. After the first dependencies are created,
# this moves to S3.
if init:
main_dict['terraform']['backend']['local'] = {
'path': 'terraform.tfstate'}
else:
main_dict['terraform']['backend']['s3'] = {
'bucket': '{}.streamalert.terraform.state'.format(
config['global']['account']['prefix']),
'key': 'stream_alert_state/terraform.tfstate',
'region': config['global']['account']['region'],
'encrypt': True,
'acl': 'private',
'kms_key_id': 'alias/{}'.format(config['global']['account']['kms_key_alias'])}
logging_bucket = '{}.streamalert.s3-logging'.format(
config['global']['account']['prefix'])
logging_bucket_lifecycle = {
'prefix': '/',
'enabled': True,
'transition': {
'days': 30,
'storage_class': 'GLACIER'}}
# Configure initial S3 buckets
main_dict['resource']['aws_s3_bucket'] = {
'lambda_source': generate_s3_bucket(
bucket=config['lambda']['rule_processor_config']['source_bucket'],
logging=logging_bucket
),
'stream_alert_secrets': generate_s3_bucket(
bucket='{}.streamalert.secrets'.format(config['global']['account']['prefix']),
logging=logging_bucket
),
'terraform_remote_state': generate_s3_bucket(
bucket=config['global']['terraform']['tfstate_bucket'],
logging=logging_bucket
),
'logging_bucket': generate_s3_bucket(
bucket=logging_bucket,
logging=logging_bucket,
acl='log-delivery-write',
lifecycle_rule=logging_bucket_lifecycle
),
'streamalerts': generate_s3_bucket(
bucket='{}.streamalerts'.format(config['global']['account']['prefix']),
logging=logging_bucket
)
}
# Setup Firehose Delivery Streams
generate_firehose(config, main_dict, logging_bucket)
# Configure global resources like Firehose alert delivery and alerts table
main_dict['module']['globals'] = {
'source': 'modules/tf_stream_alert_globals',
'account_id': config['global']['account']['aws_account_id'],
'region': config['global']['account']['region'],
'prefix': config['global']['account']['prefix'],
'alerts_table_read_capacity': (
config['global']['infrastructure']['alerts_table']['read_capacity']),
'alerts_table_write_capacity': (
config['global']['infrastructure']['alerts_table']['write_capacity']),
'rules_table_read_capacity': (
config['global']['infrastructure']['rules_table']['read_capacity']),
'rules_table_write_capacity': (
config['global']['infrastructure']['rules_table']['write_capacity'])
}
# KMS Key and Alias creation
main_dict['resource']['aws_kms_key']['stream_alert_secrets'] = {
'enable_key_rotation': True,
'description': 'StreamAlert secret management'
}
main_dict['resource']['aws_kms_alias']['stream_alert_secrets'] = {
'name': 'alias/{}'.format(config['global']['account']['kms_key_alias']),
'target_key_id': '${aws_kms_key.stream_alert_secrets.key_id}'
}
# Global infrastructure settings
infrastructure_config = config['global'].get('infrastructure')
if infrastructure_config and 'monitoring' in infrastructure_config:
if infrastructure_config['monitoring'].get('create_sns_topic'):
main_dict['resource']['aws_sns_topic']['stream_alert_monitoring'] = {
'name': DEFAULT_SNS_MONITORING_TOPIC
}
# Add any global cloudwatch alarms to the main.tf
monitoring_config = config['global']['infrastructure'].get('monitoring')
if not monitoring_config:
return main_dict
global_metrics = monitoring_config.get('metric_alarms')
if not global_metrics:
return main_dict
sns_topic_arn = monitoring_topic_arn(config)
formatted_alarms = {}
# Add global metric alarms for the rule and alert processors
for func in FUNC_PREFIXES:
if func not in global_metrics:
continue
for name, settings in global_metrics[func].iteritems():
alarm_info = settings.copy()
alarm_info['alarm_name'] = name
alarm_info['namespace'] = 'StreamAlert'
alarm_info['alarm_actions'] = [sns_topic_arn]
# Terraform only allows certain characters in resource names
acceptable_chars = ''.join([string.digits, string.letters, '_-'])
name = filter(acceptable_chars.__contains__, name)
formatted_alarms['metric_alarm_{}'.format(name)] = alarm_info
if formatted_alarms:
main_dict['resource']['aws_cloudwatch_metric_alarm'] = formatted_alarms
return main_dict
def test_generate_classifier(self):
"""CLI - Terraform Generate, Classifier"""
cluster_dict = common.infinitedict()
classifier.generate_classifier('test', cluster_dict, self.config)
expected_result = {
'module': {
'classifier_test_iam': {
'source':
'modules/tf_classifier',
'account_id':
'123456789012',
'region':
'us-east-1',
'function_role_id':
'${module.classifier_test_lambda.role_id}',
'function_alias_arn':
'${module.classifier_test_lambda.function_alias_arn}',
'function_name':
'${module.classifier_test_lambda.function_name}',
'classifier_sqs_queue_arn':
'${module.globals.classifier_sqs_queue_arn}',
'classifier_sqs_sse_kms_key_arn':
('${module.globals.classifier_sqs_sse_kms_key_arn}'),
'input_sns_topics':
['arn:aws:sns:us-east-1:123456789012:foo_bar']
},
'classifier_test_lambda': {
'alarm_actions':
['arn:aws:sns:us-east-1:123456789012:test_topic'],
'description':
'Unit-Test Streamalert Classifier Test',
'environment_variables': {
'CLUSTER': 'test',
'SQS_QUEUE_URL':
'${module.globals.classifier_sqs_queue_url}',
'LOGGER_LEVEL': 'info',
'ENABLE_METRICS': '0'
},
'tags': {
'Cluster': 'test'
},
'errors_alarm_enabled':
True,
'errors_alarm_evaluation_periods':
1,
'errors_alarm_period_secs':
120,
'errors_alarm_threshold':
0,
'filename':
'classifier.zip',
'function_name':
'unit-test_streamalert_classifier_test',
'handler':
'stream_alert.classifier.main.handler',
'log_retention_days':
14,
'memory_size_mb':
128,
'source':
'modules/tf_lambda',
'throttles_alarm_enabled':
True,
'throttles_alarm_evaluation_periods':
1,
'throttles_alarm_period_secs':
120,
'throttles_alarm_threshold':
0,
'timeout_sec':
60,
'vpc_security_group_ids': [],
'vpc_subnet_ids': [],
'input_sns_topics':
['arn:aws:sns:us-east-1:123456789012:foo_bar']
}
}
}
assert_equal(cluster_dict, expected_result)
def setup(self):
"""Setup before each method"""
self.cluster_dict = common.infinitedict()
self.config = CLIConfig(config_path='tests/unit/conf')