def test_generate_cloudwatch_monitoring_custom_sns():
"""CLI - Terraform Generate Cloudwatch Monitoring with Existing SNS Topic"""
# Test a custom SNS topic name
CONFIG['clusters']['test']['modules']['cloudwatch_monitoring'] = {
'enabled': True
}
CONFIG['global']['infrastructure']['monitoring'][
'sns_topic_name'] = 'unit_test_monitoring'
cluster_dict = common.infinitedict()
result = monitoring.generate_monitoring('test', cluster_dict, CONFIG)
expected_cloudwatch_tf_custom = {
'source': './modules/tf_monitoring',
'sns_topic_arn':
'arn:aws:sns:us-west-1:12345678910:unit_test_monitoring',
'lambda_functions': ['unit-test_test_streamalert_classifier'],
'kinesis_stream': '${module.kinesis_test.stream_name}',
'lambda_alarms_enabled': True,
'kinesis_alarms_enabled': True
}
assert_true(result)
assert_equal(cluster_dict['module']['cloudwatch_monitoring_test'],
expected_cloudwatch_tf_custom)
def test_firehose_enabled_log(self):
"""CLI - Terraform Generate Kinesis Firehose, Enabled Log"""
cluster_dict = common.infinitedict()
# Add an enabled log, with no alarm configuration (aka: alarms disabled)
self.config['global']['infrastructure']['firehose']['enabled_logs'] = {
'json:embedded': {}
}
firehose.generate_firehose(self._logging_bucket_name, cluster_dict,
self.config)
expected_result = {
'module': {
'kinesis_firehose_setup': self._default_firehose_config(),
'kinesis_firehose_json_embedded': {
'source': './modules/tf_kinesis_firehose_delivery_stream',
'buffer_size': 128,
'buffer_interval': 900,
'file_format': 'parquet',
'stream_name': 'unit_test_streamalert_json_embedded',
'role_arn':
'${module.kinesis_firehose_setup.firehose_role_arn}',
's3_bucket_name': 'unit-test-streamalert-data',
'kms_key_arn': '${aws_kms_key.server_side_encryption.arn}',
'glue_catalog_db_name': 'unit-test_streamalert',
'glue_catalog_table_name': 'json_embedded',
'schema': self._get_expected_schema()
}
}
}
assert_equal(cluster_dict, expected_result)
def generate_aggregate_cloudwatch_metric_alarms(config):
"""Return any CloudWatch Metric Alarms for aggregate metrics
Args:
config (dict): The loaded config from the 'conf/' directory
"""
result = infinitedict()
sns_topic_arn = monitoring_topic_arn(config)
for func, func_config in config['lambda'].items():
metric_alarms = func_config.get('custom_metric_alarms')
if not metric_alarms:
continue
func = func.replace('_config', '')
for idx, name in enumerate(sorted(metric_alarms)):
alarm_settings = metric_alarms[name]
alarm_settings['source'] = './modules/tf_metric_alarms'
alarm_settings['sns_topic_arn'] = sns_topic_arn
alarm_settings['alarm_name'] = name
result['module']['metric_alarm_{}_{}'.format(func,
idx)] = alarm_settings
return result
def generate_alert_merger(config):
"""Generate Terraform for the Alert Merger
Args:
config (dict): The loaded config from the 'conf/' directory
Returns:
dict: Alert Merger Terraform definition to be marshaled to JSON
"""
prefix = config['global']['account']['prefix']
result = infinitedict()
# Set variables for the alert merger's IAM permissions
result['module']['alert_merger_iam'] = {
'source': './modules/tf_alert_merger_iam',
'account_id': config['global']['account']['aws_account_id'],
'region': config['global']['account']['region'],
'prefix': config['global']['account']['prefix'],
'role_id': '${module.alert_merger_lambda.role_id}'
}
# Set variables for the Lambda module
result['module']['alert_merger_lambda'] = generate_lambda(
'{}_streamalert_{}'.format(config['global']['account']['prefix'], ALERT_MERGER_NAME),
AlertMergerPackage.package_name + '.zip',
AlertMergerPackage.lambda_handler,
config['lambda']['alert_merger_config'],
config,
environment={
'ALERTS_TABLE': '{}_streamalert_alerts'.format(prefix),
'ALERT_PROCESSOR': '{}_streamalert_alert_processor'.format(prefix),
'ALERT_PROCESSOR_TIMEOUT_SEC': config['lambda']['alert_processor_config']['timeout'],
}
)
return result
def test_firehose_enabled_log_alarm_custom(self):
"""CLI - Terraform Generate Kinesis Firehose, Enabled Alarm - Custom Settings"""
cluster_dict = common.infinitedict()
# Add an enabled log, with alarms on with custom settings
self.config['global']['infrastructure']['firehose']['enabled_logs'] = {
'json:embedded': {
'enable_alarm': True,
'evaluation_periods': 10,
'period_seconds': 3600,
'log_min_count_threshold': 100000
}
}
firehose.generate_firehose(self._logging_bucket_name, cluster_dict,
self.config)
expected_result = {
'module': {
'kinesis_firehose_setup': self._default_firehose_config(),
'kinesis_firehose_json_embedded': {
'source':
'./modules/tf_kinesis_firehose_delivery_stream',
'buffer_size':
128,
'buffer_interval':
900,
'file_format':
'parquet',
'stream_name':
'unit_test_streamalert_json_embedded',
'role_arn':
'${module.kinesis_firehose_setup.firehose_role_arn}',
's3_bucket_name':
'unit-test-streamalert-data',
'kms_key_arn':
'${aws_kms_key.server_side_encryption.arn}',
'enable_alarm':
True,
'evaluation_periods':
10,
'period_seconds':
3600,
'alarm_threshold':
100000,
'alarm_actions': [
'arn:aws:sns:us-west-1:12345678910:unit-test_streamalert_monitoring'
],
'glue_catalog_db_name':
'unit-test_streamalert',
'glue_catalog_table_name':
'json_embedded',
'schema':
self._get_expected_schema()
}
}
}
assert_equal(cluster_dict, expected_result)
def generate_cluster(config, cluster_name):
"""Generate a StreamAlert cluster file.
Args:
config (dict): The loaded config from the 'conf/' directory
cluster_name (str): The name of the currently generating cluster
Returns:
dict: generated Terraform cluster dictionary
"""
modules = config['clusters'][cluster_name]['modules']
cluster_dict = infinitedict()
generate_classifier(cluster_name, cluster_dict, config)
generate_cluster_cloudwatch_metric_filters(cluster_name, cluster_dict,
config)
generate_cluster_cloudwatch_metric_alarms(cluster_name, cluster_dict,
config)
if modules.get('cloudwatch_monitoring', {}).get('enabled'):
if not generate_monitoring(cluster_name, cluster_dict, config):
return
if modules.get('kinesis'):
if not generate_kinesis_streams(cluster_name, cluster_dict, config):
return
if modules.get('kinesis_events'):
if not generate_kinesis_events(cluster_name, cluster_dict, config):
return
if modules.get('cloudtrail'):
if not generate_cloudtrail(cluster_name, cluster_dict, config):
return
# purposely not using .get, since no extra settings are required for this module
if 'cloudwatch_events' in modules:
if not generate_cloudwatch_events(cluster_name, cluster_dict, config):
return
if modules.get('cloudwatch_logs_destination'):
if not generate_cloudwatch_destinations(cluster_name, cluster_dict,
config):
return
if modules.get('flow_logs'):
if not generate_flow_logs(cluster_name, cluster_dict, config):
return
if modules.get('s3_events'):
if not generate_s3_events(cluster_name, cluster_dict, config):
return
generate_apps(cluster_name, cluster_dict, config)
return cluster_dict
def test_generate_cloudwatch_monitoring_invalid_config(mock_logging):
"""CLI - Terraform Generate Cloudwatch Monitoring with Invalid Config"""
CONFIG['global']['infrastructure'] = {}
cluster_dict = common.infinitedict()
result = monitoring.generate_monitoring('test', cluster_dict, CONFIG)
assert_true(mock_logging.error.called)
assert_false(result)
def test_generate_cloudwatch_monitoring_disabled():
"""CLI - Terraform Generate Cloudwatch Monitoring Disabled"""
cluster_dict = common.infinitedict()
cluster = 'trusted'
result = monitoring.generate_monitoring(cluster, cluster_dict, CONFIG)
assert_true(result)
assert_true('cloudwatch_monitoring_{}'.format(cluster) not in
cluster_dict['module'])
def generate_alert_processor(config):
"""Generate Terraform for the Alert Processor
Args:
config (dict): The loaded config from the 'conf/' directory
Returns:
dict: Alert Processor dict to be marshaled to JSON
"""
prefix = config['global']['account']['prefix']
result = infinitedict()
# Set variables for the IAM permissions module
result['module']['alert_processor_iam'] = {
'source':
'./modules/tf_alert_processor_iam',
'account_id':
config['global']['account']['aws_account_id'],
'region':
config['global']['account']['region'],
'prefix':
prefix,
'role_id':
'${module.alert_processor_lambda.role_id}',
'kms_key_arn':
'${aws_kms_key.streamalert_secrets.arn}',
'sse_kms_key_arn':
'${aws_kms_key.server_side_encryption.arn}',
'output_lambda_functions': [
# Strip qualifiers: only the function name is needed for the IAM permissions
func.split(':')[0]
for func in list(config['outputs'].get('aws-lambda', {}).values())
],
'output_s3_buckets':
list(config['outputs'].get('aws-s3', {}).values()),
'output_sns_topics':
list(config['outputs'].get('aws-sns', {}).values()),
'output_sqs_queues':
list(config['outputs'].get('aws-sqs', {}).values())
}
# Set variables for the Lambda module
result['module']['alert_processor_lambda'] = generate_lambda(
'{}_streamalert_{}'.format(config['global']['account']['prefix'],
ALERT_PROCESSOR_NAME),
AlertProcessorPackage.package_name + '.zip',
AlertProcessorPackage.lambda_handler,
config['lambda']['alert_processor_config'],
config,
environment={
'ALERTS_TABLE': '{}_streamalert_alerts'.format(prefix),
'AWS_ACCOUNT_ID': config['global']['account']['aws_account_id'],
'STREAMALERT_PREFIX': prefix
})
return result
def _terraform_defaults(region):
return infinitedict({
'terraform': {
'required_version': TERRAFORM_VERSION,
},
'provider': {
'aws': {
'region': region,
'version': TERRAFORM_PROVIDER_VERSION,
},
},
})
def generate_rule_promotion(config):
"""Generate Terraform for the Rule Promotion function
Args:
config (dict): The loaded config from the 'conf/' directory
Returns:
dict: Rule Promotion dict to be marshaled to JSON
"""
# The Rule Promotion Lambda function is dependent on the rule staging feature being
# enabled, so do not generate the code for this Lambda function if it not enabled
if not config['global']['infrastructure']['rule_staging'].get(
'enabled', False):
return False
result = infinitedict()
alerts_bucket = firehose_alerts_bucket(config)
# Set variables for the IAM permissions, etc module
result['module']['rule_promotion_iam'] = {
'source':
'./modules/tf_rule_promotion_iam',
'send_digest_schedule_expression':
config['lambda']['rule_promotion_config']
['send_digest_schedule_expression'],
'digest_sns_topic':
StatsPublisher.formatted_sns_topic_arn(config).split(':')[-1],
'role_id':
'${module.rule_promotion_lambda.role_id}',
'rules_table_arn':
'${module.globals.rules_table_arn}',
'function_alias_arn':
'${module.rule_promotion_lambda.function_alias_arn}',
'function_name':
'${module.rule_promotion_lambda.function_name}',
'athena_results_bucket_arn':
'${module.streamalert_athena.results_bucket_arn}',
'alerts_bucket':
alerts_bucket,
's3_kms_key_arn':
'${aws_kms_key.server_side_encryption.arn}'
}
# Set variables for the Lambda module
result['module']['rule_promotion_lambda'] = generate_lambda(
'{}_streamalert_{}'.format(config['global']['account']['prefix'],
RULE_PROMOTION_NAME),
RulePromotionPackage.package_name + '.zip',
RulePromotionPackage.lambda_handler,
config['lambda']['rule_promotion_config'], config)
return result
def test_firehose_defaults(self):
"""CLI - Terraform Generate Kinesis Firehose, Defaults"""
cluster_dict = common.infinitedict()
firehose.generate_firehose(self._logging_bucket_name, cluster_dict,
self.config)
expected_result = {
'module': {
'kinesis_firehose_setup': self._default_firehose_config(),
}
}
assert_equal(cluster_dict, expected_result)
def generate_threat_intel_downloader(config):
"""Generate Threat Intel Downloader Terrafrom
Args:
config (dict): The loaded config from the 'conf/' directory
Returns:
dict: Athena dict to be marshalled to JSON
"""
# Use the monitoring topic as a dead letter queue
dlq_topic, _ = monitoring_topic_name(config)
prefix = config['global']['account']['prefix']
# Threat Intel Downloader module
tid_config = config['lambda']['threat_intel_downloader_config']
# old format of config used interval, but tf_lambda expects 'schedule_expression'
if 'schedule_expression' not in tid_config:
tid_config['schedule_expression'] = tid_config.get(
'interval', 'rate(1 day)')
result = infinitedict()
# Set variables for the threat intel downloader configuration
result['module']['threat_intel_downloader_iam'] = {
'source': './modules/tf_threat_intel_downloader',
'account_id': config['global']['account']['aws_account_id'],
'region': config['global']['account']['region'],
'prefix': prefix,
'function_role_id': '${module.threat_intel_downloader.role_id}',
'function_alias_arn':
'${module.threat_intel_downloader.function_alias_arn}',
'function_cloudwatch_log_group_name':
'${module.threat_intel_downloader.log_group_name}',
'monitoring_sns_topic': dlq_topic,
'table_rcu': tid_config.get('table_rcu', '10'),
'table_wcu': tid_config.get('table_wcu', '10'),
'max_read_capacity': tid_config.get('max_read_capacity', '5'),
'min_read_capacity': tid_config.get('min_read_capacity', '5'),
'target_utilization': tid_config.get('target_utilization', '70')
}
result['module']['threat_intel_downloader'] = generate_lambda(
'{}_streamalert_{}'.format(prefix, THREAT_INTEL_DOWNLOADER_NAME),
'streamalert.threat_intel_downloader.main.handler',
tid_config,
config,
)
return result
def generate_rules_engine(config):
"""Generate Terraform for the Rules Engine
Args:
config (dict): The loaded config from the 'conf/' directory
Returns:
dict: Rules Engine Terraform definition to be marshaled to JSON
"""
prefix = config['global']['account']['prefix']
result = infinitedict()
# Set variables for the rules engine IAM permissions
result['module']['rules_engine_iam'] = {
'source': './modules/tf_rules_engine',
'account_id': config['global']['account']['aws_account_id'],
'region': config['global']['account']['region'],
'prefix': prefix,
'function_role_id': '${module.rules_engine_lambda.role_id}',
'function_alias_arn': '${module.rules_engine_lambda.function_alias_arn}',
'function_name': '${module.rules_engine_lambda.function_name}',
'threat_intel_enabled': config.get('threat_intel', {}).get('enabled'),
'dynamodb_table_name': config.get('threat_intel', {}).get('dynamodb_table_name'),
'rules_table_arn': '${module.globals.rules_table_arn}',
'enable_rule_staging': config['global']['infrastructure']['rule_staging'].get(
'enabled', False
),
'classifier_sqs_queue_arn': '${module.globals.classifier_sqs_queue_arn}',
'classifier_sqs_sse_kms_key_arn': '${module.globals.classifier_sqs_sse_kms_key_arn}',
'sqs_record_batch_size': min(config.get('sqs_record_batch_size', 10), 10)
}
environment = {
'ALERTS_TABLE': '{}_streamalert_alerts'.format(prefix),
'STREAMALERT_PREFIX': prefix,
}
if config['lambda']['rules_engine_config'].get('log_rule_statistics'):
environment['STREAMALERT_TRACK_RULE_STATS'] = '1'
# Set variables for the Lambda module
result['module']['rules_engine_lambda'] = generate_lambda(
'{}_streamalert_{}'.format(prefix, RULES_ENGINE_FUNCTION_NAME),
'streamalert.rules_engine.main.handler',
config['lambda']['rules_engine_config'],
config,
environment=environment,
)
return result
def test_firehose_enabled_log_alarm_defaults(self):
"""CLI - Terraform Generate Kinesis Firehose, Enabled Alarm - Default Settings"""
cluster_dict = common.infinitedict()
# Add an enabled log, with alarms on (will use terraform default settings)
self.config['global']['infrastructure']['firehose']['enabled_logs'] = {
'json:embedded': {
'enable_alarm': True
}
}
firehose.generate_firehose(self._logging_bucket_name, cluster_dict,
self.config)
expected_result = {
'module': {
'kinesis_firehose_setup': self._default_firehose_config(),
'kinesis_firehose_json_embedded': {
'source':
'./modules/tf_kinesis_firehose_delivery_stream',
'buffer_size':
128,
'buffer_interval':
900,
'compression_format':
'GZIP',
'use_prefix':
True,
'prefix':
'unit-test',
'log_name':
'json_embedded',
'role_arn':
'${module.kinesis_firehose_setup.firehose_role_arn}',
's3_bucket_name':
'unit-test-streamalert-data',
'kms_key_arn':
'${aws_kms_key.server_side_encryption.arn}',
'enable_alarm':
True,
'alarm_actions': [
'arn:aws:sns:us-west-1:12345678910:unit-test_streamalert_monitoring'
]
}
}
}
assert_equal(cluster_dict, expected_result)
def generate_artifact_extractor(config):
"""Generate Terraform for the Artifact Extractor Lambda function
Args:
config (dict): The loaded config from the 'conf/' directory
Returns:
dict: Artifact Extractor Terraform definition to be marshaled to JSON
"""
result = infinitedict()
if not artifact_extractor_enabled(config):
return
ae_config = config['global']['infrastructure']['artifact_extractor']
stream_name = FirehoseClient.artifacts_firehose_stream_name(config)
# Set variables for the artifact extractor module
result['module']['artifact_extractor'] = {
'source':
'./modules/tf_artifact_extractor',
'account_id':
config['global']['account']['aws_account_id'],
'prefix':
config['global']['account']['prefix'],
'region':
config['global']['account']['region'],
'glue_catalog_db_name':
get_database_name(config),
'glue_catalog_table_name':
ae_config.get('table_name', DEFAULT_ARTIFACTS_TABLE_NAME),
's3_bucket_name':
firehose_data_bucket(config),
'stream_name':
stream_name,
'buffer_size':
ae_config.get('firehose_buffer_size', 128),
'buffer_interval':
ae_config.get('firehose_buffer_interval', 900),
'kms_key_arn':
'${aws_kms_key.server_side_encryption.arn}',
'schema':
generate_artifacts_table_schema()
}
return result
def test_generate_cloudwatch_monitoring():
"""CLI - Terraform Generate Cloudwatch Monitoring"""
cluster_dict = common.infinitedict()
result = monitoring.generate_monitoring('test', cluster_dict, CONFIG)
# Test the default SNS topic option
expected_cloudwatch_tf = {
'source': './modules/tf_monitoring',
'sns_topic_arn':
'arn:aws:sns:us-west-1:12345678910:unit-test_streamalert_monitoring',
'lambda_functions': ['unit-test_test_streamalert_classifier'],
'kinesis_stream': '${module.kinesis_test.stream_name}',
'lambda_alarms_enabled': True,
'kinesis_alarms_enabled': True
}
assert_true(result)
assert_equal(cluster_dict['module']['cloudwatch_monitoring_test'],
expected_cloudwatch_tf)
def test_kinesis_streams_with_custom_name():
"""CLI - Terraform Generate Kinesis Streams with Custom Name"""
cluster_dict = common.infinitedict()
stream_name = 'test-stream-name'
cluster = 'advanced'
CONFIG['clusters'][cluster]['modules']['kinesis']['streams'][
'stream_name'] = stream_name
result = kinesis_streams.generate_kinesis_streams(cluster, cluster_dict,
CONFIG)
expected_result = {
'module': {
'kinesis_advanced': {
'source': './modules/tf_kinesis_streams',
'account_id': '12345678910',
'shard_level_metrics': ["IncomingBytes"],
'region': 'us-west-1',
'prefix': 'unit-test',
'cluster': cluster,
'stream_name': stream_name,
'shards': 1,
'retention': 24,
'create_user': True,
'trusted_accounts': []
}
},
'output': {
'kinesis_advanced_access_key_id': {
'value': '${module.kinesis_advanced.access_key_id}'
},
'kinesis_advanced_secret_key': {
'value': '${module.kinesis_advanced.secret_key}'
},
'kinesis_advanced_user_arn': {
'value': '${module.kinesis_advanced.user_arn}'
}
}
}
assert_true(result)
assert_equal(cluster_dict, expected_result)
def test_generate_s3_events():
"""CLI - Terraform - S3 Events, No Module Prefix"""
cluster_dict = common.infinitedict()
result = s3_events.generate_s3_events('advanced', cluster_dict, CONFIG)
expected_config = {
'module': {
's3_events_unit-test_advanced_unit-test-bucket_data': {
'source': './modules/tf_s3_events',
'lambda_function_alias': '${module.classifier_advanced_lambda.function_alias}',
'lambda_function_alias_arn': (
'${module.classifier_advanced_lambda.function_alias_arn}'
),
'lambda_function_name': '${module.classifier_advanced_lambda.function_name}',
'bucket_name': 'unit-test-bucket.data',
'lambda_role_id': '${module.classifier_advanced_lambda.role_id}',
'filters': [
{
'filter_prefix': 'AWSLogs/123456789/CloudTrail/us-east-1/',
'filter_suffix': '.log'
}
]
},
's3_events_unit-test_advanced_unit-test_cloudtrail_data': {
'source': './modules/tf_s3_events',
'lambda_function_alias': '${module.classifier_advanced_lambda.function_alias}',
'lambda_function_alias_arn': (
'${module.classifier_advanced_lambda.function_alias_arn}'
),
'lambda_function_name': '${module.classifier_advanced_lambda.function_name}',
'bucket_name': 'unit-test.cloudtrail.data',
'lambda_role_id': '${module.classifier_advanced_lambda.role_id}',
'filters': []
}
}
}
assert_equal(result, True)
assert_equal(cluster_dict, expected_config)
def test_generate_s3_events_with_prefix():
"""CLI - Terraform - S3 Events, With Module Prefix"""
cluster_dict = common.infinitedict()
bucket_config = {
'unit-test-bucket': [
{
'filter_prefix': 'AWSLogs/123456789/CloudTrail/us-east-1/',
}
]
}
s3_events.generate_s3_events_by_bucket(
'advanced',
cluster_dict,
CONFIG,
bucket_config,
module_prefix='cloudtrail'
)
expected_config = {
'module': {
'cloudtrail_s3_events_unit-test_advanced_unit-test-bucket': {
'source': './modules/tf_s3_events',
'lambda_function_alias': '${module.classifier_advanced_lambda.function_alias}',
'lambda_function_alias_arn': (
'${module.classifier_advanced_lambda.function_alias_arn}'
),
'lambda_function_name': '${module.classifier_advanced_lambda.function_name}',
'bucket_name': 'unit-test-bucket',
'lambda_role_id': '${module.classifier_advanced_lambda.role_id}',
'filters': [
{
'filter_prefix': 'AWSLogs/123456789/CloudTrail/us-east-1/'
}
]
},
}
}
assert_equal(cluster_dict, expected_config)
def test_kinesis_events():
"""CLI - Terraform Generate Kinesis Events"""
cluster_dict = common.infinitedict()
result = kinesis_events.generate_kinesis_events('advanced', cluster_dict, CONFIG)
expected_result = {
'module': {
'kinesis_events_advanced': {
'source': './modules/tf_kinesis_events',
'batch_size': 100,
'lambda_production_enabled': True,
'lambda_role_id': '${module.classifier_advanced_lambda.role_id}',
'lambda_function_alias_arn': (
'${module.classifier_advanced_lambda.function_alias_arn}'
),
'kinesis_stream_arn': '${module.kinesis_advanced.arn}',
}
}
}
assert_true(result)
assert_equal(cluster_dict, expected_result)
def test_kinesis_streams_with_trusted_account():
"""CLI - Terraform Generate Kinesis Streams with trusted account"""
cluster_dict = common.infinitedict()
result = kinesis_streams.generate_kinesis_streams('trusted', cluster_dict,
CONFIG)
expected_result = {
'module': {
'kinesis_trusted': {
'source': './modules/tf_kinesis_streams',
'account_id': '12345678910',
'shard_level_metrics': [],
'region': 'us-west-1',
'prefix': 'unit-test',
'cluster': 'trusted',
'stream_name': 'unit-test_trusted_streamalert',
'shards': 1,
'retention': 24,
'create_user': True,
'trusted_accounts': ['98765432100']
}
},
'output': {
'kinesis_trusted_access_key_id': {
'value': '${module.kinesis_trusted.access_key_id}'
},
'kinesis_trusted_secret_key': {
'value': '${module.kinesis_trusted.secret_key}'
},
'kinesis_trusted_username': {
'value': '${module.kinesis_trusted.username}'
}
}
}
assert_true(result)
assert_equal(cluster_dict, expected_result)
def generate_threat_intel_downloader(config):
"""Generate Threat Intel Downloader Terrafrom
Args:
config (dict): The loaded config from the 'conf/' directory
Returns:
dict: Athena dict to be marshalled to JSON
"""
# Use the monitoring topic as a dead letter queue
dlq_topic, _ = monitoring_topic_name(config)
# Threat Intel Downloader module
ti_downloader_config = config['lambda']['threat_intel_downloader_config']
ti_downloader_dict = infinitedict()
ti_downloader_dict['module']['threat_intel_downloader'] = {
'account_id': config['global']['account']['aws_account_id'],
'region': config['global']['account']['region'],
'source': './modules/tf_threat_intel_downloader',
'lambda_handler': ThreatIntelDownloaderPackage.lambda_handler,
'lambda_memory': ti_downloader_config.get('memory', '128'),
'lambda_timeout': ti_downloader_config.get('timeout', '60'),
'lambda_log_level': ti_downloader_config.get('log_level', 'info'),
'interval': ti_downloader_config.get('interval', 'rate(1 day)'),
'prefix': config['global']['account']['prefix'],
'monitoring_sns_topic': dlq_topic,
'table_rcu': ti_downloader_config.get('table_rcu', '10'),
'table_wcu': ti_downloader_config.get('table_wcu', '10'),
'max_read_capacity': ti_downloader_config.get('max_read_capacity',
'5'),
'min_read_capacity': ti_downloader_config.get('min_read_capacity',
'5'),
'target_utilization': ti_downloader_config.get('target_utilization',
'70')
}
return ti_downloader_dict
def generate_main(config, init=False):
"""Generate the main.tf.json Terraform dict
Args:
config (CLIConfig): The loaded CLI config
init (bool): Terraform is running in the init phase or not (optional)
Returns:
dict: main.tf.json Terraform dict
"""
write_vars(config, region=config['global']['account']['region'])
main_dict = infinitedict()
logging_bucket, create_logging_bucket = s3_access_logging_bucket(config)
state_lock_table_name = '{}_streamalert_terraform_state_lock'.format(
config['global']['account']['prefix']
)
# Setup the Backend depending on the deployment phase.
# When first setting up StreamAlert, the Terraform statefile
# is stored locally. After the first dependencies are created,
# this moves to S3.
if init:
main_dict['terraform']['backend']['local'] = {
'path': 'terraform.tfstate',
}
else:
terraform_bucket_name, _ = terraform_state_bucket(config)
main_dict['terraform']['backend']['s3'] = {
'bucket': terraform_bucket_name,
'key': config['global'].get('terraform', {}).get(
'state_key_name',
'streamalert_state/terraform.tfstate'
),
'region': config['global']['account']['region'],
'encrypt': True,
'dynamodb_table': state_lock_table_name,
'acl': 'private',
'kms_key_id': 'alias/{}'.format(
config['global']['account'].get(
'kms_key_alias',
'{}_streamalert_secrets'.format(config['global']['account']['prefix'])
)
),
}
# Configure initial S3 buckets
main_dict['resource']['aws_s3_bucket'] = {
'streamalerts': generate_s3_bucket(
bucket=firehose_alerts_bucket(config),
logging=logging_bucket
)
}
# Configure remote state locking table
main_dict['resource']['aws_dynamodb_table'] = {
'terraform_remote_state_lock': {
'name': state_lock_table_name,
'billing_mode': 'PAY_PER_REQUEST',
'hash_key': 'LockID',
'attribute': {
'name': 'LockID',
'type': 'S'
},
'tags': {
'Name': 'StreamAlert'
}
}
}
# Create bucket for S3 access logs (if applicable)
if create_logging_bucket:
main_dict['resource']['aws_s3_bucket']['logging_bucket'] = generate_s3_bucket(
bucket=logging_bucket,
logging=logging_bucket,
acl='log-delivery-write',
lifecycle_rule={
'prefix': '/',
'enabled': True,
'transition': {
'days': 365,
'storage_class': 'GLACIER'
}
},
sse_algorithm='AES256' # SSE-KMS doesn't seem to work with access logs
)
terraform_bucket_name, create_state_bucket = terraform_state_bucket(config)
# Create bucket for Terraform state (if applicable)
if create_state_bucket:
main_dict['resource']['aws_s3_bucket']['terraform_remote_state'] = generate_s3_bucket(
bucket=terraform_bucket_name,
logging=logging_bucket
)
# Setup Firehose Delivery Streams
generate_firehose(logging_bucket, main_dict, config)
# Configure global resources like Firehose alert delivery and alerts table
main_dict['module']['globals'] = _generate_global_module(config)
# KMS Key and Alias creation
main_dict['resource']['aws_kms_key']['server_side_encryption'] = {
'enable_key_rotation': True,
'description': 'StreamAlert S3 Server-Side Encryption',
'policy': json.dumps({
'Version': '2012-10-17',
'Statement': [
{
'Sid': 'Enable IAM User Permissions',
'Effect': 'Allow',
'Principal': {
'AWS': 'arn:aws:iam::{}:root'.format(
config['global']['account']['aws_account_id']
)
},
'Action': 'kms:*',
'Resource': '*'
},
{
'Sid': 'Allow principals in the account to use the key',
'Effect': 'Allow',
'Principal': '*',
'Action': ['kms:Decrypt', 'kms:GenerateDataKey*', 'kms:Encrypt'],
'Resource': '*',
'Condition': {
'StringEquals': {
'kms:CallerAccount': config['global']['account']['aws_account_id']
}
}
}
]
})
}
main_dict['resource']['aws_kms_alias']['server_side_encryption'] = {
'name': 'alias/{}_server-side-encryption'.format(config['global']['account']['prefix']),
'target_key_id': '${aws_kms_key.server_side_encryption.key_id}'
}
main_dict['resource']['aws_kms_key']['streamalert_secrets'] = {
'enable_key_rotation': True,
'description': 'StreamAlert secret management'
}
main_dict['resource']['aws_kms_alias']['streamalert_secrets'] = {
'name': 'alias/{}'.format(
config['global']['account'].get(
'kms_key_alias',
'{}_streamalert_secrets'.format(config['global']['account']['prefix'])
)
),
'target_key_id': '${aws_kms_key.streamalert_secrets.key_id}'
}
# Global infrastructure settings
topic_name, create_topic = monitoring_topic_name(config)
if create_topic:
main_dict['resource']['aws_sns_topic']['monitoring'] = {
'name': topic_name
}
return main_dict
def setup(self):
"""Setup before each method"""
self.cluster_dict = common.infinitedict()
self.config = CLIConfig(config_path='tests/unit/conf')
def generate_athena(config):
"""Generate Athena Terraform.
Args:
config (dict): The loaded config from the 'conf/' directory
Returns:
dict: Athena dict to be marshalled to JSON
"""
athena_dict = infinitedict()
athena_config = config['lambda']['athena_partition_refresh_config']
data_buckets = sorted(athena_partition_buckets(config))
prefix = config['global']['account']['prefix']
database = athena_config.get('database_name',
'{}_streamalert'.format(prefix))
results_bucket_name = athena_query_results_bucket(config)
queue_name = athena_config.get(
'queue_name',
'{}_streamalert_athena_s3_notifications'.format(prefix)).strip()
logging_bucket, _ = s3_access_logging_bucket(config)
athena_dict['module']['streamalert_athena'] = {
's3_logging_bucket': logging_bucket,
'source': './modules/tf_athena',
'database_name': database,
'queue_name': queue_name,
'results_bucket': results_bucket_name,
'kms_key_id': '${aws_kms_key.server_side_encryption.key_id}',
'lambda_handler': AthenaPackage.lambda_handler,
'lambda_memory': athena_config.get('memory', '128'),
'lambda_timeout': athena_config.get('timeout', '60'),
'lambda_log_level': athena_config.get('log_level', 'info'),
'athena_data_buckets': data_buckets,
'concurrency_limit': athena_config.get('concurrency_limit', 10),
'account_id': config['global']['account']['aws_account_id'],
'prefix': prefix
}
# Cloudwatch monitoring setup
athena_dict['module']['athena_monitoring'] = {
'source':
'./modules/tf_monitoring',
'sns_topic_arn':
monitoring_topic_arn(config),
'lambda_functions':
['{}_streamalert_athena_partition_refresh'.format(prefix)],
'kinesis_alarms_enabled':
False
}
# Metrics setup
if not athena_config.get('enable_custom_metrics', False):
return athena_dict
# Check to see if there are any metrics configured for the athena function
current_metrics = metrics.MetricLogger.get_available_metrics()
if metrics.ATHENA_PARTITION_REFRESH_NAME not in current_metrics:
return athena_dict
metric_prefix = 'AthenaRefresh'
filter_pattern_idx, filter_value_idx = 0, 1
# Add filters for the cluster and aggregate
# Use a list of strings that represent the following comma separated values:
# <filter_name>,<filter_pattern>,<value>
filters = [
'{},{},{}'.format('{}-{}'.format(metric_prefix,
metric), settings[filter_pattern_idx],
settings[filter_value_idx]) for metric, settings in
current_metrics[metrics.ATHENA_PARTITION_REFRESH_NAME].items()
]
athena_dict['module']['streamalert_athena'][
'athena_metric_filters'] = filters
return athena_dict
def generate_aggregate_cloudwatch_metric_filters(config):
"""Return the CloudWatch Metric Filters information for aggregate metrics
Args:
config (dict): The loaded config from the 'conf/' directory
"""
functions = {
cluster: [
func.replace('_config', '') for func in CLUSTERED_FUNCTIONS
if cluster_config['{}_config'.format(func)].get(
'enable_custom_metrics')
]
for cluster, cluster_config in config['clusters'].items()
}
functions['global'] = {
func.replace('_config', '')
for func, func_config in config['lambda'].items()
if func_config.get('enable_custom_metrics')
}
if not any(funcs for funcs in list(functions.values())):
return # Nothing to add if no funcs have metrics enabled
result = infinitedict()
current_metrics = metrics.MetricLogger.get_available_metrics()
for cluster, functions in functions.items():
is_global = cluster == 'global'
for function in functions:
# This function may not actually support any custom metrics
if function not in current_metrics:
continue
metric_prefix = metrics.FUNC_PREFIXES.get(function)
if not metric_prefix:
continue
log_group_name = (
'${{module.{}_{}_lambda.log_group_name}}'.format(
function, cluster) if not is_global else
'${{module.{}_lambda.log_group_name}}'.format(function))
cluster = cluster.upper()
if not is_global:
cluster = '{}_AGGREGATE'.format(cluster)
# Add filters for the cluster and aggregate
for metric, filter_settings in current_metrics[function].items():
module_name = 'metric_filters_{}_{}_{}'.format(
metric_prefix, metric, cluster)
result['module'][module_name] = {
'source': './modules/tf_metric_filters',
'log_group_name': log_group_name,
'metric_name': '{}-{}'.format(metric_prefix, metric),
'metric_pattern': filter_settings[0],
'metric_value': filter_settings[1],
}
return result
def test_generate_classifier(self):
"""CLI - Terraform Generate, Classifier"""
cluster_dict = common.infinitedict()
classifier.generate_classifier('test', cluster_dict, self.config)
expected_result = {
'module': {
'classifier_test_iam': {
'source':
'./modules/tf_classifier',
'account_id':
'123456789012',
'region':
'us-east-1',
'prefix':
'unit-test',
'firehose_use_prefix':
True,
'function_role_id':
'${module.classifier_test_lambda.role_id}',
'function_alias_arn':
'${module.classifier_test_lambda.function_alias_arn}',
'function_name':
'${module.classifier_test_lambda.function_name}',
'classifier_sqs_queue_arn':
'${module.globals.classifier_sqs_queue_arn}',
'classifier_sqs_sse_kms_key_arn':
('${module.globals.classifier_sqs_sse_kms_key_arn}'),
'input_sns_topics':
['arn:aws:sns:us-east-1:123456789012:foo_bar']
},
'classifier_test_lambda': {
'alarm_actions':
['arn:aws:sns:us-east-1:123456789012:test_topic'],
'description':
'Unit-Test Test Streamalert Classifier',
'environment_variables': {
'CLUSTER': 'test',
'SQS_QUEUE_URL':
'${module.globals.classifier_sqs_queue_url}',
'LOGGER_LEVEL': 'info',
'ENABLE_METRICS': '0'
},
'tags': {
'Cluster': 'test'
},
'errors_alarm_enabled':
True,
'errors_alarm_evaluation_periods':
1,
'errors_alarm_period_secs':
120,
'errors_alarm_threshold':
0,
'function_name':
'unit-test_test_streamalert_classifier',
'handler':
'streamalert.classifier.main.handler',
'log_retention_days':
14,
'memory_size_mb':
128,
'source':
'./modules/tf_lambda',
'throttles_alarm_enabled':
True,
'throttles_alarm_evaluation_periods':
1,
'throttles_alarm_period_secs':
120,
'throttles_alarm_threshold':
0,
'timeout_sec':
60,
'vpc_security_group_ids': [],
'vpc_subnet_ids': [],
'input_sns_topics':
['arn:aws:sns:us-east-1:123456789012:foo_bar']
}
}
}
assert_equal(cluster_dict, expected_result)
def generate_athena(config):
"""Generate Athena Terraform.
Args:
config (dict): The loaded config from the 'conf/' directory
Returns:
dict: Athena dict to be marshalled to JSON
"""
result = infinitedict()
prefix = config['global']['account']['prefix']
athena_config = config['lambda']['athena_partitioner_config']
data_buckets = athena_partition_buckets_tf(config)
database = athena_config.get('database_name',
'{}_streamalert'.format(prefix))
results_bucket_name = athena_query_results_bucket(config)
queue_name = athena_config.get(
'queue_name',
'{}_streamalert_athena_s3_notifications'.format(prefix)).strip()
logging_bucket, _ = s3_access_logging_bucket(config)
# Set variables for the athena partitioner's IAM permissions
result['module']['athena_partitioner_iam'] = {
'source':
'./modules/tf_athena',
'account_id':
config['global']['account']['aws_account_id'],
'prefix':
prefix,
's3_logging_bucket':
logging_bucket,
'database_name':
database,
'queue_name':
queue_name,
'athena_data_buckets':
data_buckets,
'results_bucket':
results_bucket_name,
'lambda_timeout':
athena_config['timeout'],
'kms_key_id':
'${aws_kms_key.server_side_encryption.key_id}',
'function_role_id':
'${module.athena_partitioner_lambda.role_id}',
'function_name':
'${module.athena_partitioner_lambda.function_name}',
'function_alias_arn':
'${module.athena_partitioner_lambda.function_alias_arn}',
}
# Set variables for the Lambda module
result['module']['athena_partitioner_lambda'] = generate_lambda(
'{}_streamalert_{}'.format(prefix, ATHENA_PARTITIONER_NAME),
'streamalert.athena_partitioner.main.handler',
athena_config,
config,
tags={'Subcomponent': 'AthenaPartitioner'})
return result
