def test_generate_cloudwatch_monitoring_custom_sns(): """CLI - Terraform Generate Cloudwatch Monitoring with Existing SNS Topic""" # Test a custom SNS topic name CONFIG['global']['infrastructure']['monitoring'][ 'create_sns_topic'] = False CONFIG['global']['infrastructure']['monitoring']\ ['sns_topic_name'] = 'unit_test_monitoring' cluster_dict = _common.infinitedict() result = monitoring.generate_monitoring('test', cluster_dict, CONFIG) expected_cloudwatch_tf_custom = { 'source': 'modules/tf_stream_alert_monitoring', 'sns_topic_arn': 'arn:aws:sns:us-west-1:12345678910:unit_test_monitoring', 'lambda_functions': [ 'unit-testing_test_streamalert_rule_processor', 'unit-testing_test_streamalert_alert_processor' ], 'kinesis_stream': 'unit-testing_test_stream_alert_kinesis' } assert_true(result) assert_equal(cluster_dict['module']['cloudwatch_monitoring_test'], expected_cloudwatch_tf_custom)
def test_generate_s3_events(): """CLI - Terraform S3 Events with Valid Bucket""" cluster_dict = _common.infinitedict() CONFIG['clusters']['advanced']['modules']['s3_events'] = { 's3_bucket_id': 'unit-test-bucket.data' } result = s3_events.generate_s3_events('advanced', cluster_dict, CONFIG) expected_config = { 'module': { 's3_events_advanced': { 'source': 'modules/tf_stream_alert_s3_events', 'lambda_function_arn': '${module.stream_alert_advanced.lambda_arn}', 'lambda_function_name': 'unit-testing_advanced_stream_alert_processor', 's3_bucket_id': 'unit-test-bucket.data', 's3_bucket_arn': 'arn:aws:s3:::unit-test-bucket.data', 'lambda_role_id': '${module.stream_alert_advanced.lambda_role_id}', 'lambda_role_arn': '${module.stream_alert_advanced.lambda_role_arn}' } } } assert_true(result) assert_equal(cluster_dict, expected_config)
def test_generate_s3_events(): """CLI - Terraform S3 Events with Valid Buckets""" cluster_dict = _common.infinitedict() result = s3_events.generate_s3_events('advanced', cluster_dict, CONFIG) expected_config = { 'module': { 's3_events_unit-test-bucket_data': { 'source': 'modules/tf_stream_alert_s3_events', 'lambda_function_arn': '${module.stream_alert_advanced.lambda_arn}', 'bucket_id': 'unit-test-bucket.data', 'enable_events': True, 'lambda_role_id': '${module.stream_alert_advanced.lambda_role_id}', }, 's3_events_unit-test_cloudtrail_data': { 'source': 'modules/tf_stream_alert_s3_events', 'lambda_function_arn': '${module.stream_alert_advanced.lambda_arn}', 'bucket_id': 'unit-test.cloudtrail.data', 'enable_events': False, 'lambda_role_id': '${module.stream_alert_advanced.lambda_role_id}', } } } assert_true(result) assert_equal(cluster_dict, expected_config)
def test_generate_cloudwatch_monitoring_no_kinesis(): """CLI - Terraform Generate Cloudwatch Monitoring - Kinesis Disabled""" cluster_dict = _common.infinitedict() CONFIG['clusters']['test']['modules']['cloudwatch_monitoring'][ 'kinesis_alarms_enabled'] = False CONFIG['clusters']['test']['modules']['cloudwatch_monitoring'][ 'lambda_alarms_enabled'] = True result = monitoring.generate_monitoring('test', cluster_dict, CONFIG) # Test a the default SNS topic option expected_cloudwatch_tf = { 'source': 'modules/tf_stream_alert_monitoring', 'sns_topic_arn': 'arn:aws:sns:us-west-1:12345678910:stream_alert_monitoring', 'lambda_functions': [ 'unit-testing_test_streamalert_rule_processor', 'unit-testing_test_streamalert_alert_processor' ], 'lambda_alarms_enabled': True, 'kinesis_alarms_enabled': False } assert_true(result) assert_equal(cluster_dict['module']['cloudwatch_monitoring_test'], expected_cloudwatch_tf)
def test_kinesis_streams_with_trusted_account(): """CLI - Terraform Generate Kinesis Streams with trusted account""" cluster_dict = _common.infinitedict() result = kinesis_streams.generate_kinesis_streams('trusted', cluster_dict, CONFIG) expected_result = { 'module': { 'kinesis_trusted': { 'source': 'modules/tf_stream_alert_kinesis_streams', 'account_id': '12345678910', 'shard_level_metrics': [], 'region': 'us-west-1', 'prefix': 'unit-testing', 'cluster_name': 'trusted', 'stream_name': 'unit-testing_trusted_stream_alert_kinesis', 'shards': 1, 'retention': 24, 'create_user': True, 'trusted_accounts': ['98765432100'] } } } assert_true(result) assert_equal(cluster_dict, expected_result)
def test_generate_cloudwatch_monitoring_with_settings(): """CLI - Terraform Generate Cloudwatch Monitoring with Custom Settings""" cluster_dict = _common.infinitedict() result = monitoring.generate_monitoring('advanced', cluster_dict, CONFIG) # Test the default SNS topic option expected_cloudwatch_tf = { 'source': 'modules/tf_stream_alert_monitoring', 'sns_topic_arn': 'arn:aws:sns:us-west-1:12345678910:stream_alert_monitoring', 'lambda_functions': [ 'unit-testing_advanced_streamalert_rule_processor', 'unit-testing_advanced_streamalert_alert_processor' ], 'kinesis_stream': 'unit-testing_advanced_stream_alert_kinesis', 'lambda_alarms_enabled': True, 'kinesis_alarms_enabled': True, 'kinesis_iterator_age_error_threshold': '3000000' } assert_true(result) assert_equal(cluster_dict['module']['cloudwatch_monitoring_advanced'], expected_cloudwatch_tf)
def test_kinesis_streams(): """CLI - Terraform Generate Kinesis Streams""" cluster_dict = _common.infinitedict() result = kinesis_streams.generate_kinesis_streams('advanced', cluster_dict, CONFIG) expected_result = { 'module': { 'kinesis_advanced': { 'source': 'modules/tf_stream_alert_kinesis_streams', 'account_id': '12345678910', 'shard_level_metrics': ["IncomingBytes"], 'region': 'us-west-1', 'prefix': 'unit-testing', 'cluster_name': 'advanced', 'stream_name': 'unit-testing_advanced_stream_alert_kinesis', 'shards': 1, 'retention': 24, 'create_user': True } } } assert_true(result) assert_equal(cluster_dict, expected_result)
def generate_cluster(**kwargs): """Generate a StreamAlert cluster file. Keyword Args: cluster_name (str): The name of the currently generating cluster config (dict): The loaded config from the 'conf/' directory Returns: dict: generated Terraform cluster dictionary """ config = kwargs.get('config') cluster_name = kwargs.get('cluster_name') modules = config['clusters'][cluster_name]['modules'] cluster_dict = infinitedict() if not generate_stream_alert(cluster_name, cluster_dict, config): return generate_cloudwatch_metric_filters(cluster_name, cluster_dict, config) generate_cloudwatch_metric_alarms(cluster_name, cluster_dict, config) if modules.get('cloudwatch_monitoring', {}).get('enabled'): if not generate_monitoring(cluster_name, cluster_dict, config): return if modules.get('kinesis'): if not generate_kinesis_streams(cluster_name, cluster_dict, config): return outputs = config['clusters'][cluster_name].get('outputs') if outputs: if not generate_outputs(cluster_name, cluster_dict, config): return if modules.get('kinesis_events'): if not generate_kinesis_events(cluster_name, cluster_dict, config): return cloudtrail_info = modules.get('cloudtrail') if cloudtrail_info: if not generate_cloudtrail(cluster_name, cluster_dict, config): return flow_log_info = modules.get('flow_logs') if flow_log_info: if not generate_flow_logs(cluster_name, cluster_dict, config): return s3_events_info = modules.get('s3_events') if s3_events_info: if not generate_s3_events(cluster_name, cluster_dict, config): return generate_app_integrations(cluster_name, cluster_dict, config) return cluster_dict
def test_generate_cloudwatch_monitoring_disabled(): """CLI - Terraform Generate Cloudwatch Monitoring Disabled""" cluster_dict = _common.infinitedict() cluster = 'trusted' result = monitoring.generate_monitoring(cluster, cluster_dict, CONFIG) assert_true(result) assert_true('cloudwatch_monitoring_{}'.format(cluster) not in cluster_dict['module'])
def test_generate_cloudwatch_monitoring_invalid_config(mock_logging): """CLI - Terraform Generate Cloudwatch Monitoring with Invalid Config""" CONFIG['global']['infrastructure'] = {} cluster_dict = _common.infinitedict() result = monitoring.generate_monitoring('test', cluster_dict, CONFIG) assert_true(mock_logging.error.called) assert_false(result)
def test_generate_s3_events_invalid_bucket(mock_logging): """CLI - Terraform S3 Events with Missing Bucket Key""" cluster_dict = _common.infinitedict() CONFIG['clusters']['advanced']['modules']['s3_events'] = { 'wrong_key': 'my-bucket!!!' } result = s3_events.generate_s3_events('advanced', cluster_dict, CONFIG) assert_true(mock_logging.error.called) assert_false(result)
def test_generate_s3_events_legacy(): """CLI - Terraform S3 Events - Legacy""" cluster_dict = _common.infinitedict() CONFIG['clusters']['test']['modules']['s3_events'] = { 's3_bucket_id': 'unit-test-bucket.legacy.data' } result = s3_events.generate_s3_events('test', cluster_dict, CONFIG) # Reload the config CONFIG.load() assert_true(result) assert_equal(CONFIG['clusters']['test']['modules']['s3_events'], [{ 'bucket_id': 'unit-test-bucket.legacy.data' }])
def generate_threat_intel_downloader(config): """Generate Threat Intel Downloader Terrafrom Args: config (dict): The loaded config from the 'conf/' directory Returns: dict: Athena dict to be marshalled to JSON """ # Use the monitoring topic as a dead letter queue infrastructure_config = config['global'].get('infrastructure') dlq_topic = (DEFAULT_SNS_MONITORING_TOPIC if infrastructure_config.get( 'monitoring', {}).get('create_sns_topic') else infrastructure_config.get('monitoring', {}).get( 'sns_topic_name', DEFAULT_SNS_MONITORING_TOPIC)) # Threat Intel Downloader module ti_downloader_config = config['lambda']['threat_intel_downloader_config'] ti_downloader_dict = infinitedict() ti_downloader_dict['module']['threat_intel_downloader'] = { 'account_id': config['global']['account']['aws_account_id'], 'region': config['global']['account']['region'], 'source': 'modules/tf_threat_intel_downloader', 'lambda_function_arn': '${module.threat_intel_downloader.lambda_arn}', 'lambda_handler': ti_downloader_config['handler'], 'lambda_memory': ti_downloader_config.get('memory', '128'), 'lambda_timeout': ti_downloader_config.get('timeout', '60'), 'lambda_s3_bucket': ti_downloader_config['source_bucket'], 'lambda_s3_key': ti_downloader_config['source_object_key'], 'lambda_log_level': ti_downloader_config.get('log_level', 'info'), 'interval': ti_downloader_config.get('interval', 'rate(1 day)'), 'current_version': ti_downloader_config['current_version'], 'prefix': config['global']['account']['prefix'], 'monitoring_sns_topic': dlq_topic, 'table_rcu': ti_downloader_config.get('table_rcu', '10'), 'table_wcu': ti_downloader_config.get('table_wcu', '10'), 'ioc_keys': ti_downloader_config.get('ioc_keys'), 'ioc_filters': ti_downloader_config.get('ioc_filters'), 'ioc_types': ti_downloader_config.get('ioc_types') } return ti_downloader_dict
def test_kinesis_events(): """CLI - Terraform Generate Kinesis Events""" cluster_dict = _common.infinitedict() result = kinesis_events.generate_kinesis_events('advanced', cluster_dict, CONFIG) expected_result = { 'module': { 'kinesis_events_advanced': { 'source': 'modules/tf_stream_alert_kinesis_events', 'lambda_production_enabled': True, 'lambda_role_id': '${module.stream_alert_advanced.lambda_role_id}', 'lambda_function_arn': '${module.stream_alert_advanced.lambda_arn}', 'kinesis_stream_arn': '${module.kinesis_advanced.arn}', 'role_policy_prefix': 'advanced' } } } assert_true(result) assert_equal(cluster_dict, expected_result)
def test_generate_s3_events(): """CLI - Terraform - S3 Events with Valid Buckets""" cluster_dict = _common.infinitedict() result = s3_events.generate_s3_events('advanced', cluster_dict, CONFIG) expected_config = { 'module': { 's3_events_unit-testing_advanced_0': { 'source': 'modules/tf_stream_alert_s3_events', 'lambda_function_arn': '${module.stream_alert_advanced.lambda_arn}', 'bucket_id': 'unit-test-bucket.data', 'notification_id': 'advanced_0', 'enable_events': True, 'lambda_role_id': '${module.stream_alert_advanced.lambda_role_id}', 'filter_suffix': '.log', 'filter_prefix': 'AWSLogs/123456789/CloudTrail/us-east-1/' }, 's3_events_unit-testing_advanced_1': { 'source': 'modules/tf_stream_alert_s3_events', 'lambda_function_arn': '${module.stream_alert_advanced.lambda_arn}', 'bucket_id': 'unit-test.cloudtrail.data', 'enable_events': False, 'notification_id': 'advanced_1', 'lambda_role_id': '${module.stream_alert_advanced.lambda_role_id}', 'filter_suffix': '', 'filter_prefix': '' } } } assert_true(result) assert_equal(cluster_dict, expected_config)
def generate_main(**kwargs): """Generate the main.tf.json Terraform dict Keyword Args: init (bool): If Terraform is running in the init phase or not config (CLIConfig): The loaded CLI config Returns: dict: main.tf.json Terraform dict """ init = kwargs.get('init') config = kwargs['config'] main_dict = infinitedict() # Configure provider along with the minimum version main_dict['provider']['aws'] = {'version': '~> 0.1.4'} # Configure Terraform version requirement main_dict['terraform']['required_version'] = '~> 0.10.6' # Setup the Backend dependencing on the deployment phase. # When first setting up StreamAlert, the Terraform statefile # is stored locally. After the first dependencies are created, # this moves to S3. if init: main_dict['terraform']['backend']['local'] = { 'path': 'terraform.tfstate'} else: main_dict['terraform']['backend']['s3'] = { 'bucket': '{}.streamalert.terraform.state'.format( config['global']['account']['prefix']), 'key': 'stream_alert_state/terraform.tfstate', 'region': config['global']['account']['region'], 'encrypt': True, 'acl': 'private', 'kms_key_id': 'alias/{}'.format(config['global']['account']['kms_key_alias'])} logging_bucket = '{}.streamalert.s3-logging'.format( config['global']['account']['prefix']) logging_bucket_lifecycle = { 'prefix': '/', 'enabled': True, 'transition': { 'days': 30, 'storage_class': 'GLACIER'}} # Configure initial S3 buckets main_dict['resource']['aws_s3_bucket'] = { 'lambda_source': generate_s3_bucket( bucket=config['lambda']['rule_processor_config']['source_bucket'], logging=logging_bucket ), 'stream_alert_secrets': generate_s3_bucket( bucket='{}.streamalert.secrets'.format(config['global']['account']['prefix']), logging=logging_bucket ), 'terraform_remote_state': generate_s3_bucket( bucket=config['global']['terraform']['tfstate_bucket'], logging=logging_bucket ), 'logging_bucket': generate_s3_bucket( bucket=logging_bucket, acl='log-delivery-write', logging=logging_bucket, lifecycle_rule=logging_bucket_lifecycle ), 'streamalerts': generate_s3_bucket( bucket='{}.streamalerts'.format(config['global']['account']['prefix']), logging=logging_bucket ) } # Conditionally configure Firehose if config['global']['infrastructure'].get('firehose', {}).get('enabled'): firehose_config = config['global']['infrastructure']['firehose'] firehose_s3_bucket_suffix = firehose_config.get('s3_bucket_suffix', 'streamalert.data') firehose_s3_bucket_name = '{}.{}'.format(config['global']['account']['prefix'], firehose_s3_bucket_suffix) # Add the main Firehose module main_dict['module']['kinesis_firehose'] = { 'source': 'modules/tf_stream_alert_kinesis_firehose', 'account_id': config['global']['account']['aws_account_id'], 'region': config['global']['account']['region'], 'prefix': config['global']['account']['prefix'], 'logs': enabled_firehose_logs(config), 'buffer_size': config['global']['infrastructure'] ['firehose'].get('buffer_size', 64), 'buffer_interval': config['global']['infrastructure'] ['firehose'].get('buffer_interval', 300), 'compression_format': config['global']['infrastructure'] ['firehose'].get('compression_format', 'GZIP'), 's3_logging_bucket': logging_bucket, 's3_bucket_name': firehose_s3_bucket_name } # KMS Key and Alias creation main_dict['resource']['aws_kms_key']['stream_alert_secrets'] = { 'enable_key_rotation': True, 'description': 'StreamAlert secret management' } main_dict['resource']['aws_kms_alias']['stream_alert_secrets'] = { 'name': 'alias/{}'.format(config['global']['account']['kms_key_alias']), 'target_key_id': '${aws_kms_key.stream_alert_secrets.key_id}' } # Global infrastructure settings infrastructure_config = config['global'].get('infrastructure') if infrastructure_config and 'monitoring' in infrastructure_config: if infrastructure_config['monitoring'].get('create_sns_topic'): main_dict['resource']['aws_sns_topic']['stream_alert_monitoring'] = { 'name': DEFAULT_SNS_MONITORING_TOPIC } # Add any global cloudwatch alarms to the main.tf monitoring_config = config['global']['infrastructure'].get('monitoring') if not monitoring_config: return main_dict global_metrics = monitoring_config.get('metric_alarms') if not global_metrics: return main_dict topic_name = (DEFAULT_SNS_MONITORING_TOPIC if infrastructure_config ['monitoring'].get('create_sns_topic') else infrastructure_config['monitoring'].get('sns_topic_name')) sns_topic_arn = 'arn:aws:sns:{region}:{account_id}:{topic}'.format( region=config['global']['account']['region'], account_id=config['global']['account']['aws_account_id'], topic=topic_name ) formatted_alarms = {} # Add global metric alarms for the rule and alert processors for func in FUNC_PREFIXES: if func not in global_metrics: continue for name, settings in global_metrics[func].iteritems(): alarm_info = settings.copy() alarm_info['alarm_name'] = name alarm_info['namespace'] = 'StreamAlert' alarm_info['alarm_actions'] = [sns_topic_arn] # Terraform only allows certain characters in resource names acceptable_chars = ''.join([string.digits, string.letters, '_-']) name = filter(acceptable_chars.__contains__, name) formatted_alarms['metric_alarm_{}'.format(name)] = alarm_info if formatted_alarms: main_dict['resource']['aws_cloudwatch_metric_alarm'] = formatted_alarms return main_dict
def generate_athena(config): """Generate Athena Terraform. Args: config (dict): The loaded config from the 'conf/' directory Returns: dict: Athena dict to be marshalled to JSON """ athena_dict = infinitedict() athena_config = config['lambda']['athena_partition_refresh_config'] data_buckets = set() for refresh_type in athena_config['refresh_type']: data_buckets.update(set(athena_config['refresh_type'][refresh_type])) athena_dict['module']['stream_alert_athena'] = { 'source': 'modules/tf_stream_alert_athena', 'lambda_handler': athena_config['handler'], 'lambda_memory': athena_config.get('memory', '128'), 'lambda_timeout': athena_config.get('timeout', '60'), 'lambda_s3_bucket': athena_config['source_bucket'], 'lambda_s3_key': athena_config['source_object_key'], 'lambda_log_level': athena_config.get('log_level', 'info'), 'athena_data_buckets': list(data_buckets), 'refresh_interval': athena_config.get('refresh_interval', 'rate(10 minutes)'), 'current_version': athena_config['current_version'], 'enable_metrics': athena_config.get('enable_metrics', False), 'prefix': config['global']['account']['prefix'] } # Cloudwatch monitoring setup monitoring_config = config['global'].get('infrastructure', {}).get('monitoring', {}) sns_topic_name = DEFAULT_SNS_MONITORING_TOPIC if monitoring_config.get( 'create_sns_topic') else monitoring_config.get('sns_topic_name') athena_dict['module']['athena_monitoring'] = { 'source': 'modules/tf_stream_alert_monitoring', 'sns_topic_arn': 'arn:aws:sns:{region}:{account_id}:{topic}'.format( region=config['global']['account']['region'], account_id=config['global']['account']['aws_account_id'], topic=sns_topic_name), 'lambda_functions': [ '{}_streamalert_athena_partition_refresh'.format( config['global']['account']['prefix']) ], 'kinesis_alarms_enabled': False } # Metrics setup if not athena_config.get('enable_metrics', False): return athena_dict # Check to see if there are any metrics configured for the athena function current_metrics = metrics.MetricLogger.get_available_metrics() if metrics.ATHENA_PARTITION_REFRESH_NAME not in current_metrics: return athena_dict metric_prefix = 'AthenaRefresh' filter_pattern_idx, filter_value_idx = 0, 1 # Add filters for the cluster and aggregate # Use a list of strings that represnt the following comma separated values: # <filter_name>,<filter_pattern>,<value> filters = [ '{},{},{}'.format('{}-{}'.format(metric_prefix, metric), settings[filter_pattern_idx], settings[filter_value_idx]) for metric, settings in current_metrics[metrics.ATHENA_PARTITION_REFRESH_NAME].iteritems() ] athena_dict['module']['stream_alert_athena'][ 'athena_metric_filters'] = filters return athena_dict
def setup(self): """Setup before each method""" self.cluster_dict = _common.infinitedict() self.config = CLIConfig(config_path='tests/unit/conf')