def test_firehose_enabled_log_alarm_defaults(self): """CLI - Terraform Generate Kinesis Firehose, Enabled Alarm - Default Settings""" cluster_dict = common.infinitedict() # Add an enabled log, with alarms on (will use terraform default settings) self.config['global']['infrastructure']['firehose']['enabled_logs'] = { 'json:embedded': { 'enable_alarm': True } } firehose.generate_firehose(self._logging_bucket_name, cluster_dict, self.config) expected_result = { 'module': { 'kinesis_firehose_setup': self._default_firehose_config(), 'kinesis_firehose_json_embedded': { 'source': 'modules/tf_stream_alert_kinesis_firehose_delivery_stream', 'buffer_size': 128, 'buffer_interval': 900, 'compression_format': 'GZIP', 'log_name': 'json_embedded', 'role_arn': '${module.kinesis_firehose_setup.firehose_role_arn}', 's3_bucket_name': 'unit-testing.streamalert.data', 'kms_key_arn': '${aws_kms_key.server_side_encryption.arn}', 'enable_alarm': True, 'alarm_actions': ['arn:aws:sns:us-west-1:12345678910:stream_alert_monitoring'] } } } assert_equal(cluster_dict, expected_result)
def test_firehose_enabled_log(self): """CLI - Terraform Generate Kinesis Firehose, Enabled Log""" cluster_dict = common.infinitedict() # Add an enabled log, with no alarm configuration (aka: alarms disabled) self.config['global']['infrastructure']['firehose']['enabled_logs'] = { 'json:embedded': {} } firehose.generate_firehose(self._logging_bucket_name, cluster_dict, self.config) expected_result = { 'module': { 'kinesis_firehose_setup': self._default_firehose_config(), 'kinesis_firehose_json_embedded': { 'source': 'modules/tf_stream_alert_kinesis_firehose_delivery_stream', 'buffer_size': 128, 'buffer_interval': 900, 'compression_format': 'GZIP', 'log_name': 'json_embedded', 'role_arn': '${module.kinesis_firehose_setup.firehose_role_arn}', 's3_bucket_name': 'unit-testing.streamalert.data', 'kms_key_arn': '${aws_kms_key.server_side_encryption.arn}' } } } assert_equal(cluster_dict, expected_result)
def test_firehose_defaults(self): """CLI - Terraform Generate Kinesis Firehose, Defaults""" cluster_dict = common.infinitedict() firehose.generate_firehose(self._logging_bucket_name, cluster_dict, self.config) expected_result = { 'module': { 'kinesis_firehose_setup': self._default_firehose_config(), } } assert_equal(cluster_dict, expected_result)
def generate_main(config, init=False): """Generate the main.tf.json Terraform dict Args: config (CLIConfig): The loaded CLI config init (bool): Terraform is running in the init phase or not (optional) Returns: dict: main.tf.json Terraform dict """ main_dict = infinitedict() # Configure provider along with the minimum version main_dict['provider']['aws'] = { 'version': TERRAFORM_VERSIONS['provider']['aws'], 'region': config['global']['account']['region'] } # Configure Terraform version requirement main_dict['terraform']['required_version'] = TERRAFORM_VERSIONS['application'] # Setup the Backend depending on the deployment phase. # When first setting up StreamAlert, the Terraform statefile # is stored locally. After the first dependencies are created, # this moves to S3. if init: main_dict['terraform']['backend']['local'] = { 'path': 'terraform.tfstate'} else: main_dict['terraform']['backend']['s3'] = { 'bucket': config['global']['terraform']['tfstate_bucket'], 'key': config['global']['terraform']['tfstate_s3_key'], 'region': config['global']['account']['region'], 'encrypt': True, 'acl': 'private', 'kms_key_id': 'alias/{}'.format(config['global']['account']['kms_key_alias'])} logging_bucket = config['global']['s3_access_logging']['logging_bucket'] # Configure initial S3 buckets main_dict['resource']['aws_s3_bucket'] = { 'stream_alert_secrets': generate_s3_bucket( # FIXME (derek.wang) DRY out by using OutputCredentialsProvider? bucket='{}.streamalert.secrets'.format(config['global']['account']['prefix']), logging=logging_bucket ), 'streamalerts': generate_s3_bucket( bucket='{}.streamalerts'.format(config['global']['account']['prefix']), logging=logging_bucket ) } # Create bucket for S3 access logs (if applicable) if config['global']['s3_access_logging'].get('create_bucket', True): main_dict['resource']['aws_s3_bucket']['logging_bucket'] = generate_s3_bucket( bucket=logging_bucket, logging=logging_bucket, acl='log-delivery-write', lifecycle_rule={ 'prefix': '/', 'enabled': True, 'transition': { 'days': 365, 'storage_class': 'GLACIER' } }, sse_algorithm='AES256' # SSE-KMS doesn't seem to work with access logs ) # Create bucket for Terraform state (if applicable) if config['global']['terraform'].get('create_bucket', True): main_dict['resource']['aws_s3_bucket']['terraform_remote_state'] = generate_s3_bucket( bucket=config['global']['terraform']['tfstate_bucket'], logging=logging_bucket ) # Setup Firehose Delivery Streams generate_firehose(logging_bucket, main_dict, config) # Configure global resources like Firehose alert delivery and alerts table global_module = { 'source': 'modules/tf_stream_alert_globals', 'account_id': config['global']['account']['aws_account_id'], 'region': config['global']['account']['region'], 'prefix': config['global']['account']['prefix'], 'kms_key_arn': '${aws_kms_key.server_side_encryption.arn}', 'alerts_table_read_capacity': ( config['global']['infrastructure']['alerts_table']['read_capacity']), 'alerts_table_write_capacity': ( config['global']['infrastructure']['alerts_table']['write_capacity']), 'rules_engine_timeout': config['lambda']['rules_engine_config']['timeout'] } if config['global']['infrastructure']['rule_staging'].get('enabled'): global_module['enable_rule_staging'] = True global_module['rules_table_read_capacity'] = ( config['global']['infrastructure']['rule_staging']['table']['read_capacity']) global_module['rules_table_write_capacity'] = ( config['global']['infrastructure']['rule_staging']['table']['write_capacity']) main_dict['module']['globals'] = global_module # KMS Key and Alias creation main_dict['resource']['aws_kms_key']['server_side_encryption'] = { 'enable_key_rotation': True, 'description': 'StreamAlert S3 Server-Side Encryption', 'policy': json.dumps({ 'Version': '2012-10-17', 'Statement': [ { 'Sid': 'Enable IAM User Permissions', 'Effect': 'Allow', 'Principal': { 'AWS': 'arn:aws:iam::{}:root'.format( config['global']['account']['aws_account_id'] ) }, 'Action': 'kms:*', 'Resource': '*' }, { 'Sid': 'Allow principals in the account to use the key', 'Effect': 'Allow', 'Principal': '*', 'Action': ['kms:Decrypt', 'kms:GenerateDataKey*', 'kms:Encrypt'], 'Resource': '*', 'Condition': { 'StringEquals': { 'kms:CallerAccount': config['global']['account']['aws_account_id'] } } } ] }) } main_dict['resource']['aws_kms_alias']['server_side_encryption'] = { 'name': 'alias/{}_server-side-encryption'.format(config['global']['account']['prefix']), 'target_key_id': '${aws_kms_key.server_side_encryption.key_id}' } main_dict['resource']['aws_kms_key']['stream_alert_secrets'] = { 'enable_key_rotation': True, 'description': 'StreamAlert secret management' } main_dict['resource']['aws_kms_alias']['stream_alert_secrets'] = { 'name': 'alias/{}'.format(config['global']['account']['kms_key_alias']), 'target_key_id': '${aws_kms_key.stream_alert_secrets.key_id}' } # Global infrastructure settings infrastructure_config = config['global'].get('infrastructure') if infrastructure_config and 'monitoring' in infrastructure_config: if infrastructure_config['monitoring'].get('create_sns_topic'): main_dict['resource']['aws_sns_topic']['stream_alert_monitoring'] = { 'name': DEFAULT_SNS_MONITORING_TOPIC } return main_dict
def generate_main(config, init=False): """Generate the main.tf.json Terraform dict Args: config (CLIConfig): The loaded CLI config init (bool): Terraform is running in the init phase or not (optional) Returns: dict: main.tf.json Terraform dict """ main_dict = infinitedict() # Configure provider along with the minimum version main_dict['provider']['aws'] = {'version': TERRAFORM_VERSIONS['provider']['aws']} # Configure Terraform version requirement main_dict['terraform']['required_version'] = TERRAFORM_VERSIONS['application'] # Setup the Backend depending on the deployment phase. # When first setting up StreamAlert, the Terraform statefile # is stored locally. After the first dependencies are created, # this moves to S3. if init: main_dict['terraform']['backend']['local'] = { 'path': 'terraform.tfstate'} else: main_dict['terraform']['backend']['s3'] = { 'bucket': '{}.streamalert.terraform.state'.format( config['global']['account']['prefix']), 'key': 'stream_alert_state/terraform.tfstate', 'region': config['global']['account']['region'], 'encrypt': True, 'acl': 'private', 'kms_key_id': 'alias/{}'.format(config['global']['account']['kms_key_alias'])} logging_bucket = '{}.streamalert.s3-logging'.format( config['global']['account']['prefix']) logging_bucket_lifecycle = { 'prefix': '/', 'enabled': True, 'transition': { 'days': 30, 'storage_class': 'GLACIER'}} # Configure initial S3 buckets main_dict['resource']['aws_s3_bucket'] = { 'lambda_source': generate_s3_bucket( bucket=config['lambda']['rule_processor_config']['source_bucket'], logging=logging_bucket ), 'stream_alert_secrets': generate_s3_bucket( bucket='{}.streamalert.secrets'.format(config['global']['account']['prefix']), logging=logging_bucket ), 'terraform_remote_state': generate_s3_bucket( bucket=config['global']['terraform']['tfstate_bucket'], logging=logging_bucket ), 'logging_bucket': generate_s3_bucket( bucket=logging_bucket, logging=logging_bucket, acl='log-delivery-write', lifecycle_rule=logging_bucket_lifecycle ), 'streamalerts': generate_s3_bucket( bucket='{}.streamalerts'.format(config['global']['account']['prefix']), logging=logging_bucket ) } # Setup Firehose Delivery Streams generate_firehose(config, main_dict, logging_bucket) # Configure global resources like Firehose alert delivery and alerts table main_dict['module']['globals'] = { 'source': 'modules/tf_stream_alert_globals', 'account_id': config['global']['account']['aws_account_id'], 'region': config['global']['account']['region'], 'prefix': config['global']['account']['prefix'], 'alerts_table_read_capacity': ( config['global']['infrastructure']['alerts_table']['read_capacity']), 'alerts_table_write_capacity': ( config['global']['infrastructure']['alerts_table']['write_capacity']), 'rules_table_read_capacity': ( config['global']['infrastructure']['rules_table']['read_capacity']), 'rules_table_write_capacity': ( config['global']['infrastructure']['rules_table']['write_capacity']) } # KMS Key and Alias creation main_dict['resource']['aws_kms_key']['stream_alert_secrets'] = { 'enable_key_rotation': True, 'description': 'StreamAlert secret management' } main_dict['resource']['aws_kms_alias']['stream_alert_secrets'] = { 'name': 'alias/{}'.format(config['global']['account']['kms_key_alias']), 'target_key_id': '${aws_kms_key.stream_alert_secrets.key_id}' } # Global infrastructure settings infrastructure_config = config['global'].get('infrastructure') if infrastructure_config and 'monitoring' in infrastructure_config: if infrastructure_config['monitoring'].get('create_sns_topic'): main_dict['resource']['aws_sns_topic']['stream_alert_monitoring'] = { 'name': DEFAULT_SNS_MONITORING_TOPIC } # Add any global cloudwatch alarms to the main.tf monitoring_config = config['global']['infrastructure'].get('monitoring') if not monitoring_config: return main_dict global_metrics = monitoring_config.get('metric_alarms') if not global_metrics: return main_dict sns_topic_arn = monitoring_topic_arn(config) formatted_alarms = {} # Add global metric alarms for the rule and alert processors for func in FUNC_PREFIXES: if func not in global_metrics: continue for name, settings in global_metrics[func].iteritems(): alarm_info = settings.copy() alarm_info['alarm_name'] = name alarm_info['namespace'] = 'StreamAlert' alarm_info['alarm_actions'] = [sns_topic_arn] # Terraform only allows certain characters in resource names acceptable_chars = ''.join([string.digits, string.letters, '_-']) name = filter(acceptable_chars.__contains__, name) formatted_alarms['metric_alarm_{}'.format(name)] = alarm_info if formatted_alarms: main_dict['resource']['aws_cloudwatch_metric_alarm'] = formatted_alarms return main_dict
def generate_main(config, init=False): """Generate the main.tf.json Terraform dict Args: config (CLIConfig): The loaded CLI config init (bool): Terraform is running in the init phase or not (optional) Returns: dict: main.tf.json Terraform dict """ main_dict = infinitedict() # Configure provider along with the minimum version main_dict['provider']['aws'] = { 'version': TERRAFORM_VERSIONS['provider']['aws'] } # Configure Terraform version requirement main_dict['terraform']['required_version'] = TERRAFORM_VERSIONS[ 'application'] # Setup the Backend depending on the deployment phase. # When first setting up StreamAlert, the Terraform statefile # is stored locally. After the first dependencies are created, # this moves to S3. if init: main_dict['terraform']['backend']['local'] = { 'path': 'terraform.tfstate' } else: main_dict['terraform']['backend']['s3'] = { 'bucket': config['global']['terraform']['tfstate_bucket'], 'key': config['global']['terraform']['tfstate_s3_key'], 'region': config['global']['account']['region'], 'encrypt': True, 'acl': 'private', 'kms_key_id': 'alias/{}'.format(config['global']['account']['kms_key_alias']) } logging_bucket = config['global']['s3_access_logging']['logging_bucket'] # Configure initial S3 buckets main_dict['resource']['aws_s3_bucket'] = { 'stream_alert_secrets': generate_s3_bucket(bucket='{}.streamalert.secrets'.format( config['global']['account']['prefix']), logging=logging_bucket), 'streamalerts': generate_s3_bucket(bucket='{}.streamalerts'.format( config['global']['account']['prefix']), logging=logging_bucket) } # Create bucket for S3 access logs (if applicable) if config['global']['s3_access_logging'].get('create_bucket', True): main_dict['resource']['aws_s3_bucket'][ 'logging_bucket'] = generate_s3_bucket( bucket=logging_bucket, logging=logging_bucket, acl='log-delivery-write', lifecycle_rule={ 'prefix': '/', 'enabled': True, 'transition': { 'days': 365, 'storage_class': 'GLACIER' } }, sse_algorithm= 'AES256' # SSE-KMS doesn't seem to work with access logs ) # Create bucket for Terraform state (if applicable) if config['global']['terraform'].get('create_bucket', True): main_dict['resource']['aws_s3_bucket'][ 'terraform_remote_state'] = generate_s3_bucket( bucket=config['global']['terraform']['tfstate_bucket'], logging=logging_bucket) # Setup Firehose Delivery Streams generate_firehose(config, main_dict, logging_bucket) # Configure global resources like Firehose alert delivery and alerts table global_module = { 'source': 'modules/tf_stream_alert_globals', 'account_id': config['global']['account']['aws_account_id'], 'region': config['global']['account']['region'], 'prefix': config['global']['account']['prefix'], 'kms_key_arn': '${aws_kms_key.server_side_encryption.arn}', 'alerts_table_read_capacity': (config['global']['infrastructure']['alerts_table']['read_capacity']), 'alerts_table_write_capacity': (config['global']['infrastructure']['alerts_table']['write_capacity']) } if config['global']['infrastructure']['rule_staging'].get('enabled'): global_module['enable_rule_staging'] = True global_module['rules_table_read_capacity'] = ( config['global']['infrastructure']['rule_staging']['table'] ['read_capacity']) global_module['rules_table_write_capacity'] = ( config['global']['infrastructure']['rule_staging']['table'] ['write_capacity']) main_dict['module']['globals'] = global_module # KMS Key and Alias creation main_dict['resource']['aws_kms_key']['server_side_encryption'] = { 'enable_key_rotation': True, 'description': 'StreamAlert S3 Server-Side Encryption', 'policy': json.dumps({ 'Version': '2012-10-17', 'Statement': [{ 'Sid': 'Enable IAM User Permissions', 'Effect': 'Allow', 'Principal': { 'AWS': 'arn:aws:iam::{}:root'.format( config['global']['account']['aws_account_id']) }, 'Action': 'kms:*', 'Resource': '*' }, { 'Sid': 'Allow principals in the account to use the key', 'Effect': 'Allow', 'Principal': '*', 'Action': ['kms:Decrypt', 'kms:GenerateDataKey*', 'kms:Encrypt'], 'Resource': '*', 'Condition': { 'StringEquals': { 'kms:CallerAccount': config['global']['account']['aws_account_id'] } } }] }) } main_dict['resource']['aws_kms_alias']['server_side_encryption'] = { 'name': 'alias/{}_server-side-encryption'.format( config['global']['account']['prefix']), 'target_key_id': '${aws_kms_key.server_side_encryption.key_id}' } main_dict['resource']['aws_kms_key']['stream_alert_secrets'] = { 'enable_key_rotation': True, 'description': 'StreamAlert secret management' } main_dict['resource']['aws_kms_alias']['stream_alert_secrets'] = { 'name': 'alias/{}'.format(config['global']['account']['kms_key_alias']), 'target_key_id': '${aws_kms_key.stream_alert_secrets.key_id}' } # Global infrastructure settings infrastructure_config = config['global'].get('infrastructure') if infrastructure_config and 'monitoring' in infrastructure_config: if infrastructure_config['monitoring'].get('create_sns_topic'): main_dict['resource']['aws_sns_topic'][ 'stream_alert_monitoring'] = { 'name': DEFAULT_SNS_MONITORING_TOPIC } # Add any global cloudwatch alarms to the main.tf monitoring_config = config['global']['infrastructure'].get('monitoring') if not monitoring_config: return main_dict global_metrics = monitoring_config.get('metric_alarms') if not global_metrics: return main_dict sns_topic_arn = monitoring_topic_arn(config) formatted_alarms = {} # Add global metric alarms for the rule and alert processors for func in FUNC_PREFIXES: if func not in global_metrics: continue for name, settings in global_metrics[func].iteritems(): alarm_info = settings.copy() alarm_info['alarm_name'] = name alarm_info['namespace'] = 'StreamAlert' alarm_info['alarm_actions'] = [sns_topic_arn] # Terraform only allows certain characters in resource names acceptable_chars = ''.join([string.digits, string.letters, '_-']) name = filter(acceptable_chars.__contains__, name) formatted_alarms['metric_alarm_{}'.format(name)] = alarm_info if formatted_alarms: main_dict['resource']['aws_cloudwatch_metric_alarm'] = formatted_alarms return main_dict