def test_firehose_enabled_log(self): """CLI - Terraform Generate Kinesis Firehose, Enabled Log""" cluster_dict = common.infinitedict() # Add an enabled log, with no alarm configuration (aka: alarms disabled) self.config['global']['infrastructure']['firehose']['enabled_logs'] = { 'json:embedded': {} } firehose.generate_firehose(self._logging_bucket_name, cluster_dict, self.config) expected_result = { 'module': { 'kinesis_firehose_setup': self._default_firehose_config(), 'kinesis_firehose_json_embedded': { 'source': './modules/tf_kinesis_firehose_delivery_stream', 'buffer_size': 128, 'buffer_interval': 900, 'file_format': 'parquet', 'stream_name': 'unit_test_streamalert_json_embedded', 'role_arn': '${module.kinesis_firehose_setup.firehose_role_arn}', 's3_bucket_name': 'unit-test-streamalert-data', 'kms_key_arn': '${aws_kms_key.server_side_encryption.arn}', 'glue_catalog_db_name': 'unit-test_streamalert', 'glue_catalog_table_name': 'json_embedded', 'schema': self._get_expected_schema() } } } assert_equal(cluster_dict, expected_result)
def test_firehose_enabled_log_alarm_custom(self): """CLI - Terraform Generate Kinesis Firehose, Enabled Alarm - Custom Settings""" cluster_dict = common.infinitedict() # Add an enabled log, with alarms on with custom settings self.config['global']['infrastructure']['firehose']['enabled_logs'] = { 'json:embedded': { 'enable_alarm': True, 'evaluation_periods': 10, 'period_seconds': 3600, 'log_min_count_threshold': 100000 } } firehose.generate_firehose(self._logging_bucket_name, cluster_dict, self.config) expected_result = { 'module': { 'kinesis_firehose_setup': self._default_firehose_config(), 'kinesis_firehose_json_embedded': { 'source': './modules/tf_kinesis_firehose_delivery_stream', 'buffer_size': 128, 'buffer_interval': 900, 'file_format': 'parquet', 'stream_name': 'unit_test_streamalert_json_embedded', 'role_arn': '${module.kinesis_firehose_setup.firehose_role_arn}', 's3_bucket_name': 'unit-test-streamalert-data', 'kms_key_arn': '${aws_kms_key.server_side_encryption.arn}', 'enable_alarm': True, 'evaluation_periods': 10, 'period_seconds': 3600, 'alarm_threshold': 100000, 'alarm_actions': [ 'arn:aws:sns:us-west-1:12345678910:unit-test_streamalert_monitoring' ], 'glue_catalog_db_name': 'unit-test_streamalert', 'glue_catalog_table_name': 'json_embedded', 'schema': self._get_expected_schema() } } } assert_equal(cluster_dict, expected_result)
def test_firehose_defaults(self): """CLI - Terraform Generate Kinesis Firehose, Defaults""" cluster_dict = common.infinitedict() firehose.generate_firehose(self._logging_bucket_name, cluster_dict, self.config) expected_result = { 'module': { 'kinesis_firehose_setup': self._default_firehose_config(), } } assert_equal(cluster_dict, expected_result)
def test_firehose_enabled_log_alarm_defaults(self): """CLI - Terraform Generate Kinesis Firehose, Enabled Alarm - Default Settings""" cluster_dict = common.infinitedict() # Add an enabled log, with alarms on (will use terraform default settings) self.config['global']['infrastructure']['firehose']['enabled_logs'] = { 'json:embedded': { 'enable_alarm': True } } firehose.generate_firehose(self._logging_bucket_name, cluster_dict, self.config) expected_result = { 'module': { 'kinesis_firehose_setup': self._default_firehose_config(), 'kinesis_firehose_json_embedded': { 'source': './modules/tf_kinesis_firehose_delivery_stream', 'buffer_size': 128, 'buffer_interval': 900, 'compression_format': 'GZIP', 'use_prefix': True, 'prefix': 'unit-test', 'log_name': 'json_embedded', 'role_arn': '${module.kinesis_firehose_setup.firehose_role_arn}', 's3_bucket_name': 'unit-test-streamalert-data', 'kms_key_arn': '${aws_kms_key.server_side_encryption.arn}', 'enable_alarm': True, 'alarm_actions': [ 'arn:aws:sns:us-west-1:12345678910:unit-test_streamalert_monitoring' ] } } } assert_equal(cluster_dict, expected_result)
def generate_main(config, init=False): """Generate the main.tf.json Terraform dict Args: config (CLIConfig): The loaded CLI config init (bool): Terraform is running in the init phase or not (optional) Returns: dict: main.tf.json Terraform dict """ write_vars(config, region=config['global']['account']['region']) main_dict = infinitedict() logging_bucket, create_logging_bucket = s3_access_logging_bucket(config) state_lock_table_name = '{}_streamalert_terraform_state_lock'.format( config['global']['account']['prefix'] ) # Setup the Backend depending on the deployment phase. # When first setting up StreamAlert, the Terraform statefile # is stored locally. After the first dependencies are created, # this moves to S3. if init: main_dict['terraform']['backend']['local'] = { 'path': 'terraform.tfstate', } else: terraform_bucket_name, _ = terraform_state_bucket(config) main_dict['terraform']['backend']['s3'] = { 'bucket': terraform_bucket_name, 'key': config['global'].get('terraform', {}).get( 'state_key_name', 'streamalert_state/terraform.tfstate' ), 'region': config['global']['account']['region'], 'encrypt': True, 'dynamodb_table': state_lock_table_name, 'acl': 'private', 'kms_key_id': 'alias/{}'.format( config['global']['account'].get( 'kms_key_alias', '{}_streamalert_secrets'.format(config['global']['account']['prefix']) ) ), } # Configure initial S3 buckets main_dict['resource']['aws_s3_bucket'] = { 'streamalerts': generate_s3_bucket( bucket=firehose_alerts_bucket(config), logging=logging_bucket ) } # Configure remote state locking table main_dict['resource']['aws_dynamodb_table'] = { 'terraform_remote_state_lock': { 'name': state_lock_table_name, 'billing_mode': 'PAY_PER_REQUEST', 'hash_key': 'LockID', 'attribute': { 'name': 'LockID', 'type': 'S' }, 'tags': { 'Name': 'StreamAlert' } } } # Create bucket for S3 access logs (if applicable) if create_logging_bucket: main_dict['resource']['aws_s3_bucket']['logging_bucket'] = generate_s3_bucket( bucket=logging_bucket, logging=logging_bucket, acl='log-delivery-write', lifecycle_rule={ 'prefix': '/', 'enabled': True, 'transition': { 'days': 365, 'storage_class': 'GLACIER' } }, sse_algorithm='AES256' # SSE-KMS doesn't seem to work with access logs ) terraform_bucket_name, create_state_bucket = terraform_state_bucket(config) # Create bucket for Terraform state (if applicable) if create_state_bucket: main_dict['resource']['aws_s3_bucket']['terraform_remote_state'] = generate_s3_bucket( bucket=terraform_bucket_name, logging=logging_bucket ) # Setup Firehose Delivery Streams generate_firehose(logging_bucket, main_dict, config) # Configure global resources like Firehose alert delivery and alerts table main_dict['module']['globals'] = _generate_global_module(config) # KMS Key and Alias creation main_dict['resource']['aws_kms_key']['server_side_encryption'] = { 'enable_key_rotation': True, 'description': 'StreamAlert S3 Server-Side Encryption', 'policy': json.dumps({ 'Version': '2012-10-17', 'Statement': [ { 'Sid': 'Enable IAM User Permissions', 'Effect': 'Allow', 'Principal': { 'AWS': 'arn:aws:iam::{}:root'.format( config['global']['account']['aws_account_id'] ) }, 'Action': 'kms:*', 'Resource': '*' }, { 'Sid': 'Allow principals in the account to use the key', 'Effect': 'Allow', 'Principal': '*', 'Action': ['kms:Decrypt', 'kms:GenerateDataKey*', 'kms:Encrypt'], 'Resource': '*', 'Condition': { 'StringEquals': { 'kms:CallerAccount': config['global']['account']['aws_account_id'] } } } ] }) } main_dict['resource']['aws_kms_alias']['server_side_encryption'] = { 'name': 'alias/{}_server-side-encryption'.format(config['global']['account']['prefix']), 'target_key_id': '${aws_kms_key.server_side_encryption.key_id}' } main_dict['resource']['aws_kms_key']['streamalert_secrets'] = { 'enable_key_rotation': True, 'description': 'StreamAlert secret management' } main_dict['resource']['aws_kms_alias']['streamalert_secrets'] = { 'name': 'alias/{}'.format( config['global']['account'].get( 'kms_key_alias', '{}_streamalert_secrets'.format(config['global']['account']['prefix']) ) ), 'target_key_id': '${aws_kms_key.streamalert_secrets.key_id}' } # Global infrastructure settings topic_name, create_topic = monitoring_topic_name(config) if create_topic: main_dict['resource']['aws_sns_topic']['monitoring'] = { 'name': topic_name } return main_dict