def test_enabled_firehose_logs_disabled(): """CLI - Terraform Common - Expected Firehose Logs - Disable""" CONFIG['global']['infrastructure']['firehose'] = { 'enabled': True, 'disabled_logs': [ 'test_log_type_json', 'test_log_type_csv', 'test_log_type_syslog' ] } firehose_logs = set(_common.enabled_firehose_logs(CONFIG)) expected_logs = { 'cloudwatch_test_match_types', # 'test_log_type_csv', 'test_log_type_csv_nested', 'test_log_type_json_nested', 'test_log_type_json_nested_with_data', # 'test_log_type_json', 'test_log_type_kv_auditd', 'test_multiple_schemas_01', 'test_multiple_schemas_02', 'test_log_type_json_2', 'test_log_type_json_nested_osquery', # 'test_log_type_syslog', 'test_cloudtrail', 'unit_test_simple_log' } assert_equal(firehose_logs, expected_logs)
def setup_mock_firehose_delivery_streams(config): """Mock Kinesis Firehose Streams for rule testing Args: config (CLIConfig): The StreamAlert config """ region = config['global']['account']['region'] for log_type in enabled_firehose_logs(config): stream_name = 'streamalert_data_{}'.format(log_type) prefix = '{}/'.format(log_type) create_delivery_stream(region, stream_name, prefix)
def test_enabled_firehose_logs(): """CLI - Terraform Common - Expected Firehose Logs """ firehose_logs = set(_common.enabled_firehose_logs(CONFIG)) expected_logs = { 'cloudwatch_test_match_types', 'test_log_type_csv', 'test_log_type_csv_nested', 'test_log_type_json_nested', 'test_log_type_json_nested_with_data', 'test_log_type_json', 'test_log_type_kv_auditd', 'test_multiple_schemas_01', 'test_multiple_schemas_02', 'test_log_type_json_2', 'test_log_type_json_nested_osquery', 'test_log_type_syslog', 'test_cloudtrail', 'unit_test_simple_log' } assert_equal(firehose_logs, expected_logs)
def setup_mock_firehose_delivery_streams(config): """Mock Kinesis Firehose Streams for rule testing Args: config (CLIConfig): The StreamAlert config """ region = config['global']['account']['region'] firehose_client = boto3.client('firehose', region_name=region) for log_type in enabled_firehose_logs(config): firehose_client.create_delivery_stream( DeliveryStreamName='streamalert_data_{}'.format(log_type), S3DestinationConfiguration={ 'RoleARN': 'arn:aws:iam::123456789012:role/firehose_delivery_role', 'BucketARN': 'arn:aws:s3:::kinesis-test', 'Prefix': '{}/'.format(log_type), 'BufferingHints': { 'SizeInMBs': 123, 'IntervalInSeconds': 124 }, 'CompressionFormat': 'Snappy', })
def generate_main(**kwargs): """Generate the main.tf.json Terraform dict Keyword Args: init (bool): If Terraform is running in the init phase or not config (CLIConfig): The loaded CLI config Returns: dict: main.tf.json Terraform dict """ init = kwargs.get('init') config = kwargs['config'] main_dict = infinitedict() # Configure provider along with the minimum version main_dict['provider']['aws'] = {'version': '~> 0.1.4'} # Configure Terraform version requirement main_dict['terraform']['required_version'] = '~> 0.10.6' # Setup the Backend dependencing on the deployment phase. # When first setting up StreamAlert, the Terraform statefile # is stored locally. After the first dependencies are created, # this moves to S3. if init: main_dict['terraform']['backend']['local'] = { 'path': 'terraform.tfstate'} else: main_dict['terraform']['backend']['s3'] = { 'bucket': '{}.streamalert.terraform.state'.format( config['global']['account']['prefix']), 'key': 'stream_alert_state/terraform.tfstate', 'region': config['global']['account']['region'], 'encrypt': True, 'acl': 'private', 'kms_key_id': 'alias/{}'.format(config['global']['account']['kms_key_alias'])} logging_bucket = '{}.streamalert.s3-logging'.format( config['global']['account']['prefix']) logging_bucket_lifecycle = { 'prefix': '/', 'enabled': True, 'transition': { 'days': 30, 'storage_class': 'GLACIER'}} # Configure initial S3 buckets main_dict['resource']['aws_s3_bucket'] = { 'lambda_source': generate_s3_bucket( bucket=config['lambda']['rule_processor_config']['source_bucket'], logging=logging_bucket ), 'stream_alert_secrets': generate_s3_bucket( bucket='{}.streamalert.secrets'.format(config['global']['account']['prefix']), logging=logging_bucket ), 'terraform_remote_state': generate_s3_bucket( bucket=config['global']['terraform']['tfstate_bucket'], logging=logging_bucket ), 'logging_bucket': generate_s3_bucket( bucket=logging_bucket, acl='log-delivery-write', logging=logging_bucket, lifecycle_rule=logging_bucket_lifecycle ), 'streamalerts': generate_s3_bucket( bucket='{}.streamalerts'.format(config['global']['account']['prefix']), logging=logging_bucket ) } # Conditionally configure Firehose if config['global']['infrastructure'].get('firehose', {}).get('enabled'): firehose_config = config['global']['infrastructure']['firehose'] firehose_s3_bucket_suffix = firehose_config.get('s3_bucket_suffix', 'streamalert.data') firehose_s3_bucket_name = '{}.{}'.format(config['global']['account']['prefix'], firehose_s3_bucket_suffix) # Add the main Firehose module main_dict['module']['kinesis_firehose'] = { 'source': 'modules/tf_stream_alert_kinesis_firehose', 'account_id': config['global']['account']['aws_account_id'], 'region': config['global']['account']['region'], 'prefix': config['global']['account']['prefix'], 'logs': enabled_firehose_logs(config), 'buffer_size': config['global']['infrastructure'] ['firehose'].get('buffer_size', 64), 'buffer_interval': config['global']['infrastructure'] ['firehose'].get('buffer_interval', 300), 'compression_format': config['global']['infrastructure'] ['firehose'].get('compression_format', 'GZIP'), 's3_logging_bucket': logging_bucket, 's3_bucket_name': firehose_s3_bucket_name } # KMS Key and Alias creation main_dict['resource']['aws_kms_key']['stream_alert_secrets'] = { 'enable_key_rotation': True, 'description': 'StreamAlert secret management' } main_dict['resource']['aws_kms_alias']['stream_alert_secrets'] = { 'name': 'alias/{}'.format(config['global']['account']['kms_key_alias']), 'target_key_id': '${aws_kms_key.stream_alert_secrets.key_id}' } # Global infrastructure settings infrastructure_config = config['global'].get('infrastructure') if infrastructure_config and 'monitoring' in infrastructure_config: if infrastructure_config['monitoring'].get('create_sns_topic'): main_dict['resource']['aws_sns_topic']['stream_alert_monitoring'] = { 'name': DEFAULT_SNS_MONITORING_TOPIC } # Add any global cloudwatch alarms to the main.tf monitoring_config = config['global']['infrastructure'].get('monitoring') if not monitoring_config: return main_dict global_metrics = monitoring_config.get('metric_alarms') if not global_metrics: return main_dict topic_name = (DEFAULT_SNS_MONITORING_TOPIC if infrastructure_config ['monitoring'].get('create_sns_topic') else infrastructure_config['monitoring'].get('sns_topic_name')) sns_topic_arn = 'arn:aws:sns:{region}:{account_id}:{topic}'.format( region=config['global']['account']['region'], account_id=config['global']['account']['aws_account_id'], topic=topic_name ) formatted_alarms = {} # Add global metric alarms for the rule and alert processors for func in FUNC_PREFIXES: if func not in global_metrics: continue for name, settings in global_metrics[func].iteritems(): alarm_info = settings.copy() alarm_info['alarm_name'] = name alarm_info['namespace'] = 'StreamAlert' alarm_info['alarm_actions'] = [sns_topic_arn] # Terraform only allows certain characters in resource names acceptable_chars = ''.join([string.digits, string.letters, '_-']) name = filter(acceptable_chars.__contains__, name) formatted_alarms['metric_alarm_{}'.format(name)] = alarm_info if formatted_alarms: main_dict['resource']['aws_cloudwatch_metric_alarm'] = formatted_alarms return main_dict
def athena_handler(options): """Handle Athena operations""" athena_client = StreamAlertAthenaClient( CONFIG, results_key_prefix='stream_alert_cli') if options.subcommand == 'init': CONFIG.generate_athena() elif options.subcommand == 'enable': CONFIG.set_athena_lambda_enable() elif options.subcommand == 'create-db': if athena_client.check_database_exists(): LOGGER_CLI.info( 'The \'streamalert\' database already exists, nothing to do') return create_db_success, create_db_result = athena_client.run_athena_query( query='CREATE DATABASE streamalert') if create_db_success and create_db_result['ResultSet'].get('Rows'): LOGGER_CLI.info('streamalert database successfully created!') LOGGER_CLI.info('results: %s', create_db_result['ResultSet']['Rows']) elif options.subcommand == 'create-table': if not options.bucket: LOGGER_CLI.error('Missing command line argument --bucket') return if not options.refresh_type: LOGGER_CLI.error('Missing command line argument --refresh_type') return if options.type == 'data': if not options.table_name: LOGGER_CLI.error('Missing command line argument --table_name') return if options.table_name not in enabled_firehose_logs(CONFIG): LOGGER_CLI.error( 'Table name %s missing from configuration or ' 'is not enabled.', options.table_name) return if athena_client.check_table_exists(options.table_name): LOGGER_CLI.info('The \'%s\' table already exists.', options.table_name) return log_info = CONFIG['logs'][options.table_name.replace('_', ':', 1)] schema = dict(log_info['schema']) schema_statement = '' sanitized_schema = StreamAlert.sanitize_keys(schema) athena_schema = {} schema_type_mapping = { 'string': 'string', 'integer': 'int', 'boolean': 'boolean', 'float': 'decimal', dict: 'map<string, string>', list: 'array<string>' } def add_to_athena_schema(schema, root_key=''): """Helper function to add sanitized schemas to the Athena table schema""" # Setup the root_key dict if root_key and not athena_schema.get(root_key): athena_schema[root_key] = {} for key_name, key_type in schema.iteritems(): # When using special characters in the beginning or end # of a column name, they have to be wrapped in backticks key_name = '`{}`'.format(key_name) special_key = None # Transform the {} or [] into hashable types if key_type == {}: special_key = dict elif key_type == []: special_key = list # Cast nested dict as a string for now # TODO(jacknagz): support recursive schemas elif isinstance(key_type, dict): special_key = 'string' # Account for envelope keys if root_key: if special_key is not None: athena_schema[root_key][ key_name] = schema_type_mapping[special_key] else: athena_schema[root_key][ key_name] = schema_type_mapping[key_type] else: if special_key is not None: athena_schema[key_name] = schema_type_mapping[ special_key] else: athena_schema[key_name] = schema_type_mapping[ key_type] add_to_athena_schema(sanitized_schema) # Support envelope keys configuration_options = log_info.get('configuration') if configuration_options: envelope_keys = configuration_options.get('envelope_keys') if envelope_keys: sanitized_envelope_keys = StreamAlert.sanitize_keys( envelope_keys) # Note: this key is wrapped in backticks to be Hive compliant add_to_athena_schema(sanitized_envelope_keys, '`streamalert:envelope_keys`') for key_name, key_type in athena_schema.iteritems(): # Account for nested structs if isinstance(key_type, dict): struct_schema = ''.join([ '{0}:{1},'.format(sub_key, sub_type) for sub_key, sub_type in key_type.iteritems() ]) nested_schema_statement = '{0} struct<{1}>, '.format( key_name, # Use the minus index to remove the last comma struct_schema[:-1]) schema_statement += nested_schema_statement else: schema_statement += '{0} {1},'.format(key_name, key_type) query = ( 'CREATE EXTERNAL TABLE {table_name} ({schema}) ' 'PARTITIONED BY (dt string) ' 'ROW FORMAT SERDE \'org.openx.data.jsonserde.JsonSerDe\' ' 'LOCATION \'s3://{bucket}/{table_name}/\''.format( table_name=options.table_name, # Use the minus index to remove the last comma schema=schema_statement[:-1], bucket=options.bucket)) elif options.type == 'alerts': if athena_client.check_table_exists(options.type): LOGGER_CLI.info('The \'alerts\' table already exists.') return query = ('CREATE EXTERNAL TABLE alerts (' 'log_source string,' 'log_type string,' 'outputs array<string>,' 'record string,' 'rule_description string,' 'rule_name string,' 'source_entity string,' 'source_service string)' 'PARTITIONED BY (dt string)' 'ROW FORMAT SERDE \'org.openx.data.jsonserde.JsonSerDe\'' 'LOCATION \'s3://{bucket}/alerts/\''.format( bucket=options.bucket)) if query: create_table_success, _ = athena_client.run_athena_query( query=query, database='streamalert') if create_table_success: CONFIG['lambda']['athena_partition_refresh_config'] \ ['refresh_type'][options.refresh_type][options.bucket] = options.type CONFIG.write() table_name = options.type if options.type == 'alerts' else options.table_name LOGGER_CLI.info('The %s table was successfully created!', table_name)
def athena_handler(options): """Handle Athena operations""" athena_client = StreamAlertAthenaClient( CONFIG, results_key_prefix='stream_alert_cli') if options.subcommand == 'init': CONFIG.generate_athena() elif options.subcommand == 'enable': CONFIG.set_athena_lambda_enable() elif options.subcommand == 'create-db': if athena_client.check_database_exists(): LOGGER_CLI.info( 'The \'streamalert\' database already exists, nothing to do') return create_db_success, create_db_result = athena_client.run_athena_query( query='CREATE DATABASE streamalert') if create_db_success and create_db_result['ResultSet'].get('Rows'): LOGGER_CLI.info('streamalert database successfully created!') LOGGER_CLI.info('results: %s', create_db_result['ResultSet']['Rows']) elif options.subcommand == 'create-table': if not options.bucket: LOGGER_CLI.error('Missing command line argument --bucket') return if not options.refresh_type: LOGGER_CLI.error('Missing command line argument --refresh_type') return if options.type == 'data': if not options.table_name: LOGGER_CLI.error('Missing command line argument --table_name') return if options.table_name not in enabled_firehose_logs(CONFIG): LOGGER_CLI.error( 'Table name %s missing from configuration or ' 'is not enabled.', options.table_name) return if athena_client.check_table_exists(options.table_name): LOGGER_CLI.info('The \'%s\' table already exists.', options.table_name) return schema = CONFIG['logs'][options.table_name.replace('_', ':')]['schema'] sanitized_schema = StreamAlert.sanitize_keys(schema) athena_schema = {} schema_type_mapping = { 'string': 'string', 'integer': 'int', 'boolean': 'boolean', 'float': 'decimal', dict: 'map<string, string>', list: 'array<string>' } for key_name, key_type in sanitized_schema.iteritems(): # Transform the {} or [] into hashable types if key_type == {}: key_type = dict elif key_type == []: key_type = list athena_schema[key_name] = schema_type_mapping[key_type] schema_statement = ''.join([ '{0} {1},'.format(key_name, key_type) for key_name, key_type in athena_schema.iteritems() ])[:-1] query = ('CREATE EXTERNAL TABLE {table_name} ({schema})' 'PARTITIONED BY (dt string)' 'ROW FORMAT SERDE \'org.openx.data.jsonserde.JsonSerDe\'' 'LOCATION \'s3://{bucket}/{table_name}/\''.format( table_name=options.table_name, schema=schema_statement, bucket=options.bucket)) elif options.type == 'alerts': if athena_client.check_table_exists(options.type): LOGGER_CLI.info('The \'alerts\' table already exists.') return query = ('CREATE EXTERNAL TABLE alerts (' 'log_source string,' 'log_type string,' 'outputs array<string>,' 'record string,' 'rule_description string,' 'rule_name string,' 'source_entity string,' 'source_service string)' 'PARTITIONED BY (dt string)' 'ROW FORMAT SERDE \'org.openx.data.jsonserde.JsonSerDe\'' 'LOCATION \'s3://{bucket}/alerts/\''.format( bucket=options.bucket)) if query: create_table_success, _ = athena_client.run_athena_query( query=query, database='streamalert') if create_table_success: CONFIG['lambda']['athena_partition_refresh_config'] \ ['refresh_type'][options.refresh_type][options.bucket] = options.type CONFIG.write() LOGGER_CLI.info('The %s table was successfully created!', options.type)
def create_table(athena_client, options, config): """Create a 'streamalert' Athena table Args: athena_client (boto3.client): Instantiated CLI AthenaClient options (namedtuple): The parsed args passed from the CLI config (CLIConfig): Loaded StreamAlert CLI """ if not options.bucket: LOGGER_CLI.error('Missing command line argument --bucket') return if not options.refresh_type: LOGGER_CLI.error('Missing command line argument --refresh_type') return if options.type == 'data': if not options.table_name: LOGGER_CLI.error('Missing command line argument --table_name') return if options.table_name not in terraform_cli_helpers.enabled_firehose_logs( config): LOGGER_CLI.error( 'Table name %s missing from configuration or ' 'is not enabled.', options.table_name) return if athena_client.check_table_exists(options.table_name): LOGGER_CLI.info('The \'%s\' table already exists.', options.table_name) return log_info = config['logs'][options.table_name.replace('_', ':', 1)] schema = dict(log_info['schema']) schema_statement = '' sanitized_schema = StreamAlert.sanitize_keys(schema) athena_schema = {} _add_to_athena_schema(sanitized_schema, athena_schema) # Support envelope keys configuration_options = log_info.get('configuration') if configuration_options: envelope_keys = configuration_options.get('envelope_keys') if envelope_keys: sanitized_envelope_key_schema = StreamAlert.sanitize_keys( envelope_keys) # Note: this key is wrapped in backticks to be Hive compliant _add_to_athena_schema(sanitized_envelope_key_schema, athena_schema, '`streamalert:envelope_keys`') for key_name, key_type in athena_schema.iteritems(): # Account for nested structs if isinstance(key_type, dict): struct_schema = ''.join([ '{0}:{1},'.format(sub_key, sub_type) for sub_key, sub_type in key_type.iteritems() ]) nested_schema_statement = '{0} struct<{1}>, '.format( key_name, # Use the minus index to remove the last comma struct_schema[:-1]) schema_statement += nested_schema_statement else: schema_statement += '{0} {1},'.format(key_name, key_type) query = ( 'CREATE EXTERNAL TABLE {table_name} ({schema}) ' 'PARTITIONED BY (dt string) ' 'ROW FORMAT SERDE \'org.openx.data.jsonserde.JsonSerDe\' ' 'WITH SERDEPROPERTIES ( \'ignore.malformed.json\' = \'true\') ' 'LOCATION \'s3://{bucket}/{table_name}/\''.format( table_name=options.table_name, # Use the minus index to remove the last comma schema=schema_statement[:-1], bucket=options.bucket)) elif options.type == 'alerts': if athena_client.check_table_exists(options.type): LOGGER_CLI.info('The \'alerts\' table already exists.') return query = ('CREATE EXTERNAL TABLE alerts (' 'log_source string,' 'log_type string,' 'outputs array<string>,' 'record string,' 'rule_description string,' 'rule_name string,' 'source_entity string,' 'source_service string)' 'PARTITIONED BY (dt string)' 'ROW FORMAT SERDE \'org.openx.data.jsonserde.JsonSerDe\'' 'LOCATION \'s3://{bucket}/alerts/\''.format( bucket=options.bucket)) if query: create_table_success, _ = athena_client.run_athena_query( query=query, database='streamalert') if create_table_success: # Update the CLI config config['lambda']['athena_partition_refresh_config'] \ ['refresh_type'][options.refresh_type][options.bucket] = options.type config.write() table_name = options.type if options.type == 'alerts' else options.table_name LOGGER_CLI.info('The %s table was successfully created!', table_name)