def test_apply_envelope_none(self):
"""ParserBase - Apply Envelope, None"""
record = {'key': 'value'}
ParserBase._apply_envelope(record, {})
assert_equal(record, {'key': 'value'})
def test_add_optional_keys(self):
"""ParserBase - Add Optional Keys"""
schema = {'key': 'string', 'optional_key': 'integer'}
optionals = {'optional_key'}
record = {'key': 'data'}
ParserBase._add_optional_keys(record, schema, optionals)
assert_equal(record, {'key': 'data', 'optional_key': 0})
def test_extract_envelope_none(self):
"""ParserBase - Extract Envelope, None"""
record = {'key': 'value'}
parser = ParserBase(None)
envelope = parser._extract_envelope(record)
assert_equal(envelope, None)
def test_extract_envelope(self):
"""ParserBase - Extract Envelope"""
options = {'configuration': {'envelope_keys': ['env_key_01']}}
record = {'env_key_01': 'foo', 'non_env_key': 'bar'}
parser = ParserBase(options)
envelope = parser._extract_envelope(record)
assert_equal(envelope, {'env_key_01': 'foo'})
def test_json_path_records(self):
"""ParserBase - JSON Path Records"""
expected_records = [{'foo': 'bar'}, {'bar': 'baz'}]
record = {'key': [{'value': expected_records}]}
options = {'configuration': {'json_path': 'key[].value[]'}}
parser = ParserBase(options)
records = parser._json_path_records(record)
assert_equal(records, expected_records)
def test_apply_envelope(self):
"""ParserBase - Apply Envelope"""
record = {'key': 'value'}
envelope = {'env_key_01': 'value'}
ParserBase._apply_envelope(record, envelope)
expected_record = {
'key': 'value',
'streamalert:envelope_keys': {
'env_key_01': 'value'
}
}
assert_equal(record, expected_record)
def test_matches_log_patterns_str(self, log_mock):
"""ParserBase - Matches Log Patterns, Str"""
record = {'key': 'matching pattern'}
patterns = {'key': '*pattern'}
assert_equal(ParserBase._matches_log_patterns(record, patterns), True)
log_mock.assert_any_call('Transforming flat pattern \'%s\' into list',
'*pattern')
def test_validate_schema_invalid(self):
"""ParserBase - Validate Schema, Invalid"""
options = {
'schema': {
'timestamp': 'string',
'host': 'string',
'application': 'string',
'message': 'string'
},
'configuration': {
'optional_top_level_keys': ['date']
}
}
parser = ParserBase(options)
assert_equal(parser._validate_schema(), False)
def test_key_check_nested_invalid(self, log_mock):
"""ParserBase - Key Check, Invalid Nested"""
schema = {'key': 'string', 'nested': {'key_02': 'integer'}}
record = {'key': 'value', 'nested': {'key_01': 'value'}}
assert_equal(ParserBase._key_check(record, schema), False)
log_mock.assert_any_call('Expected keys not found in record: %s',
'key_02')
def test_key_check_mismatch(self, log_mock):
"""ParserBase - Key Check, Mismatch"""
schema = {'key': 'string'}
record = {'not_key': 'test'}
assert_equal(ParserBase._key_check(record, schema), False)
log_mock.assert_called_with('Found keys not expected in record: %s',
'not_key')
def test_matches_log_patterns_invalid_key(self, log_mock):
"""ParserBase - Matches Log Patterns, Invalid Key"""
record = {'key': 'value'}
patterns = {'not_key': '*pattern'}
assert_equal(ParserBase._matches_log_patterns(record, patterns), False)
log_mock.assert_any_call(
'Declared log pattern key [%s] does exist in record:\n%s',
'not_key', record)
def test_convert_type_none(self, log_mock):
"""ParserBase - Convert Type, NoneType Value"""
schema = {'key': 'string'}
record = {'key': None}
assert_equal(ParserBase._convert_type(record, schema), True)
assert_equal(record, {'key': None})
log_mock.assert_called_with(
'Skipping NoneType value in record for key: %s', 'key')
def test_convert_type_optionals(self, log_mock):
"""ParserBase - Convert Type, With Optionals Missing"""
schema = {'required_key': 'string', 'optional_key': 'string'}
optionals = {'optional_key'}
record = {'required_key': 'required_value'}
assert_equal(ParserBase._convert_type(record, schema, optionals), True)
assert_equal(record, {'required_key': 'required_value'})
log_mock.assert_called_with(
'Skipping optional key not found in record: %s', 'optional_key')
def test_convert_type_unsupported_type(self, log_mock):
"""ParserBase - Convert Type, Unsupported Type"""
schema = {'key': 'foobar'}
record = {'key': 'foobarbaz'}
assert_equal(ParserBase._convert_type(record, schema), False)
assert_equal(record, {'key': 'foobarbaz'})
log_mock.assert_called_with(
'Unsupported value type in schema for key \'%s\': %s', 'key',
'foobar')
def test_key_check_bad_envelope_subset(self, log_mock):
"""ParserBase - Key Check, Invalid Envelope Subset"""
envelope_schema = {'env_key_01': 'string'}
record = {'env_key_02': 'test'}
assert_equal(
ParserBase._key_check(record, envelope_schema, is_envelope=True),
False)
log_mock.assert_called_with('Missing keys in record envelope: %s',
{'env_key_01'})
def test_key_check_debug(self, log_mock):
"""ParserBase - Key Check, Debug Failure"""
schema = {'key': 'string', 'nested': {'key_02': 'integer'}}
record = {'key': 'value', 'nested': {'key_01': 100}}
with patch.object(parsers, 'LOGGER_DEBUG_ENABLED', True):
assert_equal(ParserBase._key_check(record, schema), False)
log_mock.assert_called_with(
'Nested key check failure. Schema:\n%s\nRecord:\n%s',
json.dumps(schema, indent=2, sort_keys=True),
json.dumps(record, indent=2, sort_keys=True))
def test_parse(self):
"""ParserBase - Parse, Invalid Schema"""
options = {
'schema': {
'key': 'string'
},
'configuration': {
'optional_top_level_keys': ['non_key']
}
}
assert_equal(ParserBase(options).parse(None), False)
def test_validate_schema_all(self):
"""ParserBase - Validate Schema, All Options"""
options = {
'schema': {
'timestamp': 'string',
'host': 'string',
'application': 'string',
'message': 'string'
},
'configuration': {
'envelope_keys': {
'env_key_01': 'string',
'env_key_02': 'string'
},
'optional_envelope_keys': ['env_key_01'],
'optional_top_level_keys': ['host']
}
}
parser = ParserBase(options)
assert_equal(parser._validate_schema(), True)
def setup(self):
"""Setup before each method"""
# pylint: disable=abstract-class-instantiated,attribute-defined-outside-init
options = {
'schema': {
'timestamp': 'string',
'host': 'string'
},
'configuration': {
'optional_top_level_keys': ['host'],
'log_patterns': {
'host': ['foo*']
},
'json_path': 'logEvents[].message',
'envelope_keys': {
'env_key_01': 'string'
},
'optional_envelope_keys': ['env_key_01']
}
}
self._parser = ParserBase(options)
def _apply_defaults(self, test_event):
"""Apply default values to the given test event
Args:
test_event (dict): The loaded test event
"""
if 'override_record' not in test_event:
return
event_log = self._config['logs'].get(test_event['log'])
configuration = event_log.get('configuration', {})
schema = configuration.get('envelope_keys', event_log['schema'])
# Add apply default values based on the declared schema
default_test_event = {
key: ParserBase.default_optional_values(value)
for key, value in schema.iteritems()
}
# Overwrite the fields included in the 'override_record' field,
# and update the test event with a full 'data' key
default_test_event.update(test_event['override_record'])
test_event['data'] = default_test_event
def test_valid_property_false(self):
"""ParserBase - Valid Property"""
parser = ParserBase(None)
assert_equal(parser.valid, False)
def test_default_optional_values_float(self):
"""ParserBase - Default Optional Type, Float"""
assert_equal(ParserBase.default_optional_values('float'), 0.0)
def test_default_optional_values_boolean(self):
"""ParserBase - Default Optional Type, Boolean"""
assert_equal(ParserBase.default_optional_values('boolean'), False)
def test_default_optional_values_str(self):
"""ParserBase - Default Optional Type, Str"""
assert_equal(ParserBase.default_optional_values('string'), '')
def test_default_optional_values_int(self):
"""ParserBase - Default Optional Type, Int"""
assert_equal(ParserBase.default_optional_values('integer'), 0)
def test_log_schema_type_property(self):
"""ParserBase - Log Schema Type Property"""
log_type = 'foobar'
parser = ParserBase(None, log_type)
assert_equal(parser.log_schema_type, log_type)
def test_valid_property(self):
"""ParserBase - Valid Property"""
parser = ParserBase(None)
parser._valid_parses.append('foobar')
assert_equal(parser.valid, True)
def test_parses_property(self):
"""ParserBase - Parses Property"""
item = 'foobar'
parser = ParserBase(None)
parser._valid_parses.append(item)
assert_equal(parser.parsed_records, [item])
def test_invalid_parses_property(self):
"""ParserBase - Invalid Parses Property"""
item = 'foobar'
parser = ParserBase(None)
parser._invalid_parses.append(item)
assert_equal(parser.invalid_parses, [item])
def test_add_parse_result_invalid(self):
"""ParserBase - Add Parse Result, Valid"""
parser = ParserBase(None)
record = {'key': 'value'}
parser._add_parse_result(record, False, None)
assert_equal(parser._invalid_parses, [record])