def test_json_path_records(self):
"""ParserBase - JSON Path Records"""
expected_records = [
{
'foo': 'bar'
},
{
'bar': 'baz'
}
]
record = {
'key': [
{
'value': expected_records
}
]
}
options = {
'configuration': {
'json_path': 'key[].value[]'
}
}
parser = ParserBase(options)
records = parser._json_path_records(record)
assert_equal(records, expected_records)
def test_validate_schema_all(self):
"""ParserBase - Validate Schema, All Options"""
options = {
'schema': {
'timestamp': 'string',
'host': 'string',
'application': 'string',
'message': 'string'
},
'configuration': {
'envelope_keys': {
'env_key_01': 'string',
'env_key_02': 'string'
},
'optional_envelope_keys': [
'env_key_01'
],
'optional_top_level_keys': [
'host'
]
}
}
parser = ParserBase(options)
assert_equal(parser._validate_schema(), True)
def test_add_parse_result_invalid(self):
"""ParserBase - Add Parse Result, Valid"""
parser = ParserBase(None)
record = {
'key': 'value'
}
parser._add_parse_result(record, False, None)
assert_equal(parser._invalid_parses, [record])
def test_extract_envelope_none(self):
"""ParserBase - Extract Envelope, None"""
record = {
'key': 'value'
}
parser = ParserBase(None)
envelope = parser._extract_envelope(record)
assert_equal(envelope, None)
def test_apply_envelope_none(self):
"""ParserBase - Apply Envelope, None"""
record = {
'key': 'value'
}
ParserBase._apply_envelope(record, {})
assert_equal(record, {'key': 'value'})
def test_add_optional_keys_none(self):
"""ParserBase - Add Optional Keys, None"""
schema = {
'key': 'string'
}
record = {
'key': 'data'
}
ParserBase._add_optional_keys(record, schema, None)
assert_equal(record, {'key': 'data'})
def test_add_optional_keys(self):
"""ParserBase - Add Optional Keys"""
schema = {
'key': 'string',
'optional_key': 'integer'
}
optionals = {'optional_key'}
record = {
'key': 'data'
}
ParserBase._add_optional_keys(record, schema, optionals)
assert_equal(record, {'key': 'data', 'optional_key': 0})
def setup(self):
"""Setup before each method"""
# pylint: disable=abstract-class-instantiated,attribute-defined-outside-init
options = {
'schema': {
'timestamp': 'string',
'host': 'string'
},
'configuration': {
'optional_top_level_keys': [
'host'
],
'log_patterns': {
'host': [
'foo*'
]
},
'json_path': 'logEvents[].message',
'envelope_keys': {
'env_key_01': 'string'
},
'optional_envelope_keys': [
'env_key_01'
]
}
}
self._parser = ParserBase(options)
def test_extract_envelope(self):
"""ParserBase - Extract Envelope"""
options = {
'configuration': {
'envelope_keys': [
'env_key_01'
]
}
}
record = {
'env_key_01': 'foo',
'non_env_key': 'bar'
}
parser = ParserBase(options)
envelope = parser._extract_envelope(record)
assert_equal(envelope, {'env_key_01': 'foo'})
def test_validate_schema_invalid(self):
"""ParserBase - Validate Schema, Invalid"""
options = {
'schema': {
'timestamp': 'string',
'host': 'string',
'application': 'string',
'message': 'string'
},
'configuration': {
'optional_top_level_keys': [
'date'
]
}
}
parser = ParserBase(options)
assert_equal(parser._validate_schema(), False)
def test_convert_type_list(self):
"""ParserBase - Convert Type, Invalid List"""
schema = {
'key': []
}
record = {
'key': 'not a list'
}
assert_equal(ParserBase._convert_type(record, schema), False)
assert_equal(record, {'key': 'not a list'})
def test_convert_type_int_invalid(self):
"""ParserBase - Convert Type, Invalid Int"""
schema = {
'key': 'integer'
}
record = {
'key': 'not an int'
}
assert_equal(ParserBase._convert_type(record, schema), False)
assert_equal(record, {'key': 'not an int'})
def test_convert_type_float_invalid(self):
"""ParserBase - Convert Type, Invalid Float"""
schema = {
'key': 'float'
}
record = {
'key': 'not a float'
}
assert_equal(ParserBase._convert_type(record, schema), False)
assert_equal(record, {'key': 'not a float'})
def test_convert_type_bool(self):
"""ParserBase - Convert Type, Boolean"""
schema = {
'key': 'boolean'
}
record = {
'key': 'True'
}
assert_equal(ParserBase._convert_type(record, schema), True)
assert_equal(record, {'key': True})
def test_matches_log_patterns_str(self, log_mock):
"""ParserBase - Matches Log Patterns, Str"""
record = {
'key': 'matching pattern'
}
patterns = {
'key': '*pattern'
}
assert_equal(ParserBase._matches_log_patterns(record, patterns), True)
log_mock.assert_any_call('Transforming flat pattern \'%s\' into list', '*pattern')
def test_convert_type_float(self):
"""ParserBase - Convert Type, Float"""
schema = {
'key': 'float'
}
record = {
'key': '0.9'
}
assert_equal(ParserBase._convert_type(record, schema), True)
assert_equal(record, {'key': 0.9})
def test_convert_type_int(self):
"""ParserBase - Convert Type, Int"""
schema = {
'key': 'integer'
}
record = {
'key': '100'
}
assert_equal(ParserBase._convert_type(record, schema), True)
assert_equal(record, {'key': 100})
def test_convert_type_unicode_str(self):
"""ParserBase - Convert Type, Unicode Str"""
schema = {
'key': 'string'
}
record = {
'key': '\ue82a'
}
assert_equal(ParserBase._convert_type(record, schema), True)
assert_equal(record, {'key': '\ue82a'})
def test_convert_type_str(self):
"""ParserBase - Convert Type, Str"""
schema = {
'key': 'string'
}
record = {
'key': 100
}
assert_equal(ParserBase._convert_type(record, schema), True)
assert_equal(record, {'key': '100'})
def test_key_check_bad_envelope_subset(self, log_mock):
"""ParserBase - Key Check, Invalid Envelope Subset"""
envelope_schema = {
'env_key_01': 'string'
}
record = {
'env_key_02': 'test'
}
assert_equal(ParserBase._key_check(record, envelope_schema, is_envelope=True), False)
log_mock.assert_called_with('Missing keys in record envelope: %s', {'env_key_01'})
def test_key_check_mismatch(self, log_mock):
"""ParserBase - Key Check, Mismatch"""
schema = {
'key': 'string'
}
record = {
'not_key': 'test'
}
assert_equal(ParserBase._key_check(record, schema), False)
log_mock.assert_called_with('Found keys not expected in record: %s', 'not_key')
def test_apply_envelope(self):
"""ParserBase - Apply Envelope"""
record = {
'key': 'value'
}
envelope = {
'env_key_01': 'value'
}
ParserBase._apply_envelope(record, envelope)
expected_record = {
'key': 'value',
'streamalert:envelope_keys': {
'env_key_01': 'value'
}
}
assert_equal(record, expected_record)
def test_matches_log_patterns_invalid_key(self, log_mock):
"""ParserBase - Matches Log Patterns, Invalid Key"""
record = {
'key': 'value'
}
patterns = {
'not_key': '*pattern'
}
assert_equal(ParserBase._matches_log_patterns(record, patterns), False)
log_mock.assert_any_call(
'Declared log pattern key [%s] does exist in record:\n%s', 'not_key', record)
def test_matches_log_patterns_list(self):
"""ParserBase - Matches Log Patterns, List"""
record = {
'key': 'matching pattern'
}
patterns = {
'key': [
'*pattern'
]
}
assert_equal(ParserBase._matches_log_patterns(record, patterns), True)
def test_key_check_mismatch_non_str_key(self, log_mock):
"""ParserBase - Key Check, Mismatch; Non-String Key"""
schema = {
'key': 'string'
}
record = {
100: 'test',
200: 'test'
}
assert_equal(ParserBase._key_check(record, schema), False)
log_mock.assert_called_with('Found keys not expected in record: %s', '100, 200')
def test_convert_type_none(self, log_mock):
"""ParserBase - Convert Type, NoneType Value"""
schema = {
'key': 'string'
}
record = {
'key': None
}
assert_equal(ParserBase._convert_type(record, schema), True)
assert_equal(record, {'key': None})
log_mock.assert_called_with('Skipping NoneType value in record for key: %s', 'key')
def test_key_check_nested_loose(self):
"""ParserBase - Key Check, Loose Nested Schema"""
schema = {
'key': 'string',
'nested': {}
}
record = {
'key': 'value',
'nested': {
'key_01': 100
}
}
assert_equal(ParserBase._key_check(record, schema), True)
def test_convert_type_unsupported_type(self, log_mock):
"""ParserBase - Convert Type, Unsupported Type"""
schema = {
'key': 'foobar'
}
record = {
'key': 'foobarbaz'
}
assert_equal(ParserBase._convert_type(record, schema), False)
assert_equal(record, {'key': 'foobarbaz'})
log_mock.assert_called_with(
'Unsupported value type in schema for key \'%s\': %s', 'key', 'foobar'
)
def test_parse(self):
"""ParserBase - Parse, Invalid Schema"""
options = {
'schema': {
'key': 'string'
},
'configuration': {
'optional_top_level_keys': [
'non_key'
]
}
}
assert_equal(ParserBase(options).parse(None), False)
def test_convert_type_optionals(self, log_mock):
"""ParserBase - Convert Type, With Optionals Missing"""
schema = {
'required_key': 'string',
'optional_key': 'string'
}
optionals = {'optional_key'}
record = {
'required_key': 'required_value'
}
assert_equal(ParserBase._convert_type(record, schema, optionals), True)
assert_equal(record, {'required_key': 'required_value'})
log_mock.assert_called_with(
'Skipping optional key not found in record: %s', 'optional_key')