def test_json_path_records(self):
        """ParserBase - JSON Path Records"""
        expected_records = [
            {
                'foo': 'bar'
            },
            {
                'bar': 'baz'
            }
        ]
        record = {
            'key': [
                {
                    'value': expected_records
                }
            ]
        }
        options = {
            'configuration': {
                'json_path': 'key[].value[]'
            }
        }

        parser = ParserBase(options)
        records = parser._json_path_records(record)
        assert_equal(records, expected_records)
    def test_validate_schema_all(self):
        """ParserBase - Validate Schema, All Options"""
        options = {
            'schema': {
                'timestamp': 'string',
                'host': 'string',
                'application': 'string',
                'message': 'string'
            },
            'configuration': {
                'envelope_keys': {
                    'env_key_01': 'string',
                    'env_key_02': 'string'
                },
                'optional_envelope_keys': [
                    'env_key_01'
                ],
                'optional_top_level_keys': [
                    'host'
                ]
            }
        }

        parser = ParserBase(options)
        assert_equal(parser._validate_schema(), True)
 def test_add_parse_result_invalid(self):
     """ParserBase - Add Parse Result, Valid"""
     parser = ParserBase(None)
     record = {
         'key': 'value'
     }
     parser._add_parse_result(record, False, None)
     assert_equal(parser._invalid_parses, [record])
    def test_extract_envelope_none(self):
        """ParserBase - Extract Envelope, None"""
        record = {
            'key': 'value'
        }

        parser = ParserBase(None)
        envelope = parser._extract_envelope(record)
        assert_equal(envelope, None)
    def test_apply_envelope_none(self):
        """ParserBase - Apply Envelope, None"""
        record = {
            'key': 'value'
        }

        ParserBase._apply_envelope(record, {})

        assert_equal(record, {'key': 'value'})
 def test_add_optional_keys_none(self):
     """ParserBase - Add Optional Keys, None"""
     schema = {
         'key': 'string'
     }
     record = {
         'key': 'data'
     }
     ParserBase._add_optional_keys(record, schema, None)
     assert_equal(record, {'key': 'data'})
 def test_add_optional_keys(self):
     """ParserBase - Add Optional Keys"""
     schema = {
         'key': 'string',
         'optional_key': 'integer'
     }
     optionals = {'optional_key'}
     record = {
         'key': 'data'
     }
     ParserBase._add_optional_keys(record, schema, optionals)
     assert_equal(record, {'key': 'data', 'optional_key': 0})
 def setup(self):
     """Setup before each method"""
     # pylint: disable=abstract-class-instantiated,attribute-defined-outside-init
     options = {
         'schema': {
             'timestamp': 'string',
             'host': 'string'
         },
         'configuration': {
             'optional_top_level_keys': [
                 'host'
             ],
             'log_patterns': {
                 'host': [
                     'foo*'
                 ]
             },
             'json_path': 'logEvents[].message',
             'envelope_keys': {
                 'env_key_01': 'string'
             },
             'optional_envelope_keys': [
                 'env_key_01'
             ]
         }
     }
     self._parser = ParserBase(options)
    def test_extract_envelope(self):
        """ParserBase - Extract Envelope"""
        options = {
            'configuration': {
                'envelope_keys': [
                    'env_key_01'
                ]
            }
        }
        record = {
            'env_key_01': 'foo',
            'non_env_key': 'bar'
        }

        parser = ParserBase(options)
        envelope = parser._extract_envelope(record)
        assert_equal(envelope, {'env_key_01': 'foo'})
Example #10
0
    def test_validate_schema_invalid(self):
        """ParserBase - Validate Schema, Invalid"""
        options = {
            'schema': {
                'timestamp': 'string',
                'host': 'string',
                'application': 'string',
                'message': 'string'
            },
            'configuration': {
                'optional_top_level_keys': [
                    'date'
                ]
            }
        }

        parser = ParserBase(options)
        assert_equal(parser._validate_schema(), False)
Example #11
0
 def test_convert_type_list(self):
     """ParserBase - Convert Type, Invalid List"""
     schema = {
         'key': []
     }
     record = {
         'key': 'not a list'
     }
     assert_equal(ParserBase._convert_type(record, schema), False)
     assert_equal(record, {'key': 'not a list'})
Example #12
0
 def test_convert_type_int_invalid(self):
     """ParserBase - Convert Type, Invalid Int"""
     schema = {
         'key': 'integer'
     }
     record = {
         'key': 'not an int'
     }
     assert_equal(ParserBase._convert_type(record, schema), False)
     assert_equal(record, {'key': 'not an int'})
Example #13
0
 def test_convert_type_float_invalid(self):
     """ParserBase - Convert Type, Invalid Float"""
     schema = {
         'key': 'float'
     }
     record = {
         'key': 'not a float'
     }
     assert_equal(ParserBase._convert_type(record, schema), False)
     assert_equal(record, {'key': 'not a float'})
Example #14
0
 def test_convert_type_bool(self):
     """ParserBase - Convert Type, Boolean"""
     schema = {
         'key': 'boolean'
     }
     record = {
         'key': 'True'
     }
     assert_equal(ParserBase._convert_type(record, schema), True)
     assert_equal(record, {'key': True})
Example #15
0
 def test_matches_log_patterns_str(self, log_mock):
     """ParserBase - Matches Log Patterns, Str"""
     record = {
         'key': 'matching pattern'
     }
     patterns = {
         'key': '*pattern'
     }
     assert_equal(ParserBase._matches_log_patterns(record, patterns), True)
     log_mock.assert_any_call('Transforming flat pattern \'%s\' into list', '*pattern')
Example #16
0
 def test_convert_type_float(self):
     """ParserBase - Convert Type, Float"""
     schema = {
         'key': 'float'
     }
     record = {
         'key': '0.9'
     }
     assert_equal(ParserBase._convert_type(record, schema), True)
     assert_equal(record, {'key': 0.9})
Example #17
0
 def test_convert_type_int(self):
     """ParserBase - Convert Type, Int"""
     schema = {
         'key': 'integer'
     }
     record = {
         'key': '100'
     }
     assert_equal(ParserBase._convert_type(record, schema), True)
     assert_equal(record, {'key': 100})
Example #18
0
 def test_convert_type_unicode_str(self):
     """ParserBase - Convert Type, Unicode Str"""
     schema = {
         'key': 'string'
     }
     record = {
         'key': '\ue82a'
     }
     assert_equal(ParserBase._convert_type(record, schema), True)
     assert_equal(record, {'key': '\ue82a'})
Example #19
0
 def test_convert_type_str(self):
     """ParserBase - Convert Type, Str"""
     schema = {
         'key': 'string'
     }
     record = {
         'key': 100
     }
     assert_equal(ParserBase._convert_type(record, schema), True)
     assert_equal(record, {'key': '100'})
Example #20
0
 def test_key_check_bad_envelope_subset(self, log_mock):
     """ParserBase - Key Check, Invalid Envelope Subset"""
     envelope_schema = {
         'env_key_01': 'string'
     }
     record = {
         'env_key_02': 'test'
     }
     assert_equal(ParserBase._key_check(record, envelope_schema, is_envelope=True), False)
     log_mock.assert_called_with('Missing keys in record envelope: %s', {'env_key_01'})
Example #21
0
 def test_key_check_mismatch(self, log_mock):
     """ParserBase - Key Check, Mismatch"""
     schema = {
         'key': 'string'
     }
     record = {
         'not_key': 'test'
     }
     assert_equal(ParserBase._key_check(record, schema), False)
     log_mock.assert_called_with('Found keys not expected in record: %s', 'not_key')
Example #22
0
    def test_apply_envelope(self):
        """ParserBase - Apply Envelope"""
        record = {
            'key': 'value'
        }
        envelope = {
            'env_key_01': 'value'
        }

        ParserBase._apply_envelope(record, envelope)

        expected_record = {
            'key': 'value',
            'streamalert:envelope_keys': {
                'env_key_01': 'value'
            }
        }

        assert_equal(record, expected_record)
Example #23
0
 def test_matches_log_patterns_invalid_key(self, log_mock):
     """ParserBase - Matches Log Patterns, Invalid Key"""
     record = {
         'key': 'value'
     }
     patterns = {
         'not_key': '*pattern'
     }
     assert_equal(ParserBase._matches_log_patterns(record, patterns), False)
     log_mock.assert_any_call(
         'Declared log pattern key [%s] does exist in record:\n%s', 'not_key', record)
Example #24
0
 def test_matches_log_patterns_list(self):
     """ParserBase - Matches Log Patterns, List"""
     record = {
         'key': 'matching pattern'
     }
     patterns = {
         'key': [
             '*pattern'
         ]
     }
     assert_equal(ParserBase._matches_log_patterns(record, patterns), True)
Example #25
0
 def test_key_check_mismatch_non_str_key(self, log_mock):
     """ParserBase - Key Check, Mismatch; Non-String Key"""
     schema = {
         'key': 'string'
     }
     record = {
         100: 'test',
         200: 'test'
     }
     assert_equal(ParserBase._key_check(record, schema), False)
     log_mock.assert_called_with('Found keys not expected in record: %s', '100, 200')
Example #26
0
 def test_convert_type_none(self, log_mock):
     """ParserBase - Convert Type, NoneType Value"""
     schema = {
         'key': 'string'
     }
     record = {
         'key': None
     }
     assert_equal(ParserBase._convert_type(record, schema), True)
     assert_equal(record, {'key': None})
     log_mock.assert_called_with('Skipping NoneType value in record for key: %s', 'key')
Example #27
0
 def test_key_check_nested_loose(self):
     """ParserBase - Key Check, Loose Nested Schema"""
     schema = {
         'key': 'string',
         'nested': {}
     }
     record = {
         'key': 'value',
         'nested': {
             'key_01': 100
         }
     }
     assert_equal(ParserBase._key_check(record, schema), True)
Example #28
0
 def test_convert_type_unsupported_type(self, log_mock):
     """ParserBase - Convert Type, Unsupported Type"""
     schema = {
         'key': 'foobar'
     }
     record = {
         'key': 'foobarbaz'
     }
     assert_equal(ParserBase._convert_type(record, schema), False)
     assert_equal(record, {'key': 'foobarbaz'})
     log_mock.assert_called_with(
         'Unsupported value type in schema for key \'%s\': %s', 'key', 'foobar'
     )
Example #29
0
    def test_parse(self):
        """ParserBase - Parse, Invalid Schema"""
        options = {
            'schema': {
                'key': 'string'
            },
            'configuration': {
                'optional_top_level_keys': [
                    'non_key'
                ]
            }
        }

        assert_equal(ParserBase(options).parse(None), False)
Example #30
0
 def test_convert_type_optionals(self, log_mock):
     """ParserBase - Convert Type, With Optionals Missing"""
     schema = {
         'required_key': 'string',
         'optional_key': 'string'
     }
     optionals = {'optional_key'}
     record = {
         'required_key': 'required_value'
     }
     assert_equal(ParserBase._convert_type(record, schema, optionals), True)
     assert_equal(record, {'required_key': 'required_value'})
     log_mock.assert_called_with(
         'Skipping optional key not found in record: %s', 'optional_key')