示例#1
0
 def test_normalize_none_defined(self, log_mock):
     """Normalizer - Normalize, No Types Defined"""
     log_type = 'cloudtrail'
     Normalizer._types_config = {}
     Normalizer.normalize(self._test_record(), log_type)
     log_mock.assert_called_with(
         'No normalized types defined for log type: %s', log_type)
示例#2
0
    def test_normalize_corner_case(self):
        """Normalizer - Normalize - Corner Case"""
        log_type = 'cloudtrail'
        Normalizer._types_config = {
            log_type: {
                'normalized_key':
                NormalizedType(log_type, 'normalized_key',
                               ['original_key', 'original_key']),
                'account':
                self._normalized_type_account()
            }
        }
        record = {
            'unrelated_key': 'foobar',
            'original_key': {
                'original_key': 'fizzbuzz',
            }
        }
        Normalizer.normalize(record, log_type)

        expected_record = {
            'unrelated_key': 'foobar',
            'original_key': {
                'original_key': 'fizzbuzz',
            },
            'streamalert_normalization': {
                'streamalert_record_id': MOCK_RECORD_ID,
                'normalized_key': [{
                    'values': ['fizzbuzz'],
                    'function': None
                }]
            }
        }

        assert_equal(record, expected_record)
示例#3
0
    def test_normalize(self):
        """Normalizer - Normalize"""
        log_type = 'cloudtrail'
        Normalizer._types_config = {
            log_type: {
                'region': {'region', 'awsRegion'},
                'sourceAccount': {'account', 'accountId'}
            }
        }
        record = self._test_record()
        Normalizer.normalize(record, log_type)

        expected_record = {
            'account': 123456,
            'region': 'region_name',
            'detail': {
                'awsRegion': 'region_name',
                'source': '1.1.1.2',
                'userIdentity': {
                    "userName": "******",
                    "invokedBy": "signin.amazonaws.com"
                }
            },
            'sourceIPAddress': '1.1.1.3',
            'streamalert:normalization': {
                'region': ['region_name'],
                'sourceAccount': [123456]
            }
        }

        assert_equal(record, expected_record)
示例#4
0
    def test_normalize_corner_case(self):
        """Normalizer - Normalize - Corner Case"""
        log_type = 'cloudtrail'
        Normalizer._types_config = {
            log_type: {
                'normalized_key': {'normalized_key', 'original_key'},
                'sourceAccount': {'account', 'accountId'}
            }
        }
        record = {
            'unrelated_key': 'foobar',
            'original_key': {
                'original_key': 'fizzbuzz',
            }
        }
        Normalizer.normalize(record, log_type)

        expected_record = {
            'unrelated_key': 'foobar',
            'original_key': {
                'original_key': 'fizzbuzz',
            },
            'streamalert:normalization': {
                'normalized_key': ['fizzbuzz']
            }
        }

        assert_equal(record, expected_record)
示例#5
0
    def _classify_payload(self, payload):
        """Run the payload through the classification logic to determine the data type

        Args:
            payload (StreamPayload): StreamAlert payload object being processed
        """
        # Get logs defined for the service/entity in the config
        logs_config = self._load_logs_for_resource(payload.service(), payload.resource)
        if not logs_config:
            LOGGER.error(
                'No log types defined for resource [%s] in sources configuration for service [%s]',
                payload.resource,
                payload.service()
            )
            return

        for record in payload.pre_parse():
            # Increment the processed size using the length of this record
            self._processed_size += len(record)

            # Get the parser for this data
            self._process_log_schemas(record, logs_config)

            LOGGER.debug('Parsed and classified payload: %s', bool(record))

            payload.fully_classified = payload.fully_classified and record
            if not record:
                self._log_bad_records(record, 1)
                continue

            LOGGER.debug(
                'Classified %d record(s) with schema: %s',
                len(record.parsed_records),
                record.log_schema_type
            )

            # Even if the parser was successful, there's a chance it
            # could not parse all records, so log them here as invalid
            self._log_bad_records(record, len(record.invalid_records))

            for parsed_rec in record.parsed_records:
                #
                # In Normalization v1, the normalized types are defined based on log source
                # (e.g. osquery, cloudwatch etc) and this will be deprecated.
                # In Normalization v2, the normalized types are defined based on log type
                # (e.g. osquery:differential, cloudwatch:cloudtrail, cloudwatch:events etc)
                #
                Normalizer.normalize(parsed_rec, record.log_schema_type)

            self._payloads.append(record)
示例#6
0
    def test_normalize(self):
        """Normalizer - Normalize"""
        log_type = 'cloudtrail'
        Normalizer._types_config = {
            log_type: {
                'region': self._normalized_type_region(),
                'ipv4': self._normalized_type_ip()
            }
        }
        record = self._test_record()
        Normalizer.normalize(record, log_type)

        expected_record = {
            'account': 123456,
            'region': 'region_name',
            'detail': {
                'awsRegion': 'region_name',
                'source': '1.1.1.2',
                'userIdentity': {
                    "userName": "******",
                    "invokedBy": "signin.amazonaws.com"
                }
            },
            'sourceIPAddress': '1.1.1.3',
            'streamalert_normalization': {
                'streamalert_record_id':
                MOCK_RECORD_ID,
                'region': [{
                    'values': ['region_name'],
                    'function': 'AWS region'
                }, {
                    'values': ['region_name'],
                    'function': 'AWS region'
                }],
                'ipv4': [{
                    'values': ['1.1.1.3'],
                    'function': 'source ip address'
                }, {
                    'values': ['1.1.1.2'],
                    'function': 'source ip address'
                }]
            }
        }

        assert_equal(record, expected_record)
示例#7
0
    def test_normalize_condition(self):
        """Normalizer - Test normalization when condition applied"""
        log_type = 'cloudtrail'

        region = NormalizedType(
            'test_log_type', 'region', [{
                'path': ['region'],
                'function': 'AWS region'
            }, {
                'path': ['detail', 'awsRegion'],
                'function': 'AWS region',
                'condition': {
                    'path': ['detail', 'userIdentity', 'userName'],
                    'not_in': ['alice', 'bob']
                }
            }])

        ipv4 = NormalizedType('test_log_type', 'ip_address',
                              [{
                                  'path': ['sourceIPAddress'],
                                  'function': 'source ip address',
                                  'condition': {
                                      'path': ['account'],
                                      'is': '123456'
                                  }
                              }, {
                                  'path': ['detail', 'source'],
                                  'function': 'source ip address',
                                  'condition': {
                                      'path': ['account'],
                                      'is_not': '123456'
                                  }
                              }])

        Normalizer._types_config = {log_type: {'region': region, 'ipv4': ipv4}}
        record = self._test_record()
        Normalizer.normalize(record, log_type)

        expected_record = {
            'account': 123456,
            'region': 'region_name',
            'detail': {
                'awsRegion': 'region_name',
                'source': '1.1.1.2',
                'userIdentity': {
                    "userName": "******",
                    "invokedBy": "signin.amazonaws.com"
                }
            },
            'sourceIPAddress': '1.1.1.3',
            'streamalert_normalization': {
                'streamalert_record_id': MOCK_RECORD_ID,
                'region': [{
                    'values': ['region_name'],
                    'function': 'AWS region'
                }],
                'ipv4': [{
                    'values': ['1.1.1.3'],
                    'function': 'source ip address'
                }]
            }
        }
        assert_equal(record, expected_record)