def test_false_positives(self): """Test that expected results return against false positives.""" for rule in rule_loader.get_production_rules(): if isinstance(rule.contents.data, KQLRuleData): for fp_name, merged_data in get_fp_data_files().items(): msg = 'Unexpected FP match for: {} - {}, against: {}'.format( rule.id, rule.name, fp_name) self.evaluate(copy.deepcopy(merged_data), rule, 0, msg)
def test_false_positives(self): """Test that expected results return against false positives.""" for rule in rule_loader.get_production_rules(): if rule.type == 'query' and rule.contents['language'] == 'kuery': for fp_name, merged_data in get_fp_data_files().items(): msg = 'Unexpected FP match for: {} - {}, against: {}'.format( rule.id, rule.name, fp_name) self.evaluate(copy.deepcopy(merged_data), rule, 0, msg)
def test_package_summary(self): """Test the generation of the package summary.""" rules = rule_loader.get_production_rules() package = Package(rules, 'test-package') changed_rule_ids, new_rule_ids, deprecated_rule_ids = package.bump_versions( save_changes=False) package.generate_summary_and_changelog(changed_rule_ids, new_rule_ids, deprecated_rule_ids)
def test_production_rules_have_rta(self): """Ensure that all production rules have RTAs.""" mappings = load_etc_dump('rule-mapping.yml') ttp_names = get_ttp_names() for rule in rule_loader.get_production_rules(): if rule.type == 'query' and rule.id in mappings: matching_rta = mappings[rule.id].get('rta_name') self.assertIsNotNone(matching_rta, "Rule {} ({}) does not have RTAs".format(rule.name, rule.id)) rta_name, ext = os.path.splitext(matching_rta) if rta_name not in ttp_names: self.fail("{} ({}) references unknown RTA: {}".format(rule.name, rule.id, rta_name))
def test_true_positives(self): """Test that expected results return against true positives.""" mismatched_ecs = [] mappings = load_etc_dump('rule-mapping.yml') for rule in rule_loader.get_production_rules(): if isinstance(rule.contents.data, KQLRuleData): if rule.id not in mappings: continue mapping = mappings[rule.id] expected = mapping['count'] sources = mapping.get('sources') rta_file = mapping['rta_name'] # ensure sources is defined and not empty; schema allows it to not be set since 'pending' bypasses self.assertTrue( sources, 'No sources defined for: {} - {} '.format( rule.id, rule.name)) msg = 'Expected TP results did not match for: {} - {}'.format( rule.id, rule.name) data_files = [ get_data_files('true_positives', rta_file).get(s) for s in sources ] data_file = combine_sources(*data_files) results = self.evaluate(data_file, rule, expected, msg) ecs_versions = set( [r.get('ecs', {}).get('version') for r in results]) rule_ecs = set(rule.metadata.get('ecs_version').copy()) if not ecs_versions & rule_ecs: msg = '{} - {} ecs_versions ({}) not in source data versions ({})'.format( rule.id, rule.name, ', '.join(rule_ecs), ', '.join(ecs_versions)) mismatched_ecs.append(msg) if mismatched_ecs: msg = 'Rules detected with source data from ecs versions not listed within the rule: \n{}'.format( '\n'.join(mismatched_ecs)) warnings.warn(msg)
def setUpClass(cls): cls.rule_files = rule_loader.load_rule_files(verbose=False) cls.rule_lookup = rule_loader.load_rules(verbose=False) cls.rules = cls.rule_lookup.values() cls.production_rules = rule_loader.get_production_rules()