def test_stack_schema_map(self): """Test to ensure that an entry exists in the stack-schema-map for the current package version.""" package_version = Version(load_current_package_version()) stack_map = utils.load_etc_dump('stack-schema-map.yaml') err_msg = f'There is no entry defined for the current package ({package_version}) in the stack-schema-map' self.assertIn(package_version, [Version(v)[:2] for v in stack_map], err_msg)
def test_deprecated_rules(self): """Test that deprecated rules are properly handled.""" versions = load_versions() deprecations = load_etc_dump('deprecated_rules.json') deprecated_rules = {} rules_path = get_path('rules') deprecated_path = get_path("rules", "_deprecated") misplaced_rules = [ r for r in self.all_rules if r.path.relative_to(rules_path).parts[-2] == '_deprecated' and # noqa: W504 r.contents.metadata.maturity != 'deprecated' ] misplaced = '\n'.join( f'{self.rule_str(r)} {r.contents.metadata.maturity}' for r in misplaced_rules) err_str = f'The following rules are stored in {deprecated_path} but are not marked as deprecated:\n{misplaced}' self.assertListEqual(misplaced_rules, [], err_str) for rule in self.deprecated_rules: meta = rule.contents.metadata deprecated_rules[rule.id] = rule err_msg = f'{self.rule_str(rule)} cannot be deprecated if it has not been version locked. ' \ f'Convert to `development` or delete the rule file instead' self.assertIn(rule.id, versions, err_msg) rule_path = rule.path.relative_to(rules_path) err_msg = f'{self.rule_str(rule)} deprecated rules should be stored in ' \ f'"{deprecated_path}" folder' self.assertEqual('_deprecated', rule_path.parts[-2], err_msg) err_msg = f'{self.rule_str(rule)} missing deprecation date' self.assertIsNotNone(meta['deprecation_date'], err_msg) err_msg = f'{self.rule_str(rule)} deprecation_date and updated_date should match' self.assertEqual(meta['deprecation_date'], meta['updated_date'], err_msg) # skip this so the lock file can be shared across branches # # missing_rules = sorted(set(versions).difference(set(self.rule_lookup))) # missing_rule_strings = '\n '.join(f'{r} - {versions[r]["rule_name"]}' for r in missing_rules) # err_msg = f'Deprecated rules should not be removed, but moved to the rules/_deprecated folder instead. ' \ # f'The following rules have been version locked and are missing. ' \ # f'Re-add to the deprecated folder and update maturity to "deprecated": \n {missing_rule_strings}' # self.assertEqual([], missing_rules, err_msg) for rule_id, entry in deprecations.items(): rule_str = f'{rule_id} - {entry["rule_name"]} ->' self.assertIn( rule_id, deprecated_rules, f'{rule_str} is logged in "deprecated_rules.json" but is missing' )
def test_production_rules_have_rta(self): """Ensure that all production rules have RTAs.""" mappings = load_etc_dump('rule-mapping.yml') ttp_names = get_ttp_names() for rule in rule_loader.get_production_rules(): if rule.type == 'query' and rule.id in mappings: matching_rta = mappings[rule.id].get('rta_name') self.assertIsNotNone(matching_rta, "Rule {} ({}) does not have RTAs".format(rule.name, rule.id)) rta_name, ext = os.path.splitext(matching_rta) if rta_name not in ttp_names: self.fail("{} ({}) references unknown RTA: {}".format(rule.name, rule.id, rta_name))
def test_production_rules_have_rta(self): """Ensure that all production rules have RTAs.""" mappings = load_etc_dump('rule-mapping.yml') ttp_names = get_ttp_names() for rule in self.production_rules: if isinstance(rule.contents.data, QueryRuleData) and rule.id in mappings: matching_rta = mappings[rule.id].get('rta_name') self.assertIsNotNone( matching_rta, f'{self.rule_str(rule)} does not have RTAs') rta_name, ext = os.path.splitext(matching_rta) if rta_name not in ttp_names: self.fail( f'{self.rule_str(rule)} references unknown RTA: {rta_name}' )
def test_true_positives(self): """Test that expected results return against true positives.""" mismatched_ecs = [] mappings = load_etc_dump('rule-mapping.yml') for rule in rule_loader.get_production_rules(): if isinstance(rule.contents.data, KQLRuleData): if rule.id not in mappings: continue mapping = mappings[rule.id] expected = mapping['count'] sources = mapping.get('sources') rta_file = mapping['rta_name'] # ensure sources is defined and not empty; schema allows it to not be set since 'pending' bypasses self.assertTrue( sources, 'No sources defined for: {} - {} '.format( rule.id, rule.name)) msg = 'Expected TP results did not match for: {} - {}'.format( rule.id, rule.name) data_files = [ get_data_files('true_positives', rta_file).get(s) for s in sources ] data_file = combine_sources(*data_files) results = self.evaluate(data_file, rule, expected, msg) ecs_versions = set( [r.get('ecs', {}).get('version') for r in results]) rule_ecs = set(rule.metadata.get('ecs_version').copy()) if not ecs_versions & rule_ecs: msg = '{} - {} ecs_versions ({}) not in source data versions ({})'.format( rule.id, rule.name, ', '.join(rule_ecs), ', '.join(ecs_versions)) mismatched_ecs.append(msg) if mismatched_ecs: msg = 'Rules detected with source data from ecs versions not listed within the rule: \n{}'.format( '\n'.join(mismatched_ecs)) warnings.warn(msg)
def test_deprecated_rules(self): """Test that deprecated rules are properly handled.""" versions = load_versions() deprecations = load_etc_dump('deprecated_rules.json') deprecated_rules = {} for rule in self.all_rules: meta = rule.contents.metadata maturity = meta.maturity if maturity == 'deprecated': deprecated_rules[rule.id] = rule err_msg = f'{self.rule_str(rule)} cannot be deprecated if it has not been version locked. ' \ f'Convert to `development` or delete the rule file instead' self.assertIn(rule.id, versions, err_msg) rule_path = rule.path.relative_to(get_path('rules')) err_msg = f'{self.rule_str(rule)} deprecated rules should be stored in ' \ f'"{get_path("rules", "_deprecated")}" folder' self.assertEqual('_deprecated', rule_path.parts[0], err_msg) err_msg = f'{self.rule_str(rule)} missing deprecation date' self.assertIsNotNone(meta.deprecation_date, err_msg) err_msg = f'{self.rule_str(rule)} deprecation_date and updated_date should match' self.assertEqual(meta.deprecation_date, meta.updated_date, err_msg) missing_rules = sorted(set(versions).difference(set(self.rule_lookup))) missing_rule_strings = '\n '.join(f'{r} - {versions[r]["rule_name"]}' for r in missing_rules) err_msg = f'Deprecated rules should not be removed, but moved to the rules/_deprecated folder instead. ' \ f'The following rules have been version locked and are missing. ' \ f'Re-add to the deprecated folder and update maturity to "deprecated": \n {missing_rule_strings}' self.assertEqual([], missing_rules, err_msg) for rule_id, entry in deprecations.items(): rule_str = f'{rule_id} - {entry["rule_name"]} ->' self.assertIn( rule_id, deprecated_rules, f'{rule_str} is logged in "deprecated_rules.json" but is missing' )
def test_deprecated_rules(self): """Test that deprecated rules are properly handled.""" from detection_rules.packaging import current_stack_version versions = default_version_lock.version_lock deprecations = load_etc_dump('deprecated_rules.json') deprecated_rules = {} rules_path = get_path('rules') deprecated_path = get_path("rules", "_deprecated") misplaced_rules = [ r for r in self.all_rules if r.path.relative_to(rules_path).parts[-2] == '_deprecated' and # noqa: W504 r.contents.metadata.maturity != 'deprecated' ] misplaced = '\n'.join( f'{self.rule_str(r)} {r.contents.metadata.maturity}' for r in misplaced_rules) err_str = f'The following rules are stored in {deprecated_path} but are not marked as deprecated:\n{misplaced}' self.assertListEqual(misplaced_rules, [], err_str) for rule in self.deprecated_rules: meta = rule.contents.metadata deprecated_rules[rule.id] = rule err_msg = f'{self.rule_str(rule)} cannot be deprecated if it has not been version locked. ' \ f'Convert to `development` or delete the rule file instead' self.assertIn(rule.id, versions, err_msg) rule_path = rule.path.relative_to(rules_path) err_msg = f'{self.rule_str(rule)} deprecated rules should be stored in ' \ f'"{deprecated_path}" folder' self.assertEqual('_deprecated', rule_path.parts[-2], err_msg) err_msg = f'{self.rule_str(rule)} missing deprecation date' self.assertIsNotNone(meta['deprecation_date'], err_msg) err_msg = f'{self.rule_str(rule)} deprecation_date and updated_date should match' self.assertEqual(meta['deprecation_date'], meta['updated_date'], err_msg) # skip this so the lock file can be shared across branches # # missing_rules = sorted(set(versions).difference(set(self.rule_lookup))) # missing_rule_strings = '\n '.join(f'{r} - {versions[r]["rule_name"]}' for r in missing_rules) # err_msg = f'Deprecated rules should not be removed, but moved to the rules/_deprecated folder instead. ' \ # f'The following rules have been version locked and are missing. ' \ # f'Re-add to the deprecated folder and update maturity to "deprecated": \n {missing_rule_strings}' # self.assertEqual([], missing_rules, err_msg) stack_version = Version(current_stack_version()) for rule_id, entry in deprecations.items(): # if a rule is deprecated and not backported in order to keep the rule active in older branches, then it # will exist in the deprecated_rules.json file and not be in the _deprecated folder - this is expected. # However, that should not occur except by exception - the proper way to handle this situation is to # "fork" the existing rule by adding a new min_stack_version. if stack_version < Version(entry['stack_version']): continue rule_str = f'{rule_id} - {entry["rule_name"]} ->' self.assertIn( rule_id, deprecated_rules, f'{rule_str} is logged in "deprecated_rules.json" but is missing' )