def test_stack_schema_map(self):
"""Test to ensure that an entry exists in the stack-schema-map for the current package version."""
package_version = Version(load_current_package_version())
stack_map = utils.load_etc_dump('stack-schema-map.yaml')
err_msg = f'There is no entry defined for the current package ({package_version}) in the stack-schema-map'
self.assertIn(package_version, [Version(v)[:2] for v in stack_map],
err_msg)
def test_deprecated_rules(self):
"""Test that deprecated rules are properly handled."""
versions = load_versions()
deprecations = load_etc_dump('deprecated_rules.json')
deprecated_rules = {}
rules_path = get_path('rules')
deprecated_path = get_path("rules", "_deprecated")
misplaced_rules = [
r for r in self.all_rules
if r.path.relative_to(rules_path).parts[-2] == '_deprecated'
and # noqa: W504
r.contents.metadata.maturity != 'deprecated'
]
misplaced = '\n'.join(
f'{self.rule_str(r)} {r.contents.metadata.maturity}'
for r in misplaced_rules)
err_str = f'The following rules are stored in {deprecated_path} but are not marked as deprecated:\n{misplaced}'
self.assertListEqual(misplaced_rules, [], err_str)
for rule in self.deprecated_rules:
meta = rule.contents.metadata
deprecated_rules[rule.id] = rule
err_msg = f'{self.rule_str(rule)} cannot be deprecated if it has not been version locked. ' \
f'Convert to `development` or delete the rule file instead'
self.assertIn(rule.id, versions, err_msg)
rule_path = rule.path.relative_to(rules_path)
err_msg = f'{self.rule_str(rule)} deprecated rules should be stored in ' \
f'"{deprecated_path}" folder'
self.assertEqual('_deprecated', rule_path.parts[-2], err_msg)
err_msg = f'{self.rule_str(rule)} missing deprecation date'
self.assertIsNotNone(meta['deprecation_date'], err_msg)
err_msg = f'{self.rule_str(rule)} deprecation_date and updated_date should match'
self.assertEqual(meta['deprecation_date'], meta['updated_date'],
err_msg)
# skip this so the lock file can be shared across branches
#
# missing_rules = sorted(set(versions).difference(set(self.rule_lookup)))
# missing_rule_strings = '\n '.join(f'{r} - {versions[r]["rule_name"]}' for r in missing_rules)
# err_msg = f'Deprecated rules should not be removed, but moved to the rules/_deprecated folder instead. ' \
# f'The following rules have been version locked and are missing. ' \
# f'Re-add to the deprecated folder and update maturity to "deprecated": \n {missing_rule_strings}'
# self.assertEqual([], missing_rules, err_msg)
for rule_id, entry in deprecations.items():
rule_str = f'{rule_id} - {entry["rule_name"]} ->'
self.assertIn(
rule_id, deprecated_rules,
f'{rule_str} is logged in "deprecated_rules.json" but is missing'
)
def test_production_rules_have_rta(self):
"""Ensure that all production rules have RTAs."""
mappings = load_etc_dump('rule-mapping.yml')
ttp_names = get_ttp_names()
for rule in rule_loader.get_production_rules():
if rule.type == 'query' and rule.id in mappings:
matching_rta = mappings[rule.id].get('rta_name')
self.assertIsNotNone(matching_rta, "Rule {} ({}) does not have RTAs".format(rule.name, rule.id))
rta_name, ext = os.path.splitext(matching_rta)
if rta_name not in ttp_names:
self.fail("{} ({}) references unknown RTA: {}".format(rule.name, rule.id, rta_name))
def test_production_rules_have_rta(self):
"""Ensure that all production rules have RTAs."""
mappings = load_etc_dump('rule-mapping.yml')
ttp_names = get_ttp_names()
for rule in self.production_rules:
if isinstance(rule.contents.data,
QueryRuleData) and rule.id in mappings:
matching_rta = mappings[rule.id].get('rta_name')
self.assertIsNotNone(
matching_rta, f'{self.rule_str(rule)} does not have RTAs')
rta_name, ext = os.path.splitext(matching_rta)
if rta_name not in ttp_names:
self.fail(
f'{self.rule_str(rule)} references unknown RTA: {rta_name}'
)
def test_true_positives(self):
"""Test that expected results return against true positives."""
mismatched_ecs = []
mappings = load_etc_dump('rule-mapping.yml')
for rule in rule_loader.get_production_rules():
if isinstance(rule.contents.data, KQLRuleData):
if rule.id not in mappings:
continue
mapping = mappings[rule.id]
expected = mapping['count']
sources = mapping.get('sources')
rta_file = mapping['rta_name']
# ensure sources is defined and not empty; schema allows it to not be set since 'pending' bypasses
self.assertTrue(
sources, 'No sources defined for: {} - {} '.format(
rule.id, rule.name))
msg = 'Expected TP results did not match for: {} - {}'.format(
rule.id, rule.name)
data_files = [
get_data_files('true_positives', rta_file).get(s)
for s in sources
]
data_file = combine_sources(*data_files)
results = self.evaluate(data_file, rule, expected, msg)
ecs_versions = set(
[r.get('ecs', {}).get('version') for r in results])
rule_ecs = set(rule.metadata.get('ecs_version').copy())
if not ecs_versions & rule_ecs:
msg = '{} - {} ecs_versions ({}) not in source data versions ({})'.format(
rule.id, rule.name, ', '.join(rule_ecs),
', '.join(ecs_versions))
mismatched_ecs.append(msg)
if mismatched_ecs:
msg = 'Rules detected with source data from ecs versions not listed within the rule: \n{}'.format(
'\n'.join(mismatched_ecs))
warnings.warn(msg)
def test_deprecated_rules(self):
"""Test that deprecated rules are properly handled."""
versions = load_versions()
deprecations = load_etc_dump('deprecated_rules.json')
deprecated_rules = {}
for rule in self.all_rules:
meta = rule.contents.metadata
maturity = meta.maturity
if maturity == 'deprecated':
deprecated_rules[rule.id] = rule
err_msg = f'{self.rule_str(rule)} cannot be deprecated if it has not been version locked. ' \
f'Convert to `development` or delete the rule file instead'
self.assertIn(rule.id, versions, err_msg)
rule_path = rule.path.relative_to(get_path('rules'))
err_msg = f'{self.rule_str(rule)} deprecated rules should be stored in ' \
f'"{get_path("rules", "_deprecated")}" folder'
self.assertEqual('_deprecated', rule_path.parts[0], err_msg)
err_msg = f'{self.rule_str(rule)} missing deprecation date'
self.assertIsNotNone(meta.deprecation_date, err_msg)
err_msg = f'{self.rule_str(rule)} deprecation_date and updated_date should match'
self.assertEqual(meta.deprecation_date, meta.updated_date,
err_msg)
missing_rules = sorted(set(versions).difference(set(self.rule_lookup)))
missing_rule_strings = '\n '.join(f'{r} - {versions[r]["rule_name"]}'
for r in missing_rules)
err_msg = f'Deprecated rules should not be removed, but moved to the rules/_deprecated folder instead. ' \
f'The following rules have been version locked and are missing. ' \
f'Re-add to the deprecated folder and update maturity to "deprecated": \n {missing_rule_strings}'
self.assertEqual([], missing_rules, err_msg)
for rule_id, entry in deprecations.items():
rule_str = f'{rule_id} - {entry["rule_name"]} ->'
self.assertIn(
rule_id, deprecated_rules,
f'{rule_str} is logged in "deprecated_rules.json" but is missing'
)
def test_deprecated_rules(self):
"""Test that deprecated rules are properly handled."""
from detection_rules.packaging import current_stack_version
versions = default_version_lock.version_lock
deprecations = load_etc_dump('deprecated_rules.json')
deprecated_rules = {}
rules_path = get_path('rules')
deprecated_path = get_path("rules", "_deprecated")
misplaced_rules = [
r for r in self.all_rules
if r.path.relative_to(rules_path).parts[-2] == '_deprecated'
and # noqa: W504
r.contents.metadata.maturity != 'deprecated'
]
misplaced = '\n'.join(
f'{self.rule_str(r)} {r.contents.metadata.maturity}'
for r in misplaced_rules)
err_str = f'The following rules are stored in {deprecated_path} but are not marked as deprecated:\n{misplaced}'
self.assertListEqual(misplaced_rules, [], err_str)
for rule in self.deprecated_rules:
meta = rule.contents.metadata
deprecated_rules[rule.id] = rule
err_msg = f'{self.rule_str(rule)} cannot be deprecated if it has not been version locked. ' \
f'Convert to `development` or delete the rule file instead'
self.assertIn(rule.id, versions, err_msg)
rule_path = rule.path.relative_to(rules_path)
err_msg = f'{self.rule_str(rule)} deprecated rules should be stored in ' \
f'"{deprecated_path}" folder'
self.assertEqual('_deprecated', rule_path.parts[-2], err_msg)
err_msg = f'{self.rule_str(rule)} missing deprecation date'
self.assertIsNotNone(meta['deprecation_date'], err_msg)
err_msg = f'{self.rule_str(rule)} deprecation_date and updated_date should match'
self.assertEqual(meta['deprecation_date'], meta['updated_date'],
err_msg)
# skip this so the lock file can be shared across branches
#
# missing_rules = sorted(set(versions).difference(set(self.rule_lookup)))
# missing_rule_strings = '\n '.join(f'{r} - {versions[r]["rule_name"]}' for r in missing_rules)
# err_msg = f'Deprecated rules should not be removed, but moved to the rules/_deprecated folder instead. ' \
# f'The following rules have been version locked and are missing. ' \
# f'Re-add to the deprecated folder and update maturity to "deprecated": \n {missing_rule_strings}'
# self.assertEqual([], missing_rules, err_msg)
stack_version = Version(current_stack_version())
for rule_id, entry in deprecations.items():
# if a rule is deprecated and not backported in order to keep the rule active in older branches, then it
# will exist in the deprecated_rules.json file and not be in the _deprecated folder - this is expected.
# However, that should not occur except by exception - the proper way to handle this situation is to
# "fork" the existing rule by adding a new min_stack_version.
if stack_version < Version(entry['stack_version']):
continue
rule_str = f'{rule_id} - {entry["rule_name"]} ->'
self.assertIn(
rule_id, deprecated_rules,
f'{rule_str} is logged in "deprecated_rules.json" but is missing'
)