def test_false_positives(self):
"""Test that expected results return against false positives."""
for rule in rule_loader.get_production_rules():
if isinstance(rule.contents.data, KQLRuleData):
for fp_name, merged_data in get_fp_data_files().items():
msg = 'Unexpected FP match for: {} - {}, against: {}'.format(
rule.id, rule.name, fp_name)
self.evaluate(copy.deepcopy(merged_data), rule, 0, msg)
def test_false_positives(self):
"""Test that expected results return against false positives."""
for rule in rule_loader.get_production_rules():
if rule.type == 'query' and rule.contents['language'] == 'kuery':
for fp_name, merged_data in get_fp_data_files().items():
msg = 'Unexpected FP match for: {} - {}, against: {}'.format(
rule.id, rule.name, fp_name)
self.evaluate(copy.deepcopy(merged_data), rule, 0, msg)
def test_package_summary(self):
"""Test the generation of the package summary."""
rules = rule_loader.get_production_rules()
package = Package(rules, 'test-package')
changed_rule_ids, new_rule_ids, deprecated_rule_ids = package.bump_versions(
save_changes=False)
package.generate_summary_and_changelog(changed_rule_ids, new_rule_ids,
deprecated_rule_ids)
def test_production_rules_have_rta(self):
"""Ensure that all production rules have RTAs."""
mappings = load_etc_dump('rule-mapping.yml')
ttp_names = get_ttp_names()
for rule in rule_loader.get_production_rules():
if rule.type == 'query' and rule.id in mappings:
matching_rta = mappings[rule.id].get('rta_name')
self.assertIsNotNone(matching_rta, "Rule {} ({}) does not have RTAs".format(rule.name, rule.id))
rta_name, ext = os.path.splitext(matching_rta)
if rta_name not in ttp_names:
self.fail("{} ({}) references unknown RTA: {}".format(rule.name, rule.id, rta_name))
def test_true_positives(self):
"""Test that expected results return against true positives."""
mismatched_ecs = []
mappings = load_etc_dump('rule-mapping.yml')
for rule in rule_loader.get_production_rules():
if isinstance(rule.contents.data, KQLRuleData):
if rule.id not in mappings:
continue
mapping = mappings[rule.id]
expected = mapping['count']
sources = mapping.get('sources')
rta_file = mapping['rta_name']
# ensure sources is defined and not empty; schema allows it to not be set since 'pending' bypasses
self.assertTrue(
sources, 'No sources defined for: {} - {} '.format(
rule.id, rule.name))
msg = 'Expected TP results did not match for: {} - {}'.format(
rule.id, rule.name)
data_files = [
get_data_files('true_positives', rta_file).get(s)
for s in sources
]
data_file = combine_sources(*data_files)
results = self.evaluate(data_file, rule, expected, msg)
ecs_versions = set(
[r.get('ecs', {}).get('version') for r in results])
rule_ecs = set(rule.metadata.get('ecs_version').copy())
if not ecs_versions & rule_ecs:
msg = '{} - {} ecs_versions ({}) not in source data versions ({})'.format(
rule.id, rule.name, ', '.join(rule_ecs),
', '.join(ecs_versions))
mismatched_ecs.append(msg)
if mismatched_ecs:
msg = 'Rules detected with source data from ecs versions not listed within the rule: \n{}'.format(
'\n'.join(mismatched_ecs))
warnings.warn(msg)
def setUpClass(cls):
cls.rule_files = rule_loader.load_rule_files(verbose=False)
cls.rule_lookup = rule_loader.load_rules(verbose=False)
cls.rules = cls.rule_lookup.values()
cls.production_rules = rule_loader.get_production_rules()