def test_rule_file_names_by_tactic(self):
"""Test to ensure rule files have the primary tactic prepended to the filename."""
rules = rule_loader.load_rules().values()
bad_name_rules = []
for rule in rules:
rule_path = Path(rule.path).resolve()
filename = rule_path.name
if rule_path.parent.name == 'ml':
continue
threat = rule.contents.get('threat', [])
authors = rule.contents.get('author', [])
if threat and 'Elastic' in authors:
primary_tactic = threat[0]['tactic']['name']
tactic_str = primary_tactic.lower().replace(' ', '_')
if tactic_str != filename[:len(tactic_str)]:
bad_name_rules.append(
f'{rule.id} - {Path(rule.path).name} -> expected: {tactic_str}'
)
if bad_name_rules:
error_msg = 'filename does not start with the primary tactic - update the tactic or the rule filename'
rule_err_str = '\n'.join(bad_name_rules)
self.fail(f'{error_msg}:\n{rule_err_str}')
def test_technique_deprecations(self):
"""Check for use of any ATT&CK techniques that have been deprecated."""
replacement_map = attack.techniques_redirect_map
revoked = list(attack.revoked)
deprecated = list(attack.deprecated)
rules = rule_loader.load_rules().values()
for rule in rules:
revoked_techniques = {}
rule_info = f'{rule.id} - {rule.name}'
threat_mapping = rule.contents.get('threat')
if threat_mapping:
for entry in threat_mapping:
techniques = entry.get('technique', [])
for technique in techniques:
if technique['id'] in revoked + deprecated:
revoked_techniques[
technique['id']] = replacement_map.get(
technique['id'], 'DEPRECATED - DO NOT USE')
if revoked_techniques:
old_new_mapping = "\n".join(
f'Actual: {k} -> Expected {v}'
for k, v in revoked_techniques.items())
self.fail(
f'{rule_info} -> Using deprecated ATT&CK techniques: \n{old_new_mapping}'
)
def test_rule_versioning(self):
"""Test that all rules are properly versioned and tracked"""
self.maxDiff = None
rules = rule_loader.load_rules().values()
original_hashes = []
post_bump_hashes = []
# test that no rules have versions defined
for rule in rules:
self.assertGreaterEqual(
rule.contents.autobumped_version, 1,
'{} - {}: version is not being set in package')
original_hashes.append(rule.contents.sha256())
package = Package(rules, 'test-package')
# test that all rules have versions defined
# package.bump_versions(save_changes=False)
for rule in package.rules:
self.assertGreaterEqual(
rule.contents.autobumped_version, 1,
'{} - {}: version is not being set in package')
# test that rules validate with version
for rule in package.rules:
post_bump_hashes.append(rule.contents.sha256())
# test that no hashes changed as a result of the version bumps
self.assertListEqual(original_hashes, post_bump_hashes,
'Version bumping modified the hash of a rule')
def test_casing_and_spacing(self):
"""Ensure consistent and expected casing for controlled tags."""
rules = rule_loader.load_rules().values()
def normalize(s):
return ''.join(s.lower().split())
expected_tags = [
'APM', 'AWS', 'Asset Visibility', 'Azure', 'Configuration Audit',
'Continuous Monitoring', 'Data Protection', 'Elastic',
'Endpoint Security', 'GCP', 'Identity and Access', 'Linux',
'Logging', 'ML', 'macOS', 'Monitoring', 'Network', 'Okta',
'Packetbeat', 'Post-Execution', 'SecOps', 'Windows'
]
expected_case = {normalize(t): t for t in expected_tags}
for rule in rules:
rule_tags = rule.contents.get('tags')
if rule_tags:
invalid_tags = {
t: expected_case[normalize(t)]
for t in rule_tags if normalize(t) in list(expected_case)
and t != expected_case[normalize(t)]
}
if invalid_tags:
error_msg = f'{rule.id} - {rule.name} -> Invalid casing for expected tags\n'
error_msg += f'Actual tags: {", ".join(invalid_tags)}\n'
error_msg += f'Expected tags: {", ".join(invalid_tags.values())}'
self.fail(error_msg)
def test_format_of_all_rules(self):
"""Test all rules."""
rules = rule_loader.load_rules().values()
for rule in rules:
self.compare_formatted(rule.rule_format(formatted_query=False),
callback=nested_normalize)
def test_format_of_all_rules(self):
"""Test all rules."""
rules = rule_loader.load_rules().values()
for rule in rules:
is_eql_rule = rule.type == 'eql'
self.compare_formatted(rule.rule_format(formatted_query=False),
callback=nested_normalize,
kwargs={'eql_rule': is_eql_rule})
def test_tactic_to_technique_correlations(self):
"""Ensure rule threat info is properly related to a single tactic and technique."""
rules = rule_loader.load_rules().values()
for rule in rules:
threat_mapping = rule.contents.get('threat')
if threat_mapping:
for entry in threat_mapping:
tactic = entry.get('tactic')
techniques = entry.get('technique', [])
# tactic
expected_tactic = attack.tactics_map[tactic['name']]
self.assertEqual(expected_tactic, tactic['id'],
f'ATT&CK tactic mapping error for rule: {rule.id} - {rule.name} ->\n'
f'expected: {expected_tactic} for {tactic["name"]}\n'
f'actual: {tactic["id"]}')
tactic_reference_id = tactic['reference'].rstrip('/').split('/')[-1]
self.assertEqual(tactic['id'], tactic_reference_id,
f'ATT&CK tactic mapping error for rule: {rule.id} - {rule.name} ->\n'
f'tactic ID {tactic["id"]} does not match the reference URL ID '
f'{tactic["reference"]}')
# techniques
for technique in techniques:
expected_technique = attack.technique_lookup[technique['id']]['name']
self.assertEqual(expected_technique, technique['name'],
f'ATT&CK technique mapping error for rule: {rule.id} - {rule.name} ->\n'
f'expected: {expected_technique} for {technique["id"]}\n'
f'actual: {technique["name"]}')
technique_reference_id = technique['reference'].rstrip('/').split('/')[-1]
self.assertEqual(technique['id'], technique_reference_id,
f'ATT&CK technique mapping error for rule: {rule.id} - {rule.name} ->\n'
f'technique ID {technique["id"]} does not match the reference URL ID '
f'{technique["reference"]}')
# sub-techniques
sub_techniques = technique.get('subtechnique')
if sub_techniques:
for sub_technique in sub_techniques:
expected_sub_technique = attack.technique_lookup[sub_technique['id']]['name']
self.assertEqual(expected_sub_technique, sub_technique['name'],
f'ATT&CK sub-technique mapping error for rule: {rule.id} - {rule.name} ->\n' # noqa: E501
f'expected: {expected_sub_technique} for {sub_technique["id"]}\n'
f'actual: {sub_technique["name"]}')
sub_technique_reference_id = '.'.join(
sub_technique['reference'].rstrip('/').split('/')[-2:])
self.assertEqual(sub_technique['id'], sub_technique_reference_id,
f'ATT&CK sub-technique mapping error for rule: {rule.id} - {rule.name} ->\n' # noqa: E501
f'sub-technique ID {sub_technique["id"]} does not match the reference URL ID ' # noqa: E501
f'{sub_technique["reference"]}')
def test_duplicated_tactics(self):
"""Check that a tactic is only defined once."""
rules = rule_loader.load_rules().values()
for rule in rules:
rule_info = f'{rule.id} - {rule.name}'
threat_mapping = rule.contents.get('threat', [])
tactics = [t['tactic']['name'] for t in threat_mapping]
duplicates = sorted(set(t for t in tactics if tactics.count(t) > 1))
if duplicates:
self.fail(f'{rule_info} -> duplicate tactics defined for {duplicates}. '
f'Flatten to a single entry per tactic')
def test_required_tags(self):
"""Test that expected tags are present within rules."""
rules = rule_loader.load_rules().values()
# indexes considered; only those with obvious relationships included
# 'apm-*-transaction*', 'auditbeat-*', 'endgame-*', 'filebeat-*', 'logs-*', 'logs-aws*',
# 'logs-endpoint.alerts-*', 'logs-endpoint.events.*', 'logs-okta*', 'packetbeat-*', 'winlogbeat-*'
required_tags_map = {
'apm-*-transaction*': {'all': ['APM']},
'auditbeat-*': {'any': ['Windows', 'macOS', 'Linux']},
'endgame-*': {'all': ['Endpoint Security']},
'logs-aws*': {'all': ['AWS']},
'logs-endpoint.alerts-*': {'all': ['Endpoint Security']},
'logs-endpoint.events.*': {'any': ['Windows', 'macOS', 'Linux', 'Host']},
'logs-okta*': {'all': ['Okta']},
'packetbeat-*': {'all': ['Network']},
'winlogbeat-*': {'all': ['Windows']}
}
for rule in rules:
rule_tags = rule.contents.get('tags', [])
indexes = rule.contents.get('index', [])
error_msg = f'{rule.id} - {rule.name} -> Missing tags:\nActual tags: {", ".join(rule_tags)}'
consolidated_optional_tags = []
is_missing_any_tags = False
missing_required_tags = set()
if 'Elastic' not in rule_tags:
missing_required_tags.add('Elastic')
for index in indexes:
expected_tags = required_tags_map.get(index, {})
expected_all = expected_tags.get('all', [])
expected_any = expected_tags.get('any', [])
existing_any_tags = [t for t in rule_tags if t in expected_any]
if expected_any:
# consolidate optional any tags which are not in use
consolidated_optional_tags.extend(t for t in expected_any if t not in existing_any_tags)
missing_required_tags.update(set(expected_all).difference(set(rule_tags)))
is_missing_any_tags = expected_any and not set(expected_any) & set(existing_any_tags)
consolidated_optional_tags = [t for t in consolidated_optional_tags if t not in missing_required_tags]
error_msg += f'\nMissing all of: {", ".join(missing_required_tags)}' if missing_required_tags else ''
error_msg += f'\nMissing any of: {", " .join(consolidated_optional_tags)}' if is_missing_any_tags else ''
if missing_required_tags or is_missing_any_tags:
self.fail(error_msg)
def test_ecs_and_beats_opt_in_not_latest_only(self):
"""Test that explicitly defined opt-in validation is not only the latest versions to avoid stale tests."""
rules = rule_loader.load_rules().values()
for rule in rules:
beats_version = rule.metadata.get('beats_version')
ecs_versions = rule.metadata.get('ecs_versions', [])
latest_beats = str(beats.get_max_version())
latest_ecs = ecs.get_max_version()
error_prefix = f'{rule.id} - {rule.name} ->'
error_msg = f'{error_prefix} it is unnecessary to define the current latest beats version: {latest_beats}'
self.assertNotEqual(latest_beats, beats_version, error_msg)
if len(ecs_versions) == 1:
error_msg = f'{error_prefix} it is unnecessary to define the current latest ecs version if only ' \
f'one version is specified: {latest_ecs}'
self.assertNotIn(latest_ecs, ecs_versions, error_msg)
def test_timeline_has_title(self):
"""Ensure rules with timelines have a corresponding title."""
for rule in rule_loader.load_rules().values():
rule_str = f'{rule.id} - {rule.name}'
timeline_id = rule.contents.get('timeline_id')
timeline_title = rule.contents.get('timeline_title')
if (timeline_title or timeline_id) and not (timeline_title and timeline_id):
missing_err = f'{rule_str} -> timeline "title" and "id" required when timelines are defined'
self.fail(missing_err)
if timeline_id:
unknown_id = f'{rule_str} -> Unknown timeline_id: {timeline_id}.'
unknown_id += f' replace with {", ".join(self.TITLES)} or update this unit test with acceptable ids'
self.assertIn(timeline_id, list(self.TITLES), unknown_id)
unknown_title = f'{rule_str} -> unknown timeline_title: {timeline_title}'
unknown_title += f' replace with {", ".join(self.TITLES.values())}'
unknown_title += ' or update this unit test with acceptable titles'
self.assertEqual(timeline_title, self.TITLES[timeline_id], )
def test_rule_file_names_by_tactic(self):
"""Test to ensure rule files have the primary tactic prepended to the filename."""
rules = rule_loader.load_rules().values()
for rule in rules:
rule_path = Path(rule.path).resolve()
filename = rule_path.name
if rule_path.parent.name == 'ml':
continue
threat = rule.contents.get('threat', [])
authors = rule.contents.get('author', [])
if threat and 'Elastic' in authors:
primary_tactic = threat[0]['tactic']['name']
tactic_str = primary_tactic.lower().replace(' ', '_')
error_msg = 'filename does not start with the primary tactic - update the tactic or the rule filename'
self.assertEqual(tactic_str, filename[:len(tactic_str)], f'{rule.id} - {rule.name} -> {error_msg}')
def test_technique_deprecations(self):
"""Check and warn for use of any ATT&CK techniques that have been deprecated."""
deprecated = {} # {technique: rules_using_them
rules = rule_loader.load_rules().values()
for rule in rules:
rule_info = f'{rule.id} - {rule.name}'
threat_mapping = rule.contents.get('threat')
if threat_mapping:
for entry in threat_mapping:
techniques = entry.get('technique', [])
for technique in techniques:
if technique['id'] in attack.revoked:
deprecated.setdefault(technique['id'], [])
deprecated[technique['id']].append(rule_info)
if deprecated:
deprecated_str = json.dumps(deprecated, indent=2, sort_keys=True)
mitre_url = 'https://attack.mitre.org/resources/updates/'
warning_str = f'The following rules are using deprecated ATT&CK techniques ({mitre_url}):\n{deprecated_str}'
warnings.warn(warning_str)
class TestMappings(unittest.TestCase):
"""Test that all rules appropriately match against expected data sets."""
FP_FILES = get_fp_data_files()
RULES = rule_loader.load_rules().values()
def evaluate(self, documents, rule, expected, msg):
"""KQL engine to evaluate."""
filtered = evaluate(rule, documents)
self.assertEqual(expected, len(filtered), msg)
return filtered
def test_true_positives(self):
"""Test that expected results return against true positives."""
mismatched_ecs = []
mappings = load_etc_dump('rule-mapping.yml')
for rule in rule_loader.get_production_rules():
if isinstance(rule.contents.data, KQLRuleData):
if rule.id not in mappings:
continue
mapping = mappings[rule.id]
expected = mapping['count']
sources = mapping.get('sources')
rta_file = mapping['rta_name']
# ensure sources is defined and not empty; schema allows it to not be set since 'pending' bypasses
self.assertTrue(
sources, 'No sources defined for: {} - {} '.format(
rule.id, rule.name))
msg = 'Expected TP results did not match for: {} - {}'.format(
rule.id, rule.name)
data_files = [
get_data_files('true_positives', rta_file).get(s)
for s in sources
]
data_file = combine_sources(*data_files)
results = self.evaluate(data_file, rule, expected, msg)
ecs_versions = set(
[r.get('ecs', {}).get('version') for r in results])
rule_ecs = set(rule.metadata.get('ecs_version').copy())
if not ecs_versions & rule_ecs:
msg = '{} - {} ecs_versions ({}) not in source data versions ({})'.format(
rule.id, rule.name, ', '.join(rule_ecs),
', '.join(ecs_versions))
mismatched_ecs.append(msg)
if mismatched_ecs:
msg = 'Rules detected with source data from ecs versions not listed within the rule: \n{}'.format(
'\n'.join(mismatched_ecs))
warnings.warn(msg)
def test_false_positives(self):
"""Test that expected results return against false positives."""
for rule in rule_loader.get_production_rules():
if isinstance(rule.contents.data, KQLRuleData):
for fp_name, merged_data in get_fp_data_files().items():
msg = 'Unexpected FP match for: {} - {}, against: {}'.format(
rule.id, rule.name, fp_name)
self.evaluate(copy.deepcopy(merged_data), rule, 0, msg)
def test_rule_loading(self):
"""Ensure that all rule queries have ecs version."""
rule_loader.load_rules().values()
def test_rule_loading(self):
"""Ensure that all rules validate."""
rule_loader.load_rules().values()
def setUpClass(cls):
cls.rule_files = rule_loader.load_rule_files(verbose=False)
cls.rule_lookup = rule_loader.load_rules(verbose=False)
cls.rules = cls.rule_lookup.values()
cls.production_rules = rule_loader.get_production_rules()
def test_package_summary(self):
"""Test the generation of the package summary."""
rules = list(rule_loader.load_rules().values())
package = Package(rules, 'test-package')
changed_rules, new_rules = package.bump_versions(save_changes=False)
package.generate_summary(changed_rules, new_rules)
