def test_rule_file_names_by_tactic(self): """Test to ensure rule files have the primary tactic prepended to the filename.""" rules = rule_loader.load_rules().values() bad_name_rules = [] for rule in rules: rule_path = Path(rule.path).resolve() filename = rule_path.name if rule_path.parent.name == 'ml': continue threat = rule.contents.get('threat', []) authors = rule.contents.get('author', []) if threat and 'Elastic' in authors: primary_tactic = threat[0]['tactic']['name'] tactic_str = primary_tactic.lower().replace(' ', '_') if tactic_str != filename[:len(tactic_str)]: bad_name_rules.append( f'{rule.id} - {Path(rule.path).name} -> expected: {tactic_str}' ) if bad_name_rules: error_msg = 'filename does not start with the primary tactic - update the tactic or the rule filename' rule_err_str = '\n'.join(bad_name_rules) self.fail(f'{error_msg}:\n{rule_err_str}')
def test_technique_deprecations(self): """Check for use of any ATT&CK techniques that have been deprecated.""" replacement_map = attack.techniques_redirect_map revoked = list(attack.revoked) deprecated = list(attack.deprecated) rules = rule_loader.load_rules().values() for rule in rules: revoked_techniques = {} rule_info = f'{rule.id} - {rule.name}' threat_mapping = rule.contents.get('threat') if threat_mapping: for entry in threat_mapping: techniques = entry.get('technique', []) for technique in techniques: if technique['id'] in revoked + deprecated: revoked_techniques[ technique['id']] = replacement_map.get( technique['id'], 'DEPRECATED - DO NOT USE') if revoked_techniques: old_new_mapping = "\n".join( f'Actual: {k} -> Expected {v}' for k, v in revoked_techniques.items()) self.fail( f'{rule_info} -> Using deprecated ATT&CK techniques: \n{old_new_mapping}' )
def test_rule_versioning(self): """Test that all rules are properly versioned and tracked""" self.maxDiff = None rules = rule_loader.load_rules().values() original_hashes = [] post_bump_hashes = [] # test that no rules have versions defined for rule in rules: self.assertGreaterEqual( rule.contents.autobumped_version, 1, '{} - {}: version is not being set in package') original_hashes.append(rule.contents.sha256()) package = Package(rules, 'test-package') # test that all rules have versions defined # package.bump_versions(save_changes=False) for rule in package.rules: self.assertGreaterEqual( rule.contents.autobumped_version, 1, '{} - {}: version is not being set in package') # test that rules validate with version for rule in package.rules: post_bump_hashes.append(rule.contents.sha256()) # test that no hashes changed as a result of the version bumps self.assertListEqual(original_hashes, post_bump_hashes, 'Version bumping modified the hash of a rule')
def test_casing_and_spacing(self): """Ensure consistent and expected casing for controlled tags.""" rules = rule_loader.load_rules().values() def normalize(s): return ''.join(s.lower().split()) expected_tags = [ 'APM', 'AWS', 'Asset Visibility', 'Azure', 'Configuration Audit', 'Continuous Monitoring', 'Data Protection', 'Elastic', 'Endpoint Security', 'GCP', 'Identity and Access', 'Linux', 'Logging', 'ML', 'macOS', 'Monitoring', 'Network', 'Okta', 'Packetbeat', 'Post-Execution', 'SecOps', 'Windows' ] expected_case = {normalize(t): t for t in expected_tags} for rule in rules: rule_tags = rule.contents.get('tags') if rule_tags: invalid_tags = { t: expected_case[normalize(t)] for t in rule_tags if normalize(t) in list(expected_case) and t != expected_case[normalize(t)] } if invalid_tags: error_msg = f'{rule.id} - {rule.name} -> Invalid casing for expected tags\n' error_msg += f'Actual tags: {", ".join(invalid_tags)}\n' error_msg += f'Expected tags: {", ".join(invalid_tags.values())}' self.fail(error_msg)
def test_format_of_all_rules(self): """Test all rules.""" rules = rule_loader.load_rules().values() for rule in rules: self.compare_formatted(rule.rule_format(formatted_query=False), callback=nested_normalize)
def test_format_of_all_rules(self): """Test all rules.""" rules = rule_loader.load_rules().values() for rule in rules: is_eql_rule = rule.type == 'eql' self.compare_formatted(rule.rule_format(formatted_query=False), callback=nested_normalize, kwargs={'eql_rule': is_eql_rule})
def test_tactic_to_technique_correlations(self): """Ensure rule threat info is properly related to a single tactic and technique.""" rules = rule_loader.load_rules().values() for rule in rules: threat_mapping = rule.contents.get('threat') if threat_mapping: for entry in threat_mapping: tactic = entry.get('tactic') techniques = entry.get('technique', []) # tactic expected_tactic = attack.tactics_map[tactic['name']] self.assertEqual(expected_tactic, tactic['id'], f'ATT&CK tactic mapping error for rule: {rule.id} - {rule.name} ->\n' f'expected: {expected_tactic} for {tactic["name"]}\n' f'actual: {tactic["id"]}') tactic_reference_id = tactic['reference'].rstrip('/').split('/')[-1] self.assertEqual(tactic['id'], tactic_reference_id, f'ATT&CK tactic mapping error for rule: {rule.id} - {rule.name} ->\n' f'tactic ID {tactic["id"]} does not match the reference URL ID ' f'{tactic["reference"]}') # techniques for technique in techniques: expected_technique = attack.technique_lookup[technique['id']]['name'] self.assertEqual(expected_technique, technique['name'], f'ATT&CK technique mapping error for rule: {rule.id} - {rule.name} ->\n' f'expected: {expected_technique} for {technique["id"]}\n' f'actual: {technique["name"]}') technique_reference_id = technique['reference'].rstrip('/').split('/')[-1] self.assertEqual(technique['id'], technique_reference_id, f'ATT&CK technique mapping error for rule: {rule.id} - {rule.name} ->\n' f'technique ID {technique["id"]} does not match the reference URL ID ' f'{technique["reference"]}') # sub-techniques sub_techniques = technique.get('subtechnique') if sub_techniques: for sub_technique in sub_techniques: expected_sub_technique = attack.technique_lookup[sub_technique['id']]['name'] self.assertEqual(expected_sub_technique, sub_technique['name'], f'ATT&CK sub-technique mapping error for rule: {rule.id} - {rule.name} ->\n' # noqa: E501 f'expected: {expected_sub_technique} for {sub_technique["id"]}\n' f'actual: {sub_technique["name"]}') sub_technique_reference_id = '.'.join( sub_technique['reference'].rstrip('/').split('/')[-2:]) self.assertEqual(sub_technique['id'], sub_technique_reference_id, f'ATT&CK sub-technique mapping error for rule: {rule.id} - {rule.name} ->\n' # noqa: E501 f'sub-technique ID {sub_technique["id"]} does not match the reference URL ID ' # noqa: E501 f'{sub_technique["reference"]}')
def test_duplicated_tactics(self): """Check that a tactic is only defined once.""" rules = rule_loader.load_rules().values() for rule in rules: rule_info = f'{rule.id} - {rule.name}' threat_mapping = rule.contents.get('threat', []) tactics = [t['tactic']['name'] for t in threat_mapping] duplicates = sorted(set(t for t in tactics if tactics.count(t) > 1)) if duplicates: self.fail(f'{rule_info} -> duplicate tactics defined for {duplicates}. ' f'Flatten to a single entry per tactic')
def test_required_tags(self): """Test that expected tags are present within rules.""" rules = rule_loader.load_rules().values() # indexes considered; only those with obvious relationships included # 'apm-*-transaction*', 'auditbeat-*', 'endgame-*', 'filebeat-*', 'logs-*', 'logs-aws*', # 'logs-endpoint.alerts-*', 'logs-endpoint.events.*', 'logs-okta*', 'packetbeat-*', 'winlogbeat-*' required_tags_map = { 'apm-*-transaction*': {'all': ['APM']}, 'auditbeat-*': {'any': ['Windows', 'macOS', 'Linux']}, 'endgame-*': {'all': ['Endpoint Security']}, 'logs-aws*': {'all': ['AWS']}, 'logs-endpoint.alerts-*': {'all': ['Endpoint Security']}, 'logs-endpoint.events.*': {'any': ['Windows', 'macOS', 'Linux', 'Host']}, 'logs-okta*': {'all': ['Okta']}, 'packetbeat-*': {'all': ['Network']}, 'winlogbeat-*': {'all': ['Windows']} } for rule in rules: rule_tags = rule.contents.get('tags', []) indexes = rule.contents.get('index', []) error_msg = f'{rule.id} - {rule.name} -> Missing tags:\nActual tags: {", ".join(rule_tags)}' consolidated_optional_tags = [] is_missing_any_tags = False missing_required_tags = set() if 'Elastic' not in rule_tags: missing_required_tags.add('Elastic') for index in indexes: expected_tags = required_tags_map.get(index, {}) expected_all = expected_tags.get('all', []) expected_any = expected_tags.get('any', []) existing_any_tags = [t for t in rule_tags if t in expected_any] if expected_any: # consolidate optional any tags which are not in use consolidated_optional_tags.extend(t for t in expected_any if t not in existing_any_tags) missing_required_tags.update(set(expected_all).difference(set(rule_tags))) is_missing_any_tags = expected_any and not set(expected_any) & set(existing_any_tags) consolidated_optional_tags = [t for t in consolidated_optional_tags if t not in missing_required_tags] error_msg += f'\nMissing all of: {", ".join(missing_required_tags)}' if missing_required_tags else '' error_msg += f'\nMissing any of: {", " .join(consolidated_optional_tags)}' if is_missing_any_tags else '' if missing_required_tags or is_missing_any_tags: self.fail(error_msg)
def test_ecs_and_beats_opt_in_not_latest_only(self): """Test that explicitly defined opt-in validation is not only the latest versions to avoid stale tests.""" rules = rule_loader.load_rules().values() for rule in rules: beats_version = rule.metadata.get('beats_version') ecs_versions = rule.metadata.get('ecs_versions', []) latest_beats = str(beats.get_max_version()) latest_ecs = ecs.get_max_version() error_prefix = f'{rule.id} - {rule.name} ->' error_msg = f'{error_prefix} it is unnecessary to define the current latest beats version: {latest_beats}' self.assertNotEqual(latest_beats, beats_version, error_msg) if len(ecs_versions) == 1: error_msg = f'{error_prefix} it is unnecessary to define the current latest ecs version if only ' \ f'one version is specified: {latest_ecs}' self.assertNotIn(latest_ecs, ecs_versions, error_msg)
def test_timeline_has_title(self): """Ensure rules with timelines have a corresponding title.""" for rule in rule_loader.load_rules().values(): rule_str = f'{rule.id} - {rule.name}' timeline_id = rule.contents.get('timeline_id') timeline_title = rule.contents.get('timeline_title') if (timeline_title or timeline_id) and not (timeline_title and timeline_id): missing_err = f'{rule_str} -> timeline "title" and "id" required when timelines are defined' self.fail(missing_err) if timeline_id: unknown_id = f'{rule_str} -> Unknown timeline_id: {timeline_id}.' unknown_id += f' replace with {", ".join(self.TITLES)} or update this unit test with acceptable ids' self.assertIn(timeline_id, list(self.TITLES), unknown_id) unknown_title = f'{rule_str} -> unknown timeline_title: {timeline_title}' unknown_title += f' replace with {", ".join(self.TITLES.values())}' unknown_title += ' or update this unit test with acceptable titles' self.assertEqual(timeline_title, self.TITLES[timeline_id], )
def test_rule_file_names_by_tactic(self): """Test to ensure rule files have the primary tactic prepended to the filename.""" rules = rule_loader.load_rules().values() for rule in rules: rule_path = Path(rule.path).resolve() filename = rule_path.name if rule_path.parent.name == 'ml': continue threat = rule.contents.get('threat', []) authors = rule.contents.get('author', []) if threat and 'Elastic' in authors: primary_tactic = threat[0]['tactic']['name'] tactic_str = primary_tactic.lower().replace(' ', '_') error_msg = 'filename does not start with the primary tactic - update the tactic or the rule filename' self.assertEqual(tactic_str, filename[:len(tactic_str)], f'{rule.id} - {rule.name} -> {error_msg}')
def test_technique_deprecations(self): """Check and warn for use of any ATT&CK techniques that have been deprecated.""" deprecated = {} # {technique: rules_using_them rules = rule_loader.load_rules().values() for rule in rules: rule_info = f'{rule.id} - {rule.name}' threat_mapping = rule.contents.get('threat') if threat_mapping: for entry in threat_mapping: techniques = entry.get('technique', []) for technique in techniques: if technique['id'] in attack.revoked: deprecated.setdefault(technique['id'], []) deprecated[technique['id']].append(rule_info) if deprecated: deprecated_str = json.dumps(deprecated, indent=2, sort_keys=True) mitre_url = 'https://attack.mitre.org/resources/updates/' warning_str = f'The following rules are using deprecated ATT&CK techniques ({mitre_url}):\n{deprecated_str}' warnings.warn(warning_str)
class TestMappings(unittest.TestCase): """Test that all rules appropriately match against expected data sets.""" FP_FILES = get_fp_data_files() RULES = rule_loader.load_rules().values() def evaluate(self, documents, rule, expected, msg): """KQL engine to evaluate.""" filtered = evaluate(rule, documents) self.assertEqual(expected, len(filtered), msg) return filtered def test_true_positives(self): """Test that expected results return against true positives.""" mismatched_ecs = [] mappings = load_etc_dump('rule-mapping.yml') for rule in rule_loader.get_production_rules(): if isinstance(rule.contents.data, KQLRuleData): if rule.id not in mappings: continue mapping = mappings[rule.id] expected = mapping['count'] sources = mapping.get('sources') rta_file = mapping['rta_name'] # ensure sources is defined and not empty; schema allows it to not be set since 'pending' bypasses self.assertTrue( sources, 'No sources defined for: {} - {} '.format( rule.id, rule.name)) msg = 'Expected TP results did not match for: {} - {}'.format( rule.id, rule.name) data_files = [ get_data_files('true_positives', rta_file).get(s) for s in sources ] data_file = combine_sources(*data_files) results = self.evaluate(data_file, rule, expected, msg) ecs_versions = set( [r.get('ecs', {}).get('version') for r in results]) rule_ecs = set(rule.metadata.get('ecs_version').copy()) if not ecs_versions & rule_ecs: msg = '{} - {} ecs_versions ({}) not in source data versions ({})'.format( rule.id, rule.name, ', '.join(rule_ecs), ', '.join(ecs_versions)) mismatched_ecs.append(msg) if mismatched_ecs: msg = 'Rules detected with source data from ecs versions not listed within the rule: \n{}'.format( '\n'.join(mismatched_ecs)) warnings.warn(msg) def test_false_positives(self): """Test that expected results return against false positives.""" for rule in rule_loader.get_production_rules(): if isinstance(rule.contents.data, KQLRuleData): for fp_name, merged_data in get_fp_data_files().items(): msg = 'Unexpected FP match for: {} - {}, against: {}'.format( rule.id, rule.name, fp_name) self.evaluate(copy.deepcopy(merged_data), rule, 0, msg)
def test_rule_loading(self): """Ensure that all rule queries have ecs version.""" rule_loader.load_rules().values()
def test_rule_loading(self): """Ensure that all rules validate.""" rule_loader.load_rules().values()
def setUpClass(cls): cls.rule_files = rule_loader.load_rule_files(verbose=False) cls.rule_lookup = rule_loader.load_rules(verbose=False) cls.rules = cls.rule_lookup.values() cls.production_rules = rule_loader.get_production_rules()
def test_package_summary(self): """Test the generation of the package summary.""" rules = list(rule_loader.load_rules().values()) package = Package(rules, 'test-package') changed_rules, new_rules = package.bump_versions(save_changes=False) package.generate_summary(changed_rules, new_rules)