def _check_health_techniques(filename, technique_content, health_is_called):
"""
Check on errors in the provided technique administration YAML file.
:param filename: YAML file location
:param technique_content: content of the YAML file in a list of dicts
:param health_is_called: boolean that specifies if detailed errors in the file will be printed to stdout
:return:
"""
from generic import load_techniques
has_error = False
# create a list of ATT&CK technique IDs and check for duplicates
tech_ids = list(map(lambda x: x['technique_id'], technique_content['techniques']))
tech_dup = set()
for tech in tech_ids:
if tech not in tech_dup:
tech_dup.add(tech)
else:
has_error = _print_error_msg('[!] Duplicate technique ID: ' + tech, health_is_called)
# check if the technique has a valid format
if not REGEX_YAML_TECHNIQUE_ID_FORMAT.match(tech):
has_error = _print_error_msg('[!] Invalid technique ID: ' + tech, health_is_called)
all_applicable_to = set()
techniques = load_techniques(filename)
for tech, v in techniques[0].items():
for obj_type in ['detection', 'visibility']:
if obj_type not in v:
has_error = _print_error_msg('[!] Technique ID: ' + tech + ' is MISSING a key-value pair: ' + obj_type, health_is_called)
else:
for obj in v[obj_type]:
obj_keys = ['applicable_to', 'comment', 'score_logbook']
obj_keys_list = ['applicable_to']
if obj_type == 'detection':
obj_keys.append('location')
obj_keys_list.append('location')
for okey in obj_keys:
if okey not in obj:
has_error = _print_error_msg('[!] Technique ID: ' + tech + ' is MISSING a key-value pair in \'' + obj_type + '\': ' + okey, health_is_called)
for okey in obj_keys_list:
if okey in obj:
if not isinstance(obj[okey], list):
has_error = _print_error_msg('[!] Technique ID: ' + tech + ' the key-value pair \'' + okey + '\' in \'' + obj_type + '\' is NOT a list', health_is_called)
health = _check_health_score_object(obj, obj_type, tech, health_is_called)
has_error = _update_health_state(has_error, health)
if 'applicable_to' in obj and isinstance(obj['applicable_to'], list):
all_applicable_to.update(obj['applicable_to'])
# get values within the key-value pair 'applicable_to' and 'location' which are a very close match
similar = set()
for i1 in all_applicable_to:
for i2 in all_applicable_to:
match_value = SequenceMatcher(None, i1, i2).ratio()
if match_value > 0.8 and match_value != 1:
similar.add(i1)
similar.add(i2)
if len(similar) > 0:
has_error = _print_error_msg('[!] There are values in the key-value pairs for \'applicable_to\' which are very similar. Correct where necessary:', health_is_called)
for s in similar:
_print_error_msg(' - ' + s, health_is_called)
if has_error and not health_is_called:
print(HEALTH_ERROR_TXT + filename)
_update_health_state_cache(filename, has_error)
def _check_health_techniques(filename, technique_content, health_is_called):
"""
Check on errors in the provided technique administration YAML file.
:param filename: YAML file location
:param technique_content: content of the YAML file in a list of dicts
:param health_is_called: boolean that specifies if detailed errors in the file will be printed to stdout
:return:
"""
from generic import load_techniques
has_error = False
platform = technique_content.get('platform', None)
if platform != 'all' and platform != ['all']:
if isinstance(platform, str):
platform = [platform]
if platform is None or len(platform) == 0 or platform == '':
platform = ['empty']
for p in platform:
if p.lower() not in PLATFORMS.keys():
has_error = _print_error_msg(
'[!] EMPTY or INVALID value for \'platform\' within the data source admin. '
'file: %s (should be value(s) of: [%s] or all)' % (p, ', '.join(list(PLATFORMS.values()))),
health_is_called)
# create a list of ATT&CK technique IDs and check for duplicates
tech_ids = list(map(lambda x: x['technique_id'], technique_content['techniques']))
tech_dup = set()
for tech in tech_ids:
if tech not in tech_dup:
tech_dup.add(tech)
else:
has_error = _print_error_msg('[!] Duplicate technique ID: ' + tech, health_is_called)
# check if the technique has a valid format
if not REGEX_YAML_TECHNIQUE_ID_FORMAT.match(tech):
has_error = _print_error_msg('[!] Invalid technique ID: ' + tech, health_is_called)
all_applicable_to = set()
techniques = load_techniques(filename)
for tech, v in techniques[0].items():
for obj_type in ['detection', 'visibility']:
if obj_type not in v:
has_error = _print_error_msg('[!] Technique ID: ' + tech + ' is MISSING a key-value pair: ' + obj_type, health_is_called)
else:
for obj in v[obj_type]:
obj_keys = ['applicable_to', 'comment', 'score_logbook']
obj_keys_list = ['applicable_to']
obj_keys_not_none = []
obj_keys_not_none.append('applicable_to')
if obj_type == 'detection':
obj_keys.append('location')
obj_keys_list.append('location')
obj_keys_not_none.append('location')
for okey in obj_keys:
if okey not in obj:
has_error = _print_error_msg('[!] Technique ID: ' + tech +
' is MISSING a key-value pair in \'' + obj_type + '\': ' + okey, health_is_called)
for okey in obj_keys_list:
if okey in obj:
if not isinstance(obj[okey], list):
has_error = _print_error_msg('[!] Technique ID: ' + tech + ' the key-value pair \'' + okey +
'\' in \'' + obj_type + '\' is NOT a list', health_is_called)
for okey in obj_keys_not_none:
if okey in obj:
none_count = 0
for item in obj[okey]:
if item is None:
none_count += 1
if none_count == 1:
has_error = _print_error_msg('[!] Technique ID: ' + tech + ' the key-value pair \'' + okey + '\' in \'' +
obj_type + '\' has an EMPTY value (an empty string is allowed: \'\')', health_is_called)
elif none_count > 1:
has_error = _print_error_msg('[!] Technique ID: ' + tech + ' the key-value pair \'' + okey + '\' in \'' + obj_type +
'\' has multiple EMPTY values (an empty string is allowed: \'\')', health_is_called)
health = _check_health_score_object(obj, obj_type, tech, health_is_called)
has_error = _update_health_state(has_error, health)
if 'applicable_to' in obj and isinstance(obj['applicable_to'], list):
all_applicable_to.update(obj['applicable_to'])
# get values within the key-value pair 'applicable_to' and 'location' which are a very close match
similar = set()
for i1 in all_applicable_to:
for i2 in all_applicable_to:
match_value = SequenceMatcher(None, i1, i2).ratio()
if match_value > 0.8 and match_value != 1:
similar.add(i1)
similar.add(i2)
if len(similar) > 0:
has_error = _print_error_msg(
'[!] There are values in the key-value pairs for \'applicable_to\' which are very similar. Correct where necessary:', health_is_called)
for s in similar:
_print_error_msg(' - ' + s, health_is_called)
if has_error and not health_is_called:
print(HEALTH_ERROR_TXT + filename)
_update_health_state_cache(filename, has_error)
def _check_health_techniques(filename, technique_content, health_is_called):
"""
Check on errors in the provided technique administration YAML file.
:param filename: YAML file location
:param technique_content: content of the YAML file in a list of dicts
:param health_is_called: boolean that specifies if detailed errors in the file will be printed to stdout
:return:
"""
from generic import load_techniques
has_error = False
# Check domain attribute (is optional):
domain = 'enterprise-attack'
if 'domain' in technique_content:
if not technique_content['domain'].lower() in DETTECT_DOMAIN_SUPPORT:
has_error = _print_error_msg(
'[!] INVALID domain value in technique administration file: %s. Must be one of: %s'
% (technique_content['domain'],
', '.join(DETTECT_DOMAIN_SUPPORT)), health_is_called)
else:
domain = technique_content['domain']
platform = technique_content.get('platform', None)
supported_platforms = PLATFORMS_ENTERPRISE if domain == 'enterprise-attack' else PLATFORMS_ICS
if platform != 'all' and platform != ['all']:
if isinstance(platform, str):
platform = [platform]
if platform is None or len(platform) == 0 or platform == '':
platform = ['empty']
for p in platform:
if p.lower() not in supported_platforms.keys():
has_error = _print_error_msg(
'[!] EMPTY or INVALID value for \'platform\' within the technique administration '
'file: %s (should be value(s) of: [%s] or all)' %
(p, ', '.join(list(supported_platforms.values()))),
health_is_called)
# create a list of ATT&CK technique IDs and check for duplicates
tech_ids = list(
map(lambda x: x['technique_id'], technique_content['techniques']))
tech_dup = set()
for tech in tech_ids:
if tech not in tech_dup:
tech_dup.add(tech)
else:
has_error = _print_error_msg('[!] Duplicate technique ID: ' + tech,
health_is_called)
# check if the technique has a valid format
if not REGEX_YAML_TECHNIQUE_ID_FORMAT.match(tech):
has_error = _print_error_msg('[!] Invalid technique ID: ' + tech,
health_is_called)
all_applicable_to = set()
techniques = load_techniques(filename)
for tech, v in techniques[0].items():
for obj_type in ['detection', 'visibility']:
if obj_type not in v:
has_error = _print_error_msg(
'[!] Technique ID: ' + tech +
' is MISSING a key-value pair: ' + obj_type,
health_is_called)
else:
obj_applicable_to = []
for obj in v[obj_type]:
obj_keys = ['applicable_to', 'comment', 'score_logbook']
obj_keys_list = ['applicable_to']
obj_keys_not_none = ['applicable_to']
if obj_type == 'detection':
obj_keys.append('location')
obj_keys_list.append('location')
obj_keys_not_none.append('location')
for okey in obj_keys:
if okey not in obj:
has_error = _print_error_msg(
'[!] Technique ID: ' + tech +
' is MISSING a key-value pair in \'' +
obj_type + '\': ' + okey, health_is_called)
for okey in obj_keys_list:
if okey in obj:
if not isinstance(obj[okey], list):
has_error = _print_error_msg(
'[!] Technique ID: ' + tech +
' the key-value pair \'' + okey +
'\' in \'' + obj_type + '\' is NOT a list',
health_is_called)
for okey in obj_keys_not_none:
if okey in obj and isinstance(obj[okey], list):
none_count = 0
for item in obj[okey]:
if item is None:
none_count += 1
if none_count == 1:
has_error = _print_error_msg(
'[!] Technique ID: ' + tech +
' the key-value pair \'' + okey +
'\' in \'' + obj_type +
'\' has an EMPTY value (an empty string is allowed: \'\')',
health_is_called)
elif none_count > 1:
has_error = _print_error_msg(
'[!] Technique ID: ' + tech +
' the key-value pair \'' + okey +
'\' in \'' + obj_type +
'\' has multiple EMPTY values (an empty string is allowed: \'\')',
health_is_called)
health = _check_health_score_object(
obj, obj_type, tech, health_is_called)
has_error = _update_health_state(has_error, health)
if 'applicable_to' in obj and isinstance(
obj['applicable_to'], list):
all_applicable_to.update(obj['applicable_to'])
obj_applicable_to.extend(obj['applicable_to'])
if obj_type == 'visibility' and len(
set(obj['applicable_to'])) > 1 and 'all' in [
a.lower() for a in obj['applicable_to']
if a is not None
]:
has_error = _print_error_msg(
'[!] Technique ID: ' + tech +
' the key-value pair \'applicable_to\' in \'' +
obj_type +
'\' has \'all\' as a value that is not exclusively used (\'all\' can not be combined '
+
'with other applicable_to values in a visibility object).',
health_is_called)
if len(obj_applicable_to) > len(set(obj_applicable_to)):
has_error = _print_error_msg(
'[!] Technique ID: ' + tech +
' the key-value pair \'applicable_to\' in \'' +
obj_type +
'\' has DUPLICATE system values (a system can only be part of one '
+
'applicable_to key-value pair within the same technique).',
health_is_called)
has_error = has_error if not _check_for_similar_values(
all_applicable_to, 'applicable_to', health_is_called) else True
if has_error and not health_is_called:
print(HEALTH_ERROR_TXT + filename)
_update_health_state_cache(filename, has_error)