def main(**kwargs):
dn_list = ATCutils.load_yamls(kwargs['dn_path'])
lp_list = ATCutils.load_yamls(kwargs['lp_path'])
ra_list = ATCutils.load_yamls(kwargs['ra_path'])
rp_list = ATCutils.load_yamls(kwargs['rp_path'])
cu_list = ATCutils.load_yamls(ATCconfig.get('customers_directory'))
enrichments_list = ATCutils.load_yamls(kwargs['en_path'])
pivoting = []
analytics = []
result = []
dr_dirs = ATCconfig.get('detection_rules_directories')
print("[*] Iterating through Detection Rules")
# Iterate through alerts and pathes to them
for dr_path in dr_dirs:
alerts, path_to_alerts = ATCutils.load_yamls_with_paths(dr_path)
for alert, path in zip(alerts, path_to_alerts):
if not isinstance(alert.get('tags'), list):
continue
list_of_customers = []
for specific_customer in cu_list:
if alert['title'] in specific_customer[
'detectionrule'] and specific_customer[
'customer_name'] not in list_of_customers:
list_of_customers.append(
specific_customer['customer_name'])
if not isinstance(list_of_customers,
list) or len(list_of_customers) == 0:
list_of_customers = ["None"]
customer = ';'.join(list_of_customers)
threats = [
tag for tag in alert['tags'] if tag.startswith('attack')
]
tactics = [
f'{ta_mapping[threat][1]}: {ta_mapping[threat][0]}'
for threat in threats if threat in ta_mapping.keys()
]
techniques = [
threat for threat in threats if threat.startswith('attack.t')
]
enrichments = [
er for er in enrichments_list
if er['title'] in alert.get('enrichment', [])
]
dn_titles = ATCutils.main_dn_calculatoin_func(path)
alert_dns = [
data for data in dn_list if data['title'] in dn_titles
]
logging_policies = []
for dn in alert_dns:
if 'loggingpolicy' in dn:
# If there are logging policies in DN that we havent added yet - add them
logging_policies.extend([
l for l in lp_list if l['title'] in dn['loggingpolicy']
and l not in logging_policies
])
# If there are no logging policices at all - make an empty one just to make one row in csv
if not isinstance(logging_policies,
list) or len(logging_policies) == 0:
logging_policies = [{
'title': "-",
'eventID': [
-1,
]
}]
for dn in alert_dns:
pivot = [
dn['category'], dn['platform'], dn['type'], dn['channel'],
dn['provider'], dn['title'], '', ''
]
for tactic in tactics:
for technique in techniques:
technique_name = technique.replace('attack.t', 'T') + ': ' +\
ATCutils.get_attack_technique_name_by_id(
technique.replace('attack.', ''))
for lp in logging_policies:
rps = [
rp for rp in rp_list if technique in rp['tags']
or tactic in rp['tags']
]
if len(rps) < 1:
rps = [{'title': '-'}]
for rp in rps:
ras_buf = []
[
ras_buf.extend(l) for l in rp.values()
if isinstance(l, list)
]
ras = [
ra for ra in ras_buf if ra.startswith('RA')
]
if len(ras) < 1:
ras = ['title']
#if len(rp) > 1:
#todo
for ra in ras:
lp['title'] = lp['title'].replace('\n', '')
result.append([
customer, tactic, technique_name,
alert['title'], dn['category'],
dn['platform'], dn['type'],
dn['channel'], dn['provider'],
dn['title'], lp['title'], '', '',
rp['title'], ra
])
# pivoting.append(pivot)
for field in dn['fields']:
analytics.append([field] + pivot)
for er in enrichments:
for dn in [
dnn for dnn in dn_list
if dnn['title'] in er.get('data_to_enrich', [])
]:
pivot = [
dn['category'], dn['platform'], dn['type'],
dn['channel'], dn['provider'], dn['title'],
er['title'], ';'.join(er.get('requirements', []))
]
for tactic in tactics:
for technique in techniques:
technique_name = technique.replace('attack.t', 'T') + ': ' + \
ATCutils.get_attack_technique_name_by_id(
technique.replace('attack.', ''))
for lp in logging_policies:
lp['title'] = lp['title'].replace('\n', '')
result.append([
customer, tactic, technique_name,
alert['title'], dn['category'],
dn['platform'], dn['type'], dn['channel'],
dn['provider'], dn['title'], lp['title'],
er['title'],
';'.join(er.get('requirements',
[])), '-', '-'
])
# pivoting.append(pivot)
for field in er['new_fields']:
analytics.append([field] + pivot)
analytics = []
for dn in dn_list:
if 'category' in dn:
dn_category = dn['category']
else:
dn_category = "-"
if 'platform' in dn:
dn_platform = dn['platform']
else:
dn_platform = "-"
if 'type' in dn:
dn_type = dn['type']
else:
dn_type = "-"
if 'channel' in dn:
dn_channel = dn['channel']
else:
dn_channel = "-"
if 'provider' in dn:
dn_provider = dn['provider']
else:
dn_provider = "-"
if 'title' in dn:
dn_title = dn['title']
else:
dn_title = "-"
pivot = [
dn_category, dn_platform, dn_type, dn_channel, dn_provider,
dn_title, '', ''
]
for field in dn['fields']:
analytics.append([field] + pivot)
for er in enrichments_list:
for dn in [
dnn for dnn in dn_list
if dnn['title'] in er.get('data_to_enrich', [])
]:
pivot = [
dn['category'], dn['platform'], dn['type'], dn['channel'],
dn['provider'], dn['title'], er['title'],
';'.join(er.get('requirements', []))
]
for field in er['new_fields']:
analytics.append([field] + pivot)
filename = 'analytics.csv'
exported_analytics_directory = ATCconfig.get(
'exported_analytics_directory')
with open(exported_analytics_directory + '/' + filename, 'w',
newline='') as csvfile:
# maybe need some quoting
alertswriter = csv.writer(csvfile, delimiter=',')
alertswriter.writerow([
'customer', 'tactic', 'technique', 'detection_rule', 'category',
'platform', 'type', 'channel', 'provider', 'data_needed',
'logging policy', 'enrichment', 'enrichment requirements',
'response playbook', 'response action'
])
for row in result:
alertswriter.writerow(row)
print(f'[+] Created {filename}')
filename = 'pivoting.csv'
exported_analytics_directory = ATCconfig.get(
'exported_analytics_directory')
with open(exported_analytics_directory + '/' + filename, 'w',
newline='') as csvfile:
# maybe need some quoting
alertswriter = csv.writer(csvfile, delimiter=',')
alertswriter.writerow([
'field', 'category', 'platform', 'type', 'channel', 'provider',
'data_needed', 'enrichment', 'enrichment requirements'
])
for row in analytics:
alertswriter.writerow(row)
print(f'[+] Created {filename}')
def main(**kwargs):
lp_list = ATCutils.load_yamls(kwargs['lp_path'])
ra_list = ATCutils.load_yamls(kwargs['ra_path'])
rp_list = ATCutils.load_yamls(kwargs['rp_path'])
cu_list = ATCutils.load_yamls(kwargs['cu_path'])
dn_list = ATCutils.load_yamls(kwargs['dn_path'])
enrichments_list = ATCutils.load_yamls(kwargs['en_path'])
_index = {}
dr_dirs = ATCconfig.get('detection_rules_directories')
# Iterate through alerts and pathes to them
for dr_path in dr_dirs:
alerts, path_to_alerts = ATCutils.load_yamls_with_paths(dr_path)
for alert, path in zip(alerts, path_to_alerts):
tactics = []
techniques = []
list_of_customers = []
# raw Sigma rule without fields that are present separately
dr_raw = alert.copy()
fields_to_remove_from_raw_dr = [
'title', 'id', 'status', 'date', 'modified', 'description',
'references', 'author', 'tags', 'logsource', 'falsepositives',
'level'
]
for field in fields_to_remove_from_raw_dr:
dr_raw.pop(field, None)
dr_raw = str(yaml.dump((dr_raw), default_flow_style=False))
for customer in cu_list:
if 'detectionrule' in customer:
if alert['title'] in customer['detectionrule'] and customer[
'customer_name'] not in list_of_customers:
list_of_customers.append(customer['customer_name'])
if not isinstance(list_of_customers,
list) or len(list_of_customers) == 0:
list_of_customers = ["None"]
if isinstance(alert.get('tags'), list):
try:
threats = [
tag for tag in alert['tags']
if tag.startswith('attack')
]
tactics = [
f'{ta_mapping[threat][1]}: {ta_mapping[threat][0]}'
for threat in threats if threat in ta_mapping.keys()
]
except:
pass
try:
threats = [
tag for tag in alert['tags']
if tag.startswith('attack')
]
techniques = [
threat for threat in threats
if threat.startswith('attack.t')
]
except:
pass
enrichments = [
er for er in enrichments_list
if er['title'] in alert.get('enrichment', [{
'title': '-'
}])
]
if len(enrichments) < 1:
enrichments = [{'title': 'not defined'}]
dn_titles = ATCutils.main_dn_calculatoin_func(path)
alert_dns = [
data for data in dn_list if data['title'] in dn_titles
]
if len(alert_dns) < 1:
alert_dns = [{
'category': 'not defined',
'platform': 'not defined',
'provider': 'not defined',
'type': 'not defined',
'channel': 'not defined',
'title': 'not defined',
'loggingpolicy': ['not defined']
}]
logging_policies = []
for dn in alert_dns:
# If there are logging policies in DN that we havent added yet - add them
logging_policies.extend([
l for l in lp_list if l['title'] in dn['loggingpolicy']
and l not in logging_policies
])
# If there are no logging policices at all - make an empty one just to make one row in csv
if not isinstance(logging_policies,
list) or len(logging_policies) == 0:
logging_policies = [{
'title': "not defined",
'eventID': [
-1,
]
}]
# we use date of creation to have timelines of progress
if 'date' in alert:
try:
date_created = datetime.datetime.strptime(
alert['date'], '%Y/%m/%d').isoformat()
except:
pass
if not date_created:
try:
# in case somebody mixed up month and date, like in "Detection of SafetyKatz"
date_created = datetime.datetime.strptime(
alert['date'], '%Y/%d/%m').isoformat()
except:
# temporary solution to avoid errors. all DRs must have date of creation
print(
'date in ' + alert['date'] + ' is not in ' +
'%Y/%m/%d and %Y/%d/%m formats. Set up current date and time'
)
date_created = datetime.datetime.now().isoformat()
else:
# temporary solution to avoid errors. all internal DRs must have date of creation
#date_created = datetime.datetime.now().isoformat()
date_created = '2019-03-01T22:50:37.587060'
# we use date of creation to have timelines of progress
if 'modified' in alert:
try:
date_modified = datetime.datetime.strptime(
alert['modified'], '%Y/%m/%d').isoformat()
except:
pass
if not date_modified:
try:
# in case somebody mixed up month and date, like in "Detection of SafetyKatz"
date_modified = datetime.datetime.strptime(
alert['modified'], '%Y/%d/%m').isoformat()
except:
date_modified = None
else:
# temporary solution to avoid errors. all internal DRs must have date of creation
#date_created = datetime.datetime.now().isoformat()
date_modified = None
# we create index document based on DR title, which is unique (supposed to be).
# this way we will update existing documents in case of changes,
# and create new ones when new DRs will be developed
# update: well, better recreate everything from the scratch all the time.
# if name of DR will change, we will have doubles in index. todo
document_id = hash(alert['title'])
list_of_tactics = []
list_of_techniques = []
if tactics:
for tactic in tactics:
if tactic not in list_of_tactics:
list_of_tactics.append(tactic)
else:
list_of_tactics = ['not defined']
if techniques:
for technique in techniques:
technique_name = technique.replace('attack.t', 'T') + ': ' +\
ATCutils.get_attack_technique_name_by_id(technique.replace('attack.', ''))
if technique not in list_of_techniques:
list_of_techniques.append(technique_name)
else:
list_of_techniques = ['not defined']
dr_title = alert['title']
dn_titles = []
dn_categories = []
dn_platforms = []
dn_types = []
dn_channels = []
dn_providers = []
lp_titles = []
en_titles = []
en_requirements = []
if 'author' in alert:
dr_author = alert['author']
else:
dr_author = 'not defined'
if 'description' in alert:
dr_description = alert['description']
else:
dr_description = 'not defined'
if 'references' in alert:
dr_references = alert['references']
else:
dr_references = 'not defined'
if 'id' in alert:
dr_id = alert['id']
else:
dr_id = 'not defined'
if 'internal_responsible' in alert:
dr_internal_responsible = alert['internal_responsible']
else:
dr_internal_responsible = 'not defined'
if 'status' in alert:
dr_status = alert['status']
else:
dr_status = 'not defined'
if 'level' in alert:
dr_severity = alert['level']
else:
dr_severity = 'not defined'
if 'confidence' in alert:
dr_confidence = alert['confidence']
else:
dr_confidence = 'not defined'
for dn in alert_dns:
if dn['title'] not in dn_titles:
dn_titles.append(dn['title'])
if dn['category'] not in dn_categories:
dn_categories.append(dn['category'])
if dn['platform'] not in dn_platforms:
dn_platforms.append(dn['platform'])
if dn['type'] not in dn_types:
dn_types.append(dn['type'])
if dn['channel'] not in dn_channels:
dn_channels.append(dn['channel'])
if dn['provider'] not in dn_providers:
dn_providers.append(dn['provider'])
for lp in logging_policies:
if lp['title'] not in lp_titles:
lp_titles.append(lp['title'])
for er in enrichments:
if er['title'] not in en_titles:
en_titles.append(er['title'])
if 'requirements' in er:
en_requirements.append(er['requirements'])
else:
if "-" not in en_requirements:
en_requirements.append("not defined")
_index.update({
"date_created": date_created,
"sigma_rule_path": path[25:],
"date_modified": date_modified,
"description": dr_description,
"references": dr_references,
"customer": list_of_customers,
"tactic": list_of_tactics,
"dr_id": dr_id,
"technique": list_of_techniques,
"raw_detection_rule": dr_raw,
"detection_rule_title": dr_title,
"detection_rule_author": dr_author,
"detection_rule_internal_responsible": dr_internal_responsible,
"detection_rule_development_status": dr_status,
"detection_rule_severity": dr_severity,
"detection_rule_confidence": dr_confidence,
"category": dn_categories,
"platform": dn_platforms,
"type": dn_types,
"channel": dn_channels,
"provider": dn_providers,
"data_needed": dn_titles,
"logging_policy": lp_titles,
"enrichment": en_titles,
"enrichment_requirements": en_requirements
})
index_line = {"index": {"_id": document_id}}
with open(exported_analytics_directory + '/' + filename,
'a') as fp:
json.dump(index_line, fp)
fp.write("\n")
json.dump(_index, fp)
fp.write("\n")
print(f'[+] Created {filename}')
def main(**kwargs):
dn_list = load_yamls(kwargs['dn_path'])[0]
lp_list = load_yamls(kwargs['lp_path'])[0]
ra_list = load_yamls(kwargs['ra_path'])[0]
rp_list = load_yamls(kwargs['rp_path'])[0]
enrichments_list = load_yamls(kwargs['en_path'])[0]
alerts, path_to_alerts = load_yamls(kwargs['dr_path'])
pivoting = []
analytics = []
result = []
print("[*] Iterating through Detection Rules")
# Iterate through alerts and pathes to them
for alert, path in zip(alerts, path_to_alerts):
if not isinstance(alert.get('tags'), list):
continue
threats = [tag for tag in alert['tags'] if tag.startswith('attack')]
tactics = [
f'{ta_mapping[threat][1]}: {ta_mapping[threat][0]}'
for threat in threats if threat in ta_mapping.keys()
]
techniques = [
threat for threat in threats if threat.startswith('attack.t')
]
enrichments = [
er for er in enrichments_list
if er['title'] in alert.get('enrichment', [{
'title': '-'
}])
]
if len(enrichments) < 1:
enrichments = [{'title': '-'}]
dn_titles = ATCutils.main_dn_calculatoin_func(path)
#print(dn_titles)
alert_dns = [data for data in dn_list if data['title'] in dn_titles]
if len(alert_dns) < 1:
alert_dns = [{
'category': '-',
'platform': '-',
'provider': '-',
'type': '-',
'channel': '-',
'title': '-',
'loggingpolicy': ['-']
}]
logging_policies = []
for dn in alert_dns:
# If there are logging policies in DN that we havent added yet - add them
logging_policies.extend([
l for l in lp_list if l['title'] in dn['loggingpolicy']
and l not in logging_policies
])
# If there are no logging policices at all - make an empty one just to make one row in csv
if not isinstance(logging_policies,
list) or len(logging_policies) == 0:
logging_policies = [{
'title': "-",
'eventID': [
-1,
]
}]
for tactic in tactics:
for technique in techniques:
technique_name = technique.replace('attack.t', 'T') + ': ' +\
ATCutils.get_attack_technique_name_by_id(technique.replace('attack.', ''))
for dn in alert_dns:
for lp in logging_policies:
for er in enrichments:
result.append([
tactic, technique_name, alert['title'],
dn['category'], dn['platform'], dn['type'],
dn['channel'], dn['provider'], dn['title'],
lp['title'], er['title'],
';'.join(er.get('requirements', [])), '-', '-'
])
print("[*] Iterating through Response Playbooks")
for rp in rp_list:
threats = [tag for tag in rp['tags'] if tag.startswith('attack')]
tactics = [
f'{ta_mapping[threat][1]}: {ta_mapping[threat][0]}'
for threat in threats if threat in ta_mapping.keys()
]
techniques = [
threat for threat in threats if threat.startswith('attack.t')
]
ras_buf = []
[ras_buf.extend(l) for l in rp.values() if isinstance(l, list)]
ras = [ra for ra in ras_buf if ra.startswith('RA')]
indices = [
i for i, x in enumerate(result)
if x[0] in tactics or x[1] in techniques
]
if len(indices) < 1:
for tactic in tactics:
for technique in techniques:
technique_name = technique.replace('attack.t', 'T') + ': ' +\
ATCutils.get_attack_technique_name_by_id(technique.replace('attack.', ''))
for ra in ras:
result.append([
tactic, technique_name, '-', '-', '-', '-', '-',
'-', '-', '-', '-', rp['title'], ra
])
else:
for i in indices:
result[i][-2] = rp['title']
result[i][-1] = ';'.join(ras)
analytics = []
print("[*] Iterating through Data Needed")
for dn in dn_list:
pivot = [
dn['category'], dn['platform'], dn['type'], dn['channel'],
dn['provider'], dn['title'], '', ''
]
for field in dn['fields']:
analytics.append([field] + pivot)
print("[*] Iterating through Enrichments")
for er in enrichments_list:
for dn in [
dnn for dnn in dn_list
if dnn['title'] in er.get('data_to_enrich', [])
]:
pivot = [
dn['category'], dn['platform'], dn['type'], dn['channel'],
dn['provider'], dn['title'], er['title'],
';'.join(er.get('requirements', []))
]
for field in er['new_fields']:
analytics.append([field] + pivot)
with open('../analytics.csv', 'w', newline='') as csvfile:
alertswriter = csv.writer(csvfile,
delimiter=',') # maybe need some quoting
alertswriter.writerow([
'tactic', 'technique', 'detection rule', 'category', 'platform',
'type', 'channel', 'provider', 'data needed', 'logging policy',
'enrichment', 'enrichment requirements', 'response playbook',
'response action'
])
for row in result:
alertswriter.writerow(row)
print("[+] Created analytics.csv")
with open('../pivoting.csv', 'w', newline='') as csvfile:
alertswriter = csv.writer(csvfile,
delimiter=',') # maybe need some quoting
alertswriter.writerow([
'field', 'category', 'platform', 'type', 'channel', 'provider',
'data_needed', 'enrichment', 'enrichment requirements'
])
for row in analytics:
alertswriter.writerow(row)
print("[+] Created pivoting.csv")
