def enrichment(self, en_path):
"""Nothing here yet"""
print("[*] Populating Enrichments...")
if en_path:
en_list = glob.glob(en_path + '*.yml')
else:
en_dir = ATCconfig.get('enrichments_dir')
en_list = glob.glob(en_dir + '/*.yml')
for en_file in en_list:
try:
en = Enrichment(en_file,
apipath=self.apipath,
auth=self.auth,
space=self.space)
en.render_template("confluence")
confluence_data = {
"title":
en.en_parsed_file['title'],
"spacekey":
self.space,
"parentid":
str(
DATAutils.confluence_get_page_id(
self.apipath, self.auth, self.space,
"Enrichments")),
"confluencecontent":
en.content,
}
res = DATAutils.push_to_confluence(confluence_data,
self.apipath, self.auth)
if res == 'Page updated':
print("==> updated page: EN '" +
en.en_parsed_file['title'] + "'")
# print("Done: ", en.en_parsed_file['title'])
except Exception as err:
print(en_file + " failed")
print("Err message: %s" % err)
print('-' * 60)
traceback.print_exc(file=sys.stdout)
print('-' * 60)
print("[+] Enrichments populated!")
def create_markdown_dirs():
config = DATAutils.load_config('config.yml')
base_dir = Path(
config.get('md_name_of_root_directory', '../Atomic_Threat_Coverage'))
target_dir_list = ['Logging_Policies', 'Data_Needed', 'Enrichments']
for item in target_dir_list:
(base_dir / item).mkdir(parents=True, exist_ok=True)
def save_markdown_file(self, atc_dir='../Atomic_Threat_Coverage/'):
"""Write content (md template filled with data) to a file"""
base = os.path.basename(self.yaml_file)
title = os.path.splitext(base)[0]
file_path = atc_dir + self.parent_title + "/" + \
title + ".md"
return DATAutils.write_file(file_path, self.content)
def data_needed(self, dn_path):
"""Desc"""
print("[*] Populating Data Needed...")
if dn_path:
dn_list = glob.glob(dn_path + '*.yml')
else:
dn_dir = ATCconfig.get('data_needed_dir')
dn_list = glob.glob(dn_dir + '/*.yml')
for dn_file in dn_list:
try:
dn = DataNeeded(dn_file,
apipath=self.apipath,
auth=self.auth,
space=self.space)
dn.render_template("confluence")
confluence_data = {
"title":
dn.dn_fields["title"],
"spacekey":
self.space,
"parentid":
str(
DATAutils.confluence_get_page_id(
self.apipath, self.auth, self.space,
"Data Needed")),
"confluencecontent":
dn.content,
}
res = DATAutils.push_to_confluence(confluence_data,
self.apipath, self.auth)
if res == 'Page updated':
print("==> updated page: DN '" + dn.dn_fields['title'] +
"'")
# print("Done: ", dn.dn_fields['title'])
except Exception as err:
print(dn_file + " failed")
print("Err message: %s" % err)
print('-' * 60)
traceback.print_exc(file=sys.stdout)
print('-' * 60)
print("[+] Data Needed populated!")
def save_markdown_file(self,
atc_dir=ATCconfig.get('md_name_of_root_directoy')):
"""Write content (md template filled with data) to a file"""
base = os.path.basename(self.yaml_file)
title = os.path.splitext(base)[0]
file_path = atc_dir + self.parent_title + "/" + \
title + ".md"
return DATAutils.write_file(file_path, self.content)
def logging_policy(self, lp_path):
"""Desc"""
print("[*] Populating Logging Policies...")
if lp_path:
lp_list = glob.glob(lp_path + '*.yml')
else:
lp_dir = ATCconfig.get('logging_policies_dir')
lp_list = glob.glob(lp_dir + '/*.yml')
for lp_file in lp_list:
try:
lp = LoggingPolicy(lp_file)
lp.render_template("confluence")
confluence_data = {
"title":
lp.fields["title"],
"spacekey":
self.space,
"parentid":
str(
DATAutils.confluence_get_page_id(
self.apipath, self.auth, self.space,
"Logging Policies")),
"confluencecontent":
lp.content,
}
res = DATAutils.push_to_confluence(confluence_data,
self.apipath, self.auth)
if res == 'Page updated':
print("==> updated page: LP '" + lp.fields['title'] + "'")
# print("Done: ", lp.fields['title'])
except Exception as err:
print(lp_file + " failed")
print("Err message: %s" % err)
print('-' * 60)
traceback.print_exc(file=sys.stdout)
print('-' * 60)
print("[+] Logging Policies populated!")
def parse_into_fields(self, yaml_file):
"""Description"""
# self.dn_fields contains parsed fields obtained from yaml file
self.dn_fields = DATAutils.read_yaml_file(yaml_file)
"""Fill the fields with values. Put None if key not found"""
self.title = self.dn_fields.get("title")
self.author = self.dn_fields.get("author")
self.description = self.dn_fields.get("description")
self.loggingpolicy = self.dn_fields.get("loggingpolicy")
self.mitigation_policy = self.dn_fields.get("mitigation_policy")
self.platform = self.dn_fields.get("platform")
self.type = self.dn_fields.get("type")
self.channel = self.dn_fields.get("channel")
self.provider = self.dn_fields.get("provider")
self.fields = self.dn_fields.get("fields")
self.sample = self.dn_fields.get("sample")
def __init__(self):
dn_path = ATCconfig.get('data_needed_dir')
lp_path = ATCconfig.get('logging_policies_dir')
en_path = ATCconfig.get('enrichments_dir')
dn_list = DATAutils.load_yamls(dn_path)
lp_list = DATAutils.load_yamls(lp_path)
enrichments_list = DATAutils.load_yamls(en_path)
pivoting = []
analytics = []
result = []
analytics = []
for dn in dn_list:
if 'category' in dn:
dn_category = dn['category']
else:
dn_category = "-"
if 'platform' in dn:
dn_platform = dn['platform']
else:
dn_platform = "-"
if 'type' in dn:
dn_type = dn['type']
else:
dn_type = "-"
if 'channel' in dn:
dn_channel = dn['channel']
else:
dn_channel = "-"
if 'provider' in dn:
dn_provider = dn['provider']
else:
dn_provider = "-"
if 'title' in dn:
dn_title = dn['title']
else:
dn_title = "-"
pivot = [
dn_category, dn_platform, dn_type, dn_channel, dn_provider,
dn_title, '', ''
]
for field in dn['fields']:
analytics.append([field] + pivot)
for er in enrichments_list:
for dn in [
dnn for dnn in dn_list
if dnn['title'] in er.get('data_to_enrich', [])
]:
pivot = [
dn['category'], dn['platform'], dn['type'], dn['channel'],
dn['provider'], dn['title'], er['title'],
';'.join(er.get('requirements', []))
]
for field in er['new_fields']:
analytics.append([field] + pivot)
filename = 'pivoting.csv'
exported_analytics_directory = ATCconfig.get(
'exported_analytics_directory')
Path(exported_analytics_directory).mkdir(parents=True, exist_ok=True)
with open(exported_analytics_directory + '/' + filename,
'w',
newline='') as csvfile:
# maybe need some quoting
alertswriter = csv.writer(csvfile, delimiter=',')
alertswriter.writerow([
'field', 'category', 'platform', 'type', 'channel', 'provider',
'data_needed', 'enrichment', 'enrichment requirements'
])
for row in analytics:
alertswriter.writerow(row)
print(f'[+] Created {filename}')
from scripts.datautils import DATAutils
from scripts.attack_mapping import te_mapping, ta_mapping
except:
from data.atc_data.scripts.datautils import DATAutils
from data.atc_data.scripts.attack_mapping import te_mapping, ta_mapping
from pathlib import Path
import csv
import sys
import getopt
from os import listdir
from os.path import isfile, join
from yaml.scanner import ScannerError
ATCconfig = DATAutils.load_config("config.yml")
HELP_MESSAGE = """Usage: python3 yamls2csv.py [OPTIONS]\n\n\n
\t\tPossible options are --detectionrules_path, --dataneeded_path
--loggingpolicies path
\t\tDefaults are
\t\tdataneeded_path = ../data_needed/;
\t\tloggingpolicies_path=../logging_policies/"""
class GenerateCSV:
def __init__(self):
dn_path = ATCconfig.get('data_needed_dir')
lp_path = ATCconfig.get('logging_policies_dir')
en_path = ATCconfig.get('enrichments_dir')
def __init__(self,
dn_path=False,
lp_path=False,
en_path=False,
atc_dir=False,
init=False):
"""Init"""
# Check if atc_dir provided
if atc_dir:
self.atc_dir = atc_dir
else:
self.atc_dir = ATCconfig.get('md_name_of_root_directory') + '/'
# Main logic
if dn_path:
dns, dn_paths = DATAutils.load_yamls_with_paths(dn_path)
else:
dns, dn_paths = DATAutils.load_yamls_with_paths(
ATCconfig.get('data_needed_dir'))
if lp_path:
lps, lp_paths = DATAutils.load_yamls_with_paths(lp_path)
else:
lps, lp_paths = DATAutils.load_yamls_with_paths(
ATCconfig.get('logging_policies_dir'))
if en_path:
ens, en_paths = DATAutils.load_yamls_with_paths(en_path)
else:
ens, en_paths = DATAutils.load_yamls_with_paths(
ATCconfig.get('enrichments_dir'))
dn_filenames = [
dn_path.split('/')[-1].replace('.yml', '') for dn_path in dn_paths
]
lp_filenames = [
lp_path.split('/')[-1].replace('.yml', '') for lp_path in lp_paths
]
en_filenames = [
en_path.split('/')[-1].replace('.yml', '') for en_path in en_paths
]
# Point to the templates directory
env = Environment(loader=FileSystemLoader('scripts/templates'))
# Get proper template
template = env.get_template('mkdocs_config_template.yml.j2')
data_to_render = {}
data_needed_list = []
for i in range(len(dns)):
dn_updated_title = dns[i].get('title')
data_needed_list.append((dn_updated_title, dn_filenames[i]))
logging_policy_list = []
for i in range(len(lps)):
rp_updated_title = lps[i].get('title')
logging_policy_list.append((rp_updated_title, lp_filenames[i]))
enrichment_list = []
for i in range(len(ens)):
en_updated_title = ens[i].get('title')
enrichment_list.append((en_updated_title, en_filenames[i]))
data_to_render.update({'data_needed_list': sorted(data_needed_list)})
data_to_render.update(
{'logging_policy_list': sorted(logging_policy_list)})
data_to_render.update({'enrichment_list': sorted(enrichment_list)})
content = template.render(data_to_render)
try:
DATAutils.write_file('mkdocs.yml', content)
print("[+] Created mkdocs.yml")
except:
print("[-] Failed to create mkdocs.yml")
def render_template(self, template_type):
"""Description
template_type:
- "markdown"
- "confluence"
"""
if template_type not in ["markdown", "confluence"]:
raise Exception("Bad template_type. Available values:" +
" [\"markdown\", \"confluence\"]")
# Get proper template
if template_type == "markdown":
template = env\
.get_template('markdown_dataneeded_template.md.j2')
logging_policies = self.dn_fields.get("loggingpolicy")
if isinstance(logging_policies, str):
logging_policies = [logging_policies]
self.dn_fields.update({'loggingpolicy': logging_policies})
mitigation_policy = self.dn_fields.get("mitigation_policy")
if isinstance(mitigation_policy, str):
mitigation_policy = [mitigation_policy]
self.dn_fields.update({'mitigation_policy': mitigation_policy})
self.dn_fields.update(
{'description': self.dn_fields.get('description').strip()})
refs = self.dn_fields.get("references")
if isinstance(refs, str):
self.dn_fields.update({'references': [refs]})
elif template_type == "confluence":
template = env\
.get_template('confluence_dataneeded_template.html.j2')
self.dn_fields.update({
'confluence_viewpage_url':
ATCconfig.get('confluence_viewpage_url')
})
self.dn_fields.update(
{'description': self.dn_fields.get('description').strip()})
logging_policies = self.dn_fields.get("loggingpolicy")
if not logging_policies:
logging_policies = [
"None",
]
logging_policies_with_id = []
for lp in logging_policies:
if lp != "None" and self.apipath and self.auth and self.space:
logging_policies_id = str(
DATAutils.confluence_get_page_id(
self.apipath, self.auth, self.space, lp))
else:
logging_policies_id = ""
lp = (lp, logging_policies_id)
logging_policies_with_id.append(lp)
self.dn_fields.update({'loggingpolicy': logging_policies_with_id})
mitigation_policies = self.dn_fields.get("mitigation_policy")
if not mitigation_policies:
mitigation_policies = [
"None",
]
mitigation_policies_with_id = []
for mp in mitigation_policies:
if mp != "None" and self.apipath and self.auth and self.space:
mitigation_policies_id = str(
DATAutils.confluence_get_page_id(
self.apipath, self.auth, self.space, mp))
else:
mitigation_policies_id = ""
mp = (mp, mitigation_policies_id)
mitigation_policies_with_id.append(mp)
self.dn_fields.update(
{'mitigation_policy': mitigation_policies_with_id})
refs = self.dn_fields.get("references")
if isinstance(refs, str):
self.dn_fields.update({'references': [refs]})
self.content = template.render(self.dn_fields)
return True
def main(c_auth=None):
try:
ATCconfig = DATAutils.load_config("config.yml")
confluence_space_name = ATCconfig.get('confluence_space_name')
confluence_space_home_page_name = ATCconfig.get(
'confluence_space_home_page_name')
confluence_rest_api_url = ATCconfig.get('confluence_rest_api_url')
confluence_name_of_root_directory = ATCconfig.get(
'confluence_name_of_root_directory')
except Exception as e:
raise e
pass
if not c_auth:
mail = input("Login: "******""
print("[*] Creating ATC root page...")
data = {
"title":
confluence_name_of_root_directory,
"spacekey":
confluence_space_name,
"parentid":
str(
DATAutils.confluence_get_page_id(url, auth, confluence_space_name,
confluence_space_home_page_name)),
"confluencecontent":
content,
}
if not DATAutils.push_to_confluence(data, url, auth):
raise Exception("[-] Could not create or update the page. " +
"Is the parent name correct?")
pages = ["Logging Policies", "Data Needed", "Enrichments"]
for page in pages:
print("Creating %s..." % page)
data = {
"title":
page,
"spacekey":
confluence_space_name,
"parentid":
str(
DATAutils.confluence_get_page_id(
url, auth, confluence_space_name,
confluence_name_of_root_directory)),
"confluencecontent":
content,
}
if not DATAutils.push_to_confluence(data, url, auth):
raise Exception("[-] Could not create or update the page. " +
"Is the parent name correct?")
print("[+] Initial Confluence page structure created!")
return True
def render_template(self, template_type):
"""Description
template_type:
- "markdown"
- "confluence"
"""
if template_type not in ["markdown", "confluence"]:
raise Exception("Bad template_type. Available values:" +
" [\"markdown\", \"confluence\"]")
# Get proper template
if template_type == "markdown":
template = env.get_template('markdown_enrichments_template.md.j2')
self.en_parsed_file.update({
'description':
self.en_parsed_file.get('description').strip()
})
elif template_type == "confluence":
template = env.get_template(
'confluence_enrichments_template.html.j2')
self.en_parsed_file.update({
'confluence_viewpage_url':
ATCconfig.get('confluence_viewpage_url')
})
data_needed = self.en_parsed_file.get('data_needed')
if data_needed:
data_needed_with_id = []
for dn in data_needed:
data_needed_id = str(
DATAutils.confluence_get_page_id(
self.apipath, self.auth, self.space, dn))
dn = (dn, data_needed_id)
data_needed_with_id.append(dn)
self.en_parsed_file.update(
{'data_needed': data_needed_with_id})
data_to_enrich = self.en_parsed_file.get('data_to_enrich')
if data_to_enrich:
data_to_enrich_with_id = []
for de in data_to_enrich:
data_to_enrich_id = str(
DATAutils.confluence_get_page_id(
self.apipath, self.auth, self.space, de))
de = (de, data_to_enrich_id)
data_to_enrich_with_id.append(de)
self.en_parsed_file.update(
{'data_to_enrich': data_to_enrich_with_id})
requirements = self.en_parsed_file.get('requirements')
if requirements:
requirements_with_id = []
for req in requirements:
requirements_id = str(
DATAutils.confluence_get_page_id(
self.apipath, self.auth, self.space, req))
req = (req, requirements_id)
requirements_with_id.append(req)
self.en_parsed_file.update(
{'requirements': requirements_with_id})
self.en_parsed_file.update({
'description':
self.en_parsed_file.get('description').strip()
})
# Render
self.content = template.render(self.en_parsed_file)
def parse_into_fields(self, yaml_file):
"""Description"""
self.en_parsed_file = DATAutils.read_yaml_file(yaml_file)
def parse_into_fields(self, yaml_file):
"""Description"""
# self.fields contains parsed fields obtained from yaml file
self.fields = DATAutils.read_yaml_file(self.yaml_file)