def enrichment(self, en_path): """Nothing here yet""" print("[*] Populating Enrichments...") if en_path: en_list = glob.glob(en_path + '*.yml') else: en_dir = ATCconfig.get('enrichments_dir') en_list = glob.glob(en_dir + '/*.yml') for en_file in en_list: try: en = Enrichment(en_file, apipath=self.apipath, auth=self.auth, space=self.space) en.render_template("confluence") confluence_data = { "title": en.en_parsed_file['title'], "spacekey": self.space, "parentid": str( DATAutils.confluence_get_page_id( self.apipath, self.auth, self.space, "Enrichments")), "confluencecontent": en.content, } res = DATAutils.push_to_confluence(confluence_data, self.apipath, self.auth) if res == 'Page updated': print("==> updated page: EN '" + en.en_parsed_file['title'] + "'") # print("Done: ", en.en_parsed_file['title']) except Exception as err: print(en_file + " failed") print("Err message: %s" % err) print('-' * 60) traceback.print_exc(file=sys.stdout) print('-' * 60) print("[+] Enrichments populated!")
def create_markdown_dirs(): config = DATAutils.load_config('config.yml') base_dir = Path( config.get('md_name_of_root_directory', '../Atomic_Threat_Coverage')) target_dir_list = ['Logging_Policies', 'Data_Needed', 'Enrichments'] for item in target_dir_list: (base_dir / item).mkdir(parents=True, exist_ok=True)
def save_markdown_file(self, atc_dir='../Atomic_Threat_Coverage/'): """Write content (md template filled with data) to a file""" base = os.path.basename(self.yaml_file) title = os.path.splitext(base)[0] file_path = atc_dir + self.parent_title + "/" + \ title + ".md" return DATAutils.write_file(file_path, self.content)
def data_needed(self, dn_path): """Desc""" print("[*] Populating Data Needed...") if dn_path: dn_list = glob.glob(dn_path + '*.yml') else: dn_dir = ATCconfig.get('data_needed_dir') dn_list = glob.glob(dn_dir + '/*.yml') for dn_file in dn_list: try: dn = DataNeeded(dn_file, apipath=self.apipath, auth=self.auth, space=self.space) dn.render_template("confluence") confluence_data = { "title": dn.dn_fields["title"], "spacekey": self.space, "parentid": str( DATAutils.confluence_get_page_id( self.apipath, self.auth, self.space, "Data Needed")), "confluencecontent": dn.content, } res = DATAutils.push_to_confluence(confluence_data, self.apipath, self.auth) if res == 'Page updated': print("==> updated page: DN '" + dn.dn_fields['title'] + "'") # print("Done: ", dn.dn_fields['title']) except Exception as err: print(dn_file + " failed") print("Err message: %s" % err) print('-' * 60) traceback.print_exc(file=sys.stdout) print('-' * 60) print("[+] Data Needed populated!")
def save_markdown_file(self, atc_dir=ATCconfig.get('md_name_of_root_directoy')): """Write content (md template filled with data) to a file""" base = os.path.basename(self.yaml_file) title = os.path.splitext(base)[0] file_path = atc_dir + self.parent_title + "/" + \ title + ".md" return DATAutils.write_file(file_path, self.content)
def logging_policy(self, lp_path): """Desc""" print("[*] Populating Logging Policies...") if lp_path: lp_list = glob.glob(lp_path + '*.yml') else: lp_dir = ATCconfig.get('logging_policies_dir') lp_list = glob.glob(lp_dir + '/*.yml') for lp_file in lp_list: try: lp = LoggingPolicy(lp_file) lp.render_template("confluence") confluence_data = { "title": lp.fields["title"], "spacekey": self.space, "parentid": str( DATAutils.confluence_get_page_id( self.apipath, self.auth, self.space, "Logging Policies")), "confluencecontent": lp.content, } res = DATAutils.push_to_confluence(confluence_data, self.apipath, self.auth) if res == 'Page updated': print("==> updated page: LP '" + lp.fields['title'] + "'") # print("Done: ", lp.fields['title']) except Exception as err: print(lp_file + " failed") print("Err message: %s" % err) print('-' * 60) traceback.print_exc(file=sys.stdout) print('-' * 60) print("[+] Logging Policies populated!")
def parse_into_fields(self, yaml_file): """Description""" # self.dn_fields contains parsed fields obtained from yaml file self.dn_fields = DATAutils.read_yaml_file(yaml_file) """Fill the fields with values. Put None if key not found""" self.title = self.dn_fields.get("title") self.author = self.dn_fields.get("author") self.description = self.dn_fields.get("description") self.loggingpolicy = self.dn_fields.get("loggingpolicy") self.mitigation_policy = self.dn_fields.get("mitigation_policy") self.platform = self.dn_fields.get("platform") self.type = self.dn_fields.get("type") self.channel = self.dn_fields.get("channel") self.provider = self.dn_fields.get("provider") self.fields = self.dn_fields.get("fields") self.sample = self.dn_fields.get("sample")
def __init__(self): dn_path = ATCconfig.get('data_needed_dir') lp_path = ATCconfig.get('logging_policies_dir') en_path = ATCconfig.get('enrichments_dir') dn_list = DATAutils.load_yamls(dn_path) lp_list = DATAutils.load_yamls(lp_path) enrichments_list = DATAutils.load_yamls(en_path) pivoting = [] analytics = [] result = [] analytics = [] for dn in dn_list: if 'category' in dn: dn_category = dn['category'] else: dn_category = "-" if 'platform' in dn: dn_platform = dn['platform'] else: dn_platform = "-" if 'type' in dn: dn_type = dn['type'] else: dn_type = "-" if 'channel' in dn: dn_channel = dn['channel'] else: dn_channel = "-" if 'provider' in dn: dn_provider = dn['provider'] else: dn_provider = "-" if 'title' in dn: dn_title = dn['title'] else: dn_title = "-" pivot = [ dn_category, dn_platform, dn_type, dn_channel, dn_provider, dn_title, '', '' ] for field in dn['fields']: analytics.append([field] + pivot) for er in enrichments_list: for dn in [ dnn for dnn in dn_list if dnn['title'] in er.get('data_to_enrich', []) ]: pivot = [ dn['category'], dn['platform'], dn['type'], dn['channel'], dn['provider'], dn['title'], er['title'], ';'.join(er.get('requirements', [])) ] for field in er['new_fields']: analytics.append([field] + pivot) filename = 'pivoting.csv' exported_analytics_directory = ATCconfig.get( 'exported_analytics_directory') Path(exported_analytics_directory).mkdir(parents=True, exist_ok=True) with open(exported_analytics_directory + '/' + filename, 'w', newline='') as csvfile: # maybe need some quoting alertswriter = csv.writer(csvfile, delimiter=',') alertswriter.writerow([ 'field', 'category', 'platform', 'type', 'channel', 'provider', 'data_needed', 'enrichment', 'enrichment requirements' ]) for row in analytics: alertswriter.writerow(row) print(f'[+] Created {filename}')
from scripts.datautils import DATAutils from scripts.attack_mapping import te_mapping, ta_mapping except: from data.atc_data.scripts.datautils import DATAutils from data.atc_data.scripts.attack_mapping import te_mapping, ta_mapping from pathlib import Path import csv import sys import getopt from os import listdir from os.path import isfile, join from yaml.scanner import ScannerError ATCconfig = DATAutils.load_config("config.yml") HELP_MESSAGE = """Usage: python3 yamls2csv.py [OPTIONS]\n\n\n \t\tPossible options are --detectionrules_path, --dataneeded_path --loggingpolicies path \t\tDefaults are \t\tdataneeded_path = ../data_needed/; \t\tloggingpolicies_path=../logging_policies/""" class GenerateCSV: def __init__(self): dn_path = ATCconfig.get('data_needed_dir') lp_path = ATCconfig.get('logging_policies_dir') en_path = ATCconfig.get('enrichments_dir')
def __init__(self, dn_path=False, lp_path=False, en_path=False, atc_dir=False, init=False): """Init""" # Check if atc_dir provided if atc_dir: self.atc_dir = atc_dir else: self.atc_dir = ATCconfig.get('md_name_of_root_directory') + '/' # Main logic if dn_path: dns, dn_paths = DATAutils.load_yamls_with_paths(dn_path) else: dns, dn_paths = DATAutils.load_yamls_with_paths( ATCconfig.get('data_needed_dir')) if lp_path: lps, lp_paths = DATAutils.load_yamls_with_paths(lp_path) else: lps, lp_paths = DATAutils.load_yamls_with_paths( ATCconfig.get('logging_policies_dir')) if en_path: ens, en_paths = DATAutils.load_yamls_with_paths(en_path) else: ens, en_paths = DATAutils.load_yamls_with_paths( ATCconfig.get('enrichments_dir')) dn_filenames = [ dn_path.split('/')[-1].replace('.yml', '') for dn_path in dn_paths ] lp_filenames = [ lp_path.split('/')[-1].replace('.yml', '') for lp_path in lp_paths ] en_filenames = [ en_path.split('/')[-1].replace('.yml', '') for en_path in en_paths ] # Point to the templates directory env = Environment(loader=FileSystemLoader('scripts/templates')) # Get proper template template = env.get_template('mkdocs_config_template.yml.j2') data_to_render = {} data_needed_list = [] for i in range(len(dns)): dn_updated_title = dns[i].get('title') data_needed_list.append((dn_updated_title, dn_filenames[i])) logging_policy_list = [] for i in range(len(lps)): rp_updated_title = lps[i].get('title') logging_policy_list.append((rp_updated_title, lp_filenames[i])) enrichment_list = [] for i in range(len(ens)): en_updated_title = ens[i].get('title') enrichment_list.append((en_updated_title, en_filenames[i])) data_to_render.update({'data_needed_list': sorted(data_needed_list)}) data_to_render.update( {'logging_policy_list': sorted(logging_policy_list)}) data_to_render.update({'enrichment_list': sorted(enrichment_list)}) content = template.render(data_to_render) try: DATAutils.write_file('mkdocs.yml', content) print("[+] Created mkdocs.yml") except: print("[-] Failed to create mkdocs.yml")
def render_template(self, template_type): """Description template_type: - "markdown" - "confluence" """ if template_type not in ["markdown", "confluence"]: raise Exception("Bad template_type. Available values:" + " [\"markdown\", \"confluence\"]") # Get proper template if template_type == "markdown": template = env\ .get_template('markdown_dataneeded_template.md.j2') logging_policies = self.dn_fields.get("loggingpolicy") if isinstance(logging_policies, str): logging_policies = [logging_policies] self.dn_fields.update({'loggingpolicy': logging_policies}) mitigation_policy = self.dn_fields.get("mitigation_policy") if isinstance(mitigation_policy, str): mitigation_policy = [mitigation_policy] self.dn_fields.update({'mitigation_policy': mitigation_policy}) self.dn_fields.update( {'description': self.dn_fields.get('description').strip()}) refs = self.dn_fields.get("references") if isinstance(refs, str): self.dn_fields.update({'references': [refs]}) elif template_type == "confluence": template = env\ .get_template('confluence_dataneeded_template.html.j2') self.dn_fields.update({ 'confluence_viewpage_url': ATCconfig.get('confluence_viewpage_url') }) self.dn_fields.update( {'description': self.dn_fields.get('description').strip()}) logging_policies = self.dn_fields.get("loggingpolicy") if not logging_policies: logging_policies = [ "None", ] logging_policies_with_id = [] for lp in logging_policies: if lp != "None" and self.apipath and self.auth and self.space: logging_policies_id = str( DATAutils.confluence_get_page_id( self.apipath, self.auth, self.space, lp)) else: logging_policies_id = "" lp = (lp, logging_policies_id) logging_policies_with_id.append(lp) self.dn_fields.update({'loggingpolicy': logging_policies_with_id}) mitigation_policies = self.dn_fields.get("mitigation_policy") if not mitigation_policies: mitigation_policies = [ "None", ] mitigation_policies_with_id = [] for mp in mitigation_policies: if mp != "None" and self.apipath and self.auth and self.space: mitigation_policies_id = str( DATAutils.confluence_get_page_id( self.apipath, self.auth, self.space, mp)) else: mitigation_policies_id = "" mp = (mp, mitigation_policies_id) mitigation_policies_with_id.append(mp) self.dn_fields.update( {'mitigation_policy': mitigation_policies_with_id}) refs = self.dn_fields.get("references") if isinstance(refs, str): self.dn_fields.update({'references': [refs]}) self.content = template.render(self.dn_fields) return True
def main(c_auth=None): try: ATCconfig = DATAutils.load_config("config.yml") confluence_space_name = ATCconfig.get('confluence_space_name') confluence_space_home_page_name = ATCconfig.get( 'confluence_space_home_page_name') confluence_rest_api_url = ATCconfig.get('confluence_rest_api_url') confluence_name_of_root_directory = ATCconfig.get( 'confluence_name_of_root_directory') except Exception as e: raise e pass if not c_auth: mail = input("Login: "******"" print("[*] Creating ATC root page...") data = { "title": confluence_name_of_root_directory, "spacekey": confluence_space_name, "parentid": str( DATAutils.confluence_get_page_id(url, auth, confluence_space_name, confluence_space_home_page_name)), "confluencecontent": content, } if not DATAutils.push_to_confluence(data, url, auth): raise Exception("[-] Could not create or update the page. " + "Is the parent name correct?") pages = ["Logging Policies", "Data Needed", "Enrichments"] for page in pages: print("Creating %s..." % page) data = { "title": page, "spacekey": confluence_space_name, "parentid": str( DATAutils.confluence_get_page_id( url, auth, confluence_space_name, confluence_name_of_root_directory)), "confluencecontent": content, } if not DATAutils.push_to_confluence(data, url, auth): raise Exception("[-] Could not create or update the page. " + "Is the parent name correct?") print("[+] Initial Confluence page structure created!") return True
def render_template(self, template_type): """Description template_type: - "markdown" - "confluence" """ if template_type not in ["markdown", "confluence"]: raise Exception("Bad template_type. Available values:" + " [\"markdown\", \"confluence\"]") # Get proper template if template_type == "markdown": template = env.get_template('markdown_enrichments_template.md.j2') self.en_parsed_file.update({ 'description': self.en_parsed_file.get('description').strip() }) elif template_type == "confluence": template = env.get_template( 'confluence_enrichments_template.html.j2') self.en_parsed_file.update({ 'confluence_viewpage_url': ATCconfig.get('confluence_viewpage_url') }) data_needed = self.en_parsed_file.get('data_needed') if data_needed: data_needed_with_id = [] for dn in data_needed: data_needed_id = str( DATAutils.confluence_get_page_id( self.apipath, self.auth, self.space, dn)) dn = (dn, data_needed_id) data_needed_with_id.append(dn) self.en_parsed_file.update( {'data_needed': data_needed_with_id}) data_to_enrich = self.en_parsed_file.get('data_to_enrich') if data_to_enrich: data_to_enrich_with_id = [] for de in data_to_enrich: data_to_enrich_id = str( DATAutils.confluence_get_page_id( self.apipath, self.auth, self.space, de)) de = (de, data_to_enrich_id) data_to_enrich_with_id.append(de) self.en_parsed_file.update( {'data_to_enrich': data_to_enrich_with_id}) requirements = self.en_parsed_file.get('requirements') if requirements: requirements_with_id = [] for req in requirements: requirements_id = str( DATAutils.confluence_get_page_id( self.apipath, self.auth, self.space, req)) req = (req, requirements_id) requirements_with_id.append(req) self.en_parsed_file.update( {'requirements': requirements_with_id}) self.en_parsed_file.update({ 'description': self.en_parsed_file.get('description').strip() }) # Render self.content = template.render(self.en_parsed_file)
def parse_into_fields(self, yaml_file): """Description""" self.en_parsed_file = DATAutils.read_yaml_file(yaml_file)
def parse_into_fields(self, yaml_file): """Description""" # self.fields contains parsed fields obtained from yaml file self.fields = DATAutils.read_yaml_file(self.yaml_file)