import os # ########################################################################### # # ############################## Customer ################################### # # ########################################################################### # ATCconfig = ATCutils.load_config("config.yml") dr_dirs = ATCconfig.get('detection_rules_directories') all_rules = [] all_names = [] all_titles = [] for dr_path in dr_dirs: rules, paths = ATCutils.load_yamls_with_paths(dr_path) all_rules = all_rules + rules names = [path.split('/')[-1].replace('.yml', '') for path in paths] all_names = all_names + names titles = [rule.get('title') for rule in rules] all_titles = all_titles + titles _ = zip(all_rules, all_names, all_titles) rules_by_title = {title: (rule, name) for (rule, name, title) in _} class Customer: """Class for Customer entity""" def __init__(self, yaml_file, apipath=None, auth=None, space=None): """ Init method """
def main(**kwargs): dn_list = ATCutils.load_yamls(kwargs['dn_path']) lp_list = ATCutils.load_yamls(kwargs['lp_path']) ra_list = ATCutils.load_yamls(kwargs['ra_path']) rp_list = ATCutils.load_yamls(kwargs['rp_path']) cu_list = ATCutils.load_yamls(ATCconfig.get('customers_directory')) enrichments_list = ATCutils.load_yamls(kwargs['en_path']) pivoting = [] analytics = [] result = [] dr_dirs = ATCconfig.get('detection_rules_directories') print("[*] Iterating through Detection Rules") # Iterate through alerts and pathes to them for dr_path in dr_dirs: alerts, path_to_alerts = ATCutils.load_yamls_with_paths(dr_path) for alert, path in zip(alerts, path_to_alerts): if not isinstance(alert.get('tags'), list): continue list_of_customers = [] for specific_customer in cu_list: if alert['title'] in specific_customer[ 'detectionrule'] and specific_customer[ 'customer_name'] not in list_of_customers: list_of_customers.append( specific_customer['customer_name']) if not isinstance(list_of_customers, list) or len(list_of_customers) == 0: list_of_customers = ["None"] customer = ';'.join(list_of_customers) threats = [ tag for tag in alert['tags'] if tag.startswith('attack') ] tactics = [ f'{ta_mapping[threat][1]}: {ta_mapping[threat][0]}' for threat in threats if threat in ta_mapping.keys() ] techniques = [ threat for threat in threats if threat.startswith('attack.t') ] enrichments = [ er for er in enrichments_list if er['title'] in alert.get('enrichment', []) ] dn_titles = ATCutils.main_dn_calculatoin_func(path) alert_dns = [ data for data in dn_list if data['title'] in dn_titles ] logging_policies = [] for dn in alert_dns: if 'loggingpolicy' in dn: # If there are logging policies in DN that we havent added yet - add them logging_policies.extend([ l for l in lp_list if l['title'] in dn['loggingpolicy'] and l not in logging_policies ]) # If there are no logging policices at all - make an empty one just to make one row in csv if not isinstance(logging_policies, list) or len(logging_policies) == 0: logging_policies = [{ 'title': "-", 'eventID': [ -1, ] }] for dn in alert_dns: pivot = [ dn['category'], dn['platform'], dn['type'], dn['channel'], dn['provider'], dn['title'], '', '' ] for tactic in tactics: for technique in techniques: technique_name = technique.replace('attack.t', 'T') + ': ' +\ ATCutils.get_attack_technique_name_by_id( technique.replace('attack.', '')) for lp in logging_policies: rps = [ rp for rp in rp_list if technique in rp['tags'] or tactic in rp['tags'] ] if len(rps) < 1: rps = [{'title': '-'}] for rp in rps: ras_buf = [] [ ras_buf.extend(l) for l in rp.values() if isinstance(l, list) ] ras = [ ra for ra in ras_buf if ra.startswith('RA') ] if len(ras) < 1: ras = ['title'] #if len(rp) > 1: #todo for ra in ras: lp['title'] = lp['title'].replace('\n', '') result.append([ customer, tactic, technique_name, alert['title'], dn['category'], dn['platform'], dn['type'], dn['channel'], dn['provider'], dn['title'], lp['title'], '', '', rp['title'], ra ]) # pivoting.append(pivot) for field in dn['fields']: analytics.append([field] + pivot) for er in enrichments: for dn in [ dnn for dnn in dn_list if dnn['title'] in er.get('data_to_enrich', []) ]: pivot = [ dn['category'], dn['platform'], dn['type'], dn['channel'], dn['provider'], dn['title'], er['title'], ';'.join(er.get('requirements', [])) ] for tactic in tactics: for technique in techniques: technique_name = technique.replace('attack.t', 'T') + ': ' + \ ATCutils.get_attack_technique_name_by_id( technique.replace('attack.', '')) for lp in logging_policies: lp['title'] = lp['title'].replace('\n', '') result.append([ customer, tactic, technique_name, alert['title'], dn['category'], dn['platform'], dn['type'], dn['channel'], dn['provider'], dn['title'], lp['title'], er['title'], ';'.join(er.get('requirements', [])), '-', '-' ]) # pivoting.append(pivot) for field in er['new_fields']: analytics.append([field] + pivot) analytics = [] for dn in dn_list: if 'category' in dn: dn_category = dn['category'] else: dn_category = "-" if 'platform' in dn: dn_platform = dn['platform'] else: dn_platform = "-" if 'type' in dn: dn_type = dn['type'] else: dn_type = "-" if 'channel' in dn: dn_channel = dn['channel'] else: dn_channel = "-" if 'provider' in dn: dn_provider = dn['provider'] else: dn_provider = "-" if 'title' in dn: dn_title = dn['title'] else: dn_title = "-" pivot = [ dn_category, dn_platform, dn_type, dn_channel, dn_provider, dn_title, '', '' ] for field in dn['fields']: analytics.append([field] + pivot) for er in enrichments_list: for dn in [ dnn for dnn in dn_list if dnn['title'] in er.get('data_to_enrich', []) ]: pivot = [ dn['category'], dn['platform'], dn['type'], dn['channel'], dn['provider'], dn['title'], er['title'], ';'.join(er.get('requirements', [])) ] for field in er['new_fields']: analytics.append([field] + pivot) filename = 'analytics.csv' exported_analytics_directory = ATCconfig.get( 'exported_analytics_directory') with open(exported_analytics_directory + '/' + filename, 'w', newline='') as csvfile: # maybe need some quoting alertswriter = csv.writer(csvfile, delimiter=',') alertswriter.writerow([ 'customer', 'tactic', 'technique', 'detection_rule', 'category', 'platform', 'type', 'channel', 'provider', 'data_needed', 'logging policy', 'enrichment', 'enrichment requirements', 'response playbook', 'response action' ]) for row in result: alertswriter.writerow(row) print(f'[+] Created {filename}') filename = 'pivoting.csv' exported_analytics_directory = ATCconfig.get( 'exported_analytics_directory') with open(exported_analytics_directory + '/' + filename, 'w', newline='') as csvfile: # maybe need some quoting alertswriter = csv.writer(csvfile, delimiter=',') alertswriter.writerow([ 'field', 'category', 'platform', 'type', 'channel', 'provider', 'data_needed', 'enrichment', 'enrichment requirements' ]) for row in analytics: alertswriter.writerow(row) print(f'[+] Created {filename}')
def main(**kwargs): lp_list = ATCutils.load_yamls(kwargs['lp_path']) ra_list = ATCutils.load_yamls(kwargs['ra_path']) rp_list = ATCutils.load_yamls(kwargs['rp_path']) cu_list = ATCutils.load_yamls(kwargs['cu_path']) dn_list = ATCutils.load_yamls(kwargs['dn_path']) enrichments_list = ATCutils.load_yamls(kwargs['en_path']) _index = {} dr_dirs = ATCconfig.get('detection_rules_directories') # Iterate through alerts and pathes to them for dr_path in dr_dirs: alerts, path_to_alerts = ATCutils.load_yamls_with_paths(dr_path) for alert, path in zip(alerts, path_to_alerts): tactics = [] techniques = [] list_of_customers = [] # raw Sigma rule without fields that are present separately dr_raw = alert.copy() fields_to_remove_from_raw_dr = [ 'title', 'id', 'status', 'date', 'modified', 'description', 'references', 'author', 'tags', 'logsource', 'falsepositives', 'level' ] for field in fields_to_remove_from_raw_dr: dr_raw.pop(field, None) dr_raw = str(yaml.dump((dr_raw), default_flow_style=False)) for customer in cu_list: if 'detectionrule' in customer: if alert['title'] in customer['detectionrule'] and customer[ 'customer_name'] not in list_of_customers: list_of_customers.append(customer['customer_name']) if not isinstance(list_of_customers, list) or len(list_of_customers) == 0: list_of_customers = ["None"] if isinstance(alert.get('tags'), list): try: threats = [ tag for tag in alert['tags'] if tag.startswith('attack') ] tactics = [ f'{ta_mapping[threat][1]}: {ta_mapping[threat][0]}' for threat in threats if threat in ta_mapping.keys() ] except: pass try: threats = [ tag for tag in alert['tags'] if tag.startswith('attack') ] techniques = [ threat for threat in threats if threat.startswith('attack.t') ] except: pass enrichments = [ er for er in enrichments_list if er['title'] in alert.get('enrichment', [{ 'title': '-' }]) ] if len(enrichments) < 1: enrichments = [{'title': 'not defined'}] dn_titles = ATCutils.main_dn_calculatoin_func(path) alert_dns = [ data for data in dn_list if data['title'] in dn_titles ] if len(alert_dns) < 1: alert_dns = [{ 'category': 'not defined', 'platform': 'not defined', 'provider': 'not defined', 'type': 'not defined', 'channel': 'not defined', 'title': 'not defined', 'loggingpolicy': ['not defined'] }] logging_policies = [] for dn in alert_dns: # If there are logging policies in DN that we havent added yet - add them logging_policies.extend([ l for l in lp_list if l['title'] in dn['loggingpolicy'] and l not in logging_policies ]) # If there are no logging policices at all - make an empty one just to make one row in csv if not isinstance(logging_policies, list) or len(logging_policies) == 0: logging_policies = [{ 'title': "not defined", 'eventID': [ -1, ] }] # we use date of creation to have timelines of progress if 'date' in alert: try: date_created = datetime.datetime.strptime( alert['date'], '%Y/%m/%d').isoformat() except: pass if not date_created: try: # in case somebody mixed up month and date, like in "Detection of SafetyKatz" date_created = datetime.datetime.strptime( alert['date'], '%Y/%d/%m').isoformat() except: # temporary solution to avoid errors. all DRs must have date of creation print( 'date in ' + alert['date'] + ' is not in ' + '%Y/%m/%d and %Y/%d/%m formats. Set up current date and time' ) date_created = datetime.datetime.now().isoformat() else: # temporary solution to avoid errors. all internal DRs must have date of creation #date_created = datetime.datetime.now().isoformat() date_created = '2019-03-01T22:50:37.587060' # we use date of creation to have timelines of progress if 'modified' in alert: try: date_modified = datetime.datetime.strptime( alert['modified'], '%Y/%m/%d').isoformat() except: pass if not date_modified: try: # in case somebody mixed up month and date, like in "Detection of SafetyKatz" date_modified = datetime.datetime.strptime( alert['modified'], '%Y/%d/%m').isoformat() except: date_modified = None else: # temporary solution to avoid errors. all internal DRs must have date of creation #date_created = datetime.datetime.now().isoformat() date_modified = None # we create index document based on DR title, which is unique (supposed to be). # this way we will update existing documents in case of changes, # and create new ones when new DRs will be developed # update: well, better recreate everything from the scratch all the time. # if name of DR will change, we will have doubles in index. todo document_id = hash(alert['title']) list_of_tactics = [] list_of_techniques = [] if tactics: for tactic in tactics: if tactic not in list_of_tactics: list_of_tactics.append(tactic) else: list_of_tactics = ['not defined'] if techniques: for technique in techniques: technique_name = technique.replace('attack.t', 'T') + ': ' +\ ATCutils.get_attack_technique_name_by_id(technique.replace('attack.', '')) if technique not in list_of_techniques: list_of_techniques.append(technique_name) else: list_of_techniques = ['not defined'] dr_title = alert['title'] dn_titles = [] dn_categories = [] dn_platforms = [] dn_types = [] dn_channels = [] dn_providers = [] lp_titles = [] en_titles = [] en_requirements = [] if 'author' in alert: dr_author = alert['author'] else: dr_author = 'not defined' if 'description' in alert: dr_description = alert['description'] else: dr_description = 'not defined' if 'references' in alert: dr_references = alert['references'] else: dr_references = 'not defined' if 'id' in alert: dr_id = alert['id'] else: dr_id = 'not defined' if 'internal_responsible' in alert: dr_internal_responsible = alert['internal_responsible'] else: dr_internal_responsible = 'not defined' if 'status' in alert: dr_status = alert['status'] else: dr_status = 'not defined' if 'level' in alert: dr_severity = alert['level'] else: dr_severity = 'not defined' if 'confidence' in alert: dr_confidence = alert['confidence'] else: dr_confidence = 'not defined' for dn in alert_dns: if dn['title'] not in dn_titles: dn_titles.append(dn['title']) if dn['category'] not in dn_categories: dn_categories.append(dn['category']) if dn['platform'] not in dn_platforms: dn_platforms.append(dn['platform']) if dn['type'] not in dn_types: dn_types.append(dn['type']) if dn['channel'] not in dn_channels: dn_channels.append(dn['channel']) if dn['provider'] not in dn_providers: dn_providers.append(dn['provider']) for lp in logging_policies: if lp['title'] not in lp_titles: lp_titles.append(lp['title']) for er in enrichments: if er['title'] not in en_titles: en_titles.append(er['title']) if 'requirements' in er: en_requirements.append(er['requirements']) else: if "-" not in en_requirements: en_requirements.append("not defined") _index.update({ "date_created": date_created, "sigma_rule_path": path[25:], "date_modified": date_modified, "description": dr_description, "references": dr_references, "customer": list_of_customers, "tactic": list_of_tactics, "dr_id": dr_id, "technique": list_of_techniques, "raw_detection_rule": dr_raw, "detection_rule_title": dr_title, "detection_rule_author": dr_author, "detection_rule_internal_responsible": dr_internal_responsible, "detection_rule_development_status": dr_status, "detection_rule_severity": dr_severity, "detection_rule_confidence": dr_confidence, "category": dn_categories, "platform": dn_platforms, "type": dn_types, "channel": dn_channels, "provider": dn_providers, "data_needed": dn_titles, "logging_policy": lp_titles, "enrichment": en_titles, "enrichment_requirements": en_requirements }) index_line = {"index": {"_id": document_id}} with open(exported_analytics_directory + '/' + filename, 'a') as fp: json.dump(index_line, fp) fp.write("\n") json.dump(_index, fp) fp.write("\n") print(f'[+] Created {filename}')
def render_template(self, template_type): """Description template_type: - "markdown" """ if template_type not in ["confluence"]: raise Exception("Bad template_type. Available values:" + " \"confluence\"]") template = env.get_template( 'confluence_responsestage_template.html.j2') self.rs_parsed_file.update( {'description': self.rs_parsed_file.get('description').strip()}) ras, ra_paths = ATCutils.load_yamls_with_paths( ATCconfig.get('response_actions_dir')) ra_filenames = [ ra_path.split('/')[-1].replace('.yml', '') for ra_path in ra_paths ] rs_id = self.rs_parsed_file.get('id') self.rs_parsed_file.update({ 'confluence_viewpage_url': ATCconfig.get('confluence_viewpage_url') }) stage_list = [] for i in range(len(ras)): if rs_mapping[rs_id] == ATCutils.normalize_rs_name( ras[i].get('stage')): ra_id = ras[i].get('id') ra_filename = ra_filenames[i] ra_title = ATCutils.normalize_react_title(ras[i].get('title')) ra_description = ras[i].get('description').strip() ra_confluence_page_name = ra_id + ": " + ra_title print(ra_confluence_page_name) if self.apipath and self.auth and self.space: ra_confluence_page_id = str( ATCutils.confluence_get_page_id( self.apipath, self.auth, self.space, ra_confluence_page_name)) else: ra_confluence_page_id = "" print(ra_confluence_page_id) stage_list.append((ra_id, ra_filename, ra_title, ra_description, ra_confluence_page_id)) new_title = self.rs_parsed_file.get('id')\ + ": "\ + ATCutils.normalize_react_title(self.rs_parsed_file.get('title')) self.rs_parsed_file.update({'title': new_title}) self.rs_parsed_file.update({'stage_list': sorted(stage_list)}) self.content = template.render(self.rs_parsed_file)