def rel_mem_store(): cam = Campaign(id=CAMPAIGN_ID, **CAMPAIGN_KWARGS) idy = Identity(id=IDENTITY_ID, **IDENTITY_KWARGS) ind = Indicator(id=INDICATOR_ID, **INDICATOR_KWARGS) mal = Malware(id=MALWARE_ID, **MALWARE_KWARGS) rel1 = Relationship(ind, 'indicates', mal, id=RELATIONSHIP_IDS[0]) rel2 = Relationship(mal, 'targets', idy, id=RELATIONSHIP_IDS[1]) rel3 = Relationship(cam, 'uses', mal, id=RELATIONSHIP_IDS[2]) stix_objs = [cam, idy, ind, mal, rel1, rel2, rel3] yield MemoryStore(stix_objs)
def parse_mappings(mappingspath, controls, relationship_ids={}): """parse the NIST800-53 revision 4 mappings and return a STIX bundle of relationships mapping the controls to ATT&CK :param mappingspath the filepath to the mappings TSV file :param controls a stix2.Bundle represneting the controls framework :param relationship_ids is a dict of format {relationship-source-id---relationship-target-id: relationship-id} which maps relationships to desired STIX IDs """ print("reading framework config...", end="", flush=True) # load the mapping config with open(os.path.join("input", "config.json"), "r") as f: config = json.load(f) version = config["attack_version"] domain = config["attack_domain"] print("done") tqdmformat = "{desc}: {percentage:3.0f}% |{bar}| {elapsed}<{remaining}{postfix}" # load ATT&CK STIX data print("downloading ATT&CK data... ", end="", flush=True) attackdata = requests.get(f"https://raw.githubusercontent.com/mitre/cti/ATT%26CK-{version}/{domain}/{domain}.json").json()["objects"] print("done") # build mapping of attack ID to stixID attackID_to_stixID = {} for attackobject in tqdm(attackdata, desc="parsing ATT&CK data", bar_format=tqdmformat): if not attackobject["type"] == "relationship": # skip objects without IDs if "external_references" not in attackobject: continue # skip deprecated and revoked objects if "revoked" in attackobject and attackobject["revoked"]: continue if "x_mitre_deprecated" in attackobject and attackobject["x_mitre_deprecated"]: continue # map attackID to stixID attackID_to_stixID[attackobject["external_references"][0]["external_id"]] = attackobject["id"] # build mapping of control ID to stixID controlID_to_stixID = {} for sdo in tqdm(controls.objects, desc="parsing controls", bar_format=tqdmformat): if sdo.type == "course-of-action": # only do mitigations controlID_to_stixID[sdo["external_references"][0]["external_id"]] = sdo["id"] # build mapping relationships relationships = {} mappings_df = pd.read_csv(mappingspath, sep="\t", keep_default_na=False, header=0) for index, row in tqdm(list(mappings_df.iterrows()), desc="parsing mappings", bar_format=tqdmformat): # create list of control STIX IDs matching this row fromIDs = dict_regex_lookup(controlID_to_stixID, row["controlID"]) # create list of technique STIX IDs matching this row toIDs = dict_regex_lookup(attackID_to_stixID, row["techniqueID"]) # only have a description if the row does # description = row["description"] if row["description"] else None if not fromIDs: print(Fore.RED + "ERROR: cannot find controlID", row["controlID"], Fore.RESET) if not toIDs: print(Fore.RED + "ERROR: cannot find techniqueID", row["techniqueID"], Fore.RESET) if not fromIDs or not toIDs: exit() # combinatorics of every from to every to for fromID in fromIDs: for toID in toIDs: joined_id = f"{fromID}---{toID}" # build the mapping relationship r = Relationship( id=relationship_ids[joined_id] if joined_id in relationship_ids else None, source_ref=fromID, target_ref=toID, relationship_type="mitigates", ) if joined_id not in relationships: relationships[joined_id] = r # construct and return the bundle of relationships return Bundle(*relationships.values())
def parse_controls(controlpath, control_ids={}, relationship_ids={}): """parse the NIST800-53 revision 4 controls and return a STIX bundle :param controlpath the filepath to the controls TSV file :param control_ids is a dict of format {control_name: stixID} which maps control names (e.g AC-1) to desired STIX IDs :param relationship_ids is a dict of format {relationship-source-id---relationship-target-id: relationship-id}, same general purpose as control_ids """ print("reading framework config...", end="", flush=True) # load the mapping config with open(os.path.join("input", "config.json"), "r") as f: config = json.load(f) framework_id = config["framework_id"] print("done") tqdmformat = "{desc}: {percentage:3.0f}% |{bar}| {elapsed}<{remaining}{postfix}" # controls_df = pd.read_csv(controlpath, sep="\t", keep_default_na=False, header=0) with open(controlpath, "r") as controlsfile: controls_data = controlsfile.read().split("\n") columns = controls_data[0].split("\t") controls_data = controls_data[1:] controls = [] currentControl = [] for row in tqdm(controls_data, desc="parsing NIST 800-53 revision 5", bar_format=tqdmformat): row = row.strip('"') # remove leading and trailing quotation marks rowtype = row_type(row) if rowtype == "control" or rowtype == "control_enhancement": if currentControl: # otherwise first row creates an empty control controls.append( Control("\n".join(currentControl), columns, control_ids)) # finish previous control currentControl = [row] # start a new control else: currentControl.append(row) # append line to current control # finish last control controls.append(Control("\n".join(currentControl), columns, control_ids)) # parse controls into stix stixcontrols = [] for control in tqdm(controls, desc="creating controls", bar_format=tqdmformat): stixcontrols.append(control.toStix(framework_id)) # parse control relationships into stix relationships = [] for control in tqdm(list( filter(lambda c: control.parent_id or len(control.related) > 0, controls)), desc="creating control relationships", bar_format=tqdmformat): if control.parent_id: # build subcontrol-of relationships target_id = control_ids[control.parent_id] source_id = control.stix_id joined_id = f"{source_id}---{target_id}" relationships.append( Relationship(id=relationship_ids[joined_id] if joined_id in relationship_ids else None, source_ref=source_id, target_ref=target_id, relationship_type="subcontrol-of")) if len(control.related) > 0: # build related-to relationships for related_id in control.related: if related_id not in control_ids: continue # sometimes related doesn't refer to a control but rather an appendix section source_id = control.stix_id target_id = control_ids[related_id] joined_id = f"{source_id}---{target_id}" relationships.append( Relationship(id=relationship_ids[joined_id] if joined_id in relationship_ids else None, source_ref=source_id, target_ref=target_id, relationship_type="related-to")) return Bundle(*itertools.chain(stixcontrols, relationships))
def parse_controls(controlpath, control_ids={}, relationship_ids={}): """parse the NIST800-53 revision 4 controls and return a STIX bundle :param controlpath the filepath to the controls TSV file :param control_ids is a dict of format {control_name: stixID} which maps control names (e.g AC-1) to desired STIX IDs :param relationship_ids is a dict of format {relationship-source-id---relationship-target-id: relationship-id}, same general purpose as control_ids """ print("reading framework config...", end="", flush=True) # load the mapping config with open(os.path.join("input", "config.json"), "r") as f: config = json.load(f) framework_id = config["framework_id"] print("done") tqdmformat = "{desc}: {percentage:3.0f}% |{bar}| {elapsed}<{remaining}{postfix}" controls_df = pd.read_csv(controlpath, sep="\t", keep_default_na=False, header=0) controls = [] currentControl = None for index, row in tqdm(list(controls_df.iterrows()), desc="parsing NIST 800-53 revision 4", bar_format=tqdmformat): rowtype = row_type(row) if rowtype == "control": controls.append(Control(row, control_ids)) currentControl = controls[ -1] # track current control to pass to enhancements if rowtype == "control_enhancement": controls.append(Control(row, control_ids, parent=currentControl)) if rowtype == "statement": controls[-1].add_statement(row) if rowtype == "substatement": controls[-1].add_substatement(row) # parse controls into stix stixcontrols = [] for control in tqdm(controls, desc="creating controls", bar_format=tqdmformat): stixcontrols.append(control.toStix(framework_id)) # parse control relationships into stix relationships = [] for control in tqdm(list( filter(lambda c: control.parent_id or len(control.related) > 0, controls)), desc="creating control relationships", bar_format=tqdmformat): if control.parent_id: # build subcontrol-of relationships target_id = control_ids[control.parent_id] source_id = control.stix_id joined_id = f"{source_id}---{target_id}" relationships.append( Relationship(id=relationship_ids[joined_id] if joined_id in relationship_ids else None, source_ref=source_id, target_ref=target_id, relationship_type="subcontrol-of")) if len(control.related) > 0: # build related-to relationships for related_id in control.related: if related_id not in control_ids: continue # sometimes related doesn't refer to a control but rather an appendix section source_id = control.stix_id target_id = control_ids[related_id] joined_id = f"{source_id}---{target_id}" relationships.append( Relationship(id=relationship_ids[joined_id] if joined_id in relationship_ids else None, source_ref=source_id, target_ref=target_id, relationship_type="related-to")) return Bundle(*itertools.chain(stixcontrols, relationships))