def test_add_nodes_no_overlap(nx): proc = Process(process_id=10, process_image="test.exe", command_line="test.exe /c foobar") other_proc = Process(process_id=12, process_image="best.exe", command_line="best.exe /c 123456") proc.launched[other_proc].append(timestamp=1) backend = NetworkX(consolidate_edges=True, nodes=[proc, other_proc]) G = backend.graph() assert len(G.nodes()) == 2 assert len(G.edges()) == 1 # Add in a new pair of nodes. proc2 = Process(process_id=4, process_image="malware.exe", command_line="malware.exe /c foobar") f = File(file_name="foo", file_path="bar") proc2.wrote[f] G = backend.add_nodes([proc2, f]) # Graph grew assert len(G.nodes()) == 4 assert len(G.edges()) == 2
def process_create(self, event) -> Tuple[Process, File, Process]: pid = -1 command_line = None match = re.match(r"PID: (\d*), Command line: (.*)", event["params"]) if match: pid, command_line = match.groups() process_image, process_image_path = split_path(event["path"]) proc = Process( process_id=int(pid), process_image=process_image, process_image_path=process_image_path, command_line=command_line, ) proc_file = proc.get_file_node() proc_file.file_of[proc] parent = Process(process_id=int(event["process_id"]), process_image=event["process_name"]) parent.launched[proc].append(timestamp=event["event_time"]) return (proc, proc_file, parent)
def make_process(self, event: dict) -> Tuple[Process, File, Process, File]: """Accepts a process with the `EventTypes.PROCESS_LAUNCHED` event_type. For example:: { FieldNames.PARENT_PROCESS_IMAGE: "cmd.exe", FieldNames.PARENT_PROCESS_IMAGE_PATH: "\\", FieldNames.PARENT_PROCESS_ID: "2568", FieldNames.PARENT_COMMAND_LINE: '/K name.exe"', FieldNames.PROCESS_IMAGE: "find.exe", FieldNames.PROCESS_IMAGE_PATH: "\\", FieldNames.COMMAND_LINE: 'find /i "svhost.exe"', FieldNames.PROCESS_ID: "3144", FieldNames.EVENT_TYPE: EventTypes.PROCESS_LAUNCHED, } Parameters ---------- event : dict [description] Returns ------- Tuple[Process, File, Process, File] [description] """ parent = Process( process_image=event[FieldNames.PARENT_PROCESS_IMAGE], process_image_path=event[FieldNames.PARENT_PROCESS_IMAGE_PATH], process_id=int(event[FieldNames.PARENT_PROCESS_ID]), command_line=event[FieldNames.PARENT_COMMAND_LINE], ) # Create the file node. # TODO: Integrate into the Process() init function? parent_file = parent.get_file_node() parent_file.file_of[parent] child = Process( process_image=event[FieldNames.PROCESS_IMAGE], process_image_path=event[FieldNames.PROCESS_IMAGE_PATH], process_id=int(event[FieldNames.PROCESS_ID]), command_line=event[FieldNames.COMMAND_LINE], ) child_file = child.get_file_node() child_file.file_of[child] if FieldNames.TIMESTAMP in event: parent.launched[child].append( timestamp=int(event[FieldNames.TIMESTAMP])) else: parent.launched[child] return (parent, parent_file, child, child_file)
def testEqualsMoreFields(): proc = Process(process_id=10, process_image="test.exe", command_line="test.exe /c foobar") other_proc = Process(process_id=10, process_image="test.exe", command_line="test.exe /c 123456") assert proc == other_proc assert hash(proc) == hash(other_proc)
def test_edge_has_no_name(nx): proc = Process(process_id=10, process_image="test.exe", command_line=None) other_proc = Process(process_id=12, process_image="best.exe", command_line="best.exe /c 123456") # append never called proc.launched[other_proc] # This shouldn't error. G = nx(nodes=[proc, other_proc]) len(G.nodes()) == 2 len(G.edges()) == 1
def process_creation(self, event: dict) -> Tuple[Process, File, Process]: """Transformers a process creation (event ID 4688) into a set of nodes. https://www.ultimatewindowssecurity.com/securitylog/encyclopedia/event.aspx?eventID=4688 Parameters ---------- event : dict [description] Returns ------- Optional[Tuple[Process, File, Process, File]] [description] """ # Get the parent PID parent_pid = int(event["data_name_processid"], 16) # Child PID child_pid = int(event["data_name_newprocessid"], 16) proc_name, proc_path = split_path(event["data_name_newprocessname"]) child = Process( host=event["computer"], process_id=child_pid, user=event["data_name_subjectusername"], process_image=proc_name, process_image_path=proc_path, command_line=event.get("data_name_commandline"), ) child_file = child.get_file_node() child_file.file_of[child] # Map the process for later self.seen_procs[child_pid] = child parent = self.seen_procs.get(parent_pid) if parent is None: # Create a dummy proc. If we haven't already seen the parent parent = Process(host=event["computer"], process_id=parent_pid) parent.launched[child].append( timestamp=event["timecreated_systemtime"]) # Don't need to pull out the parent's file, as it will have always # been created before being put into seen_procs return (child, child_file, parent)
def testLaunchedMultipleProces(): parent = Process(process_id=1, process_image="parent.exe") child = Process(process_id=2, process_image="child.exe") parent2 = Process(process_id=4, process_image="parent.exe") child2 = Process(process_id=3, process_image="child.exe") parent.launched[child].append(timestamp=12456) parent2.launched[child2].append(timestamp=2) assert {"timestamp": 12456} in parent.launched[child] assert {"timestamp": 2} not in parent.launched[child] assert {"timestamp": 2} in parent2.launched[child2] assert {"timestamp": 12456} not in parent2.launched[child2]
def test_node_updated(nx): """After pushing in the first process, the second process which has the same hash should cause the command line attribute to update""" proc = Process(process_id=10, process_image="test.exe", command_line=None) next_proc = Process(process_id=10, process_image="test.exe", command_line="best.exe /c 123456") G = nx(nodes=[proc, next_proc]) in_graph_proc = G.nodes(data=True)[hash(proc)]["data"] assert in_graph_proc.command_line == "best.exe /c 123456" assert in_graph_proc.process_id == 10 assert in_graph_proc.process_image == "test.exe" # Should only have one node, since both nodes inserted are equal assert len(G.nodes()) == 1
def test_from_json_object(nx): proc = Process(process_id=10, process_image="test.exe", command_line=None) other_proc = Process(process_id=12, process_image="best.exe", command_line="best.exe /c 123456") proc.launched[other_proc] G = nx(nodes=[proc, other_proc]) _json_output = NetworkX.graph_to_json(G) assert isinstance(_json_output, dict) G2 = NetworkX.from_json(_json_output) # Graphs should be equal. assert networkx.is_isomorphic(G, G2)
def test_file_of(): file_node = File(file_path="c:/windows", file_name="test.exe", extension=".exe") proc = Process(process_id=0, process_image="test.exe", process_image_path="c:/windows/test.exe") file_node.file_of[proc] assert proc in file_node.file_of
def make_file_copy(self, event: dict) -> Tuple[Process, File, File, File]: process = Process( process_image=event[FieldNames.PROCESS_IMAGE], process_image_path=event[FieldNames.PROCESS_IMAGE_PATH], process_id=int(event[FieldNames.PROCESS_ID]), command_line=event[FieldNames.COMMAND_LINE], ) proc_file = process.get_file_node() proc_file.file_of[process] # Source file src_file = File( file_path=event[FieldNames.SRC_FILE][FieldNames.FILE_PATH], file_name=event[FieldNames.SRC_FILE][FieldNames.FILE_NAME], hashes=event[FieldNames.SRC_FILE].get(FieldNames.HASHES), ) # Dest file src_file.set_extension() dest_file = File( file_path=event[FieldNames.DEST_FILE][FieldNames.FILE_PATH], file_name=event[FieldNames.DEST_FILE][FieldNames.FILE_NAME], hashes=event[FieldNames.DEST_FILE].get(FieldNames.HASHES), ) dest_file.set_extension() src_file.copied_to[dest_file] process.copied[src_file] return (process, proc_file, src_file, dest_file)
def make_dnslookup( self, event: dict ) -> Union[Tuple[Process, File, Domain, IPAddress], Tuple[Process, File, Domain]]: process = Process( process_image=event[FieldNames.PROCESS_IMAGE], process_image_path=event[FieldNames.PROCESS_IMAGE_PATH], process_id=int(event[FieldNames.PROCESS_ID]), command_line=event[FieldNames.COMMAND_LINE], ) proc_file = process.get_file_node() proc_file.file_of[process] dom = Domain(event[FieldNames.HTTP_HOST]) process.dns_query_for[dom] # Sometimes we don't know what the domain resolved to. if FieldNames.IP_ADDRESS in event: addr = IPAddress(ip_address=event[FieldNames.IP_ADDRESS]) dom.resolves_to[addr] return (process, proc_file, dom, addr) else: return (process, proc_file, dom)
def make_basic_regkey(self, event: dict) -> Tuple[Process, File, RegistryKey]: process = Process( process_image=event[FieldNames.PROCESS_IMAGE], process_image_path=event[FieldNames.PROCESS_IMAGE_PATH], process_id=int(event[FieldNames.PROCESS_ID]), command_line=event[FieldNames.COMMAND_LINE], ) proc_file = process.get_file_node() proc_file.file_of[process] # RegistryKey Node Creation reg_node = RegistryKey( hive=event[FieldNames.HIVE], key_path=event[FieldNames.REG_KEY_PATH], key=event[FieldNames.REG_KEY], ) if event["event_type"] == EventTypes.REG_KEY_OPENED: process.read_key[reg_node] else: process.deleted_key[reg_node] return (process, proc_file, reg_node)
def make_connection(self, event: dict) -> Tuple[Process, File, IPAddress]: process = Process( process_image=event[FieldNames.PROCESS_IMAGE], process_image_path=event[FieldNames.PROCESS_IMAGE_PATH], process_id=int(event[FieldNames.PROCESS_ID]), command_line=event[FieldNames.COMMAND_LINE], ) proc_file = process.get_file_node() proc_file.file_of[process] addr = IPAddress(ip_address=event[FieldNames.IP_ADDRESS]) if FieldNames.PORT in event and FieldNames.PROTOCOL in event: process.connected_to[addr].append( port=int(event[FieldNames.PORT]), protocol=event[FieldNames.PROTOCOL]) elif FieldNames.PORT in event: process.connected_to[addr].append(port=int(event[FieldNames.PORT])) elif FieldNames.PROTOCOL in event: process.connected_to[addr].append( protocol=event[FieldNames.PROTOCOL]) else: process.connected_to[addr] return (process, proc_file, addr)
def make_http_req( self, event: dict ) -> Union[Tuple[Process, File, URI, Domain], Tuple[Process, File, URI, Domain, IPAddress]]: process = Process( process_image=event[FieldNames.PROCESS_IMAGE], process_image_path=event[FieldNames.PROCESS_IMAGE_PATH], process_id=int(event[FieldNames.PROCESS_ID]), command_line=event[FieldNames.COMMAND_LINE], ) proc_file = process.get_file_node() proc_file.file_of[process] dom = Domain(event[FieldNames.HTTP_HOST]) uri = URI(uri=event[FieldNames.URI]) uri.uri_of[dom] process.http_request_to[uri].append( method=event[FieldNames.HTTP_METHOD]) if FieldNames.IP_ADDRESS in event: ip = IPAddress(event[FieldNames.IP_ADDRESS]) dom.resolves_to[ip] process.connected_to[ip] return (process, proc_file, uri, dom, ip) else: return (process, proc_file, uri, dom)
def make_imageload(self, event: dict) -> Optional[Tuple[File, Process, File]]: # Pull out the process fields. process = Process( process_image=event["process"], process_image_path=event["processPath"], process_id=int(event["pid"]), user=event.get("username", None), ) # Pull out the image of the process file_node = process.get_file_node() # File - (File Of) -> Process file_node.file_of[process] # Add the drive where possible. if event.get("drive"): file_path = f"{event['drive']}:\\{event['filePath']}" else: file_path = event["filePath"] loaded_file = File(file_path=file_path, file_name=event["fileName"]) loaded_file.set_extension() process.loaded[loaded_file].append(timestamp=event["event_time"]) return (loaded_file, process, file_node)
def make_regkey_set_value( self, event: dict) -> Tuple[Process, File, RegistryKey]: process = Process( process_image=event[FieldNames.PROCESS_IMAGE], process_image_path=event[FieldNames.PROCESS_IMAGE_PATH], process_id=int(event[FieldNames.PROCESS_ID]), command_line=event[FieldNames.COMMAND_LINE], ) proc_file = process.get_file_node() proc_file.file_of[process] # RegistryKey Node Creation reg_node = RegistryKey( hive=event[FieldNames.HIVE], key_path=event[FieldNames.REG_KEY_PATH], key=event[FieldNames.REG_KEY], value=event.get(FieldNames.REG_KEY_VALUE), ) if reg_node.value: process.changed_value[reg_node].append(value=reg_node.value) else: process.changed_value[reg_node] return (process, proc_file, reg_node)
def test_one_edge(nx): proc = Process(process_id=10, process_image="test.exe", command_line="test.exe /c foobar") other_proc = Process(process_id=12, process_image="best.exe", command_line="best.exe /c 123456") proc.launched[other_proc].append(timestamp=1) G = nx(nodes=[proc, other_proc]) assert len(G.nodes()) == 2 assert len(G.edges()) == 1 u = hash(proc) v = hash(other_proc) assert networkx.has_path(G, u, v) assert "Launched" in G[u][v] assert {"timestamp": 1} == G[u][v]["Launched"]["data"][0]
def make_file(self, event: dict) -> Optional[Tuple[File, Process, File]]: """Converts a fileWriteEvent to two nodes, a file and the process manipulated the file. Generates a process - (Wrote) -> File edge. Parameters ---------- event : dict The fileWriteEvent event. Returns ------- Optional[Tuple[File, Process, File]] Returns a tuple contaning the File that this event is focused on, and the process which manipulated the file. The process has a Wrote edge to the file. Also contains the file that the process belongs to. """ if "filePath" not in event: return None # Add the drive where possible. if event.get("drive"): file_path = f"{event['drive']}:\\{event['filePath']}" else: file_path = event["filePath"] hashes: Dict[str, str] = {} if event.get("md5"): hashes = {"md5": event["md5"]} # Create the file node. file_node = File(file_path=file_path, file_name=event["fileName"], hashes=hashes) # Set the extension file_node.set_extension() # Set the process node process = Process( process_id=int(event["pid"]), process_image=event["process"], process_image_path=event["processPath"], user=event.get("username"), ) # Add a wrote edge with the contents of the file write. process.wrote[file_node].append( timestamp=int(event["event_time"]), contents=event.get("textAtLowestOffset")) # Pull out the image of the process proc_file_node = process.get_file_node() # File - (File Of) -> Process proc_file_node.file_of[process] return (file_node, process, proc_file_node)
def testNotFileOf(): file_node = File(file_path="c:/windows", file_name="test.exe", extension=".exe") proc = Process(process_id=0, process_image="best.exe", process_image_path="c:/windows/best.exe") assert proc not in file_node.file_of
def test_from_json_path(nx, tmpdir): proc = Process(process_id=10, process_image="test.exe", command_line=None) other_proc = Process(process_id=12, process_image="best.exe", command_line="best.exe /c 123456") proc.launched[other_proc] G = nx(nodes=[proc, other_proc]) _json_output = NetworkX.graph_to_json(G) # Save to file p = tmpdir.mkdir("networkx").join("data.json") p.write(json.dumps(_json_output)) G2 = NetworkX.from_json(p) # Graphs should be equal. assert networkx.is_isomorphic(G, G2)
def make_basic_file(self, event: dict) -> Tuple[Process, File, File]: """Transforms a file based event. Support events: 1. EventTypes.FILE_DELETED 2. EventTypes.FILE_OPENED 3. EventTypes.FILE_WRITTEN 4. EventTypes.LOADED_MODULE Parameters ---------- event : dict [description] Returns ------- Tuple[Process, File, File] [description] """ process = Process( process_image=event[FieldNames.PROCESS_IMAGE], process_image_path=event[FieldNames.PROCESS_IMAGE_PATH], process_id=int(event[FieldNames.PROCESS_ID]), command_line=event[FieldNames.COMMAND_LINE], ) proc_file = process.get_file_node() proc_file.file_of[process] file_node = File( file_path=event[FieldNames.FILE_PATH], file_name=event[FieldNames.FILE_NAME], hashes=event.get(FieldNames.HASHES), ) file_node.set_extension() # Switch based on the event type event_type = event[FieldNames.EVENT_TYPE] if event_type == EventTypes.FILE_OPENED: process.accessed[file_node] elif event_type == EventTypes.FILE_WRITTEN: process.wrote[file_node] elif event_type == EventTypes.LOADED_MODULE: process.loaded[file_node] else: process.deleted[file_node] return (process, proc_file, file_node)
def dns_events( self, event: dict ) -> Union[Tuple[Process, File, Domain], Tuple[Process, File, Domain, IPAddress]]: """Transforms a single DNS event Example event:: { "mode": "dns_query", "protocol_type": "udp", "hostname": "foobar", "qtype": "Host Address", "processinfo": { "imagepath": "C:\\ProgramData\\bloop\\some_proc.exe", "tainted": true, "md5sum": "....", "pid": 3020 }, "timestamp": 27648 } Optionally, if the event is "dns_query_answer", we can also extract the response. Parameters ---------- event : dict source dns_query event Returns ------- Tuple[Process, File, Domain] Process and its image, and the domain looked up """ proc_info = event["processinfo"] process_image, process_image_path = split_path(proc_info["imagepath"]) proc = Process( process_id=int(proc_info["pid"]), process_image=process_image, process_image_path=process_image_path, ) proc_file = proc.get_file_node() proc_file.file_of[proc] domain = Domain(event["hostname"]) proc.dns_query_for[domain].append(timestamp=int(event["timestamp"])) if "ipaddress" in event: addr = IPAddress(event["ipaddress"]) domain.resolves_to[addr].append(timestamp=int(event["timestamp"])) return (proc, proc_file, domain, addr) else: return (proc, proc_file, domain)
def access_file(self, event) -> Tuple[Process, File]: proc = Process(process_id=int(event["process_id"]), process_image=event["process_name"]) file_name, file_path = split_path(event["path"]) target_file = File(file_name=file_name, file_path=file_path) proc.accessed[target_file].append(timestamp=event["event_time"]) return (proc, target_file)
def make_network(self, event: dict) -> Optional[Tuple[IPAddress, Process, File]]: """Converts a network connection event into a Process, File and IP Address node. Nodes: 1. IP Address communicated to. 2. Process contacting IP. 3. File process launched from. Edges: 1. Process - (Connected To) -> IP Address 2. File - (File Of) -> Process Parameters ---------- event : dict The ipv4NetworkEvent Returns ------- Optional[Tuple[IPAddress, Process, File]] The IP Address, Process, and Process's File object. """ # Pull out the process fields. process = Process( process_image=event["process"], process_image_path=event["processPath"], process_id=int(event["pid"]), user=event.get("username", None), ) # Pull out the image of the process file_node = process.get_file_node() # File - (File Of) -> Process file_node.file_of[process] # Create the network node ip_address = IPAddress(event["remoteIP"]) # Create the connection edge # Process - (Connected To) -> IP Address process.connected_to[ip_address].append(timestamp=event["event_time"], protocol=event["protocol"], port=int(event["remotePort"])) return (ip_address, process, file_node)
def test_add_node_overlaps_existing(nx): proc = Process(process_id=10, process_image="test.exe", command_line="test.exe /c foobar") other_proc = Process(process_id=12, process_image="best.exe", command_line="best.exe /c 123456") proc.launched[other_proc].append(timestamp=1) backend = NetworkX(consolidate_edges=True, nodes=[proc, other_proc]) G = backend.graph() assert len(G.nodes()) == 2 assert len(G.edges()) == 1 # Add a new node that *overlaps* an existing node (note - not the same node object.) proc2 = Process(process_id=10, process_image="test.exe", command_line="test.exe /c foobar") f = File(file_name="foo", file_path="bar") proc2.wrote[f] G = backend.add_nodes([proc2, f]) # Graph grew, but only 3 nodes. assert len(G.nodes()) == 3 assert len(G.edges()) == 2 # Process should have both write and launched edges. u = hash(proc2) v = hash(other_proc) v2 = hash(f) assert networkx.has_path(G, u, v) assert networkx.has_path(G, u, v2) assert "Launched" in G[u][v] assert "Wrote" in G[u][v2]
def make_registry( self, event: dict) -> Optional[Tuple[RegistryKey, Process, File]]: # Pull out the process fields. process = Process( process_image=event["process"], process_image_path=event["processPath"], process_id=int(event["pid"]), user=event.get("username", None), ) # Pull out the image of the process file_node = process.get_file_node() # File - (File Of) -> Process file_node.file_of[process] # RegistryKey Node Creation reg_node = RegistryKey( hive=event["hive"], key_path=event["keyPath"], key=event.get("valueName"), value=event.get("text"), value_type=event.get("valueType"), ) # Space shuttle code for the edge setting # # EventType Mappings: # 1: Value Changed # 2: Value Deleted # 3: Key Created # 4: Key Deleted # reg_event_type = str(event["eventType"]) if reg_event_type == "1": process.changed_value[reg_node].append( timestamp=event["event_time"]) elif reg_event_type == "2": process.deleted_value[reg_node].append( timestamp=event["event_time"]) elif reg_event_type == "3": process.created_key[reg_node].append(timestamp=event["event_time"]) elif reg_event_type == "4": process.deleted_key[reg_node].append(timestamp=event["event_time"]) else: logger.warn( f"Found a new registry event type with a value of {reg_event_type}: {event}" ) return (reg_node, process, file_node)
def conn_events(self, event: dict) -> Tuple[Process, File, IPAddress]: """Transforms a single connection event Example event:: { "mode": "connect", "protocol_type": "tcp", "ipaddress": "199.168.199.123", "destination_port": 3333, "processinfo": { "imagepath": "C:\\ProgramData\\bloop\\some_proc.exe", "tainted": true, "md5sum": "....", "pid": 3020 }, "timestamp": 27648 } Parameters ---------- event : dict source dns_query event Returns ------- Tuple[Process, File, IPAddress] Process and its image, and the destination address """ proc_info = event["processinfo"] process_image, process_image_path = split_path(proc_info["imagepath"]) proc = Process( process_id=int(proc_info["pid"]), process_image=process_image, process_image_path=process_image_path, ) proc_file = proc.get_file_node() proc_file.file_of[proc] addr = IPAddress(event["ipaddress"]) proc.connected_to[addr].append( protocol=event["protocol_type"], timestamp=event["timestamp"], port=int(event["destination_port"]), ) return (proc, proc_file, addr)
def access_reg_key(self, event) -> Tuple[Process, RegistryKey]: proc = Process(process_id=int(event["process_id"]), process_image=event["process_name"]) reg_key, reg_path = split_path(event["path"]) hive = reg_path.split("\\")[0] reg_path = "\\".join(reg_path.split("\\")[1:]) reg_node = RegistryKey(hive=hive, key_path=reg_path, key=reg_key) proc.read_key[reg_node].append(timestamp=event["event_time"]) return (proc, reg_node)
def connection(self, event) -> Tuple[Process, IPAddress]: proc = Process(process_id=int(event["process_id"]), process_image=event["process_name"]) dest_addr = event["path"].split("->")[-1].lstrip() ip_addr, port = dest_addr.split(":") addr = IPAddress(ip_addr) proc.connected_to[addr].append( timestamp=event["event_time"], port=int(port), protocol=event["event_type"].split(" ") [0], # Extract protocol from event type ) return (proc, addr)