def test_add_nodes_no_overlap(nx):
proc = Process(process_id=10,
process_image="test.exe",
command_line="test.exe /c foobar")
other_proc = Process(process_id=12,
process_image="best.exe",
command_line="best.exe /c 123456")
proc.launched[other_proc].append(timestamp=1)
backend = NetworkX(consolidate_edges=True, nodes=[proc, other_proc])
G = backend.graph()
assert len(G.nodes()) == 2
assert len(G.edges()) == 1
# Add in a new pair of nodes.
proc2 = Process(process_id=4,
process_image="malware.exe",
command_line="malware.exe /c foobar")
f = File(file_name="foo", file_path="bar")
proc2.wrote[f]
G = backend.add_nodes([proc2, f])
# Graph grew
assert len(G.nodes()) == 4
assert len(G.edges()) == 2
def adhoc():
"""Allows for ad-hoc transformation of generic JSON Data based on one of two CIM models:
1. The Beagle CIM Model (defined in `constants.py`)
2. The OSSEM Model (defined in https://github.com/Cyb3rWard0g/OSSEM)
"""
valid_cim_formats = ["beagle"]
data = request.get_json()
events = data["data"]
cim_format = data.get("cim", "beagle")
if str(cim_format).lower() not in valid_cim_formats:
response = jsonify({"message": f"cim_format must be in {cim_format}"})
return response
if not isinstance(events, list):
events = [events]
logger.info(f"Beginning ad-hoc graphing request")
g = JSONData(events).to_graph(consolidate_edges=True)
logger.info(f"Completed ad-hoc graphing request")
return jsonify({"data": NetworkX.graph_to_json(g)})
def test_from_datasources():
packets_1 = [
Ether(src="ab:ab:ab:ab:ab:ab", dst="12:12:12:12:12:12") /
IP(src="127.0.0.1", dst="192.168.1.1") / TCP(sport=12345, dport=80) /
HTTP() /
HTTPRequest(Method="GET", Path="/foo", Host="https://google.com")
]
packets_2 = [
# HTTP Packet
Ether(src="ab:ab:ab:ab:ab:ab", dst="12:12:12:12:12:12") /
IP(src="127.0.0.1", dst="192.168.1.1") / TCP(sport=12345, dport=80) /
HTTP() /
HTTPRequest(Method="GET", Path="/foo", Host="https://google.com"),
# DNS Packet
Ether(src="ab:ab:ab:ab:ab:ab", dst="12:12:12:12:12:12") /
IP(src="127.0.0.1", dst="192.168.1.1") / UDP(sport=80, dport=53) /
DNS(rd=1,
qd=DNSQR(qtype="A", qname="google.com"),
an=DNSRR(rdata="123.0.0.1")),
# TCP Packet
Ether(src="ab:ab:ab:ab:ab:ab", dst="12:12:12:12:12:12") /
IP(src="127.0.0.1", dst="192.168.1.1") / TCP(sport=80, dport=5355),
]
nx = NetworkX.from_datasources([
packets_to_datasource_events(packets)
for packets in [packets_1, packets_2]
])
# Make the graph
nx.graph()
assert not nx.is_empty()
def test_from_json_object(nx):
proc = Process(process_id=10, process_image="test.exe", command_line=None)
other_proc = Process(process_id=12, process_image="best.exe", command_line="best.exe /c 123456")
proc.launched[other_proc]
G = nx(nodes=[proc, other_proc])
_json_output = NetworkX.graph_to_json(G)
assert isinstance(_json_output, dict)
G2 = NetworkX.from_json(_json_output)
# Graphs should be equal.
assert networkx.is_isomorphic(G, G2)
def test_from_json_path(nx, tmpdir):
proc = Process(process_id=10, process_image="test.exe", command_line=None)
other_proc = Process(process_id=12, process_image="best.exe", command_line="best.exe /c 123456")
proc.launched[other_proc]
G = nx(nodes=[proc, other_proc])
_json_output = NetworkX.graph_to_json(G)
# Save to file
p = tmpdir.mkdir("networkx").join("data.json")
p.write(json.dumps(_json_output))
G2 = NetworkX.from_json(p)
# Graphs should be equal.
assert networkx.is_isomorphic(G, G2)
def test_add_node_overlaps_existing(nx):
proc = Process(process_id=10,
process_image="test.exe",
command_line="test.exe /c foobar")
other_proc = Process(process_id=12,
process_image="best.exe",
command_line="best.exe /c 123456")
proc.launched[other_proc].append(timestamp=1)
backend = NetworkX(consolidate_edges=True, nodes=[proc, other_proc])
G = backend.graph()
assert len(G.nodes()) == 2
assert len(G.edges()) == 1
# Add a new node that *overlaps* an existing node (note - not the same node object.)
proc2 = Process(process_id=10,
process_image="test.exe",
command_line="test.exe /c foobar")
f = File(file_name="foo", file_path="bar")
proc2.wrote[f]
G = backend.add_nodes([proc2, f])
# Graph grew, but only 3 nodes.
assert len(G.nodes()) == 3
assert len(G.edges()) == 2
# Process should have both write and launched edges.
u = hash(proc2)
v = hash(other_proc)
v2 = hash(f)
assert networkx.has_path(G, u, v)
assert networkx.has_path(G, u, v2)
assert "Launched" in G[u][v]
assert "Wrote" in G[u][v2]
def test_from_json_fails_on_invalid(nx, tmpdir):
with pytest.raises(ValueError):
NetworkX.from_json({})
with pytest.raises(ValueError):
NetworkX.from_json({"nodes": []})
with pytest.raises(ValueError):
NetworkX.from_json({"links": []})
def _save_graph_to_db(backend: NetworkX, category: str, graph_id: int = None) -> dict:
"""Saves a graph to the database, optionally forcing an overwrite of an existing graph.
Parameters
----------
backend : NetworkX
The NetworkX object to save
category : str
The category
graph_id: int
The graph ID to override.
Returns
-------
dict
JSON to return to client with ID and path.
"""
# Take the SHA256 of the contents of the graph.
contents_hash = hashlib.sha256(
json.dumps(backend.to_json(), sort_keys=True).encode("utf-8")
).hexdigest()
# See if we have previously generated this *exact* graph.
existing = Graph.query.filter_by(meta=backend.metadata, sha256=contents_hash).first()
if existing:
logger.info(f"Graph previously generated with id {existing.id}")
return {"id": existing.id, "self": f"/{existing.category}/{existing.id}"}
dest_folder = category.replace(" ", "_").lower()
# Set up the storage directory.
dest_path = f"{Config.get('storage', 'dir')}/{dest_folder}/{contents_hash}.json"
os.makedirs(f"{Config.get('storage', 'dir')}/{dest_folder}", exist_ok=True)
json.dump(backend.to_json(), open(dest_path, "w"))
if graph_id:
db_entry = Graph.query.filter_by(id=graph_id).first()
# set the new hash.
db_entry.file_path = f"{contents_hash}.json"
db_entry.sha256 = contents_hash
# NOTE: Old path is not deleted.
else:
db_entry = Graph(
sha256=contents_hash,
meta=backend.metadata,
comment=request.form.get("comment", None),
category=dest_folder, # Categories use the lower name!
file_path=f"{contents_hash}.json",
)
# Add new entry
db.session.add(db_entry)
db.session.commit()
logger.info(f"Added graph to database with id={db_entry.id}")
logger.info(f"Saved graph to {dest_path}")
return {"id": db_entry.id, "self": f"/{dest_folder}/{db_entry.id}"}
def test_empty_graph(nx):
backend = NetworkX(nodes=[], consolidate_edges=True)
backend.graph()
assert backend.is_empty()
def _backend(*args, **kwargs) -> networkx.Graph:
return NetworkX(*args, consolidate_edges=True,
**kwargs).graph() # type: ignore