def test_add_nodes():
dg = sn.DiGraph()
nodes = {
uuid.UUID('2cdfebf3-bf95-47f1-9f04-12ccdfbe03b7'): {
"id": uuid.UUID('2cdfebf3-bf95-47f1-9f04-12ccdfbe03b7'),
'type': 'B'
},
uuid.UUID('3caaa8c0-9148-493d-bdf0-2c574b95526c'): {
"id": uuid.UUID('3caaa8c0-9148-493d-bdf0-2c574b95526c'),
'type': 'A'
},
uuid.UUID('3cd197c2-cf5e-42dc-9ccd-0c2adcaf4bc2'): {
"id": uuid.UUID('3cd197c2-cf5e-42dc-9ccd-0c2adcaf4bc2'),
'type': 'C'
}
}
dg.add_nodes(nodes)
assert dg.get_nodes() == nodes
dg = sn.DiGraph()
nodes = [{'type': 'A'}, {'type': 'B'}, {'type': 'C'}]
ids = dg.add_nodes(nodes)
dg_nodes = dg.get_nodes()
assert dg_nodes[ids[0]]['type'] == 'A'
assert dg_nodes[ids[1]]['type'] == 'B'
assert dg_nodes[ids[2]]['type'] == 'C'
def test_add_edges(populated_digraph):
g = sn.DiGraph()
g.add_nodes(populated_digraph.get_nodes())
edges = [
('3caaa8c09148493dbdf02c574b95526c',
'2cdfebf3bf9547f19f0412ccdfbe03b7', {
"type": "normal"
}, '5f5f44ec7c0144e29c5b7d513f92d9ab'),
('2cdfebf3bf9547f19f0412ccdfbe03b7',
'3caaa8c09148493dbdf02c574b95526c', {
"type": "normal"
}, 'f3674fcc691848ebbd478b1bfb3e84c3'),
('3caaa8c09148493dbdf02c574b95526c',
'3cd197c2cf5e42dc9ccd0c2adcaf4bc2', {
"type": "normal"
}, '7eb91be54d3746b89a61a282bcc207bb'),
(
'2cdfebf3bf9547f19f0412ccdfbe03b7',
'3cd197c2cf5e42dc9ccd0c2adcaf4bc2',
# include some dummy reserved attributes
# to make sure they correctly get removed
{
"type": "irregular",
"id": "foo",
"src": "bar",
"dst": "baz"
},
'c172a3599b7d4ef3bbb688277276b763'),
# (b, c). leave out id to test that mixing tuple variations works
('3cd197c2cf5e42dc9ccd0c2adcaf4bc2',
'2cdfebf3bf9547f19f0412ccdfbe03b7', {
"type": "irregular"
})
]
g.add_edges(edges)
gedges = g.get_edges()
# make sure all the edges from populated_digraph are present
for eid, attrs in populated_digraph.get_edges().iteritems():
assert eid in gedges
assert gedges[eid] == attrs
# make sure the last edge in the list was added
assert g.get_edges_between('3cd197c2cf5e42dc9ccd0c2adcaf4bc2',
'2cdfebf3bf9547f19f0412ccdfbe03b7')
g = sn.DiGraph()
g.add_node({"type": "A"}, '3caaa8c09148493dbdf02c574b95526c')
g.add_node({"type": "B"}, '2cdfebf3bf9547f19f0412ccdfbe03b7')
g.add_node({"type": "C"}, '3cd197c2cf5e42dc9ccd0c2adcaf4bc2')
edges = populated_digraph.get_edges()
g.add_edges(edges)
assert g.get_edges() == populated_digraph.get_edges()
def test_get_or_add_node():
dg = sn.DiGraph()
key = 'test_key'
# node is not in before we add it
with pytest.raises(KeyError):
dg.get_node('test_key')
# node is in after we add it
dg.get_or_add_node(key, data={"type": "test"})
correct_node = {"id": key, "type": "test"}
test_node = dg.get_node(key)
assert test_node == correct_node
# if we call again with the same key, it does not add a node, it returns it
# Will also discard the argument data
dg.get_or_add_node(key, data={"type": "test2"})
correct_nodes = {
key: {
"id": key,
"type": "test" # not test2
}
}
assert dg.get_nodes() == correct_nodes
def populated_digraph():
g = sn.DiGraph()
a = g.add_node({"type": "A"}, '3caaa8c09148493dbdf02c574b95526c')
b = g.add_node({"type": "B"}, '2cdfebf3bf9547f19f0412ccdfbe03b7')
c = g.add_node({"type": "C"}, '3cd197c2cf5e42dc9ccd0c2adcaf4bc2')
g.add_edge(a, b, {"type": "normal"}, '5f5f44ec7c0144e29c5b7d513f92d9ab')
g.add_edge(b, a, {"type": "normal"}, 'f3674fcc691848ebbd478b1bfb3e84c3')
g.add_edge(a, c, {"type": "normal"}, '7eb91be54d3746b89a61a282bcc207bb')
g.add_edge(b, c, {"type": "irregular"}, 'c172a3599b7d4ef3bbb688277276b763')
return g
def test_intersection(populated_digraph):
another_digraph = sn.DiGraph()
a = another_digraph.add_node({"type": "A"}, '3caaa8c09148493dbdf02c574b95526c')
b = another_digraph.add_node({"type": "B"}, '2cdfebf3bf9547f19f0412ccdfbe03b7')
d = another_digraph.add_node({"type": "D"}, 'da30015efe3c44dbb0b3b3862cef704a')
another_digraph.add_edge(a, b, {"type": "normal"}, '5f5f44ec7c0144e29c5b7d513f92d9ab')
another_digraph.add_edge(b, a, {"type": "normal"}, 'f3674fcc691848ebbd478b1bfb3e84c3')
another_digraph.add_edge(a, d, {"type": "normal"}, 'f3674fcc691848ebbd478b1bfb3e84c3')
another_digraph.add_edge(d, b, {"type": "irregular"}, 'f3674fcc691848ebbd478b1bfb3e84c3')
I = sn.intersection(populated_digraph, another_digraph)
correct_nodes = {
uuid.UUID('3caaa8c09148493dbdf02c574b95526c'): {
"type": "A",
"id": uuid.UUID('3caaa8c09148493dbdf02c574b95526c')
},
uuid.UUID('2cdfebf3bf9547f19f0412ccdfbe03b7'): {
"type": "B",
"id": uuid.UUID('2cdfebf3bf9547f19f0412ccdfbe03b7')
}
}
assert I.get_nodes() == correct_nodes
correct_edges = {
# a,b
uuid.UUID('5f5f44ec7c0144e29c5b7d513f92d9ab'): {
"type": "normal",
"src": uuid.UUID('3caaa8c09148493dbdf02c574b95526c'),
"dst": uuid.UUID('2cdfebf3bf9547f19f0412ccdfbe03b7'),
"id": uuid.UUID('5f5f44ec7c0144e29c5b7d513f92d9ab')
},
# b,a
uuid.UUID('f3674fcc691848ebbd478b1bfb3e84c3'): {
"type": "normal",
"src": uuid.UUID('2cdfebf3bf9547f19f0412ccdfbe03b7'),
"dst": uuid.UUID('3caaa8c09148493dbdf02c574b95526c'),
"id": uuid.UUID('f3674fcc691848ebbd478b1bfb3e84c3')
}
}
assert I.get_edges() == correct_edges
amt = float(amt)
max_amt = max(amt, max_amt)
data = {"graphiti:space:activity": amt}
except ValueError:
data = {}
else:
data = {}
graph.add_edge(acquirer, target, data)
def scale_deal_amts(graph):
global max_amt
for nid, attrs in graph.get_edges().iteritems():
if 'graphiti:space:activity' in attrs:
attrs['graphiti:space:activity'] = math.log(
attrs['graphiti:space:activity'], max_amt
)
if __name__ == "__main__":
parser = argparse.ArgumentParser("merger.py")
parser.add_argument('csv_file', type=str)
args = parser.parse_args()
global max_amt
max_amt = 0
g = sn.DiGraph()
process_csv_file(g, args.csv_file)
scale_deal_amts(g)
g.save_json("mergers.json")
logfilename = args.capture_filename
print("Opening {}".format(logfilename))
with open(logfilename) as logfile:
log = logfile.read()
print("Building graph...")
delim = log_delim(log)
vars_ = extract_vars(log)
data = extract_data(vars_['fields'], log, 500)
if args.verbose:
print_data(data)
graph = sn.DiGraph()
for ip, items in data.items():
graph.clear_node_cache()
graph.cache_nodes_by("label", build=False)
for item in items:
src_ip_node = get_node(graph, 'id.orig_h', item)
user_agent_node = get_node(graph, 'user_agent', item)
if src_ip_node == None or user_agent_node == None:
continue
connect(graph, src_ip_node, user_agent_node,
{"type": "user_agent"})
label = item['host'] + item['uri']
if __name__ == "__main__":
parser = argparse.ArgumentParser("asn_timeline")
parser.add_argument('asn_graph', help="The JSON file of the ASN graph.")
args = parser.parse_args()
global timeline_counter
global timeline_delta
global timeline_delta_small
timeline_counter = 100
timeline_delta = 100
timeline_delta_small = 50
queue = {} # maps node IDs to their queue of edges
g = sn.DiGraph(json_file=args.asn_graph) # the JSON graph
set_invisible(g) # set the nodes/edges to invisible initially
for nid, attrs in sorted(g.get_nodes().iteritems(),
key=lambda n: n[1]['registration']
if 'registration' in n[1] else n[1]['label']):
timeline_set_node_visible(g, nid)
if nid in queue:
for eid in queue[nid]:
timeline_set_edge_visible(g, eid)
queue.pop(nid) # remove from the queue, once visible
for neighbor in dict(
g.neighbors(nid).items() + g.predecessors(nid).items()):
# get all edges between this node and its neighbor
edges = [
eid for eid, attrs in g.get_edges_between(
def test_json_constructor(fixture_dir, correct_output_filename,
correct_output_graph):
g = sn.DiGraph(
json_file=os.path.join(fixture_dir, correct_output_filename))
assert g.get_nodes() == correct_output_graph.get_nodes()
assert g.get_edges() == correct_output_graph.get_edges()
def test_union_partial(populated_digraph):
g1 = populated_digraph
g2 = sn.DiGraph()
a = g2.add_node({"type": "A"}, '3caaa8c09148493dbdf02c574b95526c')
b = g2.add_node({"type": "B"}, '2cdfebf3bf9547f19f0412ccdfbe03b7')
d = g2.add_node({"type": "D"})
ab = g2.add_edge(a, b, {"type": "normal"}, '5f5f44ec7c0144e29c5b7d513f92d9ab')
bd = g2.add_edge(b, d, {"type": "irregular"})
gu = sn.union(g1, g2)
correct_nodes = {
a: {
'id': a,
'type': 'A'
},
b: {
'id': b,
'type': 'B'
},
uuid.UUID('3cd197c2cf5e42dc9ccd0c2adcaf4bc2'): {
'id': uuid.UUID('3cd197c2cf5e42dc9ccd0c2adcaf4bc2'),
'type': 'C'
},
d: {
'id': d,
'type': 'D'
}
}
assert gu.get_nodes() == correct_nodes
correct_edges = {
ab: {
'id': ab,
'src': a,
'dst': b,
'type': 'normal'
},
uuid.UUID('f3674fcc691848ebbd478b1bfb3e84c3'): {
'id': uuid.UUID('f3674fcc691848ebbd478b1bfb3e84c3'),
'src': b,
'dst': a,
'type': 'normal'
},
uuid.UUID('7eb91be54d3746b89a61a282bcc207bb'): {
'id': uuid.UUID('7eb91be54d3746b89a61a282bcc207bb'),
'src': a,
'dst': uuid.UUID('3cd197c2cf5e42dc9ccd0c2adcaf4bc2'),
'type': 'normal'
},
uuid.UUID('c172a3599b7d4ef3bbb688277276b763'): {
'id': uuid.UUID('c172a3599b7d4ef3bbb688277276b763'),
'src': uuid.UUID('2cdfebf3bf9547f19f0412ccdfbe03b7'),
'dst': uuid.UUID('3cd197c2cf5e42dc9ccd0c2adcaf4bc2'),
'type': 'irregular'
},
bd: {
'id': bd,
'src': b,
'dst': d,
'type': 'irregular'
}
}
gu.get_nodes() == correct_nodes
def test_union_disjoint(populated_digraph):
g1 = populated_digraph
g2 = sn.DiGraph()
a = g2.add_node({"label" : "A"}, 'a')
b = g2.add_node({"label" : "B"}, 'b')
c = g2.add_node({"label" : "C"}, 'c')
ab = g2.add_edge(a, b, {"type" : "belongs"}, 'belongs')
bc = g2.add_edge(b, c, {"type" : "owns"}, 'owns')
ca = g2.add_edge(c, a, {"type" : "has"}, 'has')
gu = sn.union(g1, g2)
correct_nodes = {
uuid.UUID('3caaa8c09148493dbdf02c574b95526c'): {
'id': uuid.UUID('3caaa8c09148493dbdf02c574b95526c'),
'type': 'A'
},
uuid.UUID('2cdfebf3bf9547f19f0412ccdfbe03b7'): {
'id': uuid.UUID('2cdfebf3bf9547f19f0412ccdfbe03b7'),
'type': 'B'
},
uuid.UUID('3cd197c2cf5e42dc9ccd0c2adcaf4bc2'): {
'id': uuid.UUID('3cd197c2cf5e42dc9ccd0c2adcaf4bc2'),
'type': 'C'
},
'a': {
'id': 'a',
'label': 'A'
},
'b': {
'id': 'b',
'label': 'B'
},
'c': {
'id': 'c',
'label': 'C'
}
}
assert gu.get_nodes() == correct_nodes
correct_edges = {
uuid.UUID('5f5f44ec7c0144e29c5b7d513f92d9ab'): {
'id': uuid.UUID('5f5f44ec7c0144e29c5b7d513f92d9ab'),
'src': uuid.UUID('3caaa8c09148493dbdf02c574b95526c'),
'dst': uuid.UUID('2cdfebf3bf9547f19f0412ccdfbe03b7'),
'type': 'normal'
},
uuid.UUID('f3674fcc691848ebbd478b1bfb3e84c3'): {
'id': uuid.UUID('f3674fcc691848ebbd478b1bfb3e84c3'),
'src': uuid.UUID('2cdfebf3bf9547f19f0412ccdfbe03b7'),
'dst': uuid.UUID('3caaa8c09148493dbdf02c574b95526c'),
'type': 'normal'
},
uuid.UUID('7eb91be54d3746b89a61a282bcc207bb'): {
'id': uuid.UUID('7eb91be54d3746b89a61a282bcc207bb'),
'src': uuid.UUID('3caaa8c09148493dbdf02c574b95526c'),
'dst': uuid.UUID('3cd197c2cf5e42dc9ccd0c2adcaf4bc2'),
'type': 'normal'
},
uuid.UUID('c172a3599b7d4ef3bbb688277276b763'): {
'id': uuid.UUID('c172a3599b7d4ef3bbb688277276b763'),
'src': uuid.UUID('2cdfebf3bf9547f19f0412ccdfbe03b7'),
'dst': uuid.UUID('3cd197c2cf5e42dc9ccd0c2adcaf4bc2'),
'type': 'irregular'
},
ab: {
'id': ab,
'src': a,
'dst': b,
'type': 'belongs'
},
bc: {
'id': bc,
'src': b,
'dst': c,
'type': 'owns'
},
ca: {
'id': ca,
'src': c,
'dst': a,
'type': 'has'
}
}
assert gu.get_edges() == correct_edges
args.outfile += "-m"
args.outfile += ".json"
if args.attr:
a_obj = json.load(open(args.old_graph, 'r'))
b_obj = json.load(open(args.new_graph, 'r'))
print("Converting old graph...")
attr_to_id(a_obj, args.attr)
print("Converting new graph...")
attr_to_id(b_obj, args.attr)
else:
a_obj = args.old_graph
b_obj = args.new_graph
A = sn.Graph() if args.undirected else sn.DiGraph()
A.load_json(a_obj)
B = sn.Graph() if args.undirected else sn.DiGraph()
B.load_json(b_obj)
print("Performing diff...")
if args.context:
print("and filtering out clutter...")
print
D = sn.diff(A, B, args.context, args.modifications)
print("Nodes added: {}".format(
len([
n for n, attrs in D.get_nodes().items()
if attrs['diffstatus'] == 'added'