def get_person_details(self, G):
"""
This function return unique ids and names of all people.
"""
totalExternalPersonsIds = []
totalPersonsIds = []
totalExternalPersonsName = []
totalPersonsName = []
personsName_GEI = []
personId_GEI = []
# Find total persons counts.
for node_id in search_nodes(G, {"==": [("label", ), "Person"]}):
if G.nodes[node_id]["staffType"] == "Academic":
totalPersonsIds.append(node_id)
# Getting Total external persons names in list.
totalPersonsName.append(G.nodes[node_id]["name"] + "(P)")
personsName_GEI.append(G.nodes[node_id]["name"])
personId_GEI.append(node_id)
# Find total external persons counts.
totalExternalPersonsIds = list()
for node_id in search_nodes(G,
{"==": [("label", ), "External person"]}):
totalExternalPersonsIds.append(node_id)
# Getting Total persons names in list.
totalExternalPersonsName.append(G.nodes[node_id]["name"] + "(EP)")
totalPeopleIds = totalPersonsIds + totalExternalPersonsIds
totalPeopleNames = totalPersonsName + totalExternalPersonsName
return totalPeopleIds, totalPeopleNames, personsName_GEI, personId_GEI
def test_search_nodes():
assert list(search_nodes(g, {"eq": [("product", ), "chocolate"]})) == [1]
assert list(search_nodes(g, {"eq": [("product", ), "milk"]})) == [2]
result = list(search_nodes(g, {"in": [("product", ), ["milk", "coat"]]}))
assert len(result) == 2
assert 2 in result
assert 3 in result
assert list(search_nodes(g, {"eq": [("product", ), "none"]})) == []
def find_node_by_name(self, node_name: str, label: str) -> str:
"""
Return a node id of a node with this name
:param node_name:
:param label: label/class of the node
:return:
"""
assert node_name is not None
my_graph = self.storage.get_graph(self.graph_id)
graph_nodes = list(
nxq.search_nodes(
my_graph, {
'and': [{
'eq': [ABCPropertyGraph.GRAPH_ID, self.graph_id]
}, {
'eq': [ABCPropertyGraph.PROP_NAME, node_name]
}, {
'eq': [ABCPropertyGraph.PROP_CLASS, label]
}]
}))
if len(graph_nodes) == 0:
raise PropertyGraphQueryException(
graph_id=self.graph_id,
node_id=None,
msg=f"Unable to find node with name {node_name} class {label}")
if len(graph_nodes) > 1:
raise PropertyGraphQueryException(
graph_id=self.graph_id,
node_id=None,
msg=
f"Graph contains multiple nodes with name {node_name} class {label}"
)
return my_graph.nodes[graph_nodes[0]][ABCPropertyGraph.NODE_ID]
def _find_node(self, *, node_id: str, graph_id: str = None) -> int:
"""
Find a node matching this node ID or raise a Query Exception,
returning an internal integer ID
:param node_id:
:return:
"""
assert node_id is not None
# this is a bit hacky - works for other graphs, not just us
if graph_id is None:
use_graph_id = self.graph_id
else:
use_graph_id = graph_id
query_match = list(
nxq.search_nodes(
self.storage.get_graph(use_graph_id), {
'and': [{
'eq': [ABCPropertyGraph.NODE_ID, node_id]
}, {
'eq': [ABCPropertyGraph.GRAPH_ID, use_graph_id]
}]
}))
if len(query_match) == 0:
raise PropertyGraphQueryException(graph_id=use_graph_id,
node_id=node_id,
msg="Unable to find node")
if len(query_match) > 1:
raise PropertyGraphQueryException(graph_id=use_graph_id,
node_id=node_id,
msg="Multiple matches found")
return query_match[0]
def node_exists(self, *, node_id: str, label: str) -> bool:
"""
Check if this node exists
:param node_id:
:param label:
:return:
"""
assert node_id is not None
assert label is not None
graph_nodes = list(
nxq.search_nodes(
self.storage.get_graph(self.graph_id), {
'and': [{
'eq': [ABCPropertyGraph.GRAPH_ID, self.graph_id]
}, {
'eq': [ABCPropertyGraph.NODE_ID, node_id]
}, {
'eq': [ABCPropertyGraph.PROP_CLASS, label]
}]
}))
if len(graph_nodes) > 1:
raise PropertyGraphQueryException(
node_id=node_id,
graph_id=self.graph_id,
msg="Duplicate node found while checking for node uniqueness")
return len(graph_nodes) == 1
def make_data(self):
"""Rebuild graph with emphasis on object type
:return:
"""
# get all existing object types
types = self.type_collector(self.G_labels)
print('found types: ' + str(types))
# generate pretty colors
type_colors = {}
for type in types:
type_colors[type] = "%06x" % random.randint(0, 0xFFFFFF)
# G2 = nx.Graph()
subnodes = {}
subgraphs = {}
for type in types:
subnodes[type] = []
query = {"contains": ["label", '"{}\n'.format(type)]}
print('nxquery: ' + str(query))
# find corresponsing nodes
for node_id in search_nodes(self.G, query):
# G2.add_node(node_id)
subnodes[type].append(node_id)
subgraphs[type] = self.G.subgraph(subnodes[type])
print('________')
print("ok!")
self.nx_to_pywiz(type_colors, subgraphs)
def get_tech_nodes(G):
tech_traits = ["Technology", "SystemSoftware", "Node", "Device"]
findings = []
for trait in tech_traits:
search = list(search_nodes(G, {"contains": ["type", trait]}))
for s in search:
findings.append(s)
return findings
def del_graph(self, graph_id: str) -> None:
# find all nodes of this graph and remove them
graph_nodes = list(
nxq.search_nodes(
self.graphs,
{'eq': [ABCPropertyGraph.GRAPH_ID, graph_id]}))
if graph_nodes is not None and len(graph_nodes) > 0:
self.graphs.remove_nodes_from(graph_nodes)
def graph_exists(self) -> bool:
"""
Does the graph with this ID exist?
:return:
"""
query_match = list(
nxq.search_nodes(
self.storage.get_graph(self.graph_id),
{'eq': [ABCPropertyGraph.GRAPH_ID, self.graph_id]}))
return len(query_match) > 0
def check_node_name(self, *, node_id: str, label: str, name: str) -> bool:
assert node_id is not None
assert name is not None
assert label is not None
graph_nodes = list(nxq.search_nodes(self.storage.get_graph(self.graph_id),
{'and': [
{'eq': [ABCPropertyGraphConstants.GRAPH_ID, self.graph_id]},
{'eq': [ABCPropertyGraphConstants.PROP_NAME, name]},
{'eq': [ABCPropertyGraphConstants.PROP_CLASS, label]},
{'eq': [ABCPropertyGraphConstants.NODE_ID, node_id]}
]}))
return len(graph_nodes) > 0
def _find_all_nodes(self) -> List[int]:
"""
Find all nodes in this graph returning the list of their internal int IDs
:return:
"""
query_match = list(
nxq.search_nodes(
self.storage.get_graph(self.graph_id),
{'eq': [ABCPropertyGraph.GRAPH_ID, self.graph_id]}))
if len(query_match) == 0:
raise PropertyGraphQueryException(graph_id=self.graph_id,
node_id=None,
msg="Unable to find graph nodes")
return query_match
def add_graph_direct(self, graph_id: str, graph: nx.Graph) -> None:
# check this graph_id isn't already present
existing_graph_nodes = list(
nxq.search_nodes(
self.graphs,
{'eq': [ABCPropertyGraph.GRAPH_ID, graph_id]}))
if len(existing_graph_nodes) > 0:
# graph already present, delete so we can replace
self.del_graph(graph_id)
# relabel incoming graph nodes to integers, then merge
temp_graph = nx.convert_node_labels_to_integers(
graph, first_label=self.start_id)
self.start_id = self.start_id + len(temp_graph.nodes())
self.graphs.add_nodes_from(temp_graph.nodes(data=True))
self.graphs.add_edges_from(temp_graph.edges(data=True))
def get_stitch_nodes(self) -> List[str]:
my_graph = self.storage.get_graph(self.graph_id)
graph_nodes = list(
nxq.search_nodes(
my_graph, {
'and': [{
'eq': [ABCPropertyGraph.GRAPH_ID, self.graph_id]
}, {
'eq': [ABCPropertyGraph.PROP_STITCH_NODE, 'true']
}]
}))
ret = list()
for n in graph_nodes:
ret.append(my_graph.nodes[n][ABCPropertyGraph.NODE_ID])
return ret
def connect_nodes(G: Graph, df: DataFrame) -> Graph:
"""
Populates a graph with edges based on a dataframe
"""
log.info(
f"NodeLinker: Initiating with a graph of {G.number_of_nodes()} nodes "
f"and a dataframe of shape {df.shape}", )
check_metadata(df=df)
edge_src, edge_dst, node = (
df.attrs["edge_src"],
df.attrs["edge_dst"],
df.attrs["node"],
)
n_targets, e_targets = (df.attrs["n_targets"], df.attrs["e_targets"])
log.info(
f"NodeLinker: Computing binary labels based on edge targets {e_targets}"
)
df["label"] = df[e_targets].apply(np.argmax, axis=1)
log.info(
f"NodeLinker: Adding node targets {n_targets} and edge targets {e_targets} as global attributes"
)
G.graph.update({"n_targets": n_targets, "e_targets": e_targets})
# G.graph.update({"node": node})
ebunch: List[Tuple[int, int, Dict[str, float]]] = []
for i, row in df.iterrows():
src_name: str = row[edge_src]
dst_name: str = row[edge_dst]
src_nid: int = list(search_nodes(G, {"==": [(node, ), src_name]}))[0]
dst_nid: int = list(search_nodes(G, {"==": [(node, ), dst_name]}))[0]
series_e_attrs: Series = row.drop(labels=[edge_src, edge_dst])
e_attrs: Dict[str, float] = series_e_attrs.to_dict()
ebunch.append((src_nid, dst_nid, e_attrs))
log.info(f"NodeLinker: Adding {len(ebunch)} edges to the graph")
G.add_edges_from(ebunch)
return G
def get_all_nodes_by_class(self, *, label: str) -> List[str]:
assert label is not None
my_graph = self.storage.get_graph(self.graph_id)
graph_nodes = list(
nxq.search_nodes(
my_graph, {
'and': [{
'eq': [ABCPropertyGraph.GRAPH_ID, self.graph_id]
}, {
'eq': [ABCPropertyGraph.PROP_CLASS, label]
}]
}))
ret = list()
for n in graph_nodes:
ret.append(my_graph.nodes[n][ABCPropertyGraph.NODE_ID])
return ret
def extract_graph(self, graph_id: str) -> nx.Graph or None:
# extract copy of graph from store or return None
graph_nodes = list(
nxq.search_nodes(
self.graphs,
{'eq': [ABCPropertyGraph.GRAPH_ID, graph_id]}))
if len(graph_nodes) == 0:
return None
# get adjacency (only gets edges and their properties)
edge_dict = nx.to_dict_of_dicts(self.graphs, graph_nodes)
# create new graph from edges
ret = nx.from_dict_of_dicts(edge_dict)
for n in graph_nodes:
# merge node dictionaries
ret.nodes[n].update(self.graphs.nodes[n])
return ret
def check_node_unique(self, *, label: str, name: str):
"""
Check no other node of this class/label and name exists
:param label:
:param name:
:return:
"""
graph_nodes = list(
nxq.search_nodes(
self.storage.get_graph(self.graph_id), {
'and': [{
'eq': [ABCPropertyGraph.GRAPH_ID, self.graph_id]
}, {
'eq': [ABCPropertyGraph.PROP_NAME, name]
}, {
'eq': [ABCPropertyGraph.PROP_CLASS, label]
}]
}))
return len(graph_nodes) == 0
def add_graph(self, graph_id: str, graph: nx.Graph) -> None:
# check this graph_id isn't already present
existing_graph_nodes = list(
nxq.search_nodes(
self.graphs,
{'eq': [ABCPropertyGraph.GRAPH_ID, graph_id]}))
if len(existing_graph_nodes) > 0:
# graph already present, delete it so we can replace
self.del_graph(graph_id)
# relabel incoming graph nodes to integers, then merge
temp_graph = nx.convert_node_labels_to_integers(
graph, first_label=self.start_id)
# set/overwrite GraphID property on all nodes
for n in list(temp_graph.nodes()):
if not temp_graph.nodes[n].get(ABCPropertyGraph.NODE_ID, None):
raise PropertyGraphImportException(
graph_id=graph_id,
msg=
"Some nodes are missing NodeID property, unable to import"
)
temp_graph.nodes[n][ABCPropertyGraph.GRAPH_ID] = graph_id
self.start_id = self.start_id + len(temp_graph.nodes())
self.graphs.add_nodes_from(temp_graph.nodes(data=True))
self.graphs.add_edges_from(temp_graph.edges(data=True))
def make_data(self):
"""Rebuild graph with emphasis on services and the "galaxies" they form
:return:
"""
# get all existing services
services = self.service_collector(nx.get_node_attributes(self.G, 'tags'))
print('found services: ' + str(services))
# generate pretty colors
service_colors = {}
for service in services:
service_colors[service] = "%06x" % random.randint(0, 0xFFFFFF)
# prepare first graph for node-node connection checking
G2 = nx.Graph()
# find service galaxy connections
print('building G2 reference graph...')
for service in services:
query = {"contains": ["tags", service]}
matching_sources = list(search_direct_relationships(graph=self.G, source=query))
matching_targets = list(search_direct_relationships(graph=self.G, target=query))
for r in matching_sources:
# add found nodes
G2.add_node(r[0])
G2.add_node(r[1])
# connect them
G2.add_edge(r[0], r[1])
for r in matching_targets:
# add found nodes
G2.add_node(r[0])
G2.add_node(r[1])
# connect them
G2.add_edge(r[0], r[1])
# testing: dump intermediate graph
# nx_to_pywiz(G2, [], service_colors, G_labels)
# build G3, similar to G2, but with subgraphs and referencing G2 to check for single-connection situations
# prepare galaxies graph
G3 = nx.Graph()
subgraphs = {}
subnodes = {}
# find service galaxy connections
for service in services:
query = {"contains": ["tags", service + "'"] }
print('nxquery: ' + str(query))
matching_sources = list(search_direct_relationships(graph=self.G, source=query))
matching_targets = list(search_direct_relationships(graph=self.G, target=query))
subnodes[service] = []
for r in matching_sources:
# verify that our finding is not just a substring match in tags
if service in self.get_node_service(self.G_tags, r[0]):
# add found nodes
G3.add_node(r[0])
subnodes[service].append(r[0])
G3.add_node(r[1])
# sort out the target and hwere it belongs
if self.just_one(G2, r[1]) and self.get_node_service(self.G_tags,
r[1]) == '': # situation: True, False core-network
subnodes[service].append(r[1])
elif not self.just_one(self.G, r[1]) and self.get_node_service(self.G_tags, r[1]) == '':
# okay, it might not be a singleton...
# but it's still missing a service!
if self.append_based_on_service(self.G, r[1], r[0]):
subnodes[service].append(r[1])
print('Added nodes {} and {}'.format(r[0], r[1]))
# connect them
G3.add_edge(r[0], r[1],
label='Origin: {}\nReason: {}'.format(self.get_edge_property(self.G, 'origin', r[0], r[1]),
self.get_edge_property(self.G, 'reason', r[0], r[1])))
print('Added edge {} to {}'.format(r[0], r[1]))
for r in matching_targets:
# verify that our finding is not just a substring match in tags
if service in self.get_node_service(self.G_tags, r[1]):
# add found nodes
G3.add_node(r[0])
# sort out the source
if self.just_one(G2, r[0]) and self.get_node_service(self.G_tags, r[0]) == '':
subnodes[service].append(r[0])
elif not self.just_one(self.G, r[0]) and self.get_node_service(self.G_tags, r[0]) == '':
# okay, it might not be a singleton...
# but it's still missing a service!
if self.append_based_on_service(self.G, r[0], r[1]):
subnodes[service].append(r[0])
G3.add_node(r[1])
subnodes[service].append(r[1])
print('Added nodes {} and {}'.format(r[0], r[1]))
# connect them
G3.add_edge(r[0], r[1],
label='Origin: {}\nReason: {}'.format(self.get_edge_property(self.G, 'origin', r[0], r[1]),
self.get_edge_property(self.G, 'reason', r[0], r[1])))
print('Added edge {} to {}'.format(r[0], r[1]))
# loner handling
loners = list(search_nodes(self.G, query))
for r in loners:
G3.add_node(r)
subnodes[service].append(r)
subgraphs[service] = G3.subgraph(subnodes[service])
print('________')
self.nx_to_pywiz(G3, subgraphs, service_colors, self.G_labels, self.G_tags)