def get_person_details(self, G): """ This function return unique ids and names of all people. """ totalExternalPersonsIds = [] totalPersonsIds = [] totalExternalPersonsName = [] totalPersonsName = [] personsName_GEI = [] personId_GEI = [] # Find total persons counts. for node_id in search_nodes(G, {"==": [("label", ), "Person"]}): if G.nodes[node_id]["staffType"] == "Academic": totalPersonsIds.append(node_id) # Getting Total external persons names in list. totalPersonsName.append(G.nodes[node_id]["name"] + "(P)") personsName_GEI.append(G.nodes[node_id]["name"]) personId_GEI.append(node_id) # Find total external persons counts. totalExternalPersonsIds = list() for node_id in search_nodes(G, {"==": [("label", ), "External person"]}): totalExternalPersonsIds.append(node_id) # Getting Total persons names in list. totalExternalPersonsName.append(G.nodes[node_id]["name"] + "(EP)") totalPeopleIds = totalPersonsIds + totalExternalPersonsIds totalPeopleNames = totalPersonsName + totalExternalPersonsName return totalPeopleIds, totalPeopleNames, personsName_GEI, personId_GEI
def test_search_nodes(): assert list(search_nodes(g, {"eq": [("product", ), "chocolate"]})) == [1] assert list(search_nodes(g, {"eq": [("product", ), "milk"]})) == [2] result = list(search_nodes(g, {"in": [("product", ), ["milk", "coat"]]})) assert len(result) == 2 assert 2 in result assert 3 in result assert list(search_nodes(g, {"eq": [("product", ), "none"]})) == []
def find_node_by_name(self, node_name: str, label: str) -> str: """ Return a node id of a node with this name :param node_name: :param label: label/class of the node :return: """ assert node_name is not None my_graph = self.storage.get_graph(self.graph_id) graph_nodes = list( nxq.search_nodes( my_graph, { 'and': [{ 'eq': [ABCPropertyGraph.GRAPH_ID, self.graph_id] }, { 'eq': [ABCPropertyGraph.PROP_NAME, node_name] }, { 'eq': [ABCPropertyGraph.PROP_CLASS, label] }] })) if len(graph_nodes) == 0: raise PropertyGraphQueryException( graph_id=self.graph_id, node_id=None, msg=f"Unable to find node with name {node_name} class {label}") if len(graph_nodes) > 1: raise PropertyGraphQueryException( graph_id=self.graph_id, node_id=None, msg= f"Graph contains multiple nodes with name {node_name} class {label}" ) return my_graph.nodes[graph_nodes[0]][ABCPropertyGraph.NODE_ID]
def _find_node(self, *, node_id: str, graph_id: str = None) -> int: """ Find a node matching this node ID or raise a Query Exception, returning an internal integer ID :param node_id: :return: """ assert node_id is not None # this is a bit hacky - works for other graphs, not just us if graph_id is None: use_graph_id = self.graph_id else: use_graph_id = graph_id query_match = list( nxq.search_nodes( self.storage.get_graph(use_graph_id), { 'and': [{ 'eq': [ABCPropertyGraph.NODE_ID, node_id] }, { 'eq': [ABCPropertyGraph.GRAPH_ID, use_graph_id] }] })) if len(query_match) == 0: raise PropertyGraphQueryException(graph_id=use_graph_id, node_id=node_id, msg="Unable to find node") if len(query_match) > 1: raise PropertyGraphQueryException(graph_id=use_graph_id, node_id=node_id, msg="Multiple matches found") return query_match[0]
def node_exists(self, *, node_id: str, label: str) -> bool: """ Check if this node exists :param node_id: :param label: :return: """ assert node_id is not None assert label is not None graph_nodes = list( nxq.search_nodes( self.storage.get_graph(self.graph_id), { 'and': [{ 'eq': [ABCPropertyGraph.GRAPH_ID, self.graph_id] }, { 'eq': [ABCPropertyGraph.NODE_ID, node_id] }, { 'eq': [ABCPropertyGraph.PROP_CLASS, label] }] })) if len(graph_nodes) > 1: raise PropertyGraphQueryException( node_id=node_id, graph_id=self.graph_id, msg="Duplicate node found while checking for node uniqueness") return len(graph_nodes) == 1
def make_data(self): """Rebuild graph with emphasis on object type :return: """ # get all existing object types types = self.type_collector(self.G_labels) print('found types: ' + str(types)) # generate pretty colors type_colors = {} for type in types: type_colors[type] = "%06x" % random.randint(0, 0xFFFFFF) # G2 = nx.Graph() subnodes = {} subgraphs = {} for type in types: subnodes[type] = [] query = {"contains": ["label", '"{}\n'.format(type)]} print('nxquery: ' + str(query)) # find corresponsing nodes for node_id in search_nodes(self.G, query): # G2.add_node(node_id) subnodes[type].append(node_id) subgraphs[type] = self.G.subgraph(subnodes[type]) print('________') print("ok!") self.nx_to_pywiz(type_colors, subgraphs)
def get_tech_nodes(G): tech_traits = ["Technology", "SystemSoftware", "Node", "Device"] findings = [] for trait in tech_traits: search = list(search_nodes(G, {"contains": ["type", trait]})) for s in search: findings.append(s) return findings
def del_graph(self, graph_id: str) -> None: # find all nodes of this graph and remove them graph_nodes = list( nxq.search_nodes( self.graphs, {'eq': [ABCPropertyGraph.GRAPH_ID, graph_id]})) if graph_nodes is not None and len(graph_nodes) > 0: self.graphs.remove_nodes_from(graph_nodes)
def graph_exists(self) -> bool: """ Does the graph with this ID exist? :return: """ query_match = list( nxq.search_nodes( self.storage.get_graph(self.graph_id), {'eq': [ABCPropertyGraph.GRAPH_ID, self.graph_id]})) return len(query_match) > 0
def check_node_name(self, *, node_id: str, label: str, name: str) -> bool: assert node_id is not None assert name is not None assert label is not None graph_nodes = list(nxq.search_nodes(self.storage.get_graph(self.graph_id), {'and': [ {'eq': [ABCPropertyGraphConstants.GRAPH_ID, self.graph_id]}, {'eq': [ABCPropertyGraphConstants.PROP_NAME, name]}, {'eq': [ABCPropertyGraphConstants.PROP_CLASS, label]}, {'eq': [ABCPropertyGraphConstants.NODE_ID, node_id]} ]})) return len(graph_nodes) > 0
def _find_all_nodes(self) -> List[int]: """ Find all nodes in this graph returning the list of their internal int IDs :return: """ query_match = list( nxq.search_nodes( self.storage.get_graph(self.graph_id), {'eq': [ABCPropertyGraph.GRAPH_ID, self.graph_id]})) if len(query_match) == 0: raise PropertyGraphQueryException(graph_id=self.graph_id, node_id=None, msg="Unable to find graph nodes") return query_match
def add_graph_direct(self, graph_id: str, graph: nx.Graph) -> None: # check this graph_id isn't already present existing_graph_nodes = list( nxq.search_nodes( self.graphs, {'eq': [ABCPropertyGraph.GRAPH_ID, graph_id]})) if len(existing_graph_nodes) > 0: # graph already present, delete so we can replace self.del_graph(graph_id) # relabel incoming graph nodes to integers, then merge temp_graph = nx.convert_node_labels_to_integers( graph, first_label=self.start_id) self.start_id = self.start_id + len(temp_graph.nodes()) self.graphs.add_nodes_from(temp_graph.nodes(data=True)) self.graphs.add_edges_from(temp_graph.edges(data=True))
def get_stitch_nodes(self) -> List[str]: my_graph = self.storage.get_graph(self.graph_id) graph_nodes = list( nxq.search_nodes( my_graph, { 'and': [{ 'eq': [ABCPropertyGraph.GRAPH_ID, self.graph_id] }, { 'eq': [ABCPropertyGraph.PROP_STITCH_NODE, 'true'] }] })) ret = list() for n in graph_nodes: ret.append(my_graph.nodes[n][ABCPropertyGraph.NODE_ID]) return ret
def connect_nodes(G: Graph, df: DataFrame) -> Graph: """ Populates a graph with edges based on a dataframe """ log.info( f"NodeLinker: Initiating with a graph of {G.number_of_nodes()} nodes " f"and a dataframe of shape {df.shape}", ) check_metadata(df=df) edge_src, edge_dst, node = ( df.attrs["edge_src"], df.attrs["edge_dst"], df.attrs["node"], ) n_targets, e_targets = (df.attrs["n_targets"], df.attrs["e_targets"]) log.info( f"NodeLinker: Computing binary labels based on edge targets {e_targets}" ) df["label"] = df[e_targets].apply(np.argmax, axis=1) log.info( f"NodeLinker: Adding node targets {n_targets} and edge targets {e_targets} as global attributes" ) G.graph.update({"n_targets": n_targets, "e_targets": e_targets}) # G.graph.update({"node": node}) ebunch: List[Tuple[int, int, Dict[str, float]]] = [] for i, row in df.iterrows(): src_name: str = row[edge_src] dst_name: str = row[edge_dst] src_nid: int = list(search_nodes(G, {"==": [(node, ), src_name]}))[0] dst_nid: int = list(search_nodes(G, {"==": [(node, ), dst_name]}))[0] series_e_attrs: Series = row.drop(labels=[edge_src, edge_dst]) e_attrs: Dict[str, float] = series_e_attrs.to_dict() ebunch.append((src_nid, dst_nid, e_attrs)) log.info(f"NodeLinker: Adding {len(ebunch)} edges to the graph") G.add_edges_from(ebunch) return G
def get_all_nodes_by_class(self, *, label: str) -> List[str]: assert label is not None my_graph = self.storage.get_graph(self.graph_id) graph_nodes = list( nxq.search_nodes( my_graph, { 'and': [{ 'eq': [ABCPropertyGraph.GRAPH_ID, self.graph_id] }, { 'eq': [ABCPropertyGraph.PROP_CLASS, label] }] })) ret = list() for n in graph_nodes: ret.append(my_graph.nodes[n][ABCPropertyGraph.NODE_ID]) return ret
def extract_graph(self, graph_id: str) -> nx.Graph or None: # extract copy of graph from store or return None graph_nodes = list( nxq.search_nodes( self.graphs, {'eq': [ABCPropertyGraph.GRAPH_ID, graph_id]})) if len(graph_nodes) == 0: return None # get adjacency (only gets edges and their properties) edge_dict = nx.to_dict_of_dicts(self.graphs, graph_nodes) # create new graph from edges ret = nx.from_dict_of_dicts(edge_dict) for n in graph_nodes: # merge node dictionaries ret.nodes[n].update(self.graphs.nodes[n]) return ret
def check_node_unique(self, *, label: str, name: str): """ Check no other node of this class/label and name exists :param label: :param name: :return: """ graph_nodes = list( nxq.search_nodes( self.storage.get_graph(self.graph_id), { 'and': [{ 'eq': [ABCPropertyGraph.GRAPH_ID, self.graph_id] }, { 'eq': [ABCPropertyGraph.PROP_NAME, name] }, { 'eq': [ABCPropertyGraph.PROP_CLASS, label] }] })) return len(graph_nodes) == 0
def add_graph(self, graph_id: str, graph: nx.Graph) -> None: # check this graph_id isn't already present existing_graph_nodes = list( nxq.search_nodes( self.graphs, {'eq': [ABCPropertyGraph.GRAPH_ID, graph_id]})) if len(existing_graph_nodes) > 0: # graph already present, delete it so we can replace self.del_graph(graph_id) # relabel incoming graph nodes to integers, then merge temp_graph = nx.convert_node_labels_to_integers( graph, first_label=self.start_id) # set/overwrite GraphID property on all nodes for n in list(temp_graph.nodes()): if not temp_graph.nodes[n].get(ABCPropertyGraph.NODE_ID, None): raise PropertyGraphImportException( graph_id=graph_id, msg= "Some nodes are missing NodeID property, unable to import" ) temp_graph.nodes[n][ABCPropertyGraph.GRAPH_ID] = graph_id self.start_id = self.start_id + len(temp_graph.nodes()) self.graphs.add_nodes_from(temp_graph.nodes(data=True)) self.graphs.add_edges_from(temp_graph.edges(data=True))
def make_data(self): """Rebuild graph with emphasis on services and the "galaxies" they form :return: """ # get all existing services services = self.service_collector(nx.get_node_attributes(self.G, 'tags')) print('found services: ' + str(services)) # generate pretty colors service_colors = {} for service in services: service_colors[service] = "%06x" % random.randint(0, 0xFFFFFF) # prepare first graph for node-node connection checking G2 = nx.Graph() # find service galaxy connections print('building G2 reference graph...') for service in services: query = {"contains": ["tags", service]} matching_sources = list(search_direct_relationships(graph=self.G, source=query)) matching_targets = list(search_direct_relationships(graph=self.G, target=query)) for r in matching_sources: # add found nodes G2.add_node(r[0]) G2.add_node(r[1]) # connect them G2.add_edge(r[0], r[1]) for r in matching_targets: # add found nodes G2.add_node(r[0]) G2.add_node(r[1]) # connect them G2.add_edge(r[0], r[1]) # testing: dump intermediate graph # nx_to_pywiz(G2, [], service_colors, G_labels) # build G3, similar to G2, but with subgraphs and referencing G2 to check for single-connection situations # prepare galaxies graph G3 = nx.Graph() subgraphs = {} subnodes = {} # find service galaxy connections for service in services: query = {"contains": ["tags", service + "'"] } print('nxquery: ' + str(query)) matching_sources = list(search_direct_relationships(graph=self.G, source=query)) matching_targets = list(search_direct_relationships(graph=self.G, target=query)) subnodes[service] = [] for r in matching_sources: # verify that our finding is not just a substring match in tags if service in self.get_node_service(self.G_tags, r[0]): # add found nodes G3.add_node(r[0]) subnodes[service].append(r[0]) G3.add_node(r[1]) # sort out the target and hwere it belongs if self.just_one(G2, r[1]) and self.get_node_service(self.G_tags, r[1]) == '': # situation: True, False core-network subnodes[service].append(r[1]) elif not self.just_one(self.G, r[1]) and self.get_node_service(self.G_tags, r[1]) == '': # okay, it might not be a singleton... # but it's still missing a service! if self.append_based_on_service(self.G, r[1], r[0]): subnodes[service].append(r[1]) print('Added nodes {} and {}'.format(r[0], r[1])) # connect them G3.add_edge(r[0], r[1], label='Origin: {}\nReason: {}'.format(self.get_edge_property(self.G, 'origin', r[0], r[1]), self.get_edge_property(self.G, 'reason', r[0], r[1]))) print('Added edge {} to {}'.format(r[0], r[1])) for r in matching_targets: # verify that our finding is not just a substring match in tags if service in self.get_node_service(self.G_tags, r[1]): # add found nodes G3.add_node(r[0]) # sort out the source if self.just_one(G2, r[0]) and self.get_node_service(self.G_tags, r[0]) == '': subnodes[service].append(r[0]) elif not self.just_one(self.G, r[0]) and self.get_node_service(self.G_tags, r[0]) == '': # okay, it might not be a singleton... # but it's still missing a service! if self.append_based_on_service(self.G, r[0], r[1]): subnodes[service].append(r[0]) G3.add_node(r[1]) subnodes[service].append(r[1]) print('Added nodes {} and {}'.format(r[0], r[1])) # connect them G3.add_edge(r[0], r[1], label='Origin: {}\nReason: {}'.format(self.get_edge_property(self.G, 'origin', r[0], r[1]), self.get_edge_property(self.G, 'reason', r[0], r[1]))) print('Added edge {} to {}'.format(r[0], r[1])) # loner handling loners = list(search_nodes(self.G, query)) for r in loners: G3.add_node(r) subnodes[service].append(r) subgraphs[service] = G3.subgraph(subnodes[service]) print('________') self.nx_to_pywiz(G3, subgraphs, service_colors, self.G_labels, self.G_tags)