def simulate_cascade(g, p, source=None, return_tree=False): """ graph_tool version of simulating cascade return np.ndarray on vertices as the infection time in cascade uninfected node has dist -1 """ gv = sample_graph_by_p(g, p) if source is None: # consider the largest cc infected_nodes = np.nonzero(label_largest_component(gv).a)[0] source = np.random.choice(infected_nodes) times = get_infection_time(gv, source) if return_tree: # get the tree edges _, pred_map = shortest_distance(gv, source=source, pred_map=True) edges = [(pred_map[i], i) for i in infected_nodes if i != source] # create tree tree = Graph(directed=True) tree.add_vertex(g.num_vertices()) for u, v in edges: tree.add_edge(int(u), int(v)) vfilt = tree.new_vertex_property('bool') vfilt.a = False for v in set(itertools.chain(*edges)): vfilt[v] = True tree.set_vertex_filter(vfilt) if return_tree: return source, times, tree else: return source, times
def steiner_tree_mst(g, root, infection_times, source, terminals, closure_builder=build_closure, strictly_smaller=True, return_closure=False, k=-1, debug=False, verbose=True): gc, eweight, r2pred = closure_builder(g, root, terminals, infection_times, strictly_smaller=strictly_smaller, k=k, debug=debug, verbose=verbose) # get the minimum spanning arborescence # graph_tool does not provide minimum_spanning_arborescence if verbose: print('getting mst') gx = gt2nx(gc, root, terminals, edge_attrs={'weight': eweight}) try: nx_tree = nx.minimum_spanning_arborescence(gx, 'weight') except nx.exception.NetworkXException: if debug: print('fail to find mst') if return_closure: return None, gc, None else: return None if verbose: print('returning tree') mst_tree = Graph(directed=True) for _ in range(g.num_vertices()): mst_tree.add_vertex() for u, v in nx_tree.edges(): mst_tree.add_edge(u, v) if verbose: print('extract edges from original graph') # extract the edges from the original graph # sort observations by time # and also topological order topological_index = {} for i, e in enumerate(bfs_iterator(mst_tree, source=root)): topological_index[int(e.target())] = i sorted_obs = sorted( set(terminals) - {root}, key=lambda o: (infection_times[o], topological_index[o])) tree_nodes = {root} tree_edges = set() # print('root', root) for u in sorted_obs: if u in tree_nodes: if debug: print('{} covered already'.format(u)) continue # print(u) v, u = map(int, next(mst_tree.vertex(u).in_edges())) # v is ancestor tree_nodes.add(v) late_nodes = [n for n in terminals if infection_times[n] > infection_times[u]] vis = init_visitor(g, u) # from child to any tree node, including v cpbfs_search(g, source=u, terminals=list(tree_nodes), forbidden_nodes=late_nodes, visitor=vis, count_threshold=1) # dist, pred = shortest_distance(g, source=u, pred_map=True) node_set = {v for v, d in vis.dist.items() if d > 0} reachable_tree_nodes = node_set.intersection(tree_nodes) ancestor = min(reachable_tree_nodes, key=vis.dist.__getitem__) edges = extract_edges_from_pred(g, u, ancestor, vis.pred) edges = {(j, i) for i, j in edges} # need to reverse it if debug: print('tree_nodes', tree_nodes) print('connecting {} to {}'.format(v, u)) print('using ancestor {}'.format(ancestor)) print('adding edges {}'.format(edges)) tree_nodes |= {u for e in edges for u in e} tree_edges |= edges t = Graph(directed=True) for _ in range(g.num_vertices()): t.add_vertex() for u, v in tree_edges: t.add_edge(t.vertex(u), t.vertex(v)) tree_nodes = {u for e in tree_edges for u in e} vfilt = t.new_vertex_property('bool') vfilt.a = False for v in tree_nodes: vfilt[t.vertex(v)] = True t.set_vertex_filter(vfilt) if return_closure: return t, gc, mst_tree else: return t
prev_q = cur_qual["q"] prev_c = cur_comp["c"] ver_attractions = pairs_graph.vertex_properties["attraction"] # cutting vertices with attractions in [low, hig] not_deleted = pairs_graph.vertex_properties["notdeleted"] for v in pairs_graph.vertices(): if low <= ver_attractions[v] <= hig: neutral.append(ver_names[v]) if ver_part[v] == 1: positive.remove(ver_names[v]) ver_part[v] = 0 elif ver_part[v] == -1: negative.remove(ver_names[v]) ver_part[v] = 0 not_deleted[v] = False pairs_graph.set_vertex_filter(not_deleted) pairs_graph.purge_vertices() # recalculating attractions, plotting and calculating quality and completeness pairs_graph.vertex_properties["attraction"] = get_attractions(pairs_graph) cur_qual = qual.get_quality(positive, negative, neutral) cur_comp = compl.get_completeness(positive, negative, neutral) print("-----------------------------------------------") print("Sum vol = " + str(len(neutral) + len(positive) + len(negative))) print("total neutral cutted = " + str(len(neutral))) print("pos = " + str(len(positive))) print("neg = " + str(len(negative))) print("Quality = " + str(cur_qual["q"])) print("Rude mist. % = " + str(cur_qual["rude"])) print(cur_qual["notfound"]) print("Completeness = " + str(cur_comp["c"])) print("-----------------------------------------------")
class PointerProvenancePlot(Plot): """ Base class for plots using the pointer provenance graph. """ def __init__(self, *args, **kwargs): super(PointerProvenancePlot, self).__init__(*args, **kwargs) self._cached_dataset_valid = False """Tells whether we need to rebuild the dataset when caching.""" def init_parser(self, dataset, tracefile): if self.caching and os.path.exists(self._get_cache_file()): # if caching we will nevere use this return None return PointerProvenanceParser(dataset, tracefile) def init_dataset(self): logger.debug("Init provenance graph for %s", self.tracefile) self.dataset = Graph(directed=True) vdata = self.dataset.new_vertex_property("object") self.dataset.vp["data"] = vdata return self.dataset def _get_cache_file(self): return self.tracefile + "_provenance_plot.gt" def build_dataset(self): """ Build the provenance tree """ if self.caching: try: logger.debug("Load cached provenance graph") self.dataset = load_graph(self._get_cache_file()) except IOError: self.parser.parse() self.dataset.save(self._get_cache_file()) else: self.parser.parse() num_nodes = self.dataset.num_vertices() logger.debug("Total nodes %d", num_nodes) vertex_mask = self.dataset.new_vertex_property("bool") progress = ProgressPrinter(num_nodes, desc="Search kernel nodes") for node in self.dataset.vertices(): # remove null capabilities # remove operations in kernel mode vertex_data = self.dataset.vp.data node_data = vertex_data[node] if ((node_data.pc != 0 and node_data.is_kernel) or (node_data.cap.length == 0 and node_data.cap.base == 0)): vertex_mask[node] = True progress.advance() progress.finish() self.dataset.set_vertex_filter(vertex_mask, inverted=True) vertex_mask = self.dataset.copy_property(vertex_mask) num_nodes = self.dataset.num_vertices() logger.debug("Filtered kernel nodes, remaining %d", num_nodes) progress = ProgressPrinter( num_nodes, desc="Merge (cfromptr + csetbounds) sequences") for node in self.dataset.vertices(): progress.advance() # merge cfromptr -> csetbounds subtrees num_parents = node.in_degree() if num_parents == 0: # root node continue elif num_parents > 1: logger.error("Found node with more than a single parent %s", node) raise RuntimeError("Too many parents for a node") parent = next(node.in_neighbours()) parent_data = self.dataset.vp.data[parent] node_data = self.dataset.vp.data[node] if (parent_data.origin == CheriNodeOrigin.FROMPTR and node_data.origin == CheriNodeOrigin.SETBOUNDS): # the child must be unique to avoid complex logic # when merging, it may be desirable to do so with # more complex traces node_data.origin = CheriNodeOrigin.PTR_SETBOUNDS if parent.in_degree() == 1: next_parent = next(parent.in_neighbours()) vertex_mask[parent] = True self.dataset.add_edge(next_parent, node) elif parent.in_degree() == 0: vertex_mask[parent] = True else: logger.error("Found node with more than a single parent %s", parent) raise RuntimeError("Too many parents for a node") progress.finish() self.dataset.set_vertex_filter(vertex_mask, inverted=True) vertex_mask = self.dataset.copy_property(vertex_mask) num_nodes = self.dataset.num_vertices() logger.debug("Merged (cfromptr + csetbounds), remaining %d", num_nodes) progress = ProgressPrinter(num_nodes, desc="Find short-lived cfromptr") for node in self.dataset.vertices(): progress.advance() node_data = self.dataset.vp.data[node] if node_data.origin == CheriNodeOrigin.FROMPTR: vertex_mask[node] = True # if (node_data.origin == CheriNodeOrigin.FROMPTR and # len(node_data.address) == 0 and # len(node_data.deref["load"]) == 0 and # len(node_data.deref["load"]) == 0): # # remove cfromptr that are never stored or used in # # a dereference # remove_list.append(node) progress.finish() self.dataset.set_vertex_filter(vertex_mask, inverted=True)
prev_q = cur_qual["q"] prev_c = cur_comp["c"] ver_attractions = pairs_graph.vertex_properties["attraction"] # cutting vertices with attractions in [low, hig] not_deleted = pairs_graph.vertex_properties["notdeleted"] for v in pairs_graph.vertices(): if low <= ver_attractions[v] <= hig: neutral.append(ver_names[v]) if ver_part[v] == 1: positive.remove(ver_names[v]) ver_part[v] = 0 elif ver_part[v] == -1: negative.remove(ver_names[v]) ver_part[v] = 0 not_deleted[v] = False pairs_graph.set_vertex_filter(not_deleted) pairs_graph.purge_vertices() # recalculating attractions, plotting and calculating quality and completeness pairs_graph.vertex_properties["attraction"] = get_attractions(pairs_graph) cur_qual = qual.get_quality(positive, negative, neutral) cur_comp = compl.get_completeness(positive, negative, neutral) print("-----------------------------------------------") print("Sum vol = " + str(len(neutral) + len(positive) + len(negative))) print("total neutral cutted = " + str(len(neutral))) print("pos = " + str(len(positive))) print("neg = " + str(len(negative))) print("Quality = " + str(cur_qual["q"])) print("Rude mist. % = " + str(cur_qual["rude"])) print(cur_qual["notfound"]) print("Completeness = " + str(cur_comp["c"])) print("-----------------------------------------------")
def steiner_tree_mst(g, root, infection_times, source, terminals, closure_builder=build_closure, strictly_smaller=True, return_closure=False, k=-1, debug=False, verbose=True): gc, eweight, r2pred = closure_builder(g, root, terminals, infection_times, strictly_smaller=strictly_smaller, k=k, debug=debug, verbose=verbose) # get the minimum spanning arborescence # graph_tool does not provide minimum_spanning_arborescence if verbose: print('getting mst') gx = gt2nx(gc, root, terminals, edge_attrs={'weight': eweight}) try: nx_tree = nx.minimum_spanning_arborescence(gx, 'weight') except nx.exception.NetworkXException: if debug: print('fail to find mst') if return_closure: return None, gc, None else: return None if verbose: print('returning tree') mst_tree = Graph(directed=True) for _ in range(g.num_vertices()): mst_tree.add_vertex() for u, v in nx_tree.edges(): mst_tree.add_edge(u, v) if verbose: print('extract edges from original graph') # extract the edges from the original graph # sort observations by time # and also topological order topological_index = {} for i, e in enumerate(bfs_iterator(mst_tree, source=root)): topological_index[int(e.target())] = i sorted_obs = sorted(set(terminals) - {root}, key=lambda o: (infection_times[o], topological_index[o])) tree_nodes = {root} tree_edges = set() # print('root', root) for u in sorted_obs: if u in tree_nodes: if debug: print('{} covered already'.format(u)) continue # print(u) v, u = map(int, next(mst_tree.vertex(u).in_edges())) # v is ancestor tree_nodes.add(v) late_nodes = [ n for n in terminals if infection_times[n] > infection_times[u] ] vis = init_visitor(g, u) # from child to any tree node, including v cpbfs_search(g, source=u, terminals=list(tree_nodes), forbidden_nodes=late_nodes, visitor=vis, count_threshold=1) # dist, pred = shortest_distance(g, source=u, pred_map=True) node_set = {v for v, d in vis.dist.items() if d > 0} reachable_tree_nodes = node_set.intersection(tree_nodes) ancestor = min(reachable_tree_nodes, key=vis.dist.__getitem__) edges = extract_edges_from_pred(g, u, ancestor, vis.pred) edges = {(j, i) for i, j in edges} # need to reverse it if debug: print('tree_nodes', tree_nodes) print('connecting {} to {}'.format(v, u)) print('using ancestor {}'.format(ancestor)) print('adding edges {}'.format(edges)) tree_nodes |= {u for e in edges for u in e} tree_edges |= edges t = Graph(directed=True) for _ in range(g.num_vertices()): t.add_vertex() for u, v in tree_edges: t.add_edge(t.vertex(u), t.vertex(v)) tree_nodes = {u for e in tree_edges for u in e} vfilt = t.new_vertex_property('bool') vfilt.a = False for v in tree_nodes: vfilt[t.vertex(v)] = True t.set_vertex_filter(vfilt) if return_closure: return t, gc, mst_tree else: return t