class ZonedNetwork: def __init__(self, size: Tuple[int] = (10, 10), field_size: Tuple[int] = (100, 100)): self.g = Graph(directed=True) self.n_zones = size[0] * size[1] self.fwidth = field_size[0] self.fheight = field_size[1] self.n_rows = size[0] self.n_cols = size[1] self.row_size: float = self.fheight / self.n_rows self.col_size: float = self.fwidth / self.n_cols self.g.add_vertex(self.n_zones) def get_zone(self, coords: Tuple): r = int(coords[1] / self.row_size) c = int(coords[0] / self.col_size) r = min(self.n_rows - 1, r) c = min(self.n_cols - 1, c) return self.g.vertex(r * self.n_cols + c) def add_passes(self, coords_pairs: List[Tuple]): pairs = [(self.get_zone((x1, y1)), self.get_zone((x2, y2))) for x1, y1, x2, y2 in coords_pairs] return self.g.add_edge_list(pairs) def save(self, file: str): self.g.save(file, fmt='graphml')
def vytvořím_graph_tool_graf(): from graph_tool.all import Graph graf = Graph() u1 = graf.add_vertex() u2 = graf.add_vertex() graf.add_edge(u1, u2) vprop_double = graf.new_vertex_property("double") # Double-precision floating point vprop_double[graf.vertex(1)] = 3.1416 vprop_vint = graf.new_vertex_property("vector<int>") # Vector of ints vprop_vint[graf.vertex(0)] = [1, 3, 42, 54] eprop_dict = graf.new_edge_property("object") # Arbitrary python object. In this case, a dictionary. eprop_dict[graf.edges().next()] = {"foo": "bar", "gnu": 42} gprop_bool = graf.new_graph_property("bool") # Boolean gprop_bool[graf] = True graf.save('./data/graph_tool.graphml', fmt='xml')
class Network: def __init__(self, nodes_info=None, links_info=None, file_name=None): self.g = Graph() if nodes_info and links_info: self.nodes_info = nodes_info self.links_info = links_info self.g.vertex_properties["name"] = self.g.new_vertex_property( 'string') self.g.vertex_properties["id"] = self.g.new_vertex_property( 'int32_t') self.g.edge_properties["weight"] = self.g.new_edge_property( 'int32_t') self.create_network() self.g.vertex_properties["pagerank"] = pagerank( self.g, weight=self.g.edge_properties["weight"]) self.g.vertex_properties[ "degree_centrality"] = self.degree_centrality() elif file_name: self.load_network(file_name) def create_network(self): # Add Nodes for node in self.nodes_info: self.add_n(node) # Add Links for link in self.links_info: n_loser = 0 n_winner = 0 loser = link['loser'] winner = link['winner'] weight = link['rounds'] for team_id in self.g.vertex_properties.id: if loser == team_id: break n_loser += 1 for team_id in self.g.vertex_properties.id: if winner == team_id: break n_winner += 1 self.add_l(n_loser, n_winner, 16 / weight * 100) def load_network(self, file_name): new_file_name = '..' + sep + '..' + sep + 'network-graphs' + sep + file_name self.g.load(new_file_name, fmt="gt") def get_normalized_pagerank(self): max_pgr = 0 for pgr in self.g.vertex_properties.pagerank: if pgr > max_pgr: max_pgr = pgr return [ self.g.vertex_properties.pagerank[v] / max_pgr for v in self.g.vertices() ] def add_n(self, node_info): n = self.g.add_vertex() self.g.vertex_properties.id[n] = node_info['id'] self.g.vertex_properties.name[n] = node_info['Team_Name'] def add_l(self, loser, winner, weight): n1 = self.g.vertex(loser) n2 = self.g.vertex(winner) l = self.g.add_edge(n1, n2) self.g.edge_properties.weight[l] = weight def draw(self, output_file, fmt): graph_draw(self.g, vertex_text=self.g.vertex_index, output=output_file, fmt=fmt) def save_network(self, file_name): try: new_file_name = '..' + sep + '..' + sep + 'network-graphs' + sep + file_name self.g.save(new_file_name, fmt="gt") except: return False return True def vp_pagerank(self): return self.g.vertex_properties.pagerank def vp_degree_cent(self): return self.g.vertex_properties.degree_centrality def vp_name(self): return self.g.vertex_properties.name def vp_id(self): return self.g.vertex_properties.id def ep_weight(self): return self.g.edge_properties.weight # Calcula as características básicas da rede def get_basic_info(self): info = {} try: n_vertices = self.g.num_vertices() n_edges = self.g.num_edges() density = n_edges / ((n_vertices * (n_vertices - 1)) / 2) mean_degree = (2 * n_edges) / n_vertices # Cálculo do coeficiente de clusterização "na mão", usando a média dos # coeficientes locais calculados pela Graph Tools local_cc = local_clustering(self.g) clustering_coef = fsum( [local_cc[x] for x in self.g.vertices() if local_cc[x] != 0.0]) clustering_coef /= n_vertices info["Número de times"] = n_vertices info["Número de confrontos"] = n_edges info["Densidade"] = density info["Grau médio"] = mean_degree info["Coeficiente de Clusterização"] = clustering_coef except: info.clear() return info def degree_centrality(self): degree_centrality = self.g.new_vertex_property('float') for v in self.g.vertices(): degree_centrality[v] = v.in_degree() / (self.g.num_vertices() - 1) return degree_centrality # Calcula a distribuição de graus da rede def degree_distribution(self): degree_dist = {} try: for v in self.g.vertices(): if v.in_degree() not in degree_dist.keys(): degree_dist[v.in_degree()] = 1 else: degree_dist[v.in_degree()] += 1 for k in degree_dist.keys(): degree_dist[k] /= self.g.num_vertices() except: degree_dist.clear() return degree_dist
class GeneralGraph(): """ General wrapper for graph-tool or networkx graphs to add edges and nodes according to constraints """ def __init__(self, directed=True, verbose=1): self.graphtool = GRAPH_TOOL # Initialize graph if self.graphtool: self.graph = Graph(directed=directed) self.weight = self.graph.new_edge_property("float") else: if directed: print("directed graph") self.graph = nx.DiGraph() else: self.graph = nx.Graph() # set metaparameter self.time_logs = {} self.verbose = verbose def set_edge_costs(self, layer_classes=["resistance"], class_weights=[1], **kwargs): """ Initialize edge cost variables :param classes: list of cost categories :param weights: list of weights for cost categories - must be of same shape as classes (if None, then equal weighting) """ class_weights = np.array(class_weights) # set different costs: self.cost_classes = layer_classes if self.graphtool: self.cost_props = [ self.graph.new_edge_property("float") for _ in range(len(layer_classes)) ] self.cost_weights = class_weights / np.sum(class_weights) if self.verbose: print(self.cost_classes, self.cost_weights) # save weighted instance for plotting self.instance = np.sum( np.moveaxis(self.cost_instance, 0, -1) * self.cost_weights, axis=2) * self.hard_constraints def set_shift(self, start, dest, pylon_dist_min=3, pylon_dist_max=5, max_angle=np.pi / 2, **kwargs): """ Initialize shift variable by getting the donut values :param lower, upper: min and max distance of pylons :param vec: vector of diretion of edges :param max_angle: Maximum angle of edges to vec """ vec = dest - start if self.verbose: print("SHIFT:", pylon_dist_min, pylon_dist_max, vec, max_angle) self.shifts = get_half_donut(pylon_dist_min, pylon_dist_max, vec, angle_max=max_angle) self.shift_tuples = self.shifts def set_corridor(self, dist_surface, start_inds, dest_inds, sample_func="mean", sample_method="simple", factor_or_n_edges=1): # set new corridor corridor = (dist_surface > 0).astype(int) self.factor = factor_or_n_edges self.cost_rest = self.cost_instance * (self.hard_constraints > 0).astype(int) * corridor # downsample tic = time.time() if self.factor > 1: self.cost_rest = CostUtils.downsample(self.cost_rest, self.factor, mode=sample_method, func=sample_func) self.time_logs["downsample"] = round(time.time() - tic, 3) # repeat because edge artifacts self.cost_rest = self.cost_rest * (self.hard_constraints > 0).astype(int) * corridor # add start and end TODO ugly self.cost_rest[:, dest_inds[0], dest_inds[1]] = self.cost_instance[:, dest_inds[0], dest_inds[1]] self.cost_rest[:, start_inds[0], start_inds[1]] = self.cost_instance[:, start_inds[0], start_inds[1]] def add_nodes(self, nodes): """ Add vertices to the graph param nodes: list of node names if networkx, integer if graphtool """ tic = time.time() # add nodes to graph if self.graphtool: _ = self.graph.add_vertex(nodes) self.n_nodes = len(list(self.graph.vertices())) else: self.graph.add_nodes_from(np.arange(nodes)) self.n_nodes = len(self.graph.nodes()) # verbose if self.verbose: print("Added nodes:", nodes, "in time:", time.time() - tic) self.time_logs["add_nodes"] = round(time.time() - tic, 3) def add_edges(self): tic_function = time.time() n_edges = 0 # kernels, posneg = ConstraintUtils.get_kernel(self.shifts, # self.shift_vals) # edge_array = [] times_edge_list = [] times_add_edges = [] if self.verbose: print("n_neighbors:", len(self.shift_tuples)) for i in range(len(self.shift_tuples)): tic_edges = time.time() # set cost rest if necessary (random graph) self.set_cost_rest() # compute shift and weights out = self._compute_edges(self.shift_tuples[i]) # Error if -1 entries because graph-tool crashes with -1 nodes if np.any(out[:, :2].flatten() < 0): print(np.where(out[:, :2] < 0)) raise RuntimeError n_edges += len(out) times_edge_list.append(round(time.time() - tic_edges, 3)) # add edges to graph tic_graph = time.time() if self.graphtool: self.graph.add_edge_list(out, eprops=self.cost_props) else: nx_edge_list = [(e[0], e[1], { "weight": np.sum(e[2:] * self.cost_weights) }) for e in out] self.graph.add_edges_from(nx_edge_list) times_add_edges.append(round(time.time() - tic_graph, 3)) # alternative: collect edges here and add alltogether # edge_array.append(out) # # alternative: add edges all in one go # tic_concat = time.time() # edge_lists_concat = np.concatenate(edge_array, axis=0) # self.time_logs["concatenate"] = round(time.time() - tic_concat, 3) # print("time for concatenate:", self.time_logs["concatenate"]) # tic_graph = time.time() # self.graph.add_edge_list(edge_lists_concat, eprops=[self.weight]) # self.time_logs["add_edges"] = round( # (time.time() - tic_graph) / len(shifts), 3 # ) self.n_edges = len(list(self.graph.edges())) self._update_time_logs(times_add_edges, times_edge_list, tic_function) if self.verbose: print("DONE adding", n_edges, "edges:", time.time() - tic_function) def _update_time_logs(self, times_add_edges, times_edge_list, tic_function): self.time_logs["add_edges"] = round(np.mean(times_add_edges), 3) self.time_logs["add_edges_times"] = times_add_edges self.time_logs["edge_list"] = round(np.mean(times_edge_list), 3) self.time_logs["edge_list_times"] = times_edge_list self.time_logs["add_all_edges"] = round(time.time() - tic_function, 3) if self.verbose: print("Done adding edges:", len(list(self.graph.edges()))) def sum_costs(self): """ Additive weighting of costs Take the individual edge costs, compute weighted sum --> self.weight """ # add sum of all costs if not self.graphtool: return tic = time.time() summed_costs_arr = np.zeros(self.cost_props[0].get_array().shape) for i in range(len(self.cost_props)): prop = self.cost_props[i].get_array() summed_costs_arr += prop * self.cost_weights[i] self.weight.a = summed_costs_arr self.time_logs["sum_of_costs"] = round(time.time() - tic, 3) def remove_vertices(self, dist_surface, delete_padding=0): """ Remove edges in a certain corridor (or all) to replace them by a refined surface @param dist_surface: a surface where each pixel value corresponds to the distance of the pixel to the shortest path @param delete_padding: define padding in which part of the corridor to delete vertices (cannot delete all because then graph unconnected) """ tic = time.time() self.graph.clear_edges() self.graph.shrink_to_fit() self.time_logs["remove_edges"] = round(time.time() - tic, 3) def get_pareto(self, vary, source, dest, out_path=None, compare=[0, 1], plot=1): """ Arguments: vary: how many weights to explore e.g 3 --> each cost class can have weight 0, 0.5 or 1 source, dest: as always the source and destination vertex out_path: where to save the pareto figure(s) compare: indices of cost classes to compare Returns: paths: All found paths pareto: The costs for each combination of weights """ tic = time.time() # initialize lists pareto = list() paths = list() cost_sum = list() # get the edge costs cost_arrs = [cost.get_array() for cost in self.cost_props] # [self.cost_props[comp].get_array() for comp in compare] # get vary weights between 0 and 1 var_weights = np.around(np.linspace(0, 1, vary), 2) # construct weights array if len(compare) == 2: weights = [[v, 1 - v] for v in var_weights] elif len(compare) == 3: weights = list() for w0 in var_weights: for w1 in var_weights[var_weights <= 1 - w0]: weights.append([w0, w1, 1 - w0 - w1]) else: raise ValueError("argument compare can only have length 2 or 3") # w_avail: keep weights of non-compare classes, get leftover amount w_avail = np.sum(np.asarray(self.cost_weights)[compare]) # compute paths for each combination of weights for j in range(len(weights)): # option 2: np.zeros(len(cost_arrs)) + non_compare_weight w = self.cost_weights.copy() # replace the ones we want to compare w[compare] = np.array(weights[j]) * w_avail # weighted sum of edge costs self.weight.a = np.sum( [cost_arrs[i] * w[i] for i in range(len(cost_arrs))], axis=0) # get shortest path path, path_costs, _ = self.get_shortest_path(source, dest) # don't take cost_sum bc this is sum of original weighting pareto.append(np.sum(path_costs, axis=0)[compare]) paths.append(path) # take overall sum of costs (unweighted) that this w leads to cost_sum.append(np.sum(path_costs)) # print best weighting best_weight = np.argmin(cost_sum) w = self.cost_weights.copy() w[compare] = np.array(weights[best_weight]) * w_avail print("Best weights:", w, "with (unweighted) costs:", np.min(cost_sum)) self.time_logs["pareto"] = round(time.time() - tic, 3) pareto = np.array(pareto) classes = [self.cost_classes[comp] for comp in compare] # Plotting if plot: if len(compare) == 2: plot_pareto_scatter_2d(pareto, weights, classes, cost_sum=cost_sum, out_path=out_path) elif len(compare) == 3: # plot_pareto_3d(pareto, weights, classes) plot_pareto_scatter_3d(pareto, weights, classes, cost_sum=cost_sum, out_path=out_path) return paths, weights, cost_sum def get_shortest_path(self, source, target): """ Compute shortest path from source vertex to target vertex """ tic = (time.time()) # #if source and target are given as indices: if self.graphtool: vertices_path, _ = shortest_path(self.graph, source, target, weights=self.weight, negative_weights=True) else: try: vertices_path = nx.dijkstra_path(self.graph, source, target) except nx.exception.NetworkXNoPath: return [] self.time_logs["shortest_path"] = round(time.time() - tic, 3) return vertices_path def save_graph(self, OUT_PATH): """ Save the graph in OUT_PATH """ if self.graphtool: for i, cost_class in enumerate(self.cost_classes): self.graph.edge_properties[cost_class] = self.cost_props[i] self.graph.edge_properties["weight"] = self.weight self.graph.save(OUT_PATH + ".xml.gz") else: nx.write_weighted_edgelist(self.graph, OUT_PATH + '.weighted.edgelist') def load_graph(self, IN_PATH): """ Retrieve graph from IN_PATH """ if self.graphtool: self.g_prev = load_graph(IN_PATH + ".xml.gz") self.weight_prev = self.g_prev.ep.weight # weight = G2.ep.weight[G2.edge(66, 69)] else: self.g_prev = nx.read_edgelist(IN_PATH + '.weighted.edgelist', nodetype=int, data=(('weight', float), )) # ----------------------------------------------------------------------- # INTERFACE def single_sp(self, **kwargs): """ Function for full processing until shortest path """ self.start_inds = kwargs["start_inds"] self.dest_inds = kwargs["dest_inds"] self.set_shift(self.start_inds, self.dest_inds, **kwargs) # self.set_corridor( # np.ones(self.hard_constraints.shape) * 0.5, # self.start_inds, # self.dest_inds, # factor_or_n_edges=1 # ) if self.verbose: print("1) Initialize shifts and instance (corridor)") self.set_edge_costs(**kwargs) # add vertices self.add_nodes() if self.verbose: print("2) Initialize distances to inf and predecessors") self.add_edges() if self.verbose: print("3) Compute source shortest path tree") print("number of vertices and edges:", self.n_nodes, self.n_edges) # weighted sum of all costs self.sum_costs() source_v, target_v = self.add_start_and_dest(self.start_inds, self.dest_inds) # get actual best path path, path_costs, cost_sum = self.get_shortest_path(source_v, target_v) if self.verbose: print("4) shortest path", cost_sum) return path, path_costs, cost_sum
class PointerProvenancePlot(Plot): """ Base class for plots using the pointer provenance graph. """ def __init__(self, *args, **kwargs): super(PointerProvenancePlot, self).__init__(*args, **kwargs) self._cached_dataset_valid = False """Tells whether we need to rebuild the dataset when caching.""" def init_parser(self, dataset, tracefile): if self.caching and os.path.exists(self._get_cache_file()): # if caching we will nevere use this return None return PointerProvenanceParser(dataset, tracefile) def init_dataset(self): logger.debug("Init provenance graph for %s", self.tracefile) self.dataset = Graph(directed=True) vdata = self.dataset.new_vertex_property("object") self.dataset.vp["data"] = vdata return self.dataset def _get_cache_file(self): return self.tracefile + "_provenance_plot.gt" def build_dataset(self): """ Build the provenance tree """ if self.caching: try: logger.debug("Load cached provenance graph") self.dataset = load_graph(self._get_cache_file()) except IOError: self.parser.parse() self.dataset.save(self._get_cache_file()) else: self.parser.parse() num_nodes = self.dataset.num_vertices() logger.debug("Total nodes %d", num_nodes) vertex_mask = self.dataset.new_vertex_property("bool") progress = ProgressPrinter(num_nodes, desc="Search kernel nodes") for node in self.dataset.vertices(): # remove null capabilities # remove operations in kernel mode vertex_data = self.dataset.vp.data node_data = vertex_data[node] if ((node_data.pc != 0 and node_data.is_kernel) or (node_data.cap.length == 0 and node_data.cap.base == 0)): vertex_mask[node] = True progress.advance() progress.finish() self.dataset.set_vertex_filter(vertex_mask, inverted=True) vertex_mask = self.dataset.copy_property(vertex_mask) num_nodes = self.dataset.num_vertices() logger.debug("Filtered kernel nodes, remaining %d", num_nodes) progress = ProgressPrinter( num_nodes, desc="Merge (cfromptr + csetbounds) sequences") for node in self.dataset.vertices(): progress.advance() # merge cfromptr -> csetbounds subtrees num_parents = node.in_degree() if num_parents == 0: # root node continue elif num_parents > 1: logger.error("Found node with more than a single parent %s", node) raise RuntimeError("Too many parents for a node") parent = next(node.in_neighbours()) parent_data = self.dataset.vp.data[parent] node_data = self.dataset.vp.data[node] if (parent_data.origin == CheriNodeOrigin.FROMPTR and node_data.origin == CheriNodeOrigin.SETBOUNDS): # the child must be unique to avoid complex logic # when merging, it may be desirable to do so with # more complex traces node_data.origin = CheriNodeOrigin.PTR_SETBOUNDS if parent.in_degree() == 1: next_parent = next(parent.in_neighbours()) vertex_mask[parent] = True self.dataset.add_edge(next_parent, node) elif parent.in_degree() == 0: vertex_mask[parent] = True else: logger.error("Found node with more than a single parent %s", parent) raise RuntimeError("Too many parents for a node") progress.finish() self.dataset.set_vertex_filter(vertex_mask, inverted=True) vertex_mask = self.dataset.copy_property(vertex_mask) num_nodes = self.dataset.num_vertices() logger.debug("Merged (cfromptr + csetbounds), remaining %d", num_nodes) progress = ProgressPrinter(num_nodes, desc="Find short-lived cfromptr") for node in self.dataset.vertices(): progress.advance() node_data = self.dataset.vp.data[node] if node_data.origin == CheriNodeOrigin.FROMPTR: vertex_mask[node] = True # if (node_data.origin == CheriNodeOrigin.FROMPTR and # len(node_data.address) == 0 and # len(node_data.deref["load"]) == 0 and # len(node_data.deref["load"]) == 0): # # remove cfromptr that are never stored or used in # # a dereference # remove_list.append(node) progress.finish() self.dataset.set_vertex_filter(vertex_mask, inverted=True)
class Network: def __init__(self): self.g = Graph(directed=True) self.player_id_to_vertex = {} self.pairs = {} # player pair: edge # property maps for additional information self.g.vertex_properties['player_id'] = self.g.new_vertex_property( "string") self.g.vertex_properties['player_coords'] = self.g.new_vertex_property( "vector<float>") self.g.vertex_properties[ 'average_player_coords'] = self.g.new_vertex_property( "vector<float>") self.g.vertex_properties[ 'player_n_coords'] = self.g.new_vertex_property("int") self.g.edge_properties['weight'] = self.g.new_edge_property("float") @property def edge_weights(self): return self.g.edge_properties['weight'] @property def player_id_pmap(self): return self.g.vertex_properties['player_id'] @property def player_coords_pmap(self): return self.g.vertex_properties['player_coords'] @property def player_n_coords_pmap(self): return self.g.vertex_properties['player_n_coords'] @property def average_player_coords_pmap(self): # lazy evaluation of means for v in self.g.vertices(): self.g.vertex_properties['average_player_coords'][v] = np.asarray( self.player_coords_pmap[v]) / self.player_n_coords_pmap[v] return self.g.vertex_properties['average_player_coords'] def add_players(self, pids: List[str]): n = len(pids) vs = list(self.g.add_vertex(n)) self.player_id_to_vertex.update({pids[i]: vs[i] for i in range(n)}) for i in range(n): self.player_id_pmap[vs[i]] = pids[i] return vs def add_passes(self, id_pairs: List[Tuple], coords_pairs: List[Tuple], pass_scores=None): pairs = [(self.player_id_to_vertex[i1], self.player_id_to_vertex[i2]) for i1, i2 in id_pairs] # append player coordinates n = len(coords_pairs) if pass_scores is None: pass_scores = [1 for _ in range(n)] for i in range(n): # remember orig and dest location # orig player coords = self.player_coords_pmap[pairs[i][0]] if len(coords) == 0: coords = np.asarray([coords_pairs[i][0], coords_pairs[i][1]]) else: # accumulate coords += np.asarray([coords_pairs[i][0], coords_pairs[i][1]]) self.player_coords_pmap[pairs[i][0]] = coords self.player_n_coords_pmap[pairs[i][0]] += 1 # dest player coords = self.player_coords_pmap[pairs[i][1]] if len(coords) == 0: coords = np.asarray([coords_pairs[i][2], coords_pairs[i][3]]) else: # accumulate coords += np.asarray([coords_pairs[i][2], coords_pairs[i][3]]) self.player_coords_pmap[pairs[i][1]] = coords self.player_n_coords_pmap[pairs[i][1]] += 1 # if the edge exists, increment its weight instead of creating a new edge e = self.pairs.get(pairs[i]) if e is not None: self.edge_weights[e] += pass_scores[i] else: e = self.g.add_edge(*pairs[i]) self.pairs[pairs[i]] = e self.edge_weights[e] = pass_scores[i] def cleanup(self): """remove isolated vertices""" to_remove = [] for v in self.g.vertices(): if v.in_degree() + v.out_degree() == 0: to_remove.append(v) n = len(to_remove) self.g.remove_vertex(to_remove, fast=True) print("Removed {0} isolated vertices".format(n)) def save(self, file: str): self.g.save(file, fmt='graphml')