def load_existing_networks(filename="existing_networks.shp", budget_value=0): """ load existing_networks shp into GeoGraph nodes, edges and assign budget Args: filename: existing_networks shapefile budget_value: default budget value for nodes in existing network Returns: GeoGraph of existing networks with budget attribute """ geo_net = nio.load_shp(filename) nx.set_node_attributes(geo_net, 'budget', {n: budget_value for n in geo_net.nodes()}) # determine edge weight/distance function by whether they're geocentric # (assuming coordinates are not in 3-space here) distance = gm.spherical_distance if geo_net.is_geographic else \ gm.euclidean_distance nx.set_edge_attributes(geo_net, 'weight', {(u, v): distance(map(geo_net.coords.get, [u, v])) for u, v in geo_net.edges()}) return geo_net
def CopyDAG(G_train, data_total, pmin, pmax): edges = nx.edges(G_train) kern = dict.fromkeys(edges) M = len(data_total) kern_temp = {} for edge in edges: kern[edge] = np.zeros(M) for m in range(M): print('Data item: %d' % m) data = data_total[m] for i in range(0, len(data)-pmin+1+1): for j in range(pmin-1, pmax+1): if data[i:i+j] in kern_temp: kern_temp[data[i:i+j]][m] += 1 else: kern_temp[data[i:i+j]] = np.zeros(M) kern_temp[data[i:i+j]][m] = 1 for edge in edges: key = edge[0]+edge[1][-1] if key in kern_temp: kern[edge] = kern_temp[key] G = nx.DiGraph() G.add_edges_from(edges) nx.set_edge_attributes(G, 'kern_unnorm', kern) return G
def from_trial_data(cls, circuit, cycles, pos, resultinfo, trialid, step): # several parts of this script would require reconsideration to support trials # where the vertices weren't actually deleted if resultinfo['defect_mode']['mode'] != 'direct removal': raise RuntimeError('This script was only written with "remove" mode in mind') # Find change in currents before = analysis.trial_edge_currents_at_step(circuit, cycles, resultinfo, trialid, step) after = analysis.trial_edge_currents_at_step(circuit, cycles, resultinfo, trialid, step+1) before = edict_remove_redundant_entries(before) delta = {e: after.get(e, 0.0) - before[e] for e in before} # NOTE: this reconstruction will lack some isolated vertices defects = set(get_added_defects(resultinfo, trialid, step)) defect_dict = {v:False for v in edict_vertices(delta)} defect_dict.update({v:True for v in defects}) sources = {(s,t):s for (s,t) in delta} g = nx.Graph() g.add_edges_from(delta) g.add_nodes_from(defects) # in case any of the defects are isolated vertices # ``pos`` contains all vertices from the graph's initial state. Limit it to those currently # in the modified graph (because set_node_attributes crashes on nodes that don't exist) pos = {k:v for k,v in pos.items() if k in g} nx.set_edge_attributes(g, EATTR_CHANGE, delta) nx.set_edge_attributes(g, EATTR_SOURCE, sources) nx.set_node_attributes(g, VATTR_POS, pos) nx.set_node_attributes(g, VATTR_DEFECT, defect_dict) return cls(g)
def _normlise_DAG(self, G): """ [TO BE TESTED] Normalise Fisher features on a DAG (l2 normalisation). Parameters ------- :param G: DAG of Fisher features. Attribute 'kern_': edge attribute (dictionary), key: edges, value: normalised Fisher features Normalised Fisher features. :return: G, edge attribute 'kern_' assigned. """ kern_ = nx.get_edge_attributes(G, 'kern_unnorm_') kern_ = OrderedDict(sorted(kern_.items(), key=lambda t: t[0])) val_ = np.asarray(kern_.values(), dtype=float) key_ = kern_.keys() if len(val_.shape) == 2: # normalise if there are more than one data samples assigned to the DAG kern_ = normalize(val_ * self.log_proba_[:, None], norm='l2', axis=0) else: # normalise if there is only one data sample assigned to the DAG kern_ = (val_ * self.log_proba_)/np.linalg.norm(val_ * self.log_proba_) kern_ = dict(zip(key_, kern_)) nx.set_edge_attributes(G, 'kern_', kern_) return G
def normDAGl2Test(G_test, power): kern = nx.get_edge_attributes(G_test, 'kern_unnorm') tran = nx.get_edge_attributes(G_test, 'tran') kern = OrderedDict(sorted(kern.items(), key=lambda t: t[0])) val = kern.values() key = kern.keys() tran = OrderedDict(sorted(tran.items(), key=lambda t: t[0])) tran = tran.values() val = np.asarray(val, dtype=float) tran = np.asarray(tran, dtype=float) tran = np.log(1/tran) # logarithm weighting tran[tran == np.inf] = 0 tran[np.isnan(tran)] = 0 if power == 2: tran = np.square(tran) if len(val.shape) == 2: # kern = val/tran[:, None] kern = val*tran[:, None] # avoid numeric problems when using logarithm weighting kern = normalize(kern, norm='l2', axis=0) else: kern = val*tran kern = kern/np.linalg.norm(kern) kern = dict(zip(key, kern)) nx.set_edge_attributes(G_test, 'kern', kern) return G_test
def test_effective_size_weighted_undirected(self): G = self.G.copy() nx.set_edge_attributes(G, 'weight', self.G_weights) effective_size = nx.effective_size(G, weight='weight') assert_almost_equal(round(effective_size['G'], 2), 5.47) assert_almost_equal(round(effective_size['A'], 2), 2.47) assert_almost_equal(round(effective_size['C'], 2), 1)
def df_to_graph(df): G = nx.from_numpy_matrix(df.values, create_using=nx.DiGraph()) G = nx.relabel_nodes(G, dict(enumerate(df.columns))) weights = nx.get_edge_attributes(G, 'weight') invW = dict([(k, 1/float(v)) for (k,v) in weights.items()]) nx.set_edge_attributes(G, 'distance', invW) return G
def shortest_paths(self, dists): mx = np.inf * np.ones((self.N, self.N)) nx.set_edge_attributes(self.graph, 'weight', dict(zip(self.graph.edges(), dists))) d = nx.shortest_path_length(self.graph, weight='weight') for i, v in d.iteritems(): mx[i, v.keys()] = v.values() return mx
def _aux_digraph_edge_connectivity(G): """Auxiliary digraph for computing flow based edge connectivity If the input graph is undirected, we replace each edge (u,v) with two reciprocal arcs (u,v) and (v,u) and then we set the attribute 'capacity' for each arc to 1. If the input graph is directed we simply add the 'capacity' attribute. Part of algorithm 1 in [1]_ . References ---------- .. [1] Abdol-Hossein Esfahanian. Connectivity Algorithms. (this is a chapter, look for the reference of the book). http://www.cse.msu.edu/~cse835/Papers/Graph_connectivity_revised.pdf """ if G.is_directed(): if nx.get_edge_attributes(G, "capacity"): return G D = G.copy() capacity = dict((e, 1) for e in D.edges()) nx.set_edge_attributes(D, "capacity", capacity) return D else: D = G.to_directed() capacity = dict((e, 1) for e in D.edges()) nx.set_edge_attributes(D, "capacity", capacity) return D
def create_protein_graph(ingraph): pnodes = {} pnodes = defaultdict(lambda: 0, pnodes) pedges = {} pedges = defaultdict(lambda: 0, pedges) outgraph = nx.Graph() for node in ingraph.nodes_iter(): pnodes[ingraph.node[node]['protein']]+=1 for u,v,d in ingraph.edges(data=True): key=(ingraph.node[u]['protein'], ingraph.node[v]['protein']) pedges[key]+=1 for key in pnodes.keys(): outgraph.add_node(key) edges = combinations(pnodes.keys(), 2) outgraph.add_nodes_from(pnodes.keys()) outgraph.add_edges_from(edges) nx.set_node_attributes(outgraph, 'count', pnodes) nx.set_edge_attributes(outgraph, 'weight', pedges) return outgraph
def test_constraint_weighted_undirected(self): G = self.G.copy() nx.set_edge_attributes(G, 'weight', self.G_weights) constraint = nx.constraint(G, weight='weight') assert_almost_equal(round(constraint['G'], 3), 0.299) assert_almost_equal(round(constraint['A'], 3), 0.795) assert_almost_equal(round(constraint['C'], 3), 1)
def test_effective_size_weighted_directed(self): D = self.D.copy() nx.set_edge_attributes(D, 'weight', self.D_weights) effective_size = nx.effective_size(D, weight='weight') assert_almost_equal(round(effective_size[0], 3), 1.567) assert_almost_equal(round(effective_size[1], 3), 1.083) assert_almost_equal(round(effective_size[2], 3), 1)
def test_constraint_weighted_directed(self): D = self.D.copy() nx.set_edge_attributes(D, 'weight', self.D_weights) constraint = nx.constraint(D, weight='weight') assert_almost_equal(round(constraint[0], 3), 0.840) assert_almost_equal(round(constraint[1], 3), 1.143) assert_almost_equal(round(constraint[2], 3), 1.378)
def __init__(self): self.graph = nx.Graph() self.node2min = dict() self.edge2ts = dict() nx.set_edge_attributes(self.graph, "ts", dict()) #nx.set_node_attributes( self.graph ) self.nmin = 0
def set_transition_probabilities(graph, damping, start_node): # transition probabilities for_removal = [] for node in graph.nodes_iter(): # sum the weights weights_sum = 0.0 for neighbor in graph.successors_iter(node): weights_sum += graph.get_edge_data(node, neighbor)['weight'] # set the transition probability multiplied by (1 - damping) for neighbor in graph.successors_iter(node): transition = ((1-damping) * (graph[node][neighbor]['weight'] / weights_sum)) nx.set_edge_attributes(graph, 'transition', {(node, neighbor): transition}) # add the restart if graph.has_edge(node, start_node): graph[node][start_node]['transition'] += damping else: for_removal.append((node, start_node)) if weights_sum > 0: graph.add_edge (node, start_node, {'weight': 0, 'transition': damping}) else: graph.add_edge (node, start_node, {'weight': 0, 'transition': 1.0}) return for_removal
def _build_edge_wkt(self): r = self.results # Iterate through the nodes and their parent for rank, fnode, tnode in zip(r.index, r['Sequence..Upstream.id'], r['Sequence..Vertex.id']): if fnode is not None: # Set the edge attributes with those found in sequencing self.networkplan.network.edge[fnode][tnode]['rank'] = int(rank) self.networkplan.network.edge[fnode][tnode]['distance'] = float(self.networkplan._distance(fnode, tnode)) self.networkplan.network.edge[fnode][tnode]['id'] = int(tnode) fnode_coords = self.networkplan.coords[fnode] tnode_coords = self.networkplan.coords[tnode] # Build WKT Linestring with from_node and to_node coords self.networkplan.network.edge[fnode][tnode]['Wkt'] = 'LINESTRING ({x1} {y1}, {x2} {y2})'.format(x1=fnode_coords[0], y1=fnode_coords[1], x2=tnode_coords[0], y2=tnode_coords[1]) # Filter empty edges edges = {(k1, k2): attr for k1, v in self.networkplan.network.edge.iteritems() if v!={} for k2, attr in v.iteritems() if attr!={}} self.networkplan.network.edge = edges # Clear all edges from the networkx self.networkplan.network.remove_edges_from(self.networkplan.network.edges()) # Reset with the filtered edges self.networkplan.network.add_edges_from(edges.keys()) # Set the atrributes attrs = set([v for i in edges.values() for v in i.keys()]) for attr in attrs: nx.set_edge_attributes(self.networkplan.network, attr, pd.DataFrame(edges).ix[attr].to_dict())
def create_graph_df(vtask_paths, graphs_dir_out): """ Creates a frame that maps sourcefiles to networkx digraphs in terms of DOT files :param source_path_list: :param dest_dir_path: :param relabel: :return: """ if not isdir(graphs_dir_out): raise ValueError('Invalid destination directory.') data = [] graphgen_times = [] print('Writing graph representations of verification tasks to {}'.format(graphs_dir_out), flush=True) common_prefix = commonprefix(vtask_paths) for vtask in tqdm(vtask_paths): short_prefix = dirname(common_prefix) path = join(graphs_dir_out, vtask[len(short_prefix):][1:]) if not os.path.exists(dirname(path)): os.makedirs(dirname(path)) ret_path = path + '.pickle' # DEBUG if isfile(ret_path): data.append(ret_path) continue start_time = time.time() graph_path, node_labels_path, edge_types_path, edge_truth_path, node_depths_path \ = _run_cpachecker(abspath(vtask)) nx_digraph = nx.read_graphml(graph_path) node_labels = _read_node_labeling(node_labels_path) nx.set_node_attributes(nx_digraph, 'label', node_labels) edge_types = _read_edge_labeling(edge_types_path) parsed_edge_types = _parse_edge(edge_types) nx.set_edge_attributes(nx_digraph, 'type', parsed_edge_types) edge_truth = _read_edge_labeling(edge_truth_path) parsed_edge_truth = _parse_edge(edge_truth) nx.set_edge_attributes(nx_digraph, 'truth', parsed_edge_truth) node_depths = _read_node_labeling(node_depths_path) parsed_node_depths = _parse_node_depth(node_depths) nx.set_node_attributes(nx_digraph, 'depth', parsed_node_depths) assert not isfile(ret_path) assert node_labels and parsed_edge_types and parsed_edge_truth and parsed_node_depths nx.write_gpickle(nx_digraph, ret_path) data.append(ret_path) gg_time = time.time() - start_time graphgen_times.append(gg_time) return pd.DataFrame({'graph_representation': data}, index=vtask_paths), graphgen_times
def make_hrr_vectors(G, id_vectors, edge_vectors): max_out_degree = len(edge_vectors) index_dict = {} for n in G.nodes_iter(): edges = G.edges(n) indices = random.sample(range(max_out_degree), len(edges)) index_dict.update(dict(zip(edges, indices))) nx.set_edge_attributes(G, 'index', index_dict) hrrvecs = {} for n in G.nodes_iter(): edges = G.edges(n) if not edges: hrr_vec = hrr.HRR(len(id_vectors.values()[0])) else: components = [id_vectors[e[1]].convolve(edge_vectors[index_dict[e]]) for e in edges] hrrvec = sum(components[1:], components[0]) hrrvec.normalize() hrrvecs[n] = hrrvec return hrrvecs
def test_edge_num_attribute(self): g = nx.karate_club_graph() attr = {(u, v): {"even": int((u+v) % 10)} for (u, v) in g.edges()} nx.set_edge_attributes(g, attr) model = gc.CompositeModel(g) model.add_status("Susceptible") model.add_status("Infected") c = cpm.EdgeNumericalAttribute("even", value=0, op="==", probability=1) model.add_rule("Susceptible", "Infected", c) config = mc.Configuration() config.add_model_parameter('percentage_infected', 0.1) model.set_initial_status(config) iterations = model.iteration_bunch(10) self.assertEqual(len(iterations), 10) model = gc.CompositeModel(g) model.add_status("Susceptible") model.add_status("Infected") c = cpm.EdgeNumericalAttribute("even", value=[3, 10], op="IN", probability=1) model.add_rule("Susceptible", "Infected", c) config = mc.Configuration() config.add_model_parameter('percentage_infected', 0.1) model.set_initial_status(config) iterations = model.iteration_bunch(10) self.assertEqual(len(iterations), 10)
def add_random_weights(hierarchical_graph): """ Adds random weights to a graph and normalises to ensure stochasticity. Args: hierarchical_graph (networkx.classes.digraph.DiGraph): hierarchical graph without weights. Returns: networkx.classes.digraph.DiGraph: hierarchical graph with random weights. """ #give each edge a random weight edge_weight_dict = {} for edge in hierarchical_graph.edges(): rand = random.uniform(0, 1) edge_weight_dict[edge] = rand nx.set_edge_attributes(hierarchical_graph, edge_weight_dict, 'transition probability') #give each node a random weight node_weight_dict = {} for node in hierarchical_graph.nodes(): rand = random.uniform(0, 1) node_weight_dict[node] = rand nx.set_node_attributes(hierarchical_graph, node_weight_dict, 'mean transition time') #normalise the sum of the weights of the out edges for each node. for node in hierarchical_graph.nodes(): out_edges = hierarchical_graph.out_edges(node) sum_out = sum([hierarchical_graph.edges[edge]['transition probability'] for edge in out_edges]) for edge in out_edges: hierarchical_graph.edges[edge]['transition probability'] = \ hierarchical_graph.edges[edge]['transition probability']/sum_out
def _get_test_graph(): '''test graph with known partition''' G = nx.Graph() G.add_nodes_from(range(5)) G.add_edges_from([(0, 1), (1, 2), (1, 3), (2, 4), (3, 4)]) nx.set_edge_attributes(G, 'weight', [1] * G.number_of_edges()) return G
def __init__(self, dim=None, phi=np.pi, periodic=True, phases=None): if not dim: dim = [4, 4] dim = copy(dim) self.dim = copy(dim) self.nspins = np.prod(dim) self.G = nx.grid_graph(dim, periodic) if phases is not None: self.phases = phases else: self.phases = dict() binary_disorder = True if binary_disorder: for edge in self.G.edges(): self.phases[edge] = phi * np.random.random_integers(0, 1) else: for edge in self.G.edges(): self.phases[edge] = np.random.uniform(-phi, phi) nx.set_edge_attributes(self.G, "phase", self.phases) self.indices = dict() self.index2node = dict() nodes = sorted(self.G.nodes()) for i, node in enumerate(nodes): self.indices[node] = i self.index2node[i] = node self.num_edges = self.G.number_of_edges() self.set_up_neighborlists()
def test_networkx_roundtrip(self): print('\n---------- NetworkX Data Roundtrip Test Start -----------\n') g = nx.newman_watts_strogatz_graph(100, 3, 0.5) nodes = g.nodes() edges = g.edges() # Add some attributes g.graph['name'] = 'original' g.graph['density'] = nx.density(g) nx.set_node_attributes(g, 'betweenness', nx.betweenness_centrality(g)) nx.set_node_attributes(g, 'degree', nx.degree(g)) nx.set_node_attributes(g, 'closeness', nx.closeness_centrality(g)) nx.set_edge_attributes(g, 'eb', nx.edge_betweenness(g)) cyjs1 = util.from_networkx(g) g2 = util.to_networkx(cyjs1) self.assertEqual(len(g2.nodes()), len(nodes)) self.assertEqual(len(g2.edges()), len(edges)) edge_set = set(list(map(lambda x: (int(x[0]), int(x[1])), g2.edges()))) self.assertEqual(0, len(edge_set.difference(set(edges)))) node_original = g.node[1] node_generated = g2.node['1'] print(node_original) print(node_generated) self.assertEqual(node_original['degree'], node_generated['degree']) self.assertEqual(node_original['betweenness'], node_generated['betweenness']) self.assertEqual(node_original['closeness'], node_generated['closeness'])
def test_networkx_roundtrip(self): print("\n---------- NetworkX Data Roundtrip Test Start -----------\n") g = nx.newman_watts_strogatz_graph(100, 3, 0.5) nodes = g.nodes() edges = g.edges() # Add some attributes g.graph["name"] = "original" g.graph["density"] = nx.density(g) nx.set_node_attributes(g, "betweenness", nx.betweenness_centrality(g)) nx.set_node_attributes(g, "degree", nx.degree(g)) nx.set_node_attributes(g, "closeness", nx.closeness_centrality(g)) nx.set_edge_attributes(g, "eb", nx.edge_betweenness(g)) cyjs1 = util.from_networkx(g) g2 = util.to_networkx(cyjs1) self.assertEqual(len(g2.nodes()), len(nodes)) self.assertEqual(len(g2.edges()), len(edges)) edge_set = set(list(map(lambda x: (int(x[0]), int(x[1])), g2.edges()))) self.assertEqual(0, len(edge_set.difference(set(edges)))) node_original = g.node[1] node_generated = g2.node["1"] print(node_original) print(node_generated) self.assertEqual(node_original["degree"], node_generated["degree"]) self.assertEqual(node_original["betweenness"], node_generated["betweenness"]) self.assertEqual(node_original["closeness"], node_generated["closeness"])
def write_edge2cid(e2c,filename, outFile, delimiter="\t"): # write edge2cid three-column file f = open(filename+".edge2comm.txt",'w') c2c = dict( (c,i+1) for i,c in enumerate(sorted(list(set(e2c.values())))) ) # ugly... #print c2c for e,c in sorted(e2c.iteritems(), key=itemgetter(1)): f.write( "%s%s%s%s%s\n" % (str(e[0]),delimiter,str(e[1]),delimiter,str(c2c[c])) ) f.close() cid2edges,cid2nodes = defaultdict(set),defaultdict(set) # faster to recreate here than for edge,cid in e2c.iteritems(): # to keep copying all dicts cid2edges[cid].add( edge ) # during the linkage... cid2nodes[cid] |= set(edge) cid2edges,cid2nodes = dict(cid2edges),dict(cid2nodes) # write list of edges for each comm, each comm on its own line f,g = open(filename+".comm2edges.txt", 'w'),open(filename+".comm2nodes.txt", 'w') for cid in sorted(cid2edges.keys()): nodes,edges = map(str,cid2nodes[cid]), ["%s,%s" % (ni,nj) for ni,nj in cid2edges[cid]] f.write( "\t".join(edges) ); f.write("\n") g.write( "\t".join([str(cid)] + nodes) ); g.write("\n") f.close(); g.close() #print e2c e2cRelabel=dict((edge,c2c.get(id)) for (edge, id) in e2c.items()) g=nx.Graph() g.add_edges_from(e2cRelabel.keys()) nx.set_edge_attributes(g,"label",e2cRelabel) nx.write_gml(g,outFile)
def planar_cycle_basis_impl(g): assert not g.is_directed() assert not g.is_multigraph() assert nx.is_biconnected(g) # BAND-AID # "First" nodes/edges for each cycle are chosen in an order such that the first edge # may never belong to a later cycle. # rotate to (hopefully) break any ties or near-ties along the y axis rotate_graph(g, random.random() * 2 * math.pi) # NOTE: may want to verify that no ties exist after the rotation nx.set_edge_attributes(g, 'used_once', {e: False for e in g.edges()}) # precompute edge angles angles = {} for s,t in g.edges(): angles[s,t] = edge_angle(g,s,t) % (2*math.pi) angles[t,s] = (angles[s,t] + math.pi) % (2*math.pi) # identify and clear away edges which cannot belong to cycles for v in g: if degree(g,v) == 1: remove_filament_from_tip(g, v) cycles = [] # sort ascendingly by y for root in sorted(g, key = lambda v: g.node[v]['y']): # Check edges in ccw order from the +x axis for target in sorted(g[root], key=lambda t: angles[root,t]): if not g.has_edge(root, target): continue discriminator, path = trace_ccw_path(g, root, target, angles) if discriminator == PATH_PLANAR_CYCLE: assert path[0] == path[-1] remove_cycle_edges(g, path) cycles.append(path) # Both the dead end and the initial edge belong to filaments elif discriminator == PATH_DEAD_END: remove_filament_from_edge(g, root, target) remove_filament_from_tip(g, path[-1]) # The initial edge must be part of a filament # FIXME: Not necessarily true if graph is not biconnected elif discriminator == PATH_OTHER_CYCLE: remove_filament_from_edge(g, root, target) else: assert False # complete switch assert not g.has_edge(root, target) assert len(g.edges()) == 0 return cycles
def extract_edges(self, edge_attr): """docstring for extract""" exEdges = nx.get_edge_attributes(self, edge_attr) sg = nx.Graph(data = exEdges.keys(), name = edge_attr) if len(exEdges): nx.set_edge_attributes(sg, edge_attr, exEdges) return(sg)
def delete_links(G): ''' Delete links in the graph if they disavantage the person. The link is deleted if (r1-r2)/affect(n1, n2)*deg(n1) > alpha where r1 > r2 and alpha is a random number between 0 and a given paramater beta. ''' beta = 100 reput = nx.get_node_attributes(G, "reput") affect = nx.get_edge_attributes(G, "affect") n = 0 for edge in nx.edges(G): # Calculate alpha alpha = beta*random.random() # Define who has the higher reputation n1 = edge[1] n2 = edge[0] if(reput[edge[0]] > reput[edge[1]]): n1 = edge[0] n2 = edge[1] # Compute the coefficient coef = (reput[n1]-reput[n2])/affect[edge]*G.degree(n1) # Decide wether we delete the link if(coef > alpha): G.remove_edge(edge[0], edge[1]) del affect[edge] # Set the new affect dict and return the graph nx.set_edge_attributes(G, "affect", affect) return G
def nx_from_matrix(weight_matrix, nodes=None, remove_self=True): import networkx as nx import utool as ut import numpy as np if nodes is None: nodes = list(range(len(weight_matrix))) weight_list = weight_matrix.ravel() flat_idxs_ = np.arange(weight_matrix.size) multi_idxs_ = np.unravel_index(flat_idxs_, weight_matrix.shape) # Remove 0 weight edges flags = np.logical_not(np.isclose(weight_list, 0)) weight_list = ut.compress(weight_list, flags) multi_idxs = ut.compress(list(zip(*multi_idxs_)), flags) edge_list = ut.lmap(tuple, ut.unflat_take(nodes, multi_idxs)) if remove_self: flags = [e1 != e2 for e1, e2 in edge_list] edge_list = ut.compress(edge_list, flags) weight_list = ut.compress(weight_list, flags) graph = nx.Graph() graph.add_nodes_from(nodes) graph.add_edges_from(edge_list) label_list = ['%.2f' % w for w in weight_list] nx.set_edge_attributes(graph, 'weight', dict(zip(edge_list, weight_list))) nx.set_edge_attributes(graph, 'label', dict(zip(edge_list, label_list))) return graph
def color_by_nids(graph, unique_nids=None, ibs=None, nid2_color_=None): """ Colors edges and nodes by nid """ # TODO use ut.color_nodes import plottool as pt ensure_graph_nid_labels(graph, unique_nids, ibs=ibs) node_to_nid = nx.get_node_attributes(graph, 'nid') unique_nids = ut.unique(node_to_nid.values()) ncolors = len(unique_nids) if (ncolors) == 1: unique_colors = [pt.UNKNOWN_PURP] else: if nid2_color_ is not None: unique_colors = pt.distinct_colors(ncolors + len(nid2_color_) * 2) else: unique_colors = pt.distinct_colors(ncolors) # Find edges and aids strictly between two nids nid_to_color = dict(zip(unique_nids, unique_colors)) if nid2_color_ is not None: # HACK NEED TO ENSURE COLORS ARE NOT REUSED nid_to_color.update(nid2_color_) edge_aids = list(graph.edges()) edge_nids = ut.unflat_take(node_to_nid, edge_aids) flags = [nids[0] == nids[1] for nids in edge_nids] flagged_edge_aids = ut.compress(edge_aids, flags) flagged_edge_nids = ut.compress(edge_nids, flags) flagged_edge_colors = [nid_to_color[nids[0]] for nids in flagged_edge_nids] edge_to_color = dict(zip(flagged_edge_aids, flagged_edge_colors)) node_to_color = ut.map_dict_vals(ut.partial(ut.take, nid_to_color), node_to_nid) nx.set_edge_attributes(graph, 'color', edge_to_color) nx.set_node_attributes(graph, 'color', node_to_color)
def get_firewall_rules(graph, generator_string, weights, seed=0, olsrv1=False, display=False, print_csv=False, single_interface=False): """ Given a networkx graph representing the real topology of a testbed, returns the firewall rules to be applied to obtain a synthetic topology with the generated using a given generator. In addition, the function returns a score between 0 and 1 which represents how close is the new topology to the synthetic topology. A score of 1 means that the testbed topology can be transformed to completely match the synthetic graph. A score of 0 indicates that no link can be matched :param graph: networkx graph :param generator_string: a networkx function name (e.g., "random_regular_graph") with the list of parameters. The generator function must accept the parameter n as the number of nodes, plus the additional parameters, and generate a networkx graph. The format of the string is like "generator_function_name:p1=v1,p2=v2,..." where p1 and p2 are the argument names and v1 and v2 their respective values. The arguments should not include the parameter n, which is automatically obtained from the given graph :param weights: if True, it means that the experiment should generate per-link random weights, and this must be taken into account when computing the poprow timers :param seed: seed used for random generators :param olsrv1: if set to true, generate olsrv1-compatible link costs :param display: if set to True, shows a plot with the overlapping desired topology and the experiment topology, for a graphical comparison of the two :param print_csv: if true, prints a csv table with node id, times, betweennes centrality and degree, to be used for plotting and analysis :param single_interface: if true, the degree d_i is forced to 1 to represent a single interface node :return: a pair where the first element is the firewall configuration of links to be disabled, while the second is the matching score """ # remove "id_" from node names testbed_g = graph old_names = testbed_g.nodes() new_names = dict((i, i.replace('id_', '')) for i in old_names) nx.relabel_nodes(testbed_g, new_names, copy=False) n_nodes = len(testbed_g.nodes()) testbed_am = nx.adj_matrix(testbed_g) # create a map from node id to its original position mp = dict((testbed_g.nodes()[i], i) for i in range(n_nodes)) # sort the nodes by degree testbed_d = testbed_g.degree() sd = sorted(testbed_d.items(), key=lambda x: x[1], reverse=True) testbed_sd = map(lambda x: x[0], sd) # create the sorted map. testbed_sd is a vector of node names, and we # need a vector of indexes testbed_sm = [mp[testbed_sd[i]] for i in range(len(testbed_sd))] testbed_matrix = __get_matrix(testbed_am, testbed_sm) if generator_string.find(":") != -1: generator, parameters = generator_string.split(":") gen_args = dict() for p in parameters.split(","): key, value = p.split("=") try: v = int(value) except: v = value gen_args[key] = v else: generator = generator_string gen_args = dict() # create a synthetic graph by calling the given generator # search for the generator inside networkx. if not found, use one of our # custom generators try: gen_function = getattr(nx, generator) except AttributeError: gen_function = getattr(ct, generator) synt_g = gen_function(n=n_nodes, **gen_args) synt_am = nx.adj_matrix(synt_g) synt_d = synt_g.degree() # sort by degree as for the real graph sd = sorted(synt_d.items(), key=lambda x: x[1], reverse=True) synt_sm = map(lambda x: x[0], sd) synt_matrix = __get_matrix(synt_am, synt_sm) best_score = -1 best_matrix = None best_mapping = None n_attempts = 10 random.seed(0) # if the matching using the heuristic didn't work, simply try n times # with random shuffling while best_score < 1 and n_attempts > 0: n_attempts -= 1 # transform the testbed topology in the synthetic topology by performing # a logical and experiment_matrix = __matrix_and(testbed_matrix, synt_matrix) # count the number of non matching entries. 1 means perfect match, # 0 means no link could be matched score = __matching_score(experiment_matrix, synt_matrix) if score > best_score: best_score = score best_matrix = experiment_matrix best_mapping = testbed_sm testbed_sm = range(len(testbed_g.nodes())) random.shuffle(testbed_sm) testbed_matrix = __get_matrix(testbed_am, testbed_sm) if display: # do this heretic thing to avoid installing matplotlib on testbed # nodes if not really required import matplotlib.pyplot as plt experiment_g = nx.from_scipy_sparse_matrix(best_matrix) nx.draw_circular(synt_g) nx.draw_circular(experiment_g, width=3, edge_color='r') plt.show() # given the experiment matrix and the nodes of the testbed, get the links # that must be disabled to obtain the desired topology disabled_links = __get_links(best_matrix, best_mapping, testbed_g.nodes()) # transform this into a string usable by the scripts rules = __stringify_disabled_links(disabled_links) # check whether the edges in the synthetic topology have a weight u, v = synt_g.edges()[0] has_costs = False if "weight" in synt_g.get_edge_data(u, v).keys(): has_costs = True link_costs = __add_link_costs(__get_links(best_matrix, best_mapping, testbed_g.nodes(), False), seed, olsrv1=olsrv1, has_costs=has_costs) costs = __stringify_link_costs(link_costs, olsrv1=olsrv1) graph = __get_graph(best_matrix, best_mapping, testbed_g.nodes()) if weights: for s, csts in link_costs.iteritems(): for (d, cost) in csts: nx.set_edge_attributes(graph, "weight", {(s, d): cost}) intervals = __stringify_intervals(__compute_intervals(graph, use_degree=not single_interface)) if print_csv: intv = __compute_intervals(graph, return_bc=True) print("node,h,tc,hnd,tcnd,bc,d") for n, (h, tc, h2, tc2, bc, d) in intv.iteritems(): print("%s,%f,%f,%f,%f,%f,%d" % (n, h, tc, h2, tc2, bc, d)) # we return the configuration string and the measure of the match return rules, costs, intervals, best_score
names = nx.get_node_attributes(graph, 'name') dicty1 = { key.encode('utf-8'): value.encode('utf-8') for key, value in names.iteritems() } graph = nx.relabel_nodes(graph, dicty1, copy=True) signs = nx.get_edge_attributes(graph, 'sign') activities = {} for e in graph.edges_iter(): if signs[e] == '+': activities[e] = 'a' elif signs[e] == '-': activities[e] = 'i' else: print('not found: ' + signs[e]) nx.set_edge_attributes(graph, 'signal', activities) # read in C library updateBooler = cdll.LoadLibrary('./testRun.so') boolC = updateBooler.syncBool sampleList, geneDict, cvDict = readFpkmData(dataName, results.sep) genes = Set(geneDict.keys()) newOverlap = genes.intersection(set(graph.nodes())) # graph, addLaterNodes=PA.simplifyNetworkpathwayAnalysis(graph, geneDict) print(len(graph.nodes())) sampleList = [{} for q in range(len(geneDict[list(newOverlap)[0]]))] for noder in newOverlap: for jn in range(len(sampleList)): sampleList[jn][noder] = geneDict[noder][jn] print(sampleList)
def preprocessing(): '''removing isolated nodes''' isolated=nx.isolates(G) for j in isolated: for key in nodes_data.keys(): if j == key: del nodes_data[key] G.remove_node(j) #print G.number_of_nodes(), len(nodes_data) continue for k in edges_data.keys(): '''deleting all edges from edges_data which no more exist''' if not G.has_edge(edges_data[k][1],edges_data[k][2],key=k): #print len(edges_data) del edges_data[k] gen=[] indegree=G.in_degree(G.nodes()) '''number of nodes with no in_degree''' '''removing joints,merge nodes categorised as generators from graph and node data''' t=0 for i in indegree: if indegree[i]==0: if nodes_data[i][3]=='merge' or nodes_data[i][3]=='joint': continue gen.append(i) for node,nd in nodes_data.iteritems(): '''adding plants to generators''' if nd[3]== 'plant' and nd[0] not in gen: gen.append(node) voltages={} z=0 for aa,val in edges_data.iteritems(): '''adding edge attributes''' z+=1 #print z if edges_data[aa][3].find(';')!= -1: xx=map(float,edges_data[aa][3].split(';')) edges_data[aa][3]=max(xx) del xx[:] elif edges_data[aa][3]=='': edges_data[aa][3]=np.nan else: edges_data[aa][3]=float(edges_data[aa][3]) if val[4].find(';')!= -1: xy=map(float,val[4].split(';')) val[4]=sum(xy) del xy[:] else: val[4]=float(val[4]) voltages[(val[1],val[2],val[0])]=val[3] nx.set_edge_attributes(G,'voltages',voltages) distr=distributers(); for f in gen: if f in distr: distr.remove(f) #G.remove_edges_from(G.in_edges(f)) if node in gen: nodes_data[node][3]='generators' elif node in distr: nodes_data[node][3]='distributors' else: nodes_data[node][3]='transmitters' for ab,v in nodes_data.iteritems(): '''adding nodes attributes''' if v[4].find(';')!= -1: ss=map(float,v[4].split(';')) if v[3]=='generator' or v[3]=='distributor': v[4]=min(ss) else: v[4]=max(ss) del ss[:] elif v[4]=='': v[4]=np.nan else: v[4]=float(v[4]) '''removing multi edges''' for x in G.nodes(): for y in G.nodes(): if x==y: continue if G.number_of_edges(x,y)>1: kkk=[] for ee in list(G.edges(x,keys=True)): if ee[0]==x and ee[1]==y: kkk.append(ee[2]) for l in range(1,len(kkk)): edges_data[kkk[0]][4]+=edges_data[kkk[l]][4] edges_data[kkk[0]][3]=max(edges_data[kkk[0]][3],edges_data[kkk[l]][3]) del edges_data[kkk[l]] G.remove_edge(x,y,key=kkk[l]) G[x][y][kkk[0]]['cable']=edges_data[kkk[0]][4] G[x][y][kkk[0]]['voltages']=edges_data[kkk[0]][3] #return (G,nodes_data,edges_data) o=[] with open('../../NS_project/Data/Vertices_new.csv', 'wb') as csvfile: nodewriter = csv.writer(csvfile, delimiter=',') header=['v_id','lon','lat','typ','voltage'] nodewriter.writerow(header) for da,it in nodes_data.iteritems(): for l in range(0,5): o.append(str(it[l])) nodewriter.writerow(o) del o[:] with open('../../NS_project/Data/Edges_new.csv', 'wb') as csvfile1: edgewriter = csv.writer(csvfile1, delimiter=',') header1=['l_id','v_id_1','v_id_2','voltage','cables','length_m'] edgewriter.writerow(header1) for db,itt in edges_data.iteritems(): for l in range(0,5): o.append(str(itt[l])) o.append(str(itt[10])) edgewriter.writerow(o) del o[:] return G
G = nx.MultiDiGraph() G.add_edges_from((0, i) for i in range(1, n)) # 0 is the central node # for i in range(n): # print(nx.degree(G, i)) # Set edge attributes to random values representing contractual exposure attrs = {} for i in range(n): edge = (0, i, 0) attrs[edge] = {'exposure': np.random.randn()} # print(attrs) nx.set_edge_attributes(G, attrs) # # Set node attributes to random values representing credit risk # attrs = {} for i in range(n): attrs[i] = {'rating': np.random.randn()} print(attrs) nx.set_node_attributes(G, attrs) dataset_path = cm.source_path + "/datasets/" nx.write_yaml(G, dataset_path + "star.yml") # Load a graph G1 = nx.read_yaml(dataset_path + "star.yml") nx.draw(G1)
def add_edge_speeds(G, hwy_speeds=None, fallback=None, precision=1): """ Add edge speeds (km per hour) to graph as new `speed_kph` edge attributes. Imputes free-flow travel speeds for all edges based on mean `maxspeed` value of edges, per highway type. For highway types in graph that have no `maxspeed` value on any edge, function assigns the mean of all `maxspeed` values in graph. This mean-imputation can obviously be imprecise, and the caller can override it by passing in `hwy_speeds` and/or `fallback` arguments that correspond to local speed limit standards. If edge `maxspeed` attribute has "mph" in it, value will automatically be converted from miles per hour to km per hour. Any other speed units should be manually converted to km per hour prior to running this function, otherwise there could be unexpected results. If "mph" does not appear in the edge's maxspeed attribute string, then function assumes kph, per OSM guidelines: https://wiki.openstreetmap.org/wiki/Map_Features/Units Parameters ---------- G : networkx.MultiDiGraph input graph hwy_speeds : dict dict keys = OSM highway types and values = typical speeds (km per hour) to assign to edges of that highway type for any edges missing speed data. Any edges with highway type not in `hwy_speeds` will be assigned the mean preexisting speed value of all edges of that highway type. fallback : numeric default speed value (km per hour) to assign to edges whose highway type did not appear in `hwy_speeds` and had no preexisting speed values on any edge precision : int decimal precision to round speed_kph Returns ------- G : networkx.MultiDiGraph graph with speed_kph attributes on all edges """ if fallback is None: fallback = np.nan edges = utils_graph.graph_to_gdfs(G, nodes=False, fill_edge_geometry=False) # collapse any highway lists (can happen during graph simplification) # into string values simply by keeping just the first element of the list edges["highway"] = edges["highway"].map(lambda x: x[0] if isinstance(x, list) else x) if "maxspeed" in edges.columns: # collapse any maxspeed lists (can happen during graph simplification) # into a single value edges["maxspeed"] = edges["maxspeed"].map( _collapse_multiple_maxspeed_values) # create speed_kph by cleaning maxspeed strings and converting mph to # kph if necessary edges["speed_kph"] = edges["maxspeed"].astype(str).map( _clean_maxspeed).astype(float) else: # if no edges in graph had a maxspeed attribute edges["speed_kph"] = None # if user provided hwy_speeds, use them as default values, otherwise # initialize an empty series to populate with values if hwy_speeds is None: hwy_speed_avg = pd.Series(dtype=float) else: hwy_speed_avg = pd.Series(hwy_speeds).dropna() # for each highway type that caller did not provide in hwy_speeds, impute # speed of type by taking the mean of the preexisting speed values of that # highway type for hwy, group in edges.groupby("highway"): if hwy not in hwy_speed_avg: hwy_speed_avg.loc[hwy] = group["speed_kph"].mean() # if any highway types had no preexisting speed values, impute their speed # with fallback value provided by caller. if fallback=np.nan, impute speed # as the mean speed of all highway types that did have preexisting values hwy_speed_avg = hwy_speed_avg.fillna(fallback).fillna(hwy_speed_avg.mean()) # for each edge missing speed data, assign it the imputed value for its # highway type speed_kph = (edges[["highway", "speed_kph" ]].set_index("highway").iloc[:, 0].fillna(hwy_speed_avg)) # all speeds will be null if edges had no preexisting maxspeed data and # caller did not pass in hwy_speeds or fallback arguments if pd.isnull(speed_kph).all(): raise ValueError(("this graph's edges have no preexisting `maxspeed` " "attribute values so you must pass `hwy_speeds` or " "`fallback` arguments.")) # add speed kph attribute to graph edges edges["speed_kph"] = speed_kph.round(precision).values nx.set_edge_attributes(G, values=edges["speed_kph"], name="speed_kph") return G
def _validate_all_edges(self): diams = {edge: 1 for edge in self._layout.edges} nx.set_edge_attributes(self._layout, diams, name='diam')
def generate_graph(self, node_count): eps = 0.02 p = ((1 + eps) * np.log(node_count)) / node_count graph = nx.generators.gnp_random_graph(node_count, p, directed=True) nx.set_edge_attributes(graph, 1, "feature") return graph
def calcul_sim_chaines_2_4(): with open("graphe_complet_pondere_sim.pickle", 'rb') as fichier_graphe_complet: mon_depickler_complet = pickle.Unpickler(fichier_graphe_complet) graphe_complet = mon_depickler_complet.load() print(graphe_complet.edges.data()) nx.set_node_attributes(graphe_complet, [], "chaines") for u in graphe_complet.nodes(): graphe_complet.nodes[u]["chaines"] = recherche_chaines( graphe_complet, u) print(graphe_complet.nodes.data()) nx.set_edge_attributes(graphe_complet, -1.0, "sim_chaines_2_4") for u, v, data in graphe_complet.edges(data=True): chaine_1 = list( set(graphe_complet.nodes[u]["chaines"][1] + graphe_complet.nodes[u]["chaines"][3] + [1, 2, 3, 4, 5])) chaine_2 = list( set(graphe_complet.nodes[v]["chaines"][1] + graphe_complet.nodes[v]["chaines"][3] + [1, 2, 3, 4, 5])) with open( "graphes_extension/fichier_" + graphe_complet.nodes[u]["nom"] + ".pickle", 'rb') as fichier_graphe1: mon_depickler_graphe1 = pickle.Unpickler(fichier_graphe1) graphe1 = mon_depickler_graphe1.load() with open( "graphes_extension/fichier_" + graphe_complet.nodes[v]["nom"] + ".pickle", 'rb') as fichier_graphe2: mon_depickler_graphe2 = pickle.Unpickler(fichier_graphe2) graphe2 = mon_depickler_graphe2.load() with open("dico_graphe_epure_en_tout.pickle", 'rb') as fichier_commun: mon_depickler_commun = pickle.Unpickler(fichier_commun) dico_graphe = mon_depickler_commun.load() if ("fichier_" + graphe_complet.nodes[u]["nom"], "fichier_" + graphe_complet.nodes[v]["nom"] ) in dico_graphe.keys(): graphe_commun = dico_graphe[( "fichier_" + graphe_complet.nodes[u]["nom"], "fichier_" + graphe_complet.nodes[v]["nom"])] chaine_commun = [] for noeud in graphe_commun.nodes(): if noeud[0] in chaine_1 and noeud[1] in chaine_2: chaine_commun.append(noeud) chaine_commun = list( set(chaine_commun + [(1, 1), (2, 2), (3, 3), (4, 4), (5, 5)])) else: graphe_commun = dico_graphe[( "fichier_" + graphe_complet.nodes[v]["nom"], "fichier_" + graphe_complet.nodes[u]["nom"])] chaine_commun = [] for noeud in graphe_commun.nodes(): if noeud[1] in chaine_1 and noeud[0] in chaine_2: chaine_commun.append(noeud) chaine_commun = list( set(chaine_commun + [(1, 1), (2, 2), (3, 3), (4, 4), (5, 5)])) sim_2_4 = calcul_sim_non_cov_sans_motif( graphe1.subgraph(chaine_1), graphe2.subgraph(chaine_2), graphe_commun.subgraph(chaine_commun)) print(sim_2_4) graphe_complet.edges[u, v]["sim_chaines_2_4"] = sim_2_4 with open("graphe_complet_pondere_sim.pickle", 'wb') as fichier_graphe_complet_2: mon_pickler_complet = pickle.Pickler(fichier_graphe_complet_2) mon_pickler_complet.dump(graphe_complet)
def do_GET(self): try: print("path: " + str(self.path)) query = urlsplit(self.path).query print("query: " + str(query)) params = parse_qs(query) print("params: " + str(params)) origin = params["origin"][0].strip().split(",") dest = params["destination"][0].strip().split(",") origin_point = (float(origin[1]), float(origin[0])) origin = osmnx.get_nearest_node(G_raw, origin_point) dest_point = (float(dest[1]), float(dest[0])) destination = osmnx.get_nearest_node(G_raw, dest_point) alpha = float(params["alpha"][0]) networkx.set_edge_attributes(G, numpy.inf, "cost") for edge in G.edges: #print(begin,end) G.edges[edge]["cost"] = (1 - alpha) * G.edges[edge][ "time"] + alpha * G.edges[edge]["accidents"] cheapest_path = networkx.shortest_path(G, source=origin, target=destination, weight="cost") quickest_path = networkx.shortest_path(G, source=origin, target=destination, weight="time") # get_path_and_center(G, origin,destination,alpha,"cost") # # print("cheapest path: "+str(cheapest_path)) # print("quickest path: "+str(quickest_path)) r = networkx.shortest_path(G, origin, destination, "cost") latlng_list = [] latlng_center = [] latitude_total = 0 longitude_total = 0 for i in r: temp_list = [] temp_list.append(G_raw.node[i]['x']) temp_list.append(G_raw.node[i]['y']) latlng_list.append(temp_list) for i in latlng_list: latitude_total = latitude_total + i[0] longitude_total = longitude_total + i[1] avg_lat = latitude_total / len(latlng_list) avg_lng = longitude_total / len(latlng_list) latlng_center.append(avg_lat) latlng_center.append(avg_lng) data = { 'coordinates': latlng_list, 'center': latlng_center, 'alpha': alpha } #query = urlsplit(url).query #params = parse_qs(query) #print("parsed path: "+str(urldata)) # Send response status code self.send_response(200) outData = {} outData["coordinates"] = latlng_list outData["center"] = latlng_center outData["alpha"] = alpha # outData["cheapest_path"]=quickest_path # outData={} # outData["origin"]=origin # outData["destination"]=destination # outData["quickest_path"]=cheapest_path # outData["cheapest_path"]=quickest_path print("outData: " + str(outData)) # Send headers self.send_header('Content-type', 'text/html') self.end_headers() # Send message back to client message = json.dumps(outData) print("message: " + str(message)) #self.wfile.write("<html><body><h1>hi!</h1></body></html>") # Write content as utf-8 data self.wfile.write(bytes(message, "utf8")) except Exception as e: print("exception: " + str(e))
# self.wfile.write(bytes(message, "utf8")) if __name__ == '__main__': global G windower = config.windower() #config should default to entire dataset osmnx.config(log_file=True, log_console=True, use_cache=True) G_raw = osmnx.graph_from_place('Manhattan Island, New York, USA', network_type='drive') G = networkx.DiGraph(G_raw.copy()) print("INITIALIZING ACCIDENT DATA") accidents_raw = config.accidents(windower.timewindower) accidents = accidents_raw.copy() accidents.set_index("node", drop=True, inplace=True) accidents = accidents.groupby(level="node").size() networkx.set_edge_attributes(G, 0, "accidents") #set default value of zero accidents for node, count in accidents.iteritems(): try: for edge in G.in_edges(node): G.edges[edge]["accidents"] = count except Exception: #fails if node is not in graph pass print("INITIALIZING TRAVELTIME DATA") traveltimes_raw = config.traveltimes(windower.timewindower) traveltimes = traveltimes_raw.copy() traveltimes.set_index(["begin_node", "end_node"], drop=True, inplace=True) traveltimes_means = traveltimes["travel_time"].groupby( level=["begin_node", "end_node"]).mean() #set default value of infinite time.
def _sample_paths(self, input_rule_set, obs_name, target_polarity, max_paths=1, max_path_length=5): if max_paths == 0: raise ValueError("max_paths cannot be 0 for path sampling.") # Convert path polarity representation from 0/1 to 1/-1 def convert_polarities(path_list): return [tuple((n[0], 0 if n[1] > 0 else 1) for n in path) for path in path_list] pg_polarity = 0 if target_polarity > 0 else 1 nx_graph = _im_to_signed_digraph(self.get_im()) # Add edges from dummy node to input rules source_node = 'SOURCE_NODE' for rule in input_rule_set: nx_graph.add_edge(source_node, rule, attr_dict={'sign': 0}) # ------------------------------------------------- # Create combined paths_graph f_level, b_level = pg.get_reachable_sets(nx_graph, source_node, obs_name, max_path_length, signed=True) pg_list = [] for path_length in range(1, max_path_length+1): cfpg = pg.CFPG.from_graph( nx_graph, source_node, obs_name, path_length, f_level, b_level, signed=True, target_polarity=pg_polarity) pg_list.append(cfpg) combined_pg = pg.CombinedCFPG(pg_list) # Make sure the combined paths graph is not empty if not combined_pg.graph: pr = PathResult(False, 'NO_PATHS_FOUND', max_paths, max_path_length) pr.path_metrics = None pr.paths = [] return pr # Get a dict of rule objects rule_obj_dict = {} for ann in self.model.annotations: if ann.predicate == 'rule_has_object': rule_obj_dict[ann.subject] = ann.object # Get monomer initial conditions ic_dict = {} for mon in self.model.monomers: # FIXME: A hack that depends on the _0 convention ic_name = '%s_0' % mon.name # TODO: Wrap this in try/except? ic_param = self.model.parameters[ic_name] ic_value = ic_param.value ic_dict[mon.name] = ic_value # Set weights in PG based on model initial conditions for cur_node in combined_pg.graph.nodes(): edge_weights = {} rule_obj_list = [] edge_weights_by_gene = {} for u, v in combined_pg.graph.out_edges(cur_node): v_rule = v[1][0] # Get the object of the rule (a monomer name) rule_obj = rule_obj_dict.get(v_rule) if rule_obj: # Add to list so we can count instances by gene rule_obj_list.append(rule_obj) # Get the abundance of rule object from the initial # conditions # TODO: Wrap in try/except? ic_value = ic_dict[rule_obj] else: ic_value = 1.0 edge_weights[(u, v)] = ic_value edge_weights_by_gene[rule_obj] = ic_value # Get frequency of different rule objects rule_obj_ctr = Counter(rule_obj_list) # Normalize results by weight sum and gene frequency at this level edge_weight_sum = sum(edge_weights_by_gene.values()) edge_weights_norm = {} for e, v in edge_weights.items(): v_rule = e[1][1][0] rule_obj = rule_obj_dict.get(v_rule) if rule_obj: rule_obj_count = rule_obj_ctr[rule_obj] else: rule_obj_count = 1 edge_weights_norm[e] = ((v / float(edge_weight_sum)) / float(rule_obj_count)) # Add edge weights to paths graph nx.set_edge_attributes(combined_pg.graph, 'weight', edge_weights_norm) # Sample from the combined CFPG paths = combined_pg.sample_paths(max_paths) # ------------------------------------------------- if paths: pr = PathResult(True, 'PATHS_FOUND', max_paths, max_path_length) pr.path_metrics = None # Convert path polarity representation from 0/1 to 1/-1 pr.paths = convert_polarities(paths) # Strip off the SOURCE_NODE prefix pr.paths = [p[1:] for p in pr.paths] else: assert False pr = PathResult(False, 'NO_PATHS_FOUND', max_paths, max_path_length) pr.path_metrics = None pr.paths = [] return pr
EDGES = [] for i in range(len(source)): node1 = source[i] node2 = target[i] EDGES.append((node1, node2)) cnt = Counter(EDGES) # adding attributes to the graph d = {} for k in cnt.keys(): d[k] = {} d[k]['weight'] = cnt[k] #add additional attributes here for an edge between two nodes. The attributes can be a list with all the dates for instance nx.set_edge_attributes(G, d) #extracting edges with highest weights s = [] for (u, v) in G.edges(): w = G[u][v]['weight'] if w > 1: s.append([(u, v), w]) s = np.array(s) edge_imp = s[s[:, 1].astype('float64').argsort()[::-1]] #most important edges with open('edge_importance.csv', 'w') as f: writer = csv.writer(f) writer.writerows(edge_imp) #extracting unique relations of high weight edges
from matplotlib import pyplot as plt import scipy.cluster.vq as vq import networkx as nx import csv from tkinter import messagebox from tkinter import filedialog import seaborn as sns sns.set() file_open = filedialog.askopenfilename(initialdir='D:\PhD-Study\Experimental\SoDIP6', title='Select file', \ filetypes=(('GML files', '*.gml'),('CSV files', '*.csv'), ('All files', '*.*'))) if file_open.endswith('.gml'): try: G = nx.read_gml(file_open) if nx.get_edge_attributes(G, name='weight') is None: nx.set_edge_attributes(G,10.0,name='weight') except FileNotFoundError: print('File not found') elif file_open.endswith('.csv'): G = nx.Graph() try: with open(file_open) as graph_file: read_file = csv.DictReader(graph_file) for row in read_file: G.add_edge(row['node_from'], row['node_to'], weight=float(row['bandwidth'])) except FileNotFoundError: print('File or node error') else: messagebox.showerror('SoDIP6','Could Not Load file') with open('../dataset/generate_csv.csv', 'w', newline='') as write_csv:
def _boiler_generator_assn( eia_transformed_dfs, eia923_years=pc.WORKING_PARTITIONS['eia923']['years'], eia860_years=pc.WORKING_PARTITIONS['eia860']['years'], debug=False): """ Creates a set of more complete boiler generator associations. Creates a unique unit_id_pudl for each collection of boilers and generators within a plant that have ever been associated with each other, based on the boiler generator associations reported in EIA860. Unfortunately, this information is not complete for years before 2014, as the gas turbine portion of combined cycle power plants in those earlier years were not reporting their fuel consumption, or existence as part of the plants. For years 2014 and on, EIA860 contains a unit_id_eia value, allowing the combined cycle plant compoents to be associated with each other. For many plants not listed in the reported boiler generator associations, it is nonetheless possible to associate boilers and generators on a one-to-one basis, as they use identical strings to describe the units. In the end, between the reported BGA table, the string matching, and the unit_id_eia values, it's possible to create a nearly complete mapping of the generation units, at least for 2014 and later. Args: eia_transformed_dfs (dict): a dictionary of post-transform dataframes representing the EIA database tables. eia923_years (list-like): a list of the years of EIA 923 data that should be used to infer the boiler-generator associations. By default it is all the working years of data. eia860_years (list-like): a list of the years of EIA 860 data that should be used to infer the boiler-generator associations. By default it is all the working years of data. debug (bool): If True, include columns in the returned dataframe indicating by what method the individual boiler generator associations were inferred. Returns: eia_transformed_dfs (dict): Returns the same dictionary of dataframes that was passed in, and adds a new dataframe to it representing the boiler-generator associations as records containing plant_id_eia, generator_id, boiler_id, and unit_id_pudl Raises: AssertionError: If the boiler - generator association graphs are not bi-partite, meaning generators only connect to boilers, and boilers only connect to generators. AssertionError: If all the boilers do not end up with the same unit_id each year. AssertionError: If all the generators do not end up with the same unit_id each year. """ # if you're not ingesting both 860 and 923, the bga is not compilable if not (eia860_years and eia923_years): return pd.DataFrame() # compile and scrub all the parts logger.info("Inferring complete EIA boiler-generator associations.") bga_eia860 = ( eia_transformed_dfs['boiler_generator_assn_eia860'].copy().pipe( _restrict_years, eia923_years, eia860_years).astype({ 'generator_id': pd.StringDtype(), 'boiler_id': pd.StringDtype(), "plant_id_eia": int, })) # grab the generation_eia923 table, group annually, generate a new tag gen_eia923 = eia_transformed_dfs['generation_eia923'].copy() gen_eia923 = gen_eia923.set_index(pd.DatetimeIndex(gen_eia923.report_date)) gen_eia923 = (_restrict_years( gen_eia923, eia923_years, eia860_years).astype({ 'generator_id': pd.StringDtype(), "plant_id_eia": int, }).groupby([pd.Grouper(freq='AS'), 'plant_id_eia', 'generator_id' ]).net_generation_mwh.sum().reset_index().assign( missing_from_923=False)) # compile all of the generators gens_eia860 = (eia_transformed_dfs['generators_eia860'].copy().pipe( _restrict_years, eia923_years, eia860_years).astype({ 'generator_id': pd.StringDtype(), "plant_id_eia": int, })) gens = pd.merge(gen_eia923, gens_eia860, on=['plant_id_eia', 'report_date', 'generator_id'], how='outer') gens = (gens[[ 'plant_id_eia', 'report_date', 'generator_id', 'unit_id_eia', 'net_generation_mwh', 'missing_from_923' ]].drop_duplicates().astype({ 'generator_id': pd.StringDtype(), "plant_id_eia": int, })) # create the beginning of a bga compilation w/ the generators as the # background bga_compiled_1 = pd.merge( gens, bga_eia860, on=['plant_id_eia', 'generator_id', 'report_date'], how='outer') # Create a set of bga's that are linked, directly from bga8 bga_assn = bga_compiled_1[bga_compiled_1['boiler_id'].notnull()].copy() bga_assn.loc[:, 'bga_source'] = 'eia860_org' # Create a set of bga's that were not linked directly through bga8 bga_unassn = bga_compiled_1[bga_compiled_1['boiler_id'].isnull()].copy() bga_unassn = bga_unassn.drop(['boiler_id'], axis=1) # Side note: there are only 6 generators that appear in bga8 that don't # apear in gens9 or gens8 (must uncomment-out the og_tag creation above) # bga_compiled_1[bga_compiled_1['og_tag'].isnull()] bf_eia923 = (eia_transformed_dfs['boiler_fuel_eia923'].copy().pipe( _restrict_years, eia923_years, eia860_years).astype({ "boiler_id": pd.StringDtype(), "plant_id_eia": int, }).assign(total_heat_content_mmbtu=lambda x: x.fuel_consumed_units * x. fuel_mmbtu_per_unit)) bf_eia923 = (bf_eia923.set_index(pd.DatetimeIndex( bf_eia923.report_date)).groupby( [pd.Grouper(freq='AS'), 'plant_id_eia', 'boiler_id']).agg({ 'total_heat_content_mmbtu': pudl.helpers.sum_na }).reset_index().drop_duplicates( subset=['plant_id_eia', 'report_date', 'boiler_id'])) # Create a list of boilers that were not in bga8 bf9_bga = bf_eia923.merge( bga_compiled_1, on=['plant_id_eia', 'boiler_id', 'report_date'], how='outer', indicator=True, ) bf9_not_in_bga = bf9_bga[bf9_bga['_merge'] == 'left_only'] bf9_not_in_bga = bf9_not_in_bga.drop(['_merge'], axis=1) # Match the unassociated generators with unassociated boilers # This method is assuming that some the strings of the generators and the # boilers are the same bga_unassn = (bga_unassn.merge( bf9_not_in_bga[['plant_id_eia', 'boiler_id', 'report_date']], how='left', left_on=['report_date', 'plant_id_eia', 'generator_id'], right_on=['report_date', 'plant_id_eia', 'boiler_id'])) bga_unassn["bga_source"] = np.where(bga_unassn.boiler_id.notnull(), "string_assn", pd.NA) bga_compiled_2 = (bga_assn.append(bga_unassn).fillna( {'missing_from_923': True})) # Connect the gens and boilers in units bga_compiled_units = bga_compiled_2.loc[ bga_compiled_2['unit_id_eia'].notnull()] bga_gen_units = bga_compiled_units.drop(['boiler_id'], axis=1) bga_boil_units = bga_compiled_units[[ 'plant_id_eia', 'report_date', 'boiler_id', 'unit_id_eia' ]].copy() bga_boil_units.dropna(subset=['boiler_id'], inplace=True) # merge the units with the boilers bga_unit_compilation = bga_gen_units.merge( bga_boil_units, how='outer', on=['plant_id_eia', 'report_date', 'unit_id_eia'], indicator=True) # label the bga_source bga_unit_compilation.loc[bga_unit_compilation['bga_source'].isnull(), 'bga_source'] = 'unit_connection' bga_unit_compilation.drop(['_merge'], axis=1, inplace=True) bga_non_units = bga_compiled_2[bga_compiled_2['unit_id_eia'].isnull()] # combine the unit compilation and the non units bga_compiled_3 = bga_non_units.append(bga_unit_compilation) bga_compiled_3 = bga_compiled_3[[ 'plant_id_eia', 'report_date', 'generator_id', 'boiler_id', 'unit_id_eia', 'bga_source', 'net_generation_mwh', 'missing_from_923' ]] # label plants that have 'bad' generator records (generators that have MWhs # in gens9 but don't have connected boilers) create a df with just the bad # plants by searching for the 'bad' generators bad_plants = bga_compiled_3[(bga_compiled_3['boiler_id'].isnull()) & (bga_compiled_3['net_generation_mwh'] > 0)].\ drop_duplicates(subset=['plant_id_eia', 'report_date']) bad_plants = bad_plants[['plant_id_eia', 'report_date']] # merge the 'bad' plants back into the larger frame bga_compiled_3 = bga_compiled_3.merge(bad_plants, how='outer', on=['plant_id_eia', 'report_date'], indicator=True) # use the indicator to create labels bga_compiled_3['plant_w_bad_generator'] = np.where( bga_compiled_3["_merge"] == 'both', True, False) # Note: At least one gen has reported MWh in 923, but could not be # programmatically mapped to a boiler # we don't need this one anymore bga_compiled_3 = bga_compiled_3.drop(['_merge'], axis=1) # create a label for generators that are unmapped but in 923 bga_compiled_3['unmapped_but_in_923'] = np.where( (bga_compiled_3.boiler_id.isnull()) & ~bga_compiled_3.missing_from_923 & (bga_compiled_3.net_generation_mwh == 0), True, False) # create a label for generators that are unmapped bga_compiled_3['unmapped'] = np.where(bga_compiled_3.boiler_id.isnull(), True, False) bga_out = bga_compiled_3.drop('net_generation_mwh', axis=1) bga_out.loc[bga_out.unit_id_eia.isnull(), 'unit_id_eia'] = pd.NA bga_for_nx = bga_out[[ 'plant_id_eia', 'report_date', 'generator_id', 'boiler_id', 'unit_id_eia' ]] # If there's no boiler... there's no boiler-generator association bga_for_nx = bga_for_nx.dropna(subset=['boiler_id']).drop_duplicates() # Need boiler & generator specific ID strings, or they look like # the same node to NX bga_for_nx['generators'] = ( 'p' + bga_for_nx.plant_id_eia.astype(int).astype(str) + '_g' + bga_for_nx.generator_id.astype(pd.StringDtype())) bga_for_nx['boilers'] = ('p' + bga_for_nx.plant_id_eia.astype(int).astype(str) + '_b' + bga_for_nx.boiler_id.astype(pd.StringDtype())) # dataframe to accumulate the unit_ids in bga_w_units = pd.DataFrame() # We want to start our unit_id counter anew for each plant: for pid in bga_for_nx.plant_id_eia.unique(): bga_byplant = bga_for_nx[bga_for_nx.plant_id_eia == pid].copy() # Create a graph from the dataframe of boilers and generators. It's a # multi-graph, meaning the same nodes can be connected by more than one # edge -- this allows us to preserve multiple years worth of boiler # generator association information for later inspection if need be: bga_graph = nx.from_pandas_edgelist(bga_byplant, source='generators', target='boilers', edge_attr=True, create_using=nx.MultiGraph()) # Each connected sub-graph is a generation unit: gen_units = [ bga_graph.subgraph(c).copy() for c in nx.connected_components(bga_graph) ] # Assign a unit_id to each subgraph, and extract edges into a dataframe for unit_id, unit in zip(range(len(gen_units)), gen_units): # All the boiler-generator association graphs should be bi-partite, # meaning generators only connect to boilers, and boilers only # connect to generators. if not nx.algorithms.bipartite.is_bipartite(unit): raise AssertionError( f"Non-bipartite generation unit graph found." f"plant_id_eia={pid}, unit_id_pudl={unit_id}.") nx.set_edge_attributes(unit, name='unit_id_pudl', values=unit_id + 1) new_unit_df = nx.to_pandas_edgelist(unit) bga_w_units = bga_w_units.append(new_unit_df) bga_w_units = bga_w_units.drop(['source', 'target'], axis=1) # Check whether the PUDL unit_id values we've inferred conflict with # the unit_id_eia values that were reported to EIA. Are there any PUDL # unit_id values that have more than 1 EIA unit_id_eia within them? bga_unit_id_eia_counts = (bga_w_units.groupby( ['plant_id_eia', 'unit_id_pudl'])['unit_id_eia'].nunique().to_frame().reset_index()) bga_unit_id_eia_counts = bga_unit_id_eia_counts.rename( columns={'unit_id_eia': 'unit_id_eia_count'}) bga_unit_id_eia_counts = pd.merge(bga_w_units, bga_unit_id_eia_counts, on=['plant_id_eia', 'unit_id_pudl']) too_many_codes = ( bga_unit_id_eia_counts[bga_unit_id_eia_counts.unit_id_eia_count > 1]) too_many_codes = ( too_many_codes[~too_many_codes.unit_id_eia.isnull()].groupby( ['plant_id_eia', 'unit_id_pudl'])['unit_id_eia'].unique()) for row in too_many_codes.iteritems(): logger.warning(f"Multiple EIA unit codes:" f"plant_id_eia={row[0][0]}, " f"unit_id_pudl={row[0][1]}, " f"unit_id_eia={row[1]}") bga_w_units = bga_w_units.drop('unit_id_eia', axis=1) # These assertions test that all boilers and generators ended up in the # same unit_id across all the years of reporting: pgu_gb = bga_w_units.groupby(['plant_id_eia', 'generator_id'])['unit_id_pudl'] if not (pgu_gb.nunique() <= 1).all(): # raise AssertionError("Inconsistent inter-annual BGA assignment!") logger.error("Inconsistent inter-annual plant-generator-units!") pbu_gb = bga_w_units.groupby(['plant_id_eia', 'boiler_id'])['unit_id_pudl'] if not (pbu_gb.nunique() <= 1).all(): # raise AssertionError("Inconsistent inter-annual BGA assignment!") logger.error("Inconsistent inter-annual plant-boiler-units!") bga_w_units = (bga_w_units.drop( 'report_date', axis=1 ).loc[:, ['plant_id_eia', 'unit_id_pudl', 'generator_id', 'boiler_id']]. drop_duplicates()) bga_out = (pd.merge(left=bga_out, right=bga_w_units, how='left', on=['plant_id_eia', 'generator_id', 'boiler_id' ]).astype({"unit_id_pudl": pd.Int64Dtype()})) if not debug: bga_out = (bga_out[~bga_out.missing_from_923 & ~bga_out.plant_w_bad_generator & ~bga_out.unmapped_but_in_923 & ~bga_out.unmapped].drop( [ 'missing_from_923', 'plant_w_bad_generator', 'unmapped_but_in_923', 'unmapped' ], axis=1).drop_duplicates(subset=[ 'report_date', 'plant_id_eia', 'boiler_id', 'generator_id' ])) bga_out = pudl.helpers.convert_cols_dtypes(bga_out, "eia") eia_transformed_dfs['boiler_generator_assn_eia860'] = bga_out return eia_transformed_dfs
def prep_for_learning(ep_len, m, n, h, init_states, obstacles, pick_up_state, delivery_state, rewards, rew_val, custom_flag, custom_task): # Create the environment and get the TS # ts_start_time = timeit.default_timer() disc = 1 TS, obs_mat, state_mat = create_ts(m,n,h) path = '../data/ts_' + str(m) + 'x' + str(n) + 'x' + str(h) + '_1Ag_1.txt' paths = [path] bases = {init_states[0]: 'Base1'} obs_mat = update_obs_mat(obs_mat, state_mat, m, obstacles, init_states[0]) TS = update_adj_mat_3D(m, n, h, TS, obs_mat) create_input_file(TS, state_mat, obs_mat, paths[0], bases, disc, m, n, h, 0) ts_file = paths ts_dict = Ts(directed=True, multi=False) ts_dict.read_from_file(ts_file[0]) ts = expand_duration_ts(ts_dict) ts_timecost = timeit.default_timer() - ts_start_time # Get the DFA # dfa_start_time = timeit.default_timer() pick_up = str(pick_up_state[0][0] * n + pick_up_state[0][1]) # Check later delivery = str(delivery_state[0][0] * n + delivery_state[0][1]) tf = str((ep_len-1)/2) # time bound if custom_flag == 1: phi = custom_task else: phi = '[H^1 r' + pick_up + ']^[0, ' + tf + '] * [H^1 r' + delivery + ']^[0,' + tf + ']' # Construc the task according to pickup/delivery )^[0, ' + tf + ']' _, dfa_inf, bdd = twtl.translate(phi, kind=DFAType.Infinity, norm=True) # states and sim. time ex. phi = '([H^1 r47]^[0, 30] * [H^1 r31]^[0, 30])^[0, 30]' dfa_timecost = timeit.default_timer() - dfa_start_time # DFAType.Normal for normal, DFAType.Infinity for relaxed # Get the PA # pa_start_time = timeit.default_timer() alpha = 1 nom_weight_dict = {} weight_dict = {} pa_or = ts_times_fsa(ts, dfa_inf) # Original pa edges_all = nx.get_edge_attributes(ts_dict.g,'edge_weight') max_edge = max(edges_all, key=edges_all.get) norm_factor = edges_all[max_edge] for pa_edge in pa_or.g.edges(): edge = (pa_edge[0][0], pa_edge[1][0], 0) nom_weight_dict[pa_edge] = edges_all[edge]/norm_factor nx.set_edge_attributes(pa_or.g, 'edge_weight', nom_weight_dict) nx.set_edge_attributes(pa_or.g, 'weight', 1) pa = copy.deepcopy(pa_or) # copy the pa time_weight = nx.get_edge_attributes(pa.g,'weight') edge_weight = nx.get_edge_attributes(pa.g,'edge_weight') for pa_edge in pa.g.edges(): weight_dict[pa_edge] = alpha*time_weight[pa_edge] + (1-alpha)*edge_weight[pa_edge] nx.set_edge_attributes(pa.g, 'new_weight', weight_dict) pa_timecost = timeit.default_timer() - pa_start_time # Compute the energy of the states # energy_time = timeit.default_timer() compute_energy(pa) energy_dict = nx.get_node_attributes(pa.g,'energy') energy_pa = [] for ind in range(len(pa.g.nodes())): energy_pa.append(pa.g.nodes([0])[ind][1].values()[0]) # projection of pa on ts # init_state = [init_states[0][0] * n + init_states[0][1]] pa2ts = [] for i in range(len(pa.g.nodes())): if pa.g.nodes()[i][0] != 'Base1': pa2ts.append(int(pa.g.nodes()[i][0].replace("r",""))) else: pa2ts.append(init_state[0]) i_s = i # Agent's initial location in pa energy_timecost = timeit.default_timer() - pa_start_time # TS adjacency matrix and source-target TS_adj = TS TS_s = [] TS_t = [] for i in range(len(TS_adj)): for j in range(len(TS_adj)): if TS_adj[i,j] != 0: TS_s.append(i) TS_t.append(j) # pa adjacency matrix and source-target pa_adj_st = nx.adjacency_matrix(pa.g) pa_adj = pa_adj_st.todense() pa_s = [] # source node pa_t = [] # target node for i in range(len(pa_adj)): for j in range(len(pa_adj)): if pa_adj[i,j] == 1: pa_s.append(i) pa_t.append(j) # PA rewards matrix rewards_ts = np.zeros(m * n)# - 1 # penalty? rewards_pa = np.zeros(len(pa2ts)) rewards_ts_indexes = [] for i in range(len(rewards)): rewards_ts_indexes.append(rewards[i][0] * n + rewards[i][1]) # rewards_ts_indexes[i] = rewards[i][0] * n + rewards[i][1] rewards_ts[rewards_ts_indexes[i]] = rew_val for i in range(len(rewards_pa)): rewards_pa[i] = rewards_ts[pa2ts[i]] # # Display some important info print('##### PICK-UP and DELIVERY MISSION #####' + "\n") print('Initial Location : ' + str(init_states[0]) + ' <---> Region ' + str(init_state[0])) print('Pick-up Location : ' + str(pick_up_state[0]) + ' <---> Region ' + pick_up) print('Delivery Location : ' + str(delivery_state[0]) + ' <---> Regions ' + delivery) print('Reward Locations : ' + str(rewards) + ' <---> Regions ' + str(rewards_ts_indexes) + "\n") print('State Matrix : ') print(state_mat) print("\n") print('Mission Duration : ' + str(ep_len) + ' time steps') print('TWTL Task : ' + phi + "\n") print('Computational Costst : TS created in ' + str(ts_timecost) + ' seconds') # print(' TS created in ' + str(ts_timecost) + ' seconds') print(' DFA created in ' + str(dfa_timecost) + ' seconds') print(' PA created in ' + str(pa_timecost) + ' seconds') print(' Energy of PA states calculated in ' + str(energy_timecost) + ' seconds') return i_s, pa, pa_s, pa_t, pa2ts, energy_pa, rewards_pa, pick_up, delivery
def reweight_by_selection(G, vertices_selected, reweight_value): """Assumes an unweighted graph is passed in""" nx.set_edge_attributes(G, 1, WEIGHT_ATTRIBUTE) for v in vertices_selected: for neighbour in G[v]: G[v][neighbour][WEIGHT_ATTRIBUTE] = reweight_value
def bind_properties_to_network(network, network_properties, bind_node_positions=True, bind_node_color=True, bind_node_radius=True, bind_node_stroke_color=True, bind_node_stroke_width=True, bind_link_width=True, bind_link_color=True, bind_link_alpha=True): """ Binds calculated positional values to the network as node attributes `x` and `y`. Parameters ---------- network : networkx.Graph or something alike The network object to which the position should be bound network_properties : dict The network properties which are returned from the interactive visualization. bind_node_positions : bool (default: True) bind_node_color : bool (default: True) bind_node_radius : bool (default: True) bind_node_stroke_color : bool (default: True) bind_node_stroke_width : bool (default: True) bind_link_width : bool (default: True) bind_link_color : bool (default: True) bind_link_alpha : bool (default: True) Example ------- >>> props, _ = netwulf.visualize(G) >>> netwulf.bind_properties_to_network(G, props) """ # Add individial node attributes if bind_node_positions: x = {node['id']: node['x'] for node in network_properties['nodes']} y = {node['id']: node['y'] for node in network_properties['nodes']} nx.set_node_attributes(network, x, 'x') nx.set_node_attributes(network, y, 'y') network.graph['rescale'] = False if bind_node_color: color = { node['id']: node['color'] for node in network_properties['nodes'] } nx.set_node_attributes(network, color, 'color') if bind_node_radius: radius = { node['id']: node['radius'] for node in network_properties['nodes'] } nx.set_node_attributes(network, radius, 'radius') # Add individual link attributes if bind_link_width: width = {(link['source'], link['target']): link['width'] for link in network_properties['links']} nx.set_edge_attributes(network, width, 'width') # Add global style properties if bind_node_stroke_color: network.graph['nodeStrokeColor'] = network_properties[ 'nodeStrokeColor'] if bind_node_stroke_width: network.graph['nodeStrokeWidth'] = network_properties[ 'nodeStrokeWidth'] if bind_link_color: network.graph['linkColor'] = network_properties['linkColor'] if bind_link_alpha: network.graph['linkAlpha'] = network_properties['linkAlpha']
def randcost(G,k): nx.set_edge_attributes(G, 'cost', 0) #set all costs to 0 edges = random.sample(G.edges(),k) #obtain k edges dic = {key: 1 for key in edges} #dictionary, the k edges have cost 1 nx.set_edge_attributes(G, 'cost', dic) #set unit cost for k edges
def _compute_ricci_flow(G: nx.Graph, weight="weight", iterations=100, step=1, delta=1e-4, surgery=(lambda G, *args, **kwargs: G, 100), **kwargs): """Compute the given Ricci flow metric of each edge of a given connected NetworkX graph. Parameters ---------- G : NetworkX graph A given directional or undirectional NetworkX graph. weight : str The edge weight used to compute Ricci curvature. (Default value = "weight") iterations : int Iterations to require Ricci flow metric. (Default value = 100) step : float step size for gradient decent process. (Default value = 1) delta : float process stop when difference of Ricci curvature is within delta. (Default value = 1e-4) surgery : (function, int) A tuple of user define surgery function that will execute every certain iterations. (Default value = (lambda G, *args, **kwargs: G, 100)) **kwargs Additional keyword arguments passed to `_compute_ricci_curvature`. Returns ------- G: NetworkX graph A NetworkX graph with ``weight`` as Ricci flow metric. """ if not nx.is_connected(G): logger.warning( "Not connected graph detected, compute on the largest connected component instead." ) G = nx.Graph(G.subgraph(max(nx.connected_components(G), key=len))) # Set normalized weight to be the number of edges. normalized_weight = float(G.number_of_edges()) global _apsp # Start compute edge Ricci flow t0 = time.time() if nx.get_edge_attributes(G, "original_RC"): logger.warning( "original_RC detected, continue to refine the ricci flow.") else: _compute_ricci_curvature(G, weight=weight, **kwargs) for (v1, v2) in G.edges(): G[v1][v2]["original_RC"] = G[v1][v2]["ricciCurvature"] # clear the APSP since the graph have changed. _apsp = {} # Start the Ricci flow process for i in range(iterations): for (v1, v2) in G.edges(): G[v1][v2][weight] -= step * ( G[v1][v2]["ricciCurvature"]) * G[v1][v2][weight] # Do normalization on all weight to prevent weight expand to infinity w = nx.get_edge_attributes(G, weight) sumw = sum(w.values()) for k, v in w.items(): w[k] = w[k] * (normalized_weight / sumw) nx.set_edge_attributes(G, values=w, name=weight) logger.info(" === Ricci flow iteration %d === " % i) _compute_ricci_curvature(G, weight=weight, **kwargs) rc = nx.get_edge_attributes(G, "ricciCurvature") diff = max(rc.values()) - min(rc.values()) logger.info("Ricci curvature difference: %f" % diff) logger.info("max:%f, min:%f | maxw:%f, minw:%f" % (max( rc.values()), min(rc.values()), max(w.values()), min(w.values()))) if diff < delta: logger.info("Ricci curvature converged, process terminated.") break # do surgery or any specific evaluation surgery_func, do_surgery = surgery if i != 0 and i % do_surgery == 0: G = surgery_func(G, weight) normalized_weight = float(G.number_of_edges()) for n1, n2 in G.edges(): logger.debug("%s %s %s" % (n1, n2, G[n1][n2])) # clear the APSP since the graph have changed. _apsp = {} logger.info("\n%8f secs for Ricci flow computation." % (time.time() - t0)) return G
def gtfs_db_to_schedule_graph(stop_times_db, stops_db, trips_db, routes_db, services): def get_time(time): # return time as datetime.datetime, account for 24 in %H time_list = time.split(':') if int(time_list[0]) >= 24: days = int(time_list[0]) // 24 time_list[0] = int(time_list[0]) % 24 if time_list[0] < 10: time_list[0] = '0{}'.format(time_list[0]) else: time_list[0] = str(time_list[0]) return datetime.strptime(':'.join(time_list), '%H:%M:%S') + timedelta(days=days) else: return datetime.strptime(time, '%H:%M:%S') def timedelta_to_hms(td): return str(td).split('days')[-1].strip(' ') def generate_stop_sequence(group): group = group.sort_values(by='stop_sequence') # remove stops that are loopy (consecutively duplicated) unique_stops_mask = group['stop_id'].shift() != group['stop_id'] if not unique_stops_mask.all(): logging.warning( 'Your GTFS has (a) looooop edge(s)! A zero link between a node and itself, edge affected ' '\nThis edge will not be considered for computation, the stop will be deleted and the ' f'schedule will be changed. Affected stops: {group[~unique_stops_mask]["stop_id"].to_list()}' ) group = group.loc[unique_stops_mask] flattened = group.iloc[0, :][list( set(group.columns) - { 'trip_id', 'stop_sequence', 'stop_id', 'arrival_time', 'departure_time' })] departure_time = group.iloc[0, :]['arrival_time'] flattened['trip_departure_time'] = departure_time.strftime("%H:%M:%S") flattened['ordered_stops'] = group['stop_id'].to_list() flattened['stops_str'] = ','.join(group['stop_id'].to_list()) flattened['arrival_offsets'] = [ timedelta_to_hms(t - departure_time) for t in group['arrival_time'] ] flattened['departure_offsets'] = [ timedelta_to_hms(t - departure_time) for t in group['departure_time'] ] return flattened def generate_trips(group): flattened = group.iloc[0, :][list( set(group.columns) - { 'route_id', 'stops_str', 'trip_id', 'vehicle_id', 'trip_departure_time' })] trip_id = group['trip_id'].to_list() trip_departure_time = group['trip_departure_time'].to_list() vehicle_id = group['vehicle_id'].to_list() flattened['trips'] = { 'trip_id': trip_id, 'trip_departure_time': trip_departure_time, 'vehicle_id': vehicle_id } return flattened def generate_routes(group): service_id = group.iloc[0, :]['service_id'] group['route_id'] = [f'{service_id}_{i}' for i in range(len(group))] return group trips_db = trips_db[trips_db['service_id'].isin(services)] df = trips_db[['route_id', 'trip_id']].merge(routes_db[{ 'route_id', 'route_type', 'route_short_name', 'route_long_name', 'route_color' } and set(routes_db.columns)], on='route_id', how='left') df['mode'] = df['route_type'].apply(lambda x: get_mode(x)) df = df.merge(stop_times_db[[ 'trip_id', 'stop_id', 'arrival_time', 'departure_time', 'stop_sequence' ]], on='trip_id', how='left') df['arrival_time'] = df['arrival_time'].apply(lambda x: get_time(x)) df['departure_time'] = df['departure_time'].apply(lambda x: get_time(x)) df = df.groupby('trip_id').apply(generate_stop_sequence).reset_index() # drop stop sequences that are single stops df = df[df['ordered_stops'].str.len() > 1] df['vehicle_id'] = [f'veh_{i}' for i in range(len(df))] df = df.groupby(['route_id', 'stops_str']).apply(generate_trips).reset_index() df = df.drop('stops_str', axis=1) df['service_id'] = df['route_id'].astype(str) df = df.groupby(['service_id']).apply(generate_routes) g = nx.DiGraph(name='Schedule graph') g.graph['crs'] = 'epsg:4326' g.graph['route_to_service_map'] = df.set_index( 'route_id')['service_id'].T.to_dict() g.graph['service_to_route_map'] = df.groupby( 'service_id')['route_id'].apply(list).to_dict() g.graph['change_log'] = change_log.ChangeLog() df['id'] = df['route_id'] g.graph['routes'] = df.set_index('route_id').T.to_dict() df['id'] = df['service_id'] df = df.rename(columns={'route_short_name': 'name'}) g.graph['services'] = df[['service_id', 'id', 'name' ]].groupby('service_id').first().T.to_dict() # finally nodes stops = pd.DataFrame({ col: np.repeat(df[col].values, df['ordered_stops'].str.len()) for col in {'route_id', 'service_id'} }).assign(stop_id=np.concatenate(df['ordered_stops'].values)) stop_groups = stops.groupby('stop_id') stops = set(stop_groups.groups) g.add_nodes_from(stops) stops_db = stops_db.rename(columns={ 'stop_lat': 'lat', 'stop_lon': 'lon', 'stop_name': 'name' }) stops_db['id'] = stops_db['stop_id'] stops_db['x'] = stops_db['lon'] stops_db['y'] = stops_db['lat'] stops_db['epsg'] = 'epsg:4326' stops_db['s2_id'] = stops_db.apply(lambda x: spatial.generate_index_s2( lat=float(x['lat']), lng=float(x['lon'])), axis=1) nx.set_node_attributes( g, stops_db[stops_db['stop_id'].isin(stops)].set_index( 'stop_id').T.to_dict()) nx.set_node_attributes( g, pd.DataFrame( stop_groups['route_id'].apply(set)).rename(columns={ 'route_id': 'routes' }).T.to_dict()) nx.set_node_attributes( g, pd.DataFrame(stop_groups['service_id'].apply(set)).rename( columns={ 'service_id': 'services' }).T.to_dict()) # and edges df['ordered_stops'] = df['ordered_stops'].apply( lambda x: list(zip(x[:-1], x[1:]))) stop_cols = np.concatenate(df['ordered_stops'].values) edges = pd.DataFrame({ col: np.repeat(df[col].values, df['ordered_stops'].str.len()) for col in {'route_id', 'service_id'} }).assign(from_stop=stop_cols[:, 0], to_stop=stop_cols[:, 1]) edge_groups = edges.groupby(['from_stop', 'to_stop']) g.add_edges_from(edge_groups.groups) nx.set_edge_attributes( g, pd.DataFrame( edge_groups['route_id'].apply(set)).rename(columns={ 'route_id': 'routes' }).T.to_dict()) nx.set_edge_attributes( g, pd.DataFrame(edge_groups['service_id'].apply(set)).rename( columns={ 'service_id': 'services' }).T.to_dict()) return g
test_set = (0.99 * total_nodes) random_list = [i for i in range(total_nodes)] #shuffle(random_list) for node in range(0, total_nodes): if node < validation_set: G.node[random_list[node]]['val'] = False G.node[random_list[node]]['test'] = False elif node < test_set: G.node[random_list[node]]['val'] = True G.node[random_list[node]]['test'] = False else: G.node[random_list[node]]['val'] = False G.node[random_list[node]]['test'] = True nx.set_edge_attributes(G, 'test_removed', False) nx.set_edge_attributes(G, 'train_removed', False) json_graph_name = graph_name + "-G.json" json_id_map_name = graph_name + "-id_map.json" feats_file_name = graph_name + "-feats.npy" # #np.save(feats_file_name, features) data = json_graph.node_link_data(G) graphjson = json.dumps(data) f1 = open(json_graph_name, 'w') f1.write(graphjson) f1.close() #exit()
def load_network(self): # read METIS file self.G = utils.read_metis(self.DATA_FILENAME) self.initial_number_of_nodes = self.G.number_of_nodes( ) # used for computing metrics # Alpha value used in prediction model self.prediction_model_alpha = self.G.number_of_edges() * ( self.num_partitions / self.G.number_of_nodes()**2) if self.use_one_shot_alpha: self.prediction_model_alpha = self.one_shot_alpha # Order of nodes arriving self.arrival_order = list(range(0, self.G.number_of_nodes())) if self.SIMULATED_ARRIVAL_FILE == "": # mark all nodes as needing a shelter self.simulated_arrival_list = [1] * self.G.number_of_nodes() else: with open(self.SIMULATED_ARRIVAL_FILE, "r") as ar: self.simulated_arrival_list = [ int(line.rstrip('\n')) for line in ar ] # count the number of people arriving in the simulation self.number_simulated_arrivals = 0 for arrival in self.simulated_arrival_list: self.number_simulated_arrivals += arrival if self.verbose > 0: print("Graph loaded...") print(nx.info(self.G)) if nx.is_directed(self.G): print("Graph is directed") else: print("Graph is undirected") self.reset() # load displacement prediction weights if (self.PREDICTION_LIST_FILE == ""): self.predicted_displacement_weights = [ 1 ] * self.G.number_of_nodes() else: with open(self.PREDICTION_LIST_FILE, 'r') as plf: self.predicted_displacement_weights = [ float(line.rstrip('\n')) for line in plf ] # preserve original node/edge weight when modification functions are applied if self.graph_modification_functions: node_weights = { n[0]: n[1]['weight'] for n in self.G.nodes_iter(data=True) } nx.set_node_attributes(self.G, 'weight_orig', node_weights) edge_weights = {(e[0], e[1]): e[2]['weight'] for e in self.G.edges_iter(data=True)} nx.set_edge_attributes(self.G, 'weight_orig', edge_weights) # make a copy of the original graph since we may alter the graph later on # and may want to refer to information from the original graph self.originalG = self.G.copy()
} def slice_network(G, T, copy=True): """ Remove all edges witth weight<T from G or its copy, """ F = G.copy() if copy else G F.remove_edges_from( (n1, n2) for n1, n2, w in F.edges(data="weight") if w < T) return F # Draw different degrees of slicing for a random graph ergraph = nx.erdos_renyi_graph(100, 0.1) nx.set_edge_attributes(ergraph, {(n1, n2): random.random() for n1, n2 in ergraph.edges()}, "weight") pos = graphviz_layout(ergraph) for i, threshold in enumerate([0, 0.7, 0.8, 0.9, 0.95, 1.0]): ax = plt.subplots(3, 2, i + 1) slice_network(ergraph, threshold, copy=False) nx.draw_networkx_edges(ergraph, pos, alpha=0.65, ax=ax, **dzcnapy.small_attrs) nx.draw_networkx_nodes(ergraph, pos, ax=ax, **dzcnapy.small_attrs) dzcnapy.set_extent(pos, ax, "Threshold={}".format(threshold)) dzcnapy.plot("slicing")
def edit_graph(): # provide a different graph for each user graphname = "static/data/graph_login_test" if current_user.get_id() != None: graphname += "_" + current_user.get_id() + ".json" else: graphname += ".json" form1 = node_form(request.form) form2 = edge_form(request.form) # open graph json file: load into graph G with open(graphname, "r") as read_file: data = json.load(read_file) G = json_graph.node_link_graph(data) graph_info = [] num_nodes = "Number of Nodes: " + str(G.number_of_nodes()) num_edges = "Number of Edges: " + str(G.number_of_edges()) density = "Density of Graph: " + str(round(nx.density(G), 5)) graph_info.append(num_nodes) graph_info.append(num_edges) graph_info.append(density) if request.method == 'POST': # add node if form1.add_node.data and form1.validate(): if G.has_node(form1.node_name.data): print("add failed: such node already exist") flash(u"Add Failed: such node already exist", "danger") else: G.add_node( form1.node_name.data, category=form1.category.data, degree=0, viz={'size': 10}, # label=form1.node_name.data, url=form1.url.data, content=form1.content.data, notes=form1.notes.data) print("added a node") flash(u"Added Node: " + "'" + form1.node_name.data + "'", 'success') # else: # print(form1.validate()) # edit node if form1.edit_node.data and form1.validate(): if G.has_node(form1.node_name.data): attrs = {} if form1.category.data != "": attrs.update({"category": form1.category.data}) if form1.url.data != "": attrs.update({"url": form1.url.data}) if form1.content.data != "": attrs.update({"content": form1.content.data}) if form1.notes.data != "": attrs.update({"notes": form1.notes.data}) attr = {form1.node_name.data: attrs} nx.set_node_attributes(G, attr) print("updated a node") flash(u"Updated Node: " + "'" + form1.node_name.data + "'", 'success') else: print("edit failed: such node does not exist") flash(u"Edit Failed: such node does not exist", "danger") # delete node if form1.delete_node.data and form1.validate(): if G.has_node(form1.node_name.data): G.remove_node(form1.node_name.data) print("removed a node") flash(u"Removed Node: " + "'" + form1.node_name.data + "'", 'success') else: print("remove failed: such node does not exist") flash(u"Remove Failed: such node does not exist", "danger") # else: # print(form1.validate()) # add edge if form2.add_edge.data and form2.validate(): if G.has_node(form2.source_name.data) and G.has_node( form2.target_name.data): G.add_edge(form2.source_name.data, form2.target_name.data, relationship=form2.relationship.data, key=str(G.number_of_edges()), content=form2.content.data, notes=form2.notes.data) update_attr(G, form2) print("added an edge") flash(u"Added Edge (key: " + str(G.number_of_edges()) + "): " + "'" + form2.source_name.data + "' -> " + "'" + form2.target_name.data + "'" + 'success') else: print("add failed: source/target node does not exist") flash(u"Add Failed: source/target node does not exist", "danger") # edit edge if form2.edit_edge.data and form2.validate(): if form2.key_num.data == "": print("you need to input the key of the edge") flash(u"Edit Failed: you need to input the key of the edge", "danger") elif G.has_edge(form2.source_name.data, form2.target_name.data, key=form2.key_num.data): attrs = {} if form2.relationship.data != "": attrs.update({"relationship": form2.relationship.data}) if form2.content.data != "": attrs.update({"content": form2.content.data}) if form2.notes.data != "": attrs.update({"notes": form2.notes.data}) attr = { (form2.source_name.data, form2.target_name.data, form2.key_num.data): attrs } nx.set_edge_attributes(G, attr) print("updated an edge") flash(u"Updated Edge (key: " + str(G.number_of_edges()) + "): " + "'" + form2.source_name.data + "' -> " + "'" + form2.target_name.data + "'" + 'success') else: print("edit failed: such edge does not exist") flash(u"Edit Failed: such edge does not exist", "danger") # delete edge if form2.delete_edge.data and form2.validate(): if form2.key_num.data == "": print("you need to input the key of the edge") flash(u"Edit Failed: you need to input the key of the edge", "danger") if G.has_edge(form2.source_name.data, form2.target_name.data, key=form2.key_num.data): G.remove_edge(form2.source_name.data, form2.target_name.data, key=form2.key_num.data) update_attr(G, form2) print("removed an edge") flash(u"Eemoved Edge (key: " + str(G.number_of_edges()) + "): " + "'" + form2.source_name.data + "' -> " + "'" + form2.target_name.data + "'" + 'success') else: print("remove failed: such edge does not exist") flash(u"Edit Failed: such edge does not exist", "danger") # save edited graph G into json file data = json_graph.node_link_data(G) with open(graphname, "w") as write_file: json.dump(data, write_file) return redirect(url_for('edit_graph')) return render_template('edit-graph.html', form_node=form1, form_edge=form2, graph_info=graph_info, title="Edit Graph")
'San Francisco, CA', 'Saint Augustine, FL', 'Spokane, WA', 'Worcester, MA', 'Tucson, AZ'] # make subgraph of cities H = G.subgraph(cities) print("Subgraph has %d nodes with %d edges" % (len(H), H.size())) print(H.nodes) # draw with grave plt.figure(figsize=(8, 8)) # create attribute for label nx.set_edge_attributes(H, {e: G.edges[e]['weight'] for e in H.edges}, 'label') # create stylers def transfer_G_layout(network): return {n: G.position[n] for n in network} def elabel_base_style(attr): return {'font_size': 4, 'font_weight': 'bold', 'font_family': 'sans-serif', 'font_color': 'b', 'rotate': True, # TODO: make rotation less granular } elabel_style = grave.style_merger(grave.use_attributes('label'),
def set_edge_attribute(G, attr, new_attrs): nx.set_edge_attributes( G, {(u, v): va for (u, v, a), va in zip(G.edges(data=True), new_attrs)}, attr)
def solve(self, G): """ Find the pipes diameter for a given aqueduct topology respecting the following constraints: - pipes velocities between 0.5 and 1 m/s - pipes diameters commercially available """ import z3 # add a solver solver = z3.Solver() # +-----------------------+ # | variables declaration | # +-----------------------+ # water demand in each node X = dict([(node, z3.Real("demand_%s" % i)) for i, (node, datadict) in enumerate(G.nodes.items())]) # water flow in each pipe Q = dict([(edge, z3.Real("Q_%s" % i)) for i, (edge, datadict) in enumerate(G.edges.items())]) # water speed in each pipe V = dict([(edge, z3.Real("V_%s" % i)) for i, (edge, datadict) in enumerate(G.edges.items())]) # pipes diameter D = dict([(edge, z3.Real("D_%s" % i)) for i, (edge, datadict) in enumerate(G.edges.items())]) # +---------------------+ # | boundary conditions | # +---------------------+ # water demand in each node boudary_c = [] for node, datadict in G.nodes.items(): if not datadict["Tank"]: boudary_c.append(X[node] == datadict["DEMAND"]) # closing equation in each pipe: Q = V * A def sym_abs(x): return z3.If(x >= 0, x, -x) closing_c = [ sym_abs(Q[edge]) / 1000 == V[edge] * (D[edge] / 1000 / 2)**2 * math.pi for edge in G.edges ] # speed limits speed_c = [] for edge, datadict in G.edges.items(): if datadict["LEVEL"] == 1: speed_c.append(z3.And(V[edge] >= 0.1, V[edge] <= 1.)) else: speed_c.append(z3.And(V[edge] >= 0.01, V[edge] <= 1.5)) # pipes diameters diameter_c = [] available_measures = [ 75., 90., 110., 125., 140., 160., 180., 200., 225., 250., 280., 315., 355., 400. ] for edge, datadict in G.edges.items(): if datadict["LEVEL"] == 1: diameter_c += [ z3.Or( [D[edge] == measure for measure in available_measures]) ] else: diameter_c += [ z3.Or([ D[edge] == measure for measure in available_measures + [15., 25., 50.] ]) ] # kirchoff lows in the nodes kirchoff_c = [] for n1 in G.nodes: kirchoff_c.append( z3.Sum([X[n1]] + [ Q[(n1, n2)] if (n1, n2) in Q else -Q[(n2, n1)] for n2 in G.neighbors(n1) ]) == 0) # add conditons solver.add(boudary_c + closing_c + speed_c + kirchoff_c + diameter_c) def compute_head(): """ Computes head in meters for each node in the graph. The value is calculated imposing the head in the tank and than subtracting the loss on the pipes """ for node, datadict in G.nodes.items(): if datadict["Tank"] == True: start = node H = {start: 164} visited, queue = set(), [start] while queue: node = queue.pop(0) if node not in visited: visited.add(node) queue.extend(set(G.neighbors(node)) - visited) for neighbour in G.neighbors(node): l = G[node][neighbour]["LENGHT"] K = 10.67 / 120**1.852 def contain(d, k1, k2): if k1 in d: if k2 in d[k1]: return True return False if (node, neighbour) in dict(G.nodes): q = G[node][neighbour]["Q"] else: q = -G[neighbour][node]["Q"] d = G[node][neighbour]["DIAMETER"] H[neighbour] = H[node] - (K * l * math.copysign( (abs(q) / 1000)**1.852, q) / (d / 1000)**4.8704) return H if solver.check() == z3.sat: print("solved") m = solver.model() solver.add( z3.Or([D[edge] != m.evaluate(D[edge]) for edge in G.edges])) X = dict([(node, float(m.evaluate(X[node]).numerator_as_long()) / float(m.evaluate(X[node]).denominator_as_long())) for node in X]) nx.set_node_attributes(G, X, "Q") Q = dict([(edge, float(m.evaluate(Q[edge]).numerator_as_long()) / float(m.evaluate(Q[edge]).denominator_as_long())) for edge in Q]) V = dict([(edge, float(m.evaluate(V[edge]).numerator_as_long()) / float(m.evaluate(V[edge]).denominator_as_long())) for edge in V]) D = dict([(edge, float(m.evaluate(D[edge]).numerator_as_long()) / float(m.evaluate(D[edge]).denominator_as_long())) for edge in D]) data = dict([("Q", Q), ("V", V), ("DIAMETER", D)]) for key, datadict in data.items(): nx.set_edge_attributes(G, datadict, key) H = compute_head() nx.set_node_attributes(G, H, "H") else: print("no solutions")
def _order_edges_in_block(self, block_data, drop_augmented): """Produce an edge sequence for all edges in the component. Parameters ---------- block_data : pandas.DataFrame A DataFrame representing all the edges within a single block. drop_augmented : bool Whether or not to keep any edges that needed to be added to the source edges in order to navigate the network. Returns ------- edges : pandas.DataFrame The same edges that were input with the edge order and route type as new columns. """ logger.debug("order_edges_by_block started") logger.debug("Received edge data of shape %s", block_data.shape) # Sort the DataFrame to load right hand arcs into NetworkX first. # Note that Eulerian paths work in reverse order. block_data = block_data.sort_values(self.leftrightflag_field, ascending=False) block_g = nx.from_pandas_edgelist(block_data, source=self.source_field, target=self.target_field, edge_attr=True, create_using=self.graph_type) logger.debug("Block contains %s edges and %s nodes", block_g.number_of_edges(), block_g.number_of_nodes()) # if the graph is empty it means there is a problem with the source data # an error is logged, but other blocks are still processed if nx.is_empty(block_g): logger.error("Block contains no edges and cannot be sequenced") return # Scale nodes that are mid-segment by looking for duplicated ngd_str_uid values logger.debug( "Looking for nodes that fall in the middle of a road segment") block_data['same_ngd_str_uid'] = block_data.duplicated( subset=[self.struid_field], keep=False) mid_arc_start_nodes = set( block_data.loc[block_data['same_ngd_str_uid'] == True, self.source_field]) mid_arc_end_nodes = set( block_data.loc[block_data['same_ngd_str_uid'] == True, self.target_field]) mid_arc_nodes = mid_arc_start_nodes.intersection(mid_arc_end_nodes) if mid_arc_nodes: logger.debug("Found mid-segment nodes: %s", mid_arc_nodes) self._apply_node_scaling_factor(mid_arc_nodes, factor=-0.5) # initialize the edge sequence counter edge_sequence = 0 # record what type of path was used to determine the circuit path_indicator_name = self.path_indicator path_indicator_edges = {} # blocks don't necessarily form fully connected graphs, so cycle through the components logger.debug("Block contains %s connected components", nx.number_weakly_connected_components(block_g)) for block_comp in sorted(nx.weakly_connected_components(block_g), key=len, reverse=True): logger.debug( "Creating subgraph from connected component with %s nodes", len(block_comp)) block_g_comp = block_g.subgraph(block_comp) # determine the preferred start node for this block component preferred_sp = self._get_preferred_start_node(block_g_comp.nodes) logger.debug("Preferred start node for this block: %s", preferred_sp) logger.debug("Component contains %s edges and %s nodes", block_g_comp.number_of_edges(), len(block_g_comp)) # Need to pick an approach to processing this component depending on what type of circuit it forms. # Ideally things are a Eulerian circuit that can be walked and return to start, but not all blocks form # these nice circuits. If no good circuit can be found, then sequence numbers are just applied but may # not form a logical order. # Track the sequence value in case the enumeration method needs to be reset. This gets used when using # the preferred start point fails, and also controls if the start node for this component is marked as a # point we want to cluster on. seq_val_at_start = edge_sequence # the preferred option is a Eulerian circuit, so try that first # logger.debug("Available edges: %s", block_g_comp.edges) if nx.is_eulerian(block_g_comp): logger.debug("Block component is eulerian.") # record all these edges as being eulerian indicator = dict( zip(block_g_comp.edges, ['circuit'] * block_g_comp.size())) path_indicator_edges.update(indicator) # enumerate the edges and order them directly logger.debug("Creating Eulerian circuit from node %s", preferred_sp) for u, v, k in nx.eulerian_circuit(block_g_comp, source=preferred_sp, keys=True): edge_sequence += 1 block_g.edges[u, v, k][self.eo_name] = edge_sequence # logger.debug("Sequence applied: (%s, %s, %s) = %s", u, v, k, edge_sequence) # next best option is a path that stops at a different location from the start point elif nx.has_eulerian_path(block_g_comp): logger.debug("Block component forms Eulerian path") # record all these edges as being a eulerian path indicator = dict( zip(block_g_comp.edges, ['path'] * block_g_comp.size())) path_indicator_edges.update(indicator) try: logger.debug( "Trying to create path from preferred start node %s", preferred_sp) for u, v, k in nx.eulerian_path(block_g_comp, source=preferred_sp, keys=True): edge_sequence += 1 # check if the start point is actually in the first edge if edge_sequence == 1 and not (preferred_sp == u or preferred_sp == v): logger.debug( "Preferred start point not present on starting edge, throwing KeyError." ) raise KeyError("Invalid starting edge") # Sometimes the preferred start point means walking over the same edge twice, which will leave # a data gap (the previous edge order value will be overwritten). If this happens, throw a # KeyError if block_g.edges[u, v, k].get(self.eo_name): logger.debug("Edge already sequenced.") raise KeyError( "Preferred start point results in backtracking." ) block_g.edges[u, v, k][self.eo_name] = edge_sequence # logger.debug("Sequence applied: (%s, %s, %s) = %s", u, v, k, edge_sequence) if edge_sequence < block_g_comp.number_of_edges(): logger.debug("It looks like some edges got missed") raise KeyError("Missing edges on path") logger.debug( "Path was created from desired start point %s", preferred_sp) except KeyError: # preferred start point failed; let networkx pick and start over logger.debug( "Preferred start node did not create a path. Trying a different one." ) # reset the path listing since a new point will be picked logger.debug("Reset edge_sequence value to %s", seq_val_at_start) edge_sequence = seq_val_at_start for u, v, k in nx.eulerian_path(block_g_comp, keys=True): edge_sequence += 1 block_g.edges[u, v, k][self.eo_name] = edge_sequence # logger.debug("Sequence applied: (%s, %s, %s) = %s", u, v, k, edge_sequence) # No good path exists, which means someone will have to backtrack else: logger.debug( "Non-eulerian block is not easily traversable. Eulerizing it." ) # Record all these edges as being augmented. indicator = dict( zip(block_g_comp.edges, ['augmented'] * block_g_comp.size())) path_indicator_edges.update(indicator) # Send this data to the anomaly folder so that it can be investigated later. It could have addressable # issues that operations can correct for the next run. logger.debug("Writing anomaly set for this block") bf_uid_set = list( nx.get_edge_attributes( block_g_comp, self.edge_uid_field).values()).pop() anomaly_file_name = f"anomaly_block_component.{bf_uid_set}.yaml" nx.write_yaml(block_g_comp, (self.anomaly_folder / anomaly_file_name).as_posix()) # You cannot eulerize a directed graph, so create an undirected one logger.debug("Creating MultiGraph from directed graph.") temp_graph = nx.MultiGraph() for u, v, data in block_g_comp.edges(data=True): key = temp_graph.add_edge(u, v, **data) # logger.debug("Adding edge (%s, %s, %s) to temporary graph.", u, v, key) logger.debug("Created temporary MultiGraph with %s edges", temp_graph.number_of_edges()) # Convert the temporary graph to a proper Euler circuit so that it can be traversed. logger.debug("Eulerizing MultiGraph") euler_block = nx.eulerize(temp_graph) logger.debug("Added %s edges to the block", (euler_block.size() - temp_graph.size())) logger.debug("Number of vertices in eulerized graph: %s", euler_block.number_of_nodes()) # As we try to traverse the undirected graph, we need to keep track of places already visited to make # sure arcs are not skipped. visited_edges = Counter() # augmented edges will throw the node weights off, so don't bother trying the preferred start node logger.debug("Generating path through augmented block") for u, v, k in nx.eulerian_circuit(euler_block, preferred_sp, keys=True): # augmented edges have no attributes, so look for one and skip the edge if nothing is returned if drop_augmented and not euler_block.edges[u, v, k].get( self.edge_uid_field): logger.debug("Ignoring augmented edge (%s, %s, %s)", u, v, k) continue # Increment the sequence value for each edge we see. edge_sequence += 1 # Since we formed an undirected MultiGraph we need to check the orientation of the nodes on the # edge to assign the sequence back to the directed graph. start_node = u end_node = v available_edge_count = block_g.number_of_edges( start_node, end_node) # If no edges exist, invert the nodes and check again. # This also checks to see if we've already encountered all the edges between these nodes, indicating # we need to process the inverse of the start and end values if available_edge_count == 0 or ( ((start_node, end_node) in visited_edges) and (available_edge_count == visited_edges[(start_node, end_node)])): logger.debug( "Nothing to process between (%s, %s), inverting nodes.", start_node, end_node) start_node = v end_node = u available_edge_count = block_g.number_of_edges( start_node, end_node) logger.debug( "Number of edges available between (%s, %s): %s", start_node, end_node, available_edge_count) # Apply the edge_sequence to the first edge that hasn't received one yet for ki in range(available_edge_count): if not block_g.edges[start_node, end_node, ki].get( self.eo_name): logger.debug( "Edge sequence applied: (%s, %s, %s) = %s", start_node, end_node, ki, edge_sequence) block_g.edges[start_node, end_node, ki][self.eo_name] = edge_sequence visited_edges[(start_node, end_node)] += 1 break # At this point every edge should be accounted for, but in case something somehow slips through the cracks # it needs to be given a sequence label. The label almost certainly won't make much sense in terms of a # logical ordering, but this is just trying ot make sure it is counted. logger.debug("Looking for any missed edges in block component") for u, v, k in block_g_comp.edges: if not block_g.edges[u, v, k].get(self.eo_name): edge_sequence += 1 block_g.edges[u, v, k][self.eo_name] = edge_sequence logger.warning( "Applied out of order sequence to component edge (%s, %s, %s): %s", u, v, k, edge_sequence) # just log the last sequence value to make tracing easier logger.debug("Final edge sequence value for component: %s", edge_sequence) # apply a sequence value to all the edges that were discovered logger.debug("Edge order results: %s", nx.get_edge_attributes(block_g_comp, self.eo_name)) # To help cluster the start nodes, mark which node was used as the start point in this block if seq_val_at_start == 1: self._mark_chosen_start_node(block_g_comp, preferred_sp) logger.debug("Finished processing component") # record that block processing is finished logger.debug("Block processing complete") # nx.set_edge_attributes(block_g, block_sequence_labels, self.eo_name) nx.set_edge_attributes(block_g, path_indicator_edges, path_indicator_name) # check to see if the counts line up if not block_g.number_of_edges() == edge_sequence: logger.debug( "Edge sequence (%s) and edge count (%s) do not match in block", edge_sequence, block_g.number_of_edges()) # help start point clustering by apply a scaling factor to all nodes that were touched logger.debug( "Applying scaling factor to nodes in this block, except start point" ) nodes_in_block = set(block_g.nodes()) nodes_in_block.remove( preferred_sp) # don't scale the preferred start point self._apply_node_scaling_factor(nodes_in_block) logger.debug("Final node data for block: %s", self.graph.subgraph(block_g.nodes).nodes(data=True)) logger.debug("Returning pandas DataFrame from block graph.") return nx.to_pandas_edgelist(block_g, source=self.source_field, target=self.target_field)
def design_aqueduct(self, LEVEL=0): """ Performs automatic water network distribution design. """ # Costante di conversion dall'unità di misura dalla carta a metri CONVERSION = 6373044.737 * math.pi / 180 # clustering labels, cluster_centers = self.cluster(self.graph) # --- ADDUCTION --- adduction = nx.Graph() for node in cluster_centers: adduction.add_node(node) self.complete_graph(adduction) adduction = self.mesh_graph(adduction, weight='dist') self.acqueduct.add_edges_from(adduction.edges()) nx.set_node_attributes(self.acqueduct, -1, "CLUSTER") nx.set_node_attributes(self.acqueduct, False, "Tank") nx.set_edge_attributes(self.acqueduct, 1, "LEVEL") # --- DISTRIBUTION --- # initialize distribution graphs distribution = [nx.Graph() for cluster in cluster_centers] # add nodes to the respective cluster for node in labels: cluster = labels[node] distribution[cluster].add_node(node) # add center to the respective cluster for cluster, center in enumerate(cluster_centers): distribution[cluster].add_node(center) # for each distribution sub-network for dist_graph in distribution: # connect nodes in the graph self.complete_graph(dist_graph) dist_graph = self.mesh_graph(dist_graph, weight='dist') # mark as distribution nx.set_edge_attributes(dist_graph, 2, "LEVEL") # add to acqueduct self.acqueduct.add_edges_from(dist_graph.edges()) # add label info to the graph nx.set_node_attributes(self.acqueduct, labels, 'CLUSTER') # add elevation elevdict = nx.get_node_attributes(self.graph, "mean") nx.set_node_attributes(self.acqueduct, elevdict, "ELEVATION") for center in adduction.nodes: self.acqueduct.nodes[center]["ELEVATION"] = 0 for center in adduction.nodes: min_dist = float("+inf") min_alt = 0 for nbr in self.acqueduct.neighbors(center): if self.distance(center, nbr) < min_dist: min_dist = self.distance(center, nbr) min_alt = self.acqueduct.nodes[nbr]["ELEVATION"] self.acqueduct.nodes[center]["ELEVATION"] = min_alt self.acqueduct.nodes[center]["DEMAND"] = 0 # add tank info to the graph namedict = nx.get_node_attributes(self.graph, "name") tankdict = dict([(node, True if namedict[node] == "tank" else False) for node in namedict]) nx.set_node_attributes(self.acqueduct, tankdict, 'Tank') initial_level = 3 nx.set_node_attributes( self.acqueduct, dict([(node, initial_level) for node in tankdict if tankdict[node]]), "Linit") # add level for dist_graph in distribution: leveldict = nx.get_node_attributes(dist_graph, "LEVEL") nx.set_node_attributes(self.acqueduct, leveldict, "LEVEL") # add node id for ID, node in enumerate(self.acqueduct.nodes.items()): node, datadict = node nx.set_node_attributes(self.acqueduct, {node: ID}, "ID") # add demande building_type = nx.get_node_attributes(self.graph, "building") for ID, node in enumerate(self.acqueduct.nodes.items()): node, datadict = node building2demand = {"appartment": 0.11} default = 0.11 demand = building2demand.get(building_type.get(node, ''), default) nx.set_node_attributes(self.acqueduct, {node: demand}, "DEMAND") # add edge attributes IDdict = nx.get_node_attributes(self.acqueduct, "ID") for ID, edge in enumerate(self.acqueduct.edges.items()): edge, datadict = edge node1 = edge[0] node2 = edge[1] datadict["DC_ID"] = ID datadict["LENGHT"] = self.distance(node1, node2) * CONVERSION datadict["NODE1"] = IDdict[node1] datadict["NODE2"] = IDdict[node2] datadict["DIAMETER"] = 500 if "LEVEL" in self.acqueduct[node1][ node2] else 100 datadict["ROUGHNESS"] = 120 if datadict["DIAMETER"] <= 200 else 130 datadict["MINORLOSS"] = 0 datadict["STATUS"] = 1 datadict["LEVEL"] = 1 if edge in adduction.edges else 2 for key in datadict: nx.set_edge_attributes(self.acqueduct, {edge: datadict[key]}, key) # case we only want the first level network if LEVEL == 1: adduction = self.acqueduct.copy() distribution = [ node for node, _ in self.acqueduct.nodes.items() if not node in cluster_centers ] adduction.remove_nodes_from(distribution) for cluster, center in enumerate(cluster_centers): self.acqueduct.nodes[center]["DEMAND"] = 0 for node in labels: if labels[node] == cluster: adduction.nodes[center][ "DEMAND"] += self.acqueduct.nodes[node]["DEMAND"] self.acqueduct = adduction
for node in G.nodes(): state_init[node]='S' state_init[failnode]='F' nx.set_node_attributes(G, state_init, 'state') #node_weight set, could use random as well weightlist = [1,4,4,3.5,3,2,1.5,2,3,0.5,2.5,1,2,2,3,1,0.5,3.5,2,2,1,2,3,2,2,1,1,3.5] node_weight = {} for i in range(28): node_weight[i] = weightlist[i] nx.set_node_attributes(G, node_weight, 'weight') #edge weight edge_weight = {} for edge in G.edges(): edge_weight[edge] = (G.node[edge[0]]['weight'] + G.node[edge[1]]['weight'])/2 nx.set_edge_attributes(G, edge_weight, 'weight') #initial load initial_load = {} for node in G.nodes(): initial_load[node] = G.degree(node, weight = 'weight')**alpha nx.set_node_attributes(G, initial_load, 'initial') #initial capacity node_capacity = {} for node in G.nodes(): node_capacity[node] = (1+beta)*G.node[node]['initial'] nx.set_node_attributes(G, node_capacity, 'capacity') #compute the node with highest degree, closeness and betweenness in project2 #sorted(nx.degree_centrality(G).items(),key=lambda item:item[1]) #17 #sorted(nx.betweenness_centrality(G,weight = 'weight').items(),key=lambda item:item[1]) #11