def load_existing_networks(filename="existing_networks.shp", budget_value=0):
"""
load existing_networks shp into GeoGraph nodes, edges and assign budget
Args:
filename: existing_networks shapefile
budget_value: default budget value for nodes in existing network
Returns:
GeoGraph of existing networks with budget attribute
"""
geo_net = nio.load_shp(filename)
nx.set_node_attributes(geo_net, 'budget', {n: budget_value
for n in geo_net.nodes()})
# determine edge weight/distance function by whether they're geocentric
# (assuming coordinates are not in 3-space here)
distance = gm.spherical_distance if geo_net.is_geographic else \
gm.euclidean_distance
nx.set_edge_attributes(geo_net, 'weight', {(u, v):
distance(map(geo_net.coords.get, [u, v]))
for u, v in geo_net.edges()})
return geo_net
def CopyDAG(G_train, data_total, pmin, pmax):
edges = nx.edges(G_train)
kern = dict.fromkeys(edges)
M = len(data_total)
kern_temp = {}
for edge in edges:
kern[edge] = np.zeros(M)
for m in range(M):
print('Data item: %d' % m)
data = data_total[m]
for i in range(0, len(data)-pmin+1+1):
for j in range(pmin-1, pmax+1):
if data[i:i+j] in kern_temp:
kern_temp[data[i:i+j]][m] += 1
else:
kern_temp[data[i:i+j]] = np.zeros(M)
kern_temp[data[i:i+j]][m] = 1
for edge in edges:
key = edge[0]+edge[1][-1]
if key in kern_temp:
kern[edge] = kern_temp[key]
G = nx.DiGraph()
G.add_edges_from(edges)
nx.set_edge_attributes(G, 'kern_unnorm', kern)
return G
def from_trial_data(cls, circuit, cycles, pos, resultinfo, trialid, step):
# several parts of this script would require reconsideration to support trials
# where the vertices weren't actually deleted
if resultinfo['defect_mode']['mode'] != 'direct removal':
raise RuntimeError('This script was only written with "remove" mode in mind')
# Find change in currents
before = analysis.trial_edge_currents_at_step(circuit, cycles, resultinfo, trialid, step)
after = analysis.trial_edge_currents_at_step(circuit, cycles, resultinfo, trialid, step+1)
before = edict_remove_redundant_entries(before)
delta = {e: after.get(e, 0.0) - before[e] for e in before}
# NOTE: this reconstruction will lack some isolated vertices
defects = set(get_added_defects(resultinfo, trialid, step))
defect_dict = {v:False for v in edict_vertices(delta)}
defect_dict.update({v:True for v in defects})
sources = {(s,t):s for (s,t) in delta}
g = nx.Graph()
g.add_edges_from(delta)
g.add_nodes_from(defects) # in case any of the defects are isolated vertices
# ``pos`` contains all vertices from the graph's initial state. Limit it to those currently
# in the modified graph (because set_node_attributes crashes on nodes that don't exist)
pos = {k:v for k,v in pos.items() if k in g}
nx.set_edge_attributes(g, EATTR_CHANGE, delta)
nx.set_edge_attributes(g, EATTR_SOURCE, sources)
nx.set_node_attributes(g, VATTR_POS, pos)
nx.set_node_attributes(g, VATTR_DEFECT, defect_dict)
return cls(g)
def _normlise_DAG(self, G):
"""
[TO BE TESTED]
Normalise Fisher features on a DAG (l2 normalisation).
Parameters
-------
:param G: DAG of Fisher features.
Attribute 'kern_': edge attribute (dictionary), key: edges, value: normalised Fisher features
Normalised Fisher features.
:return: G, edge attribute 'kern_' assigned.
"""
kern_ = nx.get_edge_attributes(G, 'kern_unnorm_')
kern_ = OrderedDict(sorted(kern_.items(), key=lambda t: t[0]))
val_ = np.asarray(kern_.values(), dtype=float)
key_ = kern_.keys()
if len(val_.shape) == 2: # normalise if there are more than one data samples assigned to the DAG
kern_ = normalize(val_ * self.log_proba_[:, None], norm='l2', axis=0)
else: # normalise if there is only one data sample assigned to the DAG
kern_ = (val_ * self.log_proba_)/np.linalg.norm(val_ * self.log_proba_)
kern_ = dict(zip(key_, kern_))
nx.set_edge_attributes(G, 'kern_', kern_)
return G
def normDAGl2Test(G_test, power):
kern = nx.get_edge_attributes(G_test, 'kern_unnorm')
tran = nx.get_edge_attributes(G_test, 'tran')
kern = OrderedDict(sorted(kern.items(), key=lambda t: t[0]))
val = kern.values()
key = kern.keys()
tran = OrderedDict(sorted(tran.items(), key=lambda t: t[0]))
tran = tran.values()
val = np.asarray(val, dtype=float)
tran = np.asarray(tran, dtype=float)
tran = np.log(1/tran) # logarithm weighting
tran[tran == np.inf] = 0
tran[np.isnan(tran)] = 0
if power == 2:
tran = np.square(tran)
if len(val.shape) == 2:
# kern = val/tran[:, None]
kern = val*tran[:, None] # avoid numeric problems when using logarithm weighting
kern = normalize(kern, norm='l2', axis=0)
else:
kern = val*tran
kern = kern/np.linalg.norm(kern)
kern = dict(zip(key, kern))
nx.set_edge_attributes(G_test, 'kern', kern)
return G_test
def test_effective_size_weighted_undirected(self):
G = self.G.copy()
nx.set_edge_attributes(G, 'weight', self.G_weights)
effective_size = nx.effective_size(G, weight='weight')
assert_almost_equal(round(effective_size['G'], 2), 5.47)
assert_almost_equal(round(effective_size['A'], 2), 2.47)
assert_almost_equal(round(effective_size['C'], 2), 1)
def df_to_graph(df):
G = nx.from_numpy_matrix(df.values, create_using=nx.DiGraph())
G = nx.relabel_nodes(G, dict(enumerate(df.columns)))
weights = nx.get_edge_attributes(G, 'weight')
invW = dict([(k, 1/float(v)) for (k,v) in weights.items()])
nx.set_edge_attributes(G, 'distance', invW)
return G
def shortest_paths(self, dists):
mx = np.inf * np.ones((self.N, self.N))
nx.set_edge_attributes(self.graph, 'weight', dict(zip(self.graph.edges(), dists)))
d = nx.shortest_path_length(self.graph, weight='weight')
for i, v in d.iteritems():
mx[i, v.keys()] = v.values()
return mx
def _aux_digraph_edge_connectivity(G):
"""Auxiliary digraph for computing flow based edge connectivity
If the input graph is undirected, we replace each edge (u,v) with
two reciprocal arcs (u,v) and (v,u) and then we set the attribute
'capacity' for each arc to 1. If the input graph is directed we simply
add the 'capacity' attribute. Part of algorithm 1 in [1]_ .
References
----------
.. [1] Abdol-Hossein Esfahanian. Connectivity Algorithms. (this is a
chapter, look for the reference of the book).
http://www.cse.msu.edu/~cse835/Papers/Graph_connectivity_revised.pdf
"""
if G.is_directed():
if nx.get_edge_attributes(G, "capacity"):
return G
D = G.copy()
capacity = dict((e, 1) for e in D.edges())
nx.set_edge_attributes(D, "capacity", capacity)
return D
else:
D = G.to_directed()
capacity = dict((e, 1) for e in D.edges())
nx.set_edge_attributes(D, "capacity", capacity)
return D
def create_protein_graph(ingraph):
pnodes = {}
pnodes = defaultdict(lambda: 0, pnodes)
pedges = {}
pedges = defaultdict(lambda: 0, pedges)
outgraph = nx.Graph()
for node in ingraph.nodes_iter():
pnodes[ingraph.node[node]['protein']]+=1
for u,v,d in ingraph.edges(data=True):
key=(ingraph.node[u]['protein'], ingraph.node[v]['protein'])
pedges[key]+=1
for key in pnodes.keys():
outgraph.add_node(key)
edges = combinations(pnodes.keys(), 2)
outgraph.add_nodes_from(pnodes.keys())
outgraph.add_edges_from(edges)
nx.set_node_attributes(outgraph, 'count', pnodes)
nx.set_edge_attributes(outgraph, 'weight', pedges)
return outgraph
def test_constraint_weighted_undirected(self):
G = self.G.copy()
nx.set_edge_attributes(G, 'weight', self.G_weights)
constraint = nx.constraint(G, weight='weight')
assert_almost_equal(round(constraint['G'], 3), 0.299)
assert_almost_equal(round(constraint['A'], 3), 0.795)
assert_almost_equal(round(constraint['C'], 3), 1)
def test_effective_size_weighted_directed(self):
D = self.D.copy()
nx.set_edge_attributes(D, 'weight', self.D_weights)
effective_size = nx.effective_size(D, weight='weight')
assert_almost_equal(round(effective_size[0], 3), 1.567)
assert_almost_equal(round(effective_size[1], 3), 1.083)
assert_almost_equal(round(effective_size[2], 3), 1)
def test_constraint_weighted_directed(self):
D = self.D.copy()
nx.set_edge_attributes(D, 'weight', self.D_weights)
constraint = nx.constraint(D, weight='weight')
assert_almost_equal(round(constraint[0], 3), 0.840)
assert_almost_equal(round(constraint[1], 3), 1.143)
assert_almost_equal(round(constraint[2], 3), 1.378)
def __init__(self):
self.graph = nx.Graph()
self.node2min = dict()
self.edge2ts = dict()
nx.set_edge_attributes(self.graph, "ts", dict())
#nx.set_node_attributes( self.graph )
self.nmin = 0
def set_transition_probabilities(graph, damping, start_node):
# transition probabilities
for_removal = []
for node in graph.nodes_iter():
# sum the weights
weights_sum = 0.0
for neighbor in graph.successors_iter(node):
weights_sum += graph.get_edge_data(node, neighbor)['weight']
# set the transition probability multiplied by (1 - damping)
for neighbor in graph.successors_iter(node):
transition = ((1-damping) * (graph[node][neighbor]['weight'] / weights_sum))
nx.set_edge_attributes(graph, 'transition', {(node, neighbor): transition})
# add the restart
if graph.has_edge(node, start_node):
graph[node][start_node]['transition'] += damping
else:
for_removal.append((node, start_node))
if weights_sum > 0:
graph.add_edge (node, start_node, {'weight': 0, 'transition': damping})
else:
graph.add_edge (node, start_node, {'weight': 0, 'transition': 1.0})
return for_removal
def _build_edge_wkt(self):
r = self.results
# Iterate through the nodes and their parent
for rank, fnode, tnode in zip(r.index, r['Sequence..Upstream.id'], r['Sequence..Vertex.id']):
if fnode is not None:
# Set the edge attributes with those found in sequencing
self.networkplan.network.edge[fnode][tnode]['rank'] = int(rank)
self.networkplan.network.edge[fnode][tnode]['distance'] = float(self.networkplan._distance(fnode, tnode))
self.networkplan.network.edge[fnode][tnode]['id'] = int(tnode)
fnode_coords = self.networkplan.coords[fnode]
tnode_coords = self.networkplan.coords[tnode]
# Build WKT Linestring with from_node and to_node coords
self.networkplan.network.edge[fnode][tnode]['Wkt'] = 'LINESTRING ({x1} {y1}, {x2} {y2})'.format(x1=fnode_coords[0], y1=fnode_coords[1],
x2=tnode_coords[0], y2=tnode_coords[1])
# Filter empty edges
edges = {(k1, k2): attr for k1, v in self.networkplan.network.edge.iteritems() if v!={}
for k2, attr in v.iteritems() if attr!={}}
self.networkplan.network.edge = edges
# Clear all edges from the networkx
self.networkplan.network.remove_edges_from(self.networkplan.network.edges())
# Reset with the filtered edges
self.networkplan.network.add_edges_from(edges.keys())
# Set the atrributes
attrs = set([v for i in edges.values() for v in i.keys()])
for attr in attrs:
nx.set_edge_attributes(self.networkplan.network, attr, pd.DataFrame(edges).ix[attr].to_dict())
def create_graph_df(vtask_paths, graphs_dir_out):
"""
Creates a frame that maps sourcefiles to networkx digraphs in terms of DOT files
:param source_path_list:
:param dest_dir_path:
:param relabel:
:return:
"""
if not isdir(graphs_dir_out):
raise ValueError('Invalid destination directory.')
data = []
graphgen_times = []
print('Writing graph representations of verification tasks to {}'.format(graphs_dir_out), flush=True)
common_prefix = commonprefix(vtask_paths)
for vtask in tqdm(vtask_paths):
short_prefix = dirname(common_prefix)
path = join(graphs_dir_out, vtask[len(short_prefix):][1:])
if not os.path.exists(dirname(path)):
os.makedirs(dirname(path))
ret_path = path + '.pickle'
# DEBUG
if isfile(ret_path):
data.append(ret_path)
continue
start_time = time.time()
graph_path, node_labels_path, edge_types_path, edge_truth_path, node_depths_path \
= _run_cpachecker(abspath(vtask))
nx_digraph = nx.read_graphml(graph_path)
node_labels = _read_node_labeling(node_labels_path)
nx.set_node_attributes(nx_digraph, 'label', node_labels)
edge_types = _read_edge_labeling(edge_types_path)
parsed_edge_types = _parse_edge(edge_types)
nx.set_edge_attributes(nx_digraph, 'type', parsed_edge_types)
edge_truth = _read_edge_labeling(edge_truth_path)
parsed_edge_truth = _parse_edge(edge_truth)
nx.set_edge_attributes(nx_digraph, 'truth', parsed_edge_truth)
node_depths = _read_node_labeling(node_depths_path)
parsed_node_depths = _parse_node_depth(node_depths)
nx.set_node_attributes(nx_digraph, 'depth', parsed_node_depths)
assert not isfile(ret_path)
assert node_labels and parsed_edge_types and parsed_edge_truth and parsed_node_depths
nx.write_gpickle(nx_digraph, ret_path)
data.append(ret_path)
gg_time = time.time() - start_time
graphgen_times.append(gg_time)
return pd.DataFrame({'graph_representation': data}, index=vtask_paths), graphgen_times
def make_hrr_vectors(G, id_vectors, edge_vectors):
max_out_degree = len(edge_vectors)
index_dict = {}
for n in G.nodes_iter():
edges = G.edges(n)
indices = random.sample(range(max_out_degree), len(edges))
index_dict.update(dict(zip(edges, indices)))
nx.set_edge_attributes(G, 'index', index_dict)
hrrvecs = {}
for n in G.nodes_iter():
edges = G.edges(n)
if not edges:
hrr_vec = hrr.HRR(len(id_vectors.values()[0]))
else:
components = [id_vectors[e[1]].convolve(edge_vectors[index_dict[e]])
for e
in edges]
hrrvec = sum(components[1:], components[0])
hrrvec.normalize()
hrrvecs[n] = hrrvec
return hrrvecs
def test_edge_num_attribute(self):
g = nx.karate_club_graph()
attr = {(u, v): {"even": int((u+v) % 10)} for (u, v) in g.edges()}
nx.set_edge_attributes(g, attr)
model = gc.CompositeModel(g)
model.add_status("Susceptible")
model.add_status("Infected")
c = cpm.EdgeNumericalAttribute("even", value=0, op="==", probability=1)
model.add_rule("Susceptible", "Infected", c)
config = mc.Configuration()
config.add_model_parameter('percentage_infected', 0.1)
model.set_initial_status(config)
iterations = model.iteration_bunch(10)
self.assertEqual(len(iterations), 10)
model = gc.CompositeModel(g)
model.add_status("Susceptible")
model.add_status("Infected")
c = cpm.EdgeNumericalAttribute("even", value=[3, 10], op="IN", probability=1)
model.add_rule("Susceptible", "Infected", c)
config = mc.Configuration()
config.add_model_parameter('percentage_infected', 0.1)
model.set_initial_status(config)
iterations = model.iteration_bunch(10)
self.assertEqual(len(iterations), 10)
def add_random_weights(hierarchical_graph):
""" Adds random weights to a graph and normalises to ensure stochasticity.
Args:
hierarchical_graph (networkx.classes.digraph.DiGraph): hierarchical
graph without weights.
Returns:
networkx.classes.digraph.DiGraph: hierarchical graph with random weights.
"""
#give each edge a random weight
edge_weight_dict = {}
for edge in hierarchical_graph.edges():
rand = random.uniform(0, 1)
edge_weight_dict[edge] = rand
nx.set_edge_attributes(hierarchical_graph, edge_weight_dict, 'transition probability')
#give each node a random weight
node_weight_dict = {}
for node in hierarchical_graph.nodes():
rand = random.uniform(0, 1)
node_weight_dict[node] = rand
nx.set_node_attributes(hierarchical_graph, node_weight_dict, 'mean transition time')
#normalise the sum of the weights of the out edges for each node.
for node in hierarchical_graph.nodes():
out_edges = hierarchical_graph.out_edges(node)
sum_out = sum([hierarchical_graph.edges[edge]['transition probability']
for edge in out_edges])
for edge in out_edges:
hierarchical_graph.edges[edge]['transition probability'] = \
hierarchical_graph.edges[edge]['transition probability']/sum_out
def _get_test_graph():
'''test graph with known partition'''
G = nx.Graph()
G.add_nodes_from(range(5))
G.add_edges_from([(0, 1), (1, 2), (1, 3), (2, 4), (3, 4)])
nx.set_edge_attributes(G, 'weight', [1] * G.number_of_edges())
return G
def __init__(self, dim=None, phi=np.pi, periodic=True, phases=None):
if not dim: dim = [4, 4]
dim = copy(dim)
self.dim = copy(dim)
self.nspins = np.prod(dim)
self.G = nx.grid_graph(dim, periodic)
if phases is not None:
self.phases = phases
else:
self.phases = dict()
binary_disorder = True
if binary_disorder:
for edge in self.G.edges():
self.phases[edge] = phi * np.random.random_integers(0, 1)
else:
for edge in self.G.edges():
self.phases[edge] = np.random.uniform(-phi, phi)
nx.set_edge_attributes(self.G, "phase", self.phases)
self.indices = dict()
self.index2node = dict()
nodes = sorted(self.G.nodes())
for i, node in enumerate(nodes):
self.indices[node] = i
self.index2node[i] = node
self.num_edges = self.G.number_of_edges()
self.set_up_neighborlists()
def test_networkx_roundtrip(self):
print('\n---------- NetworkX Data Roundtrip Test Start -----------\n')
g = nx.newman_watts_strogatz_graph(100, 3, 0.5)
nodes = g.nodes()
edges = g.edges()
# Add some attributes
g.graph['name'] = 'original'
g.graph['density'] = nx.density(g)
nx.set_node_attributes(g, 'betweenness', nx.betweenness_centrality(g))
nx.set_node_attributes(g, 'degree', nx.degree(g))
nx.set_node_attributes(g, 'closeness', nx.closeness_centrality(g))
nx.set_edge_attributes(g, 'eb', nx.edge_betweenness(g))
cyjs1 = util.from_networkx(g)
g2 = util.to_networkx(cyjs1)
self.assertEqual(len(g2.nodes()), len(nodes))
self.assertEqual(len(g2.edges()), len(edges))
edge_set = set(list(map(lambda x: (int(x[0]), int(x[1])), g2.edges())))
self.assertEqual(0, len(edge_set.difference(set(edges))))
node_original = g.node[1]
node_generated = g2.node['1']
print(node_original)
print(node_generated)
self.assertEqual(node_original['degree'], node_generated['degree'])
self.assertEqual(node_original['betweenness'], node_generated['betweenness'])
self.assertEqual(node_original['closeness'], node_generated['closeness'])
def test_networkx_roundtrip(self):
print("\n---------- NetworkX Data Roundtrip Test Start -----------\n")
g = nx.newman_watts_strogatz_graph(100, 3, 0.5)
nodes = g.nodes()
edges = g.edges()
# Add some attributes
g.graph["name"] = "original"
g.graph["density"] = nx.density(g)
nx.set_node_attributes(g, "betweenness", nx.betweenness_centrality(g))
nx.set_node_attributes(g, "degree", nx.degree(g))
nx.set_node_attributes(g, "closeness", nx.closeness_centrality(g))
nx.set_edge_attributes(g, "eb", nx.edge_betweenness(g))
cyjs1 = util.from_networkx(g)
g2 = util.to_networkx(cyjs1)
self.assertEqual(len(g2.nodes()), len(nodes))
self.assertEqual(len(g2.edges()), len(edges))
edge_set = set(list(map(lambda x: (int(x[0]), int(x[1])), g2.edges())))
self.assertEqual(0, len(edge_set.difference(set(edges))))
node_original = g.node[1]
node_generated = g2.node["1"]
print(node_original)
print(node_generated)
self.assertEqual(node_original["degree"], node_generated["degree"])
self.assertEqual(node_original["betweenness"], node_generated["betweenness"])
self.assertEqual(node_original["closeness"], node_generated["closeness"])
def write_edge2cid(e2c,filename, outFile, delimiter="\t"):
# write edge2cid three-column file
f = open(filename+".edge2comm.txt",'w')
c2c = dict( (c,i+1) for i,c in enumerate(sorted(list(set(e2c.values())))) ) # ugly...
#print c2c
for e,c in sorted(e2c.iteritems(), key=itemgetter(1)):
f.write( "%s%s%s%s%s\n" % (str(e[0]),delimiter,str(e[1]),delimiter,str(c2c[c])) )
f.close()
cid2edges,cid2nodes = defaultdict(set),defaultdict(set) # faster to recreate here than
for edge,cid in e2c.iteritems(): # to keep copying all dicts
cid2edges[cid].add( edge ) # during the linkage...
cid2nodes[cid] |= set(edge)
cid2edges,cid2nodes = dict(cid2edges),dict(cid2nodes)
# write list of edges for each comm, each comm on its own line
f,g = open(filename+".comm2edges.txt", 'w'),open(filename+".comm2nodes.txt", 'w')
for cid in sorted(cid2edges.keys()):
nodes,edges = map(str,cid2nodes[cid]), ["%s,%s" % (ni,nj) for ni,nj in cid2edges[cid]]
f.write( "\t".join(edges) ); f.write("\n")
g.write( "\t".join([str(cid)] + nodes) ); g.write("\n")
f.close(); g.close()
#print e2c
e2cRelabel=dict((edge,c2c.get(id)) for (edge, id) in e2c.items())
g=nx.Graph()
g.add_edges_from(e2cRelabel.keys())
nx.set_edge_attributes(g,"label",e2cRelabel)
nx.write_gml(g,outFile)
def planar_cycle_basis_impl(g):
assert not g.is_directed()
assert not g.is_multigraph()
assert nx.is_biconnected(g) # BAND-AID
# "First" nodes/edges for each cycle are chosen in an order such that the first edge
# may never belong to a later cycle.
# rotate to (hopefully) break any ties or near-ties along the y axis
rotate_graph(g, random.random() * 2 * math.pi)
# NOTE: may want to verify that no ties exist after the rotation
nx.set_edge_attributes(g, 'used_once', {e: False for e in g.edges()})
# precompute edge angles
angles = {}
for s,t in g.edges():
angles[s,t] = edge_angle(g,s,t) % (2*math.pi)
angles[t,s] = (angles[s,t] + math.pi) % (2*math.pi)
# identify and clear away edges which cannot belong to cycles
for v in g:
if degree(g,v) == 1:
remove_filament_from_tip(g, v)
cycles = []
# sort ascendingly by y
for root in sorted(g, key = lambda v: g.node[v]['y']):
# Check edges in ccw order from the +x axis
for target in sorted(g[root], key=lambda t: angles[root,t]):
if not g.has_edge(root, target):
continue
discriminator, path = trace_ccw_path(g, root, target, angles)
if discriminator == PATH_PLANAR_CYCLE:
assert path[0] == path[-1]
remove_cycle_edges(g, path)
cycles.append(path)
# Both the dead end and the initial edge belong to filaments
elif discriminator == PATH_DEAD_END:
remove_filament_from_edge(g, root, target)
remove_filament_from_tip(g, path[-1])
# The initial edge must be part of a filament
# FIXME: Not necessarily true if graph is not biconnected
elif discriminator == PATH_OTHER_CYCLE:
remove_filament_from_edge(g, root, target)
else: assert False # complete switch
assert not g.has_edge(root, target)
assert len(g.edges()) == 0
return cycles
def extract_edges(self, edge_attr): """docstring for extract""" exEdges = nx.get_edge_attributes(self, edge_attr) sg = nx.Graph(data = exEdges.keys(), name = edge_attr) if len(exEdges): nx.set_edge_attributes(sg, edge_attr, exEdges) return(sg)
def delete_links(G): ''' Delete links in the graph if they disavantage the person. The link is deleted if (r1-r2)/affect(n1, n2)*deg(n1) > alpha where r1 > r2 and alpha is a random number between 0 and a given paramater beta. ''' beta = 100 reput = nx.get_node_attributes(G, "reput") affect = nx.get_edge_attributes(G, "affect") n = 0 for edge in nx.edges(G): # Calculate alpha alpha = beta*random.random() # Define who has the higher reputation n1 = edge[1] n2 = edge[0] if(reput[edge[0]] > reput[edge[1]]): n1 = edge[0] n2 = edge[1] # Compute the coefficient coef = (reput[n1]-reput[n2])/affect[edge]*G.degree(n1) # Decide wether we delete the link if(coef > alpha): G.remove_edge(edge[0], edge[1]) del affect[edge] # Set the new affect dict and return the graph nx.set_edge_attributes(G, "affect", affect) return G
def nx_from_matrix(weight_matrix, nodes=None, remove_self=True):
import networkx as nx
import utool as ut
import numpy as np
if nodes is None:
nodes = list(range(len(weight_matrix)))
weight_list = weight_matrix.ravel()
flat_idxs_ = np.arange(weight_matrix.size)
multi_idxs_ = np.unravel_index(flat_idxs_, weight_matrix.shape)
# Remove 0 weight edges
flags = np.logical_not(np.isclose(weight_list, 0))
weight_list = ut.compress(weight_list, flags)
multi_idxs = ut.compress(list(zip(*multi_idxs_)), flags)
edge_list = ut.lmap(tuple, ut.unflat_take(nodes, multi_idxs))
if remove_self:
flags = [e1 != e2 for e1, e2 in edge_list]
edge_list = ut.compress(edge_list, flags)
weight_list = ut.compress(weight_list, flags)
graph = nx.Graph()
graph.add_nodes_from(nodes)
graph.add_edges_from(edge_list)
label_list = ['%.2f' % w for w in weight_list]
nx.set_edge_attributes(graph, 'weight', dict(zip(edge_list,
weight_list)))
nx.set_edge_attributes(graph, 'label', dict(zip(edge_list,
label_list)))
return graph
def color_by_nids(graph, unique_nids=None, ibs=None, nid2_color_=None):
""" Colors edges and nodes by nid """
# TODO use ut.color_nodes
import plottool as pt
ensure_graph_nid_labels(graph, unique_nids, ibs=ibs)
node_to_nid = nx.get_node_attributes(graph, 'nid')
unique_nids = ut.unique(node_to_nid.values())
ncolors = len(unique_nids)
if (ncolors) == 1:
unique_colors = [pt.UNKNOWN_PURP]
else:
if nid2_color_ is not None:
unique_colors = pt.distinct_colors(ncolors + len(nid2_color_) * 2)
else:
unique_colors = pt.distinct_colors(ncolors)
# Find edges and aids strictly between two nids
nid_to_color = dict(zip(unique_nids, unique_colors))
if nid2_color_ is not None:
# HACK NEED TO ENSURE COLORS ARE NOT REUSED
nid_to_color.update(nid2_color_)
edge_aids = list(graph.edges())
edge_nids = ut.unflat_take(node_to_nid, edge_aids)
flags = [nids[0] == nids[1] for nids in edge_nids]
flagged_edge_aids = ut.compress(edge_aids, flags)
flagged_edge_nids = ut.compress(edge_nids, flags)
flagged_edge_colors = [nid_to_color[nids[0]] for nids in flagged_edge_nids]
edge_to_color = dict(zip(flagged_edge_aids, flagged_edge_colors))
node_to_color = ut.map_dict_vals(ut.partial(ut.take, nid_to_color), node_to_nid)
nx.set_edge_attributes(graph, 'color', edge_to_color)
nx.set_node_attributes(graph, 'color', node_to_color)
def get_firewall_rules(graph, generator_string, weights, seed=0,
olsrv1=False, display=False, print_csv=False,
single_interface=False):
"""
Given a networkx graph representing the real topology of a testbed, returns
the firewall rules to be applied to obtain a synthetic topology with the
generated using a given generator. In addition, the function returns a
score between 0 and 1 which represents how close is the new topology to the
synthetic topology. A score of 1 means that the testbed topology
can be transformed to completely match the synthetic graph. A score of 0
indicates that no link can be matched
:param graph: networkx graph
:param generator_string: a networkx function name (e.g.,
"random_regular_graph") with the list of parameters. The generator
function must accept the parameter n as the number of nodes, plus
the additional parameters, and generate a networkx graph. The format of
the string is like
"generator_function_name:p1=v1,p2=v2,..."
where p1 and p2 are the argument names and v1 and v2 their respective
values. The arguments should not include the parameter n, which is
automatically obtained from the given graph
:param weights: if True, it means that the experiment should generate
per-link random weights, and this must be taken into account when
computing the poprow timers
:param seed: seed used for random generators
:param olsrv1: if set to true, generate olsrv1-compatible link costs
:param display: if set to True, shows a plot with the overlapping desired
topology and the experiment topology, for a graphical comparison of the two
:param print_csv: if true, prints a csv table with node id, times,
betweennes centrality and degree, to be used for plotting and analysis
:param single_interface: if true, the degree d_i is forced to 1 to
represent a single interface node
:return: a pair where the first element is the firewall configuration of
links to be disabled, while the second is the matching score
"""
# remove "id_" from node names
testbed_g = graph
old_names = testbed_g.nodes()
new_names = dict((i, i.replace('id_', '')) for i in old_names)
nx.relabel_nodes(testbed_g, new_names, copy=False)
n_nodes = len(testbed_g.nodes())
testbed_am = nx.adj_matrix(testbed_g)
# create a map from node id to its original position
mp = dict((testbed_g.nodes()[i], i) for i in range(n_nodes))
# sort the nodes by degree
testbed_d = testbed_g.degree()
sd = sorted(testbed_d.items(), key=lambda x: x[1], reverse=True)
testbed_sd = map(lambda x: x[0], sd)
# create the sorted map. testbed_sd is a vector of node names, and we
# need a vector of indexes
testbed_sm = [mp[testbed_sd[i]] for i in range(len(testbed_sd))]
testbed_matrix = __get_matrix(testbed_am, testbed_sm)
if generator_string.find(":") != -1:
generator, parameters = generator_string.split(":")
gen_args = dict()
for p in parameters.split(","):
key, value = p.split("=")
try:
v = int(value)
except:
v = value
gen_args[key] = v
else:
generator = generator_string
gen_args = dict()
# create a synthetic graph by calling the given generator
# search for the generator inside networkx. if not found, use one of our
# custom generators
try:
gen_function = getattr(nx, generator)
except AttributeError:
gen_function = getattr(ct, generator)
synt_g = gen_function(n=n_nodes, **gen_args)
synt_am = nx.adj_matrix(synt_g)
synt_d = synt_g.degree()
# sort by degree as for the real graph
sd = sorted(synt_d.items(), key=lambda x: x[1], reverse=True)
synt_sm = map(lambda x: x[0], sd)
synt_matrix = __get_matrix(synt_am, synt_sm)
best_score = -1
best_matrix = None
best_mapping = None
n_attempts = 10
random.seed(0)
# if the matching using the heuristic didn't work, simply try n times
# with random shuffling
while best_score < 1 and n_attempts > 0:
n_attempts -= 1
# transform the testbed topology in the synthetic topology by performing
# a logical and
experiment_matrix = __matrix_and(testbed_matrix, synt_matrix)
# count the number of non matching entries. 1 means perfect match,
# 0 means no link could be matched
score = __matching_score(experiment_matrix, synt_matrix)
if score > best_score:
best_score = score
best_matrix = experiment_matrix
best_mapping = testbed_sm
testbed_sm = range(len(testbed_g.nodes()))
random.shuffle(testbed_sm)
testbed_matrix = __get_matrix(testbed_am, testbed_sm)
if display:
# do this heretic thing to avoid installing matplotlib on testbed
# nodes if not really required
import matplotlib.pyplot as plt
experiment_g = nx.from_scipy_sparse_matrix(best_matrix)
nx.draw_circular(synt_g)
nx.draw_circular(experiment_g, width=3, edge_color='r')
plt.show()
# given the experiment matrix and the nodes of the testbed, get the links
# that must be disabled to obtain the desired topology
disabled_links = __get_links(best_matrix, best_mapping, testbed_g.nodes())
# transform this into a string usable by the scripts
rules = __stringify_disabled_links(disabled_links)
# check whether the edges in the synthetic topology have a weight
u, v = synt_g.edges()[0]
has_costs = False
if "weight" in synt_g.get_edge_data(u, v).keys():
has_costs = True
link_costs = __add_link_costs(__get_links(best_matrix, best_mapping,
testbed_g.nodes(),
False), seed, olsrv1=olsrv1,
has_costs=has_costs)
costs = __stringify_link_costs(link_costs, olsrv1=olsrv1)
graph = __get_graph(best_matrix, best_mapping, testbed_g.nodes())
if weights:
for s, csts in link_costs.iteritems():
for (d, cost) in csts:
nx.set_edge_attributes(graph, "weight", {(s, d): cost})
intervals = __stringify_intervals(__compute_intervals(graph,
use_degree=not
single_interface))
if print_csv:
intv = __compute_intervals(graph, return_bc=True)
print("node,h,tc,hnd,tcnd,bc,d")
for n, (h, tc, h2, tc2, bc, d) in intv.iteritems():
print("%s,%f,%f,%f,%f,%f,%d" % (n, h, tc, h2, tc2, bc, d))
# we return the configuration string and the measure of the match
return rules, costs, intervals, best_score
names = nx.get_node_attributes(graph, 'name')
dicty1 = {
key.encode('utf-8'): value.encode('utf-8')
for key, value in names.iteritems()
}
graph = nx.relabel_nodes(graph, dicty1, copy=True)
signs = nx.get_edge_attributes(graph, 'sign')
activities = {}
for e in graph.edges_iter():
if signs[e] == '+':
activities[e] = 'a'
elif signs[e] == '-':
activities[e] = 'i'
else:
print('not found: ' + signs[e])
nx.set_edge_attributes(graph, 'signal', activities)
# read in C library
updateBooler = cdll.LoadLibrary('./testRun.so')
boolC = updateBooler.syncBool
sampleList, geneDict, cvDict = readFpkmData(dataName, results.sep)
genes = Set(geneDict.keys())
newOverlap = genes.intersection(set(graph.nodes()))
# graph, addLaterNodes=PA.simplifyNetworkpathwayAnalysis(graph, geneDict)
print(len(graph.nodes()))
sampleList = [{} for q in range(len(geneDict[list(newOverlap)[0]]))]
for noder in newOverlap:
for jn in range(len(sampleList)):
sampleList[jn][noder] = geneDict[noder][jn]
print(sampleList)
def preprocessing():
'''removing isolated nodes'''
isolated=nx.isolates(G)
for j in isolated:
for key in nodes_data.keys():
if j == key:
del nodes_data[key]
G.remove_node(j)
#print G.number_of_nodes(), len(nodes_data)
continue
for k in edges_data.keys():
'''deleting all edges from edges_data which no more exist'''
if not G.has_edge(edges_data[k][1],edges_data[k][2],key=k):
#print len(edges_data)
del edges_data[k]
gen=[]
indegree=G.in_degree(G.nodes())
'''number of nodes with no in_degree'''
'''removing joints,merge nodes categorised as generators from graph and node data'''
t=0
for i in indegree:
if indegree[i]==0:
if nodes_data[i][3]=='merge' or nodes_data[i][3]=='joint':
continue
gen.append(i)
for node,nd in nodes_data.iteritems():
'''adding plants to generators'''
if nd[3]== 'plant' and nd[0] not in gen:
gen.append(node)
voltages={}
z=0
for aa,val in edges_data.iteritems():
'''adding edge attributes'''
z+=1
#print z
if edges_data[aa][3].find(';')!= -1:
xx=map(float,edges_data[aa][3].split(';'))
edges_data[aa][3]=max(xx)
del xx[:]
elif edges_data[aa][3]=='':
edges_data[aa][3]=np.nan
else:
edges_data[aa][3]=float(edges_data[aa][3])
if val[4].find(';')!= -1:
xy=map(float,val[4].split(';'))
val[4]=sum(xy)
del xy[:]
else:
val[4]=float(val[4])
voltages[(val[1],val[2],val[0])]=val[3]
nx.set_edge_attributes(G,'voltages',voltages)
distr=distributers();
for f in gen:
if f in distr:
distr.remove(f)
#G.remove_edges_from(G.in_edges(f))
if node in gen:
nodes_data[node][3]='generators'
elif node in distr:
nodes_data[node][3]='distributors'
else:
nodes_data[node][3]='transmitters'
for ab,v in nodes_data.iteritems():
'''adding nodes attributes'''
if v[4].find(';')!= -1:
ss=map(float,v[4].split(';'))
if v[3]=='generator' or v[3]=='distributor':
v[4]=min(ss)
else:
v[4]=max(ss)
del ss[:]
elif v[4]=='':
v[4]=np.nan
else:
v[4]=float(v[4])
'''removing multi edges'''
for x in G.nodes():
for y in G.nodes():
if x==y:
continue
if G.number_of_edges(x,y)>1:
kkk=[]
for ee in list(G.edges(x,keys=True)):
if ee[0]==x and ee[1]==y:
kkk.append(ee[2])
for l in range(1,len(kkk)):
edges_data[kkk[0]][4]+=edges_data[kkk[l]][4]
edges_data[kkk[0]][3]=max(edges_data[kkk[0]][3],edges_data[kkk[l]][3])
del edges_data[kkk[l]]
G.remove_edge(x,y,key=kkk[l])
G[x][y][kkk[0]]['cable']=edges_data[kkk[0]][4]
G[x][y][kkk[0]]['voltages']=edges_data[kkk[0]][3]
#return (G,nodes_data,edges_data)
o=[]
with open('../../NS_project/Data/Vertices_new.csv', 'wb') as csvfile:
nodewriter = csv.writer(csvfile, delimiter=',')
header=['v_id','lon','lat','typ','voltage']
nodewriter.writerow(header)
for da,it in nodes_data.iteritems():
for l in range(0,5):
o.append(str(it[l]))
nodewriter.writerow(o)
del o[:]
with open('../../NS_project/Data/Edges_new.csv', 'wb') as csvfile1:
edgewriter = csv.writer(csvfile1, delimiter=',')
header1=['l_id','v_id_1','v_id_2','voltage','cables','length_m']
edgewriter.writerow(header1)
for db,itt in edges_data.iteritems():
for l in range(0,5):
o.append(str(itt[l]))
o.append(str(itt[10]))
edgewriter.writerow(o)
del o[:]
return G
G = nx.MultiDiGraph()
G.add_edges_from((0, i) for i in range(1, n))
# 0 is the central node
# for i in range(n):
# print(nx.degree(G, i))
# Set edge attributes to random values representing contractual exposure
attrs = {}
for i in range(n):
edge = (0, i, 0)
attrs[edge] = {'exposure': np.random.randn()}
# print(attrs)
nx.set_edge_attributes(G, attrs)
#
# Set node attributes to random values representing credit risk
#
attrs = {}
for i in range(n):
attrs[i] = {'rating': np.random.randn()}
print(attrs)
nx.set_node_attributes(G, attrs)
dataset_path = cm.source_path + "/datasets/"
nx.write_yaml(G, dataset_path + "star.yml")
# Load a graph
G1 = nx.read_yaml(dataset_path + "star.yml")
nx.draw(G1)
def add_edge_speeds(G, hwy_speeds=None, fallback=None, precision=1):
"""
Add edge speeds (km per hour) to graph as new `speed_kph` edge attributes.
Imputes free-flow travel speeds for all edges based on mean `maxspeed`
value of edges, per highway type. For highway types in graph that have no
`maxspeed` value on any edge, function assigns the mean of all `maxspeed`
values in graph.
This mean-imputation can obviously be imprecise, and the caller can
override it by passing in `hwy_speeds` and/or `fallback` arguments that
correspond to local speed limit standards.
If edge `maxspeed` attribute has "mph" in it, value will automatically be
converted from miles per hour to km per hour. Any other speed units should
be manually converted to km per hour prior to running this function,
otherwise there could be unexpected results. If "mph" does not appear in
the edge's maxspeed attribute string, then function assumes kph, per OSM
guidelines: https://wiki.openstreetmap.org/wiki/Map_Features/Units
Parameters
----------
G : networkx.MultiDiGraph
input graph
hwy_speeds : dict
dict keys = OSM highway types and values = typical speeds (km per
hour) to assign to edges of that highway type for any edges missing
speed data. Any edges with highway type not in `hwy_speeds` will be
assigned the mean preexisting speed value of all edges of that highway
type.
fallback : numeric
default speed value (km per hour) to assign to edges whose highway
type did not appear in `hwy_speeds` and had no preexisting speed
values on any edge
precision : int
decimal precision to round speed_kph
Returns
-------
G : networkx.MultiDiGraph
graph with speed_kph attributes on all edges
"""
if fallback is None:
fallback = np.nan
edges = utils_graph.graph_to_gdfs(G, nodes=False, fill_edge_geometry=False)
# collapse any highway lists (can happen during graph simplification)
# into string values simply by keeping just the first element of the list
edges["highway"] = edges["highway"].map(lambda x: x[0]
if isinstance(x, list) else x)
if "maxspeed" in edges.columns:
# collapse any maxspeed lists (can happen during graph simplification)
# into a single value
edges["maxspeed"] = edges["maxspeed"].map(
_collapse_multiple_maxspeed_values)
# create speed_kph by cleaning maxspeed strings and converting mph to
# kph if necessary
edges["speed_kph"] = edges["maxspeed"].astype(str).map(
_clean_maxspeed).astype(float)
else:
# if no edges in graph had a maxspeed attribute
edges["speed_kph"] = None
# if user provided hwy_speeds, use them as default values, otherwise
# initialize an empty series to populate with values
if hwy_speeds is None:
hwy_speed_avg = pd.Series(dtype=float)
else:
hwy_speed_avg = pd.Series(hwy_speeds).dropna()
# for each highway type that caller did not provide in hwy_speeds, impute
# speed of type by taking the mean of the preexisting speed values of that
# highway type
for hwy, group in edges.groupby("highway"):
if hwy not in hwy_speed_avg:
hwy_speed_avg.loc[hwy] = group["speed_kph"].mean()
# if any highway types had no preexisting speed values, impute their speed
# with fallback value provided by caller. if fallback=np.nan, impute speed
# as the mean speed of all highway types that did have preexisting values
hwy_speed_avg = hwy_speed_avg.fillna(fallback).fillna(hwy_speed_avg.mean())
# for each edge missing speed data, assign it the imputed value for its
# highway type
speed_kph = (edges[["highway", "speed_kph"
]].set_index("highway").iloc[:,
0].fillna(hwy_speed_avg))
# all speeds will be null if edges had no preexisting maxspeed data and
# caller did not pass in hwy_speeds or fallback arguments
if pd.isnull(speed_kph).all():
raise ValueError(("this graph's edges have no preexisting `maxspeed` "
"attribute values so you must pass `hwy_speeds` or "
"`fallback` arguments."))
# add speed kph attribute to graph edges
edges["speed_kph"] = speed_kph.round(precision).values
nx.set_edge_attributes(G, values=edges["speed_kph"], name="speed_kph")
return G
def _validate_all_edges(self):
diams = {edge: 1 for edge in self._layout.edges}
nx.set_edge_attributes(self._layout, diams, name='diam')
def generate_graph(self, node_count):
eps = 0.02
p = ((1 + eps) * np.log(node_count)) / node_count
graph = nx.generators.gnp_random_graph(node_count, p, directed=True)
nx.set_edge_attributes(graph, 1, "feature")
return graph
def calcul_sim_chaines_2_4():
with open("graphe_complet_pondere_sim.pickle",
'rb') as fichier_graphe_complet:
mon_depickler_complet = pickle.Unpickler(fichier_graphe_complet)
graphe_complet = mon_depickler_complet.load()
print(graphe_complet.edges.data())
nx.set_node_attributes(graphe_complet, [], "chaines")
for u in graphe_complet.nodes():
graphe_complet.nodes[u]["chaines"] = recherche_chaines(
graphe_complet, u)
print(graphe_complet.nodes.data())
nx.set_edge_attributes(graphe_complet, -1.0, "sim_chaines_2_4")
for u, v, data in graphe_complet.edges(data=True):
chaine_1 = list(
set(graphe_complet.nodes[u]["chaines"][1] +
graphe_complet.nodes[u]["chaines"][3] + [1, 2, 3, 4, 5]))
chaine_2 = list(
set(graphe_complet.nodes[v]["chaines"][1] +
graphe_complet.nodes[v]["chaines"][3] + [1, 2, 3, 4, 5]))
with open(
"graphes_extension/fichier_" +
graphe_complet.nodes[u]["nom"] + ".pickle",
'rb') as fichier_graphe1:
mon_depickler_graphe1 = pickle.Unpickler(fichier_graphe1)
graphe1 = mon_depickler_graphe1.load()
with open(
"graphes_extension/fichier_" +
graphe_complet.nodes[v]["nom"] + ".pickle",
'rb') as fichier_graphe2:
mon_depickler_graphe2 = pickle.Unpickler(fichier_graphe2)
graphe2 = mon_depickler_graphe2.load()
with open("dico_graphe_epure_en_tout.pickle",
'rb') as fichier_commun:
mon_depickler_commun = pickle.Unpickler(fichier_commun)
dico_graphe = mon_depickler_commun.load()
if ("fichier_" + graphe_complet.nodes[u]["nom"],
"fichier_" + graphe_complet.nodes[v]["nom"]
) in dico_graphe.keys():
graphe_commun = dico_graphe[(
"fichier_" + graphe_complet.nodes[u]["nom"],
"fichier_" + graphe_complet.nodes[v]["nom"])]
chaine_commun = []
for noeud in graphe_commun.nodes():
if noeud[0] in chaine_1 and noeud[1] in chaine_2:
chaine_commun.append(noeud)
chaine_commun = list(
set(chaine_commun +
[(1, 1), (2, 2), (3, 3), (4, 4), (5, 5)]))
else:
graphe_commun = dico_graphe[(
"fichier_" + graphe_complet.nodes[v]["nom"],
"fichier_" + graphe_complet.nodes[u]["nom"])]
chaine_commun = []
for noeud in graphe_commun.nodes():
if noeud[1] in chaine_1 and noeud[0] in chaine_2:
chaine_commun.append(noeud)
chaine_commun = list(
set(chaine_commun +
[(1, 1), (2, 2), (3, 3), (4, 4), (5, 5)]))
sim_2_4 = calcul_sim_non_cov_sans_motif(
graphe1.subgraph(chaine_1), graphe2.subgraph(chaine_2),
graphe_commun.subgraph(chaine_commun))
print(sim_2_4)
graphe_complet.edges[u, v]["sim_chaines_2_4"] = sim_2_4
with open("graphe_complet_pondere_sim.pickle",
'wb') as fichier_graphe_complet_2:
mon_pickler_complet = pickle.Pickler(fichier_graphe_complet_2)
mon_pickler_complet.dump(graphe_complet)
def do_GET(self):
try:
print("path: " + str(self.path))
query = urlsplit(self.path).query
print("query: " + str(query))
params = parse_qs(query)
print("params: " + str(params))
origin = params["origin"][0].strip().split(",")
dest = params["destination"][0].strip().split(",")
origin_point = (float(origin[1]), float(origin[0]))
origin = osmnx.get_nearest_node(G_raw, origin_point)
dest_point = (float(dest[1]), float(dest[0]))
destination = osmnx.get_nearest_node(G_raw, dest_point)
alpha = float(params["alpha"][0])
networkx.set_edge_attributes(G, numpy.inf, "cost")
for edge in G.edges:
#print(begin,end)
G.edges[edge]["cost"] = (1 - alpha) * G.edges[edge][
"time"] + alpha * G.edges[edge]["accidents"]
cheapest_path = networkx.shortest_path(G,
source=origin,
target=destination,
weight="cost")
quickest_path = networkx.shortest_path(G,
source=origin,
target=destination,
weight="time")
# get_path_and_center(G, origin,destination,alpha,"cost")
#
# print("cheapest path: "+str(cheapest_path))
# print("quickest path: "+str(quickest_path))
r = networkx.shortest_path(G, origin, destination, "cost")
latlng_list = []
latlng_center = []
latitude_total = 0
longitude_total = 0
for i in r:
temp_list = []
temp_list.append(G_raw.node[i]['x'])
temp_list.append(G_raw.node[i]['y'])
latlng_list.append(temp_list)
for i in latlng_list:
latitude_total = latitude_total + i[0]
longitude_total = longitude_total + i[1]
avg_lat = latitude_total / len(latlng_list)
avg_lng = longitude_total / len(latlng_list)
latlng_center.append(avg_lat)
latlng_center.append(avg_lng)
data = {
'coordinates': latlng_list,
'center': latlng_center,
'alpha': alpha
}
#query = urlsplit(url).query
#params = parse_qs(query)
#print("parsed path: "+str(urldata))
# Send response status code
self.send_response(200)
outData = {}
outData["coordinates"] = latlng_list
outData["center"] = latlng_center
outData["alpha"] = alpha
# outData["cheapest_path"]=quickest_path
# outData={}
# outData["origin"]=origin
# outData["destination"]=destination
# outData["quickest_path"]=cheapest_path
# outData["cheapest_path"]=quickest_path
print("outData: " + str(outData))
# Send headers
self.send_header('Content-type', 'text/html')
self.end_headers()
# Send message back to client
message = json.dumps(outData)
print("message: " + str(message))
#self.wfile.write("<html><body><h1>hi!</h1></body></html>")
# Write content as utf-8 data
self.wfile.write(bytes(message, "utf8"))
except Exception as e:
print("exception: " + str(e))
# self.wfile.write(bytes(message, "utf8"))
if __name__ == '__main__':
global G
windower = config.windower() #config should default to entire dataset
osmnx.config(log_file=True, log_console=True, use_cache=True)
G_raw = osmnx.graph_from_place('Manhattan Island, New York, USA',
network_type='drive')
G = networkx.DiGraph(G_raw.copy())
print("INITIALIZING ACCIDENT DATA")
accidents_raw = config.accidents(windower.timewindower)
accidents = accidents_raw.copy()
accidents.set_index("node", drop=True, inplace=True)
accidents = accidents.groupby(level="node").size()
networkx.set_edge_attributes(G, 0, "accidents")
#set default value of zero accidents
for node, count in accidents.iteritems():
try:
for edge in G.in_edges(node):
G.edges[edge]["accidents"] = count
except Exception: #fails if node is not in graph
pass
print("INITIALIZING TRAVELTIME DATA")
traveltimes_raw = config.traveltimes(windower.timewindower)
traveltimes = traveltimes_raw.copy()
traveltimes.set_index(["begin_node", "end_node"], drop=True, inplace=True)
traveltimes_means = traveltimes["travel_time"].groupby(
level=["begin_node", "end_node"]).mean()
#set default value of infinite time.
def _sample_paths(self, input_rule_set, obs_name, target_polarity,
max_paths=1, max_path_length=5):
if max_paths == 0:
raise ValueError("max_paths cannot be 0 for path sampling.")
# Convert path polarity representation from 0/1 to 1/-1
def convert_polarities(path_list):
return [tuple((n[0], 0 if n[1] > 0 else 1)
for n in path)
for path in path_list]
pg_polarity = 0 if target_polarity > 0 else 1
nx_graph = _im_to_signed_digraph(self.get_im())
# Add edges from dummy node to input rules
source_node = 'SOURCE_NODE'
for rule in input_rule_set:
nx_graph.add_edge(source_node, rule, attr_dict={'sign': 0})
# -------------------------------------------------
# Create combined paths_graph
f_level, b_level = pg.get_reachable_sets(nx_graph, source_node,
obs_name, max_path_length,
signed=True)
pg_list = []
for path_length in range(1, max_path_length+1):
cfpg = pg.CFPG.from_graph(
nx_graph, source_node, obs_name, path_length, f_level,
b_level, signed=True, target_polarity=pg_polarity)
pg_list.append(cfpg)
combined_pg = pg.CombinedCFPG(pg_list)
# Make sure the combined paths graph is not empty
if not combined_pg.graph:
pr = PathResult(False, 'NO_PATHS_FOUND', max_paths, max_path_length)
pr.path_metrics = None
pr.paths = []
return pr
# Get a dict of rule objects
rule_obj_dict = {}
for ann in self.model.annotations:
if ann.predicate == 'rule_has_object':
rule_obj_dict[ann.subject] = ann.object
# Get monomer initial conditions
ic_dict = {}
for mon in self.model.monomers:
# FIXME: A hack that depends on the _0 convention
ic_name = '%s_0' % mon.name
# TODO: Wrap this in try/except?
ic_param = self.model.parameters[ic_name]
ic_value = ic_param.value
ic_dict[mon.name] = ic_value
# Set weights in PG based on model initial conditions
for cur_node in combined_pg.graph.nodes():
edge_weights = {}
rule_obj_list = []
edge_weights_by_gene = {}
for u, v in combined_pg.graph.out_edges(cur_node):
v_rule = v[1][0]
# Get the object of the rule (a monomer name)
rule_obj = rule_obj_dict.get(v_rule)
if rule_obj:
# Add to list so we can count instances by gene
rule_obj_list.append(rule_obj)
# Get the abundance of rule object from the initial
# conditions
# TODO: Wrap in try/except?
ic_value = ic_dict[rule_obj]
else:
ic_value = 1.0
edge_weights[(u, v)] = ic_value
edge_weights_by_gene[rule_obj] = ic_value
# Get frequency of different rule objects
rule_obj_ctr = Counter(rule_obj_list)
# Normalize results by weight sum and gene frequency at this level
edge_weight_sum = sum(edge_weights_by_gene.values())
edge_weights_norm = {}
for e, v in edge_weights.items():
v_rule = e[1][1][0]
rule_obj = rule_obj_dict.get(v_rule)
if rule_obj:
rule_obj_count = rule_obj_ctr[rule_obj]
else:
rule_obj_count = 1
edge_weights_norm[e] = ((v / float(edge_weight_sum)) /
float(rule_obj_count))
# Add edge weights to paths graph
nx.set_edge_attributes(combined_pg.graph, 'weight',
edge_weights_norm)
# Sample from the combined CFPG
paths = combined_pg.sample_paths(max_paths)
# -------------------------------------------------
if paths:
pr = PathResult(True, 'PATHS_FOUND', max_paths, max_path_length)
pr.path_metrics = None
# Convert path polarity representation from 0/1 to 1/-1
pr.paths = convert_polarities(paths)
# Strip off the SOURCE_NODE prefix
pr.paths = [p[1:] for p in pr.paths]
else:
assert False
pr = PathResult(False, 'NO_PATHS_FOUND', max_paths, max_path_length)
pr.path_metrics = None
pr.paths = []
return pr
EDGES = []
for i in range(len(source)):
node1 = source[i]
node2 = target[i]
EDGES.append((node1, node2))
cnt = Counter(EDGES)
# adding attributes to the graph
d = {}
for k in cnt.keys():
d[k] = {}
d[k]['weight'] = cnt[k]
#add additional attributes here for an edge between two nodes. The attributes can be a list with all the dates for instance
nx.set_edge_attributes(G, d)
#extracting edges with highest weights
s = []
for (u, v) in G.edges():
w = G[u][v]['weight']
if w > 1:
s.append([(u, v), w])
s = np.array(s)
edge_imp = s[s[:, 1].astype('float64').argsort()[::-1]] #most important edges
with open('edge_importance.csv', 'w') as f:
writer = csv.writer(f)
writer.writerows(edge_imp)
#extracting unique relations of high weight edges
from matplotlib import pyplot as plt
import scipy.cluster.vq as vq
import networkx as nx
import csv
from tkinter import messagebox
from tkinter import filedialog
import seaborn as sns
sns.set()
file_open = filedialog.askopenfilename(initialdir='D:\PhD-Study\Experimental\SoDIP6', title='Select file', \
filetypes=(('GML files', '*.gml'),('CSV files', '*.csv'), ('All files', '*.*')))
if file_open.endswith('.gml'):
try:
G = nx.read_gml(file_open)
if nx.get_edge_attributes(G, name='weight') is None:
nx.set_edge_attributes(G,10.0,name='weight')
except FileNotFoundError:
print('File not found')
elif file_open.endswith('.csv'):
G = nx.Graph()
try:
with open(file_open) as graph_file:
read_file = csv.DictReader(graph_file)
for row in read_file:
G.add_edge(row['node_from'], row['node_to'], weight=float(row['bandwidth']))
except FileNotFoundError:
print('File or node error')
else:
messagebox.showerror('SoDIP6','Could Not Load file')
with open('../dataset/generate_csv.csv', 'w', newline='') as write_csv:
def _boiler_generator_assn(
eia_transformed_dfs,
eia923_years=pc.WORKING_PARTITIONS['eia923']['years'],
eia860_years=pc.WORKING_PARTITIONS['eia860']['years'],
debug=False):
"""
Creates a set of more complete boiler generator associations.
Creates a unique unit_id_pudl for each collection of boilers and generators
within a plant that have ever been associated with each other, based on
the boiler generator associations reported in EIA860. Unfortunately, this
information is not complete for years before 2014, as the gas turbine
portion of combined cycle power plants in those earlier years were not
reporting their fuel consumption, or existence as part of the plants.
For years 2014 and on, EIA860 contains a unit_id_eia value, allowing the
combined cycle plant compoents to be associated with each other. For many
plants not listed in the reported boiler generator associations, it is
nonetheless possible to associate boilers and generators on a one-to-one
basis, as they use identical strings to describe the units.
In the end, between the reported BGA table, the string matching, and the
unit_id_eia values, it's possible to create a nearly complete mapping of
the generation units, at least for 2014 and later.
Args:
eia_transformed_dfs (dict): a dictionary of post-transform dataframes
representing the EIA database tables.
eia923_years (list-like): a list of the years of EIA 923 data that
should be used to infer the boiler-generator associations. By
default it is all the working years of data.
eia860_years (list-like): a list of the years of EIA 860 data that
should be used to infer the boiler-generator associations. By
default it is all the working years of data.
debug (bool): If True, include columns in the returned dataframe
indicating by what method the individual boiler generator
associations were inferred.
Returns:
eia_transformed_dfs (dict): Returns the same dictionary of dataframes
that was passed in, and adds a new dataframe to it representing
the boiler-generator associations as records containing
plant_id_eia, generator_id, boiler_id, and unit_id_pudl
Raises:
AssertionError: If the boiler - generator association graphs are not
bi-partite, meaning generators only connect to boilers, and boilers
only connect to generators.
AssertionError: If all the boilers do not end up with the same unit_id
each year.
AssertionError: If all the generators do not end up with the same
unit_id each year.
"""
# if you're not ingesting both 860 and 923, the bga is not compilable
if not (eia860_years and eia923_years):
return pd.DataFrame()
# compile and scrub all the parts
logger.info("Inferring complete EIA boiler-generator associations.")
bga_eia860 = (
eia_transformed_dfs['boiler_generator_assn_eia860'].copy().pipe(
_restrict_years, eia923_years, eia860_years).astype({
'generator_id':
pd.StringDtype(),
'boiler_id':
pd.StringDtype(),
"plant_id_eia":
int,
}))
# grab the generation_eia923 table, group annually, generate a new tag
gen_eia923 = eia_transformed_dfs['generation_eia923'].copy()
gen_eia923 = gen_eia923.set_index(pd.DatetimeIndex(gen_eia923.report_date))
gen_eia923 = (_restrict_years(
gen_eia923, eia923_years, eia860_years).astype({
'generator_id':
pd.StringDtype(),
"plant_id_eia":
int,
}).groupby([pd.Grouper(freq='AS'), 'plant_id_eia', 'generator_id'
]).net_generation_mwh.sum().reset_index().assign(
missing_from_923=False))
# compile all of the generators
gens_eia860 = (eia_transformed_dfs['generators_eia860'].copy().pipe(
_restrict_years, eia923_years, eia860_years).astype({
'generator_id':
pd.StringDtype(),
"plant_id_eia":
int,
}))
gens = pd.merge(gen_eia923,
gens_eia860,
on=['plant_id_eia', 'report_date', 'generator_id'],
how='outer')
gens = (gens[[
'plant_id_eia', 'report_date', 'generator_id', 'unit_id_eia',
'net_generation_mwh', 'missing_from_923'
]].drop_duplicates().astype({
'generator_id': pd.StringDtype(),
"plant_id_eia": int,
}))
# create the beginning of a bga compilation w/ the generators as the
# background
bga_compiled_1 = pd.merge(
gens,
bga_eia860,
on=['plant_id_eia', 'generator_id', 'report_date'],
how='outer')
# Create a set of bga's that are linked, directly from bga8
bga_assn = bga_compiled_1[bga_compiled_1['boiler_id'].notnull()].copy()
bga_assn.loc[:, 'bga_source'] = 'eia860_org'
# Create a set of bga's that were not linked directly through bga8
bga_unassn = bga_compiled_1[bga_compiled_1['boiler_id'].isnull()].copy()
bga_unassn = bga_unassn.drop(['boiler_id'], axis=1)
# Side note: there are only 6 generators that appear in bga8 that don't
# apear in gens9 or gens8 (must uncomment-out the og_tag creation above)
# bga_compiled_1[bga_compiled_1['og_tag'].isnull()]
bf_eia923 = (eia_transformed_dfs['boiler_fuel_eia923'].copy().pipe(
_restrict_years, eia923_years, eia860_years).astype({
"boiler_id":
pd.StringDtype(),
"plant_id_eia":
int,
}).assign(total_heat_content_mmbtu=lambda x: x.fuel_consumed_units * x.
fuel_mmbtu_per_unit))
bf_eia923 = (bf_eia923.set_index(pd.DatetimeIndex(
bf_eia923.report_date)).groupby(
[pd.Grouper(freq='AS'), 'plant_id_eia', 'boiler_id']).agg({
'total_heat_content_mmbtu':
pudl.helpers.sum_na
}).reset_index().drop_duplicates(
subset=['plant_id_eia', 'report_date', 'boiler_id']))
# Create a list of boilers that were not in bga8
bf9_bga = bf_eia923.merge(
bga_compiled_1,
on=['plant_id_eia', 'boiler_id', 'report_date'],
how='outer',
indicator=True,
)
bf9_not_in_bga = bf9_bga[bf9_bga['_merge'] == 'left_only']
bf9_not_in_bga = bf9_not_in_bga.drop(['_merge'], axis=1)
# Match the unassociated generators with unassociated boilers
# This method is assuming that some the strings of the generators and the
# boilers are the same
bga_unassn = (bga_unassn.merge(
bf9_not_in_bga[['plant_id_eia', 'boiler_id', 'report_date']],
how='left',
left_on=['report_date', 'plant_id_eia', 'generator_id'],
right_on=['report_date', 'plant_id_eia', 'boiler_id']))
bga_unassn["bga_source"] = np.where(bga_unassn.boiler_id.notnull(),
"string_assn", pd.NA)
bga_compiled_2 = (bga_assn.append(bga_unassn).fillna(
{'missing_from_923': True}))
# Connect the gens and boilers in units
bga_compiled_units = bga_compiled_2.loc[
bga_compiled_2['unit_id_eia'].notnull()]
bga_gen_units = bga_compiled_units.drop(['boiler_id'], axis=1)
bga_boil_units = bga_compiled_units[[
'plant_id_eia', 'report_date', 'boiler_id', 'unit_id_eia'
]].copy()
bga_boil_units.dropna(subset=['boiler_id'], inplace=True)
# merge the units with the boilers
bga_unit_compilation = bga_gen_units.merge(
bga_boil_units,
how='outer',
on=['plant_id_eia', 'report_date', 'unit_id_eia'],
indicator=True)
# label the bga_source
bga_unit_compilation.loc[bga_unit_compilation['bga_source'].isnull(),
'bga_source'] = 'unit_connection'
bga_unit_compilation.drop(['_merge'], axis=1, inplace=True)
bga_non_units = bga_compiled_2[bga_compiled_2['unit_id_eia'].isnull()]
# combine the unit compilation and the non units
bga_compiled_3 = bga_non_units.append(bga_unit_compilation)
bga_compiled_3 = bga_compiled_3[[
'plant_id_eia', 'report_date', 'generator_id', 'boiler_id',
'unit_id_eia', 'bga_source', 'net_generation_mwh', 'missing_from_923'
]]
# label plants that have 'bad' generator records (generators that have MWhs
# in gens9 but don't have connected boilers) create a df with just the bad
# plants by searching for the 'bad' generators
bad_plants = bga_compiled_3[(bga_compiled_3['boiler_id'].isnull()) &
(bga_compiled_3['net_generation_mwh'] > 0)].\
drop_duplicates(subset=['plant_id_eia', 'report_date'])
bad_plants = bad_plants[['plant_id_eia', 'report_date']]
# merge the 'bad' plants back into the larger frame
bga_compiled_3 = bga_compiled_3.merge(bad_plants,
how='outer',
on=['plant_id_eia', 'report_date'],
indicator=True)
# use the indicator to create labels
bga_compiled_3['plant_w_bad_generator'] = np.where(
bga_compiled_3["_merge"] == 'both', True, False)
# Note: At least one gen has reported MWh in 923, but could not be
# programmatically mapped to a boiler
# we don't need this one anymore
bga_compiled_3 = bga_compiled_3.drop(['_merge'], axis=1)
# create a label for generators that are unmapped but in 923
bga_compiled_3['unmapped_but_in_923'] = np.where(
(bga_compiled_3.boiler_id.isnull())
& ~bga_compiled_3.missing_from_923
& (bga_compiled_3.net_generation_mwh == 0), True, False)
# create a label for generators that are unmapped
bga_compiled_3['unmapped'] = np.where(bga_compiled_3.boiler_id.isnull(),
True, False)
bga_out = bga_compiled_3.drop('net_generation_mwh', axis=1)
bga_out.loc[bga_out.unit_id_eia.isnull(), 'unit_id_eia'] = pd.NA
bga_for_nx = bga_out[[
'plant_id_eia', 'report_date', 'generator_id', 'boiler_id',
'unit_id_eia'
]]
# If there's no boiler... there's no boiler-generator association
bga_for_nx = bga_for_nx.dropna(subset=['boiler_id']).drop_duplicates()
# Need boiler & generator specific ID strings, or they look like
# the same node to NX
bga_for_nx['generators'] = (
'p' + bga_for_nx.plant_id_eia.astype(int).astype(str) + '_g' +
bga_for_nx.generator_id.astype(pd.StringDtype()))
bga_for_nx['boilers'] = ('p' +
bga_for_nx.plant_id_eia.astype(int).astype(str) +
'_b' +
bga_for_nx.boiler_id.astype(pd.StringDtype()))
# dataframe to accumulate the unit_ids in
bga_w_units = pd.DataFrame()
# We want to start our unit_id counter anew for each plant:
for pid in bga_for_nx.plant_id_eia.unique():
bga_byplant = bga_for_nx[bga_for_nx.plant_id_eia == pid].copy()
# Create a graph from the dataframe of boilers and generators. It's a
# multi-graph, meaning the same nodes can be connected by more than one
# edge -- this allows us to preserve multiple years worth of boiler
# generator association information for later inspection if need be:
bga_graph = nx.from_pandas_edgelist(bga_byplant,
source='generators',
target='boilers',
edge_attr=True,
create_using=nx.MultiGraph())
# Each connected sub-graph is a generation unit:
gen_units = [
bga_graph.subgraph(c).copy()
for c in nx.connected_components(bga_graph)
]
# Assign a unit_id to each subgraph, and extract edges into a dataframe
for unit_id, unit in zip(range(len(gen_units)), gen_units):
# All the boiler-generator association graphs should be bi-partite,
# meaning generators only connect to boilers, and boilers only
# connect to generators.
if not nx.algorithms.bipartite.is_bipartite(unit):
raise AssertionError(
f"Non-bipartite generation unit graph found."
f"plant_id_eia={pid}, unit_id_pudl={unit_id}.")
nx.set_edge_attributes(unit,
name='unit_id_pudl',
values=unit_id + 1)
new_unit_df = nx.to_pandas_edgelist(unit)
bga_w_units = bga_w_units.append(new_unit_df)
bga_w_units = bga_w_units.drop(['source', 'target'], axis=1)
# Check whether the PUDL unit_id values we've inferred conflict with
# the unit_id_eia values that were reported to EIA. Are there any PUDL
# unit_id values that have more than 1 EIA unit_id_eia within them?
bga_unit_id_eia_counts = (bga_w_units.groupby(
['plant_id_eia',
'unit_id_pudl'])['unit_id_eia'].nunique().to_frame().reset_index())
bga_unit_id_eia_counts = bga_unit_id_eia_counts.rename(
columns={'unit_id_eia': 'unit_id_eia_count'})
bga_unit_id_eia_counts = pd.merge(bga_w_units,
bga_unit_id_eia_counts,
on=['plant_id_eia', 'unit_id_pudl'])
too_many_codes = (
bga_unit_id_eia_counts[bga_unit_id_eia_counts.unit_id_eia_count > 1])
too_many_codes = (
too_many_codes[~too_many_codes.unit_id_eia.isnull()].groupby(
['plant_id_eia', 'unit_id_pudl'])['unit_id_eia'].unique())
for row in too_many_codes.iteritems():
logger.warning(f"Multiple EIA unit codes:"
f"plant_id_eia={row[0][0]}, "
f"unit_id_pudl={row[0][1]}, "
f"unit_id_eia={row[1]}")
bga_w_units = bga_w_units.drop('unit_id_eia', axis=1)
# These assertions test that all boilers and generators ended up in the
# same unit_id across all the years of reporting:
pgu_gb = bga_w_units.groupby(['plant_id_eia',
'generator_id'])['unit_id_pudl']
if not (pgu_gb.nunique() <= 1).all():
# raise AssertionError("Inconsistent inter-annual BGA assignment!")
logger.error("Inconsistent inter-annual plant-generator-units!")
pbu_gb = bga_w_units.groupby(['plant_id_eia', 'boiler_id'])['unit_id_pudl']
if not (pbu_gb.nunique() <= 1).all():
# raise AssertionError("Inconsistent inter-annual BGA assignment!")
logger.error("Inconsistent inter-annual plant-boiler-units!")
bga_w_units = (bga_w_units.drop(
'report_date', axis=1
).loc[:, ['plant_id_eia', 'unit_id_pudl', 'generator_id', 'boiler_id']].
drop_duplicates())
bga_out = (pd.merge(left=bga_out,
right=bga_w_units,
how='left',
on=['plant_id_eia', 'generator_id', 'boiler_id'
]).astype({"unit_id_pudl": pd.Int64Dtype()}))
if not debug:
bga_out = (bga_out[~bga_out.missing_from_923
& ~bga_out.plant_w_bad_generator
& ~bga_out.unmapped_but_in_923
& ~bga_out.unmapped].drop(
[
'missing_from_923', 'plant_w_bad_generator',
'unmapped_but_in_923', 'unmapped'
],
axis=1).drop_duplicates(subset=[
'report_date', 'plant_id_eia', 'boiler_id',
'generator_id'
]))
bga_out = pudl.helpers.convert_cols_dtypes(bga_out, "eia")
eia_transformed_dfs['boiler_generator_assn_eia860'] = bga_out
return eia_transformed_dfs
def prep_for_learning(ep_len, m, n, h, init_states, obstacles, pick_up_state, delivery_state, rewards, rew_val, custom_flag, custom_task):
# Create the environment and get the TS #
ts_start_time = timeit.default_timer()
disc = 1
TS, obs_mat, state_mat = create_ts(m,n,h)
path = '../data/ts_' + str(m) + 'x' + str(n) + 'x' + str(h) + '_1Ag_1.txt'
paths = [path]
bases = {init_states[0]: 'Base1'}
obs_mat = update_obs_mat(obs_mat, state_mat, m, obstacles, init_states[0])
TS = update_adj_mat_3D(m, n, h, TS, obs_mat)
create_input_file(TS, state_mat, obs_mat, paths[0], bases, disc, m, n, h, 0)
ts_file = paths
ts_dict = Ts(directed=True, multi=False)
ts_dict.read_from_file(ts_file[0])
ts = expand_duration_ts(ts_dict)
ts_timecost = timeit.default_timer() - ts_start_time
# Get the DFA #
dfa_start_time = timeit.default_timer()
pick_up = str(pick_up_state[0][0] * n + pick_up_state[0][1]) # Check later
delivery = str(delivery_state[0][0] * n + delivery_state[0][1])
tf = str((ep_len-1)/2) # time bound
if custom_flag == 1:
phi = custom_task
else:
phi = '[H^1 r' + pick_up + ']^[0, ' + tf + '] * [H^1 r' + delivery + ']^[0,' + tf + ']' # Construc the task according to pickup/delivery )^[0, ' + tf + ']'
_, dfa_inf, bdd = twtl.translate(phi, kind=DFAType.Infinity, norm=True) # states and sim. time ex. phi = '([H^1 r47]^[0, 30] * [H^1 r31]^[0, 30])^[0, 30]'
dfa_timecost = timeit.default_timer() - dfa_start_time # DFAType.Normal for normal, DFAType.Infinity for relaxed
# Get the PA #
pa_start_time = timeit.default_timer()
alpha = 1
nom_weight_dict = {}
weight_dict = {}
pa_or = ts_times_fsa(ts, dfa_inf) # Original pa
edges_all = nx.get_edge_attributes(ts_dict.g,'edge_weight')
max_edge = max(edges_all, key=edges_all.get)
norm_factor = edges_all[max_edge]
for pa_edge in pa_or.g.edges():
edge = (pa_edge[0][0], pa_edge[1][0], 0)
nom_weight_dict[pa_edge] = edges_all[edge]/norm_factor
nx.set_edge_attributes(pa_or.g, 'edge_weight', nom_weight_dict)
nx.set_edge_attributes(pa_or.g, 'weight', 1)
pa = copy.deepcopy(pa_or) # copy the pa
time_weight = nx.get_edge_attributes(pa.g,'weight')
edge_weight = nx.get_edge_attributes(pa.g,'edge_weight')
for pa_edge in pa.g.edges():
weight_dict[pa_edge] = alpha*time_weight[pa_edge] + (1-alpha)*edge_weight[pa_edge]
nx.set_edge_attributes(pa.g, 'new_weight', weight_dict)
pa_timecost = timeit.default_timer() - pa_start_time
# Compute the energy of the states #
energy_time = timeit.default_timer()
compute_energy(pa)
energy_dict = nx.get_node_attributes(pa.g,'energy')
energy_pa = []
for ind in range(len(pa.g.nodes())):
energy_pa.append(pa.g.nodes([0])[ind][1].values()[0])
# projection of pa on ts #
init_state = [init_states[0][0] * n + init_states[0][1]]
pa2ts = []
for i in range(len(pa.g.nodes())):
if pa.g.nodes()[i][0] != 'Base1':
pa2ts.append(int(pa.g.nodes()[i][0].replace("r","")))
else:
pa2ts.append(init_state[0])
i_s = i # Agent's initial location in pa
energy_timecost = timeit.default_timer() - pa_start_time
# TS adjacency matrix and source-target
TS_adj = TS
TS_s = []
TS_t = []
for i in range(len(TS_adj)):
for j in range(len(TS_adj)):
if TS_adj[i,j] != 0:
TS_s.append(i)
TS_t.append(j)
# pa adjacency matrix and source-target
pa_adj_st = nx.adjacency_matrix(pa.g)
pa_adj = pa_adj_st.todense()
pa_s = [] # source node
pa_t = [] # target node
for i in range(len(pa_adj)):
for j in range(len(pa_adj)):
if pa_adj[i,j] == 1:
pa_s.append(i)
pa_t.append(j)
# PA rewards matrix
rewards_ts = np.zeros(m * n)# - 1 # penalty?
rewards_pa = np.zeros(len(pa2ts))
rewards_ts_indexes = []
for i in range(len(rewards)):
rewards_ts_indexes.append(rewards[i][0] * n + rewards[i][1]) # rewards_ts_indexes[i] = rewards[i][0] * n + rewards[i][1]
rewards_ts[rewards_ts_indexes[i]] = rew_val
for i in range(len(rewards_pa)):
rewards_pa[i] = rewards_ts[pa2ts[i]]
# # Display some important info
print('##### PICK-UP and DELIVERY MISSION #####' + "\n")
print('Initial Location : ' + str(init_states[0]) + ' <---> Region ' + str(init_state[0]))
print('Pick-up Location : ' + str(pick_up_state[0]) + ' <---> Region ' + pick_up)
print('Delivery Location : ' + str(delivery_state[0]) + ' <---> Regions ' + delivery)
print('Reward Locations : ' + str(rewards) + ' <---> Regions ' + str(rewards_ts_indexes) + "\n")
print('State Matrix : ')
print(state_mat)
print("\n")
print('Mission Duration : ' + str(ep_len) + ' time steps')
print('TWTL Task : ' + phi + "\n")
print('Computational Costst : TS created in ' + str(ts_timecost) + ' seconds')
# print(' TS created in ' + str(ts_timecost) + ' seconds')
print(' DFA created in ' + str(dfa_timecost) + ' seconds')
print(' PA created in ' + str(pa_timecost) + ' seconds')
print(' Energy of PA states calculated in ' + str(energy_timecost) + ' seconds')
return i_s, pa, pa_s, pa_t, pa2ts, energy_pa, rewards_pa, pick_up, delivery
def reweight_by_selection(G, vertices_selected, reweight_value):
"""Assumes an unweighted graph is passed in"""
nx.set_edge_attributes(G, 1, WEIGHT_ATTRIBUTE)
for v in vertices_selected:
for neighbour in G[v]:
G[v][neighbour][WEIGHT_ATTRIBUTE] = reweight_value
def bind_properties_to_network(network,
network_properties,
bind_node_positions=True,
bind_node_color=True,
bind_node_radius=True,
bind_node_stroke_color=True,
bind_node_stroke_width=True,
bind_link_width=True,
bind_link_color=True,
bind_link_alpha=True):
"""
Binds calculated positional values to the network as node attributes `x` and `y`.
Parameters
----------
network : networkx.Graph or something alike
The network object to which the position should be bound
network_properties : dict
The network properties which are returned from the
interactive visualization.
bind_node_positions : bool (default: True)
bind_node_color : bool (default: True)
bind_node_radius : bool (default: True)
bind_node_stroke_color : bool (default: True)
bind_node_stroke_width : bool (default: True)
bind_link_width : bool (default: True)
bind_link_color : bool (default: True)
bind_link_alpha : bool (default: True)
Example
-------
>>> props, _ = netwulf.visualize(G)
>>> netwulf.bind_properties_to_network(G, props)
"""
# Add individial node attributes
if bind_node_positions:
x = {node['id']: node['x'] for node in network_properties['nodes']}
y = {node['id']: node['y'] for node in network_properties['nodes']}
nx.set_node_attributes(network, x, 'x')
nx.set_node_attributes(network, y, 'y')
network.graph['rescale'] = False
if bind_node_color:
color = {
node['id']: node['color']
for node in network_properties['nodes']
}
nx.set_node_attributes(network, color, 'color')
if bind_node_radius:
radius = {
node['id']: node['radius']
for node in network_properties['nodes']
}
nx.set_node_attributes(network, radius, 'radius')
# Add individual link attributes
if bind_link_width:
width = {(link['source'], link['target']): link['width']
for link in network_properties['links']}
nx.set_edge_attributes(network, width, 'width')
# Add global style properties
if bind_node_stroke_color:
network.graph['nodeStrokeColor'] = network_properties[
'nodeStrokeColor']
if bind_node_stroke_width:
network.graph['nodeStrokeWidth'] = network_properties[
'nodeStrokeWidth']
if bind_link_color:
network.graph['linkColor'] = network_properties['linkColor']
if bind_link_alpha:
network.graph['linkAlpha'] = network_properties['linkAlpha']
def randcost(G,k):
nx.set_edge_attributes(G, 'cost', 0) #set all costs to 0
edges = random.sample(G.edges(),k) #obtain k edges
dic = {key: 1 for key in edges} #dictionary, the k edges have cost 1
nx.set_edge_attributes(G, 'cost', dic) #set unit cost for k edges
def _compute_ricci_flow(G: nx.Graph,
weight="weight",
iterations=100,
step=1,
delta=1e-4,
surgery=(lambda G, *args, **kwargs: G, 100),
**kwargs):
"""Compute the given Ricci flow metric of each edge of a given connected NetworkX graph.
Parameters
----------
G : NetworkX graph
A given directional or undirectional NetworkX graph.
weight : str
The edge weight used to compute Ricci curvature. (Default value = "weight")
iterations : int
Iterations to require Ricci flow metric. (Default value = 100)
step : float
step size for gradient decent process. (Default value = 1)
delta : float
process stop when difference of Ricci curvature is within delta. (Default value = 1e-4)
surgery : (function, int)
A tuple of user define surgery function that will execute every certain iterations.
(Default value = (lambda G, *args, **kwargs: G, 100))
**kwargs
Additional keyword arguments passed to `_compute_ricci_curvature`.
Returns
-------
G: NetworkX graph
A NetworkX graph with ``weight`` as Ricci flow metric.
"""
if not nx.is_connected(G):
logger.warning(
"Not connected graph detected, compute on the largest connected component instead."
)
G = nx.Graph(G.subgraph(max(nx.connected_components(G), key=len)))
# Set normalized weight to be the number of edges.
normalized_weight = float(G.number_of_edges())
global _apsp
# Start compute edge Ricci flow
t0 = time.time()
if nx.get_edge_attributes(G, "original_RC"):
logger.warning(
"original_RC detected, continue to refine the ricci flow.")
else:
_compute_ricci_curvature(G, weight=weight, **kwargs)
for (v1, v2) in G.edges():
G[v1][v2]["original_RC"] = G[v1][v2]["ricciCurvature"]
# clear the APSP since the graph have changed.
_apsp = {}
# Start the Ricci flow process
for i in range(iterations):
for (v1, v2) in G.edges():
G[v1][v2][weight] -= step * (
G[v1][v2]["ricciCurvature"]) * G[v1][v2][weight]
# Do normalization on all weight to prevent weight expand to infinity
w = nx.get_edge_attributes(G, weight)
sumw = sum(w.values())
for k, v in w.items():
w[k] = w[k] * (normalized_weight / sumw)
nx.set_edge_attributes(G, values=w, name=weight)
logger.info(" === Ricci flow iteration %d === " % i)
_compute_ricci_curvature(G, weight=weight, **kwargs)
rc = nx.get_edge_attributes(G, "ricciCurvature")
diff = max(rc.values()) - min(rc.values())
logger.info("Ricci curvature difference: %f" % diff)
logger.info("max:%f, min:%f | maxw:%f, minw:%f" % (max(
rc.values()), min(rc.values()), max(w.values()), min(w.values())))
if diff < delta:
logger.info("Ricci curvature converged, process terminated.")
break
# do surgery or any specific evaluation
surgery_func, do_surgery = surgery
if i != 0 and i % do_surgery == 0:
G = surgery_func(G, weight)
normalized_weight = float(G.number_of_edges())
for n1, n2 in G.edges():
logger.debug("%s %s %s" % (n1, n2, G[n1][n2]))
# clear the APSP since the graph have changed.
_apsp = {}
logger.info("\n%8f secs for Ricci flow computation." % (time.time() - t0))
return G
def gtfs_db_to_schedule_graph(stop_times_db, stops_db, trips_db, routes_db,
services):
def get_time(time):
# return time as datetime.datetime, account for 24 in %H
time_list = time.split(':')
if int(time_list[0]) >= 24:
days = int(time_list[0]) // 24
time_list[0] = int(time_list[0]) % 24
if time_list[0] < 10:
time_list[0] = '0{}'.format(time_list[0])
else:
time_list[0] = str(time_list[0])
return datetime.strptime(':'.join(time_list),
'%H:%M:%S') + timedelta(days=days)
else:
return datetime.strptime(time, '%H:%M:%S')
def timedelta_to_hms(td):
return str(td).split('days')[-1].strip(' ')
def generate_stop_sequence(group):
group = group.sort_values(by='stop_sequence')
# remove stops that are loopy (consecutively duplicated)
unique_stops_mask = group['stop_id'].shift() != group['stop_id']
if not unique_stops_mask.all():
logging.warning(
'Your GTFS has (a) looooop edge(s)! A zero link between a node and itself, edge affected '
'\nThis edge will not be considered for computation, the stop will be deleted and the '
f'schedule will be changed. Affected stops: {group[~unique_stops_mask]["stop_id"].to_list()}'
)
group = group.loc[unique_stops_mask]
flattened = group.iloc[0, :][list(
set(group.columns) - {
'trip_id', 'stop_sequence', 'stop_id', 'arrival_time',
'departure_time'
})]
departure_time = group.iloc[0, :]['arrival_time']
flattened['trip_departure_time'] = departure_time.strftime("%H:%M:%S")
flattened['ordered_stops'] = group['stop_id'].to_list()
flattened['stops_str'] = ','.join(group['stop_id'].to_list())
flattened['arrival_offsets'] = [
timedelta_to_hms(t - departure_time) for t in group['arrival_time']
]
flattened['departure_offsets'] = [
timedelta_to_hms(t - departure_time)
for t in group['departure_time']
]
return flattened
def generate_trips(group):
flattened = group.iloc[0, :][list(
set(group.columns) - {
'route_id', 'stops_str', 'trip_id', 'vehicle_id',
'trip_departure_time'
})]
trip_id = group['trip_id'].to_list()
trip_departure_time = group['trip_departure_time'].to_list()
vehicle_id = group['vehicle_id'].to_list()
flattened['trips'] = {
'trip_id': trip_id,
'trip_departure_time': trip_departure_time,
'vehicle_id': vehicle_id
}
return flattened
def generate_routes(group):
service_id = group.iloc[0, :]['service_id']
group['route_id'] = [f'{service_id}_{i}' for i in range(len(group))]
return group
trips_db = trips_db[trips_db['service_id'].isin(services)]
df = trips_db[['route_id', 'trip_id']].merge(routes_db[{
'route_id', 'route_type', 'route_short_name', 'route_long_name',
'route_color'
} and set(routes_db.columns)],
on='route_id',
how='left')
df['mode'] = df['route_type'].apply(lambda x: get_mode(x))
df = df.merge(stop_times_db[[
'trip_id', 'stop_id', 'arrival_time', 'departure_time', 'stop_sequence'
]],
on='trip_id',
how='left')
df['arrival_time'] = df['arrival_time'].apply(lambda x: get_time(x))
df['departure_time'] = df['departure_time'].apply(lambda x: get_time(x))
df = df.groupby('trip_id').apply(generate_stop_sequence).reset_index()
# drop stop sequences that are single stops
df = df[df['ordered_stops'].str.len() > 1]
df['vehicle_id'] = [f'veh_{i}' for i in range(len(df))]
df = df.groupby(['route_id',
'stops_str']).apply(generate_trips).reset_index()
df = df.drop('stops_str', axis=1)
df['service_id'] = df['route_id'].astype(str)
df = df.groupby(['service_id']).apply(generate_routes)
g = nx.DiGraph(name='Schedule graph')
g.graph['crs'] = 'epsg:4326'
g.graph['route_to_service_map'] = df.set_index(
'route_id')['service_id'].T.to_dict()
g.graph['service_to_route_map'] = df.groupby(
'service_id')['route_id'].apply(list).to_dict()
g.graph['change_log'] = change_log.ChangeLog()
df['id'] = df['route_id']
g.graph['routes'] = df.set_index('route_id').T.to_dict()
df['id'] = df['service_id']
df = df.rename(columns={'route_short_name': 'name'})
g.graph['services'] = df[['service_id', 'id', 'name'
]].groupby('service_id').first().T.to_dict()
# finally nodes
stops = pd.DataFrame({
col: np.repeat(df[col].values, df['ordered_stops'].str.len())
for col in {'route_id', 'service_id'}
}).assign(stop_id=np.concatenate(df['ordered_stops'].values))
stop_groups = stops.groupby('stop_id')
stops = set(stop_groups.groups)
g.add_nodes_from(stops)
stops_db = stops_db.rename(columns={
'stop_lat': 'lat',
'stop_lon': 'lon',
'stop_name': 'name'
})
stops_db['id'] = stops_db['stop_id']
stops_db['x'] = stops_db['lon']
stops_db['y'] = stops_db['lat']
stops_db['epsg'] = 'epsg:4326'
stops_db['s2_id'] = stops_db.apply(lambda x: spatial.generate_index_s2(
lat=float(x['lat']), lng=float(x['lon'])),
axis=1)
nx.set_node_attributes(
g, stops_db[stops_db['stop_id'].isin(stops)].set_index(
'stop_id').T.to_dict())
nx.set_node_attributes(
g,
pd.DataFrame(
stop_groups['route_id'].apply(set)).rename(columns={
'route_id': 'routes'
}).T.to_dict())
nx.set_node_attributes(
g,
pd.DataFrame(stop_groups['service_id'].apply(set)).rename(
columns={
'service_id': 'services'
}).T.to_dict())
# and edges
df['ordered_stops'] = df['ordered_stops'].apply(
lambda x: list(zip(x[:-1], x[1:])))
stop_cols = np.concatenate(df['ordered_stops'].values)
edges = pd.DataFrame({
col: np.repeat(df[col].values, df['ordered_stops'].str.len())
for col in {'route_id', 'service_id'}
}).assign(from_stop=stop_cols[:, 0], to_stop=stop_cols[:, 1])
edge_groups = edges.groupby(['from_stop', 'to_stop'])
g.add_edges_from(edge_groups.groups)
nx.set_edge_attributes(
g,
pd.DataFrame(
edge_groups['route_id'].apply(set)).rename(columns={
'route_id': 'routes'
}).T.to_dict())
nx.set_edge_attributes(
g,
pd.DataFrame(edge_groups['service_id'].apply(set)).rename(
columns={
'service_id': 'services'
}).T.to_dict())
return g
test_set = (0.99 * total_nodes)
random_list = [i for i in range(total_nodes)]
#shuffle(random_list)
for node in range(0, total_nodes):
if node < validation_set:
G.node[random_list[node]]['val'] = False
G.node[random_list[node]]['test'] = False
elif node < test_set:
G.node[random_list[node]]['val'] = True
G.node[random_list[node]]['test'] = False
else:
G.node[random_list[node]]['val'] = False
G.node[random_list[node]]['test'] = True
nx.set_edge_attributes(G, 'test_removed', False)
nx.set_edge_attributes(G, 'train_removed', False)
json_graph_name = graph_name + "-G.json"
json_id_map_name = graph_name + "-id_map.json"
feats_file_name = graph_name + "-feats.npy"
#
#np.save(feats_file_name, features)
data = json_graph.node_link_data(G)
graphjson = json.dumps(data)
f1 = open(json_graph_name, 'w')
f1.write(graphjson)
f1.close()
#exit()
def load_network(self):
# read METIS file
self.G = utils.read_metis(self.DATA_FILENAME)
self.initial_number_of_nodes = self.G.number_of_nodes(
) # used for computing metrics
# Alpha value used in prediction model
self.prediction_model_alpha = self.G.number_of_edges() * (
self.num_partitions / self.G.number_of_nodes()**2)
if self.use_one_shot_alpha:
self.prediction_model_alpha = self.one_shot_alpha
# Order of nodes arriving
self.arrival_order = list(range(0, self.G.number_of_nodes()))
if self.SIMULATED_ARRIVAL_FILE == "":
# mark all nodes as needing a shelter
self.simulated_arrival_list = [1] * self.G.number_of_nodes()
else:
with open(self.SIMULATED_ARRIVAL_FILE, "r") as ar:
self.simulated_arrival_list = [
int(line.rstrip('\n')) for line in ar
]
# count the number of people arriving in the simulation
self.number_simulated_arrivals = 0
for arrival in self.simulated_arrival_list:
self.number_simulated_arrivals += arrival
if self.verbose > 0:
print("Graph loaded...")
print(nx.info(self.G))
if nx.is_directed(self.G):
print("Graph is directed")
else:
print("Graph is undirected")
self.reset()
# load displacement prediction weights
if (self.PREDICTION_LIST_FILE == ""):
self.predicted_displacement_weights = [
1
] * self.G.number_of_nodes()
else:
with open(self.PREDICTION_LIST_FILE, 'r') as plf:
self.predicted_displacement_weights = [
float(line.rstrip('\n')) for line in plf
]
# preserve original node/edge weight when modification functions are applied
if self.graph_modification_functions:
node_weights = {
n[0]: n[1]['weight']
for n in self.G.nodes_iter(data=True)
}
nx.set_node_attributes(self.G, 'weight_orig', node_weights)
edge_weights = {(e[0], e[1]): e[2]['weight']
for e in self.G.edges_iter(data=True)}
nx.set_edge_attributes(self.G, 'weight_orig', edge_weights)
# make a copy of the original graph since we may alter the graph later on
# and may want to refer to information from the original graph
self.originalG = self.G.copy()
}
def slice_network(G, T, copy=True):
"""
Remove all edges witth weight<T from G or its copy,
"""
F = G.copy() if copy else G
F.remove_edges_from(
(n1, n2) for n1, n2, w in F.edges(data="weight") if w < T)
return F
# Draw different degrees of slicing for a random graph
ergraph = nx.erdos_renyi_graph(100, 0.1)
nx.set_edge_attributes(ergraph, {(n1, n2): random.random()
for n1, n2 in ergraph.edges()}, "weight")
pos = graphviz_layout(ergraph)
for i, threshold in enumerate([0, 0.7, 0.8, 0.9, 0.95, 1.0]):
ax = plt.subplots(3, 2, i + 1)
slice_network(ergraph, threshold, copy=False)
nx.draw_networkx_edges(ergraph,
pos,
alpha=0.65,
ax=ax,
**dzcnapy.small_attrs)
nx.draw_networkx_nodes(ergraph, pos, ax=ax, **dzcnapy.small_attrs)
dzcnapy.set_extent(pos, ax, "Threshold={}".format(threshold))
dzcnapy.plot("slicing")
def edit_graph():
# provide a different graph for each user
graphname = "static/data/graph_login_test"
if current_user.get_id() != None:
graphname += "_" + current_user.get_id() + ".json"
else:
graphname += ".json"
form1 = node_form(request.form)
form2 = edge_form(request.form)
# open graph json file: load into graph G
with open(graphname, "r") as read_file:
data = json.load(read_file)
G = json_graph.node_link_graph(data)
graph_info = []
num_nodes = "Number of Nodes: " + str(G.number_of_nodes())
num_edges = "Number of Edges: " + str(G.number_of_edges())
density = "Density of Graph: " + str(round(nx.density(G), 5))
graph_info.append(num_nodes)
graph_info.append(num_edges)
graph_info.append(density)
if request.method == 'POST':
# add node
if form1.add_node.data and form1.validate():
if G.has_node(form1.node_name.data):
print("add failed: such node already exist")
flash(u"Add Failed: such node already exist", "danger")
else:
G.add_node(
form1.node_name.data,
category=form1.category.data,
degree=0,
viz={'size': 10},
# label=form1.node_name.data,
url=form1.url.data,
content=form1.content.data,
notes=form1.notes.data)
print("added a node")
flash(u"Added Node: " + "'" + form1.node_name.data + "'",
'success')
# else:
# print(form1.validate())
# edit node
if form1.edit_node.data and form1.validate():
if G.has_node(form1.node_name.data):
attrs = {}
if form1.category.data != "":
attrs.update({"category": form1.category.data})
if form1.url.data != "":
attrs.update({"url": form1.url.data})
if form1.content.data != "":
attrs.update({"content": form1.content.data})
if form1.notes.data != "":
attrs.update({"notes": form1.notes.data})
attr = {form1.node_name.data: attrs}
nx.set_node_attributes(G, attr)
print("updated a node")
flash(u"Updated Node: " + "'" + form1.node_name.data + "'",
'success')
else:
print("edit failed: such node does not exist")
flash(u"Edit Failed: such node does not exist", "danger")
# delete node
if form1.delete_node.data and form1.validate():
if G.has_node(form1.node_name.data):
G.remove_node(form1.node_name.data)
print("removed a node")
flash(u"Removed Node: " + "'" + form1.node_name.data + "'",
'success')
else:
print("remove failed: such node does not exist")
flash(u"Remove Failed: such node does not exist", "danger")
# else:
# print(form1.validate())
# add edge
if form2.add_edge.data and form2.validate():
if G.has_node(form2.source_name.data) and G.has_node(
form2.target_name.data):
G.add_edge(form2.source_name.data,
form2.target_name.data,
relationship=form2.relationship.data,
key=str(G.number_of_edges()),
content=form2.content.data,
notes=form2.notes.data)
update_attr(G, form2)
print("added an edge")
flash(u"Added Edge (key: " + str(G.number_of_edges()) + "): " +
"'" + form2.source_name.data + "' -> " + "'" +
form2.target_name.data + "'" + 'success')
else:
print("add failed: source/target node does not exist")
flash(u"Add Failed: source/target node does not exist",
"danger")
# edit edge
if form2.edit_edge.data and form2.validate():
if form2.key_num.data == "":
print("you need to input the key of the edge")
flash(u"Edit Failed: you need to input the key of the edge",
"danger")
elif G.has_edge(form2.source_name.data,
form2.target_name.data,
key=form2.key_num.data):
attrs = {}
if form2.relationship.data != "":
attrs.update({"relationship": form2.relationship.data})
if form2.content.data != "":
attrs.update({"content": form2.content.data})
if form2.notes.data != "":
attrs.update({"notes": form2.notes.data})
attr = {
(form2.source_name.data, form2.target_name.data, form2.key_num.data):
attrs
}
nx.set_edge_attributes(G, attr)
print("updated an edge")
flash(u"Updated Edge (key: " + str(G.number_of_edges()) +
"): " + "'" + form2.source_name.data + "' -> " + "'" +
form2.target_name.data + "'" + 'success')
else:
print("edit failed: such edge does not exist")
flash(u"Edit Failed: such edge does not exist", "danger")
# delete edge
if form2.delete_edge.data and form2.validate():
if form2.key_num.data == "":
print("you need to input the key of the edge")
flash(u"Edit Failed: you need to input the key of the edge",
"danger")
if G.has_edge(form2.source_name.data,
form2.target_name.data,
key=form2.key_num.data):
G.remove_edge(form2.source_name.data,
form2.target_name.data,
key=form2.key_num.data)
update_attr(G, form2)
print("removed an edge")
flash(u"Eemoved Edge (key: " + str(G.number_of_edges()) +
"): " + "'" + form2.source_name.data + "' -> " + "'" +
form2.target_name.data + "'" + 'success')
else:
print("remove failed: such edge does not exist")
flash(u"Edit Failed: such edge does not exist", "danger")
# save edited graph G into json file
data = json_graph.node_link_data(G)
with open(graphname, "w") as write_file:
json.dump(data, write_file)
return redirect(url_for('edit_graph'))
return render_template('edit-graph.html',
form_node=form1,
form_edge=form2,
graph_info=graph_info,
title="Edit Graph")
'San Francisco, CA',
'Saint Augustine, FL',
'Spokane, WA',
'Worcester, MA',
'Tucson, AZ']
# make subgraph of cities
H = G.subgraph(cities)
print("Subgraph has %d nodes with %d edges" % (len(H), H.size()))
print(H.nodes)
# draw with grave
plt.figure(figsize=(8, 8))
# create attribute for label
nx.set_edge_attributes(H,
{e: G.edges[e]['weight'] for e in H.edges},
'label')
# create stylers
def transfer_G_layout(network):
return {n: G.position[n] for n in network}
def elabel_base_style(attr):
return {'font_size': 4,
'font_weight': 'bold',
'font_family': 'sans-serif',
'font_color': 'b',
'rotate': True, # TODO: make rotation less granular
}
elabel_style = grave.style_merger(grave.use_attributes('label'),
def set_edge_attribute(G, attr, new_attrs):
nx.set_edge_attributes(
G, {(u, v): va
for (u, v, a), va in zip(G.edges(data=True), new_attrs)}, attr)
def solve(self, G):
"""
Find the pipes diameter for a given aqueduct topology respecting the following constraints:
- pipes velocities between 0.5 and 1 m/s
- pipes diameters commercially available
"""
import z3
# add a solver
solver = z3.Solver()
# +-----------------------+
# | variables declaration |
# +-----------------------+
# water demand in each node
X = dict([(node, z3.Real("demand_%s" % i))
for i, (node, datadict) in enumerate(G.nodes.items())])
# water flow in each pipe
Q = dict([(edge, z3.Real("Q_%s" % i))
for i, (edge, datadict) in enumerate(G.edges.items())])
# water speed in each pipe
V = dict([(edge, z3.Real("V_%s" % i))
for i, (edge, datadict) in enumerate(G.edges.items())])
# pipes diameter
D = dict([(edge, z3.Real("D_%s" % i))
for i, (edge, datadict) in enumerate(G.edges.items())])
# +---------------------+
# | boundary conditions |
# +---------------------+
# water demand in each node
boudary_c = []
for node, datadict in G.nodes.items():
if not datadict["Tank"]:
boudary_c.append(X[node] == datadict["DEMAND"])
# closing equation in each pipe: Q = V * A
def sym_abs(x):
return z3.If(x >= 0, x, -x)
closing_c = [
sym_abs(Q[edge]) / 1000 == V[edge] * (D[edge] / 1000 / 2)**2 *
math.pi for edge in G.edges
]
# speed limits
speed_c = []
for edge, datadict in G.edges.items():
if datadict["LEVEL"] == 1:
speed_c.append(z3.And(V[edge] >= 0.1, V[edge] <= 1.))
else:
speed_c.append(z3.And(V[edge] >= 0.01, V[edge] <= 1.5))
# pipes diameters
diameter_c = []
available_measures = [
75., 90., 110., 125., 140., 160., 180., 200., 225., 250., 280.,
315., 355., 400.
]
for edge, datadict in G.edges.items():
if datadict["LEVEL"] == 1:
diameter_c += [
z3.Or(
[D[edge] == measure for measure in available_measures])
]
else:
diameter_c += [
z3.Or([
D[edge] == measure
for measure in available_measures + [15., 25., 50.]
])
]
# kirchoff lows in the nodes
kirchoff_c = []
for n1 in G.nodes:
kirchoff_c.append(
z3.Sum([X[n1]] + [
Q[(n1, n2)] if (n1, n2) in Q else -Q[(n2, n1)]
for n2 in G.neighbors(n1)
]) == 0)
# add conditons
solver.add(boudary_c + closing_c + speed_c + kirchoff_c + diameter_c)
def compute_head():
"""
Computes head in meters for each node in the graph. The value is calculated imposing the head in the tank
and than subtracting the loss on the pipes
"""
for node, datadict in G.nodes.items():
if datadict["Tank"] == True:
start = node
H = {start: 164}
visited, queue = set(), [start]
while queue:
node = queue.pop(0)
if node not in visited:
visited.add(node)
queue.extend(set(G.neighbors(node)) - visited)
for neighbour in G.neighbors(node):
l = G[node][neighbour]["LENGHT"]
K = 10.67 / 120**1.852
def contain(d, k1, k2):
if k1 in d:
if k2 in d[k1]:
return True
return False
if (node, neighbour) in dict(G.nodes):
q = G[node][neighbour]["Q"]
else:
q = -G[neighbour][node]["Q"]
d = G[node][neighbour]["DIAMETER"]
H[neighbour] = H[node] - (K * l * math.copysign(
(abs(q) / 1000)**1.852, q) / (d / 1000)**4.8704)
return H
if solver.check() == z3.sat:
print("solved")
m = solver.model()
solver.add(
z3.Or([D[edge] != m.evaluate(D[edge]) for edge in G.edges]))
X = dict([(node, float(m.evaluate(X[node]).numerator_as_long()) /
float(m.evaluate(X[node]).denominator_as_long()))
for node in X])
nx.set_node_attributes(G, X, "Q")
Q = dict([(edge, float(m.evaluate(Q[edge]).numerator_as_long()) /
float(m.evaluate(Q[edge]).denominator_as_long()))
for edge in Q])
V = dict([(edge, float(m.evaluate(V[edge]).numerator_as_long()) /
float(m.evaluate(V[edge]).denominator_as_long()))
for edge in V])
D = dict([(edge, float(m.evaluate(D[edge]).numerator_as_long()) /
float(m.evaluate(D[edge]).denominator_as_long()))
for edge in D])
data = dict([("Q", Q), ("V", V), ("DIAMETER", D)])
for key, datadict in data.items():
nx.set_edge_attributes(G, datadict, key)
H = compute_head()
nx.set_node_attributes(G, H, "H")
else:
print("no solutions")
def _order_edges_in_block(self, block_data, drop_augmented):
"""Produce an edge sequence for all edges in the component.
Parameters
----------
block_data : pandas.DataFrame
A DataFrame representing all the edges within a single block.
drop_augmented : bool
Whether or not to keep any edges that needed to be added to the source edges in order to navigate the
network.
Returns
-------
edges : pandas.DataFrame
The same edges that were input with the edge order and route type as new columns.
"""
logger.debug("order_edges_by_block started")
logger.debug("Received edge data of shape %s", block_data.shape)
# Sort the DataFrame to load right hand arcs into NetworkX first.
# Note that Eulerian paths work in reverse order.
block_data = block_data.sort_values(self.leftrightflag_field,
ascending=False)
block_g = nx.from_pandas_edgelist(block_data,
source=self.source_field,
target=self.target_field,
edge_attr=True,
create_using=self.graph_type)
logger.debug("Block contains %s edges and %s nodes",
block_g.number_of_edges(), block_g.number_of_nodes())
# if the graph is empty it means there is a problem with the source data
# an error is logged, but other blocks are still processed
if nx.is_empty(block_g):
logger.error("Block contains no edges and cannot be sequenced")
return
# Scale nodes that are mid-segment by looking for duplicated ngd_str_uid values
logger.debug(
"Looking for nodes that fall in the middle of a road segment")
block_data['same_ngd_str_uid'] = block_data.duplicated(
subset=[self.struid_field], keep=False)
mid_arc_start_nodes = set(
block_data.loc[block_data['same_ngd_str_uid'] == True,
self.source_field])
mid_arc_end_nodes = set(
block_data.loc[block_data['same_ngd_str_uid'] == True,
self.target_field])
mid_arc_nodes = mid_arc_start_nodes.intersection(mid_arc_end_nodes)
if mid_arc_nodes:
logger.debug("Found mid-segment nodes: %s", mid_arc_nodes)
self._apply_node_scaling_factor(mid_arc_nodes, factor=-0.5)
# initialize the edge sequence counter
edge_sequence = 0
# record what type of path was used to determine the circuit
path_indicator_name = self.path_indicator
path_indicator_edges = {}
# blocks don't necessarily form fully connected graphs, so cycle through the components
logger.debug("Block contains %s connected components",
nx.number_weakly_connected_components(block_g))
for block_comp in sorted(nx.weakly_connected_components(block_g),
key=len,
reverse=True):
logger.debug(
"Creating subgraph from connected component with %s nodes",
len(block_comp))
block_g_comp = block_g.subgraph(block_comp)
# determine the preferred start node for this block component
preferred_sp = self._get_preferred_start_node(block_g_comp.nodes)
logger.debug("Preferred start node for this block: %s",
preferred_sp)
logger.debug("Component contains %s edges and %s nodes",
block_g_comp.number_of_edges(), len(block_g_comp))
# Need to pick an approach to processing this component depending on what type of circuit it forms.
# Ideally things are a Eulerian circuit that can be walked and return to start, but not all blocks form
# these nice circuits. If no good circuit can be found, then sequence numbers are just applied but may
# not form a logical order.
# Track the sequence value in case the enumeration method needs to be reset. This gets used when using
# the preferred start point fails, and also controls if the start node for this component is marked as a
# point we want to cluster on.
seq_val_at_start = edge_sequence
# the preferred option is a Eulerian circuit, so try that first
# logger.debug("Available edges: %s", block_g_comp.edges)
if nx.is_eulerian(block_g_comp):
logger.debug("Block component is eulerian.")
# record all these edges as being eulerian
indicator = dict(
zip(block_g_comp.edges, ['circuit'] * block_g_comp.size()))
path_indicator_edges.update(indicator)
# enumerate the edges and order them directly
logger.debug("Creating Eulerian circuit from node %s",
preferred_sp)
for u, v, k in nx.eulerian_circuit(block_g_comp,
source=preferred_sp,
keys=True):
edge_sequence += 1
block_g.edges[u, v, k][self.eo_name] = edge_sequence
# logger.debug("Sequence applied: (%s, %s, %s) = %s", u, v, k, edge_sequence)
# next best option is a path that stops at a different location from the start point
elif nx.has_eulerian_path(block_g_comp):
logger.debug("Block component forms Eulerian path")
# record all these edges as being a eulerian path
indicator = dict(
zip(block_g_comp.edges, ['path'] * block_g_comp.size()))
path_indicator_edges.update(indicator)
try:
logger.debug(
"Trying to create path from preferred start node %s",
preferred_sp)
for u, v, k in nx.eulerian_path(block_g_comp,
source=preferred_sp,
keys=True):
edge_sequence += 1
# check if the start point is actually in the first edge
if edge_sequence == 1 and not (preferred_sp == u
or preferred_sp == v):
logger.debug(
"Preferred start point not present on starting edge, throwing KeyError."
)
raise KeyError("Invalid starting edge")
# Sometimes the preferred start point means walking over the same edge twice, which will leave
# a data gap (the previous edge order value will be overwritten). If this happens, throw a
# KeyError
if block_g.edges[u, v, k].get(self.eo_name):
logger.debug("Edge already sequenced.")
raise KeyError(
"Preferred start point results in backtracking."
)
block_g.edges[u, v, k][self.eo_name] = edge_sequence
# logger.debug("Sequence applied: (%s, %s, %s) = %s", u, v, k, edge_sequence)
if edge_sequence < block_g_comp.number_of_edges():
logger.debug("It looks like some edges got missed")
raise KeyError("Missing edges on path")
logger.debug(
"Path was created from desired start point %s",
preferred_sp)
except KeyError:
# preferred start point failed; let networkx pick and start over
logger.debug(
"Preferred start node did not create a path. Trying a different one."
)
# reset the path listing since a new point will be picked
logger.debug("Reset edge_sequence value to %s",
seq_val_at_start)
edge_sequence = seq_val_at_start
for u, v, k in nx.eulerian_path(block_g_comp, keys=True):
edge_sequence += 1
block_g.edges[u, v, k][self.eo_name] = edge_sequence
# logger.debug("Sequence applied: (%s, %s, %s) = %s", u, v, k, edge_sequence)
# No good path exists, which means someone will have to backtrack
else:
logger.debug(
"Non-eulerian block is not easily traversable. Eulerizing it."
)
# Record all these edges as being augmented.
indicator = dict(
zip(block_g_comp.edges,
['augmented'] * block_g_comp.size()))
path_indicator_edges.update(indicator)
# Send this data to the anomaly folder so that it can be investigated later. It could have addressable
# issues that operations can correct for the next run.
logger.debug("Writing anomaly set for this block")
bf_uid_set = list(
nx.get_edge_attributes(
block_g_comp, self.edge_uid_field).values()).pop()
anomaly_file_name = f"anomaly_block_component.{bf_uid_set}.yaml"
nx.write_yaml(block_g_comp, (self.anomaly_folder /
anomaly_file_name).as_posix())
# You cannot eulerize a directed graph, so create an undirected one
logger.debug("Creating MultiGraph from directed graph.")
temp_graph = nx.MultiGraph()
for u, v, data in block_g_comp.edges(data=True):
key = temp_graph.add_edge(u, v, **data)
# logger.debug("Adding edge (%s, %s, %s) to temporary graph.", u, v, key)
logger.debug("Created temporary MultiGraph with %s edges",
temp_graph.number_of_edges())
# Convert the temporary graph to a proper Euler circuit so that it can be traversed.
logger.debug("Eulerizing MultiGraph")
euler_block = nx.eulerize(temp_graph)
logger.debug("Added %s edges to the block",
(euler_block.size() - temp_graph.size()))
logger.debug("Number of vertices in eulerized graph: %s",
euler_block.number_of_nodes())
# As we try to traverse the undirected graph, we need to keep track of places already visited to make
# sure arcs are not skipped.
visited_edges = Counter()
# augmented edges will throw the node weights off, so don't bother trying the preferred start node
logger.debug("Generating path through augmented block")
for u, v, k in nx.eulerian_circuit(euler_block,
preferred_sp,
keys=True):
# augmented edges have no attributes, so look for one and skip the edge if nothing is returned
if drop_augmented and not euler_block.edges[u, v, k].get(
self.edge_uid_field):
logger.debug("Ignoring augmented edge (%s, %s, %s)", u,
v, k)
continue
# Increment the sequence value for each edge we see.
edge_sequence += 1
# Since we formed an undirected MultiGraph we need to check the orientation of the nodes on the
# edge to assign the sequence back to the directed graph.
start_node = u
end_node = v
available_edge_count = block_g.number_of_edges(
start_node, end_node)
# If no edges exist, invert the nodes and check again.
# This also checks to see if we've already encountered all the edges between these nodes, indicating
# we need to process the inverse of the start and end values
if available_edge_count == 0 or (
((start_node, end_node) in visited_edges) and
(available_edge_count
== visited_edges[(start_node, end_node)])):
logger.debug(
"Nothing to process between (%s, %s), inverting nodes.",
start_node, end_node)
start_node = v
end_node = u
available_edge_count = block_g.number_of_edges(
start_node, end_node)
logger.debug(
"Number of edges available between (%s, %s): %s",
start_node, end_node, available_edge_count)
# Apply the edge_sequence to the first edge that hasn't received one yet
for ki in range(available_edge_count):
if not block_g.edges[start_node, end_node, ki].get(
self.eo_name):
logger.debug(
"Edge sequence applied: (%s, %s, %s) = %s",
start_node, end_node, ki, edge_sequence)
block_g.edges[start_node, end_node,
ki][self.eo_name] = edge_sequence
visited_edges[(start_node, end_node)] += 1
break
# At this point every edge should be accounted for, but in case something somehow slips through the cracks
# it needs to be given a sequence label. The label almost certainly won't make much sense in terms of a
# logical ordering, but this is just trying ot make sure it is counted.
logger.debug("Looking for any missed edges in block component")
for u, v, k in block_g_comp.edges:
if not block_g.edges[u, v, k].get(self.eo_name):
edge_sequence += 1
block_g.edges[u, v, k][self.eo_name] = edge_sequence
logger.warning(
"Applied out of order sequence to component edge (%s, %s, %s): %s",
u, v, k, edge_sequence)
# just log the last sequence value to make tracing easier
logger.debug("Final edge sequence value for component: %s",
edge_sequence)
# apply a sequence value to all the edges that were discovered
logger.debug("Edge order results: %s",
nx.get_edge_attributes(block_g_comp, self.eo_name))
# To help cluster the start nodes, mark which node was used as the start point in this block
if seq_val_at_start == 1:
self._mark_chosen_start_node(block_g_comp, preferred_sp)
logger.debug("Finished processing component")
# record that block processing is finished
logger.debug("Block processing complete")
# nx.set_edge_attributes(block_g, block_sequence_labels, self.eo_name)
nx.set_edge_attributes(block_g, path_indicator_edges,
path_indicator_name)
# check to see if the counts line up
if not block_g.number_of_edges() == edge_sequence:
logger.debug(
"Edge sequence (%s) and edge count (%s) do not match in block",
edge_sequence, block_g.number_of_edges())
# help start point clustering by apply a scaling factor to all nodes that were touched
logger.debug(
"Applying scaling factor to nodes in this block, except start point"
)
nodes_in_block = set(block_g.nodes())
nodes_in_block.remove(
preferred_sp) # don't scale the preferred start point
self._apply_node_scaling_factor(nodes_in_block)
logger.debug("Final node data for block: %s",
self.graph.subgraph(block_g.nodes).nodes(data=True))
logger.debug("Returning pandas DataFrame from block graph.")
return nx.to_pandas_edgelist(block_g,
source=self.source_field,
target=self.target_field)
def design_aqueduct(self, LEVEL=0):
"""
Performs automatic water network distribution design.
"""
# Costante di conversion dall'unità di misura dalla carta a metri
CONVERSION = 6373044.737 * math.pi / 180
# clustering
labels, cluster_centers = self.cluster(self.graph)
# --- ADDUCTION ---
adduction = nx.Graph()
for node in cluster_centers:
adduction.add_node(node)
self.complete_graph(adduction)
adduction = self.mesh_graph(adduction, weight='dist')
self.acqueduct.add_edges_from(adduction.edges())
nx.set_node_attributes(self.acqueduct, -1, "CLUSTER")
nx.set_node_attributes(self.acqueduct, False, "Tank")
nx.set_edge_attributes(self.acqueduct, 1, "LEVEL")
# --- DISTRIBUTION ---
# initialize distribution graphs
distribution = [nx.Graph() for cluster in cluster_centers]
# add nodes to the respective cluster
for node in labels:
cluster = labels[node]
distribution[cluster].add_node(node)
# add center to the respective cluster
for cluster, center in enumerate(cluster_centers):
distribution[cluster].add_node(center)
# for each distribution sub-network
for dist_graph in distribution:
# connect nodes in the graph
self.complete_graph(dist_graph)
dist_graph = self.mesh_graph(dist_graph, weight='dist')
# mark as distribution
nx.set_edge_attributes(dist_graph, 2, "LEVEL")
# add to acqueduct
self.acqueduct.add_edges_from(dist_graph.edges())
# add label info to the graph
nx.set_node_attributes(self.acqueduct, labels, 'CLUSTER')
# add elevation
elevdict = nx.get_node_attributes(self.graph, "mean")
nx.set_node_attributes(self.acqueduct, elevdict, "ELEVATION")
for center in adduction.nodes:
self.acqueduct.nodes[center]["ELEVATION"] = 0
for center in adduction.nodes:
min_dist = float("+inf")
min_alt = 0
for nbr in self.acqueduct.neighbors(center):
if self.distance(center, nbr) < min_dist:
min_dist = self.distance(center, nbr)
min_alt = self.acqueduct.nodes[nbr]["ELEVATION"]
self.acqueduct.nodes[center]["ELEVATION"] = min_alt
self.acqueduct.nodes[center]["DEMAND"] = 0
# add tank info to the graph
namedict = nx.get_node_attributes(self.graph, "name")
tankdict = dict([(node, True if namedict[node] == "tank" else False)
for node in namedict])
nx.set_node_attributes(self.acqueduct, tankdict, 'Tank')
initial_level = 3
nx.set_node_attributes(
self.acqueduct,
dict([(node, initial_level) for node in tankdict
if tankdict[node]]), "Linit")
# add level
for dist_graph in distribution:
leveldict = nx.get_node_attributes(dist_graph, "LEVEL")
nx.set_node_attributes(self.acqueduct, leveldict, "LEVEL")
# add node id
for ID, node in enumerate(self.acqueduct.nodes.items()):
node, datadict = node
nx.set_node_attributes(self.acqueduct, {node: ID}, "ID")
# add demande
building_type = nx.get_node_attributes(self.graph, "building")
for ID, node in enumerate(self.acqueduct.nodes.items()):
node, datadict = node
building2demand = {"appartment": 0.11}
default = 0.11
demand = building2demand.get(building_type.get(node, ''), default)
nx.set_node_attributes(self.acqueduct, {node: demand}, "DEMAND")
# add edge attributes
IDdict = nx.get_node_attributes(self.acqueduct, "ID")
for ID, edge in enumerate(self.acqueduct.edges.items()):
edge, datadict = edge
node1 = edge[0]
node2 = edge[1]
datadict["DC_ID"] = ID
datadict["LENGHT"] = self.distance(node1, node2) * CONVERSION
datadict["NODE1"] = IDdict[node1]
datadict["NODE2"] = IDdict[node2]
datadict["DIAMETER"] = 500 if "LEVEL" in self.acqueduct[node1][
node2] else 100
datadict["ROUGHNESS"] = 120 if datadict["DIAMETER"] <= 200 else 130
datadict["MINORLOSS"] = 0
datadict["STATUS"] = 1
datadict["LEVEL"] = 1 if edge in adduction.edges else 2
for key in datadict:
nx.set_edge_attributes(self.acqueduct, {edge: datadict[key]},
key)
# case we only want the first level network
if LEVEL == 1:
adduction = self.acqueduct.copy()
distribution = [
node for node, _ in self.acqueduct.nodes.items()
if not node in cluster_centers
]
adduction.remove_nodes_from(distribution)
for cluster, center in enumerate(cluster_centers):
self.acqueduct.nodes[center]["DEMAND"] = 0
for node in labels:
if labels[node] == cluster:
adduction.nodes[center][
"DEMAND"] += self.acqueduct.nodes[node]["DEMAND"]
self.acqueduct = adduction
for node in G.nodes():
state_init[node]='S'
state_init[failnode]='F'
nx.set_node_attributes(G, state_init, 'state')
#node_weight set, could use random as well
weightlist = [1,4,4,3.5,3,2,1.5,2,3,0.5,2.5,1,2,2,3,1,0.5,3.5,2,2,1,2,3,2,2,1,1,3.5]
node_weight = {}
for i in range(28):
node_weight[i] = weightlist[i]
nx.set_node_attributes(G, node_weight, 'weight')
#edge weight
edge_weight = {}
for edge in G.edges():
edge_weight[edge] = (G.node[edge[0]]['weight'] + G.node[edge[1]]['weight'])/2
nx.set_edge_attributes(G, edge_weight, 'weight')
#initial load
initial_load = {}
for node in G.nodes():
initial_load[node] = G.degree(node, weight = 'weight')**alpha
nx.set_node_attributes(G, initial_load, 'initial')
#initial capacity
node_capacity = {}
for node in G.nodes():
node_capacity[node] = (1+beta)*G.node[node]['initial']
nx.set_node_attributes(G, node_capacity, 'capacity')
#compute the node with highest degree, closeness and betweenness in project2
#sorted(nx.degree_centrality(G).items(),key=lambda item:item[1]) #17
#sorted(nx.betweenness_centrality(G,weight = 'weight').items(),key=lambda item:item[1]) #11
