def __init__(self, model_graph, node_sizes, node_cpds=[]):
"""
Initializes BNET object.
model_graph: Numpy array or Scipy.sparse matrix
A matrix defining the edges between nodes in the network. If
graph[i, j] = 1 there exists a directed edge from node i to j.
node_sizes: List or Int
A list of the possible number of values a discrete
node can have. If node_sizes[i] = 2, then the discrete node i
can have one of 2 possible values, such as True or False. If
this parameter is passed as an integer, it indicates that all
nodes have the size indicated by the integer.
node_cpds: List of CPD objects (cpds.py)
A list of the CPDs for each node in the BNET.
"""
"""Set the data members to the input values"""
self.model_graph = model_graph
self.num_nodes = model_graph.shape[0]
self.node_sizes = node_sizes.copy()
self.cpds = node_cpds
"""Convert the graph to a sparse matrix"""
if ((type(model_graph) == type(np.matrix([0]))) or
(type(model_graph) == type(np.array([0])))):
model_graph = sparse.lil_matrix(model_graph)
"""Obtain topological order"""
self.order = graph.topological_sort(self.model_graph)
def test_topological_sort(self):
"""
FUNCTION: topological_sort, in graph.py.
"""
"""Execute the function"""
order = graph.topological_sort(self.dag)
"""Assert that the functions output matches the expected output"""
assert order == [0, 1, 2, 3]
def test_topo_sort(self):
g = Graph(n_vertices=3, edges=[(1, 2)])
l = topological_sort(g)
self.verify_linearization(l, g)
g = Graph(n_vertices=5, edges=[(1, 2), (3, 1), (3, 4)])
l = topological_sort(g)
self.verify_linearization(l, g)
g = Graph(n_vertices=6,
edges=[(1, 3), (2, 1), (2, 4), (4, 3), (3, 6), (3, 5)])
l = topological_sort(g)
print(l)
self.verify_linearization(l, g)
# Raise exception if cycle
g = Graph(n_vertices=3, edges=[(1, 2), (2, 3), (3, 1)])
self.assertRaises(ValueError, topological_sort, g)
def test_topological_sort(self):
"""
FUNCTION: topological_sort, in graph.py.
"""
"""Execute the function"""
order = graph.topological_sort(self.dag)
"""Assert that the functions output matches the expected output"""
assert order == [0, 1, 2, 3]
def __init__(self,
adj_mat,
node_sizes,
discrete_nodes=None,
continuous_nodes=None,
node_cpds=None):
"""
Initializes BNET object.
adj_mat: Numpy array or Scipy.sparse matrix
A matrix defining the edges between nodes in the network. If
graph[i, j] = 1 there exists a directed edge from node i to j.
node_sizes: List or Int
A list of the possible number of values a discrete
node can have. If node_sizes[i] = 2, then the discrete node i
can have one of 2 possible values, such as True or False. If
this parameter is passed as an integer, it indicates that all
nodes have the size indicated by the integer.
node_cpds: List of CPD objects (node_cpds.py)
A list of the CPDs for each node in the BNET.
TODO:
- ability for different nodes to share the same CPD
"""
"""Set the data members to the input values"""
super(bnet, self).__init__(node_sizes, discrete_nodes,
continuous_nodes)
self.adj_mat = adj_mat
self.num_nodes = adj_mat.shape[0]
self.node_sizes = node_sizes.copy()
self.node_cpds_N = len(self.node_sizes)
if node_cpds is not None:
#self.node_cpds_N = len(set(node_cpds)) determine shared node_cpds using set
assert len(node_cpds)==self.node_cpds_N,\
"Number of node CPDs inconsistent with model"
self.node_cpds = np.array(node_cpds)
else:
self.node_cpds = np.array([], dtype=object)
self.node_cpds_meta = [None] * self.node_cpds_N
"""Convert the graph to a sparse matrix"""
if ((type(adj_mat) == type(np.matrix([0])))
or (type(adj_mat) == type(np.array([0])))):
adj_mat = sparse.lil_matrix(adj_mat)
"""Obtain topological order"""
self.order = graph.topological_sort(self.adj_mat)
def __init__(self, adj_mat, node_sizes, clqs, lattice=False):
"""
Initializes MRF object.
adj_mat: Numpy array or Scipy.sparse matrix
A matrix defining the edges between nodes in the network. If
graph[i, j] = 1 there exists a undirected edge from node i to j.
node_sizes: List or Int
A list of the possible number of values a discrete
node can have. If node_sizes[i] = 2, then the discrete node i
can have one of 2 possible values, such as True or False. If
this parameter is passed as an integer, it indicates that all
nodes have the size indicated by the integer.
clqs: List of clique objects (cliques.py)
A list of the cliques in the MRF.
lattice: Bool
Lattice is true if this MRF has a lattice graph structure, and
false otherwise.
"""
"""Assign the input values to their respective internal data members"""
self.lattice = lattice
self.num_nodes = adj_mat.shape[0]
self.cliques = clqs
self.node_sizes = np.array(node_sizes)
self.node_sizes_org = np.array(self.node_sizes)
"""Convert the graph to a sparse matrix"""
if ((type(adj_mat) == type(np.matrix([0]))) or
(type(adj_mat) == type(np.array([0])))):
adj_mat = sparse.lil_matrix(adj_mat)
"""In an MRF, all edges are bi-directional"""
# self.adj_mat = adj_mat - \
# sparse.lil_diags([sparse.extract_diagonal(\
# adj_mat)], [0], (adj_mat.shape[0], \
# adj_mat.shape[0]))\
# + adj_mat.T
self.adj_mat = adj_mat
"""
Obtain elimination order, which is just the input order in the case
of a lattice.
"""
if self.lattice == True:
self.order = range(0, self.adj_mat.shape[0])
else:
self.order = graph.topological_sort(self.adj_mat)
def __init__(self, adj_mat, node_sizes, clqs, lattice=False):
"""
Initializes MRF object.
adj_mat: Numpy array or Scipy.sparse matrix
A matrix defining the edges between nodes in the network. If
graph[i, j] = 1 there exists a undirected edge from node i to j.
node_sizes: List or Int
A list of the possible number of values a discrete
node can have. If node_sizes[i] = 2, then the discrete node i
can have one of 2 possible values, such as True or False. If
this parameter is passed as an integer, it indicates that all
nodes have the size indicated by the integer.
clqs: List of clique objects (cliques.py)
A list of the cliques in the MRF.
lattice: Bool
Lattice is true if this MRF has a lattice graph structure, and
false otherwise.
"""
"""Assign the input values to their respective internal data members"""
self.lattice = lattice
self.num_nodes = adj_mat.shape[0]
self.cliques = clqs
self.node_sizes = np.array(node_sizes)
self.node_sizes_org = np.array(self.node_sizes)
"""Convert the graph to a sparse matrix"""
if ((type(adj_mat) == type(np.matrix([0])))
or (type(adj_mat) == type(np.array([0])))):
adj_mat = sparse.lil_matrix(adj_mat)
"""In an MRF, all edges are bi-directional"""
# self.adj_mat = adj_mat - \
# sparse.lil_diags([sparse.extract_diagonal(\
# adj_mat)], [0], (adj_mat.shape[0], \
# adj_mat.shape[0]))\
# + adj_mat.T
self.adj_mat = adj_mat
"""
Obtain elimination order, which is just the input order in the case
of a lattice.
"""
if self.lattice == True:
self.order = range(0, self.adj_mat.shape[0])
else:
self.order = graph.topological_sort(self.adj_mat)
def __init__(self, adj_mat, node_sizes, discrete_nodes=None, continuous_nodes=None, node_cpds=None):
"""
Initializes BNET object.
adj_mat: Numpy array or Scipy.sparse matrix
A matrix defining the edges between nodes in the network. If
graph[i, j] = 1 there exists a directed edge from node i to j.
node_sizes: List or Int
A list of the possible number of values a discrete
node can have. If node_sizes[i] = 2, then the discrete node i
can have one of 2 possible values, such as True or False. If
this parameter is passed as an integer, it indicates that all
nodes have the size indicated by the integer.
node_cpds: List of CPD objects (node_cpds.py)
A list of the CPDs for each node in the BNET.
TODO:
- ability for different nodes to share the same CPD
"""
"""Set the data members to the input values"""
super(bnet, self).__init__(node_sizes, discrete_nodes, continuous_nodes)
self.adj_mat = adj_mat
self.num_nodes = adj_mat.shape[0]
self.node_sizes = node_sizes.copy()
self.node_cpds_N = len(self.node_sizes)
if node_cpds is not None:
#self.node_cpds_N = len(set(node_cpds)) determine shared node_cpds using set
assert len(node_cpds)==self.node_cpds_N,\
"Number of node CPDs inconsistent with model"
self.node_cpds = np.array(node_cpds)
else:
self.node_cpds = np.array([],dtype=object)
self.node_cpds_meta = [None]*self.node_cpds_N
"""Convert the graph to a sparse matrix"""
if ((type(adj_mat) == type(np.matrix([0]))) or
(type(adj_mat) == type(np.array([0])))):
adj_mat = sparse.lil_matrix(adj_mat)
"""Obtain topological order"""
self.order = graph.topological_sort(self.adj_mat)
def dag_shortest_path(self, g : _g.Graph, s : _g.Vertex):
'''
按顶点的拓扑序列对某加权dag图(有向无回路图)G=(V,E)的边进行松弛后
就可以在Θ(V+E)时间内计算出单源最短路径.
Args
===
`g` : 有向无回路图G=(V,E)
`s` : 源顶点
'''
sort_list = _g.topological_sort(g)
self.initialize_single_source(g, s)
for u in sort_list:
u = g.veterxs_atkey(u)
adj = g.getvertexadj(u)
for v in adj:
edge = g.getedge(u, v)
self.relax(u, v, edge.weight)
def main(filename):
with open(filename, 'r') as f:
edges = parse(get_lines(filename))
graph = make_graph(edges)
source = raw_input("Source vertex:")
while not in_graph(graph, source):
source = raw_input("Bad source\nSource vertex:")
vec, back_edges = topological_sort(graph)
negs = False
for _, _, w in edges:
if w < 0:
negs = True
cycle = len(back_edges) > 0
if not cycle:
print "Graph is a DAG, running DAG SP algorithm"
d, parent = dag_sp(graph, vec, source)
elif not negs:
print "Graph has no negative edges, running Dijkstra's algorithm"
d, parent = dijkstra(graph, source)
else:
print "Running Bellman-Ford algorithm"
d, parent = bellman_ford(graph, source)
if d == None: ## Sufficent to assume parent is also None
print "Graph contains a negative-weight cycle."
sys.exit(0)
while True:
dest = raw_input("Destination vertex:")
while not in_graph(graph, dest):
dest = raw_input("Bad destination\nDestination vertex:")
dist, order = get_shortest_path(d, parent, dest)
if dist >= INFINITY:
print "There is no way to reach the destination from the source"
else:
print "The distance between the vertices is %d" % dist
print "The shortest path is %s" % pprint_path(order)
def compute_p0_matrix_single_r(net, ncnodes, R, r_index):
# MATLAB: function p0matrix = compute_p0_matrix(net, ncnodes, R)
# Computes the loss probability p0 for every node and for every receiver
# p0(node, receiver) = the probability that a packet sent by node 'node' will not reach receiver 'receiver', even if it is duplicated along the way
# (i.e. neither the sent packet nor any of its possible duplicates won't reach the receiver)
#
# Inputs:
# net
# ncnodes = vector of NC nodes (to exclude form normal
# computations)
# Outputs:
# p0matrix = array holding the p0 value of every node, for every
# receiver
#
# Nicolae Cleju, EPFL, 2008/2009, TUIASI, 2009/2010
#==========================================================================
#-------------------------------------------
# Init
#-------------------------------------------
# Total number of nodes
#N = size(net.capacities,1);
N = net['capacities'].shape[0]
# Number of input and output packets
#b_i = sum(net.capacities .* (1-net.errorrates), 1);
#b_o = sum(net.capacities, 2)';
b_i = numpy.sum(net['capacities'] * (1 - net['errorrates']), 0)
b_o = numpy.sum(net['capacities'], 1)
# Hold the transpose of capacities for faster access
capT = net['capacities'].T
# The topological order
#order = topological_order(sparse(net.capacities));
#xs, ys = numpy.nonzero(net['capacities'])
#order = graph.topological_sort([(xs[i], ys[i]) for i in range(xs.size)])
order = graph.topological_sort(net['capacities'])
# Init output matrix
#p0matrix = zeros(N, numel(net.receivers));
#p0matrix = zeros(N, 1);
#p0matrix = numpy.zeros(N, net['rceivers'].size)
p0matrix = numpy.zeros(N)
# Holds floor(replication rate)
#M = zeros(1,N);
M = numpy.zeros(N)
# Holds the probability that an input packet is replicated M times in node n
#p_rep_M = zeros(1,N);
p_rep_M = numpy.zeros(N)
# Holds the probability that an input packet is not lost in a buffer overflow in node n
#p_ov = zeros(1,N);
p_ov = numpy.zeros(N)
# for each client c seperately
#for r_index = 1:numel(net.receivers)
#r = net.receivers(r_index);
#for r_index in range(net['receivers'].size):
r = net['receivers'][r_index]
# Prepare data
# For every node in topo order
#for n = order'
for n in order:
# If not source
#if b_i(n) ~= 0
if b_i[n] != 0:
# M = floor(replication rate)
#M(n) = floor(R(r_index,n));
M[n] = math.floor(R[r_index,n])
# Probability of M(n) - for linear interpolation
#p_rep_M(n) = 1 - ( R(r_index,n) - M(n) );
p_rep_M[n] = 1. - ( R[r_index,n] - M[n] )
# Probability of not buffer overflow
#if b_o(n) > b_i(n)
if b_o[n] > b_i[n]:
#p_ov(n) = 0; # No overflow
p_ov[n] = 0.
else:
#p_ov(n) = 1 - b_o(n) /b_i(n) ; # Buffer overflow
p_ov[n] = 1. - b_o[n] / b_i[n]
#end
#end
#end
# restrict list only to nodes ahead of the receiver (without the receiver itself)
#partial_limit = find(order == r, 1) - 1;
partial_limit = numpy.nonzero(order == r)[0] - 1;
# for the receiver, initialize p0 to 0
#p0matrix(r, r_index) = 0; #0# losses, 100# one copy
#p0matrix(r) = 0; %0% losses, 100% one copy
#p0matrix[r, r_index] = 0.
p0matrix[r] = 0. #0% losses, 100% one copy
# for all nodes after the receiver (which cannot reach the receiver) set the p0 to 1 (100# loss)
#for n_index = (find(order == r, 1) + 1) : N
for n_index in range((numpy.nonzero(order == r)[0] + 1), N):
#n = order(n_index);
n = order[n_index]
#p0matrix(n, r_index) = 1;
#p0matrix(n) = 1;
#p0matrix[n, r_index] = 1.
p0matrix[n] = 1.
#end
# go only from the receiver r backwards
#for u_index = partial_limit:-1:1
for u_index in range(partial_limit,-1,-1): # need to include the end, 0
#u = order(u_index);
u = order[u_index]
# find child nodes of node u
#children = find(net.capacities(u,:));
#children = find(capT(:,u));
#children = numpy.nonzero(net['capacities'][u,:])
children = numpy.nonzero(capT[:,u])[0]
# if u has no children, set p0 to 1 (100# loss)
# this may happen if u is another receiver sitting before the
# current one in the topo order
#if (isempty(children))
if children.size == 0:
#p0matrix(u,r_index) = 1;
#p0matrix(u) = 1;
#p0matrix[u,r_index] = 1.
p0matrix[u] = 1.
else:
# initialize sum of children probabilities
#p0_partsum = 0;
p0_partsum = 0.
# for each child
#for child_index = 1:numel(children)
for child_index in range(children.size):
#child = children(child_index);
child = children[child_index]
# initialize p0 of u through this child
#p0_child = 0;
p0_child = 0.
# probability that a packet from node 'u' is sent on the link to child 'child'
#rho = net.capacities(u,child) / b_o(u);
rho = net['capacities'][u,child] / b_o[u]
#------------------------------------------------------------------
# if child is receiver, loss prob = rho * errorrate
#if child == r
if child == r:
#p0_child = rho * net.errorrates(u,child);
p0_child = rho * net['errorrates'][u,child]
else:
# child is not the receiver
# make sure child is not a NC node (not in ncnodes)
#if any(ncnodes == child)
#if child == ncnodes
#if numpy.any(ncnodes == child):
#if ncnodes.contains(child):
if child in ncnodes:
# child is NC node, make p0 equal to 1, because we exclude paths which go through NC nodes
#p0_child = rho * 1; # 100# losses
p0_child = rho * 1. # 100# losses
else:
# child is not a NC node, process it
# p0 = rho * (probab. lost on the link from u to child +
# + not lost on the link. but lost in a buffer overflow in child
# + not lost on the link, survives buffer overflow, is replicated M times and all M+1 copies are lost after the child node
# or is replicated M+1 times and all M+2 copies are lost after the chlid node)
# if is replicating node, no chance of buffer
# overflow
# if new_p_ov(child) == 1
# p0_child = rho * (...
# errorrates(u,child) + ...
# (1-errorrates(u,child)) * ( new_p_rep_M(child)*p0(child, r_index)^(new_M(child)) + (1-new_p_rep_M(child))*p0(child, r_index)^(new_M(child) + 1)));
# else
# p0_child = rho * (...
# errorrates(u,child) + ...
# (1-errorrates(u,child))*(1-new_p_ov(child)) + ...
# (1-errorrates(u,child)) * new_p_ov(child) * p0(child, r_index));
# end
#pi = net.errorrates(u,child);
pi = net['errorrates'][u,child]
#Beta = p_ov(child);
Beta = p_ov[child]
#exponent = R(r_index, child);
exponent = R[r_index, child]
#if exponent < 1 exponent = 1; end
if exponent < 1:
exponent = 1
#p0_child = rho * (pi + (1-pi)*Beta + (1-pi)*(1-Beta)* p0matrix(child,r_index)^exponent);
#p0_child = rho * (pi + (1-pi)*Beta + (1-pi)*(1-Beta)* p0matrix(child)^exponent);
#p0_child = rho * (pi + (1-pi)*Beta + (1-pi)*(1-Beta)* p0matrix[child,r_index]**exponent)
p0_child = rho * (pi + (1-pi)*Beta + (1-pi)*(1-Beta)* p0matrix[child]**exponent)
# Sanity check
# Comment this to increase speed
#assert(~isnan(p0_child));
#end
#end
# add p0 to partial sum
#p0_partsum = p0_partsum + p0_child;
p0_partsum = p0_partsum + p0_child
#end
# check 0 < p0 < 1, allow for some margin of numerical error
#if p0_partsum < -1e-6 || p0_partsum > (1 + 1e-6)
if p0_partsum < -1e-6 or p0_partsum > (1 + 1e-6):
#disp(['p0 = ' num2str(p0_partsum)]);
print('p0 = ' + str(p0_partsum))
#error('Error: p0 not ok!');
raise
#end
# round small numerical errors
#if -1e-6 < p0_partsum && p0_partsum < 0
if -1e-6 < p0_partsum and p0_partsum < 0:
#p0_partsum = 0;
p0_partsum = 0.
#end
#if 1 < p0_partsum && p0_partsum < (1+(1e-6))
if 1 < p0_partsum and p0_partsum < (1+(1e-6)):
#p0_partsum = 1;
p0_partsum = 1.
#end
# Save final probability density function
#p0matrix(u,r_index) = p0_partsum;
#p0matrix(u) = p0_partsum;
#p0matrix[u,r_index] = p0_partsum
p0matrix[u] = p0_partsum
#end
#end
#end
return p0matrix
#!/usr/bin/env python from input import get_lines, parse from graph import make_graph, topological_sort import sys from os.path import isfile filename = sys.argv[1] if not isfile(filename): exit(1) lines = get_lines(filename) edges = parse(lines) graph = make_graph(edges) path, back_edges = topological_sort(graph) if not back_edges: print "The path is: " + str(path) else: print "The graph is cyclic. Back edges: " + str(back_edges)
from input import get_lines, parse
from graph import make_graph, topological_sort
from sp import dag_sp, bellman_ford, dijkstra, get_shortest_path
edges = parse(get_lines("dag.txt"))
graph = make_graph(edges)
vec, backs = topological_sort(graph)
negs = False
for _, _, w in edges:
if w < 0:
negs = True
cycle = len(backs) > 0
if negs:
print "Negatives"
if cycle:
print "Cyclic"
print backs
dag_sol = dag_sp(graph, vec, '1')
bf_sol = bellman_ford(graph, '1')
dijk_sol = dijkstra(graph, '1')
print dag_sol
print bf_sol
print dijk_sol
d, parent = dag_sol
print get_shortest_path(d, parent, '5')
def make_node_silent(net,node):
#MATLAB function newnet = make_node_silent(net, node)
# Generates a new net structure by removing all outgoing links of node 'node' and then adjusting all following links of the graphs as to preserve
# the same replication rate of all nodes
# The replication rate of all nodes is thus preserved in the new net (except of course node 'node' whose outgoing links are removed)
#
# Inputs:
# net = network configuration structure, with the fields:
# capacities = capacities matrix, in packets (A(i,j) = x means there is an edge from node i to node j of capacity x packets)
# errorrates = error rates matrix
# sources = vector containing the source nodes
# helpers = vector containing the helper nodes (helper = not source and not client)
# receivers = vector containing the client (receiver) nodes
# node = the node whose outgoing links are to be removed
#
# Outputs:
# newnet = the new network configuration structure, containing the same fields as the input one
#
# Nicolae Cleju, EPFL, 2008/2009, TUIASI, 2009/2010
#==========================================================================
# Create output structure
#newnet = net;
#newnet = net.copy()
# BUG: dict.copy() is shallow copy, so the underlying arrays were not copied
# use copy.deepcopy() instead
newnet = copy.deepcopy(net)
# Compute the original NI and NO
#NI = sum(net.capacities .* (1-net.errorrates), 1);
#NO = sum(net.capacities, 2);
NI = numpy.sum(net['capacities'] * (1.-net['errorrates']), 0)
NO = numpy.sum(net['capacities'], 1)
# Compute the original Nrepl
#Nrepl = NO' - NI;
#Nrepl(Nrepl < 0) = 0;
#Nrepl(net.sources) = 0;
Nrepl = NO - NI
Nrepl[Nrepl < 0] = 0
Nrepl[net['sources']] = 0
# Compute topological order
#order = topological_order(sparse(net.capacities));
#xs, ys = numpy.nonzero(net['capacities'])
#order = graph.topological_sort([(xs[i], ys[i]) for i in range(xs.size)])
order = graph.topological_sort(net['capacities'])
# remove node's outgoing links
#newnet.capacities(node, :) = zeros(1, newnet.nnodes);
#newnet.errorrates(node, :) = zeros(1, newnet.nnodes);
newnet['capacities'][node, :] = numpy.zeros(newnet['nnodes'])
newnet['errorrates'][node, :] = numpy.zeros(newnet['nnodes'])
# For all following nodes in the topo order, adjust outgoing links to maintain the original replication rate
# generate partial list with all nodes following 'node' in the topo order,
#partial_list = order( (find(order == node, 1) + 1) : numel(order) );
partial_list = order[ (numpy.nonzero(order == node)[0] + 1) : ]
#newNO = NO;
#newNI = NI;
newNO = NO.copy()
newNI = NI.copy()
#for n_index = 1:numel(partial_list)
for n_index in range(partial_list.size ):
#n = partial_list(n_index);
n = partial_list[n_index]
# compute the new NI
#newNI(n) = sum(newnet.capacities(:,n) .* (1-newnet.errorrates(:,n)));
newNI[n] = numpy.sum(newnet['capacities'][:,n] * (1-newnet['errorrates'][:,n]))
# if new NI < original NI, adjust outgoing links with the same factor
#if (newNI(n) < NI(n))
if newNI[n] < NI[n]:
#newnet.capacities(n,:) = (newNI(n)/NI(n)) * newnet.capacities(n,:);
newnet['capacities'][n,:] = (newNI[n]/NI[n]) * newnet['capacities'][n,:]
#end
# compute the new NO
#newNO(n) = sum(newnet.capacities(n,:));
newNO[n] = numpy.sum(newnet['capacities'][n,:])
#end
# checking
#f1 = (NI ./ NO');
f1 = NI / NO
#f1(isinf(f1)) = 0;
f1[numpy.isinf(f1)] = 0
#f1(isnan(f1)) = 0;
f1[numpy.isnan(f1)] = 0
#f2 = (newNI ./ newNO');
f2 = newNI / newNO
#f2(isinf(f2)) = 0;
f2[numpy.isinf(f2)] = 0
#f2(isnan(f2)) = f1(isnan(f2));
f2[numpy.isnan(f2)] = f1[numpy.isnan(f2)]
#diff = f1 - f2;
diff = f1 - f2
#assert(all(diff.^2 < 0.001));
assert(numpy.all(diff**2 < 0.001))
return newnet
import re
from graph import Graph, is_dag, topological_sort, find_longest_path_distance
n_vertices = None
edges = []
print("Specify a graph. (A blank line stops input)")
while True:
value = input()
# Stop input line only contains white space
if re.match('^\s*$', value):
break
if n_vertices is None:
n_vertices = int(value)
else:
u, v = map(int, value.split(','))
edges.append((u, v))
graph = Graph(n_vertices, edges)
print('YES') if is_dag(graph) else print('NO')
print(*topological_sort(graph))
print(find_longest_path_distance(graph, 1))
def convert_subgraph_to_module(graph, full_graph, num_subgraphs, module_name,
initialize_weights, model_template_filename,
output_filename):
model_template = open(model_template_filename, 'r').read()
nodes = graph.topological_sort()
f1 = "../../profiler/image_classification/models/language_modelling"
f2 = "../../profiler/image_classification/models/ImageNet"
import_statements = [
f"import sys \nsys.path.insert(0, \"{f1}\")\nimport transformer as transformer_layers\n \nsys.path.insert(0, \"{f2}\")\nimport resnet\n"
]
module_methods = []
counter = 0
layer_names = {}
layer_names_and_declarations = []
function_definition = []
input_names = get_input_names(graph, full_graph)
num_inputs = len(input_names)
output_names = input_names.copy()
sources = graph.sources()
# Now, generate expressions for each node.
# Iterate through nodes in topological order, and add output_name mappings for
# each expression. Use this output_name mapping when generating expressions
# in the model's implementation file.
# TODO: Make sure that nodes with multiple inputs have the inputs in the
# right order (even though this probably does not matter in practice).
for node_id in input_names:
output_name = "out%d" % counter
function_definition.append("%s = %s.clone()" %
(output_name, input_names[node_id]))
output_names[node_id] = output_name
counter += 1
for node in nodes:
layer_call = None
layer_name = "self.layer%d" % counter
output_name = "out%d" % counter
# layer_declaration = "torch.nn.%s" % (
# node.node_desc.replace("inplace", "inplace=True"))
print("=== ", node.node_desc[:node.node_desc.find('(')])
in_resnet = hasattr(resnet, node.node_desc[:node.node_desc.find('(')])
if in_resnet:
layer_declaration = "resnet.%s" % (node.node_desc)
print("found ", node.node_desc[:node.node_desc.find('(')],
"in resnet")
elif hasattr(torch.nn, node.node_desc[:node.node_desc.find('(')]):
layer_declaration = "torch.nn.%s" % (node.node_desc)
print("found in torch.nn")
elif hasattr(transformer_layers,
node.node_desc[:node.node_desc.find('(')]):
layer_declaration = "transformer_layers.%s" % (node.node_desc)
print("found in transformer layers")
elif node.node_desc.startswith("Input"):
layer_declaration = "torch.nn.%s" % (node.node_desc)
else:
print("not found ", node.node_desc[:node.node_desc.find('(')],
"anywhere")
print(node.node_desc)
exit(-1)
layer_names[node.node_id] = layer_name
if node.node_id not in output_names:
output_names[node.node_id] = output_name
# Skip layers that don't need a declaration (example: '+=').
for declaration in declaration_specialcase:
if node.node_desc.startswith(declaration):
found = True
if declaration == "EmuBidirLSTM":
m = re.search(r'.*LSTM\((\d+), (\d+)\).*', node.node_desc)
input_size = int(m.group(1))
hidden_size = int(m.group(2))
layer_declaration = "EmuBidirLSTM(%d, %d)" % (input_size,
hidden_size)
import_statements.append(
"from seq2seq.models.encoder import EmuBidirLSTM")
elif declaration == "RecurrentAttention":
m = re.search(r'.*LSTM\((\d+), (\d+)\).*', node.node_desc)
input_size = int(m.group(1))
hidden_size = int(m.group(2))
m = re.search(r'.*in_features=(\d+), out_features=(\d+).*',
node.node_desc)
context_size = int(m.group(1))
layer_declaration = "RecurrentAttention(%d, %d, %d)" % (
input_size, hidden_size, context_size)
import_statements.append(
"from seq2seq.models.decoder import RecurrentAttention"
)
elif declaration == "Classifier":
m = re.search(r'.*in_features=(\d+), out_features=(\d+).*',
node.node_desc)
in_features = int(m.group(1))
out_features = int(m.group(2))
layer_declaration = "Classifier(%d, %d)" % (in_features,
out_features)
import_statements.append(
"from seq2seq.models.decoder import Classifier")
elif declaration == "MaskConv":
node_desc = node.node_desc
modules = node_desc.split(" ")[1:-1]
module_declarations = []
for module in modules:
module_declaration = "torch.nn." + module.split(
": ")[1].replace("inplace", "inplace=True")
module_declarations.append(module_declaration)
layer_declaration = "MaskConv(torch.nn.Sequential(%s))" % ",\n ".join(
module_declarations)
import_statements.append("from model import MaskConv")
module_methods.append(
"""def get_seq_lens(self, input_length):
seq_len = input_length
for m in %s.modules():
if type(m) == torch.nn.modules.conv.Conv2d:
seq_len = ((seq_len + 2 * m.padding[1] - m.dilation[1] * (m.kernel_size[1] - 1) - 1) / m.stride[1] + 1)
return seq_len.int()""" % layer_name)
elif declaration == "BatchRNN":
if "batch_norm" in node.node_desc:
batch_norm = True
else:
batch_norm = False
if "LSTM" in node.node_desc:
rnn_type = "torch.nn.LSTM"
m = re.search(
r'LSTM\((\d+), (\d+), bidirectional=([a-zA-Z]+)\)',
node.node_desc)
input_size = int(m.group(1))
hidden_size = int(m.group(2))
bidirectional = m.group(3)
elif "GRU" in node.node_desc:
rnn_type = "torch.nn.GRU"
m = re.search(
r'GRU\((\d+), (\d+), bidirectional=([a-zA-Z]+)\)',
node.node_desc)
input_size = int(m.group(1))
hidden_size = int(m.group(2))
bidirectional = m.group(3)
else:
# TODO: Do something else?
pass
# TODO: Pass remaining arguments.
# TODO: Get hidden and input size.
layer_declaration = "BatchRNN(%d, %d, rnn_type=%s, batch_norm=%s, bidirectional=%s)" % (
input_size, hidden_size, rnn_type, batch_norm,
bidirectional)
import_statements.append("from model import BatchRNN")
elif declaration == "ResizeInput":
layer_declaration = "ResizeInput()"
import_statements.append("from model import ResizeInput")
elif declaration == "SequenceWise":
node_desc = node.node_desc
modules = node_desc[:-2].split(" ")[1:]
module_declarations = []
for module in modules:
module_declaration = "torch.nn." + module.split(
": ")[1].replace("inplace", "inplace=True")
module_declarations.append(module_declaration)
layer_declaration = "SequenceWise(torch.nn.Sequential(%s))" % ",\n ".join(
module_declarations)
import_statements.append("from model import SequenceWise")
elif declaration == "InferenceBatchSoftmax":
layer_declaration = "InferenceBatchSoftmax()"
import_statements.append(
"from model import InferenceBatchSoftmax")
break
import_statements = list(set(import_statements))
found = False
for declaration in declaration_whitelist:
if node.node_desc.startswith(declaration):
found = True
if not found:
layer_names_and_declarations.append(
(layer_name, layer_declaration))
if node.node_id in full_graph.in_edges:
in_edges = full_graph.in_edges[node.node_id]
else:
in_edges = []
if len(in_edges) == 0 and node.node_desc.startswith("Input"):
pass # Don't need to do anything for this case.
else:
if node.node_desc.startswith("Size"):
assert (len(in_edges) == 1)
m = re.search(r'Size\((-?\d+)\)', node.node_desc)
idx = int(m.group(1))
layer_call = "%s = %s.size(%d)" % (
output_name, output_names[in_edges[0].node_id], idx)
elif node.node_desc.startswith("View"):
size_node_ids = []
input_node_id = None
for i in range(len(in_edges)):
if in_edges[i].node_desc.startswith("Size"):
size_node_id = in_edges[i].node_id
size_node_ids.append(size_node_id)
else:
input_node_id = in_edges[i].node_id
m = re.search(r'View\((-?\d+)\)', node.node_desc)
if m is None:
size_output_names = [
output_names[size_node_id]
for size_node_id in size_node_ids
]
layer_call = "%s = %s.view(%s)" % (
output_name, output_names[input_node_id],
", ".join(size_output_names))
else:
size = int(m.group(1))
layer_call = "%s = %s.view(%s, %d)" % (
output_name, output_names[input_node_id],
output_names[size_node_id], size)
elif node.node_desc.startswith("__getitem__"):
assert (len(in_edges) == 1)
m = re.search(r'__getitem__\((\d+)\)', node.node_desc)
idx = int(m.group(1))
if "hidden" in in_edges[0].node_desc:
layer_call = "%s = None" % output_name
else:
layer_call = "%s = %s[%d]" % (
output_name, output_names[in_edges[0].node_id], idx)
elif node.node_desc.startswith("Add"):
assert (len(in_edges) == 2)
node1 = in_edges[0]
node2 = in_edges[1]
if len(full_graph.edges[node1.node_id]) > 1:
tmp = node1
node1 = node2
node2 = tmp
layer_call = "%s = %s + %s" % (output_names[node1.node_id],
output_names[node1.node_id],
output_names[node2.node_id])
output_names[node.node_id] = output_names[node1.node_id]
elif node.node_desc.startswith("Mul"):
assert (len(in_edges) == 2)
node1 = in_edges[0]
node2 = in_edges[1]
if len(full_graph.edges[node1.node_id]) > 1:
tmp = node1
node1 = node2
node2 = tmp
layer_call = "%s = %s * %s" % (output_names[node1.node_id],
output_names[node1.node_id],
output_names[node2.node_id])
output_names[node.node_id] = output_names[node1.node_id]
elif node.node_desc.startswith("Concat"):
m = re.search(r'Concat\((-?\d+)\)', node.node_desc)
dim = int(m.group(1))
layer_call = "%s = torch.cat([%s], %d)" % (
output_name, ", ".join([
output_names[in_node.node_id] for in_node in in_edges
]), dim)
elif node.node_desc.startswith("Transpose"):
m = re.search(r'Transpose\((.+)\)', node.node_desc)
args = m.group(1)
assert (len(in_edges) == 1)
node1 = in_edges[0]
layer_call = "%s = %s.transpose(%s)" % (
output_name, output_names[node1.node_id], args)
elif node.node_desc.startswith("hidden"):
pass
elif node.node_desc == "self.get_seq_lens":
assert (len(in_edges) == 1)
in_node = in_edges[0]
layer_call = "%s = %s(%s)" % (output_name, node.node_desc,
output_names[in_node.node_id])
else:
layer_call = "%s = %s(%s)" % (output_name, layer_name,
", ".join([
output_names[in_node.node_id]
for in_node in in_edges
]))
if layer_call is not None:
function_definition.append(layer_call)
counter += 1
# Ensure that outputs of a module are returned in the same order as
# the original model implementation.
# TODO: This might not work as intended for sub-graphs.
full_graph.populate_depths()
graph_output_names, _ = get_output_names(graph, full_graph, 0)
for key in graph_output_names:
graph_output_names[key] = output_names[key]
output_names_list = get_tensor_names_list(graph_output_names)
num_outputs = len(output_names_list)
function_definition.append("return %s" %
get_output_tuple_str(output_names_list))
# Layer declarations are added to the constructor of the module.
# Function definitions are added to the `forward()' method of the
# module.
layer_declarations_str = "\n ".join(
["%s = %s" % (x[0], x[1]) for x in layer_names_and_declarations])
if initialize_weights:
layer_declarations_str += "\n self._initialize_weights()"
module_methods.append("""def _initialize_weights(self):
for m in self.modules():
if isinstance(m, torch.nn.Conv2d):
torch.nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')
if m.bias is not None:
torch.nn.init.constant_(m.bias, 0)
elif isinstance(m, torch.nn.BatchNorm2d):
torch.nn.init.constant_(m.weight, 1)
torch.nn.init.constant_(m.bias, 0)
elif isinstance(m, torch.nn.Linear):
torch.nn.init.normal_(m.weight, 0, 0.01)
torch.nn.init.constant_(m.bias, 0)""")
function_definition_str = "\n ".join(function_definition)
input_names_list = get_tensor_names_list(input_names)
input_names = ", ".join(input_names_list)
model = model_template % {
"layer_declarations": layer_declarations_str,
"function_definition": function_definition_str,
"module_name": module_name,
"inputs": input_names,
"import_statements": "\n".join(import_statements),
"module_methods": "\n\n".join(module_methods)
}
with open(output_filename, 'w') as f:
f.write(model)
return num_inputs, num_outputs
def Algo1_delay_computation(net, sim, ncnodes, prev_estim):
#MATLAB function tc = Algo1_delay_computation(net, sim, ncnodes, prev_estim)
##compute_p0_matrix = @compute_p0_matrix;
##compute_p0_matrix = @compute_p0_matrix_optimized;
# Precision for iterations
#precision = 5e-2;
precision = 5e-2
# Number of nodes
#N = size(net.capacities,1);
N = net['capacities'].shape[0]
# Nodes' input capacities
#b_i = sum(net.capacities .* (1-net.errorrates), 1);
b_i = numpy.sum(net['capacities'] * (1-net['errorrates']), 0)
# Nodes' output capacities
#b_o = sum(net.capacities, 2)';
b_o = numpy.sum(net['capacities'], 1)
# Generation size
#gensize = sim.N;
gensize = sim['N']
# Initialize replication rates
# Careful to avoid NaN = 0/0
#R = zeros(1, numel(b_i));
R = numpy.zeros(b_i.size)
#R(b_i ~= 0) = b_o(b_i ~= 0) ./ b_i(b_i~=0);
R[b_i!=0] = b_o[b_i != 0] / b_i[b_i!=0]
#R = repmat(R, numel(net.receivers), 1);
R = numpy.tile(R, (net['receivers'].size, 1))
#Rinit = R;
Rinit = R.copy()
# Better estimate replication rates if previous delay estimates available
#if ~isempty(prev_estim)
#if prev_estim.size != 0:
if prev_estim:
#p0matrix_prevestim = compute_p0_matrix(net, prev_estim.ncnodes, R);
p0matrix_prevestim = computings.compute_p0_matrix(net, prev_estim['ncnodes'], R)
# Update R
#R = update_R(R, net, p0matrix_prevestim, prev_estim.ncnodes, prev_estim.tc);
R = updateR.update_R(R, net, p0matrix_prevestim, prev_estim['ncnodes'], prev_estim['tc'])
else:
#p0matrix_prevestim = compute_p0_matrix(net, [], R);
p0matrix_prevestim = computings.compute_p0_matrix(net, numpy.array([]), R)
#end
# Nic: Python says: init tc
tc = numpy.Inf * numpy.ones(net['receivers'].size)
# Repeat until converged
#converged_tc = false;
converged_tc = False
#prev_tc = Inf * ones(1, numel(net.receivers));
prev_tc = numpy.Inf * numpy.ones(net['receivers'].size)
#max_tc = zeros(1, numel(net.receivers));
max_tc = numpy.zeros(net['receivers'].size)
#niter = 0;
niter = 0
#maxiter = 50;
maxiter = 50
#while ~converged_tc && niter < maxiter
while (not converged_tc) and (niter < maxiter):
# Compute p0 matrix for all clients at once
#p0matrix_allncnodes = compute_p0_matrix(net, ncnodes, R);
p0matrix_allncnodes = computings.compute_p0_matrix(net, ncnodes, R)
# Repeat for each client r
#for r_idx = 1:numel(net.receivers)
for r_idx in range(net['receivers'].size):
#r = net.receivers(r_idx);
#r = net['receivers'][r_idx]
#Nc = zeros(1, N);
Nc = numpy.zeros(N)
#Nc(net.sources) = b_o(net.sources)' .* (1 - p0matrix_allncnodes(net.sources, r_idx));
Nc[net['sources']] = b_o[net['sources']] * (1 - p0matrix_allncnodes[net['sources'], r_idx])
# For each NC node u
# Go in topo order
#order = topological_order(sparse(net.capacities));
#xs, ys = numpy.nonzero(net['capacities'])
#order = graph.topological_sort([(xs[i], ys[i]) for i in range(xs.size)])
order = graph.topological_sort(net['capacities'])
# is member returns a logical array of size(order) with 1 on the
# positions where the node is in ncnodes
#ncnodes_topo = order(ismember(order, ncnodes));
ncnodes_topo = order[numpy.in1d(order,ncnodes)]
# For each NC node u
#for u_idx = 1:numel(ncnodes_topo)
for u_idx in range(ncnodes_topo.size):
#u = ncnodes_topo(u_idx);
u = ncnodes_topo[u_idx]
# Compute p0 matrix with only the previous NC ndoes
##p0matrix_prevNC = compute_p0_matrix(net, ncnodes_topo(1:u_idx-1), R);
#p0matrix_prevNC_single_r = compute_p0_matrix_single_r(net, ncnodes_topo(1:u_idx-1), R, r_idx);
p0matrix_prevNC_single_r = computings.compute_p0_matrix_single_r(net, ncnodes_topo[:u_idx], R, r_idx)
# Compute t_c(u) with u SF
##tc1 = compute_tc(net, sim, ncnodes_topo(1:u_idx-1), Nc, p0matrix_prevNC(:,r_idx));
#tc1 = compute_tc(net, sim, ncnodes_topo(1:u_idx-1), Nc, p0matrix_prevNC_single_r);
tc1 = computings.compute_tc(net, sim, ncnodes_topo[:u_idx], Nc, p0matrix_prevNC_single_r)
# Make node u silent
#silent_net = make_node_silent(net,u);
silent_net = dist_specific.make_node_silent(net,u)
# Compute p0 matrix for silent network
##p0matrix_silent = compute_p0_matrix(silent_net, ncnodes_topo(1:u_idx-1), R);
#p0matrix_silent_single_r = compute_p0_matrix_single_r(silent_net, ncnodes_topo(1:u_idx-1), R, r_idx);
p0matrix_silent_single_r = computings.compute_p0_matrix_single_r(silent_net, ncnodes_topo[:u_idx], R, r_idx)
# Compute t_c(u) with u silent
##tc2 = compute_tc(silent_net, sim, ncnodes_topo(1:u_idx-1), Nc, p0matrix_silent(:,r_idx));
#tc2 = compute_tc(silent_net, sim, ncnodes_topo(1:u_idx-1), Nc, p0matrix_silent_single_r);
tc2 = computings.compute_tc(silent_net, sim, ncnodes_topo[:u_idx], Nc, p0matrix_silent_single_r)
#delta_tc = tc2 - tc1;
#delta_tc = tc2 - tc1
#delta_Nc = gensize * (1/tc1 - 1/tc2);
delta_Nc = gensize * (1./tc1 - 1./tc2)
#sanity check
#if delta_Nc < 0
if delta_Nc < 0:
#delta_Nc = 0;
delta_Nc = 0.
#end
# Compute Nc(u) using equation 8
# sanity check (avoid division by 0)
#if abs( p0matrix_prevNC(u,r_idx) - 1) < 1e-6
if abs( p0matrix_prevNC_single_r[u] - 1) < 1e-6:
#Nc(u) = 0;
Nc[u] = 0
else:
#if b_o(u) > b_i(u)
if b_o[u] > b_i[u]:
##Nc(u) = delta_Nc / (1 - p0matrix_prevNC(u,r_idx)^R(r_idx,u));
#Nc(u) = delta_Nc / (1 - p0matrix_prevNC_single_r(u)^R(r_idx,u));
Nc[u] = delta_Nc / (1. - p0matrix_prevNC_single_r[u]**R[r_idx,u])
else:
##Nc(u) = delta_Nc / ( b_o(u) / b_i(u) * (1 - p0matrix_prevNC(u,r_idx)));
#Nc(u) = delta_Nc / ( b_o(u) / b_i(u) * (1 - p0matrix_prevNC_single_r(u)));
Nc[u] = delta_Nc / ( b_o[u] / b_i[u] * (1. - p0matrix_prevNC_single_r[u]))
#end
#end
# sanity check
#if abs(Nc(u)) < 1e-6
if abs(Nc[u]) < 1e-6:
#Nc(u) = 0;
Nc[u] = 0
#end
#end
# Compute the average decoding delay tc considering all sources and
# NC nodes simultaneously with Eq. (16)
#tc(r_idx) = compute_tc(net, sim, ncnodes, Nc, p0matrix_allncnodes(:,r_idx));
tc[r_idx] = computings.compute_tc(net, sim, ncnodes, Nc, p0matrix_allncnodes[:,r_idx])
#end
# Update R
##R1 = update_R(R, net, p0matrix_allncnodes, ncnodes, tc);
##R = update_R_vectorized(R, net, p0matrix_allncnodes, ncnodes, tc);
##R = update_R_vectorized(Rinit, net, p0matrix_allncnodes, ncnodes, tc);
#R = update_R_vectorized(Rinit, net, p0matrix_prevestim, ncnodes, tc);
R = updateR.update_R_vectorized(Rinit, net, p0matrix_prevestim, ncnodes, tc)
##assert(norm(R1 - R2) < 1e-6);
##disp(['Average tc = ' num2str(mean(tc))]);
# If tc has Inf values (this may happen for distrib algorithm), ignore
# them
#if norm(tc(isfinite(tc)) - prev_tc(isfinite(tc))) <= precision * norm(tc(isfinite(tc)))
if numpy.linalg.norm(tc[numpy.isfinite(tc)] - prev_tc[numpy.isfinite(tc)]) <= precision * numpy.linalg.norm(tc[numpy.isfinite(tc)]):
#converged_tc = true;
converged_tc = True
#end
#prev_tc = tc;
prev_tc = tc.copy()
# Save max tc
#max_tc(tc > max_tc) = tc(tc > max_tc);
max_tc[tc > max_tc] = tc[tc > max_tc]
#niter = niter + 1;
niter = niter + 1
#end
# Check if converged or not
#if niter == maxiter
if niter == maxiter:
#tc = Inf * ones(1, numel(net.receivers)); # not converged
#If not converged, don't put Inf, put largest values instead
#tc = max_tc;
tc = max_tc.copy()
#end
return tc
