def generate_data_UE(data=None,
export=False,
SO=False,
demand=3,
m=20,
withODs=False,
NLP=122):
path = 'los_angeles_data_2.mat'
graph, x_true, l, path_wps, wp_trajs, obs, wp = synthetic_data(data,
SO,
demand,
m=m,
path=path,
fast=False)
x_true = array(x_true)
obs = obs[0]
A_full = path_solver.linkpath_incidence(graph)
A = to_sp(A_full[obs, :])
A_full = to_sp(A_full)
U, f = WP.simplex(graph, wp_trajs, withODs)
T, d = path_solver.simplex(graph)
# cell locations
cell_pos = array(wp.wp.values())
data = {
'A_full': A_full,
'b_full': A_full.dot(x_true),
'A': A,
'b': A.dot(x_true),
'x_true': x_true,
'T': to_sp(T),
'd': array(d),
'U': to_sp(U),
'f': array(f),
'cell_pos': cell_pos
}
if NLP is not None:
lp = LinkPath(graph, x_true, N=NLP)
lp.update_lp_flows()
V, g = lp.simplex_lp()
data['V'], data['g'] = V, g
# Export
if export:
if not SO:
fname = '%s/UE_graph.mat' % c.DATA_DIR
else:
fname = '%s/SO_graph.mat' % c.DATA_DIR
scipy.io.savemat(fname, data, oned_as='column')
return data, graph
def get_paths(SO, K, demand, return_paths=True, ffdelays=False, path=None, save_data=False, savepath=None):
"""This experiment does the following tests:
1. compute the UE/SO link flows using node-link formulation
2. get the link delays for the UE/SO link flow
3. find the K-shortest paths for these delays/marginal delays (used ones under UE/SO)
4. add these paths to the network
5. compute the UE/SO path flows using link-path formulation
6. check if we get the same UE/SO link flow
Parameters:
-----------
SO: if False, compute the UE, if True, compute the SO
K: number of shortest paths
demand: choice of OD demand
return_paths: if True, return paths
ffdelays: if True the k-shortest paths are obtained from ff delays
Return value:
------------
"""
print 'generate graph with demand', demand
g = los_angeles(theta, 'Polynomial', path=path)[demand]
if ffdelays: paths = get_shortest_paths(g, K)
print 'compute UE'
l1 = ue.solver(g, update=True, SO=SO)
d1 = sum([link.delay*link.flow for link in g.links.values()])
if SO:
for link in g.links.values():
link.delay = link.ffdelay*(1+0.75*(link.flow*link.delayfunc.slope)**4)
print 'get {} shortest paths'.format(K)
if not ffdelays: paths = get_shortest_paths(g, K)
if return_paths: return paths
for p in paths: g.add_path_from_nodes(p)
g.visualize(general=True)
print 'Generate link-path incidence matrix'
P = path_solver.linkpath_incidence(g)
x_true = path_solver.solver(g, update=True, SO=SO)[0]
l2 = P*x_true
d2 = sum([p.delay*p.flow for p in g.paths.values()])
if save_data:
U, f = path_solver.path_to_OD_simplex(g)
if savepath is None: savepath = 'data.pkl'
data = {}
data['U'] = np.array(matrix(U))
data['f'] = np.array(f).flatten()
data['A'] = np.array(matrix(P))
data['b'] = np.array(l2).flatten()
data['x_true'] = np.array(x_true).flatten()
pickle.dump( data, open(savepath, "wb" ) )
#for i in range(P2.shape[0]):
# print np.sum(P2[i,:])
return d1,d2,paths
def test_helper(demand, paths):
g = los_angeles(theta, 'Polynomial')[demand]
for p in paths: g.add_path_from_nodes(p)
P = path_solver.linkpath_incidence(g)
g.visualize(general=True)
l1 = ue.solver(g, update=True)
d1 = sum([link.delay*link.flow for link in g.links.values()])
l2 = P*path_solver.solver(g, update=True)
d2 = sum([p.delay*p.flow for p in g.paths.values()])
l3 = ue.solver(g, update=True, SO=True)
d3 = sum([link.delay*link.flow for link in g.links.values()])
l4 = P*path_solver.solver(g, update=True, SO=True)
d4 = sum([p.delay*p.flow for p in g.paths.values()])
return l1,l2,l3,l4,d1,d2,d3,d4
def test_feasible_pathflows(SO, demand, random=False):
"""Test function feasible_pathflows"""
paths = find_UESOpaths(SO)
g = los_angeles(theta, 'Polynomial')[demand]
l1 = ue.solver(g, update=True, SO=SO)
d1 = sum([link.delay*link.flow for link in g.links.values()])
for p in paths: g.add_path_from_nodes(p)
g.visualize(general=True)
P = linkpath_incidence(g)
l2 = P*path_solver.solver(g, update=True, SO=SO, random=random)
d2 = sum([p.delay*p.flow for p in g.paths.values()])
ind_obs, ls, ds = {}, [], []
ind_obs[0] = g.indlinks.keys()
ind_obs[1] = [(36,37,1), (13,14,1), (17,8,1), (24,17,1), (28,22,1), (14,13,1), (17,24,1), (24,40,1), (14,21,1), (16,23,1)]
ind_obs[2] = [(17,24,1), (24,40,1), (14,21,1), (16,23,1)]
for i in range(len(ind_obs)):
obs = [g.indlinks[id] for id in ind_obs[i]]
obs = [int(i) for i in list(np.sort(obs))]
ls.append(P*path_solver.feasible_pathflows(g, l1[obs], obs, True))
ds.append(sum([p.delay*p.flow for p in g.paths.values()]))
print d1,d2,ds
print np.linalg.norm(l1-l2), [np.linalg.norm(l1-ls[i]) for i in range(len(ind_obs))]
def find_UESOpaths(SO, return_paths=True, random=False, path=None):
"""
1. take the union for all optimum shortest paths for UE/SO
2. compute UE/SO using node-link and link-path formulation for all demands
3. compare results
Parameters:
-----------
SO: if False, compute the UE, if True, compute the SO
return_paths: if True, do only step 1 and return paths, if False, do steps 2 and 3
"""
paths, ls, ds, ps = [], [], [], []
if SO: K = [0,0,0,10] #[2, 2, 4, 7]
else: K = [0,0,0,5] #[2, 2, 2, 3] [5,5,5,5]
for i in range(4):
tmp = get_paths(SO, K[i], i, path=path)
for p in tmp:
if p not in paths: paths.append(p)
if return_paths: return paths
for i in range(4):
g = los_angeles(theta, 'Polynomial')[i]
for p in paths: g.add_path_from_nodes(p)
P = linkpath_incidence(g)
g.visualize(general=True)
l1 = ue.solver(g, update=True, SO=SO)
d1 = sum([link.delay*link.flow for link in g.links.values()])
p_flows = path_solver.solver(g, update=True, SO=SO, random=random)
l2 = P*p_flows
d2 = sum([p.delay*p.flow for p in g.paths.values()])
ls.append([l1,l2])
ds.append([d1,d2])
ps.append(p_flows)
for i in range(4):
print np.linalg.norm(ls[i][0] - ls[i][1])
print ds[i][0], ds[i][1]
print len(paths)
def experiment(data=None, SO=False, trials=5, demand=3, N=10, withODs=False, data_id=None):
"""Run set of experiments
Steps:
1. generate synthetic data with synthetic_data()
g: Graph object with paths
f_true: true path flow
l: UE linkflow
path_wps: dictionary {path_id: wp_ids}, with wp_ids list of waypoints
wp_trajs: waypoint trajectories [(wp_traj, path_list, flow)]
obs: dictionary {i: [link_ids]} where obs[i] are the indices of the i*n/N links with the most flow
2. Solve the linear regression problem min ||P*f-l|| s.t. U*f=r, x>=0
a. solve it with {i*n/N, i=1,...,N} observed links and OD information
b. solve it with {i*n/N, i=1,...,N} observed links and cell-path flow (+ OD flows if withODs)
3. Compute the relative link flow errors l_errors and the relative path flow errors
4. Repeat this 'trials' (=10) times and compute the average error and standard deviation
Parameters:
----------
data: (N0, N1, scale, regions, res, margin) inputs of generate_wp
SO: if True, computes SO
trials: number of trials and take the average
demand: OD demand
N: number of sets of observed links
e.g. if N=10, we run 10 experiments with observations from the 10, 20, ..., 100% most occupied links
plot: if True, plot results
withODs: include ODs in the constraints
data_id: id when figure is saved
"""
numexp = 2*N
l_errors, f_errors = [[] for i in range(numexp)], [[] for i in range(numexp)]
mean_ratio = 0
ddl_ODs, ddl_cellpaths = [[] for i in range(N)], [[] for i in range(N)]
k = 0
while k < trials:
print 'trial', k
g, f_true, l, path_wps, wp_trajs, obs = synthetic_data(data, SO, demand, N)
mean_ratio += float(len(wp_trajs)) / len(path_wps)
norm_l, norm_f = np.linalg.norm(l, 1), np.linalg.norm(f_true, 1)
err_f = lambda x: np.linalg.norm(f_true-x, 1) / norm_f
err_l = lambda x: np.linalg.norm(l-x, 1) / norm_l
P = path.linkpath_incidence(g)
ls, fs = [], []
#print 'Compute min ||P*f-l|| s.t. U*f=r, x>=0 with U=OD-path incidence matrix'
# A trial must complete successfully for all 2*N tests for it to count
failed = False
for i in range(N):
try:
f, rank, dim = path.feasible_pathflows(g, l[obs[i]], obs[i], with_ODs=True, x_true=f_true)
ddl_ODs[i].append(dim - rank)
except (ValueError, UnboundLocalError) as e:
print e
# 'Probably your QP is either non-positively defined (for cvxopt_qp you should have xHx > 0 for any x != 0) or very ill-conditioned.' # __str__ allows args to be printed directly
failed = True
break
fs.append(f)
ls.append(P*f)
#print 'Compute min ||P*f-l|| s.t. U*f=r, x>=0 with U=waypoint-path incidence matrix'
if failed:
continue
for i in range(N):
try:
f, rank, dim = path.feasible_pathflows(g, l[obs[i]], obs[i],
with_ODs=withODs, with_cell_paths=True, x_true=f_true, wp_trajs=wp_trajs)
ddl_cellpaths[i].append(dim - rank)
if data[0] + data[1] + data[3][0][1] == 40:
string = '+OD' if withODs else ''
print 'rank={}, num_obs={}, cellpath'.format(rank, len(obs[i])) + string
# Throw out trials for which any domain error is caught
#for j in range(U.size[0]):
# path.feasible_pathflows(g, l[obs[i]], obs[i], eq_constraints=(U[:j,:],r[:j]))
except (ValueError, UnboundLocalError) as e:
print e
# 'Probably your QP is either non-positively defined (for cvxopt_qp you should have xHx > 0 for any x != 0) or very ill-conditioned.' # __str__ allows args to be printed directly
failed = True
break
fs.append(f)
ls.append(P*f)
if failed:
continue
k += 1
l_error = [err_l(ls[i]) for i in range(numexp)]
f_error = [err_f(fs[i]) for i in range(numexp)]
"""
if withODs:
print f_error[:N], f_error[N:]
assert np.all([bo<=od for od,bo in zip(f_error[:N],f_error[N:])]), \
'Configurations with both OD+cellpath should be performing ' + \
'at least as well as with just OD'
assert np.all([bo<=od for od,bo in zip(l_error[:N],l_error[N:])]), \
'Configurations with both OD+cellpath should be performing ' + \
'at least as well as with just OD'
"""
for i in range(numexp):
l_errors[i].append(l_error[i])
f_errors[i].append(f_error[i])
mean_l_errors = [np.mean(l_errors[i]) for i in range(numexp)]
mean_f_errors = [np.mean(f_errors[i]) for i in range(numexp)]
std_l_errors = [np.std(l_errors[i]) for i in range(numexp)]
std_f_errors = [np.std(f_errors[i]) for i in range(numexp)]
mean_ddl_ODs = [np.mean(ddl_ODs[i]) for i in range(N)]
mean_ddl_cellpaths = [np.mean(ddl_cellpaths[i]) for i in range(N)]
return mean_l_errors, mean_f_errors, std_l_errors, std_f_errors,\
mean_ratio/trials, mean_ddl_ODs, mean_ddl_cellpaths