def experiment(indlinks_obs, delaytype, noise=0.0, display=False, soft=1000.0): """find parameters that minimizes the distance between x^obs_true in NOISY case and x^obs generated by each candidate function with PARTIAL observation Parameters ---------- indlinks_obs: indices of the observed links delaytype: type of the delay noise: std of the noise added to the measured link flows, ff delays, OD demands display: if True, display results soft: weight put on the observation """ if delaytype == 'Polynomial': true_theta = coef if delaytype == 'Hyperbolic': true_theta = (a,b) print 'generate graph...' g1, g2, g3, g4 = los_angeles(true_theta, delaytype) print 'compute ue...' l1, l2, l3, l4 = ue.solver(g1, update=True), ue.solver(g2, update=True), \ ue.solver(g3, update=True), ue.solver(g4, update=True) c1 = sum([link.delay*link.flow for link in g1.links.values()]) c2 = sum([link.delay*link.flow for link in g2.links.values()]) c3 = sum([link.delay*link.flow for link in g3.links.values()]) c4 = sum([link.delay*link.flow for link in g4.links.values()]) print 'ue costs: ', c1, c2, c3, c4 obs = [g1.indlinks[id] for id in indlinks_obs] obs = [int(i) for i in list(np.sort(obs))] x1,x2,x3,x4 = l1,l2,l3,l4 if noise > 0.0: x1, x2, x3, x4 = add_noise(l1,noise), add_noise(l2,noise), add_noise(l3,noise), add_noise(l4,noise) g1, g2, g3, g4 = los_angeles(true_theta, 'Polynomial', noise) theta, xs = invopt.main_solver([g1,g2,g3,g4], [x1[obs],x2[obs],x3[obs],x4[obs]], obs, degree, soft) u, v = matrix([l1,l2,l3,l4]), matrix(xs) error = np.linalg.norm(u-v, 1) / np.linalg.norm(u, 1) if display: display_results(error, true_theta, [theta], delaytype) return error, theta
def experiment_estimation(indlinks_obs, delaytype, degree): """Demonstrate multi-objective optimization for structural estimation 1. Generate the graph of L.A. 2. Generate synthetic data in UE 3. Apply multi-objective solver Parameters ---------- indlinks_obs = indices of the observed links delaytype: type of the delay display: if True, display results """ if delaytype == 'Polynomial': true_theta = coef if delaytype == 'Hyperbolic': true_theta = (a, b) g1, g2, g3, g4 = los_angeles(true_theta, delaytype) x1, x2, x3, x4 = ue.solver(g1), ue.solver(g2), ue.solver(g3), ue.solver(g4) obs = [g1.indlinks[id] for id in indlinks_obs] obs = [int(i) for i in list(np.sort(obs))] w_multi = [0.001, .01, .1, .5, .9, .99, 0.999] # weight on the observation residual r_gap, r_obs, x_est, thetas = invopt.multi_objective_solver( [g1, g2, g3, g4], [x1[obs], x2[obs], x3[obs], x4[obs]], obs, degree, w_multi) print r_gap print r_obs u = matrix([x1, x2, x3, x4]) r_est = [np.linalg.norm(u - x, 1) / np.linalg.norm(u, 1) for x in x_est] print r_est display_results(r_est[4], true_theta, [thetas[4]], delaytype, w_multi[4])
def experiment_estimation(indlinks_obs, delaytype, degree): """Demonstrate multi-objective optimization for structural estimation 1. Generate the graph of L.A. 2. Generate synthetic data in UE 3. Apply multi-objective solver Parameters ---------- indlinks_obs = indices of the observed links delaytype: type of the delay display: if True, display results """ if delaytype == 'Polynomial': true_theta = coef if delaytype == 'Hyperbolic': true_theta = (a,b) g1, g2, g3, g4 = los_angeles(true_theta, delaytype) x1, x2, x3, x4 = ue.solver(g1), ue.solver(g2), ue.solver(g3), ue.solver(g4) obs = [g1.indlinks[id] for id in indlinks_obs] obs = [int(i) for i in list(np.sort(obs))] w_multi = [0.001, .01, .1, .5, .9, .99, 0.999] # weight on the observation residual r_gap, r_obs, x_est, thetas = invopt.multi_objective_solver([g1,g2,g3,g4], [x1[obs],x2[obs],x3[obs],x4[obs]], obs, degree, w_multi) print r_gap print r_obs u = matrix([x1,x2,x3,x4]) r_est = [np.linalg.norm(u-x, 1) / np.linalg.norm(u, 1) for x in x_est] print r_est display_results(r_est[4], true_theta, [thetas[4]], delaytype, w_multi[4])
def test2(delaytype): if delaytype == 'Polynomial': theta = matrix([0.0, 0.0, 0.0, 0.15, 0.0, 0.0]) if delaytype == 'Hyperbolic': theta = (3.5, 3.0) g = los_angeles_2(theta, delaytype) l, x = ue.solver(g, update=True, full=True) d.draw_delays(g) print (max(mul(l,g.get_slopes()))) print ('cost UE:', sum([link.delay*link.flow for link in g.links.values()])) l2, x2 = ue.solver(g, update=True, full=True, SO=True) print (max(mul(l2,g.get_slopes()))) print ('cost SO:', sum([link.delay*link.flow for link in g.links.values()]))
def test_helper(demand, paths): g = los_angeles(theta, 'Polynomial')[demand] for p in paths: g.add_path_from_nodes(p) P = path_solver.linkpath_incidence(g) g.visualize(general=True) l1 = ue.solver(g, update=True) d1 = sum([link.delay*link.flow for link in g.links.values()]) l2 = P*path_solver.solver(g, update=True) d2 = sum([p.delay*p.flow for p in g.paths.values()]) l3 = ue.solver(g, update=True, SO=True) d3 = sum([link.delay*link.flow for link in g.links.values()]) l4 = P*path_solver.solver(g, update=True, SO=True) d4 = sum([p.delay*p.flow for p in g.paths.values()]) return l1,l2,l3,l4,d1,d2,d3,d4
def test2(delaytype): if delaytype == 'Polynomial': theta = matrix([0.0, 0.0, 0.0, 0.15, 0.0, 0.0]) if delaytype == 'Hyperbolic': theta = (3.5, 3.0) g = los_angeles_2(theta, delaytype) l, x = ue.solver(g, update=True, full=True) d.draw_delays(g) print(max(mul(l, g.get_slopes()))) print('cost UE:', sum([link.delay * link.flow for link in g.links.values()])) l2, x2 = ue.solver(g, update=True, full=True, SO=True) print(max(mul(l2, g.get_slopes()))) print('cost SO:', sum([link.delay * link.flow for link in g.links.values()]))
def solver(graph, update=False, data=None, SO=False, random=False): """Solve for the UE equilibrium using link-path formulation Parameters ---------- graph: graph object update: if True, update link and path flows in graph data: (P,U,r) P: link-path incidence matrix U,r: simplex constraints SO: if True compute SO random: if True, initialize with a random feasible point """ type = graph.links.values()[0].delayfunc.type if data is None: P = linkpath_incidence(graph) U, r = path_to_OD_simplex(graph) else: P, U, r = data m = graph.numpaths A, b = spmatrix(-1.0, range(m), range(m)), matrix(0.0, (m, 1)) ffdelays = graph.get_ffdelays() if type == 'Polynomial': coefs = graph.get_coefs() if not SO: coefs = coefs * spdiag( [1.0 / (j + 2) for j in range(coefs.size[1])]) parameters = matrix([[ffdelays], [coefs]]) G = ue.objective_poly if type == 'Hyperbolic': ks = graph.get_ks() parameters = matrix([[ffdelays - div(ks[:, 0], ks[:, 1])], [ks]]) G = ue.objective_hyper def F(x=None, z=None): if x is None: return 0, solver_init(U, r, random) if z is None: f, Df = G(P * x, z, parameters, 1) return f, Df * P f, Df, H = G(P * x, z, parameters, 1) return f, Df * P, P.T * H * P failed = True while failed: x = solvers.cp(F, G=A, h=b, A=U, b=r)['x'] l = ue.solver(graph, SO=SO) error = np.linalg.norm(P * x - l, 1) / np.linalg.norm(l, 1) if error > TOL: print 'error={} > {}, re-compute path_flow'.format(error, TOL) else: failed = False if update: logging.info('Update link flows, delays in Graph.') graph.update_linkflows_linkdelays(P * x) logging.info('Update path delays in Graph.') graph.update_pathdelays() logging.info('Update path flows in Graph object.') graph.update_pathflows(x) # assert if x is a valid UE/SO return x, l
def test2(delaytype): if delaytype == 'Polynomial': theta = matrix([0.0, 0.0, 0.0, 0.15, 0.0, 0.0]) if delaytype == 'Hyperbolic': theta = (3.5, 3.0) g = los_angeles(theta, delaytype)[3] n = g.numlinks l, x = ue.solver(g, update=True, full=True) d.draw_delays(g, x[:n]) d.draw_delays(g, x[n:2*n]) d.draw_delays(g, x[2*n:]) #print l print max(mul(l,g.get_slopes())) print 'cost UE:', sum([link.delay*link.flow for link in g.links.values()]) l2, x2 = ue.solver(g, update=True, full=True, SO=True) d.draw_delays(g, x2[:n]) d.draw_delays(g, x2[n:2*n]) d.draw_delays(g, x2[2*n:]) #print l2 print max(mul(l2,g.get_slopes())) print 'cost SO:', sum([link.delay*link.flow for link in g.links.values()])
def get_paths(SO, K, demand, return_paths=True, ffdelays=False, path=None, save_data=False, savepath=None): """This experiment does the following tests: 1. compute the UE/SO link flows using node-link formulation 2. get the link delays for the UE/SO link flow 3. find the K-shortest paths for these delays/marginal delays (used ones under UE/SO) 4. add these paths to the network 5. compute the UE/SO path flows using link-path formulation 6. check if we get the same UE/SO link flow Parameters: ----------- SO: if False, compute the UE, if True, compute the SO K: number of shortest paths demand: choice of OD demand return_paths: if True, return paths ffdelays: if True the k-shortest paths are obtained from ff delays Return value: ------------ """ print 'generate graph with demand', demand g = los_angeles(theta, 'Polynomial', path=path)[demand] if ffdelays: paths = get_shortest_paths(g, K) print 'compute UE' l1 = ue.solver(g, update=True, SO=SO) d1 = sum([link.delay*link.flow for link in g.links.values()]) if SO: for link in g.links.values(): link.delay = link.ffdelay*(1+0.75*(link.flow*link.delayfunc.slope)**4) print 'get {} shortest paths'.format(K) if not ffdelays: paths = get_shortest_paths(g, K) if return_paths: return paths for p in paths: g.add_path_from_nodes(p) g.visualize(general=True) print 'Generate link-path incidence matrix' P = path_solver.linkpath_incidence(g) x_true = path_solver.solver(g, update=True, SO=SO)[0] l2 = P*x_true d2 = sum([p.delay*p.flow for p in g.paths.values()]) if save_data: U, f = path_solver.path_to_OD_simplex(g) if savepath is None: savepath = 'data.pkl' data = {} data['U'] = np.array(matrix(U)) data['f'] = np.array(f).flatten() data['A'] = np.array(matrix(P)) data['b'] = np.array(l2).flatten() data['x_true'] = np.array(x_true).flatten() pickle.dump( data, open(savepath, "wb" ) ) #for i in range(P2.shape[0]): # print np.sum(P2[i,:]) return d1,d2,paths
def assign_traffic(graph, algorithm='FW', output=True): # traffic assignment l,x = ue.solver(graph, update=True, full=True) total_delay = sum([link.delay*link.flow for link in graph.links.values()]) if output: #delay = np.asarray([link.delay for link in graph.links.itervalues()]) #ffdelay = np.asarray([link.ffdelay for link in graph.links.itervalues()]) #edge_ratios = delay/ffdelay #d.draw(graph) d.draw_delays(graph, l) #print max(mul(l,graph.get_slopes())) print 'cost UE:', sum([link.delay*link.flow for link in graph.links.values()]) return total_delay
def solver(graph, update=False, data=None, SO=False, random=False): """Solve for the UE equilibrium using link-path formulation Parameters ---------- graph: graph object update: if True, update link and path flows in graph data: (P,U,r) P: link-path incidence matrix U,r: simplex constraints SO: if True compute SO random: if True, initialize with a random feasible point """ type = graph.links.values()[0].delayfunc.type if data is None: P = linkpath_incidence(graph) U,r = path_to_OD_simplex(graph) else: P,U,r = data m = graph.numpaths A, b = spmatrix(-1.0, range(m), range(m)), matrix(0.0, (m,1)) ffdelays = graph.get_ffdelays() if type == 'Polynomial': coefs = graph.get_coefs() if not SO: coefs = coefs * spdiag([1.0/(j+2) for j in range(coefs.size[1])]) parameters = matrix([[ffdelays], [coefs]]) G = ue.objective_poly if type == 'Hyperbolic': ks = graph.get_ks() parameters = matrix([[ffdelays-div(ks[:,0],ks[:,1])], [ks]]) G = ue.objective_hyper def F(x=None, z=None): if x is None: return 0, solver_init(U,r,random) if z is None: f, Df = G(P*x, z, parameters, 1) return f, Df*P f, Df, H = G(P*x, z, parameters, 1) return f, Df*P, P.T*H*P failed = True while failed: x = solvers.cp(F, G=A, h=b, A=U, b=r)['x'] l = ue.solver(graph, SO=SO) error = np.linalg.norm(P * x - l,1) / np.linalg.norm(l,1) if error > TOL: print 'error={} > {}, re-compute path_flow'.format(error, TOL) else: failed = False if update: logging.info('Update link flows, delays in Graph.'); graph.update_linkflows_linkdelays(P*x) logging.info('Update path delays in Graph.'); graph.update_pathdelays() logging.info('Update path flows in Graph object.'); graph.update_pathflows(x) # assert if x is a valid UE/SO return x, l
def experiment(indlinks_obs, delaytype, noise=0.0, display=False, soft=1000.0): """find parameters that minimizes the distance between x^obs_true in NOISY case and x^obs generated by each candidate function with PARTIAL observation Parameters ---------- indlinks_obs: indices of the observed links delaytype: type of the delay noise: std of the noise added to the measured link flows, ff delays, OD demands display: if True, display results soft: weight put on the observation """ if delaytype == 'Polynomial': true_theta = coef if delaytype == 'Hyperbolic': true_theta = (a, b) print 'generate graph...' g1, g2, g3, g4 = los_angeles(true_theta, delaytype) print 'compute ue...' l1, l2, l3, l4 = ue.solver(g1, update=True), ue.solver(g2, update=True), \ ue.solver(g3, update=True), ue.solver(g4, update=True) c1 = sum([link.delay * link.flow for link in g1.links.values()]) c2 = sum([link.delay * link.flow for link in g2.links.values()]) c3 = sum([link.delay * link.flow for link in g3.links.values()]) c4 = sum([link.delay * link.flow for link in g4.links.values()]) print 'ue costs: ', c1, c2, c3, c4 obs = [g1.indlinks[id] for id in indlinks_obs] obs = [int(i) for i in list(np.sort(obs))] x1, x2, x3, x4 = l1, l2, l3, l4 if noise > 0.0: x1, x2, x3, x4 = add_noise(l1, noise), add_noise(l2, noise), add_noise( l3, noise), add_noise(l4, noise) g1, g2, g3, g4 = los_angeles(true_theta, 'Polynomial', noise) theta, xs = invopt.main_solver([g1, g2, g3, g4], [x1[obs], x2[obs], x3[obs], x4[obs]], obs, degree, soft) u, v = matrix([l1, l2, l3, l4]), matrix(xs) error = np.linalg.norm(u - v, 1) / np.linalg.norm(u, 1) if display: display_results(error, true_theta, [theta], delaytype) return error, theta
def compute_cost(data, toll=None, SO=False): """Compute: - untolled UE - untolled SO - tolled UE Return value ------------ total cost linkflows """ Aeq, beq, ffdelays, pm, type = data if toll is None: toll = 0.0 x = ue.solver(data=(Aeq, beq, ffdelays+toll, pm, type), SO=SO) delays = compute_delays(x, ffdelays, pm, type) return (delays.T*x)[0], x
def compute_cost(data, toll=None, SO=False): """Compute: - untolled UE - untolled SO - tolled UE Return value ------------ total cost linkflows """ Aeq, beq, ffdelays, pm, type = data if toll is None: toll = 0.0 x = ue.solver(data=(Aeq, beq, ffdelays + toll, pm, type), SO=SO) delays = compute_delays(x, ffdelays, pm, type) return (delays.T * x)[0], x
def test_osm_box(delaytype): t0=time.time() box = [34.124918, 34.1718, -118.1224, -118.02524] #small box I 210 arcadia if delaytype == 'Polynomial': theta = matrix([0.0, 0.0, 0.0, 0.15]) g = osm_network(theta, delaytype, box) g.visualize(True) d.draw(g, nodes = False) n = g.numlinks print n l, x = ue.solver(g, update=True, full=True) t1=time.time() print 'time for computation for '+str(t1-t0) d.draw_ODs(g, 205) d.draw_delays(g) print max(mul(l,g.get_slopes())) print 'cost UE:', sum([link.delay*link.flow for link in g.links.values()])
def compute_scaling(graphs, ls_obs, obs, data): """Scale the objectives for multi-objective optimization Parameters ---------- graphs: list of graphs ls_obs: list of observed links obs: indlinks ids of observed links data: obtained from get_data Return value ------------ scale: vector of scaling factor such that scale[0]: scaling for the observation residual scale[1]: scaling for the gap function """ N = len(graphs) scale = matrix(0.0, (2, 1)) Aeq, beqs, ffdelays, slopes = data theta = matrix([1.0]) #initial theta coefs = compute_coefs(ffdelays, slopes, theta) type = 'Polynomial' xs = [ ue.solver(data=(Aeq, beqs[k], ffdelays, coefs, type)) for k in range(N) ] scale[0] = 2. * np.linalg.norm( matrix(ls_obs) - matrix([x[obs] for x in xs]), 2)**-2 graph = graphs[0] total_delay = 0.0 sinks = [id for id, node in graph.nodes.items() if len(node.endODs) > 0] for t in sinks: sources = [od[0] for od in graph.ODs.keys() if od[1] == t] As = sh.mainKSP(graph, sources, t, 1) for s, path in As.items(): delay = 0.0 for i in range(len(path[0]) - 1): delay += graph.links[(path[0][i], path[0][i + 1], 1)].delay demand = 0.0 for g in graphs: demand += g.ODs[(s, t)].flow total_delay += 5. * delay * demand scale[1] = 1 / total_delay return scale
def multi_objective_solver(graphs, ls_obs, obs, degree, w_multi, max_iter=3): """Multi-objective optimization for the latency inference problem with the following weights: w1: weight on the observation residual w2: weight on the gap function w1 + w2 = 1 Parameters ---------- graphs: list of graphs ls_obs: list of partially-observed link flow vectors in equilibrium obs: indlinks ids of observed links degree: degree of the polynomial function to estimate w_multi: list of weights on the observation residual max_iter: maximum number of iterations """ data = get_data(graphs) scale = compute_scaling(graphs, ls_obs, obs, data) w_obs = [scale[0] * w for w in w_multi] # weights on the observation residual w_gap = [scale[1] * (1 - w) for w in w_multi] # weights on the gap function #softs = [(scale[0]*w)/(scale[1]*(1-w)) for w in w_multi] Aeq, beqs, ffdelays, slopes = data type, N = 'Polynomial', len(beqs) r_gap, r_obs, x_est, thetas = [], [], [], [] for w1, w2 in zip(w_obs, w_gap): theta, ys, ls = solver_mis(data, ls_obs, obs, degree, 0.0 * np.ones(degree), w1, max_iter, True, w2) coefs = compute_coefs(ffdelays, slopes, theta) r_gap.append( compute_gap(ls, ys, matrix([[ffdelays], [coefs]]), beqs, scale[1])) r_obs.append( scale[0] * 0.5 * np.linalg.norm(matrix(ls_obs) - matrix([l[obs] for l in ls]), 2)**2) xs = [ ue.solver(data=(Aeq, beqs[k], ffdelays, coefs, type)) for k in range(N) ] x_est.append(matrix(xs)) thetas.append(theta) return r_gap, r_obs, x_est, thetas
def test_osm_box(delaytype): t0 = time.time() box = [34.124918, 34.1718, -118.1224, -118.02524] #small box I 210 arcadia if delaytype == 'Polynomial': theta = matrix([0.0, 0.0, 0.0, 0.15]) g = osm_network(theta, delaytype, box) g.visualize(True) d.draw(g, nodes=False) n = g.numlinks print n l, x = ue.solver(g, update=True, full=True) t1 = time.time() print 'time for computation for ' + str(t1 - t0) d.draw_ODs(g, 205) d.draw_delays(g) print max(mul(l, g.get_slopes())) print 'cost UE:', sum( [link.delay * link.flow for link in g.links.values()])
def test_feasible_pathflows(SO, demand, random=False): """Test function feasible_pathflows""" paths = find_UESOpaths(SO) g = los_angeles(theta, 'Polynomial')[demand] l1 = ue.solver(g, update=True, SO=SO) d1 = sum([link.delay*link.flow for link in g.links.values()]) for p in paths: g.add_path_from_nodes(p) g.visualize(general=True) P = linkpath_incidence(g) l2 = P*path_solver.solver(g, update=True, SO=SO, random=random) d2 = sum([p.delay*p.flow for p in g.paths.values()]) ind_obs, ls, ds = {}, [], [] ind_obs[0] = g.indlinks.keys() ind_obs[1] = [(36,37,1), (13,14,1), (17,8,1), (24,17,1), (28,22,1), (14,13,1), (17,24,1), (24,40,1), (14,21,1), (16,23,1)] ind_obs[2] = [(17,24,1), (24,40,1), (14,21,1), (16,23,1)] for i in range(len(ind_obs)): obs = [g.indlinks[id] for id in ind_obs[i]] obs = [int(i) for i in list(np.sort(obs))] ls.append(P*path_solver.feasible_pathflows(g, l1[obs], obs, True)) ds.append(sum([p.delay*p.flow for p in g.paths.values()])) print d1,d2,ds print np.linalg.norm(l1-l2), [np.linalg.norm(l1-ls[i]) for i in range(len(ind_obs))]
def main_solver(graphs, ls_obs, obs, degree, soft=1000.0, max_iter=3): """Solves solve_mis with the best parameter Parameters ---------- graphs: list of graphs ls_obs: list of partially-observed link flow vectors in equilibrium obs: indlinks ids of observed links degree: degree of the polynomial function to estimate soft: regularization parameter for soft constraints max_iter: maximum number of iterations """ data, n_obs = get_data(graphs), len(obs) Aeq, beqs, ffdelays, slopes = data type, N, min_e, n = 'Polynomial', len(beqs), np.inf, len(ffdelays) if n_obs < n: theta = solver_mis(data, ls_obs, obs, degree, soft, max_iter) #if n_obs < n: theta = solver_mis_multi_thread(data, ls_obs, obs, degree, i*np.ones(degree), soft, max_iter, 4) if n_obs == n: theta = solver(data, ls_obs, degree) coefs = compute_coefs(ffdelays, slopes, theta) xs = [ ue.solver(data=(Aeq, beqs[j], ffdelays, coefs, type)) for j in range(N) ] return theta, xs
def find_UESOpaths(SO, return_paths=True, random=False, path=None): """ 1. take the union for all optimum shortest paths for UE/SO 2. compute UE/SO using node-link and link-path formulation for all demands 3. compare results Parameters: ----------- SO: if False, compute the UE, if True, compute the SO return_paths: if True, do only step 1 and return paths, if False, do steps 2 and 3 """ paths, ls, ds, ps = [], [], [], [] if SO: K = [0,0,0,10] #[2, 2, 4, 7] else: K = [0,0,0,5] #[2, 2, 2, 3] [5,5,5,5] for i in range(4): tmp = get_paths(SO, K[i], i, path=path) for p in tmp: if p not in paths: paths.append(p) if return_paths: return paths for i in range(4): g = los_angeles(theta, 'Polynomial')[i] for p in paths: g.add_path_from_nodes(p) P = linkpath_incidence(g) g.visualize(general=True) l1 = ue.solver(g, update=True, SO=SO) d1 = sum([link.delay*link.flow for link in g.links.values()]) p_flows = path_solver.solver(g, update=True, SO=SO, random=random) l2 = P*p_flows d2 = sum([p.delay*p.flow for p in g.paths.values()]) ls.append([l1,l2]) ds.append([d1,d2]) ps.append(p_flows) for i in range(4): print np.linalg.norm(ls[i][0] - ls[i][1]) print ds[i][0], ds[i][1] print len(paths)
def test1(): graph = small_example() linkflows = ue.solver(graph) print 'links\' indices: ', graph.indlinks print 'UE flow: ' print linkflows