def init_model(self, wgt_code_list, original_weight, overall_score,
adjust_weight, suspend_stock):
stock_lower_bound = self.stock_lower_bound
stock_upper_bound = self.stock_upper_bound
model = gurobipy.Model(self.name)
weight, weight_binary = {}, {}
for code in wgt_code_list:
lb = max(0, original_weight[code] - stock_lower_bound)
ub = min(1, original_weight[code] + stock_upper_bound)
if adjust_weight and (code
in adjust_weight) and (code
in suspend_stock):
if adjust_weight[code] != 0:
lb = min(lb, adjust_weight[code])
ub = max(ub, adjust_weight[code])
weight[code] = model.addVar(lb=lb,
ub=ub,
name='optweight_' + code,
vtype=gurobipy.GRB.SEMICONT)
weight_binary[code] = model.addVar(lb=0,
ub=1,
vtype=gurobipy.GRB.BINARY,
name='biweight_' + code)
weight = gurobipy.tupledict(weight)
weight_binary = gurobipy.tupledict(weight_binary)
model.setObjective(weight.prod(overall_score), gurobipy.GRB.MAXIMIZE)
return model, weight, weight_binary
def assignFromRTVGraph(self, rtvGraph):
m = gp.Model("rtv-assignment")
# m.setParam(GRB.Param.OutputFlag, 0)
m.setParam(GRB.Param.LogToConsole, 0)
m.setParam(GRB.Param.LogFile, "output/gurobi/log.txt")
e_ij = gp.tuplelist([(edge.source, edge.target) for edge in rtvGraph.es.select(etype="veh_trip")])
x_k = gp.tuplelist([(v.index) for v in rtvGraph.vs.select(vtype="request")])
req_trips = gp.tupledict({
req.index: [rtvGraph.es[edge].target for edge in rtvGraph.incident(req, ig.ALL)]
for req in rtvGraph.vs.select(vtype="request")})
cost_ij = gp.tupledict({(edge.source, edge.target): edge["cost"] for edge in rtvGraph.es.select(etype="veh_trip")})
cost_ko = 1e6
trips = gp.tuplelist([(v.index) for v in rtvGraph.vs.select(vtype="trip")])
vehs = gp.tuplelist([(v.index) for v in rtvGraph.vs.select(vtype="vehicle")])
reqs = gp.tuplelist([(v.index) for v in rtvGraph.vs.select(vtype="request")])
veh_trip_flow = m.addVars(e_ij, vtype=GRB.BINARY, name="e")
req_ignored = m.addVars(x_k, vtype=GRB.BINARY, name="x")
m.setObjective(veh_trip_flow.prod(cost_ij) + cost_ko * req_ignored.sum(), GRB.MINIMIZE)
m.addConstrs((veh_trip_flow.sum(i, "*") <= 1 for i in vehs), "one_veh_per_trip")
for k in reqs:
m.addConstr(sum([veh_trip_flow.sum("*", j) for j in req_trips[k]]) + req_ignored[k] == 1,
"one_veh_max_per_req[{}]".format(k))
m.optimize()
if m.status == GRB.OPTIMAL:
veh_trips = m.getAttr('x', veh_trip_flow)
veh_routes = dict()
for edge in rtvGraph.es.select(etype="veh_trip"):
veh = rtvGraph.vs[edge.source]
vehID = veh.index
tripID = edge.target
if veh_trips[vehID, tripID] > 0:
veh_routes[veh["vid"]] = edge["route"]
ignored_reqs = m.getAttr('x', req_ignored)
rejected_reqs = set()
for req in reqs:
if ignored_reqs[req] > 0:
rejected_reqs.add(rtvGraph.vs[req]["rid"])
else:
return None, None
return veh_routes, rejected_reqs
def tsp_connecting_cutting_planes(lp, var_dict, graph):
# print("running connecting cutting planes")
lp.optimize()
if lp.status == grb.GRB.Status.INFEASIBLE:
return None, {}, None
soln_index = {
index: lp.getVarByName(name).X
for index, name in var_dict.items()
}
i = 0
new_constrs = []
while True:
i += 1
# print(i)
nx.set_edge_attributes(graph, soln_index, 'capacity')
soln_graph = nx.Graph(((n1, n2, attr)
for n1, n2, attr in graph.edges(data=True)
if attr['capacity'] > 0))
if nx.is_connected(soln_graph):
break
components = nx.connected_components(soln_graph)
for node_set in components:
edge_cut_list = [(n1, n2) if n1 < n2 else (n2, n1)
for n1 in node_set
for n2 in set(graph.nodes()) - node_set]
edge_cut_vars = grb.tupledict({
index: lp.getVarByName(var_dict[index])
for index in edge_cut_list
})
constr = lp.addConstr(edge_cut_vars.sum() >= 2)
new_constrs.append(("connecting plane constraint",
[var_dict[index] for index in edge_cut_list]))
lp.update()
lp.optimize()
if lp.status == grb.GRB.status.INFEASIBLE:
return None, {}, None
soln_index = {
index: lp.getVarByName(name).X
for index, name in var_dict.items()
}
# print("number of connecting plane iterations: ", i)
return lp.objVal, grb.tupledict(
{index: (var_dict[index], val)
for index, val in soln_index.items()}), new_constrs
def extract_solution(inst, model, eps=1e-6):
threshold, lvec, bvec = inst
vals = gp.tupledict([(_name2tuple(v.varName), v.x)
for v in model.getVars() if v.x > eps])
arcs = vals.keys()
solutions = []
for arc0 in arcs.select(0, '*', '*'):
while vals[arc0] > eps:
minval = vals[arc0]
start, next_start, item = arc0
current_path = [arc0]
while next_start < threshold:
next_arcs = [
arc for arc in arcs.select(next_start, '*', '*')
if vals[arc] > eps
]
assert len(next_arcs) > 0
arc1 = start, end, item = next_arcs[0]
minval = min(minval, vals[arc1])
current_path.append(arc1)
next_start = end
solution = []
for arc in current_path:
vals[arc] -= minval
solution.append(arc[2])
solutions.append((minval, solution))
return [(v, sum([lvec[k] for k in sol]), [lvec[k] for k in sol])
for v, sol in solutions]
def _add_initial_heading(self, times, initial_heading):
if get_array_greater_zero(initial_heading):
# create degrees for all initial headings
degrees = grb.tupledict()
arcs = grb.tuplelist()
for index in range(len(self.graph.vertices)):
v_a = np.array(self.graph.vertices[index])
ad_indexes = [
j for j in range(len(self.graph.vertices))
if self.graph.ad_matrix[index, j] > 0
]
vert_arr = np.array(
[self.graph.vertices[i] for i in ad_indexes])
l = len(vert_arr)
degrees = np.array([
get_angle(v_a, vert, initial_heading[index])
for vert in vert_arr
])
tuple_list = [((index, ad_indexes[i]), degrees[i])
for i in range(l)]
# Correct entries with NaN
for i in range(len(tuple_list)):
if tuple_list[i] == np.NaN:
tuple_list[i][-1] = 0
arcs_i, multidict_i = grb.multidict(
tuple_list) if tuple_list else (None, None)
degrees.update(multidict_i)
arcs.extend(arcs_i)
for arc in arcs:
self.model.addConstr(times[arc] >= degrees[arc],
name="degree_constraint_init")
def extract_solution(inst, model, eps=1e-6):
threshold, lvec, bvec = inst
vals = gp.tupledict([(_name2tuple(v.varName), v.x)
for v in model.getVars() if v.x > eps])
arcs = vals.keys()
solutions = []
for arc0 in arcs.select(0):
while vals[arc0] > eps:
minval = vals[arc0]
start, next_start, item, kind = arc0
current_path = [arc0]
vals[arc0] -= minval
while next_start < threshold:
next_arcs = [
arc for arc in arcs.select(next_start) if vals[arc] > eps
]
next_arcs.sort(key=lambda arc: arc[1])
assert len(next_arcs) > 0
arc1 = start, end, item, kind = next_arcs[0]
if vals[arc1] < minval:
diff = minval - vals[arc1]
for arc in current_path:
vals[arc] += diff
minval = vals[arc1]
vals[arc1] -= minval
current_path.append(arc1)
next_start = end
solutions.append((minval, current_path))
return [format_solution(lvec, v, sol) for v, sol in solutions]
def add_constr_vertex_in_singleton_has_edge(model, hat_vars, genome_vars,
vertex_set):
logger.info("Creating constraints that vertex has R-edge.")
for v in vertex_set:
rss = gurobipy.tupledict([((v, k), genome_vars.select(v, k)[0])
for k in vertex_set
if len(genome_vars.select(v, k)) != 0])
model.addConstr(hat_vars[v] <= rss.sum(v, '*'))
def _add_hamilton_paths(self):
sorted_edges = {}
angles = gurobipy.tupledict()
for i in range(len(self.graph.vertices)):
edges, angle = self._build_hamilton_path(i)
if edges:
angles.update(angle)
sorted_edges[i] = edges
return sorted_edges, angles
def tsp_cutting_planes(lp, var_dict, graph):
print("running cutting planes")
lp.optimize()
soln_index = {
index: lp.getVarByName(name).X
for index, name in var_dict.items()
}
while True:
nx.set_edge_attributes(graph, soln_index, 'capacity')
cut_partitions = []
for i in range(1, len(graph)):
cut_weight, partitions = nx.minimum_cut(graph, 0, i)
if cut_weight < 2 - 10**(-12):
cut_partitions.append((cut_weight, partitions))
if not cut_partitions:
break
for (cut_value, partitions) in cut_partitions:
edge_cut_list = []
for p1_node in partitions[0]:
for p2_node in partitions[1]:
if graph.has_edge(p1_node, p2_node):
if p1_node > p2_node:
edge_cut_list.append((p2_node, p1_node))
else:
edge_cut_list.append((p1_node, p2_node))
edge_cut_vars = grb.tupledict({
index: lp.getVarByName(var_dict[index])
for index in edge_cut_list
})
lp.addConstr(edge_cut_vars.sum() >= 2)
a = time.clock()
lp.update()
lp.optimize()
soln_index = {
index: lp.getVarByName(name).X
for index, name in var_dict.items()
}
return lp.objVal, grb.tupledict(
{index: (var_dict[index], val)
for index, val in soln_index.items()})
def _calculate_degrees(self) -> (grb.tuplelist, grb.tupledict):
# Calculate degrees
degrees = grb.tupledict()
arcs = grb.tuplelist()
for i in range(len(self.graph.vertices)):
arcs_i, degrees_i = self._get_angles(i)
if arcs_i:
degrees.update(degrees_i)
arcs.extend(arcs_i)
return arcs, degrees
def read_gurobi_sol(instance_name):
file = solutions_dir + instance_name + ".sol"
with open(file, 'r') as f:
f_content = [a.strip('\n') for a in f]
lines = f_content[1:]
lines = format_sol_lines(lines)
solution = dict()
non_zero_solution = dict()
for var, vars_key, vars_value in lines:
if var not in solution:
solution[var] = tupledict()
if var not in non_zero_solution:
non_zero_solution[var] = tupledict()
solution[var][vars_key] = vars_value
if vars_value != 0:
non_zero_solution[var][vars_key] = vars_value
return solution, non_zero_solution
def build_ip_and_optimize(self,
graph: Graph,
start_solution: Optional[dict] = None,
callbacks: Optional[Union[Multidict,
dict]] = None,
time_limit=None):
error_message = None
runtime = 0
is_optimal = False
times = None
try:
self.graph = graph
self.model = gurobipy.Model()
self.model.setParam("LazyConstraints", 1)
for param in self.params:
self.model.setParam(param, self.params[param])
# Override general time limit, if one is provided
if time_limit:
self.model.setParam("TimeLimit", time_limit)
times_first, times_reverse = self._add_times_variables()
self.edges, self.angles = self._add_hamilton_paths()
self._add_times_equals_constraints(times_first, times_reverse)
self.times = gurobipy.tupledict()
self.times.update(times_first)
self.times.update(times_reverse)
self._add_taken_edges_constraints()
self._set_objective()
self._add_callbacks(callbacks)
self._add_pre_solution(graph, start_solution)
self.model.update()
self.model.optimize(callback_rerouter)
runtime = self.model.Runtime
is_optimal = self.model.Status == gurobipy.GRB.OPTIMAL
times = {t: self.times[t].x for t in self.times}
except Exception as e:
if self.model.Status != gurobipy.GRB.TIME_LIMIT:
error_message = str(e)
if is_debug_env():
raise e
sol = AngularGraphSolution(self.graph,
runtime,
times=times,
solution_type=self.solution_type,
is_optimal=is_optimal,
solver=self.__class__.__name__,
error_message=error_message)
return sol
def _add_cycle_constr(self, cycle_nodes, model: grb.Model):
cycle = [nodes.value for nodes in cycle_nodes]
l = len(cycle)
cycle_edges = [(cycle[i], cycle[((i + 1) % l)]) for i in range(l)]
cycle_edges_rev = [(cycle[((i + 1) % l)], cycle[i]) for i in range(l)]
#print("CYCLE EDGES:", cycle_edges)
l = len(cycle_edges) - 1
try:
cycle_edges_vars = grb.tupledict(
{i: self.edges[i]
for i in cycle_edges})
cycle_edges_rev_vars = grb.tupledict(
{i: self.edges[i]
for i in cycle_edges_rev})
exp = grb.LinExpr(cycle_edges_vars.sum())
exp_rev = grb.LinExpr(cycle_edges_rev_vars.sum())
model.cbLazy(exp <= l)
model.cbLazy(exp_rev <= l)
#model.addConstr(exp <= l)
except grb.GurobiError as err:
print("ERROR: Gurobi error with number:", err.errno)
def solve(self, graph: Graph, **kwargs):
self.abstract_graph = convert_graph_to_angular_abstract_graph(graph)
self.graph = graph
self.model = grb.Model()
for param in self.params:
self.model.setParam(param, self.params[param])
self._add_callbacks(kwargs.pop("callbacks", None))
self.vertex_edges = grb.tupledict()
costs = self._add_variables(self.abstract_graph)
for vertex_index in len(max(self.abstract_graph.vertices)):
edges = self._build_hamilton_path(vertex_index,
self.abstract_graph)
edges_dict = grb.tupledict({(vertex_index, u, v): self.edges[u, v]
for u, v in edges})
self.vertex_edges.update(edges_dict)
self._add_objective(costs)
self.model.optimize()
def optimize_information_design(self, sol_p, sol_d, sol_r, current_supply, incoming_supply):
"""
@param incoming_supply:
@param current_supply:
@param sol_p:
@param sol_d:
@param sol_r:
@return:
"""
print("inside behavioral optimization unit")
# Only consider the next time step
t_fifteen = np.min([t for i, j, t in sol_p.keys()])
one_t_ahead_sol_p = {(i, j, t): sol_p[(i, j, t)] for i, j, t in sol_p.keys() if t == t_fifteen}
one_t_ahead_sol_r = {(i, j, t): sol_r[(i, j, t)] for i, j, t in sol_r.keys() if t == t_fifteen}
# total assignment + rebalancing
# one_t_ahead_move = {}
# for k, v in one_t_ahead_sol_p.items():
# one_t_ahead_move[k] = v + one_t_ahead_sol_r[k]
# Maybe it should be just the rebalancing ones
one_t_ahead_move = one_t_ahead_sol_r
# get attractiveness of each destination, per origin
dest_attraction = defaultdict(lambda: defaultdict) # origin : {dest: att}, these are C_k^d
# get the expected number of drivers in the origin zone (i.e., current + incoming)
supply = gb.tupledict()
demand_df = self.operator.get_zonal_info_for_general()
for z_id in self.zone_ids:
dist = _get_dist_to_all_zones(z_id)
# get demand
assert dist.shape[0] == demand_df.shape[0]
# {z.id : utility}
expected_attractiveness = _compute_expected_attractiveness(demand_df, dist, unit_rebalancing_cost=FUEL_COST)
dest_attraction[z_id] = expected_attractiveness
supply[(z_id, t_fifteen)] = current_supply[z_id] + incoming_supply[(z_id, t_fifteen)]
optimal_si = {}
start_t = time.time()
# can I parallelize this?
for origin_id in self.zone_ids:
# some optimal si might be empty, maybe because there were no drivers there to begin with
# print('calling LP model')
optimal_si[origin_id] = solve_for_one_zone(origin_id, one_t_ahead_move, dest_attraction[origin_id],
supply[(origin_id, t_fifteen)], t_fifteen)
print(f"it took {(time.time() - start_t) / 60} minutes to optimize all zones")
# pass the optimal si to the operator
self.operator.get_optimal_si(optimal_si)
def solve(self, draw=False):
print("Running Branch and Bound")
lp_copy = self.lp.copy() # not doing this copy is equivalent to keeping cutting planes around
obj, lp_soln = self.node_lower_bound(lp_copy, self.var_dict, self.graph)
if all([var[1] % 1 == 0 for _, var in lp_soln.items()]) and obj < self.best_soln[0]:
self.best_soln = (obj, lp_soln)
print("Updated best solution to: ", obj)
self.num_branch_nodes += 1
self.queue.put((obj, self.num_branch_nodes, grb.tupledict(), lp_soln))
while not self.queue.empty():
self.branch_step(draw)
return self.best_soln
def init_model(weigth_num, original_weight, overall_score):
model = gurobipy.Model('factor_model')
weight = {}
for j in range(weigth_num):
lb = max(0, original_weight[j] - stock_lower_bound)
ub = min(1, original_weight[j] + stock_upper_bound)
weight[j] = model.addVar(lb=lb,
ub=ub,
name='weight_' + str(j),
vtype=gurobipy.GRB.SEMICONT)
weight = gurobipy.tupledict(weight)
weight_binary = model.addVars(weigth_num,
lb=0,
ub=1,
vtype=gurobipy.GRB.BINARY,
name='weight_binary')
model.setObjective(weight.prod(overall_score), gurobipy.GRB.MAXIMIZE)
return model, weight, weight_binary
def read_nonzero_gurobi_sol(instance_name):
if instance_name + ".nonzero" in listdir(solutions_dir):
file = solutions_dir + instance_name + ".nonzero"
with open(file, 'r') as f:
f_content = [a.strip('\n') for a in f]
lines = f_content[1:]
lines = format_sol_lines(lines)
non_zero_solution = dict()
for var, vars_key, vars_value in lines:
if var not in non_zero_solution:
non_zero_solution[var] = tupledict()
non_zero_solution[var][vars_key] = vars_value
else:
non_zero_solution = read_gurobi_sol(instance_name)[1]
return non_zero_solution
def _general_circle_elimination(self, model: grb.Model):
edges_solution = model.cbGetSolution(self.edges)
used_edges = grb.tupledict({key: edges_solution[key] for key in edges_solution if not math.isclose(0, edges_solution[key], abs_tol=10**-5)})
for edge in used_edges:
if used_edges[edge] < 0.7:
print("Found an edge with value less than 0.7")
self._check_for_cycle(used_edges, model)
return
# Turn used_edges into a dependency graph
dep_graph = self._get_dep_graph(used_edges)
try:
calculate_order(dep_graph, calculate_circle_dep=True)
except CircularDependencyException as dep_exception:
self._add_cycle_constr(dep_exception.circle_nodes, model)
# For now we try to ignore this disconnected graphs
except DisconnectedDependencyGraphException as disc_exception:
cycle = calculate_cycle(disc_exception.disconnected_nodes)
self._add_cycle_constr(cycle, model)
def make_tupledict(matrix, *names):
"""Converts a matrix (in list of lists form) to a dictionary with tuples
as keys.
Arguments:
matrix {List[List[...]]} -- matrix as list of lists (row-major order).
names {List[List[Any]]} -- row/column names, in the order they are
in the list format.
Returns:
Dict[Tuple[...], int|float] -- dictionary indexed by name tuples.
"""
d = tupledict()
ranges = [range(len(x)) for x in names]
for r in product(*ranges):
if len(names) == 1:
key = names[0][r[0]]
else:
key = tuple(name[i] for name, i in zip(names, r))
d[key] = get_elem(matrix, r)
return d
def tsp_lp_initializer(graph):
lp = grb.Model("TSP Degree LP")
x = lp.addVars(
nx.edges(graph),
lb=0,
ub=1,
obj=[graph[edge[0]][edge[1]]['weight'] for edge in nx.edges(graph)],
vtype=grb.GRB.CONTINUOUS,
name='edgevars')
for n in nx.nodes(graph):
lp.addConstr(x.sum(n, '*') + x.sum('*', n),
grb.GRB.EQUAL,
2,
name=str(n))
x = grb.tupledict({index: x[index].VarName for index in x.keys()})
lp.update()
return lp, x
def _build_hamilton_path(self, current_node_index: int,
abstract_graph: Graph):
edges = grb.tupledict({
(u, v): self.edges[u, v]
for u, v in self.edges
if current_node_index in abstract_graph.vertices[u]
or current_node_index in abstract_graph.vertices[v]
})
vertices_index = {
i
for i in len(abstract_graph.vertices)
if current_node_index in abstract_graph.vertices[i]
}
for vertex_index in vertices_index:
self.model.addConstr(
edges.sum(vertex_index, '*') +
edges.sum('*', vertex_index) == 2)
# Only |V_i|-1 edges shall be used
self.model.addConstr(sum(edges) == len(vertices_index) - 1)
return edges
def gather_solution(self, DPP_sol_row, initial_solution):
counter = 0
s = gurobipy.tupledict()
tr = gurobipy.tupledict()
r = gurobipy.tupledict()
er = gurobipy.tupledict()
e = gurobipy.tupledict()
Tlen = self.problem_data.get_names("wildfire")
Ilen = self.problem_data.get_names("resources")
Glen = self.problem_data.get_names("groups")
for res in Ilen:
DPP = DPP_sol_row[counter]
for tt in Tlen:
s[res,tt] = DPP.get_variables().get_variable('s')[res,tt].X
tr[res,tt] = DPP.get_variables().get_variable('tr')[res,tt].X
r[res,tt] = DPP.get_variables().get_variable('r')[res,tt].X
er[res,tt] = DPP.get_variables().get_variable('er')[res,tt].X
e[res,tt] = DPP.get_variables().get_variable('e')[res,tt].X
counter = counter + 1
vars = gurobipy.tupledict()
vars["s"] = s
vars["tr"] = tr
vars["r"] = r
vars["er"] = er
vars["e"] = e
modelcopy = initial_solution.get_model().copy()
modelcopy.update()
sol1 = _sol.Solution(modelcopy, vars)
counter=0
for res in Ilen:
DPP = DPP_sol_row[counter]
for tt in Tlen:
sol1.get_model().getVarByName("start["+res+","+str(tt)+"]").start = DPP.get_variables().get_variable('s')[res,tt].X
sol1.get_model().getVarByName("travel["+str(res)+","+str(tt)+"]").start = DPP.get_variables().get_variable('tr')[res,tt].X
sol1.get_model().getVarByName("rest["+str(res)+","+str(tt)+"]").start = DPP.get_variables().get_variable('r')[res,tt].X
sol1.get_model().getVarByName("end_rest["+str(res)+","+str(tt)+"]").start = DPP.get_variables().get_variable('er')[res,tt].X
sol1.get_model().getVarByName("end["+str(res)+","+str(tt)+"]").start = DPP.get_variables().get_variable('e')[res,tt].X
counter = counter + 1
#for gro in Glen:
# for tt in Tlen:
# sol1.get_model().getVarByName("missing_resources["+gro+","+str(tt)+"]").start = DPP.get_variables().get_variable('mu')[gro,tt].X
sol1.get_model().update()
#print(sol1.get_model().getVars())
# mu[res,tt] = DPP.get_variables().get_variable('mu')[res,tt]
return sol1
for i in range(num_of_task):
task_i = input_content[3 + num_of_arc + i].split()
start_node = task_i[0]
end_node = task_i[1]
task_period = float(task_i[2])
task_deadline = float(task_i[3])
task_length = int(task_i[4])
full_S[str(i)] = [task_period, task_deadline, task_length]
route = nx.shortest_path(topology_graph,
source=start_node,
target=end_node)
for t in range(len(route) - 1):
full_F.append((route[t], route[t + 1], str(i)))
E, R, LE = multidict(E)
old_κ = tupledict({})
for va, vb in E:
old_κ[va, vb] = -1
old_φ = tupledict({})
old_ρ = tupledict({})
old_F = tuplelist([])
old_T = tupledict({})
old_e2e = tupledict({})
old_f = tuplelist([])
batch = 0
while len(full_S) > batch * size_of_batch:
#有序存储帧实例及属性
S = dict(
list(full_S.items())[size_of_batch * batch:size_of_batch *
(batch + 1)])
for i in range(numofarc):
arci = grid[3+i].split()
E.append((arci[0],arci[1]))
G.add_edge(arci[0],arci[1])
superT = max(superT,int(arci[3]))
draw(G)
fullS = {}
fullF = tuplelist([])
for i in range(numoftask):
taski = grid[3+numofarc+i].split()
fullS[str(i)] = [superT,int(taski[2])]
route = nx.shortest_path(G,source=taski[0],target=taski[1])
for t in range(len(route)-1):
fullF.append((route[t],route[t+1],str(i)))
oldκ = tupledict({})
for va,vb in E:
oldκ[va,vb] = -1
oldφ = tupledict({})
oldρ = tupledict({})
oldF = tuplelist([])
oldT = tupledict({})
oldL = tupledict({})
oldf = tuplelist([])
while len(fullS) > tur*batch:
#有序存储帧实例及属性
S = dict(list(fullS.items())[batch*tur:batch*(tur+1)])
S,e2e,size = multidict(S)
F = tuplelist([])
def solve(self, graph: Graph, **kwargs):
error_message = None
returned_order = None
is_optimal = False
runtime = 0
try:
self.build_model(graph)
if "time_limit" in kwargs:
self.model.setParam("TimeLimit", kwargs.pop("time_limit"))
self.add_start_solution(graph, kwargs.pop("start_solution", None))
self._add_callbacks(kwargs.pop("callbacks", None))
if kwargs.pop("relax", False):
old_edges = self.edges
used_edges = None
rel_model = self.model.relax()
keys, self.edges = grb.multidict({key: rel_model.getVarByName(self.edges[key].VarName) for key in self.edges})
rel_model.optimize(callback_rerouter)
runtime = self.model.Runtime
else:
circle_found = True
max_runtime = self.params["TimeLimit"]
while(circle_found and max_runtime > 0):
self.model.optimize(callback_rerouter)
max_runtime -= self.model.Runtime
runtime = abs(self.params["TimeLimit"] - max_runtime)
try:
used_edges = grb.tupledict({key: self.edges[key] for key in self.edges if not math.isclose(0, self.edges[key].x, abs_tol=10**-6)})
circle_found = self._check_for_cycle(used_edges, self.model, lazy=False)
if circle_found and max_runtime > 0:
self.model.setParam("TimeLimit", max_runtime)
except AttributeError as e:
# Can happen if no solution was found in the time limit
# If not, raise error
if runtime < self.params["TimeLimit"]:
raise e
is_optimal = self.model.Status == grb.GRB.OPTIMAL
if is_debug_env():
local_subtours = 0
for circle in self.found_circles:
verts = [key[1] for key in circle]
for v_i in self.v_incident_edges:
if self.v_incident_edges[v_i].issuperset(verts):
local_subtours += 1
break
print("Overall subtours:", len(self.found_circles), "local subtours:", local_subtours,\
"in percent:", local_subtours*100/len(self.found_circles))
try:
used_edges = {key: self.edges[key] for key in self.edges if not math.isclose(0, self.edges[key].x, abs_tol=10**-6)}
dep_graph = self._get_dep_graph(used_edges)
order = calculate_order(dep_graph, calculate_circle_dep=True)
returned_order = [tuple(self.abstract_graph.vertices[i]) for i in order]
except (CircularDependencyException, AttributeError) as e:
# If we have a circular dependency after the time limit we just didnt managed to get a feasable solution in time
# Else something went wrong and the error should be raised
if runtime < self.params["TimeLimit"]:
raise e
except Exception as e:
error_message = str(e)
if is_debug_env():
raise e
#times = calculate_times(returned_order, self.graph)
sol = AngularGraphSolution(self.graph,
runtime,
solution_type=self.solution_type,
solver=self.__class__.__name__,
is_optimal=is_optimal,
order=returned_order,
error_message=error_message)
return sol
def branch_step(self, draw=False):
model_copy = self.lp.copy()
queue_node = self.queue.get()
lp_value = queue_node[0]
# print("lp_value compared to best so far: ", lp_value, self.best_soln[0])
if lp_value > self.best_soln[0]:
print("branch node " + str(queue_node[1]) + " discarded directly from queue")
return
print("processing branch node: ", queue_node[1])
branches = queue_node[2]
soln = queue_node[3]
for index, var in branches.items():
model_var = model_copy.getVarByName(var[0])
# print("adding branch constraint: ", model_var, var[1])
model_copy.addConstr(model_var == var[1])
if draw:
edge_solution = grb.tupledict([(index, soln[index][1]) for index in soln.keys()])
drawsolution.draw(self.graph, edge_solution)
model_copy.update()
branch_var = self.branch_rule(model_copy, self.graph, (lp_value, soln))
print("branch_var: ", branch_var)
if branch_var is None:
return
model_copy.update()
lp0 = model_copy.copy()
lp1 = model_copy.copy()
var0 = lp0.getVarByName(branch_var[1])
lp0.addConstr(var0 == 0)
branches0 = dict(branches)
branches0[branch_var[0]] = (branch_var[1], 0)
var1 = lp1.getVarByName(branch_var[1])
lp1.addConstr(var1 == 1)
branches1 = dict(branches)
branches1[branch_var[0]] = (branch_var[1], 1)
lp0.update()
lp1.update()
# print("lp0 model constraint RHS: ", list(map(lambda x: x.getAttr('RHS'), lp0.getConstrs())))
# print("lp1 model constraint RHS: ", list(map(lambda x: x.getAttr('RHS'), lp1.getConstrs())))
a = time.clock()
obj0, lp0_soln = self.node_lower_bound(lp0, self.var_dict, self.graph)
obj1, lp1_soln = self.node_lower_bound(lp1, self.var_dict, self.graph)
print("Time to solve new LPs: ", time.clock() - a)
print("new lp objective values: ", lp0.objVal, lp1.objVal)
if all([var[1] % 1 == 0 for _, var in lp0_soln.items()]) and obj0 < self.best_soln[0]:
self.best_soln = (obj0, lp0_soln)
print("Updated best solution to: ", obj0)
elif obj0 < self.best_soln[0]:
self.num_branch_nodes += 1
print("adding branch node: ", self.num_branch_nodes)
self.queue.put((obj0, self.num_branch_nodes, branches0, lp0_soln))
# print("fractional variables when adding to queue: ",
# [(index, var[0], var[1]) for index, var in lp1_soln.items() if 0 < var[1] < 1])
# print("queue node and branches for insert: ", self.num_branch_nodes, branches1)
else:
print("Discarding solution; obj value compared to best solution: ", obj0, self.best_soln[0])
if all([var[1] % 1 == 0 for _, var in lp1_soln.items()]) and obj1 < self.best_soln[0]:
self.best_soln = (obj1, lp1_soln)
print("Updated best solution to: ", obj1)
elif obj1 < self.best_soln[0]:
self.num_branch_nodes += 1
print("adding branch node: ", self.num_branch_nodes)
self.queue.put((obj1, self.num_branch_nodes, branches1, lp1_soln))
# print("fractional variables when adding to queue: ",
# [(index, var[0], var[1]) for index, var in lp1_soln.items() if 0 < var[1] < 1])
# print("queue node and branches for insert: ", self.num_branch_nodes, branches1)
else:
print("Discarding solution; obj value compared to best solution: ", obj1, self.best_soln[0])
def getSolution(problem):
n = len(problem)
s = int(np.sqrt(n))
model = gp.Model('Sudoku')
model.setParam('OutputFlag', False)
# create 9 binary variables for each cell without given value
var = gp.tupledict()
for i in range(n):
for j in range(n):
if problem[i, j] == 0:
for k in range(n):
var[(i, j,
k)] = model.addVar(0,
1,
0,
GRB.BINARY,
name='v[{},{},{}]'.format(i, j, k))
# Assert each cell assumes one value
model.addConstrs((var.sum(i, j, '*') == 1 for i in range(n)
for j in range(n) if problem[i, j] == 0),
name='cell')
# Assert there are no duplicate values in a row
model.addConstrs((var.sum(i, '*', k) == 1 for i in range(n)
for k in range(n) if k + 1 not in problem[i, :]),
name='row')
# Assert there are no duplicate values in a column
model.addConstrs((var.sum('*', j, k) == 1 for j in range(n)
for k in range(n) if k + 1 not in problem[:, j]),
name='column')
# Assert there are no duplicate values in a subgrid
for si in range(s):
for sj in range(s):
for k in range(n):
if k + 1 not in problem[si * s:(si + 1) * s,
sj * s:(sj + 1) * s]:
model.addConstr(gp.quicksum(
var[i, j, k] for i in range(si * s, (si + 1) * s)
for j in range(sj * s, (sj + 1) * s)
if problem[i, j] == 0) == 1,
name='Sub[{},{},{}]'.format(si, sj, k))
# optimize model
model.optimize()
if model.Status == GRB.OPTIMAL:
# get solution
sol = model.getAttr('X', var)
solution = problem.copy()
for key, val in sol.items():
if val > 0.5:
solution[key[0], key[1]] = int(key[2] + 1)
else:
solution = np.zeros([n, n], dtype='int')
return model, solution
def combine_representations(dist_matrix, voi_index, S_indices, return_new_dists=True, threshold=None, solver='coin_cmd', api='py-mip', variable_num_tol=0.001, max_seconds=np.inf):
"""
A function to find the weights of optimal linear combination of representations.
Input
dist_matrix - np.array (shape=(n, J))
Array containing the distances between the vertex of interest and the other n - 1
vertices.
voi_index - int
Index of vertex of interest.
S_indices - array-like
Indices of the vertices that should be at the top of the
nomination list for the vertex of interest.
return_new_dists - bool
If true, returns both the weights and the corresponding distance matrix.
threshold - maximum value of the rank of an element of S_indices.
solver - solver to use. in {'coin_cmd'}
api - api to use. in {'gurobi', 'py-mip', 'pulp'}
variable_num_tol - float in (0, 1]. resolution of the approximation of the continuous weights.
If None, continuous weights are found.
max_seconds - float in (0, inf)
Return
weights - np.array (length=J)
Array containing the coefficients for the optimal distance function.
-- optional -- new_dists - np.array (length=n)
Array containing the distances after applying the learned weight vector.
"""
# Grab the shape of the data that was passed int
n, J = dist_matrix.shape
# Pre-process the data so that there are no elements of S_indices that are above threshold
if threshold is not None:
ranks = evaluate_best_vertices(dist_matrix, vertices=np.arange(J), s_star=S_indices)
dist_matrix = edit_dist_matrices(dist_matrix, S_indices, ranks, threshold)
# Grab the maximum value of dist_matrix (to be used as an upper bound later)
M = np.max(abs(dist_matrix))
# Grab the number of elements known to be similar to voi_index
S = len(S_indices)
# Define an array of integers corresponding to elements not in S_indices
Q_indices = np.array([int(i) for i in np.concatenate((range(0, voi_index), range(voi_index+1, n))) if i not in S_indices])
# Grab the number of elements not known to be similar to voi_index
Q = len(Q_indices)
# We can either use continuous weights for the representations or use discrete approximation
if variable_num_tol is not None:
# Here, we are using a discrete approximation
# variable_num_tol is in the interval (0, 1]. If variable_num_tol is close to 0 that means
# we want our approximation to be of a high resolution. To achieve this, we define 1 / variable_num_tol
# to be the maximum value that a particular weight can take on.
# i.e. with variable_num_tol = 0.1, the weights can take on values 0.0, 0.1, 0.2, 0.3, .., 1.
# We normalize later.
up_bound = int(np.math.ceil(1 / variable_num_tol))
variable_num_tol = 1 / up_bound
cat='Integer'
else:
# Here, we let the weights be continuous
up_bound=1
cat='Continuous'
# We've implemented the ILP in 3 different APIs: pulp, py-mip and gurobi.
# Gurobi seems to be the industry standard but licensing is a bit prohibitive.
# Each API is relatively similar. First, you define a set of variables.
# Then, using these variables, you define an objective function and a set of constraints that you assign to a model object.
if api == 'pulp':
# Define a model.
model=pulp.LpProblem(sense=pulp.LpMinimize)
# Define indicator variables (as defined in section 1.2 of https://arxiv.org/pdf/2005.10700.pdf) for every vertex
# That is not in S_indices.
# NB: i am more than happy to walk through the paper if that would be helpful!
inds = pulp.LpVariable.dicts("indicators for elements not in S",
(q for q in Q_indices),
cat='Integer',
upBound=1,
lowBound=0
)
# Define non-negative weight variables for each of the representations.
weights = pulp.LpVariable.dicts("weights for representations",
(j for j in range(J)),
cat=cat,
upBound=up_bound,
lowBound=0
)
# Set the objective function.
model += (
pulp.lpSum(
[inds[(q)] for q in Q_indices]
)
)
# Add constraint that the weights must sum to the upper bound defined by variable_num_tol.
model += (
pulp.lpSum(
[weights[(j)] for j in range(J)]
) == up_bound
)
# Add constraint that elements of S_indices should be closer than elements not in S_indices (or, the Q_indices)
for s in S_indices:
for q in Q_indices:
model += (
pulp.lpSum(
[weights[(j)] * dist_matrix[s, j] for j in range(J)]
)
<=
pulp.lpSum(
[weights[(j)] * dist_matrix[q, j] for j in range(J)]
)
+
pulp.lpSum(inds[(q)] * M * up_bound)
)
# Different solvers for api == 'pulp'
try:
if solver == 'pulp':
model.solve()
elif solver == 'coin_cmd':
model.solve(solver=pulp.COIN_CMD())
alpha_hat = np.array([w.varValue for w in weights.values()])
except Exception as e:
print(e)
return None
alpha_hat = np.array([w.varValue for w in weights.values()])
elif api=='gurobi':
model= gp.Model()
model.setParam('OutputFlag', 0)
model.setParam('TimeLimit', max_seconds)
ind = model.addVars(Q, vtype=GRB.BINARY, name='ind')
model.setObjective(gp.quicksum(ind), GRB.MINIMIZE)
w = model.addVars(J, lb=0, ub=1, vtype=GRB.CONTINUOUS, name='w')
model.addConstr(w.sum() == 1)
for s in S_indices:
temp_s = gp.tupledict([((i), dist_matrix[s, i]) for i in range(J)])
for i, q in enumerate(Q_indices):
temp_q = gp.tupledict([((i), dist_matrix[q, i]) for i in range(J)])
model.addConstr(w.prod(temp_s) <= w.prod(temp_q) + ind[i]*M)
model.optimize()
alpha_hat = np.array([i.X for i in list(w.values())])
elif api=='py-mip':
model = mip.Model(sense=mip.MINIMIZE)
inds = [model.add_var(name='inds', var_type=mip.BINARY) for q in range(Q)]
# if variable_num_tol is None:
weights = [model.add_var(name='weights', lb=0.0, ub=up_bound, var_type=cat) for j in range(J)]
model += mip.xsum(w for w in weights) == 1
# else:
# weights = [model.add_var(name='weights', lb=0, ub=up_bound, var_type='I') for j in range(J)]
# model += mip.xsum(w for w in weights) == up_bound
for s in S_indices:
for i, q in enumerate(Q_indices):
model += mip.xsum(weights[j] * dist_matrix[s, j] for j in range(J)) <= mip.xsum(weights[j] * dist_matrix[q, j] for j in range(J)) + inds[i]*M*up_bound
model.objective = mip.xsum(ind for ind in inds)
model.optimize(max_seconds=max_seconds)
alpha_hat = np.array([w.x for w in weights])
else:
raise ValueError("api %s not implemented"%(api))
# Return the new distances if told to do so.
if return_new_dists:
if np.sum(alpha_hat == 0) == J:
return alpha_hat, dist_matrix
else:
return alpha_hat, np.average(dist_matrix, axis=1, weights=alpha_hat)
if alpha_hat[0] is None:
return None
else:
# Normalize
return alpha_hat / np.sum(alpha_hat)
def construct_model_layers(self, gurobi_vars, layers, lower_bounds, upper_bounds, var_name_str=''):
layer_idx = 1 # this starts at 1 -- i.e. the second element of lower_bounds/upper_bounds (0 is input)
# and is only incremented after hitting a ReLU. Thus passing the lower_bound and not
# relu_lower_bounds from lin_net is correct.
for layer in layers:
if isinstance(layer, nn.Linear):
layer_nb_out = layer.out_features
pre_vars = gurobi_vars[-1]
if isinstance(pre_vars, grb.tupledict):
pre_vars = [var for key, var in sorted(pre_vars.items())]
# Build all the outputs of the linear layer
new_vars = self.model.addVars([i for i in range(layer_nb_out)],
lb=lower_bounds[layer_idx],
ub=upper_bounds[layer_idx],
name=f'zhat{layer_idx}_{var_name_str}')
new_layer_gurobi_vars = [var for key, var in new_vars.items()]
self.model.addConstrs(
((grb.LinExpr(layer.weight[neuron_idx, :], pre_vars)
+ layer.bias[neuron_idx].item()) == new_vars[neuron_idx]
for neuron_idx in range(layer.out_features)),
name=f'lay{layer_idx}_{var_name_str}'
)
elif isinstance(layer, nn.ReLU):
pre_lbs = lower_bounds[layer_idx]
pre_ubs = upper_bounds[layer_idx]
if isinstance(gurobi_vars[-1], grb.tupledict):
amb_mask = (pre_lbs < 0) & (pre_ubs > 0)
if amb_mask.sum().item() != 0:
to_new_preubs = pre_ubs[amb_mask]
to_new_prelbs = pre_lbs[amb_mask]
new_var_idxs = torch.nonzero((pre_lbs < 0) & (pre_ubs > 0)).numpy().tolist()
new_var_idxs = [tuple(idxs) for idxs in new_var_idxs]
new_layer_gurobi_vars = self.model.addVars(new_var_idxs,
lb=0,
ub=to_new_preubs,
name=f'z{layer_idx}_{var_name_str}')
new_binary_vars = self.model.addVars(new_var_idxs,
lb=0, ub=1,
vtype=grb.GRB.BINARY,
name=f'delta{layer_idx}_{var_name_str}')
flat_new_vars = [new_layer_gurobi_vars[idx] for idx in new_var_idxs]
flat_binary_vars = [new_binary_vars[idx] for idx in new_var_idxs]
pre_amb_vars = [gurobi_vars[-1][idx] for idx in new_var_idxs]
# C1: Superior to 0
# C2: Add the constraint that it's superior to the inputs
self.model.addConstrs(
(flat_new_vars[idx] >= pre_amb_vars[idx]
for idx in range(len(flat_new_vars))),
name=f'ReLU_lb{layer_idx}_{var_name_str}'
)
# C3: Below binary*upper_bound
self.model.addConstrs(
(flat_new_vars[idx] <= to_new_preubs[idx].item() * flat_binary_vars[idx]
for idx in range(len(flat_new_vars))),
name=f'ReLU{layer_idx}_ub1-{var_name_str}'
)
# C4: Below binary*lower_bound
self.model.addConstrs(
(flat_new_vars[idx] <= (pre_amb_vars[idx]
- to_new_prelbs[idx].item() * (1 - flat_binary_vars[idx]))
for idx in range(len(flat_new_vars))),
name=f'ReLU{layer_idx}_ub2-{var_name_str}'
)
else:
new_layer_gurobi_vars = grb.tupledict()
for pos in torch.nonzero(pre_lbs >= 0).numpy().tolist():
pos = tuple(pos)
new_layer_gurobi_vars[pos] = gurobi_vars[-1][pos]
for pos in torch.nonzero(pre_ubs <= 0).numpy().tolist():
new_layer_gurobi_vars[tuple(pos)] = self.zero_var
else:
assert isinstance(gurobi_vars[-1][0], grb.Var)
amb_mask = (pre_lbs < 0) & (pre_ubs > 0)
if amb_mask.sum().item() == 0:
pass
# print("WARNING: No ambiguous ReLU at a layer")
else:
to_new_preubs = pre_ubs[amb_mask]
new_var_idxs = torch.nonzero(amb_mask).squeeze(1).numpy().tolist()
new_vars = self.model.addVars(new_var_idxs,
lb=0,
ub=to_new_preubs,
name=f'z{layer_idx}_{var_name_str}')
new_binary_vars = self.model.addVars(new_var_idxs,
lb=0, ub=1,
vtype=grb.GRB.BINARY,
name=f'delta{layer_idx}_{var_name_str}')
# C1: Superior to 0
# C2: Add the constraint that it's superior to the inputs
self.model.addConstrs(
(new_vars[idx] >= gurobi_vars[-1][idx]
for idx in new_var_idxs),
name=f'ReLU_lb{layer_idx}_{var_name_str}'
)
# C3: Below binary*upper_bound
self.model.addConstrs(
(new_vars[idx] <= pre_ubs[idx].item() * new_binary_vars[idx]
for idx in new_var_idxs),
name=f'ReLU{layer_idx}_ub1-{var_name_str}'
)
# C4: Below binary*lower_bound
self.model.addConstrs(
(new_vars[idx] <= (gurobi_vars[-1][idx]
- pre_lbs[idx].item() * (1 - new_binary_vars[idx]))
for idx in new_var_idxs),
name=f'ReLU{layer_idx}_ub2-{var_name_str}'
)
# Get all the variables in a list, such that we have the
# output of the layer
new_layer_gurobi_vars = []
new_idx = 0
for idx in range(layer_nb_out):
if pre_lbs[idx] >= 0:
# Pass through variable
new_layer_gurobi_vars.append(gurobi_vars[-1][idx])
elif pre_ubs[idx] <= 0:
# Blocked variable
new_layer_gurobi_vars.append(self.zero_var)
else:
new_layer_gurobi_vars.append(new_vars[idx])
layer_idx += 1
elif isinstance(layer, ibp.Sparsemax):
pre_vars = gurobi_vars[-1] # these pre_vars should be of dim 2
assert len(pre_vars.shape) == 2
# note we have to subtract 1 from sparsemax dim to remove batch dimension
dim = layer.dim - 1
if dim == 0:
# slide over rows
other_dim = 1
d = pre_vars.shape[dim]
sparsemax_grb_vars = []
for col_dim in range(pre_vars.shape[other_dim]): # iterate over columns
curr_column = pre_vars[:, col_dim]
# sparsemax with these input variables
# create sparsemax out_vars, set them to sum to 1
sparsemax_z = self.model.addVars(list(range(d)), lb=0, ub=1, name=f'sparsemax_col{col_dim}_{var_name_str}')
sparsemax_grb_vars.append([sparsemax_z[i] for i in range(d)])
self.model.addConstr(grb.quicksum(sparsemax_z) == 1, name=f'sparsemax_col{col_dim}_sum_{var_name_str}')
mu1 = self.model.addVars(list(range(d)), lb=0, name=f'sparsemax_mu1_{col_dim}_{var_name_str}')
mu2 = self.model.addVars(list(range(d)), lb=0, name=f'sparsemax_mu2_{col_dim}_{var_name_str}')
lam = self.model.addVar(lb=-grb.GRB.INFINITY, ub=grb.GRB.INFINITY, name=f'sparsemax_lambda_{col_dim}_{var_name_str}')
self.model.addConstrs( (((sparsemax_z[i] - curr_column[i]) + mu1[i] - mu2[i] + lam == 0) for i in range(d)), name=f'sparsemax_stationarity_{col_dim}_{var_name_str}')
# add z minus 1 and negative z
# not sure if we actually need negative z
zminusone = self.model.addVars([i for i in range(d)], lb=-1, ub=0, name=f'sparsemax_zminusone_{col_dim}_{var_name_str}')
self.model.addConstrs((zminusone[i] == (sparsemax_z[i] - 1) for i in range(d)), name=f'sparsemax_zminusone_{col_dim}_constr_{var_name_str}')
negz = self.model.addVars([i for i in range(d)], lb=-1, ub=0, name=f'sparsemax_negz_{col_dim}_{var_name_str}')
self.model.addConstrs((-negz[i] == sparsemax_z[i] for i in range(d)), name=f'sparsemax_negz_{col_dim}_{var_name_str}')
# complementary slackness
for i in range(d):
self.model.addSOS(grb.GRB.SOS_TYPE1, [zminusone[i], mu1[i]], wts=[1, 2])
self.model.addSOS(grb.GRB.SOS_TYPE1, [negz[i], mu2[i]], wts=[1, 2])
# at this point we have sparsemax_grb_vars, with 1 nested list. need to transpose for columns
new_layer_gurobi_vars = np.array(sparsemax_grb_vars).transpose()
elif dim == 1:
raise NotImplementedError("row-wise not implemented")
else:
raise NotImplementedError("sparsemax for more than 2D not implemented")
elif isinstance(layer, ibp.View):
pre_vars = gurobi_vars[-1] # previous activations
viewed_vars = np.reshape(pre_vars, layer.shape[1:]) # [1:] is to drop batch dim
new_layer_gurobi_vars = viewed_vars
elif isinstance(layer, ibp.View_Cut):
pre_vars = gurobi_vars[-1]
viewed_vars = pre_vars[:-1, :]
new_layer_gurobi_vars = viewed_vars
elif isinstance(layer, Flatten):
raise NotImplementedError("flatten should be manually removed right now")
else:
raise NotImplementedError
gurobi_vars.append(new_layer_gurobi_vars)
