class InterfaceToSolver:
"""A wrapper for the mip model class, allows interaction with mip using pd.DataFrames."""
def __init__(self, solver_name='CBC'):
self.variables = {}
self.linear_mip_variables = {}
self.solver_name = solver_name
if solver_name == 'CBC':
self.mip_model = Model("market", solver_name=CBC)
self.linear_mip_model = Model("market", solver_name=CBC)
elif solver_name == 'GUROBI':
self.mip_model = Model("market", solver_name=GUROBI)
self.linear_mip_model = Model("market", solver_name=GUROBI)
else:
raise ValueError("Solver '{}' not recognised.")
self.mip_model.verbose = 0
self.mip_model.solver.set_mip_gap_abs(1e-10)
self.mip_model.solver.set_mip_gap(1e-20)
self.mip_model.lp_method = LP_Method.DUAL
self.linear_mip_model.verbose = 0
self.linear_mip_model.solver.set_mip_gap_abs(1e-10)
self.linear_mip_model.solver.set_mip_gap(1e-20)
self.linear_mip_model.lp_method = LP_Method.DUAL
def add_variables(self, decision_variables):
"""Add decision variables to the model.
Examples
--------
>>> decision_variables = pd.DataFrame({
... 'variable_id': [0, 1],
... 'lower_bound': [0.0, 0.0],
... 'upper_bound': [6.0, 1.0],
... 'type': ['continuous', 'binary']})
>>> si = InterfaceToSolver()
>>> si.add_variables(decision_variables)
The underlying mip_model should now have 2 variables.
>>> print(si.mip_model.num_cols)
2
The first one should have the following properties.
>>> print(si.mip_model.var_by_name('0').var_type)
C
>>> print(si.mip_model.var_by_name('0').lb)
0.0
>>> print(si.mip_model.var_by_name('0').ub)
6.0
The second one should have the following properties.
>>> print(si.mip_model.var_by_name('1').var_type)
B
>>> print(si.mip_model.var_by_name('1').lb)
0.0
>>> print(si.mip_model.var_by_name('1').ub)
1.0
"""
# Create a mapping between the nempy level names for variable types and the mip representation.
variable_types = {'continuous': CONTINUOUS, 'binary': BINARY}
# Add each variable to the mip model.
for variable_id, lower_bound, upper_bound, variable_type in zip(
list(decision_variables['variable_id']),
list(decision_variables['lower_bound']),
list(decision_variables['upper_bound']),
list(decision_variables['type'])):
self.variables[variable_id] = self.mip_model.add_var(
lb=lower_bound,
ub=upper_bound,
var_type=variable_types[variable_type],
name=str(variable_id))
self.linear_mip_variables[
variable_id] = self.linear_mip_model.add_var(
lb=lower_bound,
ub=upper_bound,
var_type=variable_types[variable_type],
name=str(variable_id))
def add_sos_type_2(self, sos_variables, sos_id_columns, position_column):
"""Add groups of special ordered sets of type 2 two the mip model.
Examples
--------
>>> decision_variables = pd.DataFrame({
... 'variable_id': [0, 1, 2, 3, 4, 5],
... 'lower_bound': [0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
... 'upper_bound': [5.0, 5.0, 5.0, 5.0, 5.0, 5.0],
... 'type': ['continuous', 'continuous', 'continuous',
... 'continuous', 'continuous', 'continuous']})
>>> sos_variables = pd.DataFrame({
... 'variable_id': [0, 1, 2, 3, 4, 5],
... 'sos_id': ['A', 'A', 'A', 'B', 'B', 'B'],
... 'position': [0, 1, 2, 0, 1, 2]})
>>> si = InterfaceToSolver()
>>> si.add_variables(decision_variables)
>>> si.add_sos_type_2(sos_variables, 'sos_id', 'position')
"""
# Function that adds sets to mip model.
def add_sos_vars(sos_group):
self.mip_model.add_sos(
list(zip(sos_group['vars'], sos_group[position_column])), 2)
# For each variable_id get the variable object from the mip model
sos_variables['vars'] = sos_variables['variable_id'].apply(
lambda x: self.variables[x])
# Break up the sets based on their id and add them to the model separately.
sos_variables.groupby(sos_id_columns).apply(add_sos_vars)
# This is a hack to make sure mip knows there are binary constraints.
self.mip_model.add_var(var_type=BINARY, obj=0.0)
def add_sos_type_1(self, sos_variables):
# Function that adds sets to mip model.
def add_sos_vars(sos_group):
self.mip_model.add_sos(
list(
zip(sos_group['vars'],
[1.0 for i in range(len(sos_variables['vars']))])), 1)
# For each variable_id get the variable object from the mip model
sos_variables['vars'] = sos_variables['variable_id'].apply(
lambda x: self.variables[x])
# Break up the sets based on their id and add them to the model separately.
sos_variables.groupby('sos_id').apply(add_sos_vars)
# This is a hack to make mip knows there are binary constraints.
self.mip_model.add_var(var_type=BINARY, obj=0.0)
def add_objective_function(self, objective_function):
"""Add the objective function to the mip model.
Examples
--------
>>> decision_variables = pd.DataFrame({
... 'variable_id': [0, 1, 2, 3, 4, 5],
... 'lower_bound': [0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
... 'upper_bound': [5.0, 5.0, 5.0, 5.0, 5.0, 5.0],
... 'type': ['continuous', 'continuous', 'continuous',
... 'continuous', 'continuous', 'continuous']})
>>> objective_function = pd.DataFrame({
... 'variable_id': [0, 1, 3, 4, 5],
... 'cost': [1.0, 2.0, -1.0, 5.0, 0.0]})
>>> si = InterfaceToSolver()
>>> si.add_variables(decision_variables)
>>> si.add_objective_function(objective_function)
>>> print(si.mip_model.var_by_name('0').obj)
1.0
>>> print(si.mip_model.var_by_name('5').obj)
0.0
"""
objective_function = objective_function.sort_values('variable_id')
objective_function = objective_function.set_index('variable_id')
obj = minimize(
xsum(objective_function['cost'][i] * self.variables[i]
for i in list(objective_function.index)))
self.mip_model.objective = obj
self.linear_mip_model.objective = obj
def add_constraints(self, constraints_lhs, constraints_type_and_rhs):
"""Add constraints to the mip model.
Examples
--------
>>> decision_variables = pd.DataFrame({
... 'variable_id': [0, 1, 2, 3, 4, 5],
... 'lower_bound': [0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
... 'upper_bound': [5.0, 5.0, 10.0, 10.0, 5.0, 5.0],
... 'type': ['continuous', 'continuous', 'continuous',
... 'continuous', 'continuous', 'continuous']})
>>> constraints_lhs = pd.DataFrame({
... 'constraint_id': [1, 1, 2, 2],
... 'variable_id': [0, 1, 3, 4],
... 'coefficient': [1.0, 0.5, 1.0, 2.0]})
>>> constraints_type_and_rhs = pd.DataFrame({
... 'constraint_id': [1, 2],
... 'type': ['<=', '='],
... 'rhs': [10.0, 20.0]})
>>> si = InterfaceToSolver()
>>> si.add_variables(decision_variables)
>>> si.add_constraints(constraints_lhs, constraints_type_and_rhs)
>>> print(si.mip_model.constr_by_name('1'))
1: +1.0 0 +0.5 1 <= 10.0
>>> print(si.mip_model.constr_by_name('2'))
2: +1.0 3 +2.0 4 = 20.0
"""
constraints_lhs = constraints_lhs.groupby(
['constraint_id', 'variable_id'],
as_index=False).agg({'coefficient': 'sum'})
rows = constraints_lhs.groupby(['constraint_id'], as_index=False)
# Make a dictionary so constraint rhs values can be accessed using the constraint id.
rhs = dict(
zip(constraints_type_and_rhs['constraint_id'],
constraints_type_and_rhs['rhs']))
# Make a dictionary so constraint type can be accessed using the constraint id.
enq_type = dict(
zip(constraints_type_and_rhs['constraint_id'],
constraints_type_and_rhs['type']))
var_ids = constraints_lhs['variable_id'].to_numpy()
vars = np.asarray([
self.variables[k] if k in self.variables.keys() else None
for k in range(0,
max(var_ids) + 1)
])
coefficients = constraints_lhs['coefficient'].to_numpy()
for row_id, row in rows.indices.items():
# Use the variable_ids to get mip variable objects present in the constraints
lhs_variables = vars[var_ids[row]]
# Use the positions of the non nan values to the lhs coefficients.
lhs = coefficients[row]
# Multiply and the variables by their coefficients and sum to create the lhs of the constraint.
exp = lhs_variables * lhs
exp = exp.tolist()
exp = xsum(exp)
# Add based on inequality type.
if enq_type[row_id] == '<=':
new_constraint = exp <= rhs[row_id]
elif enq_type[row_id] == '>=':
new_constraint = exp >= rhs[row_id]
elif enq_type[row_id] == '=':
new_constraint = exp == rhs[row_id]
else:
raise ValueError(
"Constraint type not recognised should be one of '<=', '>=' or '='."
)
self.mip_model.add_constr(new_constraint, name=str(row_id))
self.linear_mip_model.add_constr(new_constraint, name=str(row_id))
def optimize(self):
"""Optimize the mip model.
If an optimal solution cannot be found and the investigate_infeasibility flag is set to True then remove
constraints until a feasible solution is found.
Examples
--------
>>> decision_variables = pd.DataFrame({
... 'variable_id': [0, 1, 2, 3, 4, 5],
... 'lower_bound': [0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
... 'upper_bound': [5.0, 5.0, 10.0, 10.0, 5.0, 5.0],
... 'type': ['continuous', 'continuous', 'continuous',
... 'continuous', 'continuous', 'continuous']})
>>> constraints_lhs = pd.DataFrame({
... 'constraint_id': [1, 1, 2, 2],
... 'variable_id': [0, 1, 3, 4],
... 'coefficient': [1.0, 0.5, 1.0, 2.0]})
>>> constraints_type_and_rhs = pd.DataFrame({
... 'constraint_id': [1, 2],
... 'type': ['<=', '='],
... 'rhs': [10.0, 20.0]})
>>> si = InterfaceToSolver()
>>> si.add_variables(decision_variables)
>>> si.add_constraints(constraints_lhs, constraints_type_and_rhs)
>>> si.optimize()
>>> decision_variables['value'] = si.get_optimal_values_of_decision_variables(decision_variables)
>>> print(decision_variables)
variable_id lower_bound upper_bound type value
0 0 0.0 5.0 continuous 0.0
1 1 0.0 5.0 continuous 0.0
2 2 0.0 10.0 continuous 0.0
3 3 0.0 10.0 continuous 10.0
4 4 0.0 5.0 continuous 5.0
5 5 0.0 5.0 continuous 0.0
"""
status = self.mip_model.optimize()
if status != OptimizationStatus.OPTIMAL:
# Attempt find constraint causing infeasibility.
print('Model infeasible attempting to find problem constraint.')
con_index = find_problem_constraint(self.mip_model)
print(
'Couldn\'t find an optimal solution, but removing con {} fixed INFEASIBLITY'
.format(con_index))
raise ValueError('Linear program infeasible')
def get_optimal_values_of_decision_variables(self, variable_definitions):
"""Get the optimal values for each decision variable.
Examples
--------
>>> decision_variables = pd.DataFrame({
... 'variable_id': [0, 1, 2, 3, 4, 5],
... 'lower_bound': [0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
... 'upper_bound': [5.0, 5.0, 10.0, 10.0, 5.0, 5.0],
... 'type': ['continuous', 'continuous', 'continuous',
... 'continuous', 'continuous', 'continuous']})
>>> constraints_lhs = pd.DataFrame({
... 'constraint_id': [1, 1, 2, 2],
... 'variable_id': [0, 1, 3, 4],
... 'coefficient': [1.0, 0.5, 1.0, 2.0]})
>>> constraints_type_and_rhs = pd.DataFrame({
... 'constraint_id': [1, 2],
... 'type': ['<=', '='],
... 'rhs': [10.0, 20.0]})
>>> si = InterfaceToSolver()
>>> si.add_variables(decision_variables)
>>> si.add_constraints(constraints_lhs, constraints_type_and_rhs)
>>> si.optimize()
>>> decision_variables['value'] = si.get_optimal_values_of_decision_variables(decision_variables)
>>> print(decision_variables)
variable_id lower_bound upper_bound type value
0 0 0.0 5.0 continuous 0.0
1 1 0.0 5.0 continuous 0.0
2 2 0.0 10.0 continuous 0.0
3 3 0.0 10.0 continuous 10.0
4 4 0.0 5.0 continuous 5.0
5 5 0.0 5.0 continuous 0.0
"""
values = variable_definitions['variable_id'].apply(
lambda x: self.mip_model.var_by_name(str(x)).x, self.mip_model)
return values
def get_optimal_values_of_decision_variables_lin(self,
variable_definitions):
values = variable_definitions['variable_id'].apply(
lambda x: self.linear_mip_model.var_by_name(str(x)).x,
self.mip_model)
return values
def get_slack_in_constraints(self, constraints_type_and_rhs):
"""Get the slack values in each constraint.
Examples
--------
>>> decision_variables = pd.DataFrame({
... 'variable_id': [0, 1, 2, 3, 4, 5],
... 'lower_bound': [0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
... 'upper_bound': [5.0, 5.0, 10.0, 10.0, 5.0, 5.0],
... 'type': ['continuous', 'continuous', 'continuous',
... 'continuous', 'continuous', 'continuous']})
>>> constraints_lhs = pd.DataFrame({
... 'constraint_id': [1, 1, 2, 2],
... 'variable_id': [0, 1, 3, 4],
... 'coefficient': [1.0, 0.5, 1.0, 2.0]})
>>> constraints_type_and_rhs = pd.DataFrame({
... 'constraint_id': [1, 2],
... 'type': ['<=', '='],
... 'rhs': [10.0, 20.0]})
>>> si = InterfaceToSolver()
>>> si.add_variables(decision_variables)
>>> si.add_constraints(constraints_lhs, constraints_type_and_rhs)
>>> si.optimize()
>>> constraints_type_and_rhs['slack'] = si.get_slack_in_constraints(constraints_type_and_rhs)
>>> print(constraints_type_and_rhs)
constraint_id type rhs slack
0 1 <= 10.0 10.0
1 2 = 20.0 0.0
"""
slack = constraints_type_and_rhs['constraint_id'].apply(
lambda x: self.mip_model.constr_by_name(str(x)).slack,
self.mip_model)
return slack
def price_constraints(self, constraint_ids_to_price):
"""For each constraint_id find the marginal value of the constraint.
This is done by incrementing the constraint by a value of 1.0 and re-optimizing the model, the marginal cost
of the constraint is increase in the objective function value between model runs.
Examples
--------
>>> decision_variables = pd.DataFrame({
... 'variable_id': [0, 1, 2, 3, 4, 5],
... 'lower_bound': [0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
... 'upper_bound': [5.0, 5.0, 10.0, 10.0, 5.0, 5.0],
... 'type': ['continuous', 'continuous', 'continuous',
... 'continuous', 'continuous', 'continuous']})
>>> objective_function = pd.DataFrame({
... 'variable_id': [0, 1, 2, 3, 4, 5],
... 'cost': [1.0, 3.0, 10.0, 8.0, 9.0, 7.0]})
>>> constraints_lhs = pd.DataFrame({
... 'constraint_id': [1, 1, 1, 1],
... 'variable_id': [0, 1, 3, 4],
... 'coefficient': [1.0, 1.0, 1.0, 1.0]})
>>> constraints_type_and_rhs = pd.DataFrame({
... 'constraint_id': [1],
... 'type': ['='],
... 'rhs': [20.0]})
>>> si = InterfaceToSolver()
>>> si.add_variables(decision_variables)
>>> si.add_constraints(constraints_lhs, constraints_type_and_rhs)
>>> si.add_objective_function(objective_function)
>>> si.optimize()
>>> si.linear_mip_model.optimize()
<OptimizationStatus.OPTIMAL: 0>
>>> prices = si.price_constraints([1])
>>> print(prices)
{1: 8.0}
>>> decision_variables['value'] = si.get_optimal_values_of_decision_variables(decision_variables)
>>> print(decision_variables)
variable_id lower_bound upper_bound type value
0 0 0.0 5.0 continuous 5.0
1 1 0.0 5.0 continuous 5.0
2 2 0.0 10.0 continuous 0.0
3 3 0.0 10.0 continuous 10.0
4 4 0.0 5.0 continuous 0.0
5 5 0.0 5.0 continuous 0.0
"""
costs = {}
for id in constraint_ids_to_price:
costs[id] = self.linear_mip_model.constr_by_name(str(id)).pi
return costs
def update_rhs(self, constraint_id, violation_degree):
constraint = self.linear_mip_model.constr_by_name(str(constraint_id))
constraint.rhs += violation_degree
def update_variable_bounds(self, new_bounds):
for variable_id, lb, ub in zip(new_bounds['variable_id'],
new_bounds['lower_bound'],
new_bounds['upper_bound']):
self.mip_model.var_by_name(str(variable_id)).lb = lb
self.mip_model.var_by_name(str(variable_id)).ub = ub
def disable_variables(self, variables):
for var_id in variables['variable_id']:
var = self.linear_mip_model.var_by_name(str(var_id))
var.lb = 0.0
var.ub = 0.0
plt.text((70), (78), "Region 2")
plt.plot((50, 50), (0, 80))
dist = {(f, c):
round(sqrt((pf[f][0] - pc[c][0])**2 + (pf[f][1] - pc[c][1])**2), 1)
for (f, c) in product(F, C)}
m = Model()
z = {i: m.add_var(ub=c[i]) for i in F} # plant capacity
# Type 1 SOS: only one plant per region
for r in [0, 1]:
# set of plants in region r
Fr = [i for i in F if r * 50 <= pf[i][0] <= 50 + r * 50]
m.add_sos([(z[i], i - 1) for i in Fr], 1)
# amount that plant i will supply to client j
x = {(i, j): m.add_var() for (i, j) in product(F, C)}
# satisfy demand
for j in C:
m += xsum(x[(i, j)] for i in F) == d[j]
# SOS type 2 to model installation costs for each installed plant
y = {i: m.add_var() for i in F}
for f in F:
D = 6 # nr. of discretization points, increase for more precision
v = [c[f] * (v / (D - 1)) for v in range(D)] # points
# non-linear function values for points in v
vn = [0 if k == 0 else 1520 * log(v[k]) for k in range(D)]
def re_allocate(cmap, jmin, jmax, Ns, Os, Tfwd, res_up, res_dw, time_limit, solver=CBC):
start_time = str(time.time())
joblist = cmap.index
nodelist= cmap.columns
nJ, nN = cmap.shape
J = range(nJ)
N = range(nN)
assert len(jmin) == nJ and len(jmax) == nJ and len(res_up) == nJ and len(res_dw) == nJ
# compute throughput of current map, cost will depend on it
_cNj = cmap.sum(axis=1)
c_rate = [interp1d4s(Ns[_j], Os[_j], _cNj[_j]) \
if _cNj[_j] >= jmin[_j] else 0 for _j in range(cmap.shape[0])]
with open("%s-b4.log" % start_time, 'w') as fp:
fp.write(cmap.to_string() + '\n')
fp.write(f"jmin={jmin}, jmax={jmax}, Ns={Ns}, Os={Os}, Tfwd={Tfwd}, res_up={res_up}, res_dw={res_dw}, time_limit={time_limit}\n")
c_map = cmap.values
m = Model(solver_name=solver)
# create decision variable, J x N
x = [[m.add_var('x({},{})'.format(j, n), var_type=BINARY) for n in N] for j in J]
# job scalability constraint, should be either within the internal or zero
bigM = nN + 1
dummy4Jmin = [m.add_var(var_type=BINARY) for _j in J]
dummy4Jmax = [m.add_var(var_type=BINARY) for _j in J]
for _j in J:
_nJ = xsum(x[_j][_n] for _n in N)
m += _nJ >= jmin[_j] - bigM * dummy4Jmin[_j]
m += _nJ <= 0 + bigM * (1 - dummy4Jmin[_j])
m += jmax[_j] >= _nJ - bigM * dummy4Jmax[_j]
m += _nJ <= 0 + bigM * (1 - dummy4Jmax[_j])
# node allocations constraint
for _n in N:
m += xsum(x[_j][_n] for _j in J) <= 1 # at most one application
# constraint to disallow job migration
dummy4xxorc = [[m.add_var('xxorc({},{})'.format(j, n), var_type=BINARY) for n in N] for j in J] # x XOR c
dummy4migr = [m.add_var(var_type=BINARY) for _j in J]
c = c_map # a shorter name for easy ref
for _j in J:
for _n in N:
m += dummy4xxorc[_j][_n] <= x[_j][_n] + c[_j][_n]
m += dummy4xxorc[_j][_n] >= x[_j][_n] - c[_j][_n]
m += dummy4xxorc[_j][_n] >= c[_j][_n] - x[_j][_n]
m += dummy4xxorc[_j][_n] <= 2 - x[_j][_n] - c[_j][_n]
for _j in J:
_xxorc_sumJ = xsum(dummy4xxorc[_j][_n] for _n in N)
_nj_new = xsum(x[_j][_n] for _n in N)
_nj_old = sum(c[_j])
m += _nj_new - _nj_old >= _xxorc_sumJ - bigM * dummy4migr[_j]
m += _nj_new - _nj_old <= -_xxorc_sumJ + bigM * (1 - dummy4migr[_j])
# approximate the scalability
w4sappro = [[m.add_var('x({},{})'.format(j, i), var_type=CONTINUOUS, lb=0, ub=1) for i in range(len(Ns[j]))] for j in J]
S_approx = {}
for _j in J:
I = range(len(Ns[_j]))
m += xsum(w4sappro[_j]) == 1 # convexification
_Nj = xsum(x[_j][_n] for _n in N)
m +=_Nj == xsum(Ns[_j][_i] * w4sappro[_j][_i] for _i in I)
m.add_sos([(w4sappro[_j][_i], Ns[_j][_i]) for _i in I], 2)
S_approx[_j] = xsum(Os[_j][_i] * w4sappro[_j][_i] for _i in I)
# scale up/down cost
dummy4upcost = [m.add_var(var_type=BINARY) for _j in J]
dummy4dwcost = [m.add_var(var_type=BINARY) for _j in J]
for _j in J:
_Nj = xsum(x[_j][_n] for _n in N)
_Cj = sum(c_map[_j])
m += _Nj <= _Cj + (bigM - _Cj) * dummy4upcost[_j]
m += _Nj >= (_Cj + 1) * dummy4upcost[_j]
m += _Nj <= (_Cj - 1) + (bigM - (_Cj - 1)) * (1 - dummy4dwcost[_j])
m += _Nj >= _Cj * (1 - dummy4dwcost[_j])
rescale_cost = xsum(dummy4upcost[_j] * res_up[_j] * c_rate[_j] + dummy4dwcost[_j] * res_dw[_j] * c_rate[_j] for _j in J)
# new steady performance
steady_perf = xsum(Tfwd * S_approx[_j] for _j in J)
# set objective function
m.objective = maximize(steady_perf - rescale_cost)
# solve model
status = m.optimize(max_seconds=time_limit)
if status == OptimizationStatus.OPTIMAL:
print('optimal solution cost {} found'.format(m.objective_value))
elif status == OptimizationStatus.FEASIBLE:
print('sol.cost {} found, best possible: {}'.format(m.objective_value, m.objective_bound))
elif status == OptimizationStatus.NO_SOLUTION_FOUND:
print('no feasible solution found, lower bound is: {}'.format(m.objective_bound))
else:
print('something wrong', status)
sol_map = np.zeros((nJ, nN), dtype=np.int8)
if status == OptimizationStatus.OPTIMAL:
for _n in N:
for _j in J:
sol_map[_j][_n] = 1 if x[_j][_n].x > 0.5 else 0
rate = [S_approx[_j].x for _j in J]
cost = [(dummy4upcost[_j] * res_up[_j] * c_rate[_j] + dummy4dwcost[_j] * res_dw[_j] * c_rate[_j]) for _j in J]
else:
rate, cost = [], []
sol_map_pd = pd.DataFrame(sol_map, index=joblist, columns=nodelist)
with open("%s-after.log" % start_time, 'w') as fp:
fp.write(sol_map_pd.to_string() + '\n')
fp.write(f"opt_mdl.Status={status}, rate={rate}, cost={cost}")
return status, sol_map, np.array(rate), np.array(cost)
