def _fix_nonants_at_value(self):
""" Fix the Vars subject to non-anticipativity at their current values.
Loop over the scenarios to restore, but loop over subproblems
to alert persistent solvers.
"""
for k,s in self.local_scenarios.items():
persistent_solver = None
if not self.bundling:
if (sputils.is_persistent(s._solver_plugin)):
persistent_solver = s._solver_plugin
for var in s._mpisppy_data.nonant_indices.values():
var.fix()
if not self.bundling and persistent_solver is not None:
persistent_solver.update_var(var)
if self.bundling: # we might need to update persistent solvers
rank_local = self.cylinder_rank
for k,s in self.local_subproblems.items():
if (sputils.is_persistent(s._solver_plugin)):
persistent_solver = s._solver_plugin
else:
break # all solvers should be the same
# the bundle number is the last number in the name
bunnum = sputils.extract_num(k)
# for the scenarios in this bundle, update Vars
for sname, scen in self.local_scenarios.items():
if sname not in self.names_in_bundles[rank_local][bunnum]:
break
for var in scen._mpisppy_data.nonant_indices.values():
persistent_solver.update_var(var)
def solve_extensive_form(self, solver_options=None, tee=False):
""" Solve the extensive form.
Args:
solver_options (dict, optional):
Dictionary of solver-specific options (e.g. Gurobi options,
CPLEX options, etc.).
tee (bool, optional):
If True, displays solver output. Default False.
Returns:
:class:`pyomo.opt.results.results_.SolverResults`:
Result returned by the Pyomo solve method.
"""
if "persistent" in self.options["solver"]:
self.solver.set_instance(self.ef)
# Pass solver-specifiec (e.g. Gurobi, CPLEX) options
if solver_options is not None:
for (opt, value) in solver_options.items():
self.solver.options[opt] = value
results = self.solver.solve(self.ef, tee=tee, load_solutions=False)
if len(results.solution) > 0:
if sputils.is_persistent(self.solver):
self.solver.load_vars()
else:
self.ef.solutions.load_from(results)
self.first_stage_solution_available = True
self.tree_solution_available = True
return results
def _create_solvers(self):
for sname, s in self.local_subproblems.items(): # solver creation
s._solver_plugin = SolverFactory(self.options["solvername"])
if (sputils.is_persistent(s._solver_plugin)):
dtiming = ("display_timing"
in self.options) and self.options["display_timing"]
if dtiming:
set_instance_start_time = time.time()
set_instance_retry(s, s._solver_plugin, sname)
if dtiming:
set_instance_time = time.time() - set_instance_start_time
all_set_instance_times = self.mpicomm.gather(
set_instance_time, root=0)
if self.cylinder_rank == 0:
print("Set instance times:")
print("\tmin=%4.2f mean=%4.2f max=%4.2f" %
(np.min(all_set_instance_times),
np.mean(all_set_instance_times),
np.max(all_set_instance_times)))
## if we have bundling, attach
## the solver plugin to the scenarios
## as well to avoid some gymnastics
if self.bundling:
for scen_name in s.scen_list:
scen = self.local_scenarios[scen_name]
scen._solver_plugin = s._solver_plugin
def _restore_original_nonants(self):
""" Restore nonanticipative variables to their original values.
This function works in conjunction with _save_original_nonants.
We loop over the scenarios to restore variables, but loop over
subproblems to alert persistent solvers.
Warning:
We are counting on Pyomo indices not to change order between save
and restoration. THIS WILL NOT WORK ON BUNDLES (Feb 2019) but
hopefully does not need to.
"""
for k, s in self.local_scenarios.items():
persistent_solver = None
if not self.bundling:
if (sputils.is_persistent(s._solver_plugin)):
persistent_solver = s._solver_plugin
else:
print("restore_original_nonants called for a bundle")
raise
for ci, vardata in enumerate(
s._mpisppy_data.nonant_indices.values()):
vardata._value = s._mpisppy_data.original_nonants[ci]
vardata.fixed = s._mpisppy_data.original_fixedness[ci]
if persistent_solver != None:
persistent_solver.update_var(vardata)
def miditer(self):
''' Currently hard-coded for no parallelism
'''
print('About to do the solve in iteration', self.ph._PHIter)
sols = self.get_sol()
print_sol(sols, print_all=False)
if (self.ph._PHIter < 5):
return
models = self.ph.local_scenarios
arb_model = get_arb_elt(models)
edges = arb_model.edges
num_scenarios = len(sols)
bundling = not hasattr(arb_model, '_solver_plugin')
if (bundling):
arb_bundle_model = get_arb_elt(self.ph.local_subproblems)
persistent = sputils.is_persistent(arb_bundle_model._solver_plugin)
else:
persistent = sputils.is_persistent(arb_model._solver_plugin)
for edge in edges:
if (arb_model.x[edge].fixed): # Fixed in one model = fixed in all
continue
sol_values = {name: sols[name][edge]['value'] for name in sols}
total = sum(sol_values.values())
if (total < 0.1) or (total > num_scenarios - 0.1):
''' All scenarios agree '''
pass
if (total > (num_scenarios // 2) + 1.9):
print(f'Fixing edge {edge} to 1')
for (sname, model) in models.items():
model.x[edge].fix(1.0)
if (persistent):
if (bundling):
solver = self.get_solver(sname)
else:
solver = model._solver_plugin
solver.update_var(model.x[edge])
fixed_edges = [edge for edge in edges if arb_model.x[edge].fixed]
print(f'Fixed {len(fixed_edges)} total edges')
def main(self):
self.slam_heur_prep()
self.ib = inf if self.is_minimizing else -inf
slam_iter = 1
while not self.got_kill_signal():
if (slam_iter - 1) % 10000 == 0:
logger.debug(
f' {self.__class__.__name__} loop iter={slam_iter} on rank {self.global_rank}'
)
logger.debug(
f' {self.__class__.__name__} got from opt on rank {self.global_rank}'
)
if self.new_nonants:
local_candidate = self.extract_local_candidate_soln()
global_candidate = np.empty_like(local_candidate)
self.cylinder_comm.Allreduce(local_candidate,
global_candidate,
op=self.mpi_op)
'''
## round the candidate
candidate = global_candidate.round()
'''
# Everyone has the candidate solution at this point
# Moreover, we are guaranteed that it is feasible
bundling = self.opt.bundling
for (sname, s) in self.opt.local_subproblems.items():
is_pers = sputils.is_persistent(s._solver_plugin)
solver = s._solver_plugin if is_pers else None
nonant_source = s.ref_vars.values() if bundling else \
s._mpisppy_data.nonant_indices.values()
for ix, var in enumerate(nonant_source):
var.fix(global_candidate[ix])
if (is_pers):
solver.update_var(var)
obj = self.opt.calculate_incumbent(fix_nonants=False)
update = (obj is not None) and \
((obj < self.ib) if self.is_minimizing else (self.ib < obj))
if update:
self.bound = obj
self.ib = obj
slam_iter += 1
def _restore_and_fix_best_nonants(self):
for k, s in self.opt.local_scenarios.items():
scenario_cache = s._mpisppy_data.best_nonant_cache
if scenario_cache is None:
return
is_persistent = sputils.is_persistent(s._solver_plugin)
solver = s._solver_plugin
for ndn_i, var in s._mpisppy_data.nonant_indices.items():
var.fix(scenario_cache[ndn_i])
if is_persistent:
solver.update_var(var)
self.opt.first_stage_solution_available = True
def _update_prox_approx(self):
"""
update proximal term approximation by potentially
adding a linear cut near each current xvar value
NOTE: This is badly inefficient for bundles, but works
"""
tol = self.prox_approx_tol
for sn, s in self.local_scenarios.items():
persistent_solver = (s._solver_plugin if sputils.is_persistent(
s._solver_plugin) else None)
for prox_approx_manager in s._mpisppy_data.xsqvar_prox_approx.values(
):
prox_approx_manager.check_tol_add_cut(tol, persistent_solver)
def _add_QP_column(self, model_name):
''' Add a column to the QP, with values taken from the most recent MIP
solve.
'''
mip = self.local_subproblems[model_name]
qp = self.local_QP_subproblems[model_name]
solver = qp._QP_solver_plugin
persistent = sputils.is_persistent(solver)
if hasattr(solver, 'add_column'):
new_var = qp.a.add()
coef_list = [1.]
constr_list = [qp.sum_one]
target = mip.ref_vars if self.bundling else mip.nonant_vars
for (node, ix) in qp.eqx.index_set():
coef_list.append(target[node, ix].value)
constr_list.append(qp.eqx[node, ix])
for key in mip.y_indices:
coef_list.append(mip.leaf_vars[key].value)
constr_list.append(qp.eqy[key])
solver.add_column(qp, new_var, 0, constr_list, coef_list)
return
# Add new variable and update \sum a_i = 1 constraint
new_var = qp.a.add() # Add the new convex comb. variable
if (persistent):
solver.add_var(new_var)
solver.remove_constraint(qp.sum_one)
qp.sum_one._body += new_var
solver.add_constraint(qp.sum_one)
else:
qp.sum_one._body += new_var
target = mip.ref_vars if self.bundling else mip.nonant_vars
for (node, ix) in qp.eqx.index_set():
if (persistent):
solver.remove_constraint(qp.eqx[node, ix])
qp.eqx[node, ix]._body += new_var * target[node, ix].value
solver.add_constraint(qp.eqx[node, ix])
else:
qp.eqx[node, ix]._body += new_var * target[node, ix].value
for key in mip.y_indices:
if (persistent):
solver.remove_constraint(qp.eqy[key])
qp.eqy[key]._body += new_var * pyo.value(mip.leaf_vars[key])
solver.add_constraint(qp.eqy[key])
else:
qp.eqy[key]._body += new_var * pyo.value(mip.leaf_vars[key])
def _set_QP_objective(self):
''' Attach dual weights, objective function and solver to each QP.
QP dual weights are initialized to the MIP dual weights.
'''
for name, mip in self.local_subproblems.items():
QP = self.local_QP_subproblems[name]
obj, new = self._extract_objective(mip)
## Finish setting up objective for QP
if self.bundling:
m_source = self.local_scenarios[mip.scen_list[0]]
x_source = QP.xr
else:
m_source = mip
x_source = QP.x
QP._mpisppy_model.W = pyo.Param(
m_source._mpisppy_data.nonant_indices.keys(),
mutable=True,
initialize=m_source._mpisppy_model.W)
# rhos are attached to each scenario, not each bundle (should they be?)
ph_term = pyo.quicksum(
(QP._mpisppy_model.W[nni] * x_source[nni] +
(m_source._mpisppy_model.rho[nni] / 2.) *
(x_source[nni] - m_source._mpisppy_model.xbars[nni]) *
(x_source[nni] - m_source._mpisppy_model.xbars[nni])
for nni in m_source._mpisppy_data.nonant_indices))
if obj.is_minimizing():
QP.obj = pyo.Objective(expr=new + ph_term, sense=pyo.minimize)
else:
QP.obj = pyo.Objective(expr=-new + ph_term, sense=pyo.minimize)
''' Attach a solver with various options '''
solver = pyo.SolverFactory(self.FW_options['solvername'])
if sputils.is_persistent(solver):
solver.set_instance(QP)
if 'qp_solver_options' in self.FW_options:
qp_opts = self.FW_options['qp_solver_options']
if qp_opts:
for (key, option) in qp_opts.items():
solver.options[key] = option
self.local_QP_subproblems[name]._QP_solver_plugin = solver
def main(self):
self.slam_heur_prep()
slam_iter = 1
while not self.got_kill_signal():
if (slam_iter - 1) % 10000 == 0:
logger.debug(
f' {self.__class__.__name__} loop iter={slam_iter} on rank {self.global_rank}'
)
logger.debug(
f' {self.__class__.__name__} got from opt on rank {self.global_rank}'
)
if self.new_nonants:
local_candidate = self.extract_local_candidate_soln()
global_candidate = np.empty_like(local_candidate)
self.cylinder_comm.Allreduce(local_candidate,
global_candidate,
op=self.mpi_op)
'''
## round the candidate
candidate = global_candidate.round()
'''
# Everyone has the candidate solution at this point
for s in self.opt.local_scenarios.values():
is_pers = sputils.is_persistent(s._solver_plugin)
solver = s._solver_plugin if is_pers else None
for ix, var in enumerate(
s._mpisppy_data.nonant_indices.values()):
var.fix(global_candidate[ix])
if (is_pers):
solver.update_var(var)
obj = self.opt.calculate_incumbent(fix_nonants=False)
self.update_if_improving(obj)
slam_iter += 1
def _fix_root_nonants(self, root_cache):
""" Fix the 1st stage Vars subject to non-anticipativity at given values.
Loop over the scenarios to restore, but loop over subproblems
to alert persistent solvers.
Useful for multistage to find feasible solutions with a given scenario.
Args:
root_cache (numpy vector): values at which to fix
WARNING:
We are counting on Pyomo indices not to change order between
when the cache_list is created and used.
NOTE:
You probably want to call _save_nonants right before calling this
"""
for k, s in self.local_scenarios.items():
persistent_solver = None
if (sputils.is_persistent(s._solver_plugin)):
persistent_solver = s._solver_plugin
nlens = s._mpisppy_data.nlens
rootnode = None
for node in s._mpisppy_node_list:
if node.name == 'ROOT':
rootnode = node
break
if rootnode is None:
raise RuntimeError("Could not find a 'ROOT' node in scen {}"\
.format(k))
if root_cache is None:
raise RuntimeError("Empty root cache for scen={}".format(k))
if len(root_cache) != nlens['ROOT']:
raise RuntimeError("Needed {} nonant Vars for 'ROOT', got {}"\
.format(nlens['ROOT'], len(root_cache)))
for i in range(nlens['ROOT']):
this_vardata = node.nonant_vardata_list[i]
this_vardata._value = root_cache[i]
this_vardata.fix()
if persistent_solver is not None:
persistent_solver.update_var(this_vardata)
def fix_nonants_upto_stage(self, t, cache):
""" Fix the Vars subject to non-anticipativity at given values for stages 1 to t.
Loop over the scenarios to restore, but loop over subproblems
to alert persistent solvers.
Args:
cache (ndn dict of list or numpy vector): values at which to fix
WARNING:
We are counting on Pyomo indices not to change order between
when the cache_list is created and used.
NOTE:
You probably want to call _save_nonants right before calling this
"""
self._lazy_create_solvers()
for k, s in self.local_scenarios.items():
persistent_solver = None
if (sputils.is_persistent(s._solver_plugin)):
persistent_solver = s._solver_plugin
nlens = s._mpisppy_data.nlens
for node in s._mpisppy_node_list:
if node.stage <= t:
ndn = node.name
if ndn not in cache:
raise RuntimeError("Could not find {} in {}"\
.format(ndn, cache))
if cache[ndn] is None:
raise RuntimeError(
"Empty cache for scen={}, node={}".format(k, ndn))
if len(cache[ndn]) != nlens[ndn]:
raise RuntimeError("Needed {} nonant Vars for {}, got {}"\
.format(nlens[ndn], ndn, len(cache[ndn])))
for i in range(nlens[ndn]):
this_vardata = node.nonant_vardata_list[i]
this_vardata._value = cache[ndn][i]
this_vardata.fix()
if persistent_solver is not None:
persistent_solver.update_var(this_vardata)
def post_iter0(self):
opt = self.opt
# NOTE: the LShaped code negates the objective, so
# we do the same here for consistency
if 'cross_scen_options' in opt.options and \
'valid_eta_bound' in opt.options['cross_scen_options']:
valid_eta_bound = opt.options['cross_scen_options'][
'valid_eta_bound']
if not opt.is_minimizing:
_eta_init = {k: -v for k, v in valid_eta_bound.items()}
else:
_eta_init = valid_eta_bound
_eta_bounds = lambda m, k: (_eta_init[k], None)
else:
lb = (-sys.maxsize - 1) * 1. / len(opt.all_scenario_names)
_eta_init = lambda m, k: lb
_eta_bounds = lambda m, k: (lb, None)
# eta is attached to each subproblem, regardless of bundles
bundling = opt.bundling
for k, s in opt.local_subproblems.items():
s.eta = pyo.Var(opt.all_scenario_names,
initialize=_eta_init,
bounds=_eta_bounds)
if sputils.is_persistent(s._solver_plugin):
for var in s.eta.values():
s._solver_plugin.add_var(var)
if bundling: ## create a refence to eta on each subproblem
for sn in s.scen_list:
scenario = opt.local_scenarios[sn]
scenario.eta = {
k: s.eta[k]
for k in opt.all_scenario_names
}
## hold the PH object harmless
self._disable_W_and_prox()
for k, s in opt.local_subproblems.items():
obj = find_active_objective(s)
repn = generate_standard_repn(obj.expr, quadratic=True)
if len(repn.nonlinear_vars) > 0:
raise ValueError(
"CrossScenario does not support models with nonlinear objective functions"
)
if bundling:
## NOTE: this is slighly wasteful, in that for a bundle
## the first-stage cost appears len(s.scen_list) times
## If this really made a difference, we could use s.ref_vars
## to do the substitution
nonant_vardata_list = list()
for sn in s.scen_list:
nonant_vardata_list.extend( \
opt.local_scenarios[sn]._PySPnode_list[0].nonant_vardata_list)
else:
nonant_vardata_list = s._PySPnode_list[0].nonant_vardata_list
nonant_ids = set((id(var) for var in nonant_vardata_list))
linear_coefs = list(repn.linear_coefs)
linear_vars = list(repn.linear_vars)
quadratic_coefs = list(repn.quadratic_coefs)
# adjust coefficients by scenario/bundle probability
scen_prob = s.PySP_prob
for i, var in enumerate(repn.linear_vars):
if id(var) not in nonant_ids:
linear_coefs[i] *= scen_prob
for i, (x, y) in enumerate(repn.quadratic_vars):
# only multiply through once
if id(x) not in nonant_ids:
quadratic_coefs[i] *= scen_prob
elif id(y) not in nonant_ids:
quadratic_coefs[i] *= scen_prob
# NOTE: the LShaped code negates the objective, so
# we do the same here for consistency
if not opt.is_minimizing:
for i, coef in enumerate(linear_coefs):
linear_coefs[i] = -coef
for i, coef in enumerate(quadratic_coefs):
quadratic_coefs[i] = -coef
# add the other etas
if bundling:
these_scenarios = set(s.scen_list)
else:
these_scenarios = [k]
eta_scenarios = list()
for sn in opt.all_scenario_names:
if sn not in these_scenarios:
linear_coefs.append(1)
linear_vars.append(s.eta[sn])
eta_scenarios.append(sn)
expr = LinearExpression(constant=repn.constant,
linear_coefs=linear_coefs,
linear_vars=linear_vars)
if repn.quadratic_vars:
expr += pyo.quicksum((coef * x * y for coef, (
x, y) in zip(quadratic_coefs, repn.quadratic_vars)))
s._EF_obj = pyo.Expression(expr=expr)
if opt.is_minimizing:
s._EF_Obj = pyo.Objective(expr=s._EF_obj, sense=pyo.minimize)
else:
s._EF_Obj = pyo.Objective(expr=-s._EF_obj, sense=pyo.maximize)
s._EF_Obj.deactivate()
# add cut constraint dicts
s._benders_cuts = pyo.Constraint(pyo.Any)
s._ib_constr = pyo.Constraint(pyo.Any)
self._enable_W_and_prox()
# try to get the initial eta LB cuts
# (may not be available)
opt.spcomm.get_from_cross_cuts()
def lshaped_algorithm(self, converger=None):
""" function that runs the lshaped.py algorithm
"""
if converger:
converger = converger(self, self.rank, self.n_proc)
max_iter = 30
if "max_iter" in self.options:
max_iter = self.options["max_iter"]
tol = 1e-8
if "tol" in self.options:
tol = self.options["tol"]
verbose = True
if "verbose" in self.options:
verbose = self.options["verbose"]
master_solver = self.options["master_solver"]
sp_solver = self.options["sp_solver"]
# creates the master problem
self.create_master()
m = self.master
assert hasattr(m, "obj")
# prevents problems from first stage variables becoming unconstrained
# after processing
_init_vars(self.master_vars)
# sets up the BendersCutGenerator object
m.bender = LShapedCutGenerator()
m.bender.set_input(master_vars=self.master_vars,
tol=tol,
comm=self.mpicomm)
# let the cut generator know who's using it, probably should check that this is called after set input
m.bender.set_ls(self)
# set the eta variables, removing this from the add_suproblem function so we can
# Pass all the scenarios in the problem to bender.add_subproblem
# and let it internally handle which ranks get which scenarios
if self.has_master_scens:
sub_scenarios = [
scenario_name for scenario_name in self.local_scenario_names
if scenario_name not in self.master_scenarios
]
else:
sub_scenarios = self.local_scenario_names
for scenario_name in self.local_scenario_names:
if scenario_name in sub_scenarios:
subproblem_fn_kwargs = dict()
subproblem_fn_kwargs['scenario_name'] = scenario_name
m.bender.add_subproblem(
subproblem_fn=self.create_subproblem,
subproblem_fn_kwargs=subproblem_fn_kwargs,
master_eta=m.eta[scenario_name],
subproblem_solver=sp_solver,
subproblem_solver_options=self.options["sp_solver_options"]
)
else:
self.attach_nonant_var_map(scenario_name)
# set the eta bounds if computed
# by self.create_subproblem
self.set_eta_bounds()
if self.rank == self.rank0:
opt = pyo.SolverFactory(master_solver)
if opt is None:
raise Exception("Error: Failed to Create Master Solver")
# set options
for k, v in self.options["master_solver_options"].items():
opt.options[k] = v
is_persistent = sputils.is_persistent(opt)
if is_persistent:
opt.set_instance(m)
t = time.time()
res, t1, t2 = None, None, None
# benders solve loop, repeats the benders master - subproblem
# loop until either a no more cuts can are generated
# or the maximum iterations limit is reached
for self.iter in range(max_iter):
if verbose and self.rank == self.rank0:
if self.iter > 0:
print("Current Iteration:", self.iter + 1, "Time Elapsed:",
"%7.2f" % (time.time() - t),
"Time Spent on Last Master:", "%7.2f" % t1,
"Time Spent Generating Last Cut Set:", "%7.2f" % t2,
"Current Objective:", "%7.2f" % m.obj.expr())
else:
print("Current Iteration:", self.iter + 1, "Time Elapsed:",
"%7.2f" % (time.time() - t),
"Current Objective: -Inf")
t1 = time.time()
x_vals = np.zeros(len(self.master_vars))
eta_vals = np.zeros(self.scenario_count)
outer_bound = np.zeros(1)
if self.rank == self.rank0:
if is_persistent:
res = opt.solve(tee=False)
else:
res = opt.solve(m, tee=False)
# LShaped is always minimizing
outer_bound[0] = res.Problem[0].Lower_bound
for i, var in enumerate(self.master_vars):
x_vals[i] = var.value
for i, eta in enumerate(m.eta.values()):
eta_vals[i] = eta.value
self.mpicomm.Bcast(x_vals, root=self.rank0)
self.mpicomm.Bcast(eta_vals, root=self.rank0)
self.mpicomm.Bcast(outer_bound, root=self.rank0)
if self.is_minimizing:
self._LShaped_bound = outer_bound[0]
else:
# LShaped is always minimizing, so negate
# the outer bound for sharing broadly
self._LShaped_bound = -outer_bound[0]
if self.rank != self.rank0:
for i, var in enumerate(self.master_vars):
var._value = x_vals[i]
for i, eta in enumerate(m.eta.values()):
eta._value = eta_vals[i]
t1 = time.time() - t1
# The hub object takes precedence over the converger
# We'll send the nonants now, and check for a for
# convergence
if self.spcomm:
self.spcomm.sync(send_nonants=True)
if self.spcomm.is_converged():
break
t2 = time.time()
cuts_added = m.bender.generate_cut()
t2 = time.time() - t2
if self.rank == self.rank0:
for c in cuts_added:
if is_persistent:
opt.add_constraint(c)
if verbose and len(cuts_added) == 0:
print(f"Converged in {self.iter+1} iterations.\n"
f"Total Time Elapsed: {time.time()-t:7.2f} "
f"Time Spent on Last Master: {t1:7.2f} "
f"Time spent verifying second stage: {t2:7.2f} "
f"Final Objective: {m.obj.expr():7.2f}")
break
if verbose and self.iter == max_iter - 1:
print("WARNING MAX ITERATION LIMIT REACHED !!! ")
else:
if len(cuts_added) == 0:
break
# The hub object takes precedence over the converger
if self.spcomm:
self.spcomm.sync(send_nonants=False)
if self.spcomm.is_converged():
break
if converger:
converger.convergence_value()
if converger.is_converged():
if verbose and self.rank == self.rank0:
print(
f"Converged to user criteria in {self.iter+1} iterations.\n"
f"Total Time Elapsed: {time.time()-t:7.2f} "
f"Time Spent on Last Master: {t1:7.2f} "
f"Time spent verifying second stage: {t2:7.2f} "
f"Final Objective: {m.obj.expr():7.2f}")
break
return res
def create_subproblem(self, scenario_name):
""" the subproblem creation function passed into the
BendersCutsGenerator
"""
instance = self.local_scenarios[scenario_name]
nonant_list, nonant_ids = _get_nonant_ids(instance)
# NOTE: since we use generate_standard_repn below, we need
# to unfix any nonants so they'll properly appear
# in the objective
fixed_nonants = [var for var in nonant_list if var.fixed]
for var in fixed_nonants:
var.fixed = False
# pulls the scenario objective expression, removes the first stage variables, and sets the new objective
obj = find_active_objective(instance)
if not hasattr(instance, "PySP_prob"):
instance.PySP_prob = 1. / self.scenario_count
PySP_prob = instance.PySP_prob
repn = generate_standard_repn(obj.expr, quadratic=True)
if len(repn.nonlinear_vars) > 0:
raise ValueError(
"LShaped does not support models with nonlinear objective functions"
)
linear_vars = list()
linear_coefs = list()
quadratic_vars = list()
quadratic_coefs = list()
## we'll assume the constant is part of stage 1 (wlog it is), just
## like the first-stage bits of the objective
constant = repn.constant
## only keep the second stage variables in the objective
for coef, var in zip(repn.linear_coefs, repn.linear_vars):
id_var = id(var)
if id_var not in nonant_ids:
linear_vars.append(var)
linear_coefs.append(PySP_prob * coef)
for coef, (x, y) in zip(repn.quadratic_coefs, repn.quadratic_vars):
id_x = id(x)
id_y = id(y)
if id_x not in nonant_ids or id_y not in nonant_ids:
quadratic_coefs.append(PySP_prob * coef)
quadratic_vars.append((x, y))
# checks if model sense is max, if so negates the objective
if not self.is_minimizing:
for i, coef in enumerate(linear_coefs):
linear_coefs[i] = -coef
for i, coef in enumerate(quadratic_coefs):
quadratic_coefs[i] = -coef
expr = LinearExpression(constant=constant,
linear_coefs=linear_coefs,
linear_vars=linear_vars)
if quadratic_coefs:
expr += pyo.quicksum(
(coef * x * y
for coef, (x, y) in zip(quadratic_coefs, quadratic_vars)))
instance.del_component(obj)
# set subproblem objective function
instance.obj = pyo.Objective(expr=expr, sense=pyo.minimize)
## need to do this here for validity if computing the eta bound
if self.relax_master:
# relaxes any integrality constraints for the subproblem
RelaxIntegerVars().apply_to(instance)
if self.compute_eta_bound:
for var in fixed_nonants:
var.fixed = True
opt = pyo.SolverFactory(self.options["sp_solver"])
if self.options["sp_solver_options"]:
for k, v in self.options["sp_solver_options"].items():
opt.options[k] = v
if sputils.is_persistent(opt):
opt.set_instance(instance)
res = opt.solve(tee=False)
else:
res = opt.solve(instance, tee=False)
eta_lb = res.Problem[0].Lower_bound
self.valid_eta_lb[scenario_name] = eta_lb
# if not done above
if not self.relax_master:
# relaxes any integrality constraints for the subproblem
RelaxIntegerVars().apply_to(instance)
# iterates through constraints and removes first stage constraints from the model
# the id dict is used to improve the speed of identifying the stage each variables belongs to
for constr_data in list(
itertools.chain(
instance.component_data_objects(SOSConstraint,
active=True,
descend_into=True),
instance.component_data_objects(Constraint,
active=True,
descend_into=True))):
if _first_stage_only(constr_data, nonant_ids):
_del_con(constr_data)
# creates the sub map to remove first stage variables from objective expression
complicating_vars_map = pyo.ComponentMap()
subproblem_to_master_vars_map = pyo.ComponentMap()
# creates the complicating var map that connects the first stage variables in the sub problem to those in
# the master problem -- also set the bounds on the subproblem master vars to be none for better cuts
for var, mvar in zip(nonant_list, self.master_vars):
if var.name not in mvar.name: # mvar.name may be part of a bundle
raise Exception(
"Error: Complicating variable mismatch, sub-problem variables changed order"
)
complicating_vars_map[mvar] = var
subproblem_to_master_vars_map[var] = mvar
# these are already enforced in the master
# don't need to be enfored in the subproblems
var.setlb(None)
var.setub(None)
var.fixed = False
# this is for interefacing with PH code
instance._subproblem_to_master_vars_map = subproblem_to_master_vars_map
if self.store_subproblems:
self.subproblems[scenario_name] = instance
return instance, complicating_vars_map
def SDM(self, model_name):
''' Algorithm 2 in Boland et al. (with small tweaks)
'''
mip = self.local_subproblems[model_name]
qp = self.local_QP_subproblems[model_name]
# Set the QP dual weights to the correct values If we are bundling, we
# initialize the QP dual weights to be the dual weights associated with
# the first scenario in the bundle (this should be okay, because each
# scenario in the bundle has the same dual weights, analytically--maybe
# a numerical problem).
arb_scen_mip = self.local_scenarios[mip.scen_list[0]] \
if self.bundling else mip
for (node_name, ix) in arb_scen_mip._nonant_indexes:
qp._Ws[node_name, ix]._value = \
arb_scen_mip._Ws[node_name, ix].value
alpha = self.FW_options['FW_weight']
# Algorithm 3 line 6
xt = {
ndn_i: (1 - alpha) * pyo.value(arb_scen_mip._xbars[ndn_i]) +
alpha * pyo.value(xvar)
for ndn_i, xvar in arb_scen_mip._nonant_indexes.items()
}
for itr in range(self.FW_options['FW_iter_limit']):
# Algorithm 2 line 4
mip_source = mip.scen_list if self.bundling else [model_name]
for scenario_name in mip_source:
scen_mip = self.local_scenarios[scenario_name]
for ndn_i, nonant in scen_mip._nonant_indexes.items():
x_source = xt[ndn_i] if itr==0 \
else nonant._value
scen_mip._Ws[ndn_i]._value = (
qp._Ws[ndn_i]._value + scen_mip._PHrho[ndn_i]._value *
(x_source - scen_mip._xbars[ndn_i]._value))
# Algorithm 2 line 5
if (sputils.is_persistent(mip._solver_plugin)):
mip_obj = find_active_objective(mip, True)
mip._solver_plugin.set_objective(mip_obj)
mip_results = mip._solver_plugin.solve(mip)
self._check_solve(mip_results, model_name + ' (MIP)')
# Algorithm 2 lines 6--8
obj = find_active_objective(mip, True)
if (itr == 0):
dual_bound = pyo.value(obj)
# Algorithm 2 line 9 (compute \Gamma^t)
val0 = pyo.value(obj)
new = replace_expressions(obj.expr, mip.mip_to_qp)
val1 = pyo.value(new)
obj.expr = replace_expressions(new, qp.qp_to_mip)
if abs(val0) > 1e-9:
stop_check = (val1 - val0) / abs(
val0) # \Gamma^t in Boland, but normalized
else:
stop_check = val1 - val0 # \Gamma^t in Boland
stop_check_tol = self.FW_options["stop_check_tol"]\
if "stop_check_tol" in self.FW_options else 1e-4
if (self.is_minimizing and stop_check < -stop_check_tol):
print(
'Warning (fwph): convergence quantity Gamma^t = '
'{sc:.2e} (should be non-negative)'.format(sc=stop_check))
print('Try decreasing the MIP gap tolerance and re-solving')
elif (not self.is_minimizing and stop_check > stop_check_tol):
print(
'Warning (fwph): convergence quantity Gamma^t = '
'{sc:.2e} (should be non-positive)'.format(sc=stop_check))
print('Try decreasing the MIP gap tolerance and re-solving')
self._add_QP_column(model_name)
if (sputils.is_persistent(qp._QP_solver_plugin)):
qp_obj = find_active_objective(qp, True)
qp._QP_solver_plugin.set_objective(qp_obj)
qp_results = qp._QP_solver_plugin.solve(qp)
self._check_solve(qp_results, model_name + ' (QP)')
if (stop_check < self.FW_options['FW_conv_thresh']):
break
# Re-set the mip._Ws so that the QP objective
# is correct in the next major iteration
mip_source = mip.scen_list if self.bundling else [model_name]
for scenario_name in mip_source:
scen_mip = self.local_scenarios[scenario_name]
for (node_name, ix) in scen_mip._nonant_indexes:
scen_mip._Ws[node_name, ix]._value = \
qp._Ws[node_name, ix]._value
return dual_bound
def solve_one(self,
solver_options,
k,
s,
dtiming=False,
gripe=False,
tee=False,
verbose=False,
disable_pyomo_signal_handling=False,
update_objective=True):
""" Solve one subproblem.
Args:
solver_options (dict or None):
The scenario solver options.
k (str):
Subproblem name.
s (ConcreteModel with appendages):
The subproblem to solve.
dtiming (boolean, optional):
If True, reports timing values. Default False.
gripe (boolean, optional):
If True, outputs a message when a solve fails. Default False.
tee (boolean, optional):
If True, displays solver output. Default False.
verbose (boolean, optional):
If True, displays verbose output. Default False.
disable_pyomo_signal_handling (boolean, optional):
True for asynchronous PH; ignored for persistent solvers.
Default False.
update_objective (boolean, optional):
If True, and a persistent solver is used, update
the persistent solver's objective
Returns:
float:
Pyomo solve time in seconds.
"""
def _vb(msg):
if verbose and self.cylinder_rank == 0:
print("(rank0) " + msg)
# if using a persistent solver plugin,
# re-compile the objective due to changed weights and x-bars
if update_objective and (sputils.is_persistent(s._solver_plugin)):
set_objective_start_time = time.time()
active_objective_datas = list(
s.component_data_objects(pyo.Objective,
active=True,
descend_into=True))
if len(active_objective_datas) > 1:
raise RuntimeError('Multiple active objectives identified '
'for scenario {sn}'.format(sn=s._name))
elif len(active_objective_datas) < 1:
raise RuntimeError('Could not find any active objectives '
'for scenario {sn}'.format(sn=s._name))
else:
s._solver_plugin.set_objective(active_objective_datas[0])
if dtiming:
set_objective_time = time.time() - set_objective_start_time
all_set_objective_times = self.mpicomm.gather(
set_objective_time, root=0)
if self.cylinder_rank == 0:
print("Set objective times (seconds):")
print("\tmin=%4.2f mean=%4.2f max=%4.2f" %
(np.mean(all_set_objective_times),
np.mean(all_set_objective_times),
np.max(all_set_objective_times)))
if self.extensions is not None:
results = self.extobject.pre_solve(s)
solve_start_time = time.time()
if (solver_options):
_vb("Using sub-problem solver options=" + str(solver_options))
for option_key, option_value in solver_options.items():
s._solver_plugin.options[option_key] = option_value
solve_keyword_args = dict()
if self.cylinder_rank == 0:
if tee is not None and tee is True:
solve_keyword_args["tee"] = True
if (sputils.is_persistent(s._solver_plugin)):
solve_keyword_args["save_results"] = False
elif disable_pyomo_signal_handling:
solve_keyword_args["use_signal_handling"] = False
try:
results = s._solver_plugin.solve(s,
**solve_keyword_args,
load_solutions=False)
solver_exception = None
except Exception as e:
results = None
solver_exception = e
if self.extensions is not None:
results = self.extobject.post_solve(s, results)
pyomo_solve_time = time.time() - solve_start_time
if (results is None) or (len(results.solution) == 0) or \
(results.solution(0).status == SolutionStatus.infeasible) or \
(results.solver.termination_condition == TerminationCondition.infeasible) or \
(results.solver.termination_condition == TerminationCondition.infeasibleOrUnbounded) or \
(results.solver.termination_condition == TerminationCondition.unbounded):
s._mpisppy_data.scenario_feasible = False
if gripe:
name = self.__class__.__name__
if self.spcomm:
name = self.spcomm.__class__.__name__
print(f"[{name}] Solve failed for scenario {s.name}")
if results is not None:
print("status=", results.solver.status)
print("TerminationCondition=",
results.solver.termination_condition)
if solver_exception is not None:
raise solver_exception
else:
if sputils.is_persistent(s._solver_plugin):
s._solver_plugin.load_vars()
else:
s.solutions.load_from(results)
if self.is_minimizing:
s._mpisppy_data.outer_bound = results.Problem[0].Lower_bound
else:
s._mpisppy_data.outer_bound = results.Problem[0].Upper_bound
s._mpisppy_data.scenario_feasible = True
# TBD: get this ready for IPopt (e.g., check feas_prob every time)
# propogate down
if self.bundling: # must be a bundle
for sname in s._ef_scenario_names:
self.local_scenarios[sname]._mpisppy_data.scenario_feasible\
= s._mpisppy_data.scenario_feasible
return pyomo_solve_time
def make_cuts(self, coefs):
# take the coefficient array and assemble cuts accordingly
# this should have already been set in the extension !
opt = self.opt
# rows are
# [ const, eta_coeff, *nonant_coeffs ]
row_len = 1 + 1 + self.nonant_len
outer_iter = int(coefs[-1])
bundling = opt.bundling
if opt.bundling:
for bn, b in opt.local_subproblems.items():
persistent_solver = sputils.is_persistent(b._solver_plugin)
## get an arbitrary scenario
s = opt.local_scenarios[b.scen_list[0]]
for idx, k in enumerate(opt.all_scenario_names):
row = coefs[row_len * idx:row_len * (idx + 1)]
# the row could be all zeros,
# which doesn't do anything
if (row == 0.).all():
continue
# rows are
# [ const, eta_coeff, *nonant_coeffs ]
linear_const = row[0]
linear_coefs = list(row[1:])
linear_vars = [b._mpisppy_model.eta[k]]
for ndn_i in s._mpisppy_data.nonant_indices:
## for bundles, we add the constrains only
## to the reference first stage variables
linear_vars.append(b.ref_vars[ndn_i])
cut_expr = LinearExpression(constant=linear_const,
linear_coefs=linear_coefs,
linear_vars=linear_vars)
b._mpisppy_model.benders_cuts[outer_iter,
k] = (None, cut_expr, 0)
if persistent_solver:
b._solver_plugin.add_constraint(
b._mpisppy_model.benders_cuts[outer_iter, k])
else:
for sn, s in opt.local_subproblems.items():
persistent_solver = sputils.is_persistent(s._solver_plugin)
for idx, k in enumerate(opt.all_scenario_names):
row = coefs[row_len * idx:row_len * (idx + 1)]
# the row could be all zeros,
# which doesn't do anything
if (row == 0.).all():
continue
# rows are
# [ const, eta_coeff, *nonant_coeffs ]
linear_const = row[0]
linear_coefs = list(row[1:])
linear_vars = [s._mpisppy_model.eta[k]]
linear_vars.extend(s._mpisppy_data.nonant_indices.values())
cut_expr = LinearExpression(constant=linear_const,
linear_coefs=linear_coefs,
linear_vars=linear_vars)
s._mpisppy_model.benders_cuts[outer_iter,
k] = (None, cut_expr, 0.)
if persistent_solver:
s._solver_plugin.add_constraint(
s._mpisppy_model.benders_cuts[outer_iter, k])
# NOTE: the LShaped code negates the objective, so
# we do the same here for consistency
ib = self.BestInnerBound
ob = self.BestOuterBound
if not opt.is_minimizing:
ib = -ib
ob = -ob
add_cut = (isfinite(ib) or isfinite(ob)) and \
((ib < self.best_inner_bound) or (ob > self.best_outer_bound))
if add_cut:
self.best_inner_bound = ib
self.best_outer_bound = ob
for sn, s in opt.local_subproblems.items():
persistent_solver = sputils.is_persistent(s._solver_plugin)
prior_outer_iter = list(
s._mpisppy_model.inner_bound_constr.keys())
s._mpisppy_model.inner_bound_constr[outer_iter] = (
ob, s._mpisppy_model.EF_obj, ib)
if persistent_solver:
s._solver_plugin.add_constraint(
s._mpisppy_model.inner_bound_constr[outer_iter])
# remove other ib constraints (we only need the tightest)
for it in prior_outer_iter:
if persistent_solver:
s._solver_plugin.remove_constraint(
s._mpisppy_model.inner_bound_constr[it])
del s._mpisppy_model.inner_bound_constr[it]
## helping the extention track cuts
self.new_cuts = True