def _fix_nonants_at_value(self): """ Fix the Vars subject to non-anticipativity at their current values. Loop over the scenarios to restore, but loop over subproblems to alert persistent solvers. """ for k,s in self.local_scenarios.items(): persistent_solver = None if not self.bundling: if (sputils.is_persistent(s._solver_plugin)): persistent_solver = s._solver_plugin for var in s._mpisppy_data.nonant_indices.values(): var.fix() if not self.bundling and persistent_solver is not None: persistent_solver.update_var(var) if self.bundling: # we might need to update persistent solvers rank_local = self.cylinder_rank for k,s in self.local_subproblems.items(): if (sputils.is_persistent(s._solver_plugin)): persistent_solver = s._solver_plugin else: break # all solvers should be the same # the bundle number is the last number in the name bunnum = sputils.extract_num(k) # for the scenarios in this bundle, update Vars for sname, scen in self.local_scenarios.items(): if sname not in self.names_in_bundles[rank_local][bunnum]: break for var in scen._mpisppy_data.nonant_indices.values(): persistent_solver.update_var(var)
def solve_extensive_form(self, solver_options=None, tee=False): """ Solve the extensive form. Args: solver_options (dict, optional): Dictionary of solver-specific options (e.g. Gurobi options, CPLEX options, etc.). tee (bool, optional): If True, displays solver output. Default False. Returns: :class:`pyomo.opt.results.results_.SolverResults`: Result returned by the Pyomo solve method. """ if "persistent" in self.options["solver"]: self.solver.set_instance(self.ef) # Pass solver-specifiec (e.g. Gurobi, CPLEX) options if solver_options is not None: for (opt, value) in solver_options.items(): self.solver.options[opt] = value results = self.solver.solve(self.ef, tee=tee, load_solutions=False) if len(results.solution) > 0: if sputils.is_persistent(self.solver): self.solver.load_vars() else: self.ef.solutions.load_from(results) self.first_stage_solution_available = True self.tree_solution_available = True return results
def _create_solvers(self): for sname, s in self.local_subproblems.items(): # solver creation s._solver_plugin = SolverFactory(self.options["solvername"]) if (sputils.is_persistent(s._solver_plugin)): dtiming = ("display_timing" in self.options) and self.options["display_timing"] if dtiming: set_instance_start_time = time.time() set_instance_retry(s, s._solver_plugin, sname) if dtiming: set_instance_time = time.time() - set_instance_start_time all_set_instance_times = self.mpicomm.gather( set_instance_time, root=0) if self.cylinder_rank == 0: print("Set instance times:") print("\tmin=%4.2f mean=%4.2f max=%4.2f" % (np.min(all_set_instance_times), np.mean(all_set_instance_times), np.max(all_set_instance_times))) ## if we have bundling, attach ## the solver plugin to the scenarios ## as well to avoid some gymnastics if self.bundling: for scen_name in s.scen_list: scen = self.local_scenarios[scen_name] scen._solver_plugin = s._solver_plugin
def _restore_original_nonants(self): """ Restore nonanticipative variables to their original values. This function works in conjunction with _save_original_nonants. We loop over the scenarios to restore variables, but loop over subproblems to alert persistent solvers. Warning: We are counting on Pyomo indices not to change order between save and restoration. THIS WILL NOT WORK ON BUNDLES (Feb 2019) but hopefully does not need to. """ for k, s in self.local_scenarios.items(): persistent_solver = None if not self.bundling: if (sputils.is_persistent(s._solver_plugin)): persistent_solver = s._solver_plugin else: print("restore_original_nonants called for a bundle") raise for ci, vardata in enumerate( s._mpisppy_data.nonant_indices.values()): vardata._value = s._mpisppy_data.original_nonants[ci] vardata.fixed = s._mpisppy_data.original_fixedness[ci] if persistent_solver != None: persistent_solver.update_var(vardata)
def miditer(self): ''' Currently hard-coded for no parallelism ''' print('About to do the solve in iteration', self.ph._PHIter) sols = self.get_sol() print_sol(sols, print_all=False) if (self.ph._PHIter < 5): return models = self.ph.local_scenarios arb_model = get_arb_elt(models) edges = arb_model.edges num_scenarios = len(sols) bundling = not hasattr(arb_model, '_solver_plugin') if (bundling): arb_bundle_model = get_arb_elt(self.ph.local_subproblems) persistent = sputils.is_persistent(arb_bundle_model._solver_plugin) else: persistent = sputils.is_persistent(arb_model._solver_plugin) for edge in edges: if (arb_model.x[edge].fixed): # Fixed in one model = fixed in all continue sol_values = {name: sols[name][edge]['value'] for name in sols} total = sum(sol_values.values()) if (total < 0.1) or (total > num_scenarios - 0.1): ''' All scenarios agree ''' pass if (total > (num_scenarios // 2) + 1.9): print(f'Fixing edge {edge} to 1') for (sname, model) in models.items(): model.x[edge].fix(1.0) if (persistent): if (bundling): solver = self.get_solver(sname) else: solver = model._solver_plugin solver.update_var(model.x[edge]) fixed_edges = [edge for edge in edges if arb_model.x[edge].fixed] print(f'Fixed {len(fixed_edges)} total edges')
def main(self): self.slam_heur_prep() self.ib = inf if self.is_minimizing else -inf slam_iter = 1 while not self.got_kill_signal(): if (slam_iter - 1) % 10000 == 0: logger.debug( f' {self.__class__.__name__} loop iter={slam_iter} on rank {self.global_rank}' ) logger.debug( f' {self.__class__.__name__} got from opt on rank {self.global_rank}' ) if self.new_nonants: local_candidate = self.extract_local_candidate_soln() global_candidate = np.empty_like(local_candidate) self.cylinder_comm.Allreduce(local_candidate, global_candidate, op=self.mpi_op) ''' ## round the candidate candidate = global_candidate.round() ''' # Everyone has the candidate solution at this point # Moreover, we are guaranteed that it is feasible bundling = self.opt.bundling for (sname, s) in self.opt.local_subproblems.items(): is_pers = sputils.is_persistent(s._solver_plugin) solver = s._solver_plugin if is_pers else None nonant_source = s.ref_vars.values() if bundling else \ s._mpisppy_data.nonant_indices.values() for ix, var in enumerate(nonant_source): var.fix(global_candidate[ix]) if (is_pers): solver.update_var(var) obj = self.opt.calculate_incumbent(fix_nonants=False) update = (obj is not None) and \ ((obj < self.ib) if self.is_minimizing else (self.ib < obj)) if update: self.bound = obj self.ib = obj slam_iter += 1
def _restore_and_fix_best_nonants(self): for k, s in self.opt.local_scenarios.items(): scenario_cache = s._mpisppy_data.best_nonant_cache if scenario_cache is None: return is_persistent = sputils.is_persistent(s._solver_plugin) solver = s._solver_plugin for ndn_i, var in s._mpisppy_data.nonant_indices.items(): var.fix(scenario_cache[ndn_i]) if is_persistent: solver.update_var(var) self.opt.first_stage_solution_available = True
def _update_prox_approx(self): """ update proximal term approximation by potentially adding a linear cut near each current xvar value NOTE: This is badly inefficient for bundles, but works """ tol = self.prox_approx_tol for sn, s in self.local_scenarios.items(): persistent_solver = (s._solver_plugin if sputils.is_persistent( s._solver_plugin) else None) for prox_approx_manager in s._mpisppy_data.xsqvar_prox_approx.values( ): prox_approx_manager.check_tol_add_cut(tol, persistent_solver)
def _add_QP_column(self, model_name): ''' Add a column to the QP, with values taken from the most recent MIP solve. ''' mip = self.local_subproblems[model_name] qp = self.local_QP_subproblems[model_name] solver = qp._QP_solver_plugin persistent = sputils.is_persistent(solver) if hasattr(solver, 'add_column'): new_var = qp.a.add() coef_list = [1.] constr_list = [qp.sum_one] target = mip.ref_vars if self.bundling else mip.nonant_vars for (node, ix) in qp.eqx.index_set(): coef_list.append(target[node, ix].value) constr_list.append(qp.eqx[node, ix]) for key in mip.y_indices: coef_list.append(mip.leaf_vars[key].value) constr_list.append(qp.eqy[key]) solver.add_column(qp, new_var, 0, constr_list, coef_list) return # Add new variable and update \sum a_i = 1 constraint new_var = qp.a.add() # Add the new convex comb. variable if (persistent): solver.add_var(new_var) solver.remove_constraint(qp.sum_one) qp.sum_one._body += new_var solver.add_constraint(qp.sum_one) else: qp.sum_one._body += new_var target = mip.ref_vars if self.bundling else mip.nonant_vars for (node, ix) in qp.eqx.index_set(): if (persistent): solver.remove_constraint(qp.eqx[node, ix]) qp.eqx[node, ix]._body += new_var * target[node, ix].value solver.add_constraint(qp.eqx[node, ix]) else: qp.eqx[node, ix]._body += new_var * target[node, ix].value for key in mip.y_indices: if (persistent): solver.remove_constraint(qp.eqy[key]) qp.eqy[key]._body += new_var * pyo.value(mip.leaf_vars[key]) solver.add_constraint(qp.eqy[key]) else: qp.eqy[key]._body += new_var * pyo.value(mip.leaf_vars[key])
def _set_QP_objective(self): ''' Attach dual weights, objective function and solver to each QP. QP dual weights are initialized to the MIP dual weights. ''' for name, mip in self.local_subproblems.items(): QP = self.local_QP_subproblems[name] obj, new = self._extract_objective(mip) ## Finish setting up objective for QP if self.bundling: m_source = self.local_scenarios[mip.scen_list[0]] x_source = QP.xr else: m_source = mip x_source = QP.x QP._mpisppy_model.W = pyo.Param( m_source._mpisppy_data.nonant_indices.keys(), mutable=True, initialize=m_source._mpisppy_model.W) # rhos are attached to each scenario, not each bundle (should they be?) ph_term = pyo.quicksum( (QP._mpisppy_model.W[nni] * x_source[nni] + (m_source._mpisppy_model.rho[nni] / 2.) * (x_source[nni] - m_source._mpisppy_model.xbars[nni]) * (x_source[nni] - m_source._mpisppy_model.xbars[nni]) for nni in m_source._mpisppy_data.nonant_indices)) if obj.is_minimizing(): QP.obj = pyo.Objective(expr=new + ph_term, sense=pyo.minimize) else: QP.obj = pyo.Objective(expr=-new + ph_term, sense=pyo.minimize) ''' Attach a solver with various options ''' solver = pyo.SolverFactory(self.FW_options['solvername']) if sputils.is_persistent(solver): solver.set_instance(QP) if 'qp_solver_options' in self.FW_options: qp_opts = self.FW_options['qp_solver_options'] if qp_opts: for (key, option) in qp_opts.items(): solver.options[key] = option self.local_QP_subproblems[name]._QP_solver_plugin = solver
def main(self): self.slam_heur_prep() slam_iter = 1 while not self.got_kill_signal(): if (slam_iter - 1) % 10000 == 0: logger.debug( f' {self.__class__.__name__} loop iter={slam_iter} on rank {self.global_rank}' ) logger.debug( f' {self.__class__.__name__} got from opt on rank {self.global_rank}' ) if self.new_nonants: local_candidate = self.extract_local_candidate_soln() global_candidate = np.empty_like(local_candidate) self.cylinder_comm.Allreduce(local_candidate, global_candidate, op=self.mpi_op) ''' ## round the candidate candidate = global_candidate.round() ''' # Everyone has the candidate solution at this point for s in self.opt.local_scenarios.values(): is_pers = sputils.is_persistent(s._solver_plugin) solver = s._solver_plugin if is_pers else None for ix, var in enumerate( s._mpisppy_data.nonant_indices.values()): var.fix(global_candidate[ix]) if (is_pers): solver.update_var(var) obj = self.opt.calculate_incumbent(fix_nonants=False) self.update_if_improving(obj) slam_iter += 1
def _fix_root_nonants(self, root_cache): """ Fix the 1st stage Vars subject to non-anticipativity at given values. Loop over the scenarios to restore, but loop over subproblems to alert persistent solvers. Useful for multistage to find feasible solutions with a given scenario. Args: root_cache (numpy vector): values at which to fix WARNING: We are counting on Pyomo indices not to change order between when the cache_list is created and used. NOTE: You probably want to call _save_nonants right before calling this """ for k, s in self.local_scenarios.items(): persistent_solver = None if (sputils.is_persistent(s._solver_plugin)): persistent_solver = s._solver_plugin nlens = s._mpisppy_data.nlens rootnode = None for node in s._mpisppy_node_list: if node.name == 'ROOT': rootnode = node break if rootnode is None: raise RuntimeError("Could not find a 'ROOT' node in scen {}"\ .format(k)) if root_cache is None: raise RuntimeError("Empty root cache for scen={}".format(k)) if len(root_cache) != nlens['ROOT']: raise RuntimeError("Needed {} nonant Vars for 'ROOT', got {}"\ .format(nlens['ROOT'], len(root_cache))) for i in range(nlens['ROOT']): this_vardata = node.nonant_vardata_list[i] this_vardata._value = root_cache[i] this_vardata.fix() if persistent_solver is not None: persistent_solver.update_var(this_vardata)
def fix_nonants_upto_stage(self, t, cache): """ Fix the Vars subject to non-anticipativity at given values for stages 1 to t. Loop over the scenarios to restore, but loop over subproblems to alert persistent solvers. Args: cache (ndn dict of list or numpy vector): values at which to fix WARNING: We are counting on Pyomo indices not to change order between when the cache_list is created and used. NOTE: You probably want to call _save_nonants right before calling this """ self._lazy_create_solvers() for k, s in self.local_scenarios.items(): persistent_solver = None if (sputils.is_persistent(s._solver_plugin)): persistent_solver = s._solver_plugin nlens = s._mpisppy_data.nlens for node in s._mpisppy_node_list: if node.stage <= t: ndn = node.name if ndn not in cache: raise RuntimeError("Could not find {} in {}"\ .format(ndn, cache)) if cache[ndn] is None: raise RuntimeError( "Empty cache for scen={}, node={}".format(k, ndn)) if len(cache[ndn]) != nlens[ndn]: raise RuntimeError("Needed {} nonant Vars for {}, got {}"\ .format(nlens[ndn], ndn, len(cache[ndn]))) for i in range(nlens[ndn]): this_vardata = node.nonant_vardata_list[i] this_vardata._value = cache[ndn][i] this_vardata.fix() if persistent_solver is not None: persistent_solver.update_var(this_vardata)
def post_iter0(self): opt = self.opt # NOTE: the LShaped code negates the objective, so # we do the same here for consistency if 'cross_scen_options' in opt.options and \ 'valid_eta_bound' in opt.options['cross_scen_options']: valid_eta_bound = opt.options['cross_scen_options'][ 'valid_eta_bound'] if not opt.is_minimizing: _eta_init = {k: -v for k, v in valid_eta_bound.items()} else: _eta_init = valid_eta_bound _eta_bounds = lambda m, k: (_eta_init[k], None) else: lb = (-sys.maxsize - 1) * 1. / len(opt.all_scenario_names) _eta_init = lambda m, k: lb _eta_bounds = lambda m, k: (lb, None) # eta is attached to each subproblem, regardless of bundles bundling = opt.bundling for k, s in opt.local_subproblems.items(): s.eta = pyo.Var(opt.all_scenario_names, initialize=_eta_init, bounds=_eta_bounds) if sputils.is_persistent(s._solver_plugin): for var in s.eta.values(): s._solver_plugin.add_var(var) if bundling: ## create a refence to eta on each subproblem for sn in s.scen_list: scenario = opt.local_scenarios[sn] scenario.eta = { k: s.eta[k] for k in opt.all_scenario_names } ## hold the PH object harmless self._disable_W_and_prox() for k, s in opt.local_subproblems.items(): obj = find_active_objective(s) repn = generate_standard_repn(obj.expr, quadratic=True) if len(repn.nonlinear_vars) > 0: raise ValueError( "CrossScenario does not support models with nonlinear objective functions" ) if bundling: ## NOTE: this is slighly wasteful, in that for a bundle ## the first-stage cost appears len(s.scen_list) times ## If this really made a difference, we could use s.ref_vars ## to do the substitution nonant_vardata_list = list() for sn in s.scen_list: nonant_vardata_list.extend( \ opt.local_scenarios[sn]._PySPnode_list[0].nonant_vardata_list) else: nonant_vardata_list = s._PySPnode_list[0].nonant_vardata_list nonant_ids = set((id(var) for var in nonant_vardata_list)) linear_coefs = list(repn.linear_coefs) linear_vars = list(repn.linear_vars) quadratic_coefs = list(repn.quadratic_coefs) # adjust coefficients by scenario/bundle probability scen_prob = s.PySP_prob for i, var in enumerate(repn.linear_vars): if id(var) not in nonant_ids: linear_coefs[i] *= scen_prob for i, (x, y) in enumerate(repn.quadratic_vars): # only multiply through once if id(x) not in nonant_ids: quadratic_coefs[i] *= scen_prob elif id(y) not in nonant_ids: quadratic_coefs[i] *= scen_prob # NOTE: the LShaped code negates the objective, so # we do the same here for consistency if not opt.is_minimizing: for i, coef in enumerate(linear_coefs): linear_coefs[i] = -coef for i, coef in enumerate(quadratic_coefs): quadratic_coefs[i] = -coef # add the other etas if bundling: these_scenarios = set(s.scen_list) else: these_scenarios = [k] eta_scenarios = list() for sn in opt.all_scenario_names: if sn not in these_scenarios: linear_coefs.append(1) linear_vars.append(s.eta[sn]) eta_scenarios.append(sn) expr = LinearExpression(constant=repn.constant, linear_coefs=linear_coefs, linear_vars=linear_vars) if repn.quadratic_vars: expr += pyo.quicksum((coef * x * y for coef, ( x, y) in zip(quadratic_coefs, repn.quadratic_vars))) s._EF_obj = pyo.Expression(expr=expr) if opt.is_minimizing: s._EF_Obj = pyo.Objective(expr=s._EF_obj, sense=pyo.minimize) else: s._EF_Obj = pyo.Objective(expr=-s._EF_obj, sense=pyo.maximize) s._EF_Obj.deactivate() # add cut constraint dicts s._benders_cuts = pyo.Constraint(pyo.Any) s._ib_constr = pyo.Constraint(pyo.Any) self._enable_W_and_prox() # try to get the initial eta LB cuts # (may not be available) opt.spcomm.get_from_cross_cuts()
def lshaped_algorithm(self, converger=None): """ function that runs the lshaped.py algorithm """ if converger: converger = converger(self, self.rank, self.n_proc) max_iter = 30 if "max_iter" in self.options: max_iter = self.options["max_iter"] tol = 1e-8 if "tol" in self.options: tol = self.options["tol"] verbose = True if "verbose" in self.options: verbose = self.options["verbose"] master_solver = self.options["master_solver"] sp_solver = self.options["sp_solver"] # creates the master problem self.create_master() m = self.master assert hasattr(m, "obj") # prevents problems from first stage variables becoming unconstrained # after processing _init_vars(self.master_vars) # sets up the BendersCutGenerator object m.bender = LShapedCutGenerator() m.bender.set_input(master_vars=self.master_vars, tol=tol, comm=self.mpicomm) # let the cut generator know who's using it, probably should check that this is called after set input m.bender.set_ls(self) # set the eta variables, removing this from the add_suproblem function so we can # Pass all the scenarios in the problem to bender.add_subproblem # and let it internally handle which ranks get which scenarios if self.has_master_scens: sub_scenarios = [ scenario_name for scenario_name in self.local_scenario_names if scenario_name not in self.master_scenarios ] else: sub_scenarios = self.local_scenario_names for scenario_name in self.local_scenario_names: if scenario_name in sub_scenarios: subproblem_fn_kwargs = dict() subproblem_fn_kwargs['scenario_name'] = scenario_name m.bender.add_subproblem( subproblem_fn=self.create_subproblem, subproblem_fn_kwargs=subproblem_fn_kwargs, master_eta=m.eta[scenario_name], subproblem_solver=sp_solver, subproblem_solver_options=self.options["sp_solver_options"] ) else: self.attach_nonant_var_map(scenario_name) # set the eta bounds if computed # by self.create_subproblem self.set_eta_bounds() if self.rank == self.rank0: opt = pyo.SolverFactory(master_solver) if opt is None: raise Exception("Error: Failed to Create Master Solver") # set options for k, v in self.options["master_solver_options"].items(): opt.options[k] = v is_persistent = sputils.is_persistent(opt) if is_persistent: opt.set_instance(m) t = time.time() res, t1, t2 = None, None, None # benders solve loop, repeats the benders master - subproblem # loop until either a no more cuts can are generated # or the maximum iterations limit is reached for self.iter in range(max_iter): if verbose and self.rank == self.rank0: if self.iter > 0: print("Current Iteration:", self.iter + 1, "Time Elapsed:", "%7.2f" % (time.time() - t), "Time Spent on Last Master:", "%7.2f" % t1, "Time Spent Generating Last Cut Set:", "%7.2f" % t2, "Current Objective:", "%7.2f" % m.obj.expr()) else: print("Current Iteration:", self.iter + 1, "Time Elapsed:", "%7.2f" % (time.time() - t), "Current Objective: -Inf") t1 = time.time() x_vals = np.zeros(len(self.master_vars)) eta_vals = np.zeros(self.scenario_count) outer_bound = np.zeros(1) if self.rank == self.rank0: if is_persistent: res = opt.solve(tee=False) else: res = opt.solve(m, tee=False) # LShaped is always minimizing outer_bound[0] = res.Problem[0].Lower_bound for i, var in enumerate(self.master_vars): x_vals[i] = var.value for i, eta in enumerate(m.eta.values()): eta_vals[i] = eta.value self.mpicomm.Bcast(x_vals, root=self.rank0) self.mpicomm.Bcast(eta_vals, root=self.rank0) self.mpicomm.Bcast(outer_bound, root=self.rank0) if self.is_minimizing: self._LShaped_bound = outer_bound[0] else: # LShaped is always minimizing, so negate # the outer bound for sharing broadly self._LShaped_bound = -outer_bound[0] if self.rank != self.rank0: for i, var in enumerate(self.master_vars): var._value = x_vals[i] for i, eta in enumerate(m.eta.values()): eta._value = eta_vals[i] t1 = time.time() - t1 # The hub object takes precedence over the converger # We'll send the nonants now, and check for a for # convergence if self.spcomm: self.spcomm.sync(send_nonants=True) if self.spcomm.is_converged(): break t2 = time.time() cuts_added = m.bender.generate_cut() t2 = time.time() - t2 if self.rank == self.rank0: for c in cuts_added: if is_persistent: opt.add_constraint(c) if verbose and len(cuts_added) == 0: print(f"Converged in {self.iter+1} iterations.\n" f"Total Time Elapsed: {time.time()-t:7.2f} " f"Time Spent on Last Master: {t1:7.2f} " f"Time spent verifying second stage: {t2:7.2f} " f"Final Objective: {m.obj.expr():7.2f}") break if verbose and self.iter == max_iter - 1: print("WARNING MAX ITERATION LIMIT REACHED !!! ") else: if len(cuts_added) == 0: break # The hub object takes precedence over the converger if self.spcomm: self.spcomm.sync(send_nonants=False) if self.spcomm.is_converged(): break if converger: converger.convergence_value() if converger.is_converged(): if verbose and self.rank == self.rank0: print( f"Converged to user criteria in {self.iter+1} iterations.\n" f"Total Time Elapsed: {time.time()-t:7.2f} " f"Time Spent on Last Master: {t1:7.2f} " f"Time spent verifying second stage: {t2:7.2f} " f"Final Objective: {m.obj.expr():7.2f}") break return res
def create_subproblem(self, scenario_name): """ the subproblem creation function passed into the BendersCutsGenerator """ instance = self.local_scenarios[scenario_name] nonant_list, nonant_ids = _get_nonant_ids(instance) # NOTE: since we use generate_standard_repn below, we need # to unfix any nonants so they'll properly appear # in the objective fixed_nonants = [var for var in nonant_list if var.fixed] for var in fixed_nonants: var.fixed = False # pulls the scenario objective expression, removes the first stage variables, and sets the new objective obj = find_active_objective(instance) if not hasattr(instance, "PySP_prob"): instance.PySP_prob = 1. / self.scenario_count PySP_prob = instance.PySP_prob repn = generate_standard_repn(obj.expr, quadratic=True) if len(repn.nonlinear_vars) > 0: raise ValueError( "LShaped does not support models with nonlinear objective functions" ) linear_vars = list() linear_coefs = list() quadratic_vars = list() quadratic_coefs = list() ## we'll assume the constant is part of stage 1 (wlog it is), just ## like the first-stage bits of the objective constant = repn.constant ## only keep the second stage variables in the objective for coef, var in zip(repn.linear_coefs, repn.linear_vars): id_var = id(var) if id_var not in nonant_ids: linear_vars.append(var) linear_coefs.append(PySP_prob * coef) for coef, (x, y) in zip(repn.quadratic_coefs, repn.quadratic_vars): id_x = id(x) id_y = id(y) if id_x not in nonant_ids or id_y not in nonant_ids: quadratic_coefs.append(PySP_prob * coef) quadratic_vars.append((x, y)) # checks if model sense is max, if so negates the objective if not self.is_minimizing: for i, coef in enumerate(linear_coefs): linear_coefs[i] = -coef for i, coef in enumerate(quadratic_coefs): quadratic_coefs[i] = -coef expr = LinearExpression(constant=constant, linear_coefs=linear_coefs, linear_vars=linear_vars) if quadratic_coefs: expr += pyo.quicksum( (coef * x * y for coef, (x, y) in zip(quadratic_coefs, quadratic_vars))) instance.del_component(obj) # set subproblem objective function instance.obj = pyo.Objective(expr=expr, sense=pyo.minimize) ## need to do this here for validity if computing the eta bound if self.relax_master: # relaxes any integrality constraints for the subproblem RelaxIntegerVars().apply_to(instance) if self.compute_eta_bound: for var in fixed_nonants: var.fixed = True opt = pyo.SolverFactory(self.options["sp_solver"]) if self.options["sp_solver_options"]: for k, v in self.options["sp_solver_options"].items(): opt.options[k] = v if sputils.is_persistent(opt): opt.set_instance(instance) res = opt.solve(tee=False) else: res = opt.solve(instance, tee=False) eta_lb = res.Problem[0].Lower_bound self.valid_eta_lb[scenario_name] = eta_lb # if not done above if not self.relax_master: # relaxes any integrality constraints for the subproblem RelaxIntegerVars().apply_to(instance) # iterates through constraints and removes first stage constraints from the model # the id dict is used to improve the speed of identifying the stage each variables belongs to for constr_data in list( itertools.chain( instance.component_data_objects(SOSConstraint, active=True, descend_into=True), instance.component_data_objects(Constraint, active=True, descend_into=True))): if _first_stage_only(constr_data, nonant_ids): _del_con(constr_data) # creates the sub map to remove first stage variables from objective expression complicating_vars_map = pyo.ComponentMap() subproblem_to_master_vars_map = pyo.ComponentMap() # creates the complicating var map that connects the first stage variables in the sub problem to those in # the master problem -- also set the bounds on the subproblem master vars to be none for better cuts for var, mvar in zip(nonant_list, self.master_vars): if var.name not in mvar.name: # mvar.name may be part of a bundle raise Exception( "Error: Complicating variable mismatch, sub-problem variables changed order" ) complicating_vars_map[mvar] = var subproblem_to_master_vars_map[var] = mvar # these are already enforced in the master # don't need to be enfored in the subproblems var.setlb(None) var.setub(None) var.fixed = False # this is for interefacing with PH code instance._subproblem_to_master_vars_map = subproblem_to_master_vars_map if self.store_subproblems: self.subproblems[scenario_name] = instance return instance, complicating_vars_map
def SDM(self, model_name): ''' Algorithm 2 in Boland et al. (with small tweaks) ''' mip = self.local_subproblems[model_name] qp = self.local_QP_subproblems[model_name] # Set the QP dual weights to the correct values If we are bundling, we # initialize the QP dual weights to be the dual weights associated with # the first scenario in the bundle (this should be okay, because each # scenario in the bundle has the same dual weights, analytically--maybe # a numerical problem). arb_scen_mip = self.local_scenarios[mip.scen_list[0]] \ if self.bundling else mip for (node_name, ix) in arb_scen_mip._nonant_indexes: qp._Ws[node_name, ix]._value = \ arb_scen_mip._Ws[node_name, ix].value alpha = self.FW_options['FW_weight'] # Algorithm 3 line 6 xt = { ndn_i: (1 - alpha) * pyo.value(arb_scen_mip._xbars[ndn_i]) + alpha * pyo.value(xvar) for ndn_i, xvar in arb_scen_mip._nonant_indexes.items() } for itr in range(self.FW_options['FW_iter_limit']): # Algorithm 2 line 4 mip_source = mip.scen_list if self.bundling else [model_name] for scenario_name in mip_source: scen_mip = self.local_scenarios[scenario_name] for ndn_i, nonant in scen_mip._nonant_indexes.items(): x_source = xt[ndn_i] if itr==0 \ else nonant._value scen_mip._Ws[ndn_i]._value = ( qp._Ws[ndn_i]._value + scen_mip._PHrho[ndn_i]._value * (x_source - scen_mip._xbars[ndn_i]._value)) # Algorithm 2 line 5 if (sputils.is_persistent(mip._solver_plugin)): mip_obj = find_active_objective(mip, True) mip._solver_plugin.set_objective(mip_obj) mip_results = mip._solver_plugin.solve(mip) self._check_solve(mip_results, model_name + ' (MIP)') # Algorithm 2 lines 6--8 obj = find_active_objective(mip, True) if (itr == 0): dual_bound = pyo.value(obj) # Algorithm 2 line 9 (compute \Gamma^t) val0 = pyo.value(obj) new = replace_expressions(obj.expr, mip.mip_to_qp) val1 = pyo.value(new) obj.expr = replace_expressions(new, qp.qp_to_mip) if abs(val0) > 1e-9: stop_check = (val1 - val0) / abs( val0) # \Gamma^t in Boland, but normalized else: stop_check = val1 - val0 # \Gamma^t in Boland stop_check_tol = self.FW_options["stop_check_tol"]\ if "stop_check_tol" in self.FW_options else 1e-4 if (self.is_minimizing and stop_check < -stop_check_tol): print( 'Warning (fwph): convergence quantity Gamma^t = ' '{sc:.2e} (should be non-negative)'.format(sc=stop_check)) print('Try decreasing the MIP gap tolerance and re-solving') elif (not self.is_minimizing and stop_check > stop_check_tol): print( 'Warning (fwph): convergence quantity Gamma^t = ' '{sc:.2e} (should be non-positive)'.format(sc=stop_check)) print('Try decreasing the MIP gap tolerance and re-solving') self._add_QP_column(model_name) if (sputils.is_persistent(qp._QP_solver_plugin)): qp_obj = find_active_objective(qp, True) qp._QP_solver_plugin.set_objective(qp_obj) qp_results = qp._QP_solver_plugin.solve(qp) self._check_solve(qp_results, model_name + ' (QP)') if (stop_check < self.FW_options['FW_conv_thresh']): break # Re-set the mip._Ws so that the QP objective # is correct in the next major iteration mip_source = mip.scen_list if self.bundling else [model_name] for scenario_name in mip_source: scen_mip = self.local_scenarios[scenario_name] for (node_name, ix) in scen_mip._nonant_indexes: scen_mip._Ws[node_name, ix]._value = \ qp._Ws[node_name, ix]._value return dual_bound
def solve_one(self, solver_options, k, s, dtiming=False, gripe=False, tee=False, verbose=False, disable_pyomo_signal_handling=False, update_objective=True): """ Solve one subproblem. Args: solver_options (dict or None): The scenario solver options. k (str): Subproblem name. s (ConcreteModel with appendages): The subproblem to solve. dtiming (boolean, optional): If True, reports timing values. Default False. gripe (boolean, optional): If True, outputs a message when a solve fails. Default False. tee (boolean, optional): If True, displays solver output. Default False. verbose (boolean, optional): If True, displays verbose output. Default False. disable_pyomo_signal_handling (boolean, optional): True for asynchronous PH; ignored for persistent solvers. Default False. update_objective (boolean, optional): If True, and a persistent solver is used, update the persistent solver's objective Returns: float: Pyomo solve time in seconds. """ def _vb(msg): if verbose and self.cylinder_rank == 0: print("(rank0) " + msg) # if using a persistent solver plugin, # re-compile the objective due to changed weights and x-bars if update_objective and (sputils.is_persistent(s._solver_plugin)): set_objective_start_time = time.time() active_objective_datas = list( s.component_data_objects(pyo.Objective, active=True, descend_into=True)) if len(active_objective_datas) > 1: raise RuntimeError('Multiple active objectives identified ' 'for scenario {sn}'.format(sn=s._name)) elif len(active_objective_datas) < 1: raise RuntimeError('Could not find any active objectives ' 'for scenario {sn}'.format(sn=s._name)) else: s._solver_plugin.set_objective(active_objective_datas[0]) if dtiming: set_objective_time = time.time() - set_objective_start_time all_set_objective_times = self.mpicomm.gather( set_objective_time, root=0) if self.cylinder_rank == 0: print("Set objective times (seconds):") print("\tmin=%4.2f mean=%4.2f max=%4.2f" % (np.mean(all_set_objective_times), np.mean(all_set_objective_times), np.max(all_set_objective_times))) if self.extensions is not None: results = self.extobject.pre_solve(s) solve_start_time = time.time() if (solver_options): _vb("Using sub-problem solver options=" + str(solver_options)) for option_key, option_value in solver_options.items(): s._solver_plugin.options[option_key] = option_value solve_keyword_args = dict() if self.cylinder_rank == 0: if tee is not None and tee is True: solve_keyword_args["tee"] = True if (sputils.is_persistent(s._solver_plugin)): solve_keyword_args["save_results"] = False elif disable_pyomo_signal_handling: solve_keyword_args["use_signal_handling"] = False try: results = s._solver_plugin.solve(s, **solve_keyword_args, load_solutions=False) solver_exception = None except Exception as e: results = None solver_exception = e if self.extensions is not None: results = self.extobject.post_solve(s, results) pyomo_solve_time = time.time() - solve_start_time if (results is None) or (len(results.solution) == 0) or \ (results.solution(0).status == SolutionStatus.infeasible) or \ (results.solver.termination_condition == TerminationCondition.infeasible) or \ (results.solver.termination_condition == TerminationCondition.infeasibleOrUnbounded) or \ (results.solver.termination_condition == TerminationCondition.unbounded): s._mpisppy_data.scenario_feasible = False if gripe: name = self.__class__.__name__ if self.spcomm: name = self.spcomm.__class__.__name__ print(f"[{name}] Solve failed for scenario {s.name}") if results is not None: print("status=", results.solver.status) print("TerminationCondition=", results.solver.termination_condition) if solver_exception is not None: raise solver_exception else: if sputils.is_persistent(s._solver_plugin): s._solver_plugin.load_vars() else: s.solutions.load_from(results) if self.is_minimizing: s._mpisppy_data.outer_bound = results.Problem[0].Lower_bound else: s._mpisppy_data.outer_bound = results.Problem[0].Upper_bound s._mpisppy_data.scenario_feasible = True # TBD: get this ready for IPopt (e.g., check feas_prob every time) # propogate down if self.bundling: # must be a bundle for sname in s._ef_scenario_names: self.local_scenarios[sname]._mpisppy_data.scenario_feasible\ = s._mpisppy_data.scenario_feasible return pyomo_solve_time
def make_cuts(self, coefs): # take the coefficient array and assemble cuts accordingly # this should have already been set in the extension ! opt = self.opt # rows are # [ const, eta_coeff, *nonant_coeffs ] row_len = 1 + 1 + self.nonant_len outer_iter = int(coefs[-1]) bundling = opt.bundling if opt.bundling: for bn, b in opt.local_subproblems.items(): persistent_solver = sputils.is_persistent(b._solver_plugin) ## get an arbitrary scenario s = opt.local_scenarios[b.scen_list[0]] for idx, k in enumerate(opt.all_scenario_names): row = coefs[row_len * idx:row_len * (idx + 1)] # the row could be all zeros, # which doesn't do anything if (row == 0.).all(): continue # rows are # [ const, eta_coeff, *nonant_coeffs ] linear_const = row[0] linear_coefs = list(row[1:]) linear_vars = [b._mpisppy_model.eta[k]] for ndn_i in s._mpisppy_data.nonant_indices: ## for bundles, we add the constrains only ## to the reference first stage variables linear_vars.append(b.ref_vars[ndn_i]) cut_expr = LinearExpression(constant=linear_const, linear_coefs=linear_coefs, linear_vars=linear_vars) b._mpisppy_model.benders_cuts[outer_iter, k] = (None, cut_expr, 0) if persistent_solver: b._solver_plugin.add_constraint( b._mpisppy_model.benders_cuts[outer_iter, k]) else: for sn, s in opt.local_subproblems.items(): persistent_solver = sputils.is_persistent(s._solver_plugin) for idx, k in enumerate(opt.all_scenario_names): row = coefs[row_len * idx:row_len * (idx + 1)] # the row could be all zeros, # which doesn't do anything if (row == 0.).all(): continue # rows are # [ const, eta_coeff, *nonant_coeffs ] linear_const = row[0] linear_coefs = list(row[1:]) linear_vars = [s._mpisppy_model.eta[k]] linear_vars.extend(s._mpisppy_data.nonant_indices.values()) cut_expr = LinearExpression(constant=linear_const, linear_coefs=linear_coefs, linear_vars=linear_vars) s._mpisppy_model.benders_cuts[outer_iter, k] = (None, cut_expr, 0.) if persistent_solver: s._solver_plugin.add_constraint( s._mpisppy_model.benders_cuts[outer_iter, k]) # NOTE: the LShaped code negates the objective, so # we do the same here for consistency ib = self.BestInnerBound ob = self.BestOuterBound if not opt.is_minimizing: ib = -ib ob = -ob add_cut = (isfinite(ib) or isfinite(ob)) and \ ((ib < self.best_inner_bound) or (ob > self.best_outer_bound)) if add_cut: self.best_inner_bound = ib self.best_outer_bound = ob for sn, s in opt.local_subproblems.items(): persistent_solver = sputils.is_persistent(s._solver_plugin) prior_outer_iter = list( s._mpisppy_model.inner_bound_constr.keys()) s._mpisppy_model.inner_bound_constr[outer_iter] = ( ob, s._mpisppy_model.EF_obj, ib) if persistent_solver: s._solver_plugin.add_constraint( s._mpisppy_model.inner_bound_constr[outer_iter]) # remove other ib constraints (we only need the tightest) for it in prior_outer_iter: if persistent_solver: s._solver_plugin.remove_constraint( s._mpisppy_model.inner_bound_constr[it]) del s._mpisppy_model.inner_bound_constr[it] ## helping the extention track cuts self.new_cuts = True