def pysp_instance_creation_callback(scenario_name, use_integer=False, sense=pyo.minimize, crops_multiplier=1): # long function to create the entire model # scenario_name is a string (e.g. AboveAverageScenario0) # # Returns a concrete model for the specified scenario # scenarios come in groups of three scengroupnum = sputils.extract_num(scenario_name) scenario_base_name = scenario_name.rstrip("0123456789") model = pyo.ConcreteModel() def crops_init(m): retval = [] for i in range(crops_multiplier): retval.append("WHEAT" + str(i)) retval.append("CORN" + str(i)) retval.append("SUGAR_BEETS" + str(i)) return retval model.CROPS = pyo.Set(initialize=crops_init) # # Parameters # model.TOTAL_ACREAGE = 500.0 * crops_multiplier def _scale_up_data(indict): outdict = {} for i in range(crops_multiplier): for crop in ['WHEAT', 'CORN', 'SUGAR_BEETS']: outdict[crop + str(i)] = indict[crop] return outdict model.PriceQuota = _scale_up_data({ 'WHEAT': 100000.0, 'CORN': 100000.0, 'SUGAR_BEETS': 6000.0 }) model.SubQuotaSellingPrice = _scale_up_data({ 'WHEAT': 170.0, 'CORN': 150.0, 'SUGAR_BEETS': 36.0 }) model.SuperQuotaSellingPrice = _scale_up_data({ 'WHEAT': 0.0, 'CORN': 0.0, 'SUGAR_BEETS': 10.0 }) model.CattleFeedRequirement = _scale_up_data({ 'WHEAT': 200.0, 'CORN': 240.0, 'SUGAR_BEETS': 0.0 }) model.PurchasePrice = _scale_up_data({ 'WHEAT': 238.0, 'CORN': 210.0, 'SUGAR_BEETS': 100000.0 }) model.PlantingCostPerAcre = _scale_up_data({ 'WHEAT': 150.0, 'CORN': 230.0, 'SUGAR_BEETS': 260.0 }) # # Stochastic Data # Yield = {} Yield['BelowAverageScenario'] = \ {'WHEAT':2.0,'CORN':2.4,'SUGAR_BEETS':16.0} Yield['AverageScenario'] = \ {'WHEAT':2.5,'CORN':3.0,'SUGAR_BEETS':20.0} Yield['AboveAverageScenario'] = \ {'WHEAT':3.0,'CORN':3.6,'SUGAR_BEETS':24.0} def Yield_init(m, cropname): # yield as in "crop yield" crop_base_name = cropname.rstrip("0123456789") if scengroupnum != 0: return Yield[scenario_base_name][ crop_base_name] + farmerstream.rand() else: return Yield[scenario_base_name][crop_base_name] model.Yield = pyo.Param(model.CROPS, within=pyo.NonNegativeReals, initialize=Yield_init, mutable=True) # # Variables # if (use_integer): model.DevotedAcreage = pyo.Var(model.CROPS, within=pyo.NonNegativeIntegers, bounds=(0.0, model.TOTAL_ACREAGE)) else: model.DevotedAcreage = pyo.Var(model.CROPS, bounds=(0.0, model.TOTAL_ACREAGE)) model.QuantitySubQuotaSold = pyo.Var(model.CROPS, bounds=(0.0, None)) model.QuantitySuperQuotaSold = pyo.Var(model.CROPS, bounds=(0.0, None)) model.QuantityPurchased = pyo.Var(model.CROPS, bounds=(0.0, None)) # # Constraints # def ConstrainTotalAcreage_rule(model): return pyo.sum_product(model.DevotedAcreage) <= model.TOTAL_ACREAGE model.ConstrainTotalAcreage = pyo.Constraint( rule=ConstrainTotalAcreage_rule) def EnforceCattleFeedRequirement_rule(model, i): return model.CattleFeedRequirement[i] <= ( model.Yield[i] * model.DevotedAcreage[i] ) + model.QuantityPurchased[i] - model.QuantitySubQuotaSold[ i] - model.QuantitySuperQuotaSold[i] model.EnforceCattleFeedRequirement = pyo.Constraint( model.CROPS, rule=EnforceCattleFeedRequirement_rule) def LimitAmountSold_rule(model, i): return model.QuantitySubQuotaSold[i] + model.QuantitySuperQuotaSold[ i] - (model.Yield[i] * model.DevotedAcreage[i]) <= 0.0 model.LimitAmountSold = pyo.Constraint(model.CROPS, rule=LimitAmountSold_rule) def EnforceQuotas_rule(model, i): return (0.0, model.QuantitySubQuotaSold[i], model.PriceQuota[i]) model.EnforceQuotas = pyo.Constraint(model.CROPS, rule=EnforceQuotas_rule) # Stage-specific cost computations; def ComputeFirstStageCost_rule(model): return pyo.sum_product(model.PlantingCostPerAcre, model.DevotedAcreage) model.FirstStageCost = pyo.Expression(rule=ComputeFirstStageCost_rule) def ComputeSecondStageCost_rule(model): expr = pyo.sum_product(model.PurchasePrice, model.QuantityPurchased) expr -= pyo.sum_product(model.SubQuotaSellingPrice, model.QuantitySubQuotaSold) expr -= pyo.sum_product(model.SuperQuotaSellingPrice, model.QuantitySuperQuotaSold) return expr model.SecondStageCost = pyo.Expression(rule=ComputeSecondStageCost_rule) def total_cost_rule(model): if (sense == pyo.minimize): return model.FirstStageCost + model.SecondStageCost return -model.FirstStageCost - model.SecondStageCost model.Total_Cost_Objective = pyo.Objective(rule=total_cost_rule, sense=sense) return model
def pysp2_callback(scenario_name, node_names=None, cb_data=None): """ mpisppy signature for scenario creation. Then find a starting solution for the scenario if solver option is not None. Note that stage numbers are one-based. Args: scenario_name (str): put the scenario number on the end node_names (int): not used cb_data: (dict) "etree", "solver", "epath", "tee", "acstream", "convex_relaxation" Returns: scenario (pyo.ConcreteModel): the scenario instance Attaches: _enodes (ACtree nodes): a list of the ACtree tree nodes _egret_md (egret tuple with dict as [1]) egret model data """ # pull the number off the end of the scenario name scen_num = sputils.extract_num(scenario_name) etree = cb_data["etree"] solver = cb_data["solver"] acstream = cb_data["acstream"] convex_relaxation = cb_data["convex_relaxation"] if "convex_relaxation"\ in cb_data else False def lines_up_and_down(stage_md_dict, enode): # local routine to configure the lines in stage_md_dict for the scenario LinesDown = [] for f in enode.FailedLines: LinesDown.append(f[0]) for this_branch in stage_md_dict.elements("branch"): if this_branch[0] in enode.LinesUp: this_branch[1]["in_service"] = True elif this_branch[0] in LinesDown: this_branch[1]["in_service"] = False else: print("enode.LinesUp=", enode.LinesUp) print("enode.FailedLines=", enode.FailedLines) raise RuntimeError("Branch (line) {} neither up nor down in scenario {}".\ format(this_branch[0], scenario_name)) def _egret_model(md_dict): # the exact acopf model is hard-wired here: if not convex_relaxation: pyomod, mdict = eac.create_riv_acopf_model( md_dict, include_feasibility_slack=True) else: pyomod, mdict = eac_relax.create_soc_relaxation( md_dict, include_feasibility_slack=True, use_linear_relaxation=False) return pyomod, mdict # pull the number off the end of the scenario name scen_num = sputils.extract_num(scenario_name) #print ("debug scen_num=",scen_num) numstages = etree.NumStages enodes = etree.Nodes_for_Scenario(scen_num) full_scenario_model = pyo.ConcreteModel() full_scenario_model.stage_models = dict() # look at egret/data/model_data.py for the format specification of md_dict first_stage_md_dict = _md_dict(cb_data) generator_set = first_stage_md_dict.attributes("generator") generator_names = generator_set["names"] # the following creates the first stage model full_scenario_model.stage_models[1], model_dict = _egret_model( first_stage_md_dict) full_scenario_model.stage_models[1].obj.deactivate() setattr(full_scenario_model, "stage_models_" + str(1), full_scenario_model.stage_models[1]) for stage in range(2, numstages + 1): #print ("stage={}".format(stage)) stage_md_dict = copy.deepcopy(first_stage_md_dict) #print ("debug: processing node {}".format(enodes[stage-1].Name)) lines_up_and_down(stage_md_dict, enodes[stage - 1]) full_scenario_model.stage_models[stage], model_dict = \ _egret_model(stage_md_dict) full_scenario_model.stage_models[stage].obj.deactivate() setattr(full_scenario_model, "stage_models_" + str(stage), full_scenario_model.stage_models[stage]) def aggregate_ramping_rule(m): """ We are adding ramping to the obj instead of a constraint for now because we may not have ramp limit data. """ retval = 0 for stage in range(1, numstages): retval += sum((full_scenario_model.stage_models[stage+1].pg[this_gen]\ - full_scenario_model.stage_models[stage].pg[this_gen])**2\ for this_gen in generator_names) return retval full_scenario_model.ramping = pyo.Expression(rule=aggregate_ramping_rule) full_scenario_model.objective = pyo.Objective(expr=\ 1000000.0 * full_scenario_model.ramping+\ sum(full_scenario_model.stage_models[stage].obj.expr\ for stage in range(1,numstages+1))) inst = full_scenario_model # end code from PySP1 node_list = list() parent_name = None for sm1, enode in enumerate(etree.Nodes_for_Scenario(scen_num)): stage = sm1 + 1 if stage < etree.NumStages: node_list.append( scenario_tree.ScenarioNode( name=enode.Name, cond_prob=enode.CondProb, stage=stage, cost_expression=inst.stage_models[stage].obj, scen_name_list=enode.ScenarioList, nonant_list=[ inst.stage_models[stage].pg, inst.stage_models[stage].qg ], scen_model=inst, parent_name=parent_name)) parent_name = enode.Name inst._PySPnode_list = node_list # Optionally assign probability to PySP_prob inst.PySP_prob = 1 / etree.numscens # solve it so subsequent code will have a good start if solver is not None: print( f"scenario creation callback is solving {scenario_name} on rank {rank}" ) solver.solve( inst) #, tee=True) #symbolic_solver_labels=True, keepfiles=True) # attachments inst._enodes = enodes inst._egret_md = first_stage_md_dict return inst
def iterk(self, PHIter): """ Before iter k>1 solves, but after x-bar update. """ # first, do some persistent solver with bundles gymnastics have_bundles = hasattr(self.ph, "saved_objs") # indicates bundles if have_bundles: subpname = next(iter(self.ph.local_subproblems)) subp = self.ph.local_subproblems[subpname] solver_is_persistent = isinstance( subp._solver_plugin, pyo.pyomo.solvers.plugins.solvers. persistent_solver.PersistentSolver) if solver_is_persistent: vars_to_update = {} fixoptions = self.fixeroptions raw_fixed_so_far = 0 self._update_fix_counts() for sname, s in self.local_scenarios.items(): if self.fixer_tuples[s] is None: print("MAJOR WARNING: No Iter k fixer tuple for s.name=", s.name) return if not have_bundles: solver_is_persistent = isinstance( s._solver_plugin, pyo.pyomo.solvers.plugins.solvers. persistent_solver.PersistentSolver) for (varid, th, nb, lb, ub) in self.fixer_tuples[s]: was_fixed = False try: (ndn, i) = s._varid_to_nonant_index[varid] except: print("Are you trying to fix a Var that is not nonant?") raise tolval = self.threshold[(ndn, i)] xvar = s._nonant_indexes[ndn, i] if not xvar.is_fixed(): xb = pyo.value(s._xbars[(ndn, i)]) fx = s._PySP_conv_iter_count[(ndn, i)] if fx > 0: xbar = pyo.value(s._xbars[(ndn, i)]) was_fixed = False if nb is not None and nb <= fx: xvar.fix(xbar) self._vb("Fixed nb %s %s at %s" % \ (s.name, xvar.name, str(xvar._value))) was_fixed = True elif lb is not None and lb < fx \ and xb - xvar.lb < self.boundtol: xvar.fix(xvar.lb) self._vb("Fixed lb %s %s at %s" % \ (s.name, xvar.name, str(xvar._value))) was_fixed = True elif ub is not None and ub < fx \ and xvar.ub - xb < self.boundtol: xvar.fix(xvar.ub) self._vb("Fixed ub %s %s at %s" % \ (s.name, xvar.name, str(xvar._value))) was_fixed = True if was_fixed: raw_fixed_so_far += 1 if not have_bundles and solver_is_persistent: s._solver_plugin.update_var(xvar) if have_bundles and solver_is_persistent: if sname not in vars_to_update: vars_to_update[sname] = [] vars_to_update[sname].append(xvar) if have_bundles and solver_is_persistent: for k, subp in self.ph.local_subproblems.items(): subpnum = sputils.extract_num(k) rank_local = self.ph.rank for sname in self.ph.names_in_bundles[rank_local][subpnum]: if sname in vars_to_update: for xvar in vars_to_update[sname]: subp._solver_plugin.update_var(xvar) self.fixed_so_far += raw_fixed_so_far / len(self.local_scenarios) self._dp("Unique Vars fixed so far %s" % (self.fixed_so_far)) if raw_fixed_so_far % len(self.local_scenarios) != 0: raise RuntimeError("Variation in fixing across scenarios detected " "in fixer.py")
def scenario_creator( scenario_name, use_integer=False, sense=pyo.minimize, crops_multiplier=1, ): """ Create a scenario for the (scalable) farmer example. Args: scenario_name (str): Name of the scenario to construct. use_integer (bool, optional): If True, restricts variables to be integer. Default is False. sense (int, optional): Model sense (minimization or maximization). Must be either pyo.minimize or pyo.maximize. Default is pyo.minimize. crops_multiplier (int, optional): Factor to control scaling. There will be three times this many crops. Default is 1. """ # scenario_name has the form <str><int> e.g. scen12, foobar7 # The digits are scraped off the right of scenario_name using regex then # converted mod 3 into one of the below avg./avg./above avg. scenarios scennum = sputils.extract_num(scenario_name) basenames = [ 'BelowAverageScenario', 'AverageScenario', 'AboveAverageScenario' ] basenum = scennum % 3 groupnum = scennum // 3 scenname = basenames[basenum] + str(groupnum) # The RNG is seeded with the scenario number so that it is # reproducible when used with multiple threads. # NOTE: if you want to do replicates, you will need to pass a seed # as a kwarg to scenario_creator then use seed+scennum as the seed argument. farmerstream.seed(scennum) # Check for minimization vs. maximization if sense not in [pyo.minimize, pyo.maximize]: raise ValueError("Model sense Not recognized") # Create the concrete model object model = pysp_instance_creation_callback( scenname, use_integer=use_integer, sense=sense, crops_multiplier=crops_multiplier, ) # Create the list of nodes associated with the scenario (for two stage, # there is only one node associated with the scenario--leaf nodes are # ignored). model._PySPnode_list = [ scenario_tree.ScenarioNode( name="ROOT", cond_prob=1.0, stage=1, cost_expression=model.FirstStageCost, scen_name_list=None, # Deprecated? nonant_list=[model.DevotedAcreage], scen_model=model, ) ] return model
def iter0(self, local_scenarios): # first, do some persistent solver with bundles gymnastics have_bundles = hasattr(self.ph, "saved_objs") # indicates bundles if have_bundles: subpname = next(iter(self.ph.local_subproblems)) subp = self.ph.local_subproblems[subpname] solver_is_persistent = isinstance( subp._solver_plugin, pyo.pyomo.solvers.plugins.solvers. persistent_solver.PersistentSolver) if solver_is_persistent: vars_to_update = {} fixoptions = self.fixeroptions raw_fixed_so_far = 0 # count those fixed in each scenario for sname, s in self.ph.local_scenarios.items(): if self.iter0_fixer_tuples[s] is None: print("WARNING: No Iter0 fixer tuple for s.name=", s.name) return if not have_bundles: solver_is_persistent = isinstance( s._solver_plugin, pyo.pyomo.solvers.plugins.solvers. persistent_solver.PersistentSolver) for (varid, th, nb, lb, ub) in self.iter0_fixer_tuples[s]: was_fixed = False try: (ndn, i) = s._varid_to_nonant_index[varid] except: print("Are you trying to fix a Var that is not nonant?") raise xvar = s._nonant_indexes[ndn, i] if not xvar.is_fixed(): xb = pyo.value(s._xbars[(ndn, i)]) diff = xb * xb - pyo.value(s._xsqbars[(ndn, i)]) tolval = self.iter0_threshold[(ndn, i)] sqtolval = tolval * tolval # the tol is on sqrt if -diff > sqtolval or diff > sqtolval: ##print ("debug0 NO fix diff, sqtolval", diff, sqtolval) continue else: ##print ("debug0 fix diff, sqtolval", diff, sqtolval) # if we are still here, it is converged if nb is not None: xvar.fix(xb) self._vb("Fixed0 nb %s %s at %s" % \ (s.name, xvar.name, str(xvar._value))) was_fixed = True elif lb is not None and xb - xvar.lb < self.boundtol: xvar.fix(xvar.lb) self._vb("Fixed0 lb %s %s at %s" % \ (s.name, xvar.name, str(xvar._value))) was_fixed = True elif ub is not None and xvar.ub - xb < self.boundtol: xvar.fix(xvar.ub) self._vb("Fixed0 ub %s %s at %s" % \ (s.name, xvar.name, str(xvar._value))) was_fixed = True if was_fixed: raw_fixed_so_far += 1 if not have_bundles and solver_is_persistent: s._solver_plugin.update_var(xvar) if have_bundles and solver_is_persistent: if sname not in vars_to_update: vars_to_update[sname] = [] vars_to_update[sname].append(xvar) if have_bundles and solver_is_persistent: for k, subp in self.ph.local_subproblems.items(): subpnum = sputils.extract_num(k) rank_local = self.ph.rank for sname in self.ph.names_in_bundles[rank_local][subpnum]: if sname in vars_to_update: for xvar in vars_to_update[sname]: subp._solver_plugin.update_var(xvar) self.fixed_so_far += raw_fixed_so_far / len(local_scenarios) self._dp("Unique Vars fixed so far %s" % (self.fixed_so_far)) if raw_fixed_so_far % len(local_scenarios) != 0: raise RuntimeError("Variation in fixing across scenarios detected " "in fixer.py (iter0)")
except: print(msg, "\n bad number of bundles per rank=", sys.argv[2]) quit() try: maxit = int(sys.argv[3]) except: print(msg, "\n bad max iterations=", sys.argv[3]) quit() try: rho = int(sys.argv[4]) except: print(msg, "\n bad rho=", sys.argv[4]) quit() # The number of scenarios is the last number in the instance name ScenCount = sputils.extract_num(instname) start_time = dt.datetime.now() PHoptions = {} PHoptions["solvername"] = "gurobi_persistent" PHoptions["PHIterLimit"] = maxit PHoptions["defaultPHrho"] = rho PHoptions["convthresh"] = -1 PHoptions["subsolvedirectives"] = None PHoptions["verbose"] = False PHoptions["display_timing"] = False PHoptions["display_progress"] = True ### async section ### PHoptions["asynchronous"] = True PHoptions["async_frac_needed"] = 0.5
def pysp2_callback( scenario_name, etree=None, solver=None, epath=None, tee=False, acstream=None, convex_relaxation=False, verbose=False, ramp_coeff=1000000, load_mismatch_cost=1000, q_load_mismatch_cost=None, ): """ mpisppy signature for scenario creation. Then find a starting solution for the scenario if solver option is not None. Note that stage numbers are one-based. Args: scenario_name (str): Put the scenario number on the end etree (): Default is None. solver (str): Solver to use. epath (str): Path to the egret data tee (bool): If True, displays solver output. Default is False. acstream (): Default is None. convex_relaxation (bool): If True, build the convex relaxation. Default is False. verbose (bool, optional): If True, display verbose output. Default is False. ramp_coeff (int, optional): Default is 1e6. load_mismatch_cost (float, optional): Default is 1000. q_load_mismatch_cost (float, optional): Deafult is load_mismatch_cost. Returns: scenario (pyo.ConcreteModel): the scenario instance Attaches: _enodes (ACtree nodes): a list of the ACtree tree nodes _egret_md (egret tuple with dict as [1]) egret model data """ print("Debug: convex_relaxation=", convex_relaxation) # pull the number off the end of the scenario name scen_num = sputils.extract_num(scenario_name) if q_load_mismatch_cost is None: q_load_mismatch_cost = load_mismatch_cost def lines_up_and_down(stage_md_dict, enode): # local routine to configure the lines in stage_md_dict for the scenario LinesDown = [] for f in enode.FailedLines: LinesDown.append(f[0]) for this_branch in stage_md_dict.elements("branch"): if this_branch[0] in enode.LinesUp: this_branch[1]["in_service"] = True elif this_branch[0] in LinesDown: this_branch[1]["in_service"] = False else: print("enode.LinesUp=", enode.LinesUp) print("enode.FailedLines=", enode.FailedLines) raise RuntimeError("Branch (line) {} neither up nor down in scenario {}".\ format(this_branch[0], scenario_name)) def _egret_model(md_dict): # the exact acopf model is hard-wired here: md_dict.data['system']['load_mismatch_cost'] = load_mismatch_cost md_dict.data['system']['q_load_mismatch_cost'] = q_load_mismatch_cost if not convex_relaxation: pyomod, mdict = eac.create_riv_acopf_model( md_dict, include_feasibility_slack=True) else: pyomod, mdict = eac_relax.create_soc_relaxation( md_dict, include_feasibility_slack=True, use_linear_relaxation=False) return pyomod, mdict # pull the number off the end of the scenario name scen_num = sputils.extract_num(scenario_name) numstages = etree.NumStages enodes = etree.Nodes_for_Scenario(scen_num) full_scenario_model = pyo.ConcreteModel() full_scenario_model.stage_models = dict() # look at egret/data/model_data.py for the format specification of md_dict first_stage_md_dict = _md_dict(epath) generator_set = first_stage_md_dict.attributes("generator") generator_names = generator_set["names"] # the following creates the first stage model full_scenario_model.stage_models[1], model_dict = _egret_model( first_stage_md_dict) full_scenario_model.stage_models[1].obj.deactivate() setattr(full_scenario_model, "stage_models_" + str(1), full_scenario_model.stage_models[1]) for stage in range(2, numstages + 1): #print ("stage={}".format(stage)) stage_md_dict = copy.deepcopy(first_stage_md_dict) #print ("debug: processing node {}".format(enodes[stage-1].Name)) lines_up_and_down(stage_md_dict, enodes[stage - 1]) full_scenario_model.stage_models[stage], model_dict = \ _egret_model(stage_md_dict) full_scenario_model.stage_models[stage].obj.deactivate() setattr(full_scenario_model, "stage_models_" + str(stage), full_scenario_model.stage_models[stage]) def aggregate_ramping_rule(m): """ We are adding ramping to the obj instead of a constraint for now because we may not have ramp limit data. """ retval = 0 for stage in range(1, numstages): retval += sum((full_scenario_model.stage_models[stage+1].pg[this_gen]\ - full_scenario_model.stage_models[stage].pg[this_gen])**2\ for this_gen in generator_names) return retval full_scenario_model.ramping = pyo.Expression(rule=aggregate_ramping_rule) full_scenario_model.objective = pyo.Objective(expr=\ ramp_coeff * full_scenario_model.ramping+\ sum(full_scenario_model.stage_models[stage].obj.expr\ for stage in range(1,numstages+1))) inst = full_scenario_model # end code from PySP1 node_list = list() parent_name = None for sm1, enode in enumerate(etree.Nodes_for_Scenario(scen_num)): stage = sm1 + 1 if stage < etree.NumStages: node_list.append( scenario_tree.ScenarioNode( name=enode.Name, cond_prob=enode.CondProb, stage=stage, cost_expression=inst.stage_models[stage].obj, scen_name_list=enode.ScenarioList, nonant_list=[ inst.stage_models[stage].pg, inst.stage_models[stage].qg ], scen_model=inst, parent_name=parent_name)) parent_name = enode.Name if verbose: print(f"\nScenario creation for {scenario_name} on rank {rank}" f" FailedLines (line,minutes)={enode.FailedLines}") inst._PySPnode_list = node_list inst.PySP_prob = 1 / etree.numscens # solve it so subsequent code will have a good start if solver is not None: print( f"scenario creation callback is solving {scenario_name} on rank {rank}" ) solver.solve(inst, tee=tee) #symbolic_solver_labels=True, keepfiles=True) # attachments inst._enodes = enodes inst._egret_md = first_stage_md_dict return inst
def gap_estimators(xhat_one, mname, solving_type="EF-2stage", scenario_names=None, sample_options=None, ArRP=1, scenario_creator_kwargs={}, scenario_denouement=None, solvername=None, solver_options=None, verbose=False, objective_gap=False): ''' Given a xhat, scenario names, a scenario creator and options, gap_estimators creates a scenario tree and the associatd estimators G and s from §2 of [bm2011]. Returns G and s evaluated at xhat. If ArRP>1, G and s are pooled, from a number ArRP of estimators, computed with different scenario trees. Parameters ---------- xhat_one : dict A candidate first stage solution mname: str Name of the reference model, e.g. 'mpisppy.tests.examples.farmer'. solving_type: str, optional The way we solve the approximate problem. Can be "EF-2stage" (default) or "EF-mstage". scenario_names: list, optional List of scenario names used to compute G_n and s_n. Default is None Must be specified for 2 stage, but can be missing for multistage sample_options: dict, optional Only for multistage. Must contain a 'seed' and a 'branching_factors' attribute, specifying the starting seed and the branching factors of the scenario tree ArRP:int,optional Number of batches (we create a ArRP model). Default is 1 (one batch). scenario_creator_kwargs: dict, optional Additional arguments for scenario_creator. Default is {} scenario_denouement: function, optional Function to run after scenario creation. Default is None. solvername : str, optional Solver. Default is None solver_options: dict, optional Solving options. Default is None verbose: bool, optional Should it print the gap estimator ? Default is True objective_gap: bool, optional Returns a gap estimate around approximate objective value branching_factors: list, optional Only for multistage. List of branching factors of the sample scenario tree. Returns ------- G_k and s_k, gap estimator and associated standard deviation estimator. ''' global_toc("Enter gap_estimators") if solving_type not in ["EF-2stage", "EF-mstage"]: raise RuntimeError( "Only EF solve for the approximate problem is supported yet.") else: is_multi = (solving_type == "EF-mstage") if is_multi: try: branching_factors = sample_options['branching_factors'] start = sample_options['seed'] except (TypeError, KeyError, RuntimeError): raise RuntimeError( 'For multistage problems, sample_options must be a dict with branching_factors and seed attributes.' ) else: start = sputils.extract_num(scenario_names[0]) if ArRP > 1: #Special case : ArRP, G and s are pooled from r>1 estimators. if is_multi: raise RuntimeError( "Pooled estimators are not supported for multistage problems yet." ) n = len(scenario_names) if (n % ArRP != 0): raise RuntimeWarning("You put as an input a number of scenarios"+\ f" which is not a mutliple of {ArRP}.") n = n - n % ArRP G = [] s = [] for k in range(ArRP): scennames = scenario_names[k * (n // ArRP):(k + 1) * (n // ArRP)] tmp = gap_estimators( xhat_one, mname, solvername=solvername, scenario_names=scennames, ArRP=1, scenario_creator_kwargs=scenario_creator_kwargs, scenario_denouement=scenario_denouement, solver_options=solver_options, solving_type=solving_type) G.append(tmp['G']) s.append(tmp['s']) global_toc(f"ArRP {k} of {ArRP}") #Pooling G = np.mean(G) s = np.linalg.norm(s) / np.sqrt(n // ArRP) return {"G": G, "s": s, "seed": start} #A1RP #We start by computing the optimal solution to the approximate problem induced by our scenarios if is_multi: #Sample a scenario tree: this is a subtree, but starting from stage 1 samp_tree = sample_tree.SampleSubtree( mname, xhats=[], root_scen=None, starting_stage=1, branching_factors=branching_factors, seed=start, options=scenario_creator_kwargs, solvername=solvername, solver_options=solver_options) samp_tree.run() start += sputils.number_of_nodes(branching_factors) ama_object = samp_tree.ama else: #We use amalgamator to do it ama_options = dict(scenario_creator_kwargs) ama_options['start'] = start ama_options['num_scens'] = len(scenario_names) ama_options['EF_solver_name'] = solvername ama_options['EF_solver_options'] = solver_options ama_options[solving_type] = True ama_object = ama.from_module(mname, ama_options, use_command_line=False) ama_object.scenario_names = scenario_names ama_object.verbose = False ama_object.run() start += len(scenario_names) #Optimal solution of the approximate problem zstar = ama_object.best_outer_bound #Associated policies xstars = sputils.nonant_cache_from_ef(ama_object.ef) #Then, we evaluate the fonction value induced by the scenario at xstar. if is_multi: # Find feasible policies (i.e. xhats) for every non-leaf nodes if len(samp_tree.ef._ef_scenario_names) > 1: local_scenarios = { sname: getattr(samp_tree.ef, sname) for sname in samp_tree.ef._ef_scenario_names } else: local_scenarios = { samp_tree.ef._ef_scenario_names[0]: samp_tree.ef } xhats, start = sample_tree.walking_tree_xhats( mname, local_scenarios, xhat_one['ROOT'], branching_factors, start, scenario_creator_kwargs, solvername=solvername, solver_options=solver_options) #Compute then the average function value with this policy scenario_creator_kwargs = samp_tree.ama.kwargs all_nodenames = sputils.create_nodenames_from_branching_factors( branching_factors) else: #In a 2 stage problem, the only non-leaf is the ROOT node xhats = xhat_one all_nodenames = None xhat_eval_options = { "iter0_solver_options": None, "iterk_solver_options": None, "display_timing": False, "solvername": solvername, "verbose": False, "solver_options": solver_options } ev = xhat_eval.Xhat_Eval(xhat_eval_options, scenario_names, ama_object.scenario_creator, scenario_denouement, scenario_creator_kwargs=scenario_creator_kwargs, all_nodenames=all_nodenames) #Evaluating xhat and xstar and getting the value of the objective function #for every (local) scenario zhat = ev.evaluate(xhats) objs_at_xhat = ev.objs_dict zstar = ev.evaluate(xstars) objs_at_xstar = ev.objs_dict eval_scen_at_xhat = [] eval_scen_at_xstar = [] scen_probs = [] for k, s in ev.local_scenarios.items(): eval_scen_at_xhat.append(objs_at_xhat[k]) eval_scen_at_xstar.append(objs_at_xstar[k]) scen_probs.append(s._mpisppy_probability) scen_gaps = np.array(eval_scen_at_xhat) - np.array(eval_scen_at_xstar) local_gap = np.dot(scen_gaps, scen_probs) local_ssq = np.dot(scen_gaps**2, scen_probs) local_prob_sqnorm = np.linalg.norm(scen_probs)**2 local_obj_at_xhat = np.dot(eval_scen_at_xhat, scen_probs) local_estim = np.array( [local_gap, local_ssq, local_prob_sqnorm, local_obj_at_xhat]) global_estim = np.zeros(4) ev.mpicomm.Allreduce(local_estim, global_estim, op=mpi.SUM) G, ssq, prob_sqnorm, obj_at_xhat = global_estim if global_rank == 0 and verbose: print(f"G = {G}") sample_var = (ssq - G**2) / (1 - prob_sqnorm) #Unbiased sample variance s = np.sqrt(sample_var) use_relative_error = (np.abs(zstar) > 1) G = correcting_numeric(G, objfct=obj_at_xhat, relative_error=use_relative_error) if objective_gap: if is_multi: return { "G": G, "s": s, "zhats": [zhat], "zstars": [zstar], "seed": start } else: return { "G": G, "s": s, "zhats": eval_scen_at_xhat, "zstars": eval_scen_at_xstar, "seed": start } else: return {"G": G, "s": s, "seed": start}
def iter0(self, local_scenarios): # first, do some persistent solver with bundles gymnastics have_bundles = hasattr(self.ph, "saved_objs") # indicates bundles if have_bundles: subpname = next(iter(self.ph.local_subproblems)) subp = self.ph.local_subproblems[subpname] solver_is_persistent = isinstance(subp._solver_plugin, pyo.pyomo.solvers.plugins.solvers.persistent_solver.PersistentSolver) if solver_is_persistent: vars_to_update = {} fixoptions = self.fixeroptions # modelers might have already fixed variables - count those up and output the result raw_fixed_on_arrival = 0 raw_fixed_this_iter = 0 for sname,s in self.ph.local_scenarios.items(): if self.iter0_fixer_tuples[s] is None: print ("WARNING: No Iter0 fixer tuple for s.name=",s.name) return if not have_bundles: solver_is_persistent = isinstance(s._solver_plugin, pyo.pyomo.solvers.plugins.solvers.persistent_solver.PersistentSolver) for (varid, th, nb, lb, ub) in self.iter0_fixer_tuples[s]: was_fixed = False try: (ndn, i) = s._mpisppy_data.varid_to_nonant_index[varid] except: print ("Are you trying to fix a Var that is not nonant?") raise xvar = s._mpisppy_data.nonant_indices[ndn,i] if not xvar.is_fixed(): xb = pyo.value(s._mpisppy_model.xbars[(ndn,i)]) diff = xb * xb - pyo.value(s._mpisppy_model.xsqbars[(ndn,i)]) tolval = self.iter0_threshold[(ndn, i)] sqtolval = tolval*tolval # the tol is on sqrt if -diff > sqtolval or diff > sqtolval: ##print ("debug0 NO fix diff, sqtolval", diff, sqtolval) continue else: ##print ("debug0 fix diff, sqtolval", diff, sqtolval) # if we are still here, it is converged if nb is not None: xvar.fix(xb) self._vb("Fixed0 nb %s %s at %s" % \ (s.name, xvar.name, str(xvar._value))) was_fixed = True elif lb is not None and xb - xvar.lb < self.boundtol: xvar.fix(xvar.lb) self._vb("Fixed0 lb %s %s at %s" % \ (s.name, xvar.name, str(xvar._value))) was_fixed = True elif ub is not None and xvar.ub - xb < self.boundtol: xvar.fix(xvar.ub) self._vb("Fixed0 ub %s %s at %s" % \ (s.name, xvar.name, str(xvar._value))) was_fixed = True if was_fixed: raw_fixed_this_iter += 1 if not have_bundles and solver_is_persistent: s._solver_plugin.update_var(xvar) if have_bundles and solver_is_persistent: if sname not in vars_to_update: vars_to_update[sname] = [] vars_to_update[sname].append(xvar) else: # TODO: a paranoid would and should put a check to ensure # that variables are fixed in all scenarios and are fixed # to the same value. raw_fixed_on_arrival += 1 if have_bundles and solver_is_persistent: for k,subp in self.ph.local_subproblems.items(): subpnum = sputils.extract_num(k) rank_local = self.ph.cylinder_rank for sname in self.ph.names_in_bundles[rank_local][subpnum]: if sname in vars_to_update: for xvar in vars_to_update[sname]: subp._solver_plugin.update_var(xvar) self.fixed_prior_iter0 += raw_fixed_on_arrival / len(local_scenarios) self.fixed_so_far += raw_fixed_this_iter / len(local_scenarios) self._dp("Unique vars fixed so far - %d (%d prior to iteration 0)" % (self.fixed_so_far+self.fixed_prior_iter0, self.fixed_prior_iter0)) if raw_fixed_this_iter % len(local_scenarios) != 0: raise RuntimeError ("Variation in fixing across scenarios detected " "in fixer.py (iter0)") # maybe to do mpicomm.abort() if raw_fixed_on_arrival % len(local_scenarios) != 0: raise RuntimeError ("Variation in fixing across scenarios prior to iteration 0 detected " "in fixer.py (iter0)")