def main(): scenario_creator = farmer.scenario_creator CropsMult = int(sys.argv[1]) scen_count = int(sys.argv[2]) solver_name = sys.argv[3] cb_data = {'use_integer': False, "CropsMult": CropsMult} scenario_names = ['Scenario' + str(i) for i in range(scen_count)] ef = sputils.create_EF(scenario_names, scenario_creator, creator_options={'cb_data': cb_data}) solver = pyo.SolverFactory(solver_name) if 'persistent' in solver_name: solver.set_instance(ef, symbolic_solver_labels=True) solver.solve(tee=True) else: solver.solve( ef, tee=True, symbolic_solver_labels=True, ) print(f"EF objective: {pyo.value(ef.EF_Obj)}")
def main(): scenario_creator = farmer.scenario_creator crops_multiplier = int(sys.argv[1]) scen_count = int(sys.argv[2]) solver_name = sys.argv[3] scenario_creator_kwargs = { "use_integer": False, "crops_multiplier": crops_multiplier, } scenario_names = ['Scenario' + str(i) for i in range(scen_count)] ef = sputils.create_EF( scenario_names, scenario_creator, scenario_creator_kwargs=scenario_creator_kwargs, ) solver = pyo.SolverFactory(solver_name) if 'persistent' in solver_name: solver.set_instance(ef, symbolic_solver_labels=True) solver.solve(tee=True) else: solver.solve( ef, tee=True, symbolic_solver_labels=True, ) return ef
def __init__(self, options, all_scenario_names, scenario_creator, model_name=None, scenario_creator_options=None, suppress_warnings=False): cb_data = None if scenario_creator_options is not None: if "cb_data" in scenario_creator_options: cb_data = scenario_creator_options["cb_data"] super().__init__(options, all_scenario_names, scenario_creator, cb_data=cb_data) if self.n_proc > 1 and self.rank == self.rank0: logger.warning( "Creating an ExtensiveForm object in parallel. Why?") if scenario_creator_options is None: scenario_creator_options = dict() required = ["solver"] self._options_check(required, self.options) self.solver = pyo.SolverFactory(self.options["solver"]) self.ef = create_EF( all_scenario_names, scenario_creator, creator_options=scenario_creator_options, EF_name=model_name, suppress_warnings=suppress_warnings, )
def post_everything(self, PHIter, conv): if (self.fixer_object): self.fixer_object.post_everything(PHIter, conv) self.reader_object.post_everything(PHIter, conv) self.writer_object.post_everything(PHIter, conv) # Custom fixer post-processing if (self.rank != 0): return if (not self.fixer_object): return print('Fixed', self.fixer_object.fixed_so_far, 'unique variables') # Solve the resulting EF with all variables fixed scenario_names = self.ph.all_scenario_names scenario_creator = self.ph.scenario_creator ef = create_EF(scenario_names, scenario_creator, creator_options=None) # Pick out any random scenario sname = list(self.ph.local_scenarios.keys())[0] scenario = self.ph.local_scenarios[sname] nonant_vars = { var.name: var for node in scenario._PySPnode_list for var in node.nonant_vardata_list } # Fix all the nonant vars in the EF for (pair, var) in ef.ref_vars.items(): vname = extract_varname(var.name) if nonant_vars[vname].is_fixed(): var.fix(nonant_vars[vname].value) solver = pyo.SolverFactory('gurobi') solver.solve(ef, tee=True)
def create_extensive_form(self): self.creator_options = { "cb_data": self.cb_data, "md_dict": self.md_dict, "first_time": 0, "relaxation": self.relaxation } self.scenario_names = [ "Scenario_" + str(i) for i in range( 1, len(self.cb_data["etree"].rootnode.ScenarioList) + 1) ] self.ef = sputils.create_EF(self.scenario_names, util.pysp2_callback, self.creator_options) return self.ef
lines = list() for this_branch in md_dict.elements("branch"): lines.append(this_branch[0]) acstream = np.random.RandomState() cb_data["etree"] = ET.ACTree(number_of_stages, branching_factors, seed, acstream, a_line_fails_prob, stage_duration_minutes, repair_fct, lines) cb_data["epath"] = egret_path_to_data cb_data["acstream"] = acstream creator_options = {"cb_data": cb_data} scenario_names=["Scenario_"+str(i)\ for i in range(1,len(cb_data["etree"].rootnode.ScenarioList)+1)] # end options ef = sputils.create_EF(scenario_names, pysp2_callback, creator_options) ###solver.options["BarHomogeneous"] = 1 if "gurobi" in solvername: solver.options["BarHomogeneous"] = 1 results = solver.solve(ef, tee=True) pyo.assert_optimal_termination(results) print('EF objective value:', pyo.value(ef.EF_Obj)) sputils.ef_nonants_csv(ef, "vardump.csv") for (sname, smodel) in sputils.ef_scenarios(ef): print(sname) for stage in smodel.stage_models: print(" Stage {}".format(stage)) for gen in smodel.stage_models[stage].pg: print (" gen={} pg={}, qg={}"\
BFs = options["branching_factors"] ScenCount = BFs[0] * BFs[1] all_scenario_names = list() for sn in range(ScenCount): all_scenario_names.append("Scen" + str(sn + 1)) # end hardwire # This is multi-stage, so we need to supply node names all_nodenames = sputils.create_nodenames_from_BFs(BFs) # **** ef **** solver = pyo.SolverFactory(options["solvername"]) ef = sputils.create_EF( all_scenario_names, scenario_creator, scenario_creator_kwargs={"branching_factors": BFs}, ) results = solver.solve(ef, tee=options["verbose"]) print('EF objective value:', pyo.value(ef.EF_Obj)) sputils.ef_nonants_csv(ef, "vardump.csv") # **** ph **** options["xhat_specific_options"] = {"xhat_solver_options": options["iterk_solver_options"], "xhat_scenario_dict": \ {"ROOT": "Scen1", "ROOT_0": "Scen1", "ROOT_1": "Scen4", "ROOT_2": "Scen7"}, "csvname": "specific.csv"}
def run(self): """ Top-level execution.""" if self.is_EF: ef = sputils.create_EF( self.scenario_names, self.scenario_creator, scenario_creator_kwargs=self.kwargs, suppress_warnings=True, ) solvername = self.solvername solver = pyo.SolverFactory(solvername) if hasattr(self, "solver_options") and (self.solver_options is not None): for option_key, option_value in self.solver_options.items(): if option_value is not None: solver.options[option_key] = option_value if self.verbose: global_toc("Starting EF solve") if 'persistent' in solvername: solver.set_instance(ef, symbolic_solver_labels=True) results = solver.solve(tee=False) else: results = solver.solve( ef, tee=False, symbolic_solver_labels=True, ) if self.verbose: global_toc("Completed EF solve") self.EF_Obj = pyo.value(ef.EF_Obj) objs = sputils.get_objs(ef) self.is_minimizing = objs[0].is_minimizing #TBD : Write a function doing this if self.is_minimizing: self.best_outer_bound = results.Problem[0]['Lower bound'] self.best_inner_bound = results.Problem[0]['Upper bound'] else: self.best_inner_bound = results.Problem[0]['Upper bound'] self.best_outer_bound = results.Problem[0]['Lower bound'] self.ef = ef if 'write_solution' in self.options: if 'first_stage_solution' in self.options['write_solution']: sputils.write_ef_first_stage_solution( self.ef, self.options['write_solution']['first_stage_solution']) if 'tree_solution' in self.options['write_solution']: sputils.write_ef_tree_solution( self.ef, self.options['write_solution']['tree_solution']) self.xhats = sputils.nonant_cache_from_ef(ef) self.local_xhats = self.xhats #Every scenario is local for EF self.first_stage_solution = {"ROOT": self.xhats["ROOT"]} else: self.ef = None args = argparse.Namespace(**self.options) #Create a hub dict hub_name = find_hub(self.options['cylinders'], self.is_multi) hub_creator = getattr(vanilla, hub_name + '_hub') beans = { "args": args, "scenario_creator": self.scenario_creator, "scenario_denouement": self.scenario_denouement, "all_scenario_names": self.scenario_names, "scenario_creator_kwargs": self.kwargs } if self.is_multi: beans["all_nodenames"] = self.options["all_nodenames"] hub_dict = hub_creator(**beans) #Add extensions if 'extensions' in self.options: for extension in self.options['extensions']: extension_creator = getattr(vanilla, 'add_' + extension) hub_dict = extension_creator(hub_dict, args) #Create spoke dicts potential_spokes = find_spokes(self.options['cylinders'], self.is_multi) #We only use the spokes with an associated command line arg set to True spokes = [ spoke for spoke in potential_spokes if self.options['with_' + spoke] ] list_of_spoke_dict = list() for spoke in spokes: spoke_creator = getattr(vanilla, spoke + '_spoke') spoke_beans = copy.deepcopy(beans) if spoke == "xhatspecific": spoke_beans["scenario_dict"] = self.options[ "scenario_dict"] spoke_dict = spoke_creator(**spoke_beans) list_of_spoke_dict.append(spoke_dict) spcomm, opt_dict = sputils.spin_the_wheel(hub_dict, list_of_spoke_dict) self.opt = spcomm.opt self.cylinder_rank = self.opt.cylinder_rank self.on_hub = ("hub_class" in opt_dict) if self.on_hub: # we are on a hub rank self.best_inner_bound = spcomm.BestInnerBound self.best_outer_bound = spcomm.BestOuterBound #NOTE: We do not get bounds on every rank, only on hub # This should change if we want to use cylinders for MMW if 'write_solution' in self.options: if 'first_stage_solution' in self.options['write_solution']: sputils.write_spin_the_wheel_first_stage_solution( spcomm, opt_dict, self.options['write_solution']['first_stage_solution']) if 'tree_solution' in self.options['write_solution']: sputils.write_spin_the_wheel_tree_solution( spcomm, opt_dict, self.options['write_solution']['tree_solution']) if self.on_hub: #we are on a hub rank a_sname = self.opt.local_scenario_names[0] root = self.opt.local_scenarios[a_sname]._mpisppy_node_list[0] self.first_stage_solution = { "ROOT": [pyo.value(var) for var in root.nonant_vardata_list] } self.local_xhats = sputils.local_nonant_cache(spcomm)
def _create_master_with_scenarios(self): ef_scenarios = self.master_scenarios ## we want the correct probabilities to be set when ## calling create_EF if len(ef_scenarios) > 1: def scenario_creator_wrapper(name, **creator_options): scenario = self.scenario_creator(name, **creator_options) if not hasattr(scenario, 'PySP_prob'): scenario.PySP_prob = 1. / len(self.all_scenario_names) return scenario master = sputils.create_EF(ef_scenarios, scenario_creator_wrapper, creator_options={ 'node_names': None, 'cb_data': self.cb_data }) nonant_list, nonant_ids = _get_nonant_ids_EF(master) else: master = self.scenario_creator(ef_scenarios[0], node_names=None, cb_data=self.cb_data) if not hasattr(master, 'PySP_prob'): master.PySP_prob = 1. / len(self.all_scenario_names) nonant_list, nonant_ids = _get_nonant_ids(master) self.master_vars = nonant_list # creates the eta variables for scenarios that are NOT selected to be # included in the master problem eta_indx = [ scenario_name for scenario_name in self.all_scenario_names if scenario_name not in self.master_scenarios ] self._add_master_etas(master, eta_indx) obj = find_active_objective(master) repn = generate_standard_repn(obj.expr, quadratic=True) if len(repn.nonlinear_vars) > 0: raise ValueError( "LShaped does not support models with nonlinear objective functions" ) linear_vars = list(repn.linear_vars) linear_coefs = list(repn.linear_coefs) quadratic_coefs = list(repn.quadratic_coefs) # adjust coefficients by scenario/bundle probability scen_prob = master.PySP_prob for i, var in enumerate(repn.linear_vars): if id(var) not in nonant_ids: linear_coefs[i] *= scen_prob for i, (x, y) in enumerate(repn.quadratic_vars): # only multiply through once if id(x) not in nonant_ids: quadratic_coefs[i] *= scen_prob elif id(y) not in nonant_ids: quadratic_coefs[i] *= scen_prob # NOTE: the LShaped code negates the objective, so # we do the same here for consistency if not self.is_minimizing: for i, coef in enumerate(linear_coefs): linear_coefs[i] = -coef for i, coef in enumerate(quadratic_coefs): quadratic_coefs[i] = -coef # add the etas for var in master.eta.values(): linear_vars.append(var) linear_coefs.append(1) expr = LinearExpression(constant=repn.constant, linear_coefs=linear_coefs, linear_vars=linear_vars) if repn.quadratic_vars: expr += pyo.quicksum( (coef * x * y for coef, (x, y) in zip(quadratic_coefs, repn.quadratic_vars))) master.del_component(obj) # set master objective function master.obj = pyo.Objective(expr=expr, sense=pyo.minimize) self.master = master
all_scenario_names.append("Scen"+str(sn+1)) # end hardwire # This is multi-stage, so we need to supply node names all_nodenames = ["ROOT"] # all trees must have this node # The rest is a naming convention invented for this problem. # Note that mpisppy does not have nodes at the leaves, # and node names must end in a serial number. for b in range(BFs[0]): all_nodenames.append("ROOT_"+str(b)) # **** ef **** solver = pyo.SolverFactory(PHoptions["solvername"]) ef = sputils.create_EF(all_scenario_names, scenario_creator, creator_options={"cb_data": BFs}) results = solver.solve(ef, tee=PHoptions["verbose"]) print('EF objective value:', pyo.value(ef.EF_Obj)) sputils.ef_nonants_csv(ef, "vardump.csv") # **** ph **** PHoptions["xhat_specific_options"] = {"xhat_solver_options": PHoptions["iterk_solver_options"], "xhat_scenario_dict": \ {"ROOT": "Scen1", "ROOT_0": "Scen1", "ROOT_1": "Scen4", "ROOT_2": "Scen7"}, "csvname": "specific.csv"}
'verbose': False, 'display_progress': True, 'display_timing': False, 'iter0_solver_options': None, 'iterk_solver_options': None } ph = PH( options=phoptions, all_scenario_names=farmer.all_scenario_names, scenario_creator=farmer.scenario_creator, scenario_denouement=farmer.scenario_denouement, ) ph.ph_main() ef = sputils.create_EF(farmer.all_scenario_names, farmer.scenario_creator) solver = pyo.SolverFactory(solver_name) if 'persistent' in solver_name: solver.set_instance(ef, symbolic_solver_labels=True) solver.solve(tee=True) else: solver.solve( ef, tee=True, symbolic_solver_labels=True, ) print(f"EF objective: {pyo.value(ef.EF_Obj)}") farmer.close()
def _Q_opt(self, ThetaVals=None, solver="ef_ipopt", return_values=[], bootlist=None, calc_cov=False, cov_n=None): """ Set up all thetas as first stage Vars, return resulting theta values as well as the objective function value. """ if (solver == "k_aug"): raise RuntimeError("k_aug no longer supported.") # (Bootstrap scenarios will use indirection through the bootlist) if bootlist is None: senario_numbers = list(range(len(self.callback_data))) scen_names = ["Scenario{}".format(i) for i in senario_numbers] else: scen_names = ["Scenario{}".format(i) for i in range(len(bootlist))] # tree_model.CallbackModule = None outer_cb_data = dict() outer_cb_data["callback"] = self._instance_creation_callback if ThetaVals is not None: outer_cb_data["ThetaVals"] = ThetaVals if bootlist is not None: outer_cb_data["BootList"] = bootlist outer_cb_data["cb_data"] = self.callback_data # None is OK outer_cb_data["theta_names"] = self.theta_names options = {"solver": "ipopt"} scenario_creator_options = {"cb_data": outer_cb_data} if use_mpisppy: ef = sputils.create_EF( scen_names, _experiment_instance_creation_callback, EF_name="_Q_opt", suppress_warnings=True, scenario_creator_kwargs=scenario_creator_options) else: ef = local_ef.create_EF( scen_names, _experiment_instance_creation_callback, EF_name="_Q_opt", suppress_warnings=True, scenario_creator_kwargs=scenario_creator_options) self.ef_instance = ef # Solve the extensive form with ipopt if solver == "ef_ipopt": if not calc_cov: # Do not calculate the reduced hessian solver = SolverFactory('ipopt') if self.solver_options is not None: for key in self.solver_options: solver.options[key] = self.solver_options[key] solve_result = solver.solve(ef, tee=self.tee) # The import error will be raised when we attempt to use # inv_reduced_hessian_barrier below. # #elif not asl_available: # raise ImportError("parmest requires ASL to calculate the " # "covariance matrix with solver 'ipopt'") else: # parmest makes the fitted parameters stage 1 variables ind_vars = [] for ndname, Var, solval in ef_nonants(ef): ind_vars.append(Var) # calculate the reduced hessian solve_result, inv_red_hes = \ inverse_reduced_hessian.inv_reduced_hessian_barrier( self.ef_instance, independent_variables= ind_vars, solver_options=self.solver_options, tee=self.tee) if self.diagnostic_mode: print(' Solver termination condition = ', str(solve_result.solver.termination_condition)) # assume all first stage are thetas... thetavals = {} for ndname, Var, solval in ef_nonants(ef): # process the name # the scenarios are blocks, so strip the scenario name vname = Var.name[Var.name.find(".") + 1:] thetavals[vname] = solval objval = pyo.value(ef.EF_Obj) if calc_cov: # Calculate the covariance matrix # Number of data points considered n = cov_n # Extract number of fitted parameters l = len(thetavals) # Assumption: Objective value is sum of squared errors sse = objval '''Calculate covariance assuming experimental observation errors are independent and follow a Gaussian distribution with constant variance. The formula used in parmest was verified against equations (7-5-15) and (7-5-16) in "Nonlinear Parameter Estimation", Y. Bard, 1974. This formula is also applicable if the objective is scaled by a constant; the constant cancels out. (was scaled by 1/n because it computes an expected value.) ''' cov = 2 * sse / (n - l) * inv_red_hes cov = pd.DataFrame(cov, index=thetavals.keys(), columns=thetavals.keys()) thetavals = pd.Series(thetavals) if len(return_values) > 0: var_values = [] for exp_i in self.ef_instance.component_objects( Block, descend_into=False): vals = {} for var in return_values: exp_i_var = exp_i.find_component(str(var)) if exp_i_var is None: # we might have a block such as _mpisppy_data continue temp = [pyo.value(_) for _ in exp_i_var.values()] if len(temp) == 1: vals[var] = temp[0] else: vals[var] = temp if len(vals) > 0: var_values.append(vals) var_values = pd.DataFrame(var_values) if calc_cov: return objval, thetavals, var_values, cov else: return objval, thetavals, var_values if calc_cov: return objval, thetavals, cov else: return objval, thetavals else: raise RuntimeError("Unknown solver in Q_Opt=" + solver)
for this_branch in md_dict.elements("branch"): lines.append(this_branch[0]) acstream = np.random.RandomState() scenario_creator_kwargs["etree"] = ET.ACTree(number_of_stages, branching_factors, seed, acstream, a_line_fails_prob, stage_duration_minutes, repair_fct, lines) scenario_creator_kwargs["acstream"] = acstream scenario_names=["Scenario_"+str(i)\ for i in range(1,len(scenario_creator_kwargs["etree"].rootnode.ScenarioList)+1)] # end options ef = sputils.create_EF(scenario_names, pysp2_callback, scenario_creator_kwargs=scenario_creator_kwargs) ###solver.options["BarHomogeneous"] = 1 if "gurobi" in solvername: solver.options["BarHomogeneous"] = 1 results = solver.solve(ef, tee=True) pyo.assert_optimal_termination(results) print('EF objective value:', pyo.value(ef.EF_Obj)) sputils.ef_nonants_csv(ef, "vardump.csv") for (sname, smodel) in sputils.ef_scenarios(ef): print(sname) for stage in smodel.stage_models: print(" Stage {}".format(stage)) for gen in smodel.stage_models[stage].pg:
solvername = ama_object.options['EF_solver_name'] scenario_creator_kwargs = { "use_integer": False, "crops_multiplier": crops_multiplier, "num_scens": ScenCount } # different scenarios for xhat scenario_names = [ 'Scenario' + str(i) for i in range(ScenCount, 2 * ScenCount) ] ef = sputils.create_EF( scenario_names, scenario_creator, scenario_creator_kwargs=scenario_creator_kwargs, ) solver = pyo.SolverFactory(solvername) if 'persistent' in solvername: solver.set_instance(ef, symbolic_solver_labels=True) solver.solve(tee=False) else: solver.solve( ef, tee=False, symbolic_solver_labels=True, ) print(f"Xhat in-sample objective: {pyo.value(ef.EF_Obj)}")