def xhat_generator_farmer(scenario_names, solvername="gurobi", solver_options=None, use_integer=False, crops_multiplier=1, start_seed=None): ''' For sequential sampling. Takes scenario names as input and provide the best solution for the approximate problem associated with the scenarios. Parameters ---------- scenario_names: list of str Names of the scenario we use solvername: str, optional Name of the solver used. The default is "gurobi" solver_options: dict, optional Solving options. The default is None. use_integer: boolean indicates the integer farmer version crops_multiplier: int mulitplied by three to get the total number of crops start_seed: int, optional The starting seed, used to create different sample scenario trees. The default is 0. Returns ------- xhat: str A generated xhat, solution to the approximate problem induced by scenario_names. NOTE: This tool only works when the file is in mpisppy. In SPInstances, you must change the from_module line. ''' num_scens = len(scenario_names) ama_options = { "EF-2stage": True, "EF_solver_name": solvername, "EF_solver_options": solver_options, "num_scens": num_scens, "_mpisppy_probability": 1 / num_scens, "start_seed": start_seed, } #We use from_module to build easily an Amalgamator object ama = amalgamator.from_module("afarmer", ama_options, use_command_line=False) #Correcting the building by putting the right scenarios. ama.scenario_names = scenario_names ama.verbose = False ama.run() # get the xhat xhat = sputils.nonant_cache_from_ef(ama.ef) return {'ROOT': xhat['ROOT']}
def xhat_generator_farmer(scenario_names, solvername="gurobi", solver_options=None, crops_multiplier=1): ''' For developer testing: Given scenario names and options, create the scenarios and compute the xhat that is minimizing the approximate problem associated with these scenarios. Parameters ---------- scenario_names: int Names of the scenario we use solvername: str, optional Name of the solver used. The default is "gurobi". solver_options: dict, optional Solving options. The default is None. crops_multiplier: int, optional A parameter of the farmer model. The default is 1. Returns ------- xhat: xhat object (dict containing a 'ROOT' key with a np.array) A generated xhat. NOTE: this is here for testing during development. ''' num_scens = len(scenario_names) ama_options = { "EF-2stage": True, "EF_solver_name": solvername, "EF_solver_options": solver_options, "use_integer": False, "crops_multiplier": crops_multiplier, "num_scens": num_scens, "_mpisppy_probability": 1 / num_scens, } #We use from_module to build easily an Amalgamator object ama = amalgamator.from_module("mpisppy.tests.examples.farmer", ama_options, use_command_line=False) #Correcting the building by putting the right scenarios. ama.scenario_names = scenario_names ama.run() # get the xhat xhat = sputils.nonant_cache_from_ef(ama.ef) return xhat
def xhat_generator_apl1p(scenario_names, solvername="gurobi", solver_options=None): ''' For sequential sampling. Takes scenario names as input and provide the best solution for the approximate problem associated with the scenarios. Parameters ---------- scenario_names: int Names of the scenario we use solvername: str, optional Name of the solver used. The default is "gurobi" solver_options: dict, optional Solving options. The default is None. Returns ------- xhat: str A generated xhat, solution to the approximate problem induced by scenario_names. ''' num_scens = len(scenario_names) ama_options = { "EF-2stage": True, "EF_solver_name": solvername, "EF_solver_options": solver_options, "num_scens": num_scens, "_mpisppy_probability": 1 / num_scens, } #We use from_module to build easily an Amalgomator object ama = amalgomator.from_module("mpisppy.tests.examples.apl1p", ama_options, use_command_line=False) #Correcting the building by putting the right scenarios. ama.scenario_names = scenario_names ama.run() # get the xhat xhat = sputils.nonant_cache_from_ef(ama.ef) return xhat
def run(self): """ Top-level execution.""" if self.is_EF: ef = sputils.create_EF( self.scenario_names, self.scenario_creator, scenario_creator_kwargs=self.kwargs, suppress_warnings=True, ) solvername = self.solvername solver = pyo.SolverFactory(solvername) if hasattr(self, "solver_options") and (self.solver_options is not None): for option_key, option_value in self.solver_options.items(): if option_value is not None: solver.options[option_key] = option_value if self.verbose: global_toc("Starting EF solve") if 'persistent' in solvername: solver.set_instance(ef, symbolic_solver_labels=True) results = solver.solve(tee=False) else: results = solver.solve( ef, tee=False, symbolic_solver_labels=True, ) if self.verbose: global_toc("Completed EF solve") self.EF_Obj = pyo.value(ef.EF_Obj) objs = sputils.get_objs(ef) self.is_minimizing = objs[0].is_minimizing #TBD : Write a function doing this if self.is_minimizing: self.best_outer_bound = results.Problem[0]['Lower bound'] self.best_inner_bound = results.Problem[0]['Upper bound'] else: self.best_inner_bound = results.Problem[0]['Upper bound'] self.best_outer_bound = results.Problem[0]['Lower bound'] self.ef = ef if 'write_solution' in self.options: if 'first_stage_solution' in self.options['write_solution']: sputils.write_ef_first_stage_solution( self.ef, self.options['write_solution']['first_stage_solution']) if 'tree_solution' in self.options['write_solution']: sputils.write_ef_tree_solution( self.ef, self.options['write_solution']['tree_solution']) self.xhats = sputils.nonant_cache_from_ef(ef) self.local_xhats = self.xhats #Every scenario is local for EF self.first_stage_solution = {"ROOT": self.xhats["ROOT"]} else: self.ef = None args = argparse.Namespace(**self.options) #Create a hub dict hub_name = find_hub(self.options['cylinders'], self.is_multi) hub_creator = getattr(vanilla, hub_name + '_hub') beans = { "args": args, "scenario_creator": self.scenario_creator, "scenario_denouement": self.scenario_denouement, "all_scenario_names": self.scenario_names, "scenario_creator_kwargs": self.kwargs } if self.is_multi: beans["all_nodenames"] = self.options["all_nodenames"] hub_dict = hub_creator(**beans) #Add extensions if 'extensions' in self.options: for extension in self.options['extensions']: extension_creator = getattr(vanilla, 'add_' + extension) hub_dict = extension_creator(hub_dict, args) #Create spoke dicts potential_spokes = find_spokes(self.options['cylinders'], self.is_multi) #We only use the spokes with an associated command line arg set to True spokes = [ spoke for spoke in potential_spokes if self.options['with_' + spoke] ] list_of_spoke_dict = list() for spoke in spokes: spoke_creator = getattr(vanilla, spoke + '_spoke') spoke_beans = copy.deepcopy(beans) if spoke == "xhatspecific": spoke_beans["scenario_dict"] = self.options[ "scenario_dict"] spoke_dict = spoke_creator(**spoke_beans) list_of_spoke_dict.append(spoke_dict) spcomm, opt_dict = sputils.spin_the_wheel(hub_dict, list_of_spoke_dict) self.opt = spcomm.opt self.cylinder_rank = self.opt.cylinder_rank self.on_hub = ("hub_class" in opt_dict) if self.on_hub: # we are on a hub rank self.best_inner_bound = spcomm.BestInnerBound self.best_outer_bound = spcomm.BestOuterBound #NOTE: We do not get bounds on every rank, only on hub # This should change if we want to use cylinders for MMW if 'write_solution' in self.options: if 'first_stage_solution' in self.options['write_solution']: sputils.write_spin_the_wheel_first_stage_solution( spcomm, opt_dict, self.options['write_solution']['first_stage_solution']) if 'tree_solution' in self.options['write_solution']: sputils.write_spin_the_wheel_tree_solution( spcomm, opt_dict, self.options['write_solution']['tree_solution']) if self.on_hub: #we are on a hub rank a_sname = self.opt.local_scenario_names[0] root = self.opt.local_scenarios[a_sname]._mpisppy_node_list[0] self.first_stage_solution = { "ROOT": [pyo.value(var) for var in root.nonant_vardata_list] } self.local_xhats = sputils.local_nonant_cache(spcomm)
def xhat_generator_aircond(scenario_names, solvername="gurobi", solver_options=None, branching_factors=None, mudev=0, sigmadev=40, start_ups=None, start_seed=0): ''' For sequential sampling. Takes scenario names as input and provide the best solution for the approximate problem associated with the scenarios. Parameters ---------- scenario_names: list of str Names of the scenario we use solvername: str, optional Name of the solver used. The default is "gurobi" solver_options: dict, optional Solving options. The default is None. branching_factors: list, optional Branching factors of the scenario 3. The default is [3,2,3] (a 4 stage model with 18 different scenarios) mudev: float, optional The average deviation of demand between two stages; The default is 0. sigma_dev: float, optional The standard deviation from mudev for the demand difference between two stages. The default is 40. start_seed: int, optional The starting seed, used to create different sample scenario trees. The default is 0. Returns ------- xhat: str A generated xhat, solution to the approximate problem induced by scenario_names. NOTE: This tool only works when the file is in mpisppy. In SPInstances, you must change the from_module line. ''' num_scens = len(scenario_names) ama_options = { "EF-mstage": True, "EF_solver_name": solvername, "EF_solver_options": solver_options, "num_scens": num_scens, "_mpisppy_probability": 1 / num_scens, "branching_factors": branching_factors, "mudev": mudev, "start_ups": start_ups, "start_seed": start_seed, "sigmadev": sigmadev } #We use from_module to build easily an Amalgamator object ama = amalgamator.from_module("mpisppy.tests.examples.aircond", ama_options, use_command_line=False) #Correcting the building by putting the right scenarios. ama.scenario_names = scenario_names ama.verbose = False ama.run() # get the xhat xhat = sputils.nonant_cache_from_ef(ama.ef) return xhat
"num_scens": num_scens, "branching_factors": bfs, "mudev": 0, "sigmadev": 40, "start_ups": start_ups, "start_seed": 0 } refmodel = "mpisppy.tests.examples.aircond" # WARNING: Change this in SPInstances #We use from_module to build easily an Amalgamator object t0 = time.time() ama = amalgamator.from_module(refmodel, ama_options, use_command_line=False) ama.run() print('start ups costs: ', start_ups) print('branching factors: ', bfs) print('run time: ', time.time() - t0) print(f"inner bound =", ama.best_inner_bound) print(f"outer bound =", ama.best_outer_bound) xhat = sputils.nonant_cache_from_ef(ama.ef) print('xhat_one = ', xhat['ROOT']) if save_xhat: bf_string = '' for bf in bfs: bf_string = bf_string + bf + '_' np.savetxt( 'aircond_start_ups=' + str(start_ups) + bf_string + 'zhatstar.txt', xhat['ROOT'])
dest="batch_size", type=int, default=None) #None means take batch_size=num_scens ama_object = ama.from_module(refmodel, ama_options, extraargs=ama_extraargs) ama_object.run() if global_rank == 0: print("inner bound=", ama_object.best_inner_bound) # This the xhat of the left term of LHS of MMW (9) print("outer bound=", ama_object.best_outer_bound) ########### get the nonants (the xhat) nonant_cache = sputils.nonant_cache_from_ef(ama_object.ef) ciutils.write_xhat(nonant_cache, path="xhat.npy") #Set parameters for run() options = ama_object.options options['solver_options'] = options['EF_solver_options'] xhat = ciutils.read_xhat("xhat.npy") num_batches = ama_object.options['num_batches'] batch_size = ama_object.options['batch_size'] mmw = MMWConfidenceIntervals(refmodel, options, xhat, num_batches,
def gap_estimators(xhat_one, mname, solving_type="EF-2stage", scenario_names=None, sample_options=None, ArRP=1, scenario_creator_kwargs={}, scenario_denouement=None, solvername=None, solver_options=None, verbose=False, objective_gap=False): ''' Given a xhat, scenario names, a scenario creator and options, gap_estimators creates a scenario tree and the associatd estimators G and s from §2 of [bm2011]. Returns G and s evaluated at xhat. If ArRP>1, G and s are pooled, from a number ArRP of estimators, computed with different scenario trees. Parameters ---------- xhat_one : dict A candidate first stage solution mname: str Name of the reference model, e.g. 'mpisppy.tests.examples.farmer'. solving_type: str, optional The way we solve the approximate problem. Can be "EF-2stage" (default) or "EF-mstage". scenario_names: list, optional List of scenario names used to compute G_n and s_n. Default is None Must be specified for 2 stage, but can be missing for multistage sample_options: dict, optional Only for multistage. Must contain a 'seed' and a 'branching_factors' attribute, specifying the starting seed and the branching factors of the scenario tree ArRP:int,optional Number of batches (we create a ArRP model). Default is 1 (one batch). scenario_creator_kwargs: dict, optional Additional arguments for scenario_creator. Default is {} scenario_denouement: function, optional Function to run after scenario creation. Default is None. solvername : str, optional Solver. Default is None solver_options: dict, optional Solving options. Default is None verbose: bool, optional Should it print the gap estimator ? Default is True objective_gap: bool, optional Returns a gap estimate around approximate objective value branching_factors: list, optional Only for multistage. List of branching factors of the sample scenario tree. Returns ------- G_k and s_k, gap estimator and associated standard deviation estimator. ''' global_toc("Enter gap_estimators") if solving_type not in ["EF-2stage", "EF-mstage"]: raise RuntimeError( "Only EF solve for the approximate problem is supported yet.") else: is_multi = (solving_type == "EF-mstage") if is_multi: try: branching_factors = sample_options['branching_factors'] start = sample_options['seed'] except (TypeError, KeyError, RuntimeError): raise RuntimeError( 'For multistage problems, sample_options must be a dict with branching_factors and seed attributes.' ) else: start = sputils.extract_num(scenario_names[0]) if ArRP > 1: #Special case : ArRP, G and s are pooled from r>1 estimators. if is_multi: raise RuntimeError( "Pooled estimators are not supported for multistage problems yet." ) n = len(scenario_names) if (n % ArRP != 0): raise RuntimeWarning("You put as an input a number of scenarios"+\ f" which is not a mutliple of {ArRP}.") n = n - n % ArRP G = [] s = [] for k in range(ArRP): scennames = scenario_names[k * (n // ArRP):(k + 1) * (n // ArRP)] tmp = gap_estimators( xhat_one, mname, solvername=solvername, scenario_names=scennames, ArRP=1, scenario_creator_kwargs=scenario_creator_kwargs, scenario_denouement=scenario_denouement, solver_options=solver_options, solving_type=solving_type) G.append(tmp['G']) s.append(tmp['s']) global_toc(f"ArRP {k} of {ArRP}") #Pooling G = np.mean(G) s = np.linalg.norm(s) / np.sqrt(n // ArRP) return {"G": G, "s": s, "seed": start} #A1RP #We start by computing the optimal solution to the approximate problem induced by our scenarios if is_multi: #Sample a scenario tree: this is a subtree, but starting from stage 1 samp_tree = sample_tree.SampleSubtree( mname, xhats=[], root_scen=None, starting_stage=1, branching_factors=branching_factors, seed=start, options=scenario_creator_kwargs, solvername=solvername, solver_options=solver_options) samp_tree.run() start += sputils.number_of_nodes(branching_factors) ama_object = samp_tree.ama else: #We use amalgamator to do it ama_options = dict(scenario_creator_kwargs) ama_options['start'] = start ama_options['num_scens'] = len(scenario_names) ama_options['EF_solver_name'] = solvername ama_options['EF_solver_options'] = solver_options ama_options[solving_type] = True ama_object = ama.from_module(mname, ama_options, use_command_line=False) ama_object.scenario_names = scenario_names ama_object.verbose = False ama_object.run() start += len(scenario_names) #Optimal solution of the approximate problem zstar = ama_object.best_outer_bound #Associated policies xstars = sputils.nonant_cache_from_ef(ama_object.ef) #Then, we evaluate the fonction value induced by the scenario at xstar. if is_multi: # Find feasible policies (i.e. xhats) for every non-leaf nodes if len(samp_tree.ef._ef_scenario_names) > 1: local_scenarios = { sname: getattr(samp_tree.ef, sname) for sname in samp_tree.ef._ef_scenario_names } else: local_scenarios = { samp_tree.ef._ef_scenario_names[0]: samp_tree.ef } xhats, start = sample_tree.walking_tree_xhats( mname, local_scenarios, xhat_one['ROOT'], branching_factors, start, scenario_creator_kwargs, solvername=solvername, solver_options=solver_options) #Compute then the average function value with this policy scenario_creator_kwargs = samp_tree.ama.kwargs all_nodenames = sputils.create_nodenames_from_branching_factors( branching_factors) else: #In a 2 stage problem, the only non-leaf is the ROOT node xhats = xhat_one all_nodenames = None xhat_eval_options = { "iter0_solver_options": None, "iterk_solver_options": None, "display_timing": False, "solvername": solvername, "verbose": False, "solver_options": solver_options } ev = xhat_eval.Xhat_Eval(xhat_eval_options, scenario_names, ama_object.scenario_creator, scenario_denouement, scenario_creator_kwargs=scenario_creator_kwargs, all_nodenames=all_nodenames) #Evaluating xhat and xstar and getting the value of the objective function #for every (local) scenario zhat = ev.evaluate(xhats) objs_at_xhat = ev.objs_dict zstar = ev.evaluate(xstars) objs_at_xstar = ev.objs_dict eval_scen_at_xhat = [] eval_scen_at_xstar = [] scen_probs = [] for k, s in ev.local_scenarios.items(): eval_scen_at_xhat.append(objs_at_xhat[k]) eval_scen_at_xstar.append(objs_at_xstar[k]) scen_probs.append(s._mpisppy_probability) scen_gaps = np.array(eval_scen_at_xhat) - np.array(eval_scen_at_xstar) local_gap = np.dot(scen_gaps, scen_probs) local_ssq = np.dot(scen_gaps**2, scen_probs) local_prob_sqnorm = np.linalg.norm(scen_probs)**2 local_obj_at_xhat = np.dot(eval_scen_at_xhat, scen_probs) local_estim = np.array( [local_gap, local_ssq, local_prob_sqnorm, local_obj_at_xhat]) global_estim = np.zeros(4) ev.mpicomm.Allreduce(local_estim, global_estim, op=mpi.SUM) G, ssq, prob_sqnorm, obj_at_xhat = global_estim if global_rank == 0 and verbose: print(f"G = {G}") sample_var = (ssq - G**2) / (1 - prob_sqnorm) #Unbiased sample variance s = np.sqrt(sample_var) use_relative_error = (np.abs(zstar) > 1) G = correcting_numeric(G, objfct=obj_at_xhat, relative_error=use_relative_error) if objective_gap: if is_multi: return { "G": G, "s": s, "zhats": [zhat], "zstars": [zstar], "seed": start } else: return { "G": G, "s": s, "zhats": eval_scen_at_xhat, "zstars": eval_scen_at_xstar, "seed": start } else: return {"G": G, "s": s, "seed": start}
BFs = [3, 2, 4, 4] num_scens = np.prod(BFs) mname = "mpisppy.tests.examples.aircond_submodels" ama_options = { "EF-mstage": True, "num_scens": num_scens, "_mpisppy_probability": 1 / num_scens, "BFs": BFs, } #We use from_module to build easily an Amalgomator object ama = amalgomator.from_module(mname, ama_options, use_command_line=False) ama.run() # get the xhat xhat_one = sputils.nonant_cache_from_ef(ama.ef)['ROOT'] #----------Find a feasible solution for a single scenario------------- scenario = ama.ef.scen0 seed = sputils.number_of_nodes(BFs) options = dict() #We take default aircond options xhats, seed = feasible_solution(mname, scenario, xhat_one, BFs, seed, options) print(xhats) #----------Find feasible solutions for every scenario ------------ #Fetching scenarios from EF scenarios = dict() for k in ama.ef._ef_scenario_names:
def _gap_estimators_with_independent_scenarios(self, xhat_k, nk, estimator_scenario_names, scenario_denouement): """ Sample a scenario tree: this is a subtree, but starting from stage 1. Args: xhat_k (dict[nodename] of list): the solution to lead the walk nk (int): number of scenarios, estimator_scenario_names(list of str): scenario names scenario_denouement (fct): called for each scenario at the end (TBD: drop this arg and just use the function in refmodel) Returns: Gk, Sk (float): mean and standard devation of the gap estimate Note: Seed management is mainly in the form of updates to SeedCount """ ama_options = self.options.copy() ama_options['EF-mstage'] = True ama_options['EF_solver_name'] = self.solvername if self.solver_options is not None: ama_options['EF_solver_options'] = self.solver_options ama_options['num_scens'] = nk ama_options['_mpisppy_probability'] = 1 / nk #Probably not used ama_options['start_seed'] = self.SeedCount pseudo_branching_factors = [nk] + [1] * (self.numstages - 2) ama_options['branching_factors'] = pseudo_branching_factors ama = amalgamator.Amalgamator( options=ama_options, scenario_names=estimator_scenario_names, scenario_creator=self.refmodel.scenario_creator, kw_creator=self.refmodel.kw_creator, scenario_denouement=scenario_denouement) ama.run() #Optimal solution of the approximate problem zstar = ama.best_outer_bound #Associated policies xstars = sputils.nonant_cache_from_ef(ama.ef) scenario_creator_kwargs = ama.kwargs # Find feasible policies (i.e. xhats) for every non-leaf nodes local_scenarios = { sname: getattr(ama.ef, sname) for sname in ama.ef._ef_scenario_names } xhats, start = sample_tree.walking_tree_xhats( self.refmodelname, local_scenarios, xhat_k['ROOT'], self.options['branching_factors'], self.SeedCount, self.options, # not scenario_creator_kwargs, solvername=self.solvername, solver_options=self.solver_options) #Compute then the average function value with this policy all_nodenames = sputils.create_nodenames_from_branching_factors( pseudo_branching_factors) xhat_eval_options = { "iter0_solver_options": None, "iterk_solver_options": None, "display_timing": False, "solvername": self.solvername, "verbose": False, "solver_options": self.solver_options } ev = xhat_eval.Xhat_Eval( xhat_eval_options, estimator_scenario_names, self.refmodel.scenario_creator, scenario_denouement, scenario_creator_kwargs=scenario_creator_kwargs, all_nodenames=all_nodenames) #Evaluating xhat and xstar and getting the value of the objective function #for every (local) scenario ev.evaluate(xhats) objs_at_xhat = ev.objs_dict ev.evaluate(xstars) objs_at_xstar = ev.objs_dict eval_scen_at_xhat = [] eval_scen_at_xstar = [] scen_probs = [] for k, s in ev.local_scenarios.items(): eval_scen_at_xhat.append(objs_at_xhat[k]) eval_scen_at_xstar.append(objs_at_xstar[k]) scen_probs.append(s._mpisppy_probability) scen_gaps = np.array(eval_scen_at_xhat) - np.array(eval_scen_at_xstar) local_gap = np.dot(scen_gaps, scen_probs) local_ssq = np.dot(scen_gaps**2, scen_probs) local_prob_sqnorm = np.linalg.norm(scen_probs)**2 local_obj_at_xhat = np.dot(eval_scen_at_xhat, scen_probs) local_estim = np.array( [local_gap, local_ssq, local_prob_sqnorm, local_obj_at_xhat]) global_estim = np.zeros(4) ev.mpicomm.Allreduce(local_estim, global_estim, op=mpi.SUM) G, ssq, prob_sqnorm, obj_at_xhat = global_estim if global_rank == 0: print(f"G = {G}") sample_var = (ssq - G**2) / (1 - prob_sqnorm ) #Unbiased sample variance sk = np.sqrt(sample_var) use_relative_error = (np.abs(zstar) > 1) Gk = ciutils.correcting_numeric(G, objfct=obj_at_xhat, relative_error=use_relative_error) self.SeedCount = start return Gk, sk
solver = pyo.SolverFactory(solvername) if 'persistent' in solvername: solver.set_instance(ef, symbolic_solver_labels=True) solver.solve(tee=False) else: solver.solve( ef, tee=False, symbolic_solver_labels=True, ) print(f"Xhat in-sample objective: {pyo.value(ef.EF_Obj)}") ########### get the nonants (the xhat) # NOTE: we probably should do an assert or two to make sure Vars match nonant_cache = sputils.nonant_cache_from_ef(ef) # Create the eval object for the left term of the LHS of (9) in MMW # but we are back to using the first scenarios MMW_scenario_names = ['scen' + str(i) for i in range(ScenCount)] # The options need to be re-done (and phase needs to be split up) options = { "iter0_solver_options": None, "iterk_solver_options": None, "solvername": solvername, "verbose": False } # TBD: set solver options ev = Xhat_Eval( options,