def test_init7(self): self.assertTrue("reference_test_model" not in sys.modules) scenario_tree_model = CreateAbstractScenarioTreeModel().\ create_instance( join(testdatadir, "reference_test_scenario_tree.dat")) with self.assertRaises(ValueError): with ScenarioTreeInstanceFactory( model=reference_test_model, scenario_tree=scenario_tree_model) as factory: pass with ScenarioTreeInstanceFactory( model=reference_test_model, scenario_tree=scenario_tree_model, data=testdatadir) as factory: self.assertTrue(factory.model_directory() is None) self.assertTrue(factory.scenario_tree_directory() is None) self._check_factory(factory) self.assertEqual(factory._closed, True) with ScenarioTreeInstanceFactory( model=reference_test_model, scenario_tree=join(testdatadir, "reference_test_scenario_tree.dat")) as factory: self.assertTrue(factory.model_directory() is None) self.assertTrue(factory.scenario_tree_directory() is not None) self._check_factory(factory) self.assertEqual(len(factory._archives), 0)
def test_init20(self): self.assertTrue("reference_test_model" not in sys.modules) def scenario_model_callback(scenario_tree, scenario_name, node_list): self.assertTrue(isinstance(scenario_tree, ScenarioTree)) instance = reference_test_model.create_instance() if scenario_name == "s1": instance.p = 1.0 elif scenario_name == "s2": instance.p = 2.0 else: assert scenario_name == "s3" instance.p = 3.0 return instance with ScenarioTreeInstanceFactory( model=scenario_model_callback, scenario_tree=join(testdatadir, "reference_test_scenario_tree.dat")) as factory: my_scenario_tree = factory.generate_scenario_tree() with ScenarioTreeInstanceFactory( model=scenario_model_callback, scenario_tree=my_scenario_tree) as factory: self.assertTrue(factory.model_directory() is None) self.assertTrue(factory.scenario_tree_directory() is None) self._check_factory(factory) self.assertEqual(factory._closed, True) self.assertEqual(len(factory._archives), 0)
def test_init2_default(self): self.assertTrue("ReferenceModel" not in sys.modules) with ScenarioTreeInstanceFactory( model=join(testdatadir, "archive_test.tgz,"), scenario_tree=testdatadir) as factory: self.assertEqual(len(factory._archives), 1) self.assertTrue(factory.model_directory() is not None) self.assertTrue(factory.scenario_tree_directory() is not None) self._check_factory(factory) self.assertTrue("ReferenceModel" in sys.modules)
def test_init14(self): self.assertTrue("reference_test_model" not in sys.modules) scenario_tree_model = CreateAbstractScenarioTreeModel().\ create_instance( join(testdatadir, "reference_test_scenario_tree.dat")) with ScenarioTreeInstanceFactory( model=reference_test_model, scenario_tree=scenario_tree_model, data=testdatadir) as factory: scenario_tree = factory.generate_scenario_tree() self.assertEqual(factory._closed, True) self.assertEqual(len(factory._archives), 0) # start with a scenario tree (not a scenario tree model) with ScenarioTreeInstanceFactory( model=reference_test_model, scenario_tree=scenario_tree, data=testdatadir) as factory: self._check_factory(factory) self.assertEqual(factory._closed, True) self.assertEqual(len(factory._archives), 0)
def __init__(self, fsfile, fsfct = None, tree_model = None, phopts = None): """Initialize a StochSolver object. """ if fsfct is None: # Changed in October 2018: None implies AbstractModel args_list = _optiondict_2_list(phopts) parser = construct_ph_options_parser("") options = parser.parse_args(args_list) scenario_instance_factory = \ ScenarioTreeInstanceFactory(fsfile, tree_model) try: self.scenario_tree = \ GenerateScenarioTreeForPH(options, scenario_instance_factory) except: print ("ERROR in StochSolver called from",inspect.stack()[1][3]) raise RuntimeError("fsfct is None, so assuming", "AbstractModel but could not find all ingredients.") else: # concrete model if callable(fsfct): scen_function = fsfct else: # better be a string fsfile = fsfile.replace('.py','') # import does not like .py # __import__ only gives the top level module # probably need to be dealing with modules installed via setup.py m = __import__(fsfile) for n in fsfile.split(".")[1:]: m = getattr(m, n) scen_function = getattr(m, fsfct) if tree_model is None: treecbname = "pysp_scenario_tree_model_callback" tree_maker = getattr(m, treecbname) tree = tree_maker() scenario_instance_factory = ScenarioTreeInstanceFactory(scen_function, tree_model) else: # DLW March 21: still not correct scenario_instance_factory = \ ScenarioTreeInstanceFactory(scen_function, tree_model) kwargs = _kwfromphopts(phopts) self.scenario_tree = \ scenario_instance_factory.generate_scenario_tree(**kwargs) #verbose = True) instances = scenario_instance_factory. \ construct_instances_for_scenario_tree(self.scenario_tree) self.scenario_tree.linkInInstances(instances)
def test_init15(self): self.assertTrue("reference_test_model" not in sys.modules) with ScenarioTreeInstanceFactory( model=join(testdatadir, "reference_test_model.py"), scenario_tree=join(testdatadir, "ScenarioStructure.py")) as factory: self.assertEqual(len(factory._archives), 0) self.assertTrue(factory.model_directory() is not None) self.assertTrue(factory.scenario_tree_directory() is not None) self._check_factory(factory) self.assertTrue("reference_test_model" in sys.modules)
def test_init22(self): self.assertTrue("both_callbacks" not in sys.modules) with ScenarioTreeInstanceFactory( model=join(testdatadir, "both_callbacks.py"), scenario_tree=None) as factory: self.assertEqual(len(factory._archives), 0) self.assertTrue(factory.model_directory() is not None) self.assertTrue(factory.scenario_tree_directory() is not None) self._check_factory(factory) self.assertEqual(len(factory._archives), 0) self.assertTrue("both_callbacks" in sys.modules)
def test_init10(self): with ScenarioTreeInstanceFactory( model=testdatadir, scenario_tree=join(testdatadir, "reference_test_scenario_tree.dat"), data=join(testdatadir, "yaml_data")) as factory: self.assertEqual(len(factory._archives), 0) self.assertTrue(factory.model_directory() is not None) self.assertTrue(factory.scenario_tree_directory() is not None) self.assertTrue(factory.data_directory(), join(testdatadir, "yaml_data")) self._check_factory(factory) self.assertEqual(len(factory._archives), 0)
def test_init11(self): self.assertTrue("reference_test_model" not in sys.modules) scenario_tree_model = CreateAbstractScenarioTreeModel().\ create_instance( join(testdatadir, "reference_test_scenario_tree.dat")) scenario_tree_model.ScenarioBasedData = False with ScenarioTreeInstanceFactory( model=reference_test_model, scenario_tree=scenario_tree_model, data=testdatadir) as factory: self.assertTrue(factory.model_directory() is None) self.assertTrue(factory.scenario_tree_directory() is None) self._check_factory(factory) self.assertEqual(factory._closed, True) self.assertEqual(len(factory._archives), 0)
def test_init16(self): self.assertTrue("reference_test_model" not in sys.modules) self.assertTrue("ScenarioStructure" not in sys.modules) nx_tree = load_external_module(os.path.join(testdatadir, "ScenarioStructure.py"))[0].G with ScenarioTreeInstanceFactory( model=join(testdatadir, "reference_test_model.py"), scenario_tree=nx_tree) as factory: self.assertEqual(len(factory._archives), 0) self.assertTrue(factory.model_directory() is not None) self.assertTrue(factory.scenario_tree_directory() is None) self._check_factory(factory) self.assertTrue("reference_test_model" in sys.modules) self.assertTrue("ScenarioStructure" in sys.modules)
def test_init8(self): self.assertTrue("reference_test_model" not in sys.modules) self.assertTrue("reference_test_scenario_tree_model" not in sys.modules) with ScenarioTreeInstanceFactory( model=reference_test_model, scenario_tree=join(testdatadir, "reference_test_scenario_tree_model.py"), data=testdatadir) as factory: self.assertTrue(factory.model_directory() is None) self.assertTrue(factory.scenario_tree_directory() is not None) self.assertTrue(factory._scenario_tree_module is not None) self._check_factory(factory) self.assertEqual(factory._closed, True) self.assertEqual(len(factory._archives), 0) self.assertTrue("reference_test_model" not in sys.modules) self.assertTrue("reference_test_scenario_tree_model" in sys.modules)
def test_init4_default(self): self.assertTrue("ReferenceModel" not in sys.modules) archive_copy = self._get_testfname_prefix()+".archive_copy.tgz" shutil.copyfile(join(testdatadir, "archive_test.tgz"), archive_copy) with ScenarioTreeInstanceFactory( model=join(testdatadir, "archive_test.tgz,"), scenario_tree=join(testdatadir, archive_copy+",")) as factory: self.assertEqual(len(factory._archives), 2) self.assertTrue(factory.model_directory() is not None) self.assertTrue(factory.scenario_tree_directory() is not None) self._check_factory(factory) self.assertEqual(len(factory._archives), 0) os.remove(archive_copy) self.assertTrue("ReferenceModel" in sys.modules)
def generate_sample_sp(self, size, **kwds): assert size > 0 def model_callback(scenario_name, node_list): m = self.sample(return_copy=True) return m scenario_tree_model = self._create_scenario_tree_model(size) factory = ScenarioTreeInstanceFactory( model=model_callback, scenario_tree=scenario_tree_model) options = \ ScenarioTreeManagerClientSerial.register_options() for key in kwds: options[key] = kwds[key] manager = ScenarioTreeManagerClientSerial(options, factory=factory) manager.initialize() manager.reference_model = self.reference_model.clone() return manager
def generate_scenario_tree_image(options): with ScenarioTreeInstanceFactory( options.model_location, options.scenario_tree_location) as factory: scenario_tree = factory.generate_scenario_tree( downsample_fraction=options.scenario_tree_downsample_fraction, bundles=options.scenario_bundle_specification, random_bundles=options.create_random_bundles, random_seed=options.scenario_tree_random_seed, verbose=options.verbose) with TempfileManager.push(): tmpdotfile = TempfileManager.create_tempfile(suffix=".dot") scenario_tree.save_to_dot(tmpdotfile) os.system('dot -Tpdf -o %s %s' % (options.output_file, tmpdotfile)) print("Output Saved To: %s" % (options.output_file))
def pyro_sample_sp(self, size, **kwds): assert size > 0 model = self.reference_model.clone() scenario_tree_model = \ self._create_scenario_tree_model(size) factory = ScenarioTreeInstanceFactory( model=self.reference_model, scenario_tree=scenario_tree_model) options = \ ScenarioTreeManagerClientPyro.register_options() for key in kwds: options[key] = kwds[key] manager = ScenarioTreeManagerClientPyro( options, factory=factory) try: init = manager.initialize(async_call=True) pcuids = ComponentMap() for param in self.stochastic_data: pcuids[param] = ComponentUID(param) init.complete() for scenario in manager.scenario_tree.scenarios: data = [] for param, dist in self.stochastic_data.items(): data.append((pcuids[param], dist.sample())) manager.invoke_function( "_update_data", thisfile, invocation_type=InvocationType.OnScenario(scenario.name), function_args=(data,), oneway_call=True) manager.reference_model = model except: manager.close() raise return manager
def test_init13(self): model = reference_test_model.create_instance() scenario_tree_model = CreateAbstractScenarioTreeModel().\ create_instance( join(testdatadir, "reference_test_scenario_tree.dat")) with ScenarioTreeInstanceFactory( model=model, scenario_tree=scenario_tree_model) as factory: self.assertTrue(factory.model_directory() is None) self.assertTrue(factory.scenario_tree_directory() is None) scenario_tree = factory.generate_scenario_tree() instances = factory.construct_instances_for_scenario_tree( scenario_tree, verbose=True) self.assertEqual(len(instances), 3) self.assertEqual(instances["s1"].p(), model.p()) self.assertEqual(instances["s2"].p(), model.p()) self.assertEqual(instances["s3"].p(), model.p()) self.assertEqual(factory._closed, True) self.assertEqual(len(factory._archives), 0)
def test_init17(self): self.assertTrue("reference_test_model" not in sys.modules) self.assertTrue("ScenarioStructure" not in sys.modules) nx_tree = load_external_module(os.path.join(testdatadir, "ScenarioStructure.py"))[0].G with ScenarioTreeInstanceFactory( model=join(testdatadir, "reference_test_model.py"), scenario_tree=nx_tree) as factory: self.assertEqual(len(factory._archives), 0) self.assertTrue(factory.model_directory() is not None) self.assertTrue(factory.scenario_tree_directory() is None) self._check_factory(factory) scenario_tree = factory.generate_scenario_tree() self.assertEqual(scenario_tree.contains_bundles(), False) # check that we can modify the networkx tree to redefine # bundles nx_tree.nodes["s1"]["bundle"] = 0 nx_tree.nodes["s2"]["bundle"] = 0 nx_tree.nodes["s3"]["bundle"] = 0 scenario_tree = factory.generate_scenario_tree() self.assertEqual(scenario_tree.contains_bundles(), True) self.assertEqual(len(scenario_tree.bundles), 1) nx_tree.nodes["s1"]["bundle"] = 0 nx_tree.nodes["s2"]["bundle"] = 1 nx_tree.nodes["s3"]["bundle"] = 2 scenario_tree = factory.generate_scenario_tree() self.assertEqual(scenario_tree.contains_bundles(), True) self.assertEqual(len(scenario_tree.bundles), 3) nx_tree.nodes["s1"]["bundle"] = None nx_tree.nodes["s2"]["bundle"] = None nx_tree.nodes["s3"]["bundle"] = None scenario_tree = factory.generate_scenario_tree() self.assertEqual(scenario_tree.contains_bundles(), False) self.assertTrue("reference_test_model" in sys.modules) self.assertTrue("ScenarioStructure" in sys.modules)
def get_factory(): tree = networkx.DiGraph() tree.add_node("r", variables=["x"], cost="t0_cost") for i in range(3): tree.add_node("s" + str(i), variables=["Y", "stale", "fixed"], cost="t1_cost", bundle="b" + str(i)) tree.add_edge("r", "s" + str(i), weight=1.0 / 3) model = pyo.ConcreteModel() model.x = pyo.Var() model.Y = pyo.Var([1], bounds=(None, 1)) model.stale = pyo.Var(initialize=0.0) model.fixed = pyo.Var(initialize=0.0) model.fixed.fix() model.p = pyo.Param(mutable=True) model.t0_cost = pyo.Expression(expr=model.x) model.t1_cost = pyo.Expression(expr=model.Y[1]) model.o = pyo.Objective(expr=model.t0_cost + model.t1_cost) model.c = pyo.ConstraintList() model.c.add(model.x >= 1) model.c.add(model.Y[1] >= model.p) def _create_model(scenario_name, node_names): m = model.clone() if scenario_name == "s0": m.p.value = 0.0 elif scenario_name == "s1": m.p.value = 1.0 else: assert (scenario_name == "s2") m.p.value = 2.0 return m return ScenarioTreeInstanceFactory(model=_create_model, scenario_tree=tree)
def test_init18(self): self.assertTrue("reference_test_model" not in sys.modules) self.assertTrue("ScenarioStructure" not in sys.modules) nx_tree = load_external_module(os.path.join(testdatadir, "ScenarioStructure.py"))[0].G def scenario_model_callback(scenario_tree, scenario_name, node_list): self.assertIs(scenario_tree, nx_tree) instance = reference_test_model.create_instance() if scenario_name == "s1": instance.p = 1.0 elif scenario_name == "s2": instance.p = 2.0 else: assert scenario_name == "s3" instance.p = 3.0 return instance with ScenarioTreeInstanceFactory( model=scenario_model_callback, scenario_tree=nx_tree) as factory: self.assertTrue(factory.model_directory() is None) self.assertTrue(factory.scenario_tree_directory() is None) self._check_factory(factory) self.assertEqual(factory._closed, True) self.assertEqual(len(factory._archives), 0)
def test_init12(self): self.assertTrue("reference_test_model" not in sys.modules) scenario_tree_model = CreateAbstractScenarioTreeModel().\ create_instance( join(testdatadir, "reference_test_scenario_tree.dat")) def scenario_model_callback(scenario_tree, scenario_name, node_list): self.assertIs(scenario_tree, scenario_tree_model) instance = reference_test_model.create_instance() if scenario_name == "s1": instance.p = 1.0 elif scenario_name == "s2": instance.p = 2.0 else: assert scenario_name == "s3" instance.p = 3.0 return instance with ScenarioTreeInstanceFactory( model=scenario_model_callback, scenario_tree=scenario_tree_model) as factory: self.assertTrue(factory.model_directory() is None) self.assertTrue(factory.scenario_tree_directory() is None) self._check_factory(factory) self.assertEqual(factory._closed, True) self.assertEqual(len(factory._archives), 0)
def test_init6(self): self.assertTrue("reference_test_model" not in sys.modules) scenario_tree_model = CreateAbstractScenarioTreeModel().\ create_instance( join(testdatadir, "reference_test_scenario_tree.dat")) with ScenarioTreeInstanceFactory( model=join(testdatadir, "reference_test_model.py"), scenario_tree=scenario_tree_model) as factory: self.assertTrue(factory.model_directory() is not None) self.assertTrue(factory.scenario_tree_directory() is None) self._check_factory(factory) with self.assertRaises(TypeError): with ScenarioTreeInstanceFactory( model=join(testdatadir, "reference_test_model.py"), scenario_tree=int) as f: pass with self.assertRaises(ValueError): with ScenarioTreeInstanceFactory( model=join(testdatadir, "reference_test_model.py"), scenario_tree=None) as f: pass with self.assertRaises(TypeError): with ScenarioTreeInstanceFactory( model=None, scenario_tree=scenario_tree_model) as f: pass with self.assertRaises(IOError): with ScenarioTreeInstanceFactory( model=join(testdatadir, "reference_test_model_does_not_exist.py"), scenario_tree=scenario_tree_model) as f: pass with self.assertRaises(ValueError): with ScenarioTreeInstanceFactory( model=join(testdatadir, "reference_test_model.py"), scenario_tree=CreateAbstractScenarioTreeModel()) as f: pass self.assertEqual(len(factory._archives), 0) self.assertTrue("reference_test_model" in sys.modules)
from pyomo.contrib.pynumero.sparse import BlockMatrix from pyomo.contrib.pynumero.interfaces.pyomo_nlp import PyomoNLP import matplotlib.pylab as plt from pysp.scenariotree.manager import \ ScenarioTreeManagerFactory from pysp.scenariotree.instance_factory import \ ScenarioTreeInstanceFactory from gas_network_model import (pysp_instance_creation_callback, nx_scenario_tree) from pysp.ef import create_ef_instance # define and initialize the SP instance_factory = ScenarioTreeInstanceFactory(pysp_instance_creation_callback, nx_scenario_tree) options = ScenarioTreeManagerFactory.register_options() options.scenario_tree_manager = 'serial' sp = ScenarioTreeManagerFactory(options, factory=instance_factory) sp.initialize() instance = create_ef_instance(sp.scenario_tree) #instance = create_model(1.0) nlp = PyomoNLP(instance) print("\n----------------------") print("Problem statistics:") print("----------------------") print("Number of variables: {:>25d}".format(nlp.n_primals())) print("Number of equality constraints: {:>14d}".format(nlp.n_eq_constraints())) print("Number of inequality constraints: {:>11d}".format(
def run(args=None): ##########################================================######### # to import plugins import pyomo.environ import pysp.pyro.smanager_pyro import pysp.plugins.phpyro def partialLagrangeParametric(args=None): print("lagrangeParam begins ") blanks = " " # used for formatting print statements class Object(object): pass Result = Object() # options used IndVarName = options.indicator_var_name CCStageNum = options.stage_num alphaTol = options.alpha_tol MaxMorePR = options.MaxMorePR # option to include up to this many PR points above F^* with all delta fixed outputFilePrefix = options.outputFilePrefix # We write ScenarioList = name, probability # PRoptimal = probability, min-cost, [selections] # PRmore = probability, min-cost, [selections] # ================ sorted by probability ======================== # # These can be read to avoid re-computing points ph = PHFromScratch(options) Result.ph = ph rootnode = ph._scenario_tree._stages[0]._tree_nodes[ 0] # use rootnode to loop over scenarios if find_active_objective(ph._scenario_tree._scenarios[0]._instance, safety_checks=True).is_minimizing(): print("We are solving a MINIMIZATION problem.\n") else: print("We are solving a MAXIMIZATION problem.\n") # initialize ScenarioList = [] lambdaval = 0. lagrUtil.Set_ParmValue(ph, options.lambda_parm_name, lambdaval) # IMPORTANT: Preprocess the scenario instances # before fixing variables, otherwise they # will be preprocessed out of the expressions # and the output_fixed_variable_bounds option # will have no effect when we update the # fixed variable values (and then assume we # do not need to preprocess again because # of this option). ph._preprocess_scenario_instances() lagrUtil.FixAllIndicatorVariables(ph, IndVarName, 0) for scenario in rootnode._scenarios: ScenarioList.append((scenario._name, scenario._probability)) # sorts from min to max probability ScenarioList.sort(key=operator.itemgetter(1)) with open(outputFilePrefix + 'ScenarioList.csv', 'w') as outFile: for scenario in ScenarioList: outFile.write(scenario[0] + ", " + str(scenario[1]) + "\n") Result.ScenarioList = ScenarioList print("lambda= " + str(lambdaval) + " ...run begins " + str(len(ScenarioList)) + " scenarios") SolStat, zL = lagrUtil.solve_ph_code(ph, options) print("\t...ends") bL = Compute_ExpectationforVariable(ph, IndVarName, CCStageNum) if bL > 0: print("** bL = " + str(bL) + " > 0") return Result print("Initial cost = " + str(zL) + " for bL = " + str(bL)) lagrUtil.FixAllIndicatorVariables(ph, IndVarName, 1) print("lambda= " + str(lambdaval) + " ...run begins") SolStat, zU = lagrUtil.solve_ph_code(ph, options) print("\t...ends") bU = Compute_ExpectationforVariable(ph, IndVarName, CCStageNum) if bU < 1: print("** bU = " + str(bU) + " < 1") lagrUtil.FreeAllIndicatorVariables(ph, IndVarName) Result.lbz = [[0, bL, zL], [None, bU, zU]] Result.selections = [[], ScenarioList] NumIntervals = 1 print("initial gap = " + str(1 - zL / zU) + " \n") print("End of test; this is only a test.") return Result ################################ # LagrangeParametric ends here ################################ #### start run #### AllInOne = False ########################## # options defined here ########################## try: conf_options_parser = construct_ph_options_parser("lagrange [options]") conf_options_parser.add_argument( "--alpha", help="The alpha level for the chance constraint. Default is 0.05", action="store", dest="alpha", type=float, default=0.05) conf_options_parser.add_argument( "--alpha-min", help= "The min alpha level for the chance constraint. Default is None", action="store", dest="alpha_min", type=float, default=None) conf_options_parser.add_argument( "--alpha-max", help="The alpha level for the chance constraint. Default is None", action="store", dest="alpha_max", type=float, default=None) conf_options_parser.add_argument( "--min-prob", help="Tolerance for testing probability > 0. Default is 1e-5", action="store", dest="min_prob", type=float, default=1e-5) conf_options_parser.add_argument( "--alpha-tol", help="Tolerance for testing equality to alpha. Default is 1e-5", action="store", dest="alpha_tol", type=float, default=1e-5) conf_options_parser.add_argument( "--MaxMorePR", help= "Generate up to this many additional PR points after response function. Default is 0", action="store", dest="MaxMorePR", type=int, default=0) conf_options_parser.add_argument( "--outputFilePrefix", help="Output file name. Default is ''", action="store", dest="outputFilePrefix", type=str, default="") conf_options_parser.add_argument( "--stage-num", help= "The stage number of the CC indicator variable (number, not name). Default is 2", action="store", dest="stage_num", type=int, default=2) conf_options_parser.add_argument( "--lambda-parm-name", help= "The name of the lambda parameter in the model. Default is lambdaMult", action="store", dest="lambda_parm_name", type=str, default="lambdaMult") conf_options_parser.add_argument( "--indicator-var-name", help= "The name of the indicator variable for the chance constraint. The default is delta", action="store", dest="indicator_var_name", type=str, default="delta") conf_options_parser.add_argument( "--use-Loane-cuts", help="Add the Loane cuts if there is a gap. Default is False", action="store_true", dest="add_Loane_cuts", default=False) conf_options_parser.add_argument( "--fofx-var-name", help= "(Loane) The name of the model's auxiliary variable that is constrained to be f(x). Default is fofox", action="store", dest="fofx_var_name", type=str, default="fofx") conf_options_parser.add_argument( "--solve-with-ph", help= "Perform solves via PH rather than an EF solve. Default is False", action="store_true", dest="solve_with_ph", default=False) conf_options_parser.add_argument( "--skip-graph", help= "Do not show the graph at the end. Default is False (i.e. show the graph)", action="store_true", dest="skip_graph", default=False) conf_options_parser.add_argument( "--write-xls", help="Write results into a xls file. Default is False", action="store_true", dest="write_xls", default=False) conf_options_parser.add_argument( "--skip-ExpFlip", help= "Do not show the results for flipping the indicator variable for each scenario. Default is False (i.e. show the flipping-results)", action="store_true", dest="skip_ExpFlip", default=False) conf_options_parser.add_argument( "--HeurFlip", help= "The number of solutions to evaluate after the heuristic. Default is 3. For 0 the heuristic flip gets skipped.", action="store", type=int, dest="HeurFlip", default=3) conf_options_parser.add_argument( "--HeurMIP", help= "The mipgap for the scenariowise solves in the heuristic. Default is 0.0001", action="store", type=float, dest="HeurMIP", default=0.0001) conf_options_parser.add_argument( "--interactive", help="Enable interactive version of the code. Default is False.", action="store_true", dest="interactive", default=False) conf_options_parser.add_argument( "--Lgap", help= "The (relative) Lagrangian gap acceptable for the chance constraint. Default is 10^-4", action="store", type=float, dest="LagrangeGap", default=0.0001) conf_options_parser.add_argument( "--lagrange-method", help="The Lagrange multiplier search method", action="store", dest="lagrange_search_method", type=str, default="tangential") conf_options_parser.add_argument( "--max-lambda", help="The max value of the multiplier. Default=10^10", action="store", dest="max_lambda", type=float, default=10**10) conf_options_parser.add_argument( "--min-lambda", help="The min value of the multiplier. Default=0.0", action="store", dest="min_lambda", type=float, default=0) conf_options_parser.add_argument( "--min-probability", help="The min value of scenario probability. Default=10^-15", action="store", dest="min_probability", type=float, default=10**(-15)) ################################################################ options = conf_options_parser.parse_args(args=args) # temporary hack options._ef_options = conf_options_parser._ef_options options._ef_options.import_argparse(options) except SystemExit as _exc: # the parser throws a system exit if "-h" is specified - catch # it to exit gracefully. return _exc.code # load the reference model and create the scenario tree - no # scenario instances yet. if options.verbose: print("Loading reference model and scenario tree") #scenario_instance_factory, full_scenario_tree = load_models(options) scenario_instance_factory = \ ScenarioTreeInstanceFactory(options.model_directory, options.instance_directory) full_scenario_tree = \ GenerateScenarioTreeForPH(options, scenario_instance_factory) solver_manager = SolverManagerFactory(options.solver_manager_type) if solver_manager is None: raise ValueError("Failed to create solver manager of " "type=" + options.solver_manager_type + " specified in call to PH constructor") if isinstance(solver_manager, pysp.plugins.phpyro.SolverManager_PHPyro): raise ValueError("PHPyro can not be used as the solver manager") try: if (scenario_instance_factory is None) or (full_scenario_tree is None): raise RuntimeError( "***ERROR: Failed to initialize model and/or the scenario tree data." ) # load_model gets called again, so lets make sure unarchived directories are used options.model_directory = scenario_instance_factory._model_filename options.instance_directory = scenario_instance_factory._scenario_tree_filename scenario_count = len(full_scenario_tree._stages[-1]._tree_nodes) # create ph objects for finding the solution. we do this even if # we're solving the extensive form if options.verbose: print( "Loading scenario instances and initializing scenario tree for full problem." ) ########## Here is where multiplier search is called ############ Result = partialLagrangeParametric() ##################################################################################### finally: # delete temporary unarchived directories scenario_instance_factory.close() print("\nreturned from partialLagrangeParametric")
class ScenarioTreeServerPyro(pyu_pyro.TaskWorker): # Maps name to a registered worker class to instantiate _registered_workers = {} @classmethod def get_registered_worker_type(cls, name): if name in cls._registered_workers: return cls._registered_workers[name] raise KeyError("No worker type has been registered under the name " "'%s' for ScenarioTreeServerPyro" % (name)) def __init__(self, *args, **kwds): mpi = kwds.pop('mpi', None) # add for purposes of diagnostic output. kwds["name"] = ("ScenarioTreeServerPyro_%d@%s" % (os.getpid(), socket.gethostname())) if mpi is not None: assert len(mpi) == 2 kwds["name"] += "_MPIRank_" + str(mpi[1].rank) kwds["caller_name"] = kwds["name"] self._modules_imported = kwds.pop('modules_imported', {}) pyu_pyro.TaskWorker.__init__(self, **kwds) assert hasattr(self, "_bulk_task_collection") self._bulk_task_collection = True self._contiguous_task_processing = False self.type = self.WORKERNAME self.block = True self.timeout = None self._worker_map = {} self._init_verbose = self._verbose # A reference to the mpi4py.MPI namespace self.MPI = None # The communicator and group associated with all processors self.mpi_comm_world = None self.mpi_group_world = None # The communicator associated with the workers assigned # to the current current client self.mpi_comm_workers = None if mpi is not None: assert len(mpi) == 2 self.MPI = mpi[0] self.mpi_comm_world = mpi[1] self.mpi_group_world = self.mpi_comm_world.Get_group() # # These will be used by all subsequent workers created # by this server. Their creation can eat up a nontrivial # amount of initialization time when a large number of # workers are created on this server, so we only create # them once. # self._scenario_instance_factory = None self._full_scenario_tree = None def reset(self): if self._scenario_instance_factory is not None: self._scenario_instance_factory.close() self._scenario_instance_factory = None self._full_scenario_tree = None for worker_name in list(self._worker_map): self.remove_worker(worker_name) if self.mpi_comm_workers is not None: self.mpi_comm_workers.Free() self.mpi_comm_workers = None self._verbose = self._init_verbose def remove_worker(self, name): self._worker_map[name].close() del self._worker_map[name] def process(self, data): self._worker_task_return_queue = self._current_task_client try: # The only reason we are go through this much # effort to deal with the serpent serializer # is because it is the default in Pyro4. if pyu_pyro.using_pyro4 and \ (Pyro4.config.SERIALIZER == 'serpent'): if six.PY3: assert type(data) is dict assert data['encoding'] == 'base64' data = base64.b64decode(data['data']) else: assert type(data) is unicode data = str(data) return pickle.dumps(self._process(pickle.loads(data))) except: logger.error("Scenario tree server %s caught an exception of type " "%s while processing a task. Going idle." % (self.WORKERNAME, sys.exc_info()[0].__name__)) traceback.print_exception(*sys.exc_info()) self._worker_error = True return pickle.dumps( pyu_pyro.TaskProcessingError(traceback.format_exc())) def _process(self, data): data = Bunch(**data) result = None if not data.action.startswith('ScenarioTreeServerPyro_'): result = getattr(self._worker_map[data.worker_name], data.action)\ (*data.args, **data.kwds) elif data.action == 'ScenarioTreeServerPyro_setup': model_input = data.options.pop('model', None) if model_input is None: model_input = data.options.pop('model_callback') assert dill_available model_input = dill.loads(model_input) scenario_tree_input = data.options.pop('scenario_tree') data_input = data.options.pop('data') mpi_group = data.options.pop("mpi_group", None) verbose = data.options.pop("verbose", False) assert len(data.options) == 0 self._verbose |= verbose assert self._scenario_instance_factory is None assert self._full_scenario_tree is None if self._verbose: print("Server %s received setup request." % (self.WORKERNAME)) # Make sure these are not archives assert (not isinstance(model_input, six.string_types)) or \ os.path.exists(model_input) assert isinstance(scenario_tree_input, ScenarioTree) self._scenario_instance_factory = \ ScenarioTreeInstanceFactory( model_input, scenario_tree_input, data=data_input) # # Try to prevent unnecessarily re-importing the model module # if other callbacks are in the same location. Doing so might # have serious consequences. # if self._scenario_instance_factory._model_module is not None: self._modules_imported[self._scenario_instance_factory.\ _model_filename] = \ self._scenario_instance_factory._model_module assert self._scenario_instance_factory._scenario_tree_module is None self._full_scenario_tree = \ self._scenario_instance_factory.generate_scenario_tree() assert self.mpi_comm_workers is None if self.mpi_comm_world is not None: assert self.mpi_group_world is not None assert mpi_group is not None mpi_group = self.mpi_group_world.Incl(mpi_group) self.mpi_comm_workers = \ self.mpi_comm_world.Create_group(mpi_group) else: assert mpi_group is None if self._full_scenario_tree is None: raise RuntimeError("Unable to launch scenario tree worker - " "scenario tree construction failed.") result = True elif data.action == "ScenarioTreeServerPyro_initialize": worker_name = data.worker_name if self._verbose: print("Server %s received request to initialize " "scenario tree worker with name %s." % (self.WORKERNAME, worker_name)) assert self._scenario_instance_factory is not None assert self._full_scenario_tree is not None if worker_name in self._worker_map: raise RuntimeError( "Server %s Cannot initialize worker with name '%s' " "because a worker already exists with that name." % (self.WORKERNAME, worker_name)) worker_type = self._registered_workers[data.worker_type] self._worker_map[worker_name] = worker_type( self, worker_name, *data.init_args, **data.init_kwds) result = True elif data.action == "ScenarioTreeServerPyro_release": if self._verbose: print("Server %s releasing worker: %s" % (self.WORKERNAME, data.worker_name)) self.remove_worker(data.worker_name) result = True elif data.action == "ScenarioTreeServerPyro_reset": if self._verbose: print("Server %s received reset request" % (self.WORKERNAME)) self.reset() result = True elif data.action == "ScenarioTreeServerPyro_shutdown": if self._verbose: print("Server %s received shutdown request" % (self.WORKERNAME)) self.reset() self._worker_shutdown = True result = True else: raise ValueError("Server %s: Invalid command: %s" % (self.WORKERNAME, data.action)) return result
def _process(self, data): data = Bunch(**data) result = None if not data.action.startswith('ScenarioTreeServerPyro_'): result = getattr(self._worker_map[data.worker_name], data.action)\ (*data.args, **data.kwds) elif data.action == 'ScenarioTreeServerPyro_setup': model_input = data.options.pop('model', None) if model_input is None: model_input = data.options.pop('model_callback') assert dill_available model_input = dill.loads(model_input) scenario_tree_input = data.options.pop('scenario_tree') data_input = data.options.pop('data') mpi_group = data.options.pop("mpi_group", None) verbose = data.options.pop("verbose", False) assert len(data.options) == 0 self._verbose |= verbose assert self._scenario_instance_factory is None assert self._full_scenario_tree is None if self._verbose: print("Server %s received setup request." % (self.WORKERNAME)) # Make sure these are not archives assert (not isinstance(model_input, six.string_types)) or \ os.path.exists(model_input) assert isinstance(scenario_tree_input, ScenarioTree) self._scenario_instance_factory = \ ScenarioTreeInstanceFactory( model_input, scenario_tree_input, data=data_input) # # Try to prevent unnecessarily re-importing the model module # if other callbacks are in the same location. Doing so might # have serious consequences. # if self._scenario_instance_factory._model_module is not None: self._modules_imported[self._scenario_instance_factory.\ _model_filename] = \ self._scenario_instance_factory._model_module assert self._scenario_instance_factory._scenario_tree_module is None self._full_scenario_tree = \ self._scenario_instance_factory.generate_scenario_tree() assert self.mpi_comm_workers is None if self.mpi_comm_world is not None: assert self.mpi_group_world is not None assert mpi_group is not None mpi_group = self.mpi_group_world.Incl(mpi_group) self.mpi_comm_workers = \ self.mpi_comm_world.Create_group(mpi_group) else: assert mpi_group is None if self._full_scenario_tree is None: raise RuntimeError("Unable to launch scenario tree worker - " "scenario tree construction failed.") result = True elif data.action == "ScenarioTreeServerPyro_initialize": worker_name = data.worker_name if self._verbose: print("Server %s received request to initialize " "scenario tree worker with name %s." % (self.WORKERNAME, worker_name)) assert self._scenario_instance_factory is not None assert self._full_scenario_tree is not None if worker_name in self._worker_map: raise RuntimeError( "Server %s Cannot initialize worker with name '%s' " "because a worker already exists with that name." % (self.WORKERNAME, worker_name)) worker_type = self._registered_workers[data.worker_type] self._worker_map[worker_name] = worker_type( self, worker_name, *data.init_args, **data.init_kwds) result = True elif data.action == "ScenarioTreeServerPyro_release": if self._verbose: print("Server %s releasing worker: %s" % (self.WORKERNAME, data.worker_name)) self.remove_worker(data.worker_name) result = True elif data.action == "ScenarioTreeServerPyro_reset": if self._verbose: print("Server %s received reset request" % (self.WORKERNAME)) self.reset() result = True elif data.action == "ScenarioTreeServerPyro_shutdown": if self._verbose: print("Server %s received shutdown request" % (self.WORKERNAME)) self.reset() self._worker_shutdown = True result = True else: raise ValueError("Server %s: Invalid command: %s" % (self.WORKERNAME, data.action)) return result
def run(args=None): ################################### print("RUNNING - run args=%s" % str(args)) import pyomo.environ def LagrangeParametric(args=None): class Object(object): pass Result = Object() Result.status = 'LagrangeParam begins ' + datetime_string( ) + '...running new ph' ph = None blanks = " " # used for formatting print statements # options used betaMin = options.beta_min betaMax = options.beta_max betaTol = options.beta_tol gapTol = options.Lagrange_gap minProb = options.min_prob maxIntervals = options.max_intervals maxTime = options.max_time IndVarName = options.indicator_var_name multName = options.lambda_parm_name CCStageNum = options.stage_num csvPrefix = options.csvPrefix verbosity = options.verbosity verbosity = 2 # override for debug (= 3 to get super-debug) HGdebug = 0 # special debug (not public) # local...may become option optTol = gapTol #################################################################### STARTTIME = time.time() Result.status = "options set" if verbosity > 1: print("From LagrangeParametric, status = %s\tSTARTTIME = %s" \ % (str(getattr(Result,'status')), str(STARTTIME))) ph = PHFromScratch(options) Result.ph = ph rootnode = ph._scenario_tree._stages[0]._tree_nodes[ 0] # use rootnode to loop over scenarios ReferenceInstance = ph._instances[ rootnode._scenarios[0]._name] # arbitrary scenario if find_active_objective(ph._scenario_tree._scenarios[0]._instance, safety_checks=True).is_minimizing(): sense = 'min' else: sense = 'max' scenario_count = len(full_scenario_tree._stages[-1]._tree_nodes) if options.verbosity > 0: print("%s %s scenarios" % (str(sense), str(scenario_count))) # initialize Result.status = 'starting at ' + datetime_string() if verbosity > 0: print(Result.status) ScenarioList = [] lambdaval = 0. lagrUtil.Set_ParmValue(ph, multName, lambdaval) # IMPORTANT: Preprocess the scenario instances # before fixing variables, otherwise they # will be preprocessed out of the expressions # and the output_fixed_variable_bounds option # will have no effect when we update the # fixed variable values (and then assume we # do not need to preprocess again because # of this option). ph._preprocess_scenario_instances() sumprob = 0. minprob = 1. maxprob = 0. # fixed = 0 to get PR point at b=0 lagrUtil.FixAllIndicatorVariables(ph, IndVarName, 0) for scenario in rootnode._scenarios: instance = ph._instances[scenario._name] sname = scenario._name sprob = scenario._probability sumprob = sumprob + sprob minprob = min(minprob, sprob) maxprob = max(maxprob, sprob) ScenarioList.append([sname, sprob]) ScenarioList.sort( key=operator.itemgetter(1)) # sorts from min to max probability if verbosity > 0: print("probabilities sum to %f range: %f to %f" % (sumprob, minprob, maxprob)) Result.ScenarioList = ScenarioList # Write ScenarioList = name, probability in csv file sorted by probability outName = csvPrefix + 'ScenarioList.csv' print("writing to %s" % outName) with open(outName, 'w') as outFile: for scenario in ScenarioList: outFile.write(scenario[0] + ", " + str(scenario[1]) + '\n') Result.ScenarioList = ScenarioList addstatus = 'Scenario List written to ' + csvPrefix + 'ScenarioList.csv' Result.status = Result.status + '\n' + addstatus if verbosity > 0: print(addstatus) if verbosity > 0: print("solve begins %s" % datetime_string()) print("\t- lambda = %f" % lambdaval) SolStat, zL = lagrUtil.solve_ph_code(ph, options) if verbosity > 0: print("solve ends %s" % datetime_string()) print("\t- status = %s" % str(SolStat)) print("\t- zL = %s" % str(zL)) bL = Compute_ExpectationforVariable(ph, IndVarName, CCStageNum) if bL > 0: print("** bL = %s > 0 (all %s = 0)" % (str(bL), str(IndVarName))) return Result if verbosity > 0: print("Initial optimal obj = %s for bL = %s" % (str(zL), str(bL))) # fixed = 1 to get PR point at b=1 lagrUtil.FixAllIndicatorVariables(ph, IndVarName, 1) if verbosity > 0: print("solve begins %s" % datetime_string()) print("\t- lambda = %s" % str(lambdaval)) SolStat, zU = lagrUtil.solve_ph_code(ph, options) if verbosity > 0: print("solve ends %s" % datetime_string()) print("\t- status = %s" % str(SolStat)) print("\t- zU = %s" % str(zU)) if not SolStat[0:2] == 'ok': print(str(SolStat[0:3]) + " is not 'ok'") addstatus = "** Solution is non-optimal...aborting" print(addstatus) Result.status = Result.status + "\n" + addstatus return Result bU = Compute_ExpectationforVariable(ph, IndVarName, CCStageNum) if bU < 1. - betaTol and verbosity > 0: print("** Warning: bU = %s < 1" % str(bU)) ### enumerate points in PR space (all but one scenario) # Result.lbz = [ [0,bL,zL], [None,bU,zU] ] # for scenario in rootnode._scenarios: # sname = scenario._name # instance = ph._instances[sname] # print "excluding scenario",sname # getattr(instance,IndVarName).value = 0 # print sname,"value =",getattr(instance,IndVarName).value,getattr(instance,IndVarName).fixed # SolStat, z = lagrUtil.solve_ph_code(ph, options) # b = Compute_ExpectationforVariable(ph, IndVarName, CCStageNum) # print "solve ends with status =",SolStat,"(b, z) =",b,z # getattr(instance,IndVarName).value = 1 # Result.lbz.append([None,b,z]) # for t in instance.TimePeriods: # print "Global at",t,"=",instance.posGlobalLoadGenerateMismatch[t].value, \ # '-',instance.negGlobalLoadGenerateMismatch[t].value,"=",\ # instance.GlobalLoadGenerateMismatch[t].value,\ # "\tDemand =",instance.TotalDemand[t].value, ",",\ # "Reserve =",instance.ReserveRequirement[t].value # # PrintPRpoints(Result.lbz) # return Result #### end enumeration ######################################################################## if verbosity > 1: print("We have bU = %s ...about to free all %s for %d scenarios" % \ (str(bU), str(IndVarName), len(ScenarioList))) # free scenario selection variable lagrUtil.FreeAllIndicatorVariables(ph, IndVarName) if verbosity > 1: print("\tall %s freed; elapsed time = %f" % (str(IndVarName), time.time() - STARTTIME)) # initialize with the two endpoints Result.lbz = [[0., bL, zL], [None, bU, zU]] Result.selections = [[], ScenarioList] NumIntervals = 1 if verbosity > 0: print("Initial relative Lagrangian gap = %f maxIntervals = %d" % (1 - zL / zU, maxIntervals)) if verbosity > 1: print("entering while loop %s" % datetime_string()) print("\n") ############ main loop to search intervals ############# ######################################################## while NumIntervals < maxIntervals: lapsedTime = time.time() - STARTTIME if lapsedTime > maxTime: addstatus = '** max time reached ' + str(lapsedTime) print(addstatus) Result.status = Result.status + '\n' + addstatus break if verbosity > 1: print("Top of while with %d intervals elapsed time = %f" % (NumIntervals, lapsedTime)) PrintPRpoints(Result.lbz) lambdaval = None ### loop over PR points to find first unfathomed interval to search ### for PRpoint in range(1, len(Result.lbz)): if Result.lbz[PRpoint][0] == None: # multiplier = None means interval with upper endpoint at PRpoint not fathomed bL = Result.lbz[PRpoint - 1][1] zL = Result.lbz[PRpoint - 1][2] bU = Result.lbz[PRpoint][1] zU = Result.lbz[PRpoint][2] lambdaval = (zU - zL) / (bU - bL) break ############################# # Exited from the for loop if verbosity > 1: print("exited for loop with PRpoint = %s ...lambdaval = %s" % (PRpoint, lambdaval)) if lambdaval == None: break # all intervals are fathomed if verbosity > 1: PrintPRpoints(Result.lbz) if verbosity > 0: print("Searching for b in [%s, %s] with %s = %f" % (str( round(bL, 4)), str(round(bU, 4)), multName, lambdaval)) # search interval (bL,bU) lagrUtil.Set_ParmValue(ph, multName, lambdaval) if verbosity > 0: print("solve begins %s" % datetime_string()) print("\t- %s = %f" % (multName, lambdaval)) ######################################################### SolStat, Lagrangian = lagrUtil.solve_ph_code(ph, options) ######################################################### if not SolStat[0:2] == 'ok': addstatus = "** Solution status " + SolStat + " is not optimal" print(addstatus) Result.status = Result.status + "\n" + addstatus return Result b = Compute_ExpectationforVariable(ph, IndVarName, CCStageNum) z = Lagrangian + lambdaval * b if verbosity > 0: print("solve ends %s" % datetime_string()) print("\t- Lagrangian = %f" % Lagrangian) print("\t- b = %s" % str(b)) print("\t- z = %s" % str(z)) print("\n") # We have PR point (b,z), which may be new or one of the endpoints ################################################################## ######### Begin tolerance tests ########## # Test that b is in [bL,bU] if verbosity > 1: print("\ttesting b") if b < bL - betaTol or b > bU + betaTol: addstatus = "** fatal error: probability (= " + str(b) + \ ") is outside interval, (" + str(bL) + ", " + str(bU) + ")" addstatus = addstatus + "\n\t(tolerance = " + str( betaTol) + ")" print(addstatus + '\n') Result.status = Result.status + addstatus return Result # Test that z is in [zL,zU] if verbosity > 1: print("\ttesting z") # using optTol as absolute tolerance (not relative) # ...if we reconsider, need to allow negative z-values if z < zL - optTol or z > zU + optTol: addstatus = "** fatal error: obj (= " + str(z) + \ ") is outside interval, (" + str(zL) + ", " + str(zU) + ")" print(addstatus + '\n') Result.status = Result.status + addstatus return Result # Ok, we have (b,z) in [(bL,zL), (bU,zU)], at least within tolerances oldLagrangian = zL - lambdaval * bL # ensure lambdaval set such that endpoints have same Lagrangian value # (this is probably unnecessary, but check anyway) if abs(oldLagrangian - (zU - lambdaval * bU)) > optTol * abs(oldLagrangian): addstatus = "** fatal error: Lagrangian at (bL,zL) = " + \ str(oldLagrangian) + " not= " + str(zU-lambdaval*bU) + \ "\n\t(optTol = " + str(optTol) + ")" Result.status = Result.status + addstatus return Result # no more fatal error tests...need to know if (b,z) is endpoint or new if verbosity > 1: print("No anomalies...testing if b = bL or bU") # Test if endpoint is an alternative optimum of Lagrangian # ...using optTol as *relative* tolerance # (could use other reference values -- eg, avg or max of old and new Lagrangian values) refValue = max(min(abs(oldLagrangian), abs(Lagrangian)), 1.) alternativeOpt = abs(oldLagrangian - Lagrangian) <= optTol * refValue # alternativeOpt = True means we computed point (b,z) is alternative optimum such that: # case 1: (b,z) = endpoint, in which case we simply fathom [bL,bU] by setting PRpoint # to [lambdaval,bU,zU] (the numeric value of multiplier means fathomed) # case 2: (b,z) is new PR point on line segment, in which case we split into # [bL,b] and [b,bU], with both fathomed if verbosity > 1: print("oldLagrangian = %s" % str(oldLagrangian)) if alternativeOpt: print(":= Lagrangian = %s" % str(Lagrangian)) else: print("> Lagrangian = %s" % str(Lagrangian)) if alternativeOpt: # setting multiplier of (bU,zU) to a numeric fathoms the interval [bL,bU] Result.lbz[PRpoint][0] = lambdaval # test if (b,z) is an endpoint newPRpoint = abs(b - bL) > betaTol and abs(b - bU) > betaTol if not newPRpoint: # ...(b,z) is NOT an endpoint (or sufficiently close), so split and fathom if verbosity > 1: print("\tnot an endpoint\tlbz = %s" % str(Result.lbz[PRpoint])) if verbosity > 0: print("Lagangian solution is new PR point on line segment of (" \ + str(bL) + ", " + str(bU) +")") print( "\tsplitting (bL,bU) into (bL,b) and (b,bU), both fathomed" ) # note: else ==> b = bL or bU, so we do nothing, having already fathomed [bL,bU] # (b,z) is new PR point, so split interval (still in while loop) ########################################## # alternative optimum ==> split & fathom: (bL,b), (b,bU) if verbosity > 1: print("\talternativeOpt %s newPRpoint = %s" % (alternativeOpt, newPRpoint)) if newPRpoint: NumIntervals += 1 if alternativeOpt: if verbosity > 1: print("\tInsert [lambdaval,b,z] at %f" % PRpoint) Result.lbz = Insert([lambdaval, b, z], PRpoint, Result.lbz) addstatus = "Added PR point on line segment of envelope" if verbosity > 0: print(addstatus + '\n') else: if verbosity > 1: print("\tInsert [None,b,z] at %f" % PRpoint) Result.lbz = Insert([None, b, z], PRpoint, Result.lbz) addstatus = "new envelope extreme point added (interval split, not fathomed)" Result.status = Result.status + "\n" + addstatus if verbosity > 1: print("...after insertion:") PrintPRpoints(Result.lbz) # get the selections of new point (ie, scenarios for which delta=1) Selections = [] for scenario in ScenarioList: instance = ph._instances[scenario[0]] if getattr(instance, IndVarName).value == 1: Selections.append(scenario) Result.selections = Insert(Selections, PRpoint, Result.selections) if verbosity > 0: print("Interval "+str(PRpoint)+", ["+str(bL)+", "+str(bU)+ \ "] split at ("+str(b)+", "+str(z)+")") print("\tnew PR point has " + str(len(Selections)) + " selections") if verbosity > 1: print("test that selections list aligned with lbz") if not len(Result.lbz) == len(Result.selections): print("** fatal error: lbz not= selections") PrintPRpoints(Result.lbz) print("Result.selections:") for i in range(Result.selections): print("%d %f" % (i, Result.selections[i])) return Result # ok, we have split and/or fathomed interval if NumIntervals >= maxIntervals: # we are about to leave while loop due to... addstatus = "** terminating because number of intervals = " + \ str(NumIntervals) + " >= max = " + str(maxIntervals) if verbosity > 0: print(addstatus + '\n') Result.status = Result.status + "\n" + addstatus # while loop continues if verbosity > 1: print("bottom of while loop") PrintPRpoints(Result.lbz) ################################################### # end while NumIntervals < maxIntervals: # ^ this is indentation of while loop ################ end while loop ################### if verbosity > 1: print("\nend while loop...setting multipliers") for i in range(1, len(Result.lbz)): db = Result.lbz[i][1] - Result.lbz[i - 1][1] dz = Result.lbz[i][2] - Result.lbz[i - 1][2] if dz > 0: Result.lbz[i][0] = dz / db else: #print "dz =",dz," at ",i,": ",Result.lbz[i]," -",Result.lbz[i-1] Result.lbz[i][0] = 0 if verbosity > 0: PrintPRpoints(Result.lbz) addstatus = '\nLagrange multiplier search ends' + datetime_string() if verbosity > 0: print(addstatus + '\n') Result.status = Result.status + addstatus outName = csvPrefix + "PRoptimal.csv" with open(outName, 'w') as outFile: if verbosity > 0: print("writing PR points to " + outName + '\n') for lbz in Result.lbz: outFile.write(str(lbz[1]) + ", " + str(lbz[2]) + '\n') outName = csvPrefix + "OptimalSelections.csv" with open(outName, 'w') as outFile: if verbosity > 0: print("writing optimal selections for each PR point to " + csvPrefix + 'PRoptimal.csv\n') for selections in Result.selections: char = "" thisSelection = "" for slist in selections: if slist: thisSelection = thisSelection + char + slist[0] char = "," outFile.write(thisSelection + '\n') if verbosity > 0: print("\nReturning status:\n %s \n=======================" % Result.status) ################################ if verbosity > 2: print("\nAbout to return...Result attributes: %d" % len(inspect.getmembers(Result))) for attr in inspect.getmembers(Result): print(attr[0]) print("\n===========================================") # LagrangeParametric ends here return Result ################################ ####################################### start run #################################### AllInOne = False ######################## # options defined here ######################## try: conf_options_parser = construct_ph_options_parser("lagrange [options]") conf_options_parser.add_argument( "--beta-min", help="The min beta level for the chance constraint. Default is 0", action="store", dest="beta_min", type=float, default=0.) conf_options_parser.add_argument( "--beta-max", help="The beta level for the chance constraint. Default is 1.", action="store", dest="beta_max", type=float, default=1.) conf_options_parser.add_argument( "--beta-tol", help="Tolerance for testing equality to beta. Default is 1e-5", action="store", dest="beta_tol", type=float, default=1e-5) conf_options_parser.add_argument( "--Lagrange-gap", help= "The (relative) Lagrangian gap acceptable for the chance constraint. Default is 10^-4", action="store", type=float, dest="Lagrange_gap", default=0.0001) conf_options_parser.add_argument( "--min-prob", help="Tolerance for testing probability > 0. Default is 1e-9", action="store", dest="min_prob", type=float, default=1e-5) conf_options_parser.add_argument( "--max-intervals", help= "The max number of intervals generated; if causes termination, non-fathomed intervals have multiplier=None. Default = 100.", action="store", dest="max_intervals", type=int, default=100) conf_options_parser.add_argument( "--max-time", help="Maximum time (seconds). Default is 3600.", action="store", dest="max_time", type=float, default=3600) conf_options_parser.add_argument( "--lambda-parm-name", help= "The name of the lambda parameter in the model. Default is lambdaMult", action="store", dest="lambda_parm_name", type=str, default="lambdaMult") conf_options_parser.add_argument( "--indicator-var-name", help= "The name of the indicator variable for the chance constraint. The default is delta", action="store", dest="indicator_var_name", type=str, default="delta") conf_options_parser.add_argument( "--stage-num", help= "The stage number of the CC indicator variable (number, not name). Default is 2", action="store", dest="stage_num", type=int, default=2) conf_options_parser.add_argument( "--csvPrefix", help="Output file name. Default is ''", action="store", dest="csvPrefix", type=str, default='') conf_options_parser.add_argument( "--verbosity", help= "verbosity=0 is no extra output, =1 is medium, =2 is debug, =3 super-debug. Default is 1.", action="store", dest="verbosity", type=int, default=1) # The following needed for solve_ph_code in lagrangeutils conf_options_parser.add_argument( "--solve-with-ph", help= "Perform solves via PH rather than an EF solve. Default is False", action="store_true", dest="solve_with_ph", default=False) ##HG: deleted params filed as deletedParam.py ####################################################################################################### options = conf_options_parser.parse_args(args=args) # temporary hack options._ef_options = conf_options_parser._ef_options options._ef_options.import_argparse(options) except SystemExit as _exc: # the parser throws a system exit if "-h" is specified - catch # it to exit gracefully. return _exc.code # create the reference instances and the scenario tree - no # scenario instances yet. if options.verbosity > 0: print("Loading reference model and scenario tree") # Dec 18 # scenario_instance_factory, full_scenario_tree = load_models(options) scenario_instance_factory = \ ScenarioTreeInstanceFactory(options.model_directory, options.instance_directory) full_scenario_tree = \ GenerateScenarioTreeForPH(options, scenario_instance_factory) #### try: if (scenario_instance_factory is None) or (full_scenario_tree is None): raise RuntimeError( "***ERROR: Failed to initialize the model and/or scenario tree data." ) # load_model gets called again, so lets make sure unarchived directories are used options.model_directory = scenario_instance_factory._model_filename options.instance_directory = scenario_instance_factory._scenario_tree_filename ########## Here is where multiplier search is called from run() ############ Result = LagrangeParametric() ##################################################################################### finally: # delete temporary unarchived directories scenario_instance_factory.close() if options.verbosity > 0: print("\n===========================================") print("\nreturned from LagrangeParametric") if options.verbosity > 2: print("\nFrom run, Result should have status and ph objects...") for attr in inspect.getmembers(Result): print(attr) print("\n===========================================") try: status = Result.status print("status = " + str(Result.status)) except: print("status not defined") sys.exit() try: lbz = Result.lbz PrintPRpoints(lbz) with open(options.csvPrefix + "PRoptimal.csv", 'w') as outFile: for lbz in Result.lbz: outFile.write(str(lbz[1]) + ", " + str(lbz[2]) + '\n') except: print("Result.lbz not defined") sys.exit() try: ScenarioList = Result.ScenarioList ScenarioList.sort(key=operator.itemgetter(1)) with open(options.csvPrefix + "ScenarioList.csv", 'w') as outFile: for scenario in ScenarioList: outFile.write(scenario[0] + ", " + str(scenario[1]) + '\n') except: print("Result.ScenarioList not defined") sys.exit()
def run(args=None): ################################### # to import plugins import pyomo.environ import pysp.plugins.phpyro def LagrangeMorePR(args=None): print("lagrangeMorePR begins %s" % datetime_string()) blanks = " " # used for formatting print statements class Object(object): pass Result = Object() # options used betaTol = options.beta_tol # tolerance used to separate b-values IndVarName = options.indicator_var_name multName = options.lambda_parm_name CCStageNum = options.stage_num MaxMorePR = options.max_number # max PR points to be generated (above F^* with all delta fixed) MaxTime = options.max_time # max time before terminate csvPrefix = options.csvPrefix # input filename prefix (eg, case name) probFileName = options.probFileName # name of file containing probabilities ##HG override # options.verbosity = 2 verbosity = options.verbosity Result.status = 'starting ' + datetime_string() STARTTIME = time.time() ph = PHFromScratch(options) rootnode = ph._scenario_tree._stages[0]._tree_nodes[ 0] # use rootnode to loop over scenarios if find_active_objective(ph._scenario_tree._scenarios[0]._instance, safety_checks=True).is_minimizing(): print("We are solving a MINIMIZATION problem.") else: print("We are solving a MAXIMIZATION problem.") # initialize ScenarioList = [] with open(csvPrefix + "ScenarioList.csv", 'r') as inputFile: for line in inputFile.readlines(): L = line.split(',') ScenarioList.append([L[0], float(L[1])]) addstatus = str( len(ScenarioList) ) + ' scenarios read from file: ' + csvPrefix + 'ScenarioList.csv' if verbosity > 0: print(addstatus) Result.status = Result.status + '\n' + addstatus PRoptimal = [] with open(csvPrefix + "PRoptimal.csv", 'r') as inputFile: for line in inputFile.readlines(): bzS = line.split(',') PRoptimal.append([None, float(bzS[0]), float(bzS[1])]) addstatus = str( len(PRoptimal) ) + ' PR points read from file: ' + csvPrefix + 'PRoptimal.csv (envelope function)' if verbosity > 0: print(addstatus) Result.status = Result.status + '\n' + addstatus # ensure PR points on envelope function are sorted by probability PRoptimal.sort(key=operator.itemgetter(1)) PRoptimal[0][0] = 0 # initial lambda (for b=0) for p in range(1, len(PRoptimal)): dz = PRoptimal[p][2] - PRoptimal[p - 1][2] db = PRoptimal[p][1] - PRoptimal[p - 1][1] PRoptimal[p][0] = dz / db if verbosity > 0: PrintPRpoints(PRoptimal) Result.PRoptimal = PRoptimal lambdaval = 0. lagrUtil.Set_ParmValue(ph, options.lambda_parm_name, lambdaval) # IMPORTANT: Preprocess the scenario instances # before fixing variables, otherwise they # will be preprocessed out of the expressions # and the output_fixed_variable_bounds option # will have no effect when we update the # fixed variable values (and then assume we # do not need to preprocess again because # of this option). ph._preprocess_scenario_instances() ## read scenarios to select for each PR point on envelope function with open(csvPrefix + "OptimalSelections.csv", 'r') as inputFile: OptimalSelections = [] for line in inputFile.readlines(): if len(line) == 0: break # eof selections = line.split(',') L = len(selections) Ls = len(selections[L - 1]) selections[L - 1] = selections[L - 1][0:Ls - 1] if verbosity > 1: print(str(selections)) OptimalSelections.append(selections) Result.OptimalSelections = OptimalSelections addstatus = str(len(OptimalSelections)) + ' Optimal selections read from file: ' \ + csvPrefix + 'OptimalSelections.csv' Result.status = Result.status + '\n' + addstatus if len(OptimalSelections) == len(PRoptimal): if verbosity > 0: print(addstatus) else: addstatus = addstatus + '\n** Number of selections not equal to number of PR points' print(addstatus) Result.status = Result.status + '\n' + addstatus print(str(OptimalSelections)) print((PRoptimal)) return Result ##################################################################################### # get probabilities if probFileName is None: # ...generate from widest gap regions PRlist = FindPRpoints(options, PRoptimal) else: # ...read probabilities probList = [] with open(probFileName, 'r') as inputFile: if verbosity > 0: print("reading from probList = " + probFileName) for line in inputFile.readlines(): # 1 probability per line if len(line) == 0: break prob = float(line) probList.append(prob) if verbosity > 0: print("\t " + str(len(probList)) + " probabilities") if verbosity > 1: print(str(probList)) PRlist = GetPoints(options, PRoptimal, probList) if verbosity > 1: print("PRlist:") for interval in PRlist: print(str(interval)) # We now have PRlist = [[i, b], ...], where b is in PRoptimal interval (i-1,i) addstatus = str(len(PRlist)) + ' probabilities' if verbosity > 0: print(addstatus) Result.status = Result.status + '\n' + addstatus ##################################################################################### lapsedTime = time.time() - STARTTIME addstatus = 'Initialize complete...lapsed time = ' + str(lapsedTime) if verbosity > 1: print(addstatus) Result.status = Result.status + '\n' + addstatus ##################################################################################### if verbosity > 1: print( "\nlooping over Intervals to generate PR points by flipping heuristic" ) Result.morePR = [] for interval in PRlist: lapsedTime = time.time() - STARTTIME if lapsedTime > MaxTime: addstatus = '** lapsed time = ' + str( lapsedTime) + ' > max time = ' + str(MaxTime) if verbosity > 0: print(addstatus) Result.status = Result.status + '\n' + addstatus break i = interval[0] # = PR point index b = interval[ 1] # = target probability to reach by flipping from upper endpoint bU = PRoptimal[i][1] # = upper endpoint bL = PRoptimal[i - 1][1] # = lower endpoint if verbosity > 1: print( "target probability = "+str(b)+" < bU = PRoptimal[" + str(i) + "][1]" \ " and > bL = PRoptimal["+str(i-1)+"][1]") if b < bL or b > bU: addstatus = '** probability = '+str(b) + ', not in gap interval: (' \ + str(bL) + ', ' + str(bU) + ')' print(addstatus) print(str(PRoptimal)) print(str(PRlist)) Result.status = Result.status + '\n' + addstatus return Result if verbosity > 1: print( "i = "+str(i)+" : Starting with bU = "+str(bU)+" having "+ \ str(len(OptimalSelections[i]))+ " selections:") print(str(OptimalSelections[i])) # first fix all scenarios = 0 for sname, sprob in ScenarioList: scenario = ph._scenario_tree.get_scenario(sname) lagrUtil.FixIndicatorVariableOneScenario( ph, scenario, IndVarName, 0) # now fix optimal selections = 1 for sname in OptimalSelections[i]: scenario = ph._scenario_tree.get_scenario(sname) lagrUtil.FixIndicatorVariableOneScenario( ph, scenario, IndVarName, 1) # flip scenario selections from bU until we reach b (target probability) bNew = bU for sname, sprob in ScenarioList: scenario = ph._scenario_tree.get_scenario(sname) if bNew - sprob < b: continue instance = ph._instances[sname] if getattr(instance, IndVarName).value == 0: continue bNew = bNew - sprob # flipped scenario selection lagrUtil.FixIndicatorVariableOneScenario( ph, scenario, IndVarName, 0) if verbosity > 1: print("\tflipped " + sname + " with prob = " + str(sprob) + " ...bNew = " + str(bNew)) if verbosity > 1: print("\tflipped selections reach " + str(bNew) + " >= target = " + str(b) + " (bL = " + str(bL) + ")") if bNew <= bL + betaTol or bNew >= bU - betaTol: if verbosity > 0: print( "\tNot generating PR point...flipping from bU failed") continue # to next interval in list # ready to solve to get cost for fixed scenario selections associated with probability = bNew if verbosity > 1: # check that scenarios are fixed as they should be totalprob = 0. for scenario in ScenarioList: sname = scenario[0] sprob = scenario[1] instance = ph._instances[sname] print("fix "+sname+" = "+str(getattr(instance,IndVarName).value)+\ " is "+str(getattr(instance,IndVarName).fixed)+" probability = "+str(sprob)) if getattr(instance, IndVarName).value == 1: totalprob = totalprob + sprob lambdaval = getattr(instance, multName).value print("\ttotal probability = %f" % totalprob) # solve (all delta fixed); lambda=0, so z = Lagrangian if verbosity > 0: print("solve begins %s" % datetime_string()) print("\t- lambda = %f" % lambdaval) SolStat, z = lagrUtil.solve_ph_code(ph, options) b = Compute_ExpectationforVariable(ph, IndVarName, CCStageNum) if verbosity > 0: print("solve ends %s" % datetime_string()) print("\t- SolStat = %s" % str(SolStat)) print("\t- b = %s" % str(b)) print("\t- z = %s" % str(z)) print("(adding to more PR points)") Result.morePR.append([None, b, z]) if verbosity > 1: PrintPRpoints(Result.morePR) ###################################################### # end loop over target probabilities with open(csvPrefix + "PRmore.csv", 'w') as outFile: for point in Result.morePR: outFile.write(str(point[1]) + ',' + str(point[2])) addstatus = str( len(Result.morePR )) + ' PR points written to file: ' + csvPrefix + 'PRmore.csv' if verbosity > 0: print(addstatus) Result.status = Result.status + '\n' + addstatus addstatus = 'lapsed time = ' + putcommas(time.time() - STARTTIME) if verbosity > 0: print(addstatus) Result.status = Result.status + '\n' + addstatus return Result ################################ # LagrangeMorePR ends here ################################ #### start run #### AllInOne = False # VERYSTARTTIME=time.time() # print "##############VERYSTARTTIME:",str(VERYSTARTTIME-VERYSTARTTIME) ########################## # options defined here ########################## try: conf_options_parser = construct_ph_options_parser("lagrange [options]") conf_options_parser.add_argument( "--beta-min", help= "The min beta level for the chance constraint. Default is None", action="store", dest="beta_min", type=float, default=None) conf_options_parser.add_argument( "--beta-max", help="The beta level for the chance constraint. Default is None", action="store", dest="beta_max", type=float, default=None) conf_options_parser.add_argument( "--min-prob", help="Tolerance for testing probability > 0. Default is 1e-5", action="store", dest="min_prob", type=float, default=1e-5) conf_options_parser.add_argument( "--beta-tol", help="Tolerance for testing equality to beta. Default is 10^-2", action="store", dest="beta_tol", type=float, default=1e-2) conf_options_parser.add_argument( "--Lagrange-gap", help= "The (relative) Lagrangian gap acceptable for the chance constraint. Default is 10^-4.", action="store", type=float, dest="Lagrange_gap", default=0.0001) conf_options_parser.add_argument( "--max-number", help="The max number of PR points. Default = 10.", action="store", dest="max_number", type=int, default=10) conf_options_parser.add_argument( "--max-time", help="Maximum time (seconds). Default is 3600.", action="store", dest="max_time", type=float, default=3600) conf_options_parser.add_argument( "--csvPrefix", help="Input file name prefix. Default is ''", action="store", dest="csvPrefix", type=str, default="") conf_options_parser.add_argument( "--lambda-parm-name", help= "The name of the lambda parameter in the model. Default is lambdaMult", action="store", dest="lambda_parm_name", type=str, default="lambdaMult") conf_options_parser.add_argument( "--indicator-var-name", help= "The name of the indicator variable for the chance constraint. The default is delta", action="store", dest="indicator_var_name", type=str, default="delta") conf_options_parser.add_argument( "--stage-num", help= "The stage number of the CC indicator variable (number, not name). Default is 2", action="store", dest="stage_num", type=int, default=2) conf_options_parser.add_argument( "--verbosity", help= "verbosity=0 is no extra output, =1 is medium, =2 is debug, =3 super-debug. Default is 1.", action="store", dest="verbosity", type=int, default=1) conf_options_parser.add_argument( "--prob-file", help="file name specifiying probabilities", action="store", dest="probFileName", type=str, default=None) # The following needed for solve_ph_code in lagrangeutils conf_options_parser.add_argument( "--solve-with-ph", help= "Perform solves via PH rather than an EF solve. Default is False", action="store_true", dest="solve_with_ph", default=False) ################################################################ options = conf_options_parser.parse_args(args=args) # temporary hack options._ef_options = conf_options_parser._ef_options options._ef_options.import_argparse(options) except SystemExit as _exc: # the parser throws a system exit if "-h" is specified - catch # it to exit gracefully. return _exc.code if options.verbose is True: print("Loading reference model and scenario tree") scenario_instance_factory = \ ScenarioTreeInstanceFactory(options.model_directory, options.instance_directory) full_scenario_tree = \ GenerateScenarioTreeForPH(options, scenario_instance_factory) solver_manager = SolverManagerFactory(options.solver_manager_type) if solver_manager is None: raise ValueError("Failed to create solver manager of " "type=" + options.solver_manager_type + " specified in call to PH constructor") if isinstance(solver_manager, pysp.plugins.phpyro.SolverManager_PHPyro): raise ValueError("PHPyro can not be used as the solver manager") try: if (scenario_instance_factory is None) or (full_scenario_tree is None): raise RuntimeError( "***ERROR: Failed to initialize the model and/or scenario tree data." ) # load_model gets called again, so lets make sure unarchived directories are used options.model_directory = scenario_instance_factory._model_filename options.instance_directory = scenario_instance_factory._scenario_tree_filename scenario_count = len(full_scenario_tree._stages[-1]._tree_nodes) # create ph objects for finding the solution. we do this even if # we're solving the extensive form if options.verbose is True: print( "Loading scenario instances and initializing scenario tree for full problem." ) ########## Here is where multiplier search is called ############ Result = LagrangeMorePR() ##################################################################################### finally: # delete temporary unarchived directories scenario_instance_factory.close() print("\n==================== returned from LagrangeMorePR") print(str(Result.status)) try: print("Envelope:") print(str(PrintPRpoints(Result.PRoptimal))) print("\nAdded:") PrintPRpoints(Result.morePR) except: print("from run: PrintPRpoints failed") sys.exit() # combine tables and sort by probability if len(Result.morePR) > 0: PRpoints = copy.deepcopy(Result.PRoptimal) for lbz in Result.morePR: PRpoints.append(lbz) print("Combined table of PR points (sorted):") PRpoints.sort(key=operator.itemgetter(1)) print(str(PrintPRpoints(PRpoints)))