def setUp(self): """ Get ready for tests""" ######## make a temp dir to which files can be copied ##### self.tdir = tempfile.mkdtemp() #TemporaryDirectory().name sys.path.insert(1, self.tdir) """ During debugging, local files might get in the way of finding the file in the temp dir, so we cd there.""" self.savecwd = os.getcwd() os.chdir(self.tdir) p = str(pyomoroot.__path__) l = p.find("'") r = p.find("'", l + 1) pyomorootpath = p[l + 1:r] farmpath = pyomorootpath + os.sep + ".." + os.sep + "examples" + \ os.sep + "pysp" + os.sep + "farmer" farmpath = os.path.abspath(farmpath) self.farmer_concrete_file = farmpath + os.sep + \ "concrete" + os.sep + "ReferenceModel.py" shutil.copyfile(self.farmer_concrete_file, self.tdir + os.sep + "ReferenceModel.py") abstract_tree = CreateAbstractScenarioTreeModel() shutil.copyfile( farmpath + os.sep + "scenariodata" + os.sep + "ScenarioStructure.dat", self.tdir + os.sep + "ScenarioStructure.dat") self.farmer_concrete_tree = \ abstract_tree.create_instance("ScenarioStructure.dat")
def setUp(self): """ Get ready for tests""" ######## make a temp dir to which files can be copied ##### self.tdir = tempfile.mkdtemp() #TemporaryDirectory().name sys.path.insert(1,self.tdir) """ During debugging, local files might get in the way of finding the file in the temp dir, so we cd there.""" self.savecwd = os.getcwd() os.chdir(self.tdir) p = str(pyomoroot.__path__) l = p.find("'") r = p.find("'", l+1) pyomorootpath = p[l+1:r] farmpath = pyomorootpath + os.sep + ".." + os.sep + "examples" + \ os.sep + "pysp" + os.sep + "farmer" farmpath = os.path.abspath(farmpath) self.farmer_concrete_file = farmpath + os.sep + \ "concrete" + os.sep + "ReferenceModel.py" shutil.copyfile(self.farmer_concrete_file, self.tdir + os.sep + "ReferenceModel.py") abstract_tree = CreateAbstractScenarioTreeModel() shutil.copyfile(farmpath + os.sep +"scenariodata" + os.sep + "ScenarioStructure.dat", self.tdir + os.sep + "ScenarioStructure.dat") self.farmer_concrete_tree = \ abstract_tree.create_instance("ScenarioStructure.dat")
def _create_scenario_tree_model(self, size): assert size > 0 stm = CreateAbstractScenarioTreeModel() stm.Stages.add('t1') stm.Stages.add('t2') stm.Nodes.add('root') for i in xrange(1, size+1): stm.Nodes.add('n'+str(i)) stm.Scenarios.add('s'+str(i)) stm = stm.create_instance() stm.NodeStage['root'] = 't1' stm.ConditionalProbability['root'] = 1.0 weight = 1.0/float(size) for i in xrange(1, size+1): node_name = 'n'+str(i) scen_name = 's'+str(i) stm.NodeStage[node_name] = 't2' stm.Children['root'].add(node_name) stm.Children[node_name].clear() stm.ConditionalProbability[node_name] = weight stm.ScenarioLeafNode[scen_name] = node_name stm.StageCost['t1'] = self._stage1_cost.name stm.StageCost['t2'] = self._stage2_cost.name for var, (stagenum, derived) in \ self._variable_stage_assignments.items(): stage_name = 't'+str(stagenum) if not derived: stm.StageVariables[stage_name].add(var.name) else: stm.StageDerivedVariables[stage_name].add(var.name) return stm
def solve_sto(f, IndexReturn, Tbill): #The next two lines show one way to create a concrete scenario tree. There are #others that can be found in `pyomo.pysp.scenariotree.tree_structure_model`. abstract_tree = CreateAbstractScenarioTreeModel() concrete_tree = abstract_tree.create_instance("/Users/xiaoshiguo/Desktop/model"+str(J)+"/ScenarioStructure"+str(J)+".dat") concrete_tree.IndexReturn = IndexReturn # line added by DLW concrete_tree.Tbill = Tbill stsolver = rapper.StochSolver("ReferenceModel"+str(J)+".py", fsfct = "pysp_instance_creation_callback", tree_model = concrete_tree) #stsolver = rapper.StochSolver("/Users/xiaoshiguo/Desktop/portfolio/models/ReferenceModel.py", tree_model = concrete_tree) ef_sol = stsolver.solve_ef('glpk', tee=False) # ef_sol = stsolver.solve_ph(subsolver = solvername, default_rho = 1) if ef_sol.solver.termination_condition != TerminationCondition.optimal: print ("oops! not optimal:",ef_sol.solver.termination_condition) #There is an iterator to loop over the root node solution: for varname, varval in stsolver.root_Var_solution(): if varname == "weight": weight = varval else: eta = varval #print (varname, str(varval)) #a function to compute compute the objective function value obj = stsolver.root_E_obj() #print ("Expecatation take over scenarios=", obj) # write down scenario tree in testcref file #csvw.write_csv_soln(stsolver.scenario_tree, str(f)) return obj, weight, eta
def _generate_base_scenario_tree(self, model, variable_stage_assignments): stage_cost_annotation = locate_annotations(model, PySP_StageCostAnnotation, max_allowed=1) if len(stage_cost_annotation) == 0: raise ValueError("Reference model is missing stage cost " "annotation: %s" % (PySP_StageCostAnnotation.__name__)) else: assert len(stage_cost_annotation) == 1 stage_cost_annotation = stage_cost_annotation[0][1] stage_cost_assignments = ComponentMap( stage_cost_annotation.expand_entries()) stage1_cost = None stage2_cost = None for cdata, stagenum in stage_cost_assignments.items(): if stagenum == 1: stage1_cost = cdata elif stagenum == 2: stage2_cost = cdata if stage1_cost is None: raise ValueError("Missing stage cost annotation for time stage: 1") if stage2_cost is None: raise ValueError("Missing stage cost annotation for time stage: 2") assert stage1_cost != stage2_cost # # Create a dummy 1-scenario scenario tree # stm = CreateAbstractScenarioTreeModel() stm.Stages.add('Stage1') stm.Stages.add('Stage2') stm.Nodes.add('RootNode') stm.Nodes.add('LeafNode') stm.Scenarios.add('ReferenceScenario') stm = stm.create_instance() stm.NodeStage['RootNode'] = 'Stage1' stm.ConditionalProbability['RootNode'] = 1.0 stm.NodeStage['LeafNode'] = 'Stage2' stm.Children['RootNode'].add('LeafNode') stm.Children['LeafNode'].clear() stm.ConditionalProbability['LeafNode'] = 1.0 stm.ScenarioLeafNode['ReferenceScenario'] = 'LeafNode' stm.StageCost['Stage1'] = stage1_cost.name stm.StageCost['Stage2'] = stage2_cost.name for var, (stagenum, derived) in variable_stage_assignments.items(): stagelabel = 'Stage' + str(stagenum) if not derived: stm.StageVariables[stagelabel].add(var.name) else: stm.StageDerivedVariables[second_stage].add(var.name) scenario_tree = ScenarioTree(scenariotreeinstance=stm) scenario_tree.linkInInstances( {'ReferenceScenario': self.reference_model}) return scenario_tree
def _generate_base_scenario_tree(self, model, variable_stage_assignments): stage_cost_annotation = locate_annotations( model, PySP_StageCostAnnotation, max_allowed=1) if len(stage_cost_annotation) == 0: raise ValueError("Reference model is missing stage cost " "annotation: %s" % (PySP_StageCostAnnotation.__name__)) else: assert len(stage_cost_annotation) == 1 stage_cost_annotation = stage_cost_annotation[0][1] stage_cost_assignments = ComponentMap( stage_cost_annotation.expand_entries()) stage1_cost = None stage2_cost = None for cdata, stagenum in stage_cost_assignments.items(): if stagenum == 1: stage1_cost = cdata elif stagenum == 2: stage2_cost = cdata if stage1_cost is None: raise ValueError("Missing stage cost annotation for time stage: 1") if stage2_cost is None: raise ValueError("Missing stage cost annotation for time stage: 2") assert stage1_cost != stage2_cost # # Create a dummy 1-scenario scenario tree # stm = CreateAbstractScenarioTreeModel() stm.Stages.add('Stage1') stm.Stages.add('Stage2') stm.Nodes.add('RootNode') stm.Nodes.add('LeafNode') stm.Scenarios.add('ReferenceScenario') stm = stm.create_instance() stm.NodeStage['RootNode'] = 'Stage1' stm.ConditionalProbability['RootNode'] = 1.0 stm.NodeStage['LeafNode'] = 'Stage2' stm.Children['RootNode'].add('LeafNode') stm.Children['LeafNode'].clear() stm.ConditionalProbability['LeafNode'] = 1.0 stm.ScenarioLeafNode['ReferenceScenario'] = 'LeafNode' stm.StageCost['Stage1'] = stage1_cost.name stm.StageCost['Stage2'] = stage2_cost.name for var, (stagenum, derived) in variable_stage_assignments.items(): stagelabel = 'Stage'+str(stagenum) if not derived: stm.StageVariables[stagelabel].add(var.name) else: stm.StageDerivedVariables[second_stage].add(var.name) scenario_tree = ScenarioTree(scenariotreeinstance=stm) scenario_tree.linkInInstances( {'ReferenceScenario': self.reference_model}) return scenario_tree
def test_init11(self): self.assertTrue("reference_test_model" not in sys.modules) scenario_tree_model = CreateAbstractScenarioTreeModel().\ create_instance( join(testdatadir, "reference_test_scenario_tree.dat")) scenario_tree_model.ScenarioBasedData = False with ScenarioTreeInstanceFactory(model=reference_test_model, scenario_tree=scenario_tree_model, data=testdatadir) as factory: self.assertTrue(factory.model_directory() is None) self.assertTrue(factory.scenario_tree_directory() is None) self._check_factory(factory) self.assertEqual(factory._closed, True) self.assertEqual(len(factory._archives), 0)
def createScenarioTreeModel(self): """ Construct scenario tree based on abstract scenario tree model for stochastic programming @ In, None @ Out, treeModel, Instance, pyomo scenario tree model """ treeModel = CreateAbstractScenarioTreeModel() if pyoVersion: treeModel = treeModel.create_instance() treeModel.Stages.add('FirstStage') treeModel.Stages.add('SecondStage') treeModel.Nodes.add('RootNode') for i in self.scenarios['scenario_name']: leafNode = 'leaf_' + i treeModel.Nodes.add(leafNode) treeModel.Scenarios.add(i) if not pyoVersion: treeModel = treeModel.create_instance() treeModel.NodeStage['RootNode'] = 'FirstStage' treeModel.ConditionalProbability['RootNode'] = 1.0 for node in treeModel.Nodes: if node != 'RootNode': treeModel.NodeStage[node] = 'SecondStage' treeModel.Children['RootNode'].add(node) treeModel.Children[node].clear() treeModel.ConditionalProbability[node] = self.scenarios['probabilities'][node.replace('leaf_','')] treeModel.ScenarioLeafNode[node.replace('leaf_','')] = node return treeModel
def test_init11(self): self.assertTrue("reference_test_model" not in sys.modules) scenario_tree_model = CreateAbstractScenarioTreeModel().\ create_instance( join(testdatadir, "reference_test_scenario_tree.dat")) scenario_tree_model.ScenarioBasedData = False with ScenarioTreeInstanceFactory( model=reference_test_model, scenario_tree=scenario_tree_model, data_location=testdatadir) as factory: self.assertTrue(factory.model_directory() is None) self.assertTrue(factory.scenario_tree_directory() is None) self._check_factory(factory) self.assertEqual(factory._closed, True) self.assertEqual(len(factory._archives), 0)
def test_init7(self): self.assertTrue("reference_test_model" not in sys.modules) scenario_tree_model = CreateAbstractScenarioTreeModel().\ create_instance( join(testdatadir, "reference_test_scenario_tree.dat")) with self.assertRaises(ValueError): with ScenarioTreeInstanceFactory( model=reference_test_model, scenario_tree=scenario_tree_model) as factory: pass with ScenarioTreeInstanceFactory(model=reference_test_model, scenario_tree=scenario_tree_model, data=testdatadir) as factory: self.assertTrue(factory.model_directory() is None) self.assertTrue(factory.scenario_tree_directory() is None) self._check_factory(factory) self.assertEqual(factory._closed, True) with ScenarioTreeInstanceFactory( model=reference_test_model, scenario_tree=join( testdatadir, "reference_test_scenario_tree.dat")) as factory: self.assertTrue(factory.model_directory() is None) self.assertTrue(factory.scenario_tree_directory() is not None) self._check_factory(factory) self.assertEqual(len(factory._archives), 0)
def test_init12(self): self.assertTrue("reference_test_model" not in sys.modules) scenario_tree_model = CreateAbstractScenarioTreeModel().\ create_instance( join(testdatadir, "reference_test_scenario_tree.dat")) def scenario_model_callback(scenario_tree, scenario_name, node_list): self.assertIs(scenario_tree, scenario_tree_model) instance = reference_test_model.create_instance() if scenario_name == "s1": instance.p = 1.0 elif scenario_name == "s2": instance.p = 2.0 else: assert scenario_name == "s3" instance.p = 3.0 return instance with ScenarioTreeInstanceFactory( model=scenario_model_callback, scenario_tree=scenario_tree_model) as factory: self.assertTrue(factory.model_directory() is None) self.assertTrue(factory.scenario_tree_directory() is None) self._check_factory(factory) self.assertEqual(factory._closed, True) self.assertEqual(len(factory._archives), 0)
def _treemaker(scenlist): """Makes a scenario tree (avoids dependence on daps) Parameters ---------- scenlist (list of `int`): experiment (i.e. scenario) numbers Returns ------- a `ConcreteModel` that is the scenario tree """ num_scenarios = len(scenlist) m = CreateAbstractScenarioTreeModel() m.Stages.add('Stage1') m.Stages.add('Stage2') m.Nodes.add('RootNode') for i in scenlist: m.Nodes.add('LeafNode_Experiment' + str(i)) m.Scenarios.add('Experiment' + str(i)) m = m.create_instance() m.NodeStage['RootNode'] = 'Stage1' m.ConditionalProbability['RootNode'] = 1.0 for node in m.Nodes: if node != 'RootNode': m.NodeStage[node] = 'Stage2' m.Children['RootNode'].add(node) m.Children[node].clear() m.ConditionalProbability[node] = 1.0 / num_scenarios m.ScenarioLeafNode[node.replace('LeafNode_', '')] = node return m
def _import_model_and_scenario_tree(self): model_import, module_name = \ load_external_module(self._model_filename, clear_cache=True) self._model_module = model_import dir_model_import = dir(model_import) self._model_object = None self._model_callback = None if "pysp_instance_creation_callback" in dir_model_import: callback = model_import.pysp_instance_creation_callback if not hasattr(callback, "__call__"): raise TypeError("'pysp_instance_creation_callback' object is " "not callable in model file: %s" % (self._model_filename)) self._model_callback = callback elif "model" in dir_model_import: model = model_import.model if not isinstance(model, (_BlockData, Block)): raise TypeError("'model' object has incorrect type " "in model file: %s" % (self._model_filename)) self._model_object = model else: raise AttributeError( "No 'model' or 'pysp_instance_creation_callback' " "object found in model file: %s" % (self._model_filename)) if self._scenario_tree_filename is None: assert self._scenario_tree_location is None assert self._scenario_tree_directory is None if self._model_object is not None: self._data_directory = self._model_directory if "pysp_scenario_tree_model_callback" in dir_model_import: callback = model_import.pysp_scenario_tree_model_callback if not hasattr(callback, "__call__"): raise TypeError( "'pysp_scenario_tree_model_callback' object is " "not callable in model file: %s" % (self._model_filename)) self._scenario_tree_instance = callback() if not isinstance(self._scenario_tree_instance, (_BlockData, Block)): raise TypeError( "'pysp_scenario_tree_model_callback' returned " "an object that is not of the correct type for " "a Pyomo model (e.g, _BockData, Block): %s" % (type(self._scenario_tree_instance))) else: raise ValueError( "No scenario tree file was given but no function " "named 'pysp_scenario_tree_model_callback' was " "found in the reference model file.") else: self._data_directory = self._scenario_tree_directory self._scenario_tree_instance = \ CreateAbstractScenarioTreeModel().\ create_instance(filename=self._scenario_tree_filename)
def __init__(self, scenario_creator, tree_model=None, scenarios_dir=None, scenario_creator_callback_name=None, tree_model_callback_name=None): ## first, attempt to determine abstract vs concrete ## and get a scenario instance creator ## if callable, a instance creator if callable(scenario_creator): self.pysp_instance_creator = scenario_creator self.abstract = False else: ## else, either and abstract model or a module with a callback if scenario_creator_callback_name is None: scenario_creator_callback_name = 'pysp_instance_creation_callback' module = import_file(scenario_creator) if hasattr(module, scenario_creator_callback_name): self.pysp_instance_creator = \ getattr(module, scenario_creator_callback_name) self.abstract = False else: self.pysp_instance_creator = module.model.create_instance self.abstract = True ## attempt to find and construct a tree model if tree_model is None: if tree_model_callback_name is None: tree_model_callback_name = 'pysp_scenario_tree_model_callback' tree_maker = getattr(module, tree_model_callback_name) tree_model = tree_maker() ## if we get a *.dat file, assume the scenarios are here unless ## otherwise specified if isinstance(tree_model, str): self.tree_model = CreateAbstractScenarioTreeModel(\ ).create_instance(tree_model) self.scenarios_dir = os.path.dirname(tree_model) elif hasnetworkx and isinstance(tree_model, networkx.DiGraph): self.tree_model = ScenarioTreeModelFromNetworkX(tree_model) elif isinstance(tree_model, pyo.ConcreteModel): self.tree_model = tree_model else: raise RuntimeError("Type of tree_model {} unrecongized".format( type(tree_model))) ## set the scenarios_dir if specified, but complain if ## we don't have an abstract model if scenarios_dir is not None: if not self.abstract: raise RuntimeError("An abstract model is required for " "scenarios_dir") self.scenarios_dir = scenarios_dir self._init()
def _treemaker(scenlist): """ Makes a scenario tree (avoids dependence on daps) Parameters ---------- scenlist (list of `int`): experiment (i.e. scenario) numbers Returns ------- a `ConcreteModel` that is the scenario tree """ num_scenarios = len(scenlist) m = CreateAbstractScenarioTreeModel() m.Stages.add('Stage1') m.Stages.add('Stage2') m.Nodes.add('RootNode') for i in scenlist: m.Nodes.add('LeafNode_Experiment'+str(i)) m.Scenarios.add('Experiment'+str(i)) m = m.create_instance() m.NodeStage['RootNode'] = 'Stage1' m.ConditionalProbability['RootNode'] = 1.0 for node in m.Nodes: if node != 'RootNode': m.NodeStage[node] = 'Stage2' m.Children['RootNode'].add(node) m.Children[node].clear() m.ConditionalProbability[node] = 1.0/num_scenarios m.ScenarioLeafNode[node.replace('LeafNode_','')] = node return m
def pysp_scenario_tree_model_callback(): from pyomo.pysp.scenariotree.tree_structure_model import \ CreateAbstractScenarioTreeModel #Model st_model = CreateAbstractScenarioTreeModel() #---------------------------------------- #Basic Spectification #---------------------------------------- #Stages------------------- for i in Root.stages: st_model.Stages.add(i) #Nodes-------------------- for i in sorted(Case.tree.nodes()): st_model.Nodes.add(i) #Scenarios---------------- for i in sorted(Case.names): st_model.Scenarios.add(i) st_model = st_model.create_instance() #---------------------------------------- #Advanced Spectification #---------------------------------------- #Scenario Leaf Node for i in sorted(Case.names): st_model.ScenarioLeafNode[i] = Case.scenario_node[i] #Node attributes for i in st_model.Nodes: #Conditional probability st_model.ConditionalProbability[i] = Case.tree.node[i]['probability'] #Stage st_model.NodeStage[i] = Case.tree.node[i]['stage'] #Children Node for j in Case.tree.edges(i): st_model.Children[i].add(j[1]) #------------------------------------------- #Variable /Objective Spectification #------------------------------------------- for i in Root.variables: st_model.StageVariables[Root.st_to_var[i]].add('x[' + i + ']') for i in Root.stages: st_model.StageCost[i] = 'SC[' + i + ']' return st_model
def test_init6(self): self.assertTrue("reference_test_model" not in sys.modules) scenario_tree_model = CreateAbstractScenarioTreeModel().\ create_instance( join(testdatadir, "reference_test_scenario_tree.dat")) with ScenarioTreeInstanceFactory( model=join(testdatadir, "reference_test_model.py"), scenario_tree=scenario_tree_model) as factory: self.assertTrue(factory.model_directory() is not None) self.assertTrue(factory.scenario_tree_directory() is None) self._check_factory(factory) with self.assertRaises(TypeError): with ScenarioTreeInstanceFactory(model=join( testdatadir, "reference_test_model.py"), scenario_tree=int) as f: pass with self.assertRaises(ValueError): with ScenarioTreeInstanceFactory(model=join( testdatadir, "reference_test_model.py"), scenario_tree=None) as f: pass with self.assertRaises(TypeError): with ScenarioTreeInstanceFactory( model=None, scenario_tree=scenario_tree_model) as f: pass with self.assertRaises(IOError): with ScenarioTreeInstanceFactory( model=join(testdatadir, "reference_test_model_does_not_exist.py"), scenario_tree=scenario_tree_model) as f: pass with self.assertRaises(ValueError): with ScenarioTreeInstanceFactory( model=join(testdatadir, "reference_test_model.py"), scenario_tree=CreateAbstractScenarioTreeModel()) as f: pass self.assertEqual(len(factory._archives), 0) self.assertTrue("reference_test_model" in sys.modules)
def test_init14(self): self.assertTrue("reference_test_model" not in sys.modules) scenario_tree_model = CreateAbstractScenarioTreeModel().\ create_instance( join(testdatadir, "reference_test_scenario_tree.dat")) with ScenarioTreeInstanceFactory(model=reference_test_model, scenario_tree=scenario_tree_model, data=testdatadir) as factory: scenario_tree = factory.generate_scenario_tree() self.assertEqual(factory._closed, True) self.assertEqual(len(factory._archives), 0) # start with a scenario tree (not a scenario tree model) with ScenarioTreeInstanceFactory(model=reference_test_model, scenario_tree=scenario_tree, data=testdatadir) as factory: self._check_factory(factory) self.assertEqual(factory._closed, True) self.assertEqual(len(factory._archives), 0)
def test_init13(self): model = reference_test_model.create_instance() scenario_tree_model = CreateAbstractScenarioTreeModel().\ create_instance( join(testdatadir, "reference_test_scenario_tree.dat")) with ScenarioTreeInstanceFactory( model=model, scenario_tree=scenario_tree_model) as factory: self.assertTrue(factory.model_directory() is None) self.assertTrue(factory.scenario_tree_directory() is None) scenario_tree = factory.generate_scenario_tree() instances = factory.construct_instances_for_scenario_tree( scenario_tree, verbose=True) self.assertEqual(len(instances), 3) self.assertEqual(instances["s1"].p(), model.p()) self.assertEqual(instances["s2"].p(), model.p()) self.assertEqual(instances["s3"].p(), model.p()) self.assertEqual(factory._closed, True) self.assertEqual(len(factory._archives), 0)
def return_CP_and_path(p_data): # return_CP_and_path(p_data) -> dict(), dict() # This function reads the path to the instance directory (p_data) and # returns conditional two dictionaries, the first one is the conditional # probability of a scenario, the second one is the path to all files of a # scenario. from collections import deque, defaultdict # from pyomo.pysp.util.scenariomodels import scenario_tree_model from pyomo.pysp.scenariotree.tree_structure_model import \ CreateAbstractScenarioTreeModel pwd = os.getcwd() os.chdir(p_data) s2fp_dict = defaultdict( deque) # Scenario to 'file path' dictionary, .dat not included s2cd_dict = defaultdict(float) # Scenario to conditonal density mapping # sStructure = scenario_tree_model.create_instance( filename='ScenarioStructure.dat' ) sStructure = CreateAbstractScenarioTreeModel().create_instance( filename='ScenarioStructure.dat') # The following code is borrowed from Kevin's temoa_lib.py ########################################################################### # Step 1: find the root node. PySP doesn't make this very easy ... # a child -> parent mapping, because every child has only one parent, but # not vice-versa ctpTree = dict() # Child to parent dict, one to one mapping to_process = deque() to_process.extend(sStructure.Children.keys()) while to_process: node = to_process.pop() if node in sStructure.Children: # it's a parent! new_nodes = set(sStructure.Children[node]) to_process.extend(new_nodes) ctpTree.update({n: node for n in new_nodes}) # parents - children root_node = (set(ctpTree.values()) - set(ctpTree.keys())).pop() # ptcTree = defaultdict( list ) # Parent to child node, one to multiple mapping # for c, p in ctpTree.items(): # ptcTree[ p ].append( c ) # ptcTree = dict( ptcTree ) # be slightly defensive; catch any additions # leaf_nodes = set(ctpTree.keys()) - set(ctpTree.values()) # leaf_nodes = set(sStructure.ScenarioLeafNode.values()) # Try to hack Kevin's code leaf_nodes = sStructure.ScenarioLeafNode.values( ) # Try to hack Kevin's code leaf_nodes_names = list() for n in leaf_nodes: leaf_nodes_names.append(n.value) leaf_nodes_names = set(leaf_nodes_names) scenario_nodes = dict() # Map from leafnode to 'node path' for node in leaf_nodes_names: # e.g.: {Rs0s0: [R, Rs0, Rs0s0]} s = deque() scenario_nodes[node] = s while node in ctpTree: s.append(node) node = ctpTree[node] s.append(node) s.reverse() ########################################################################### for s in sStructure.Scenarios: cp = 1.0 # Starting probability for n in scenario_nodes[value(sStructure.ScenarioLeafNode[s])]: cp = cp * value(sStructure.ConditionalProbability[n]) if not sStructure.ScenarioBasedData.value: s2fp_dict[s].append(n + '.dat') s2cd_dict[s] = cp from pyomo.core import Objective if sStructure.ScenarioBasedData.value: for s in sStructure.Scenarios: s2fp_dict[s].append(s + '.dat') os.chdir(pwd) return (s2cd_dict, s2fp_dict)
root 1.0 n1 0.33333333 n2 0.33333334 n3 0.33333333 ; set Scenarios := s1 s2 s3 ; param ScenarioLeafNode := s1 n1 s2 n2 s3 n3 ; set StageVariables[t1] := x ; param StageCost := t1 cost[1] t2 cost[2] ; """) model = CreateAbstractScenarioTreeModel().create_instance(f.name) os.remove(f.name)
class ScenarioTreeInstanceFactory(object): def __init__(self, model, scenario_tree, data=None): """Class to help manage construction of scenario tree models. This class is designed to help manage the various input formats that that are accepted by PySP and provide a unified interface for building scenario trees that are paired with a set of concrete Pyomo models. Args: model: The reference scenario model. Can be set to Pyomo model or the name of a file containing a Pyomo model. For historical reasons, this argument can also be set to a directory name where it is assumed a file named ReferenceModel.py exists. scenario_tree: The scenario tree. Can be set to a Pyomo model, a file containing a Pyomo model, or a .dat file containing data for an abstract scenario tree model representation, which defines the structure of the scenario tree. It can also be a .py file that contains a networkx scenario tree or a networkx scenario tree object. For historical reasons, this argument can also be set to a directory name where it is assumed a file named ScenarioStructure.dat exists. data: Directory containing .dat files necessary for building the scenario instances associated with the scenario tree. This argument is required if no directory information can be extracted from the first two arguments and the reference model is an abstract Pyomo model. Otherwise, it is not required or the location will be inferred from the scenario tree location (first) or from the reference model location (second), where it is assumed the data files reside in the same directory. """ self._closed = True self._archives = [] self._model_filename = None self._model_module = None self._model_object = None self._model_callback = None self._scenario_tree_filename = None self._scenario_tree_module = None self._scenario_tree_model = None self._scenario_tree = None self._data_directory = None try: self._init(model, scenario_tree, data) except: self.close() raise self._closed = False def _init(self, model, scenario_tree, data): self._model_filename = None self._model_module = None self._model_object = None self._model_callback = None if isinstance(model, six.string_types): logger.debug("A model filename was provided.") self._model_filename, self._archives = \ _extract_pathspec(model, "ReferenceModel.py", archives=self._archives) if not os.path.exists(self._model_filename): logger.error("Failed to extract reference model python file " "from path specification: %s" % (model)) raise IOError("path does not exist: %s" % (self._model_filename)) assert self._model_filename is not None assert self._model_filename.endswith(".py") (self._model_module, self._model_object, self._model_callback) = \ _find_reference_model_or_callback(self._model_filename) if (self._model_object is None) and \ (self._model_callback is None): raise AttributeError("No reference Pyomo model or " "'pysp_instance_creation_callback' " "function object found in src: %s" % (self._model_filename)) elif hasattr(model, "__call__"): logger.debug("A model callback function was provided.") self._model_callback = model else: if not isinstance(model, (_BlockData, Block)): raise TypeError( "model argument object has incorrect type: %s. " "Must be a string type, a callback, or a Pyomo model." % (type(model))) logger.debug("A model object was provided.") self._model_object = model self._scenario_tree_filename = None self._scenario_tree_model = None self._scenario_tree = None if isinstance(scenario_tree, ScenarioTree): for scenario in scenario_tree.scenarios: if scenario.instance is not None: raise ValueError( "The scenario tree can not be linked with instances") if hasattr(scenario_tree, "_scenario_instance_factory"): del scenario_tree._scenario_instance_factory self._scenario_tree = scenario_tree elif has_networkx and \ isinstance(scenario_tree, networkx.DiGraph): self._scenario_tree_model = scenario_tree elif isinstance(scenario_tree, six.string_types): logger.debug("scenario tree input is a string, attempting " "to load file specification: %s" % (scenario_tree)) self._scenario_tree_filename = None if not scenario_tree.endswith(".py"): self._scenario_tree_filename, self._archives = \ _extract_pathspec(scenario_tree, "ScenarioStructure.dat", archives=self._archives) if not os.path.exists(self._scenario_tree_filename): logger.debug("Failed to extract scenario tree structure " ".dat file from path specification: %s" % (scenario_tree)) self._scenario_tree_filename = None if self._scenario_tree_filename is None: self._scenario_tree_filename, self._archives = \ _extract_pathspec(scenario_tree, "ScenarioStructure.py", archives=self._archives) if not os.path.exists(self._scenario_tree_filename): logger.debug("Failed to locate scenario tree structure " ".py file with path specification: %s" % (scenario_tree)) self._scenario_tree_filename = None if self._scenario_tree_filename is None: raise ValueError("Failed to extract scenario tree structure " "file with .dat or .py extension from path " "specification: %s" % (scenario_tree)) elif self._scenario_tree_filename.endswith(".py"): if self._scenario_tree_filename == self._model_filename: # try not to clobber the model import (self._scenario_tree_module, self._scenario_tree, self._scenario_tree_model) = \ _find_scenariotree(module=self._model_module) else: (self._scenario_tree_module, self._scenario_tree, self._scenario_tree_model) = \ _find_scenariotree(src=self._scenario_tree_filename) if (self._scenario_tree is None) and \ (self._scenario_tree_model is None): raise AttributeError("No scenario tree or " "'pysp_scenario_tree_model_callback' " "function found in src: %s" % (self._scenario_tree_filename)) elif self._scenario_tree_filename.endswith(".dat"): self._scenario_tree_model = \ CreateAbstractScenarioTreeModel().\ create_instance(filename=self._scenario_tree_filename) else: assert False elif scenario_tree is None: if self._model_module is not None: self._scenario_tree_filename = self._model_filename (self._scenario_tree_module, self._scenario_tree, self._scenario_tree_model) = \ _find_scenariotree(module=self._model_module) if (self._scenario_tree is None) and \ (self._scenario_tree_model is None): raise ValueError( "No input was provided for the scenario tree " "and no callback or scenario tree object was " "found with the model") else: raise ValueError( "No input was provided for the scenario tree " "but there is no module to search for a " "'pysp_scenario_tree_model_callback' function " "or a ScenarioTree object.") else: self._scenario_tree_model = scenario_tree if self._scenario_tree is None: if (not isinstance(self._scenario_tree_model, (_BlockData, Block))) and \ ((not has_networkx) or \ (not isinstance(self._scenario_tree_model, networkx.DiGraph))): raise TypeError( "scenario tree model object has incorrect type: %s. " "Must be a string type, Pyomo model, or a " "networkx.DiGraph object." % (type(scenario_tree))) if isinstance(self._scenario_tree_model, (_BlockData, Block)): if not self._scenario_tree_model.is_constructed(): raise ValueError("scenario tree model is not constructed") self._data_directory = None if data is None: if self.scenario_tree_directory() is not None: logger.debug("data directory is set to the scenario tree " "directory: %s" % (self.scenario_tree_directory())) self._data_directory = self.scenario_tree_directory() elif self.model_directory() is not None: logger.debug("data directory is set to the reference model " "directory: %s" % (self.model_directory())) self._data_directory = self.model_directory() else: if (self._model_callback is None) and \ isinstance(self._model_object, AbstractModel) and \ (not self._model_object.is_constructed()): raise ValueError( "A data location is required since no model " "callback was provided and no other location could " "be inferred.") logger.debug("no data directory is required") else: logger.debug("data location is provided, attempting " "to load specification: %s" % (data)) self._data_directory, self._archives = \ _extract_pathspec(data, None, archives=self._archives) if not os.path.exists(self._data_directory): logger.error("Failed to extract data directory " "from path specification: %s" % (data)) raise IOError("path does not exist: %s" % (self._data_directory)) def __getstate__(self): self.close() raise NotImplementedError("Do not deepcopy or serialize this class") def __setstate__(self, d): self.close() raise NotImplementedError("Do not deepcopy or serialize this class") def close(self): for _, archive, tmpdir in self._archives: if os.path.exists(tmpdir): shutil.rmtree(tmpdir, True) archive.close() self._archives = [] self._closed = True # # Support "with" statements. Forgetting to call close() # on this class can result in temporary unarchived # directories being left sitting around # def __enter__(self): return self def __exit__(self, type, value, traceback): self.close() def model_directory(self): if self._model_filename is not None: return os.path.dirname(self._model_filename) else: return None def scenario_tree_directory(self): if self._scenario_tree_filename is not None: return os.path.dirname(self._scenario_tree_filename) else: return None def data_directory(self): return self._data_directory # # construct a scenario instance - just like it sounds! # def construct_scenario_instance(self, scenario_name, scenario_tree, profile_memory=False, output_instance_construction_time=False, compile_instance=False, verbose=False): assert not self._closed if not scenario_tree.contains_scenario(scenario_name): raise ValueError("ScenarioTree does not contain scenario " "with name %s." % (scenario_name)) scenario = scenario_tree.get_scenario(scenario_name) node_name_list = [n._name for n in scenario._node_list] if verbose: print("Creating instance for scenario=%s" % (scenario_name)) scenario_instance = None try: if self._model_callback is not None: assert self._model_object is None try: _scenario_tree_arg = None # new callback signature if (self._scenario_tree_filename is not None) and \ self._scenario_tree_filename.endswith('.dat'): # we started with a .dat file, so # send the PySP scenario tree _scenario_tree_arg = scenario_tree elif self._scenario_tree_model is not None: # We started from a Pyomo # scenario tree model instance, or a # networkx tree. _scenario_tree_arg = self._scenario_tree_model else: # send the PySP scenario tree _scenario_tree_arg = scenario_tree scenario_instance = self._model_callback( _scenario_tree_arg, scenario_name, node_name_list) except TypeError: # old callback signature # TODO: #logger.warning( # "DEPRECATED: The 'pysp_instance_creation_callback' function " # "signature has changed. An additional argument should be " # "added to the beginning of the arguments list that will be " # "set to the user provided scenario tree object when called " # "by PySP (e.g., a Pyomo scenario tree model instance, " # "a networkx tree, or a PySP ScenarioTree object.") scenario_instance = self._model_callback( scenario_name, node_name_list) elif self._model_object is not None: if (not isinstance(self._model_object, AbstractModel)) or \ (self._model_object.is_constructed()): scenario_instance = self._model_object.clone() elif scenario_tree._scenario_based_data: assert self.data_directory() is not None scenario_data_filename = \ os.path.join(self.data_directory(), str(scenario_name)) # JPW: The following is a hack to support # initialization of block instances, which # don't work with .dat files at the # moment. Actually, it's not that bad of a # hack - it just needs to be extended a bit, # and expanded into the node-based data read # logic (where yaml is completely ignored at # the moment. if os.path.exists(scenario_data_filename + '.dat'): scenario_data_filename = \ scenario_data_filename + ".dat" data = None elif os.path.exists(scenario_data_filename + '.yaml'): if not yaml_available: raise ValueError( "Found yaml data file for scenario '%s' " "but he PyYAML module is not available" % (scenario_name)) scenario_data_filename = \ scenario_data_filename+".yaml" with open(scenario_data_filename) as f: data = yaml.load(f, **yaml_load_args) else: raise RuntimeError( "Cannot find a data file for scenario '%s' " "in directory: %s\nRecognized formats: .dat, " ".yaml" % (scenario_name, self.data_directory())) if verbose: print("Data for scenario=%s loads from file=%s" % (scenario_name, scenario_data_filename)) if data is None: scenario_instance = \ self._model_object.create_instance( filename=scenario_data_filename, profile_memory=profile_memory, report_timing=output_instance_construction_time) else: scenario_instance = \ self._model_object.create_instance( data, profile_memory=profile_memory, report_timing=output_instance_construction_time) else: assert self.data_directory() is not None data_files = [] for node_name in node_name_list: node_data_filename = \ os.path.join(self.data_directory(), str(node_name)+".dat") if not os.path.exists(node_data_filename): raise RuntimeError( "Cannot find a data file for scenario tree " "node '%s' in directory: %s\nRecognized " "formats: .dat" % (node_name, self.data_directory())) data_files.append(node_data_filename) scenario_data = DataPortal(model=self._model_object) for data_file in data_files: if verbose: print("Node data for scenario=%s partially " "loading from file=%s" % (scenario_name, data_file)) scenario_data.load(filename=data_file) scenario_instance = self._model_object.create_instance( scenario_data, profile_memory=profile_memory, report_timing=output_instance_construction_time) else: raise RuntimeError("Unable to construct scenario instance. " "Neither a reference model or callback " "is defined.") # name each instance with the scenario name scenario_instance._name = scenario_name # apply each of the post-instance creation plugins. this # really shouldn't be associated (in terms of naming) with the # pyomo script - this should be rectified with a workflow # re-work. it is unclear how this interacts, or doesn't, with # the preprocessors. ep = ExtensionPoint(IPyomoScriptModifyInstance) for ep in ExtensionPoint(IPyomoScriptModifyInstance): logger.warning( "DEPRECATED: IPyomoScriptModifyInstance extension " "point callbacks will be ignored by PySP in the future") ep.apply(options=None, model=reference_model, instance=scenario_instance) if compile_instance: from pyomo.repn.beta.matrix import \ compile_block_linear_constraints compile_block_linear_constraints( scenario_instance, "_PySP_compiled_linear_constraints", verbose=verbose) except: logger.error("Failed to create model instance for scenario=%s" % (scenario_name)) raise return scenario_instance def construct_instances_for_scenario_tree( self, scenario_tree, profile_memory=False, output_instance_construction_time=False, compile_scenario_instances=False, verbose=False): assert not self._closed if scenario_tree._scenario_based_data: if verbose: print("Scenario-based instance initialization enabled") else: if verbose: print("Node-based instance initialization enabled") scenario_instances = {} for scenario in scenario_tree._scenarios: # the construction of instances takes little overhead in terms # of memory potentially lost in the garbage-collection sense # (mainly only that due to parsing and instance # simplification/prep-processing). to speed things along, # disable garbage collection if it enabled in the first place # through the instance construction process. # IDEA: If this becomes too much for truly large numbers of # scenarios, we could manually collect every time X # instances have been created. scenario_instance = None with PauseGC() as pgc: scenario_instance = \ self.construct_scenario_instance( scenario._name, scenario_tree, profile_memory=profile_memory, output_instance_construction_time=output_instance_construction_time, compile_instance=compile_scenario_instances, verbose=verbose) scenario_instances[scenario._name] = scenario_instance assert scenario_instance.local_name == scenario.name return scenario_instances def generate_scenario_tree(self, downsample_fraction=1.0, include_scenarios=None, bundles=None, random_bundles=None, random_seed=None, verbose=True): scenario_tree_model = self._scenario_tree_model if scenario_tree_model is not None: if has_networkx and \ isinstance(scenario_tree_model, networkx.DiGraph): scenario_tree_model = \ ScenarioTreeModelFromNetworkX(scenario_tree_model) else: assert isinstance(scenario_tree_model, (_BlockData, Block)), \ str(scenario_tree_model)+" "+str(type(scenario_tree_model)) if bundles is not None: if isinstance(bundles, six.string_types): if scenario_tree_model is None: raise ValueError("A bundles file can not be used when the " "scenario tree input was not a Pyomo " "model or ScenarioStructure.dat file.") logger.debug("attempting to locate bundle file for input: %s" % (bundles)) # we interpret the scenario bundle specification in one of # two ways. if the supplied name is a file, it is used # directly. otherwise, it is interpreted as the root of a # file with a .dat suffix to be found in the instance # directory. orig_input = bundles if not bundles.endswith(".dat"): bundles = bundles + ".dat" bundles = os.path.expanduser(bundles) if not os.path.exists(bundles): if self.data_directory() is None: raise ValueError( "Could not locate bundle .dat file from input " "'%s'. Path does not exist and there is no data " "directory to search in." % (orig_input)) bundles = os.path.join(self.data_directory(), bundles) if not os.path.exists(bundles): raise ValueError("Could not locate bundle .dat file " "from input '%s' as absolute path or " "relative to data directory: %s" % (orig_input, self.data_directory())) if verbose: print("Scenario tree bundle specification filename=%s" % (bundles)) scenario_tree_model = scenario_tree_model.clone() scenario_tree_model.Bundling = True scenario_tree_model.Bundling._constructed = False scenario_tree_model.Bundling._data.clear() scenario_tree_model.Bundles.clear() scenario_tree_model.Bundles._constructed = False scenario_tree_model.Bundles._data.clear() scenario_tree_model.BundleScenarios.clear() scenario_tree_model.BundleScenarios._constructed = False scenario_tree_model.BundleScenarios._data.clear() scenario_tree_model.load(bundles) # # construct the scenario tree # if scenario_tree_model is not None: scenario_tree = ScenarioTree( scenariotreeinstance=scenario_tree_model, scenariobundlelist=include_scenarios) else: assert self._scenario_tree is not None if include_scenarios is None: scenario_tree = copy.deepcopy(self._scenario_tree) else: # note: any bundles will be lost if self._scenario_tree.contains_bundles(): raise ValueError("Can not compress a scenario tree that " "contains bundles") scenario_tree = self._scenario_tree.make_compressed( include_scenarios, normalize=True) # compress/down-sample the scenario tree, if requested if (downsample_fraction is not None) and \ (downsample_fraction < 1.0): scenario_tree.downsample(downsample_fraction, random_seed, verbose) # # create bundles from a dict, if requested # if bundles is not None: if not isinstance(bundles, six.string_types): if verbose: print("Adding bundles to scenario tree from " "user-specified dict") if scenario_tree.contains_bundles(): if verbose: print("Scenario tree already contains bundles. " "All existing bundles will be removed.") for bundle in list(scenario_tree.bundles): scenario_tree.remove_bundle(bundle.name) for bundle_name in bundles: scenario_tree.add_bundle(bundle_name, bundles[bundle_name]) # # create random bundles, if requested # if (random_bundles is not None) and \ (random_bundles > 0): if bundles is not None: raise ValueError("Cannot specify both random " "bundles and a bundles specification") num_scenarios = len(scenario_tree._scenarios) if random_bundles > num_scenarios: raise ValueError("Cannot create more random bundles " "than there are scenarios!") if verbose: print("Creating " + str(random_bundles) + " random bundles using seed=" + str(random_seed)) scenario_tree.create_random_bundles(random_bundles, random_seed) scenario_tree._scenario_instance_factory = self return scenario_tree
def _init(self, model, scenario_tree, data): self._model_filename = None self._model_module = None self._model_object = None self._model_callback = None if isinstance(model, six.string_types): logger.debug("A model filename was provided.") self._model_filename, self._archives = \ _extract_pathspec(model, "ReferenceModel.py", archives=self._archives) if not os.path.exists(self._model_filename): logger.error("Failed to extract reference model python file " "from path specification: %s" % (model)) raise IOError("path does not exist: %s" % (self._model_filename)) assert self._model_filename is not None assert self._model_filename.endswith(".py") (self._model_module, self._model_object, self._model_callback) = \ _find_reference_model_or_callback(self._model_filename) if (self._model_object is None) and \ (self._model_callback is None): raise AttributeError( "No reference Pyomo model or " "'pysp_instance_creation_callback' " "function object found in src: %s" % (self._model_filename)) elif hasattr(model, "__call__"): logger.debug("A model callback function was provided.") self._model_callback = model else: if not isinstance(model, (_BlockData, Block)): raise TypeError( "model argument object has incorrect type: %s. " "Must be a string type, a callback, or a Pyomo model." % (type(model))) logger.debug("A model object was provided.") self._model_object = model self._scenario_tree_filename = None self._scenario_tree_model = None self._scenario_tree = None if isinstance(scenario_tree, ScenarioTree): for scenario in scenario_tree.scenarios: if scenario.instance is not None: raise ValueError( "The scenario tree can not be linked with instances") if hasattr(scenario_tree, "_scenario_instance_factory"): del scenario_tree._scenario_instance_factory self._scenario_tree = scenario_tree elif has_networkx and \ isinstance(scenario_tree, networkx.DiGraph): self._scenario_tree_model = scenario_tree elif isinstance(scenario_tree, six.string_types): logger.debug("scenario tree input is a string, attempting " "to load file specification: %s" % (scenario_tree)) self._scenario_tree_filename = None if not scenario_tree.endswith(".py"): self._scenario_tree_filename, self._archives = \ _extract_pathspec(scenario_tree, "ScenarioStructure.dat", archives=self._archives) if not os.path.exists(self._scenario_tree_filename): logger.debug("Failed to extract scenario tree structure " ".dat file from path specification: %s" % (scenario_tree)) self._scenario_tree_filename = None if self._scenario_tree_filename is None: self._scenario_tree_filename, self._archives = \ _extract_pathspec(scenario_tree, "ScenarioStructure.py", archives=self._archives) if not os.path.exists(self._scenario_tree_filename): logger.debug("Failed to locate scenario tree structure " ".py file with path specification: %s" % (scenario_tree)) self._scenario_tree_filename = None if self._scenario_tree_filename is None: raise ValueError("Failed to extract scenario tree structure " "file with .dat or .py extension from path " "specification: %s" % (scenario_tree)) elif self._scenario_tree_filename.endswith(".py"): if self._scenario_tree_filename == self._model_filename: # try not to clobber the model import (self._scenario_tree_module, self._scenario_tree, self._scenario_tree_model) = \ _find_scenariotree(module=self._model_module) else: (self._scenario_tree_module, self._scenario_tree, self._scenario_tree_model) = \ _find_scenariotree(src=self._scenario_tree_filename) if (self._scenario_tree is None) and \ (self._scenario_tree_model is None): raise AttributeError( "No scenario tree or " "'pysp_scenario_tree_model_callback' " "function found in src: %s" % (self._scenario_tree_filename)) elif self._scenario_tree_filename.endswith(".dat"): self._scenario_tree_model = \ CreateAbstractScenarioTreeModel().\ create_instance(filename=self._scenario_tree_filename) else: assert False elif scenario_tree is None: if self._model_module is not None: self._scenario_tree_filename = self._model_filename (self._scenario_tree_module, self._scenario_tree, self._scenario_tree_model) = \ _find_scenariotree(module=self._model_module) if (self._scenario_tree is None) and \ (self._scenario_tree_model is None): raise ValueError( "No input was provided for the scenario tree " "and no callback or scenario tree object was " "found with the model") else: raise ValueError( "No input was provided for the scenario tree " "but there is no module to search for a " "'pysp_scenario_tree_model_callback' function " "or a ScenarioTree object.") else: self._scenario_tree_model = scenario_tree if self._scenario_tree is None: if (not isinstance(self._scenario_tree_model, (_BlockData, Block))) and \ ((not has_networkx) or \ (not isinstance(self._scenario_tree_model, networkx.DiGraph))): raise TypeError( "scenario tree model object has incorrect type: %s. " "Must be a string type, Pyomo model, or a " "networkx.DiGraph object." % (type(scenario_tree))) if isinstance(self._scenario_tree_model, (_BlockData, Block)): if not self._scenario_tree_model.is_constructed(): raise ValueError( "scenario tree model is not constructed") self._data_directory = None if data is None: if self.scenario_tree_directory() is not None: logger.debug("data directory is set to the scenario tree " "directory: %s" % (self.scenario_tree_directory())) self._data_directory = self.scenario_tree_directory() elif self.model_directory() is not None: logger.debug("data directory is set to the reference model " "directory: %s" % (self.model_directory())) self._data_directory = self.model_directory() else: if (self._model_callback is None) and \ isinstance(self._model_object, AbstractModel) and \ (not self._model_object.is_constructed()): raise ValueError( "A data location is required since no model " "callback was provided and no other location could " "be inferred.") logger.debug("no data directory is required") else: logger.debug("data location is provided, attempting " "to load specification: %s" % (data)) self._data_directory, self._archives = \ _extract_pathspec(data, None, archives=self._archives) if not os.path.exists(self._data_directory): logger.error("Failed to extract data directory " "from path specification: %s" % (data)) raise IOError("path does not exist: %s" % (self._data_directory))
def _init(self, model, scenario_tree, data): self._model_filename = None self._model_module = None self._model_object = None self._model_callback = None if isinstance(model, six.string_types): logger.debug("A model filename was provided.") self._model_filename, self._archives = \ _extract_pathspec(model, "ReferenceModel.py", archives=self._archives) if not os.path.exists(self._model_filename): logger.error("Failed to extract reference model python file " "from path specification: %s" % (model)) raise IOError("path does not exist: %s" % (self._model_filename)) assert self._model_filename is not None assert self._model_filename.endswith(".py") (self._model_module, self._model_object, self._model_callback) = \ _find_reference_model_or_callback(self._model_filename) if (self._model_object is None) and \ (self._model_callback is None): raise AttributeError("No reference Pyomo model or " "'pysp_instance_creation_callback' " "function object found in src: %s" % (self._model_filename)) elif hasattr(model, "__call__"): logger.debug("A model callback function was provided.") self._model_callback = model else: if not isinstance(model, (_BlockData, Block)): raise TypeError( "model argument object has incorrect type: %s. " "Must be a string type, a callback, or a Pyomo model." % (type(model))) logger.debug("A model object was provided.") self._model_object = model self._scenario_tree_filename = None self._scenario_tree_model = None self._scenario_tree = None if isinstance(scenario_tree, ScenarioTree): for scenario in scenario_tree.scenarios: if scenario.instance is not None: raise ValueError( "The scenario tree can not be linked with instances") if hasattr(scenario_tree, "_scenario_instance_factory"): del scenario_tree._scenario_instance_factory self._scenario_tree = scenario_tree elif has_networkx and \ isinstance(scenario_tree, networkx.DiGraph): self._scenario_tree_model = scenario_tree elif isinstance(scenario_tree, six.string_types): logger.debug("scenario tree input is a string, attempting " "to load file specification: %s" % (scenario_tree)) self._scenario_tree_filename = None if not scenario_tree.endswith(".py"): self._scenario_tree_filename, self._archives = \ _extract_pathspec(scenario_tree, "ScenarioStructure.dat", archives=self._archives) if not os.path.exists(self._scenario_tree_filename): logger.debug("Failed to extract scenario tree structure " ".dat file from path specification: %s" % (scenario_tree)) self._scenario_tree_filename = None if self._scenario_tree_filename is None: self._scenario_tree_filename, self._archives = \ _extract_pathspec(scenario_tree, "ScenarioStructure.py", archives=self._archives) if not os.path.exists(self._scenario_tree_filename): logger.debug("Failed to locate scenario tree structure " ".py file with path specification: %s" % (scenario_tree)) self._scenario_tree_filename = None if self._scenario_tree_filename is None: raise ValueError("Failed to extract scenario tree structure " "file with .dat or .py extension from path " "specification: %s" % (scenario_tree)) elif self._scenario_tree_filename.endswith(".py"): if self._scenario_tree_filename == self._model_filename: # try not to clobber the model import (self._scenario_tree_module, self._scenario_tree, self._scenario_tree_model) = \ _find_scenariotree(module=self._model_module) else: (self._scenario_tree_module, self._scenario_tree, self._scenario_tree_model) = \ _find_scenariotree(src=self._scenario_tree_filename) if (self._scenario_tree is None) and \ (self._scenario_tree_model is None): raise AttributeError("No scenario tree or " "'pysp_scenario_tree_model_callback' " "function found in src: %s" % (self._scenario_tree_filename)) elif self._scenario_tree_filename.endswith(".dat"): self._scenario_tree_model = \ CreateAbstractScenarioTreeModel().\ create_instance(filename=self._scenario_tree_filename) else: assert False elif scenario_tree is None: if self._model_module is not None: self._scenario_tree_filename = self._model_filename (self._scenario_tree_module, self._scenario_tree, self._scenario_tree_model) = \ _find_scenariotree(module=self._model_module) if (self._scenario_tree is None) and \ (self._scenario_tree_model is None): raise ValueError( "No input was provided for the scenario tree " "and no callback or scenario tree object was " "found with the model") else: raise ValueError( "No input was provided for the scenario tree " "but there is no module to search for a " "'pysp_scenario_tree_model_callback' function " "or a ScenarioTree object.") else: self._scenario_tree_model = scenario_tree if self._scenario_tree is None: if (not isinstance(self._scenario_tree_model, (_BlockData, Block))) and \ ((not has_networkx) or \ (not isinstance(self._scenario_tree_model, networkx.DiGraph))): raise TypeError( "scenario tree model object has incorrect type: %s. " "Must be a string type, Pyomo model, or a " "networkx.DiGraph object." % (type(scenario_tree))) if isinstance(self._scenario_tree_model, (_BlockData, Block)): if not self._scenario_tree_model.is_constructed(): raise ValueError("scenario tree model is not constructed") self._data_directory = None if data is None: if self.scenario_tree_directory() is not None: logger.debug("data directory is set to the scenario tree " "directory: %s" % (self.scenario_tree_directory())) self._data_directory = self.scenario_tree_directory() elif self.model_directory() is not None: logger.debug("data directory is set to the reference model " "directory: %s" % (self.model_directory())) self._data_directory = self.model_directory() else: if (self._model_callback is None) and \ isinstance(self._model_object, AbstractModel) and \ (not self._model_object.is_constructed()): raise ValueError( "A data location is required since no model " "callback was provided and no other location could " "be inferred.") logger.debug("no data directory is required") else: logger.debug("data location is provided, attempting " "to load specification: %s" % (data)) self._data_directory, self._archives = \ _extract_pathspec(data, None, archives=self._archives) if not os.path.exists(self._data_directory): logger.error("Failed to extract data directory " "from path specification: %s" % (data)) raise IOError("path does not exist: %s" % (self._data_directory))
class ScenarioTreeInstanceFactory(object): def __init__(self, model, scenario_tree, data=None): """Class to help manage construction of scenario tree models. This class is designed to help manage the various input formats that that are accepted by PySP and provide a unified interface for building scenario trees that are paired with a set of concrete Pyomo models. Args: model: The reference scenario model. Can be set to Pyomo model or the name of a file containing a Pyomo model. For historical reasons, this argument can also be set to a directory name where it is assumed a file named ReferenceModel.py exists. scenario_tree: The scenario tree. Can be set to a Pyomo model, a file containing a Pyomo model, or a .dat file containing data for an abstract scenario tree model representation, which defines the structure of the scenario tree. It can also be a .py file that contains a networkx scenario tree or a networkx scenario tree object. For historical reasons, this argument can also be set to a directory name where it is assumed a file named ScenarioStructure.dat exists. data: Directory containing .dat files necessary for building the scenario instances associated with the scenario tree. This argument is required if no directory information can be extracted from the first two arguments and the reference model is an abstract Pyomo model. Otherwise, it is not required or the location will be inferred from the scenario tree location (first) or from the reference model location (second), where it is assumed the data files reside in the same directory. """ self._closed = True self._archives = [] self._model_filename = None self._model_module = None self._model_object = None self._model_callback = None self._scenario_tree_filename = None self._scenario_tree_module = None self._scenario_tree_model = None self._scenario_tree = None self._data_directory = None try: self._init(model, scenario_tree, data) except: self.close() raise self._closed = False def _init(self, model, scenario_tree, data): self._model_filename = None self._model_module = None self._model_object = None self._model_callback = None if isinstance(model, six.string_types): logger.debug("A model filename was provided.") self._model_filename, self._archives = \ _extract_pathspec(model, "ReferenceModel.py", archives=self._archives) if not os.path.exists(self._model_filename): logger.error("Failed to extract reference model python file " "from path specification: %s" % (model)) raise IOError("path does not exist: %s" % (self._model_filename)) assert self._model_filename is not None assert self._model_filename.endswith(".py") (self._model_module, self._model_object, self._model_callback) = \ _find_reference_model_or_callback(self._model_filename) if (self._model_object is None) and \ (self._model_callback is None): raise AttributeError( "No reference Pyomo model or " "'pysp_instance_creation_callback' " "function object found in src: %s" % (self._model_filename)) elif hasattr(model, "__call__"): logger.debug("A model callback function was provided.") self._model_callback = model else: if not isinstance(model, (_BlockData, Block)): raise TypeError( "model argument object has incorrect type: %s. " "Must be a string type, a callback, or a Pyomo model." % (type(model))) logger.debug("A model object was provided.") self._model_object = model self._scenario_tree_filename = None self._scenario_tree_model = None self._scenario_tree = None if isinstance(scenario_tree, ScenarioTree): for scenario in scenario_tree.scenarios: if scenario.instance is not None: raise ValueError( "The scenario tree can not be linked with instances") if hasattr(scenario_tree, "_scenario_instance_factory"): del scenario_tree._scenario_instance_factory self._scenario_tree = scenario_tree elif has_networkx and \ isinstance(scenario_tree, networkx.DiGraph): self._scenario_tree_model = scenario_tree elif isinstance(scenario_tree, six.string_types): logger.debug("scenario tree input is a string, attempting " "to load file specification: %s" % (scenario_tree)) self._scenario_tree_filename = None if not scenario_tree.endswith(".py"): self._scenario_tree_filename, self._archives = \ _extract_pathspec(scenario_tree, "ScenarioStructure.dat", archives=self._archives) if not os.path.exists(self._scenario_tree_filename): logger.debug("Failed to extract scenario tree structure " ".dat file from path specification: %s" % (scenario_tree)) self._scenario_tree_filename = None if self._scenario_tree_filename is None: self._scenario_tree_filename, self._archives = \ _extract_pathspec(scenario_tree, "ScenarioStructure.py", archives=self._archives) if not os.path.exists(self._scenario_tree_filename): logger.debug("Failed to locate scenario tree structure " ".py file with path specification: %s" % (scenario_tree)) self._scenario_tree_filename = None if self._scenario_tree_filename is None: raise ValueError("Failed to extract scenario tree structure " "file with .dat or .py extension from path " "specification: %s" % (scenario_tree)) elif self._scenario_tree_filename.endswith(".py"): if self._scenario_tree_filename == self._model_filename: # try not to clobber the model import (self._scenario_tree_module, self._scenario_tree, self._scenario_tree_model) = \ _find_scenariotree(module=self._model_module) else: (self._scenario_tree_module, self._scenario_tree, self._scenario_tree_model) = \ _find_scenariotree(src=self._scenario_tree_filename) if (self._scenario_tree is None) and \ (self._scenario_tree_model is None): raise AttributeError( "No scenario tree or " "'pysp_scenario_tree_model_callback' " "function found in src: %s" % (self._scenario_tree_filename)) elif self._scenario_tree_filename.endswith(".dat"): self._scenario_tree_model = \ CreateAbstractScenarioTreeModel().\ create_instance(filename=self._scenario_tree_filename) else: assert False elif scenario_tree is None: if self._model_module is not None: self._scenario_tree_filename = self._model_filename (self._scenario_tree_module, self._scenario_tree, self._scenario_tree_model) = \ _find_scenariotree(module=self._model_module) if (self._scenario_tree is None) and \ (self._scenario_tree_model is None): raise ValueError( "No input was provided for the scenario tree " "and no callback or scenario tree object was " "found with the model") else: raise ValueError( "No input was provided for the scenario tree " "but there is no module to search for a " "'pysp_scenario_tree_model_callback' function " "or a ScenarioTree object.") else: self._scenario_tree_model = scenario_tree if self._scenario_tree is None: if (not isinstance(self._scenario_tree_model, (_BlockData, Block))) and \ ((not has_networkx) or \ (not isinstance(self._scenario_tree_model, networkx.DiGraph))): raise TypeError( "scenario tree model object has incorrect type: %s. " "Must be a string type, Pyomo model, or a " "networkx.DiGraph object." % (type(scenario_tree))) if isinstance(self._scenario_tree_model, (_BlockData, Block)): if not self._scenario_tree_model.is_constructed(): raise ValueError( "scenario tree model is not constructed") self._data_directory = None if data is None: if self.scenario_tree_directory() is not None: logger.debug("data directory is set to the scenario tree " "directory: %s" % (self.scenario_tree_directory())) self._data_directory = self.scenario_tree_directory() elif self.model_directory() is not None: logger.debug("data directory is set to the reference model " "directory: %s" % (self.model_directory())) self._data_directory = self.model_directory() else: if (self._model_callback is None) and \ isinstance(self._model_object, AbstractModel) and \ (not self._model_object.is_constructed()): raise ValueError( "A data location is required since no model " "callback was provided and no other location could " "be inferred.") logger.debug("no data directory is required") else: logger.debug("data location is provided, attempting " "to load specification: %s" % (data)) self._data_directory, self._archives = \ _extract_pathspec(data, None, archives=self._archives) if not os.path.exists(self._data_directory): logger.error("Failed to extract data directory " "from path specification: %s" % (data)) raise IOError("path does not exist: %s" % (self._data_directory)) def __getstate__(self): self.close() raise NotImplementedError("Do not deepcopy or serialize this class") def __setstate__(self,d): self.close() raise NotImplementedError("Do not deepcopy or serialize this class") def close(self): for _,archive,tmpdir in self._archives: if os.path.exists(tmpdir): shutil.rmtree(tmpdir, True) archive.close() self._archives = [] self._closed = True # # Support "with" statements. Forgetting to call close() # on this class can result in temporary unarchived # directories being left sitting around # def __enter__(self): return self def __exit__(self, type, value, traceback): self.close() def model_directory(self): if self._model_filename is not None: return os.path.dirname(self._model_filename) else: return None def scenario_tree_directory(self): if self._scenario_tree_filename is not None: return os.path.dirname(self._scenario_tree_filename) else: return None def data_directory(self): return self._data_directory # # construct a scenario instance - just like it sounds! # def construct_scenario_instance(self, scenario_name, scenario_tree, profile_memory=False, output_instance_construction_time=False, compile_instance=False, verbose=False): assert not self._closed if not scenario_tree.contains_scenario(scenario_name): raise ValueError("ScenarioTree does not contain scenario " "with name %s." % (scenario_name)) scenario = scenario_tree.get_scenario(scenario_name) node_name_list = [n._name for n in scenario._node_list] if verbose: print("Creating instance for scenario=%s" % (scenario_name)) scenario_instance = None try: if self._model_callback is not None: assert self._model_object is None try: _scenario_tree_arg = None # new callback signature if (self._scenario_tree_filename is not None) and \ self._scenario_tree_filename.endswith('.dat'): # we started with a .dat file, so # send the PySP scenario tree _scenario_tree_arg = scenario_tree elif self._scenario_tree_model is not None: # We started from a Pyomo # scenario tree model instance, or a # networkx tree. _scenario_tree_arg = self._scenario_tree_model else: # send the PySP scenario tree _scenario_tree_arg = scenario_tree scenario_instance = self._model_callback(_scenario_tree_arg, scenario_name, node_name_list) except TypeError: # old callback signature # TODO: #logger.warning( # "DEPRECATED: The 'pysp_instance_creation_callback' function " # "signature has changed. An additional argument should be " # "added to the beginning of the arguments list that will be " # "set to the user provided scenario tree object when called " # "by PySP (e.g., a Pyomo scenario tree model instance, " # "a networkx tree, or a PySP ScenarioTree object.") scenario_instance = self._model_callback(scenario_name, node_name_list) elif self._model_object is not None: if (not isinstance(self._model_object, AbstractModel)) or \ (self._model_object.is_constructed()): scenario_instance = self._model_object.clone() elif scenario_tree._scenario_based_data: assert self.data_directory() is not None scenario_data_filename = \ os.path.join(self.data_directory(), str(scenario_name)) # JPW: The following is a hack to support # initialization of block instances, which # don't work with .dat files at the # moment. Actually, it's not that bad of a # hack - it just needs to be extended a bit, # and expanded into the node-based data read # logic (where yaml is completely ignored at # the moment. if os.path.exists(scenario_data_filename+'.dat'): scenario_data_filename = \ scenario_data_filename + ".dat" data = None elif os.path.exists(scenario_data_filename+'.yaml'): if not has_yaml: raise ValueError( "Found yaml data file for scenario '%s' " "but he PyYAML module is not available" % (scenario_name)) scenario_data_filename = \ scenario_data_filename+".yaml" with open(scenario_data_filename) as f: data = yaml.load(f) else: raise RuntimeError( "Cannot find a data file for scenario '%s' " "in directory: %s\nRecognized formats: .dat, " ".yaml" % (scenario_name, self.data_directory())) if verbose: print("Data for scenario=%s loads from file=%s" % (scenario_name, scenario_data_filename)) if data is None: scenario_instance = \ self._model_object.create_instance( filename=scenario_data_filename, profile_memory=profile_memory, report_timing=output_instance_construction_time) else: scenario_instance = \ self._model_object.create_instance( data, profile_memory=profile_memory, report_timing=output_instance_construction_time) else: assert self.data_directory() is not None data_files = [] for node_name in node_name_list: node_data_filename = \ os.path.join(self.data_directory(), str(node_name)+".dat") if not os.path.exists(node_data_filename): raise RuntimeError( "Cannot find a data file for scenario tree " "node '%s' in directory: %s\nRecognized " "formats: .dat" % (node_name, self.data_directory())) data_files.append(node_data_filename) scenario_data = DataPortal(model=self._model_object) for data_file in data_files: if verbose: print("Node data for scenario=%s partially " "loading from file=%s" % (scenario_name, data_file)) scenario_data.load(filename=data_file) scenario_instance = self._model_object.create_instance( scenario_data, profile_memory=profile_memory, report_timing=output_instance_construction_time) else: raise RuntimeError("Unable to construct scenario instance. " "Neither a reference model or callback " "is defined.") # name each instance with the scenario name scenario_instance._name = scenario_name # apply each of the post-instance creation plugins. this # really shouldn't be associated (in terms of naming) with the # pyomo script - this should be rectified with a workflow # re-work. it is unclear how this interacts, or doesn't, with # the preprocessors. ep = ExtensionPoint(IPyomoScriptModifyInstance) for ep in ExtensionPoint(IPyomoScriptModifyInstance): logger.warning( "DEPRECATED: IPyomoScriptModifyInstance extension " "point callbacks will be ignored by PySP in the future") ep.apply(options=None, model=reference_model, instance=scenario_instance) if compile_instance: from pyomo.repn.beta.matrix import \ compile_block_linear_constraints compile_block_linear_constraints( scenario_instance, "_PySP_compiled_linear_constraints", verbose=verbose) except: logger.error("Failed to create model instance for scenario=%s" % (scenario_name)) raise return scenario_instance def construct_instances_for_scenario_tree( self, scenario_tree, profile_memory=False, output_instance_construction_time=False, compile_scenario_instances=False, verbose=False): assert not self._closed if scenario_tree._scenario_based_data: if verbose: print("Scenario-based instance initialization enabled") else: if verbose: print("Node-based instance initialization enabled") scenario_instances = {} for scenario in scenario_tree._scenarios: # the construction of instances takes little overhead in terms # of memory potentially lost in the garbage-collection sense # (mainly only that due to parsing and instance # simplification/prep-processing). to speed things along, # disable garbage collection if it enabled in the first place # through the instance construction process. # IDEA: If this becomes too much for truly large numbers of # scenarios, we could manually collect every time X # instances have been created. scenario_instance = None with PauseGC() as pgc: scenario_instance = \ self.construct_scenario_instance( scenario._name, scenario_tree, profile_memory=profile_memory, output_instance_construction_time=output_instance_construction_time, compile_instance=compile_scenario_instances, verbose=verbose) scenario_instances[scenario._name] = scenario_instance assert scenario_instance.local_name == scenario.name return scenario_instances def generate_scenario_tree(self, downsample_fraction=1.0, include_scenarios=None, bundles=None, random_bundles=None, random_seed=None, verbose=True): scenario_tree_model = self._scenario_tree_model if scenario_tree_model is not None: if has_networkx and \ isinstance(scenario_tree_model, networkx.DiGraph): scenario_tree_model = \ ScenarioTreeModelFromNetworkX(scenario_tree_model) else: assert isinstance(scenario_tree_model, (_BlockData, Block)), \ str(scenario_tree_model)+" "+str(type(scenario_tree_model)) if bundles is not None: if isinstance(bundles, six.string_types): if scenario_tree_model is None: raise ValueError( "A bundles file can not be used when the " "scenario tree input was not a Pyomo " "model or ScenarioStructure.dat file.") logger.debug("attempting to locate bundle file for input: %s" % (bundles)) # we interpret the scenario bundle specification in one of # two ways. if the supplied name is a file, it is used # directly. otherwise, it is interpreted as the root of a # file with a .dat suffix to be found in the instance # directory. orig_input = bundles if not bundles.endswith(".dat"): bundles = bundles+".dat" bundles = os.path.expanduser(bundles) if not os.path.exists(bundles): if self.data_directory() is None: raise ValueError( "Could not locate bundle .dat file from input " "'%s'. Path does not exist and there is no data " "directory to search in." % (orig_input)) bundles = os.path.join(self.data_directory(), bundles) if not os.path.exists(bundles): raise ValueError("Could not locate bundle .dat file " "from input '%s' as absolute path or " "relative to data directory: %s" % (orig_input, self.data_directory())) if verbose: print("Scenario tree bundle specification filename=%s" % (bundles)) scenario_tree_model = scenario_tree_model.clone() scenario_tree_model.Bundling = True scenario_tree_model.Bundling._constructed = False scenario_tree_model.Bundles.clear() scenario_tree_model.Bundles._constructed = False scenario_tree_model.BundleScenarios.clear() scenario_tree_model.BundleScenarios._constructed = False scenario_tree_model.load(bundles) # # construct the scenario tree # if scenario_tree_model is not None: scenario_tree = ScenarioTree(scenariotreeinstance=scenario_tree_model, scenariobundlelist=include_scenarios) else: assert self._scenario_tree is not None if include_scenarios is None: scenario_tree = copy.deepcopy(self._scenario_tree) else: # note: any bundles will be lost if self._scenario_tree.contains_bundles(): raise ValueError( "Can not compress a scenario tree that " "contains bundles") scenario_tree = self._scenario_tree.make_compressed( include_scenarios, normalize=True) # compress/down-sample the scenario tree, if requested if (downsample_fraction is not None) and \ (downsample_fraction < 1.0): scenario_tree.downsample(downsample_fraction, random_seed, verbose) # # create bundles from a dict, if requested # if bundles is not None: if not isinstance(bundles, six.string_types): if verbose: print("Adding bundles to scenario tree from " "user-specified dict") if scenario_tree.contains_bundles(): if verbose: print("Scenario tree already contains bundles. " "All existing bundles will be removed.") for bundle in list(scenario_tree.bundles): scenario_tree.remove_bundle(bundle.name) for bundle_name in bundles: scenario_tree.add_bundle(bundle_name, bundles[bundle_name]) # # create random bundles, if requested # if (random_bundles is not None) and \ (random_bundles > 0): if bundles is not None: raise ValueError("Cannot specify both random " "bundles and a bundles specification") num_scenarios = len(scenario_tree._scenarios) if random_bundles > num_scenarios: raise ValueError("Cannot create more random bundles " "than there are scenarios!") if verbose: print("Creating "+str(random_bundles)+ " random bundles using seed=" +str(random_seed)) scenario_tree.create_random_bundles(random_bundles, random_seed) scenario_tree._scenario_instance_factory = self return scenario_tree