def create_model(data):
"""
Create instance of Pyomo model.
Return:
model: Model object.
instance: Problem instance.
symbol_map: Symbol map created when writing model to a file.
filename: Filename that a model instance was written to.
"""
#
if not data.options.runtime.logging == 'quiet':
sys.stdout.write('[%8.2f] Creating model\n' %
(time.time() - start_time))
sys.stdout.flush()
#
if (pympler_available is True) and (data.options.runtime.profile_memory >=
1):
global memory_data
mem_used = muppy.get_size(muppy.get_objects())
data.local.max_memory = mem_used
print(" Total memory = %d bytes prior to model construction" %
mem_used)
#
# Find the Model objects
#
_models = {}
for _name, _obj in iteritems(data.local.usermodel.__dict__):
if isinstance(_obj, Model):
_models[_name] = _obj
model_name = data.options.model.object_name
if len(_models) == 1:
_name = list(_models.keys())[0]
if model_name is None:
model_name = _name
elif model_name != _name:
msg = "Model '%s' is not defined in file '%s'!"
raise SystemExit(msg % (model_name, data.options.model.filename))
elif len(_models) > 1:
if model_name is None:
msg = "Multiple models defined in file '%s'!"
raise SystemExit(msg % data.options.model.filename)
elif not model_name in _models:
msg = "Unknown model '%s' in file '%s'!"
raise SystemExit(msg % (model_name, data.options.model.filename))
ep = ExtensionPoint(IPyomoScriptCreateModel)
if model_name is None:
if len(ep) == 0:
msg = "A model is not defined and the 'pyomo_create_model' is not "\
"provided in module %s"
raise SystemExit(msg % data.options.model.filename)
elif len(ep) > 1:
msg = 'Multiple model construction plugins have been registered in module %s!'
raise SystemExit(msg % data.options.model.filename)
else:
model_options = data.options.model.options.value()
model = ep.service().apply(
options=pyutilib.misc.Container(*data.options),
model_options=pyutilib.misc.Container(*model_options))
else:
if model_name not in _models:
msg = "Model '%s' is not defined in file '%s'!"
raise SystemExit(msg % (model_name, data.options.model.filename))
model = _models[model_name]
if model is None:
msg = "'%s' object is 'None' in module %s"
raise SystemExit(msg % (model_name, data.options.model.filename))
elif len(ep) > 0:
msg = "Model construction function 'create_model' defined in " \
"file '%s', but model is already constructed!"
raise SystemExit(msg % data.options.model.filename)
#
# Print model
#
for ep in ExtensionPoint(IPyomoScriptPrintModel):
ep.apply(options=data.options, model=model)
#
# Create Problem Instance
#
ep = ExtensionPoint(IPyomoScriptCreateDataPortal)
if len(ep) > 1:
msg = 'Multiple model data construction plugins have been registered!'
raise SystemExit(msg)
if len(ep) == 1:
modeldata = ep.service().apply(options=data.options, model=model)
else:
modeldata = DataPortal()
if model._constructed:
#
# TODO: use a better test for ConcreteModel
#
instance = model
elif len(data.options.data.files) > 1:
#
# Load a list of *.dat files
#
for file in data.options.data.files:
suffix = (file).split(".")[-1]
if suffix != "dat":
msg = 'When specifiying multiple data files, they must all ' \
'be *.dat files. File specified: %s'
raise SystemExit(msg % str(file))
modeldata.load(filename=file, model=model)
instance = model.create_instance(
modeldata,
namespaces=data.options.data.namespaces,
profile_memory=data.options.runtime.profile_memory,
report_timing=data.options.runtime.report_timing)
elif len(data.options.data.files) == 1:
#
# Load a *.dat file or process a *.py data file
#
suffix = (data.options.data.files[0]).split(".")[-1].lower()
if suffix == "dat":
instance = model.create_instance(
data.options.data.files[0],
namespaces=data.options.data.namespaces,
profile_memory=data.options.runtime.profile_memory,
report_timing=data.options.runtime.report_timing)
elif suffix == "py":
userdata = pyutilib.misc.import_file(data.options.data.files[0],
clear_cache=True)
if "modeldata" in dir(userdata):
if len(ep) == 1:
msg = "Cannot apply 'pyomo_create_modeldata' and use the" \
" 'modeldata' object that is provided in the model"
raise SystemExit(msg)
if userdata.modeldata is None:
msg = "'modeldata' object is 'None' in module %s"
raise SystemExit(msg % str(data.options.data.files[0]))
modeldata = userdata.modeldata
else:
if len(ep) == 0:
msg = "Neither 'modeldata' nor 'pyomo_create_dataportal' " \
'is defined in module %s'
raise SystemExit(msg % str(data.options.data.files[0]))
modeldata.read(model)
instance = model.create_instance(
modeldata,
namespaces=data.options.data.namespaces,
profile_memory=data.options.runtime.profile_memory,
report_timing=data.options.runtime.report_timing)
elif suffix == "yml" or suffix == 'yaml':
try:
import yaml
except:
msg = "Cannot apply load data from a YAML file: PyYaml is not installed"
raise SystemExit(msg)
modeldata = yaml.load(open(data.options.data.files[0]))
instance = model.create_instance(
modeldata,
namespaces=data.options.data.namespaces,
profile_memory=data.options.runtime.profile_memory,
report_timing=data.options.runtime.report_timing)
else:
raise ValueError("Unknown data file type: " +
data.options.data.files[0])
else:
instance = model.create_instance(
modeldata,
namespaces=data.options.data.namespaces,
profile_memory=data.options.runtime.profile_memory,
report_timing=data.options.runtime.report_timing)
#
modify_start_time = time.time()
for ep in ExtensionPoint(IPyomoScriptModifyInstance):
if data.options.runtime.report_timing is True:
tick = time.time()
ep.apply(options=data.options, model=model, instance=instance)
if data.options.runtime.report_timing is True:
print(" %6.2f seconds to apply %s" %
(time.time() - tick, type(ep)))
tick = time.time()
#
for transformation in data.options.transform:
with TransformationFactory(transformation) as xfrm:
instance = xfrm.create_using(instance)
if instance is None:
raise SystemExit("Unexpected error while applying "
"transformation '%s'" % transformation)
#
if data.options.runtime.report_timing is True:
total_time = time.time() - modify_start_time
print(" %6.2f seconds required for problem transformations" %
total_time)
if logger.isEnabledFor(logging.DEBUG):
print("MODEL INSTANCE")
instance.pprint()
print("")
for ep in ExtensionPoint(IPyomoScriptPrintInstance):
ep.apply(options=data.options, instance=instance)
fname = None
smap_id = None
if not data.options.model.save_file is None:
if data.options.runtime.report_timing is True:
write_start_time = time.time()
if data.options.model.save_file == True:
if data.local.model_format in (ProblemFormat.cpxlp,
ProblemFormat.lpxlp):
fname = (data.options.data.files[0])[:-3] + 'lp'
else:
fname = (data.options.data.files[0])[:-3] + str(
data.local.model_format)
format = data.local.model_format
else:
fname = data.options.model.save_file
format = data.options.model.save_format
io_options = {}
if data.options.model.symbolic_solver_labels:
io_options['symbolic_solver_labels'] = True
if data.options.model.file_determinism != 1:
io_options[
'file_determinism'] = data.options.model.file_determinism
(fname, smap_id) = instance.write(filename=fname,
format=format,
io_options=io_options)
if not data.options.runtime.logging == 'quiet':
if not os.path.exists(fname):
print("ERROR: file " + fname + " has not been created!")
else:
print("Model written to file '" + str(fname) + "'")
if data.options.runtime.report_timing is True:
total_time = time.time() - write_start_time
print(" %6.2f seconds required to write file" % total_time)
if (pympler_available is True) and (data.options.runtime.profile_memory
>= 2):
print("")
print(" Summary of objects following file output")
post_file_output_summary = summary.summarize(muppy.get_objects())
summary.print_(post_file_output_summary, limit=100)
print("")
for ep in ExtensionPoint(IPyomoScriptSaveInstance):
ep.apply(options=data.options, instance=instance)
if (pympler_available is True) and (data.options.runtime.profile_memory >=
1):
mem_used = muppy.get_size(muppy.get_objects())
if mem_used > data.local.max_memory:
data.local.max_memory = mem_used
print(" Total memory = %d bytes following Pyomo instance creation" %
mem_used)
return pyutilib.misc.Options(model=model,
instance=instance,
smap_id=smap_id,
filename=fname,
local=data.local)
def create_model(data):
"""
Create instance of Pyomo model.
Return:
model: Model object.
instance: Problem instance.
symbol_map: Symbol map created when writing model to a file.
filename: Filename that a model instance was written to.
"""
#
if not data.options.runtime.logging == 'quiet':
sys.stdout.write('[%8.2f] Creating model\n' % (time.time()-start_time))
sys.stdout.flush()
#
if (pympler_available is True) and (data.options.runtime.profile_memory >= 1):
global memory_data
mem_used = muppy.get_size(muppy.get_objects())
data.local.max_memory = mem_used
print(" Total memory = %d bytes prior to model construction" % mem_used)
#
# Create Model
#
ep = ExtensionPoint(IPyomoScriptCreateModel)
model_name = 'model'
if data.options.model.object_name is not None: model_name = data.options.model.object_name
if model_name in dir(data.local.usermodel):
if len(ep) > 0:
msg = "Model construction function 'create_model' defined in " \
"file '%s', but model is already constructed!"
raise SystemExit(msg % data.options.model.filename)
model = getattr(data.local.usermodel, data.options.model.object_name)
if model is None:
msg = "'%s' object is 'None' in module %s"
raise SystemExit(msg % (model_name, data.options.model.filename))
else:
if len(ep) == 0:
msg = "Neither '%s' nor 'pyomo_create_model' are available in " \
'module %s'
raise SystemExit(msg % ( model_name, data.options.model.filename ))
elif len(ep) > 1:
msg = 'Multiple model construction plugins have been registered!'
raise SystemExit(msg)
else:
model_options = data.options.model.options.value()
#if model_options is None:
#model_options = []
model = ep.service().apply( options = pyutilib.misc.Container(*data.options), model_options=pyutilib.misc.Container(*model_options) )
#
for ep in ExtensionPoint(IPyomoScriptPrintModel):
ep.apply( options=data.options, model=model )
#
# Create Problem Instance
#
ep = ExtensionPoint(IPyomoScriptCreateDataPortal)
if len(ep) > 1:
msg = 'Multiple model data construction plugins have been registered!'
raise SystemExit(msg)
if len(ep) == 1:
modeldata = ep.service().apply( options=data.options, model=model )
else:
modeldata = DataPortal()
if model._constructed:
#
# TODO: use a better test for ConcreteModel
#
instance = model
elif len(data.options.data.files) > 1:
#
# Load a list of *.dat files
#
for file in data.options.data.files:
suffix = (file).split(".")[-1]
if suffix != "dat":
msg = 'When specifiying multiple data files, they must all ' \
'be *.dat files. File specified: %s'
raise SystemExit(msg % str( file ))
modeldata.load(filename=file, model=model)
instance = model.create_instance(modeldata,
namespaces=data.options.data.namespaces,
profile_memory=data.options.runtime.profile_memory,
report_timing=data.options.runtime.report_timing)
elif len(data.options.data.files) == 1:
#
# Load a *.dat file or process a *.py data file
#
suffix = (data.options.data.files[0]).split(".")[-1].lower()
if suffix == "dat":
instance = model.create_instance(data.options.data.files[0],
namespaces=data.options.data.namespaces,
profile_memory=data.options.runtime.profile_memory,
report_timing=data.options.runtime.report_timing)
elif suffix == "py":
userdata = pyutilib.misc.import_file(data.options.data.files[0], clear_cache=True)
if "modeldata" in dir(userdata):
if len(ep) == 1:
msg = "Cannot apply 'pyomo_create_modeldata' and use the" \
" 'modeldata' object that is provided in the model"
raise SystemExit(msg)
if userdata.modeldata is None:
msg = "'modeldata' object is 'None' in module %s"
raise SystemExit(msg % str( data.options.data.files[0] ))
modeldata=userdata.modeldata
else:
if len(ep) == 0:
msg = "Neither 'modeldata' nor 'pyomo_create_dataportal' " \
'is defined in module %s'
raise SystemExit(msg % str( data.options.data.files[0] ))
modeldata.read(model)
instance = model.create_instance(modeldata,
namespaces=data.options.data.namespaces,
profile_memory=data.options.runtime.profile_memory,
report_timing=data.options.runtime.report_timing)
elif suffix == "yml" or suffix == 'yaml':
try:
import yaml
except:
msg = "Cannot apply load data from a YAML file: PyYaml is not installed"
raise SystemExit(msg)
modeldata = yaml.load(open(data.options.data.files[0]))
instance = model.create_instance(modeldata,
namespaces=data.options.data.namespaces,
profile_memory=data.options.runtime.profile_memory,
report_timing=data.options.runtime.report_timing)
else:
raise ValueError("Unknown data file type: "+data.options.data.files[0])
else:
instance = model.create_instance(modeldata,
namespaces=data.options.data.namespaces,
profile_memory=data.options.runtime.profile_memory,
report_timing=data.options.runtime.report_timing)
#
modify_start_time = time.time()
for ep in ExtensionPoint(IPyomoScriptModifyInstance):
if data.options.runtime.report_timing is True:
tick = time.time()
ep.apply( options=data.options, model=model, instance=instance )
if data.options.runtime.report_timing is True:
print(" %6.2f seconds to apply %s" % (time.time() - tick, type(ep)))
tick = time.time()
#
for transformation in data.options.transform:
with TransformationFactory(transformation) as xfrm:
instance = xfrm.create_using(instance)
if instance is None:
raise SystemExit("Unexpected error while applying "
"transformation '%s'" % transformation)
#
if data.options.runtime.report_timing is True:
total_time = time.time() - modify_start_time
print(" %6.2f seconds required for problem transformations" % total_time)
if logger.isEnabledFor(logging.DEBUG):
print("MODEL INSTANCE")
instance.pprint()
print("")
for ep in ExtensionPoint(IPyomoScriptPrintInstance):
ep.apply( options=data.options, instance=instance )
fname=None
smap_id=None
if not data.options.model.save_file is None:
if data.options.runtime.report_timing is True:
write_start_time = time.time()
if data.options.model.save_file == True:
if data.local.model_format in (ProblemFormat.cpxlp, ProblemFormat.lpxlp):
fname = (data.options.data.files[0])[:-3]+'lp'
else:
fname = (data.options.data.files[0])[:-3]+str(data.local.model_format)
format=data.local.model_format
else:
fname = data.options.model.save_file
format= data.options.model.save_format
io_options = {}
if data.options.model.symbolic_solver_labels:
io_options['symbolic_solver_labels'] = True
if data.options.model.file_determinism != 1:
io_options['file_determinism'] = data.options.model.file_determinism
(fname, smap_id) = instance.write(filename=fname,
format=format,
io_options=io_options)
if not data.options.runtime.logging == 'quiet':
if not os.path.exists(fname):
print("ERROR: file "+fname+" has not been created!")
else:
print("Model written to file '"+str(fname)+"'")
if data.options.runtime.report_timing is True:
total_time = time.time() - write_start_time
print(" %6.2f seconds required to write file" % total_time)
if (pympler_available is True) and (data.options.runtime.profile_memory >= 2):
print("")
print(" Summary of objects following file output")
post_file_output_summary = summary.summarize(muppy.get_objects())
summary.print_(post_file_output_summary, limit=100)
print("")
for ep in ExtensionPoint(IPyomoScriptSaveInstance):
ep.apply( options=data.options, instance=instance )
if (pympler_available is True) and (data.options.runtime.profile_memory >= 1):
mem_used = muppy.get_size(muppy.get_objects())
if mem_used > data.local.max_memory:
data.local.max_memory = mem_used
print(" Total memory = %d bytes following Pyomo instance creation" % mem_used)
return pyutilib.misc.Options(
model=model, instance=instance,
smap_id=smap_id, filename=fname, local=data.local )
def construct_scenario_instance(self,
scenario_name,
scenario_tree,
profile_memory=False,
output_instance_construction_time=False,
compile_instance=False):
if not scenario_tree.contains_scenario(scenario_name):
raise ValueError("ScenarioTree does not contain scenario "
"with name %s." % (scenario_name))
scenario = scenario_tree.get_scenario(scenario_name)
node_name_list = [n._name for n in scenario._node_list]
if self._verbose:
print("Creating instance for scenario=%s" % (scenario_name))
scenario_instance = None
try:
if self._model_callback is not None:
assert self._model_object is None
scenario_instance = self._model_callback(scenario_name,
node_name_list)
elif self._model_object is not None:
if scenario_tree._scenario_based_data:
scenario_data_filename = \
os.path.join(self._scenario_tree_directory,
str(scenario_name))
# JPW: The following is a hack to support
# initialization of block instances, which
# don't work with .dat files at the
# moment. Actually, it's not that bad of a
# hack - it just needs to be extended a bit,
# and expanded into the node-based data read
# logic (where yaml is completely ignored at
# the moment.
if os.path.exists(scenario_data_filename+'.dat'):
scenario_data_filename = \
scenario_data_filename + ".dat"
data = None
elif os.path.exists(scenario_data_filename+'.yaml'):
import yaml
scenario_data_filename = \
scenario_data_filename + ".yaml"
yaml_input_file=open(scenario_data_filename,"r")
data = yaml.load(yaml_input_file)
yaml_input_file.close()
else:
raise RuntimeError(
"Cannot find the scenario data for "
+ scenario_data_filename)
if self._verbose:
print("Data for scenario=%s loads from file=%s"
% (scenario_name, scenario_data_filename))
if data is None:
scenario_instance = \
self._model_object.create_instance(
filename=scenario_data_filename,
preprocess=False,
profile_memory=profile_memory,
report_timing=output_instance_construction_time)
else:
scenario_instance = \
self._model_object.create_instance(
data,
preprocess=False,
profile_memory=profile_memory,
report_timing=output_instance_construction_time)
else:
data_files = []
for node_name in node_name_list:
node_data_filename = \
os.path.join(self._scenario_tree_directory,
str(node_name)+".dat")
if not os.path.exists(node_data_filename):
raise RuntimeError(
"Node data file="+node_data_filename+
" does not exist or cannot be accessed")
data_files.append(node_data_filename)
scenario_data = DataPortal(model=self._model_object)
for data_file in data_files:
if self._verbose:
print("Node data for scenario=%s partially "
"loading from file=%s"
% (scenario_name, data_file))
scenario_data.load(filename=data_file)
scenario_instance = self._model_object.create_instance(
scenario_data,
preprocess=False,
profile_memory=profile_memory,
report_timing=output_instance_construction_time)
else:
raise RuntimeError("Unable to construct scenario instance. "
"Neither a reference model or callback "
"is defined.")
# name each instance with the scenario name
scenario_instance.name = scenario_name
# apply each of the post-instance creation plugins. this
# really shouldn't be associated (in terms of naming) with the
# pyomo script - this should be rectified with a workflow
# re-work. it is unclear how this interacts, or doesn't, with
# the preprocessors.
ep = ExtensionPoint(IPyomoScriptModifyInstance)
for ep in ExtensionPoint(IPyomoScriptModifyInstance):
logger.warning(
"DEPRECATED: IPyomoScriptModifyInstance extension "
"point callbacks will be ignored by PySP in the future")
ep.apply(options=None,
model=reference_model,
instance=scenario_instance)
if compile_instance:
from pyomo.repn.beta.matrix import compile_block_linear_constraints
compile_block_linear_constraints(
scenario_instance,
"_PySP_compiled_linear_constraints",
verbose=self._verbose)
except Exception as exc:
msg = ("Failed to create model instance for scenario=%s"
% (scenario_name))
print(msg)
raise
return scenario_instance
def construct_scenario_instance(self,
scenario_name,
scenario_tree,
profile_memory=False,
output_instance_construction_time=False,
compile_instance=False,
verbose=False):
assert not self._closed
if not scenario_tree.contains_scenario(scenario_name):
raise ValueError("ScenarioTree does not contain scenario "
"with name %s." % (scenario_name))
scenario = scenario_tree.get_scenario(scenario_name)
node_name_list = [n._name for n in scenario._node_list]
if verbose:
print("Creating instance for scenario=%s" % (scenario_name))
scenario_instance = None
try:
if self._model_callback is not None:
assert self._model_object is None
try:
_scenario_tree_arg = None
# new callback signature
if (self._scenario_tree_filename is not None) and \
self._scenario_tree_filename.endswith('.dat'):
# we started with a .dat file, so
# send the PySP scenario tree
_scenario_tree_arg = scenario_tree
elif self._scenario_tree_model is not None:
# We started from a Pyomo
# scenario tree model instance, or a
# networkx tree.
_scenario_tree_arg = self._scenario_tree_model
else:
# send the PySP scenario tree
_scenario_tree_arg = scenario_tree
scenario_instance = self._model_callback(
_scenario_tree_arg, scenario_name, node_name_list)
except TypeError:
# old callback signature
# TODO:
#logger.warning(
# "DEPRECATED: The 'pysp_instance_creation_callback' function "
# "signature has changed. An additional argument should be "
# "added to the beginning of the arguments list that will be "
# "set to the user provided scenario tree object when called "
# "by PySP (e.g., a Pyomo scenario tree model instance, "
# "a networkx tree, or a PySP ScenarioTree object.")
scenario_instance = self._model_callback(
scenario_name, node_name_list)
elif self._model_object is not None:
if (not isinstance(self._model_object, AbstractModel)) or \
(self._model_object.is_constructed()):
scenario_instance = self._model_object.clone()
elif scenario_tree._scenario_based_data:
assert self.data_directory() is not None
scenario_data_filename = \
os.path.join(self.data_directory(),
str(scenario_name))
# JPW: The following is a hack to support
# initialization of block instances, which
# don't work with .dat files at the
# moment. Actually, it's not that bad of a
# hack - it just needs to be extended a bit,
# and expanded into the node-based data read
# logic (where yaml is completely ignored at
# the moment.
if os.path.exists(scenario_data_filename + '.dat'):
scenario_data_filename = \
scenario_data_filename + ".dat"
data = None
elif os.path.exists(scenario_data_filename + '.yaml'):
if not has_yaml:
raise ValueError(
"Found yaml data file for scenario '%s' "
"but he PyYAML module is not available" %
(scenario_name))
scenario_data_filename = \
scenario_data_filename+".yaml"
with open(scenario_data_filename) as f:
data = yaml.load(f)
else:
raise RuntimeError(
"Cannot find a data file for scenario '%s' "
"in directory: %s\nRecognized formats: .dat, "
".yaml" % (scenario_name, self.data_directory()))
if verbose:
print("Data for scenario=%s loads from file=%s" %
(scenario_name, scenario_data_filename))
if data is None:
scenario_instance = \
self._model_object.create_instance(
filename=scenario_data_filename,
profile_memory=profile_memory,
report_timing=output_instance_construction_time)
else:
scenario_instance = \
self._model_object.create_instance(
data,
profile_memory=profile_memory,
report_timing=output_instance_construction_time)
else:
assert self.data_directory() is not None
data_files = []
for node_name in node_name_list:
node_data_filename = \
os.path.join(self.data_directory(),
str(node_name)+".dat")
if not os.path.exists(node_data_filename):
raise RuntimeError(
"Cannot find a data file for scenario tree "
"node '%s' in directory: %s\nRecognized "
"formats: .dat" %
(node_name, self.data_directory()))
data_files.append(node_data_filename)
scenario_data = DataPortal(model=self._model_object)
for data_file in data_files:
if verbose:
print("Node data for scenario=%s partially "
"loading from file=%s" %
(scenario_name, data_file))
scenario_data.load(filename=data_file)
scenario_instance = self._model_object.create_instance(
scenario_data,
profile_memory=profile_memory,
report_timing=output_instance_construction_time)
else:
raise RuntimeError("Unable to construct scenario instance. "
"Neither a reference model or callback "
"is defined.")
# name each instance with the scenario name
scenario_instance._name = scenario_name
# apply each of the post-instance creation plugins. this
# really shouldn't be associated (in terms of naming) with the
# pyomo script - this should be rectified with a workflow
# re-work. it is unclear how this interacts, or doesn't, with
# the preprocessors.
ep = ExtensionPoint(IPyomoScriptModifyInstance)
for ep in ExtensionPoint(IPyomoScriptModifyInstance):
logger.warning(
"DEPRECATED: IPyomoScriptModifyInstance extension "
"point callbacks will be ignored by PySP in the future")
ep.apply(options=None,
model=reference_model,
instance=scenario_instance)
if compile_instance:
from pyomo.repn.beta.matrix import \
compile_block_linear_constraints
compile_block_linear_constraints(
scenario_instance,
"_PySP_compiled_linear_constraints",
verbose=verbose)
except:
logger.error("Failed to create model instance for scenario=%s" %
(scenario_name))
raise
return scenario_instance
