def assignTests(cls, problem_list):
for solver,writer in testing_solvers:
for PROBLEM in problem_list:
aux_list = ['','force_pw']
for AUX in aux_list:
for REPN in PWRepn:
for BOUND_TYPE in Bound:
for SENSE in [maximize,minimize]:
if not( ((BOUND_TYPE == Bound.Lower) and (SENSE == maximize)) or \
((BOUND_TYPE == Bound.Upper) and (SENSE == minimize)) or \
((REPN in [PWRepn.BIGM_BIN,PWRepn.BIGM_SOS1,PWRepn.MC]) and ('step' in PROBLEM)) ):
kwds = {}
kwds['sense'] = SENSE
kwds['pw_repn'] = REPN
kwds['pw_constr_type'] = BOUND_TYPE
if SENSE == maximize:
attrName = "test_{0}_{1}_{2}_{3}_{4}_{5}".format(PROBLEM,REPN,BOUND_TYPE,'maximize',solver,writer)
elif SENSE == minimize:
attrName = "test_{0}_{1}_{2}_{3}_{4}_{5}".format(PROBLEM,REPN,BOUND_TYPE,'minimize',solver,writer)
if AUX != '':
kwds[AUX] = True
attrName += '_'+AUX
setattr(cls,attrName,createTestMethod(attrName,PROBLEM,solver,writer,kwds))
if yaml_available:
with open(join(thisDir,'baselines',PROBLEM+'_baseline_results.yml'),'r') as f:
baseline_results = yaml.load(f, **yaml_load_args)
setattr(cls,PROBLEM+'_results',baseline_results)
def test_pyomo_command(self):
results = os.path.join(currdir, 'result.yml')
args = [
os.path.join(currdir, 't1.py'),
'--solver-manager=neos',
'--solver=cbc',
'--symbolic-solver-labels',
'--save-results=%s' % results,
'-c',
]
try:
output = main.run(args)
self.assertEqual(output.errorcode, 0)
with open(results) as FILE:
data = yaml.load(FILE, **yaml_load_args)
self.assertEqual(data['Solver'][0]['Status'], 'ok')
self.assertAlmostEqual(data['Solution'][1]['Status'], 'optimal')
self.assertAlmostEqual(
data['Solution'][1]['Objective']['o']['Value'], 1)
self.assertAlmostEqual(
data['Solution'][1]['Variable']['x']['Value'], 0.5)
finally:
cleanup()
os.remove(results)
def getObjective(self, fname):
FILE = open(fname, 'r')
data = yaml.load(FILE, **yaml_load_args)
FILE.close()
solutions = data.get('Solution', [])
ans = []
for x in solutions:
ans.append(x.get('Objective', {}))
return ans
def assignTests(cls):
defs = yaml.load(open(os.path.join(currdir, 'examples.yml'), 'r'),
**yaml_load_args)
for package in defs:
for checkerName in defs[package]:
for key in defs[package][checkerName]:
attrName = "{0}_{1}_{2}".format(package, checkerName, key)
cls.add_fn_test(name=attrName, fn=createTestMethod(defs, package, checkerName, key))
def get_config_values(filename):
if filename.endswith('.yml') or filename.endswith('.yaml'):
if not yaml_available:
raise ValueError("ERROR: yaml configuration file specified, but pyyaml is not installed!")
INPUT = open(filename, 'r')
val = yaml.load(INPUT, **yaml_load_args)
INPUT.close()
return val
elif filename.endswith('.jsn') or filename.endswith('.json'):
INPUT = open(filename, 'r')
val = json.load(INPUT)
INPUT.close()
return val
raise IOError("ERROR: Unexpected configuration file '%s'" % filename)
def read(self):
"""
This function loads data from a YAML file and tuplizes the nested
dictionaries and lists of lists.
"""
if not os.path.exists(self.filename):
raise IOError("Cannot find file '%s'" % self.filename)
INPUT = open(self.filename, 'r')
jdata = yaml.load(INPUT, **yaml_load_args)
INPUT.close()
if jdata is None:
raise IOError("Empty YAML file")
self._info = {}
for k, v in jdata.items():
self._info[k] = tuplize(v)
def read(self, **kwds):
if 'istream' in kwds:
istream = kwds['istream']
del kwds['istream']
else:
ostream = sys.stdin
if 'filename' in kwds:
INPUT = open(kwds['filename'], "r")
del kwds['filename']
kwds['istream'] = INPUT
self.read(**kwds)
INPUT.close()
return
if not 'format' in kwds or kwds['format'] == 'yaml':
repn = yaml.load(istream, **yaml_load_args)
else:
repn = json.load(istream)
for i in range(len(self._order)):
key = self._order[i]
if not key in repn:
continue
item = dict.__getitem__(self, key)
item.load(repn[key])
def create_model(data):
"""
Create instance of Pyomo model.
Return:
model: Model object.
instance: Problem instance.
symbol_map: Symbol map created when writing model to a file.
filename: Filename that a model instance was written to.
"""
#
if not data.options.runtime.logging == 'quiet':
sys.stdout.write('[%8.2f] Creating model\n' %
(time.time() - start_time))
sys.stdout.flush()
#
if data.options.runtime.profile_memory >= 1 and pympler_available:
global memory_data
mem_used = pympler.muppy.get_size(pympler.muppy.get_objects())
data.local.max_memory = mem_used
print(" Total memory = %d bytes prior to model construction" %
mem_used)
#
# Find the Model objects
#
_models = {}
_model_IDS = set()
for _name, _obj in data.local.usermodel.__dict__.items():
if isinstance(_obj, Model) and id(_obj) not in _model_IDS:
_models[_name] = _obj
_model_IDS.add(id(_obj))
model_name = data.options.model.object_name
if len(_models) == 1:
_name = list(_models.keys())[0]
if model_name is None:
model_name = _name
elif model_name != _name:
msg = "Model '%s' is not defined in file '%s'!"
raise SystemExit(msg % (model_name, data.options.model.filename))
elif len(_models) > 1:
if model_name is None:
msg = "Multiple models defined in file '%s'!"
raise SystemExit(msg % data.options.model.filename)
elif not model_name in _models:
msg = "Unknown model '%s' in file '%s'!"
raise SystemExit(msg % (model_name, data.options.model.filename))
ep = ExtensionPoint(IPyomoScriptCreateModel)
if model_name is None:
if len(ep) == 0:
msg = "A model is not defined and the 'pyomo_create_model' is not "\
"provided in module %s"
raise SystemExit(msg % data.options.model.filename)
elif len(ep) > 1:
msg = 'Multiple model construction plugins have been registered in module %s!'
raise SystemExit(msg % data.options.model.filename)
else:
model_options = data.options.model.options.value()
tick = time.time()
model = ep.service().apply(options=Bunch(*data.options),
model_options=Bunch(*model_options))
if data.options.runtime.report_timing is True:
print(" %6.2f seconds required to construct instance" %
(time.time() - tick))
data.local.time_initial_import = None
tick = time.time()
else:
if model_name not in _models:
msg = "Model '%s' is not defined in file '%s'!"
raise SystemExit(msg % (model_name, data.options.model.filename))
model = _models[model_name]
if model is None:
msg = "'%s' object is 'None' in module %s"
raise SystemExit(msg % (model_name, data.options.model.filename))
elif len(ep) > 0:
msg = "Model construction function 'create_model' defined in " \
"file '%s', but model is already constructed!"
raise SystemExit(msg % data.options.model.filename)
#
# Print model
#
for ep in ExtensionPoint(IPyomoScriptPrintModel):
ep.apply(options=data.options, model=model)
#
# Create Problem Instance
#
ep = ExtensionPoint(IPyomoScriptCreateDataPortal)
if len(ep) > 1:
msg = 'Multiple model data construction plugins have been registered!'
raise SystemExit(msg)
if len(ep) == 1:
modeldata = ep.service().apply(options=data.options, model=model)
else:
modeldata = DataPortal()
if model._constructed:
#
# TODO: use a better test for ConcreteModel
#
instance = model
if data.options.runtime.report_timing is True and not data.local.time_initial_import is None:
print(" %6.2f seconds required to construct instance" %
(data.local.time_initial_import))
else:
tick = time.time()
if len(data.options.data.files) > 1:
#
# Load a list of *.dat files
#
for file in data.options.data.files:
suffix = (file).split(".")[-1]
if suffix != "dat":
msg = 'When specifiying multiple data files, they must all ' \
'be *.dat files. File specified: %s'
raise SystemExit(msg % str(file))
modeldata.load(filename=file, model=model)
instance = model.create_instance(
modeldata,
namespaces=data.options.data.namespaces,
profile_memory=data.options.runtime.profile_memory,
report_timing=data.options.runtime.report_timing)
elif len(data.options.data.files) == 1:
#
# Load a *.dat file or process a *.py data file
#
suffix = (data.options.data.files[0]).split(".")[-1].lower()
if suffix == "dat":
instance = model.create_instance(
data.options.data.files[0],
namespaces=data.options.data.namespaces,
profile_memory=data.options.runtime.profile_memory,
report_timing=data.options.runtime.report_timing)
elif suffix == "py":
userdata = import_file(data.options.data.files[0],
clear_cache=True)
if "modeldata" in dir(userdata):
if len(ep) == 1:
msg = "Cannot apply 'pyomo_create_modeldata' and use the" \
" 'modeldata' object that is provided in the model"
raise SystemExit(msg)
if userdata.modeldata is None:
msg = "'modeldata' object is 'None' in module %s"
raise SystemExit(msg % str(data.options.data.files[0]))
modeldata = userdata.modeldata
else:
if len(ep) == 0:
msg = "Neither 'modeldata' nor 'pyomo_create_dataportal' " \
'is defined in module %s'
raise SystemExit(msg % str(data.options.data.files[0]))
modeldata.read(model)
instance = model.create_instance(
modeldata,
namespaces=data.options.data.namespaces,
profile_memory=data.options.runtime.profile_memory,
report_timing=data.options.runtime.report_timing)
elif suffix == "yml" or suffix == 'yaml':
modeldata = yaml.load(open(data.options.data.files[0]),
**yaml_load_args)
instance = model.create_instance(
modeldata,
namespaces=data.options.data.namespaces,
profile_memory=data.options.runtime.profile_memory,
report_timing=data.options.runtime.report_timing)
else:
raise ValueError("Unknown data file type: " +
data.options.data.files[0])
else:
instance = model.create_instance(
modeldata,
namespaces=data.options.data.namespaces,
profile_memory=data.options.runtime.profile_memory,
report_timing=data.options.runtime.report_timing)
if data.options.runtime.report_timing is True:
print(" %6.2f seconds required to construct instance" %
(time.time() - tick))
#
modify_start_time = time.time()
for ep in ExtensionPoint(IPyomoScriptModifyInstance):
if data.options.runtime.report_timing is True:
tick = time.time()
ep.apply(options=data.options, model=model, instance=instance)
if data.options.runtime.report_timing is True:
print(" %6.2f seconds to apply %s" %
(time.time() - tick, type(ep)))
tick = time.time()
#
for transformation in data.options.transform:
with TransformationFactory(transformation) as xfrm:
instance = xfrm.create_using(instance)
if instance is None:
raise SystemExit("Unexpected error while applying "
"transformation '%s'" % transformation)
#
if data.options.runtime.report_timing is True:
total_time = time.time() - modify_start_time
print(" %6.2f seconds required for problem transformations" %
total_time)
if is_debug_set(logger):
print("MODEL INSTANCE")
instance.pprint()
print("")
for ep in ExtensionPoint(IPyomoScriptPrintInstance):
ep.apply(options=data.options, instance=instance)
fname = None
smap_id = None
if not data.options.model.save_file is None:
if data.options.runtime.report_timing is True:
write_start_time = time.time()
if data.options.model.save_file == True:
if data.local.model_format in (ProblemFormat.cpxlp,
ProblemFormat.lpxlp):
fname = (data.options.data.files[0])[:-3] + 'lp'
else:
fname = (data.options.data.files[0])[:-3] + str(
data.local.model_format)
format = data.local.model_format
else:
fname = data.options.model.save_file
format = data.options.model.save_format
io_options = {}
if data.options.model.symbolic_solver_labels:
io_options['symbolic_solver_labels'] = True
if data.options.model.file_determinism != 1:
io_options[
'file_determinism'] = data.options.model.file_determinism
(fname, smap_id) = instance.write(filename=fname,
format=format,
io_options=io_options)
if not data.options.runtime.logging == 'quiet':
if not os.path.exists(fname):
print("ERROR: file " + fname + " has not been created!")
else:
print("Model written to file '" + str(fname) + "'")
if data.options.runtime.report_timing is True:
total_time = time.time() - write_start_time
print(" %6.2f seconds required to write file" % total_time)
if data.options.runtime.profile_memory >= 2 and pympler_available:
print("")
print(" Summary of objects following file output")
post_file_output_summary = pympler.summary.summarize(
pympler.muppy.get_objects())
pympler.summary.print_(post_file_output_summary, limit=100)
print("")
for ep in ExtensionPoint(IPyomoScriptSaveInstance):
ep.apply(options=data.options, instance=instance)
if data.options.runtime.profile_memory >= 1 and pympler_available:
mem_used = pympler.muppy.get_size(pympler.muppy.get_objects())
if mem_used > data.local.max_memory:
data.local.max_memory = mem_used
print(" Total memory = %d bytes following Pyomo instance creation" %
mem_used)
return Bunch(model=model,
instance=instance,
smap_id=smap_id,
filename=fname,
local=data.local)
def construct_scenario_instance(self,
scenario_name,
scenario_tree,
profile_memory=False,
output_instance_construction_time=False,
compile_instance=False,
verbose=False):
assert not self._closed
if not scenario_tree.contains_scenario(scenario_name):
raise ValueError("ScenarioTree does not contain scenario "
"with name %s." % (scenario_name))
scenario = scenario_tree.get_scenario(scenario_name)
node_name_list = [n._name for n in scenario._node_list]
if verbose:
print("Creating instance for scenario=%s" % (scenario_name))
scenario_instance = None
try:
if self._model_callback is not None:
assert self._model_object is None
try:
_scenario_tree_arg = None
# new callback signature
if (self._scenario_tree_filename is not None) and \
self._scenario_tree_filename.endswith('.dat'):
# we started with a .dat file, so
# send the PySP scenario tree
_scenario_tree_arg = scenario_tree
elif self._scenario_tree_model is not None:
# We started from a Pyomo
# scenario tree model instance, or a
# networkx tree.
_scenario_tree_arg = self._scenario_tree_model
else:
# send the PySP scenario tree
_scenario_tree_arg = scenario_tree
scenario_instance = self._model_callback(
_scenario_tree_arg, scenario_name, node_name_list)
except TypeError:
# old callback signature
# TODO:
#logger.warning(
# "DEPRECATED: The 'pysp_instance_creation_callback' function "
# "signature has changed. An additional argument should be "
# "added to the beginning of the arguments list that will be "
# "set to the user provided scenario tree object when called "
# "by PySP (e.g., a Pyomo scenario tree model instance, "
# "a networkx tree, or a PySP ScenarioTree object.")
scenario_instance = self._model_callback(
scenario_name, node_name_list)
elif self._model_object is not None:
if (not isinstance(self._model_object, AbstractModel)) or \
(self._model_object.is_constructed()):
scenario_instance = self._model_object.clone()
elif scenario_tree._scenario_based_data:
assert self.data_directory() is not None
scenario_data_filename = \
os.path.join(self.data_directory(),
str(scenario_name))
# JPW: The following is a hack to support
# initialization of block instances, which
# don't work with .dat files at the
# moment. Actually, it's not that bad of a
# hack - it just needs to be extended a bit,
# and expanded into the node-based data read
# logic (where yaml is completely ignored at
# the moment.
if os.path.exists(scenario_data_filename + '.dat'):
scenario_data_filename = \
scenario_data_filename + ".dat"
data = None
elif os.path.exists(scenario_data_filename + '.yaml'):
if not yaml_available:
raise ValueError(
"Found yaml data file for scenario '%s' "
"but he PyYAML module is not available" %
(scenario_name))
scenario_data_filename = \
scenario_data_filename+".yaml"
with open(scenario_data_filename) as f:
data = yaml.load(f, **yaml_load_args)
else:
raise RuntimeError(
"Cannot find a data file for scenario '%s' "
"in directory: %s\nRecognized formats: .dat, "
".yaml" % (scenario_name, self.data_directory()))
if verbose:
print("Data for scenario=%s loads from file=%s" %
(scenario_name, scenario_data_filename))
if data is None:
scenario_instance = \
self._model_object.create_instance(
filename=scenario_data_filename,
profile_memory=profile_memory,
report_timing=output_instance_construction_time)
else:
scenario_instance = \
self._model_object.create_instance(
data,
profile_memory=profile_memory,
report_timing=output_instance_construction_time)
else:
assert self.data_directory() is not None
data_files = []
for node_name in node_name_list:
node_data_filename = \
os.path.join(self.data_directory(),
str(node_name)+".dat")
if not os.path.exists(node_data_filename):
raise RuntimeError(
"Cannot find a data file for scenario tree "
"node '%s' in directory: %s\nRecognized "
"formats: .dat" %
(node_name, self.data_directory()))
data_files.append(node_data_filename)
scenario_data = DataPortal(model=self._model_object)
for data_file in data_files:
if verbose:
print("Node data for scenario=%s partially "
"loading from file=%s" %
(scenario_name, data_file))
scenario_data.load(filename=data_file)
scenario_instance = self._model_object.create_instance(
scenario_data,
profile_memory=profile_memory,
report_timing=output_instance_construction_time)
else:
raise RuntimeError("Unable to construct scenario instance. "
"Neither a reference model or callback "
"is defined.")
# name each instance with the scenario name
scenario_instance._name = scenario_name
# apply each of the post-instance creation plugins. this
# really shouldn't be associated (in terms of naming) with the
# pyomo script - this should be rectified with a workflow
# re-work. it is unclear how this interacts, or doesn't, with
# the preprocessors.
ep = ExtensionPoint(IPyomoScriptModifyInstance)
for ep in ExtensionPoint(IPyomoScriptModifyInstance):
logger.warning(
"DEPRECATED: IPyomoScriptModifyInstance extension "
"point callbacks will be ignored by PySP in the future")
ep.apply(options=None,
model=reference_model,
instance=scenario_instance)
if compile_instance:
from pyomo.repn.beta.matrix import \
compile_block_linear_constraints
compile_block_linear_constraints(
scenario_instance,
"_PySP_compiled_linear_constraints",
verbose=verbose)
except:
logger.error("Failed to create model instance for scenario=%s" %
(scenario_name))
raise
return scenario_instance
