def apply_postprocessing(data, instance=None, results=None):
"""
Apply post-processing steps.
Required:
instance: Problem instance.
results: Optimization results object.
"""
#
if not data.options.runtime.logging == 'quiet':
sys.stdout.write('[%8.2f] Applying Pyomo postprocessing actions\n' %
(time.time() - start_time))
sys.stdout.flush()
# options are of type ConfigValue, not raw strings / atomics.
for config_value in data.options.postprocess:
postprocess = import_file(config_value, clear_cache=True)
if "pyomo_postprocess" in dir(postprocess):
postprocess.pyomo_postprocess(data.options, instance, results)
for ep in ExtensionPoint(IPyomoScriptPostprocess):
ep.apply(options=data.options, instance=instance, results=results)
if data.options.runtime.profile_memory >= 1 and pympler_available:
mem_used = pympler.muppy.get_size(pympler.muppy.get_objects())
if mem_used > data.local.max_memory:
data.local.max_memory = mem_used
print(" Total memory = %d bytes upon termination" % mem_used)
def process_results(data, instance=None, results=None, opt=None):
"""
Process optimization results.
Required:
instance: Problem instance.
results: Optimization results object.
opt: Optimizer object.
"""
#
if not data.options.runtime.logging == 'quiet':
sys.stdout.write('[%8.2f] Processing results\n' %
(time.time() - start_time))
sys.stdout.flush()
#
if data.options.postsolve.print_logfile:
print("")
print("==========================================================")
print("Solver Logfile: " + str(opt._log_file))
print("==========================================================")
print("")
with open(opt._log_file, "r") as INPUT:
for line in INPUT:
sys.stdout.write(line)
print("==========================================================")
print("Solver Logfile - END")
print("==========================================================")
#
try:
# transform the results object into human-readable names.
instance.solutions.store_to(results)
except Exception:
print("Problem updating solver results")
raise
#
if not data.options.postsolve.show_results:
if data.options.postsolve.save_results:
results_file = data.options.postsolve.save_results
elif data.options.postsolve.results_format == 'json':
results_file = 'results.json'
else:
# The ordering of the elif and else conditions is important here
# to ensure that the default file format is yaml
results_file = 'results.yml'
results.write(filename=results_file,
format=data.options.postsolve.results_format)
if not data.options.runtime.logging == 'quiet':
print(" Number of solutions: " + str(len(results.solution)))
if len(results.solution) > 0:
print(" Solution Information")
print(" Gap: " + str(results.solution[0].gap))
print(" Status: " + str(results.solution[0].status))
if len(results.solution[0].objective) == 1:
key = list(results.solution[0].objective.keys())[0]
print(" Function Value: " +
str(results.solution[0].objective[key]['Value']))
print(" Solver results file: " + results_file)
#
#ep = ExtensionPoint(IPyomoScriptPrintResults)
if data.options.postsolve.show_results:
print("")
results.write(num=1, format=data.options.postsolve.results_format)
print("")
#
if data.options.postsolve.summary:
print("")
print("==========================================================")
print("Solution Summary")
print("==========================================================")
if len(results.solution(0).variable) > 0:
print("")
display(instance)
print("")
else:
print("No solutions reported by solver.")
#
for ep in ExtensionPoint(IPyomoScriptPrintResults):
ep.apply(options=data.options, instance=instance, results=results)
#
for ep in ExtensionPoint(IPyomoScriptSaveResults):
ep.apply(options=data.options, instance=instance, results=results)
#
if data.options.runtime.profile_memory >= 1 and pympler_available:
global memory_data
mem_used = pympler.muppy.get_size(pympler.muppy.get_objects())
if mem_used > data.local.max_memory:
data.local.max_memory = mem_used
print(" Total memory = %d bytes following results processing" %
mem_used)
def apply_preprocessing(data, parser=None):
"""
Execute preprocessing files
Required:
parser: Command line parser object
Returned:
error: This is true if an error has occurred.
"""
data.local = Bunch()
#
if not data.options.runtime.logging == 'quiet':
sys.stdout.write('[%8.2f] Applying Pyomo preprocessing actions\n' %
(time.time() - start_time))
sys.stdout.flush()
#
global filter_excepthook
#
#
# Setup solver and model
#
#
if len(data.options.model.filename) == 0:
parser.print_help()
data.error = True
return data
#
if not data.options.preprocess is None:
for config_value in data.options.preprocess:
preprocess = import_file(config_value, clear_cache=True)
#
for ep in ExtensionPoint(IPyomoScriptPreprocess):
ep.apply(options=data.options)
#
# Verify that files exist
#
for file in [data.options.model.filename
] + data.options.data.files.value():
if not os.path.exists(file):
raise IOError("File " + file + " does not exist!")
#
filter_excepthook = True
tick = time.time()
data.local.usermodel = import_file(data.options.model.filename,
clear_cache=True)
data.local.time_initial_import = time.time() - tick
filter_excepthook = False
usermodel_dir = dir(data.local.usermodel)
data.local._usermodel_plugins = []
for key in modelapi:
if key in usermodel_dir:
class TMP(Plugin):
implements(modelapi[key], service=True)
def __init__(self):
self.fn = getattr(data.local.usermodel, key)
def apply(self, **kwds):
return self.fn(**kwds)
tmp = TMP()
data.local._usermodel_plugins.append(tmp)
if 'pyomo_preprocess' in usermodel_dir:
if data.options.model.object_name in usermodel_dir:
msg = "Preprocessing function 'pyomo_preprocess' defined in file" \
" '%s', but model is already constructed!"
raise SystemExit(msg % data.options.model.filename)
getattr(data.local.usermodel, 'pyomo_preprocess')(options=data.options)
#
return data
def create_model(data):
"""
Create instance of Pyomo model.
Return:
model: Model object.
instance: Problem instance.
symbol_map: Symbol map created when writing model to a file.
filename: Filename that a model instance was written to.
"""
#
if not data.options.runtime.logging == 'quiet':
sys.stdout.write('[%8.2f] Creating model\n' %
(time.time() - start_time))
sys.stdout.flush()
#
if data.options.runtime.profile_memory >= 1 and pympler_available:
global memory_data
mem_used = pympler.muppy.get_size(pympler.muppy.get_objects())
data.local.max_memory = mem_used
print(" Total memory = %d bytes prior to model construction" %
mem_used)
#
# Find the Model objects
#
_models = {}
_model_IDS = set()
for _name, _obj in data.local.usermodel.__dict__.items():
if isinstance(_obj, Model) and id(_obj) not in _model_IDS:
_models[_name] = _obj
_model_IDS.add(id(_obj))
model_name = data.options.model.object_name
if len(_models) == 1:
_name = list(_models.keys())[0]
if model_name is None:
model_name = _name
elif model_name != _name:
msg = "Model '%s' is not defined in file '%s'!"
raise SystemExit(msg % (model_name, data.options.model.filename))
elif len(_models) > 1:
if model_name is None:
msg = "Multiple models defined in file '%s'!"
raise SystemExit(msg % data.options.model.filename)
elif not model_name in _models:
msg = "Unknown model '%s' in file '%s'!"
raise SystemExit(msg % (model_name, data.options.model.filename))
ep = ExtensionPoint(IPyomoScriptCreateModel)
if model_name is None:
if len(ep) == 0:
msg = "A model is not defined and the 'pyomo_create_model' is not "\
"provided in module %s"
raise SystemExit(msg % data.options.model.filename)
elif len(ep) > 1:
msg = 'Multiple model construction plugins have been registered in module %s!'
raise SystemExit(msg % data.options.model.filename)
else:
model_options = data.options.model.options.value()
tick = time.time()
model = ep.service().apply(options=Bunch(*data.options),
model_options=Bunch(*model_options))
if data.options.runtime.report_timing is True:
print(" %6.2f seconds required to construct instance" %
(time.time() - tick))
data.local.time_initial_import = None
tick = time.time()
else:
if model_name not in _models:
msg = "Model '%s' is not defined in file '%s'!"
raise SystemExit(msg % (model_name, data.options.model.filename))
model = _models[model_name]
if model is None:
msg = "'%s' object is 'None' in module %s"
raise SystemExit(msg % (model_name, data.options.model.filename))
elif len(ep) > 0:
msg = "Model construction function 'create_model' defined in " \
"file '%s', but model is already constructed!"
raise SystemExit(msg % data.options.model.filename)
#
# Print model
#
for ep in ExtensionPoint(IPyomoScriptPrintModel):
ep.apply(options=data.options, model=model)
#
# Create Problem Instance
#
ep = ExtensionPoint(IPyomoScriptCreateDataPortal)
if len(ep) > 1:
msg = 'Multiple model data construction plugins have been registered!'
raise SystemExit(msg)
if len(ep) == 1:
modeldata = ep.service().apply(options=data.options, model=model)
else:
modeldata = DataPortal()
if model._constructed:
#
# TODO: use a better test for ConcreteModel
#
instance = model
if data.options.runtime.report_timing is True and not data.local.time_initial_import is None:
print(" %6.2f seconds required to construct instance" %
(data.local.time_initial_import))
else:
tick = time.time()
if len(data.options.data.files) > 1:
#
# Load a list of *.dat files
#
for file in data.options.data.files:
suffix = (file).split(".")[-1]
if suffix != "dat":
msg = 'When specifiying multiple data files, they must all ' \
'be *.dat files. File specified: %s'
raise SystemExit(msg % str(file))
modeldata.load(filename=file, model=model)
instance = model.create_instance(
modeldata,
namespaces=data.options.data.namespaces,
profile_memory=data.options.runtime.profile_memory,
report_timing=data.options.runtime.report_timing)
elif len(data.options.data.files) == 1:
#
# Load a *.dat file or process a *.py data file
#
suffix = (data.options.data.files[0]).split(".")[-1].lower()
if suffix == "dat":
instance = model.create_instance(
data.options.data.files[0],
namespaces=data.options.data.namespaces,
profile_memory=data.options.runtime.profile_memory,
report_timing=data.options.runtime.report_timing)
elif suffix == "py":
userdata = import_file(data.options.data.files[0],
clear_cache=True)
if "modeldata" in dir(userdata):
if len(ep) == 1:
msg = "Cannot apply 'pyomo_create_modeldata' and use the" \
" 'modeldata' object that is provided in the model"
raise SystemExit(msg)
if userdata.modeldata is None:
msg = "'modeldata' object is 'None' in module %s"
raise SystemExit(msg % str(data.options.data.files[0]))
modeldata = userdata.modeldata
else:
if len(ep) == 0:
msg = "Neither 'modeldata' nor 'pyomo_create_dataportal' " \
'is defined in module %s'
raise SystemExit(msg % str(data.options.data.files[0]))
modeldata.read(model)
instance = model.create_instance(
modeldata,
namespaces=data.options.data.namespaces,
profile_memory=data.options.runtime.profile_memory,
report_timing=data.options.runtime.report_timing)
elif suffix == "yml" or suffix == 'yaml':
modeldata = yaml.load(open(data.options.data.files[0]),
**yaml_load_args)
instance = model.create_instance(
modeldata,
namespaces=data.options.data.namespaces,
profile_memory=data.options.runtime.profile_memory,
report_timing=data.options.runtime.report_timing)
else:
raise ValueError("Unknown data file type: " +
data.options.data.files[0])
else:
instance = model.create_instance(
modeldata,
namespaces=data.options.data.namespaces,
profile_memory=data.options.runtime.profile_memory,
report_timing=data.options.runtime.report_timing)
if data.options.runtime.report_timing is True:
print(" %6.2f seconds required to construct instance" %
(time.time() - tick))
#
modify_start_time = time.time()
for ep in ExtensionPoint(IPyomoScriptModifyInstance):
if data.options.runtime.report_timing is True:
tick = time.time()
ep.apply(options=data.options, model=model, instance=instance)
if data.options.runtime.report_timing is True:
print(" %6.2f seconds to apply %s" %
(time.time() - tick, type(ep)))
tick = time.time()
#
for transformation in data.options.transform:
with TransformationFactory(transformation) as xfrm:
instance = xfrm.create_using(instance)
if instance is None:
raise SystemExit("Unexpected error while applying "
"transformation '%s'" % transformation)
#
if data.options.runtime.report_timing is True:
total_time = time.time() - modify_start_time
print(" %6.2f seconds required for problem transformations" %
total_time)
if is_debug_set(logger):
print("MODEL INSTANCE")
instance.pprint()
print("")
for ep in ExtensionPoint(IPyomoScriptPrintInstance):
ep.apply(options=data.options, instance=instance)
fname = None
smap_id = None
if not data.options.model.save_file is None:
if data.options.runtime.report_timing is True:
write_start_time = time.time()
if data.options.model.save_file == True:
if data.local.model_format in (ProblemFormat.cpxlp,
ProblemFormat.lpxlp):
fname = (data.options.data.files[0])[:-3] + 'lp'
else:
fname = (data.options.data.files[0])[:-3] + str(
data.local.model_format)
format = data.local.model_format
else:
fname = data.options.model.save_file
format = data.options.model.save_format
io_options = {}
if data.options.model.symbolic_solver_labels:
io_options['symbolic_solver_labels'] = True
if data.options.model.file_determinism != 1:
io_options[
'file_determinism'] = data.options.model.file_determinism
(fname, smap_id) = instance.write(filename=fname,
format=format,
io_options=io_options)
if not data.options.runtime.logging == 'quiet':
if not os.path.exists(fname):
print("ERROR: file " + fname + " has not been created!")
else:
print("Model written to file '" + str(fname) + "'")
if data.options.runtime.report_timing is True:
total_time = time.time() - write_start_time
print(" %6.2f seconds required to write file" % total_time)
if data.options.runtime.profile_memory >= 2 and pympler_available:
print("")
print(" Summary of objects following file output")
post_file_output_summary = pympler.summary.summarize(
pympler.muppy.get_objects())
pympler.summary.print_(post_file_output_summary, limit=100)
print("")
for ep in ExtensionPoint(IPyomoScriptSaveInstance):
ep.apply(options=data.options, instance=instance)
if data.options.runtime.profile_memory >= 1 and pympler_available:
mem_used = pympler.muppy.get_size(pympler.muppy.get_objects())
if mem_used > data.local.max_memory:
data.local.max_memory = mem_used
print(" Total memory = %d bytes following Pyomo instance creation" %
mem_used)
return Bunch(model=model,
instance=instance,
smap_id=smap_id,
filename=fname,
local=data.local)
def cleanup():
for key in modelapi:
for ep in ExtensionPoint(modelapi[key]):
ep.deactivate()