def _find_reference_model_or_callback(src):
"""Tries to find a single reference model or callback for
generating scenario models."""
module = import_file(src, clear_cache=True)
reference_model = None
callback = None
dir_module = dir(module)
if "pysp_instance_creation_callback" in dir_module:
callback = getattr(module, "pysp_instance_creation_callback")
if not hasattr(callback, "__call__"):
raise TypeError("'pysp_instance_creation_callback' "
"object found in source '%s' is not "
"callable" % (src))
else:
matching_names = []
for attr_name in dir_module:
obj = getattr(module, attr_name)
if isinstance(obj, (_BlockData, Block)):
reference_model = obj
matching_names.append(attr_name)
if len(matching_names) > 1:
raise ValueError("Multiple objects found in source '%s' "
"that could be a reference model. Make "
"sure there is only one Pyomo model in "
"the source file. Object names: %s" %
(str(matching_names)))
return module, reference_model, callback
def import_install_module(download_dest):
"""
Imports the path in download_dest (install_idaes_workshop_materials.py module)
"""
install_module = futils.import_file(download_dest)
print('... importing install module')
return install_module
def _test_disc_first(self, tname):
ofile = join(currdir, tname + '.' + self.sim_mod + '.out')
bfile = join(currdir, tname + '.' + self.sim_mod + '.txt')
with capture_output(ofile):
# create model
exmod = import_file(join(exdir, tname + '.py'))
m = exmod.create_model()
# Discretize model
discretizer = TransformationFactory('dae.collocation')
discretizer.apply_to(m, nfe=10, ncp=5)
# Simulate model
sim = Simulator(m, package=self.sim_mod)
if hasattr(m, 'var_input'):
tsim, profiles = sim.simulate(numpoints=100,
varying_inputs=m.var_input)
else:
tsim, profiles = sim.simulate(numpoints=100)
# Initialize model
sim.initialize_model()
self._print(m, profiles)
if not os.path.exists(bfile):
os.rename(ofile, bfile)
# os.system('diff ' + ofile + ' ' + bfile)
self.assertFileEqualsBaseline(ofile, bfile, tolerance=0.01)
def test_LOA_8PP_fixed_disjuncts(self):
"""Test LOA with 8PP using fixed disjuncts initialization."""
exfile = import_file(
join(exdir, 'eight_process', 'eight_proc_model.py'))
eight_process = exfile.build_eight_process_flowsheet()
initialize = [
# Use units 1, 4, 7, 8
eight_process.use_unit_1or2.disjuncts[0],
eight_process.use_unit_3ornot.disjuncts[1],
eight_process.use_unit_4or5ornot.disjuncts[0],
eight_process.use_unit_6or7ornot.disjuncts[1],
eight_process.use_unit_8ornot.disjuncts[0]
]
for disj in eight_process.component_data_objects(Disjunct):
if disj in initialize:
disj.indicator_var.set_value(1)
else:
disj.indicator_var.set_value(0)
SolverFactory('gdpopt').solve(eight_process,
strategy='LOA',
init_strategy='fix_disjuncts',
mip_solver=mip_solver,
nlp_solver=nlp_solver)
self.assertTrue(fabs(value(eight_process.profit.expr) - 68) <= 1E-2)
def run_bilevel(self, *args, **kwds):
module = import_file(args[0])
instance = module.pyomo_create_model(None, None)
xfrm = TransformationFactory('bilevel.linear_mpec')
xfrm.apply_to(instance, deterministic=True)
instance.pprint(filename=join(currdir, self.problem +
'_linear_mpec.out'))
def test_init18(self):
self.assertTrue("reference_test_model" not in sys.modules)
self.assertTrue("ScenarioStructure" not in sys.modules)
nx_tree = import_file(os.path.join(testdatadir,
"ScenarioStructure.py")).G
def scenario_model_callback(scenario_tree, scenario_name, node_list):
self.assertIs(scenario_tree, nx_tree)
instance = reference_test_model.create_instance()
if scenario_name == "s1":
instance.p = 1.0
elif scenario_name == "s2":
instance.p = 2.0
else:
assert scenario_name == "s3"
instance.p = 3.0
return instance
with ScenarioTreeInstanceFactory(model=scenario_model_callback,
scenario_tree=nx_tree) as factory:
self.assertTrue(factory.model_directory() is None)
self.assertTrue(factory.scenario_tree_directory() is None)
self._check_factory(factory)
self.assertEqual(factory._closed, True)
self.assertEqual(len(factory._archives), 0)
def test_nine_process(self):
"""Test with the nine process problem model."""
exfile = import_file(join(exdir, 'nine_process', 'small_process.py'))
simple_model = exfile.build_model()
simple_model_size = build_model_size_report(simple_model)
self.assertEqual(simple_model_size.overall.variables, 34)
self.assertEqual(simple_model_size.activated.variables, 34)
def test_8PP_deactive(self):
exfile = import_file(
join(exdir, 'eight_process', 'eight_proc_model.py'))
m = exfile.build_eight_process_flowsheet()
for djn in m.component_data_objects(ctype=Disjunction):
djn.deactivate()
self.assertTrue(satisfiable(m) is not False)
def testMethod(obj):
if not testing_solvers[solver, writer]:
obj.skipTest("Solver %s (interface=%s) is not available" %
(solver, writer))
m = import_file(os.path.join(thisDir, 'problems', problem + '.py'),
clear_cache=True)
model = m.define_model(**kwds)
opt = pyomo.opt.SolverFactory(solver, solver_io=writer)
results = opt.solve(model)
# non-recursive
new_results = ((var.name, var.value)
for var in model.component_data_objects(
Var, active=True, descend_into=False))
baseline_results = getattr(obj, problem + '_results')
for name, value in new_results:
if abs(baseline_results[name] - value) > 0.0001:
raise IOError("Difference in baseline solution values and "
"current solution values using:\n" + \
"Solver: "+solver+"\n" + \
"Writer: "+writer+"\n" + \
"Variable: "+name+"\n" + \
"Solution: "+str(value)+"\n" + \
"Baseline: "+str(baseline_results[name])+"\n")
def _test_disc_first(self, tname):
bfile = join(currdir, tname + '.' + self.sim_mod + '.json')
# create model
exmod = import_file(join(exdir, tname + '.py'))
m = exmod.create_model()
# Discretize model
discretizer = TransformationFactory('dae.collocation')
discretizer.apply_to(m, nfe=10, ncp=5)
# Simulate model
sim = Simulator(m, package=self.sim_mod)
if hasattr(m, 'var_input'):
tsim, profiles = sim.simulate(numpoints=100,
varying_inputs=m.var_input)
else:
tsim, profiles = sim.simulate(numpoints=100)
# Initialize model
sim.initialize_model()
results = self._store_results(m, profiles)
# Used to regenerate baseline files
if not os.path.exists(bfile):
with open(bfile, 'w') as f1:
json.dump(results, f1)
# Compare results to baseline
with open(bfile, 'r') as f2:
baseline = json.load(f2)
self.assertStructuredAlmostEqual(results, baseline, abstol=1e-2)
def apply_postprocessing(data, instance=None, results=None):
"""
Apply post-processing steps.
Required:
instance: Problem instance.
results: Optimization results object.
"""
#
if not data.options.runtime.logging == 'quiet':
sys.stdout.write('[%8.2f] Applying Pyomo postprocessing actions\n' %
(time.time() - start_time))
sys.stdout.flush()
# options are of type ConfigValue, not raw strings / atomics.
for config_value in data.options.postprocess:
postprocess = import_file(config_value, clear_cache=True)
if "pyomo_postprocess" in dir(postprocess):
postprocess.pyomo_postprocess(data.options, instance, results)
for ep in ExtensionPoint(IPyomoScriptPostprocess):
ep.apply(options=data.options, instance=instance, results=results)
if data.options.runtime.profile_memory >= 1 and pympler_available:
mem_used = pympler.muppy.get_size(pympler.muppy.get_objects())
if mem_used > data.local.max_memory:
data.local.max_memory = mem_used
print(" Total memory = %d bytes upon termination" % mem_used)
def test_external_grey_box_react_example_maximize_cb_outputs_scaling(self):
ex = import_file(os.path.join(example_dir, 'external_grey_box', 'react_example', 'maximize_cb_ratio_residuals.py'))
aoptions={'nlp_scaling_method': 'user-scaling',
'output_file': '_cyipopt-external-greybox-react-scaling.log',
'file_print_level':10}
m = ex.maximize_cb_ratio_residuals_with_output_scaling(additional_options=aoptions)
self.assertAlmostEqual(pyo.value(m.reactor.inputs['sv']), 1.26541996, places=3)
self.assertAlmostEqual(pyo.value(m.reactor.inputs['cb']), 1071.7410089, places=2)
self.assertAlmostEqual(pyo.value(m.reactor.outputs['cb_ratio']), 0.15190409266, places=3)
with open('_cyipopt-external-greybox-react-scaling.log', 'r') as fd:
solver_trace = fd.read()
os.remove('_cyipopt-external-greybox-react-scaling.log')
self.assertIn('nlp_scaling_method = user-scaling', solver_trace)
self.assertIn('output_file = _cyipopt-external-greybox-react-scaling.log', solver_trace)
self.assertIn('objective scaling factor = 1', solver_trace)
self.assertIn('x scaling provided', solver_trace)
self.assertIn('c scaling provided', solver_trace)
self.assertIn('d scaling provided', solver_trace)
self.assertIn('DenseVector "x scaling vector" with 7 elements:', solver_trace)
self.assertIn('x scaling vector[ 2]= 1.2000000000000000e+00', solver_trace)
self.assertIn('x scaling vector[ 7]= 1.7000000000000000e+00', solver_trace)
self.assertIn('x scaling vector[ 6]= 1.1000000000000001e+00', solver_trace)
self.assertIn('x scaling vector[ 1]= 1.3000000000000000e+00', solver_trace)
self.assertIn('x scaling vector[ 3]= 1.3999999999999999e+00', solver_trace)
self.assertIn('x scaling vector[ 4]= 1.5000000000000000e+00', solver_trace)
self.assertIn('x scaling vector[ 5]= 1.6000000000000001e+00', solver_trace)
self.assertIn('DenseVector "c scaling vector" with 6 elements:', solver_trace)
self.assertIn('c scaling vector[ 1]= 4.2000000000000000e+01', solver_trace)
self.assertIn('c scaling vector[ 2]= 1.0000000000000001e-01', solver_trace)
self.assertIn('c scaling vector[ 3]= 2.0000000000000001e-01', solver_trace)
self.assertIn('c scaling vector[ 4]= 2.9999999999999999e-01', solver_trace)
self.assertIn('c scaling vector[ 5]= 4.0000000000000002e-01', solver_trace)
self.assertIn('c scaling vector[ 6]= 1.0000000000000000e+01', solver_trace)
def test_cyipopt_functor(self):
ex = import_file(os.path.join(example_dir, 'callback', 'cyipopt_functor_callback.py'))
df = ex.main()
self.assertEqual(df.shape, (7, 5))
# check one of the residuals
s = df['ca_bal']
self.assertAlmostEqual(s.iloc[6], 0, places=3)
def test_import_vars(self):
import_ex = import_file(os.path.join(_this_file_dir, 'import_ex.py'))
try:
importvar = import_ex.a
except:
self.fail(
'test_import_vars - failed to access data in import_ex.py file.'
)
def test_cyipopt_callbacks(self):
ex = import_file(
os.path.join(example_dir, 'callback', 'cyipopt_callback.py'))
output = StringIO()
with LoggingIntercept(output, 'pyomo', logging.INFO):
ex.main()
self.assertIn("Residuals for iteration 2", output.getvalue().strip())
def test_external_grey_box_react_example_maximize_with_hessian_with_output_pyomo(
self):
ex = import_file(
os.path.join(example_dir, 'external_grey_box', 'react_example',
'maximize_cb_ratio_residuals.py'))
m = ex.maximize_cb_ratio_residuals_with_hessian_with_output_pyomo()
self.assertAlmostEqual(pyo.value(m.sv), 1.26541996, places=3)
self.assertAlmostEqual(pyo.value(m.cb), 1071.7410089, places=2)
self.assertAlmostEqual(pyo.value(m.cb_ratio), 0.15190409266, places=3)
def test_parameter_estimation(self):
data_fname = os.path.join(example_dir, 'external_grey_box', 'param_est', 'smalldata.csv')
baseline = pandas.read_csv(data_fname)
# test the data generator
ex = import_file(os.path.join(example_dir, 'external_grey_box', 'param_est', 'generate_data.py'))
df1 = ex.generate_data(5, 200, 5, 42)
df2 = ex.generate_data_external(5, 200, 5, 42)
pandas.testing.assert_frame_equal(df1, baseline, atol=1e-3)
pandas.testing.assert_frame_equal(df2, baseline, atol=1e-3)
# test the estimation
ex = import_file(os.path.join(example_dir, 'external_grey_box', 'param_est', 'perform_estimation.py'))
m = ex.perform_estimation_external(data_fname, solver_trace=False)
self.assertAlmostEqual(pyo.value(m.UA), 204.43761, places=3)
m = ex.perform_estimation_pyomo_only(data_fname, solver_trace=False)
self.assertAlmostEqual(pyo.value(m.UA), 204.43761, places=3)
def test_LOA_strip_pack_default_init(self):
"""Test logic-based outer approximation with strip packing."""
exfile = import_file(
join(exdir, 'strip_packing', 'strip_packing_concrete.py'))
strip_pack = exfile.build_rect_strip_packing_model()
SolverFactory('gdpopt').solve(strip_pack,
strategy='LOA',
mip_solver=mip_solver,
nlp_solver=nlp_solver)
self.assertTrue(fabs(value(strip_pack.total_length.expr) - 11) <= 1E-2)
def test_GLOA_nonconvex_HENS(self):
exfile = import_file(join(exdir, 'small_lit', 'nonconvex_HEN.py'))
model = exfile.build_gdp_model()
SolverFactory('gdpopt').solve(
model, strategy='GLOA',
mip_solver=mip_solver,
nlp_solver=global_nlp_solver,
nlp_solver_args=global_nlp_solver_args,
tee=False)
objective_value = value(model.objective.expr)
self.assertAlmostEqual(objective_value * 1E-5, 1.14385, 2)
def test_RIC_8PP_maxBinary(self):
"""Test logic-based OA with max_binary initialization."""
exfile = import_file(
join(exdir, 'eight_process', 'eight_proc_model.py'))
eight_process = exfile.build_eight_process_flowsheet()
SolverFactory('gdpopt').solve(
eight_process, strategy='RIC', init_strategy='max_binary',
mip_solver=mip_solver,
nlp_solver=nlp_solver)
self.assertTrue(fabs(value(eight_process.profit.expr) - 68) <= 1E-2)
def test_RIC_strip_pack_maxBinary(self):
"""Test RIC with strip packing using max_binary initialization."""
exfile = import_file(
join(exdir, 'strip_packing', 'strip_packing_concrete.py'))
strip_pack = exfile.build_rect_strip_packing_model()
SolverFactory('gdpopt').solve(
strip_pack, strategy='RIC', init_strategy='max_binary',
mip_solver=mip_solver,
nlp_solver=nlp_solver)
self.assertTrue(
fabs(value(strip_pack.total_length.expr) - 11) <= 1E-2)
def test_RIC_8PP_default_init(self):
"""Test logic-based outer approximation with 8PP."""
exfile = import_file(
join(exdir, 'eight_process', 'eight_proc_model.py'))
eight_process = exfile.build_eight_process_flowsheet()
SolverFactory('gdpopt').solve(
eight_process, strategy='RIC',
mip_solver=mip_solver,
nlp_solver=nlp_solver,
tee=False)
self.assertTrue(fabs(value(eight_process.profit.expr) - 68) <= 1E-2)
def test_RIC_8PP_force_NLP(self):
exfile = import_file(
join(exdir, 'eight_process', 'eight_proc_model.py'))
eight_process = exfile.build_eight_process_flowsheet()
SolverFactory('gdpopt').solve(
eight_process, strategy='RIC',
mip_solver=mip_solver,
nlp_solver=nlp_solver,
force_subproblem_nlp=True,
tee=False)
self.assertTrue(fabs(value(eight_process.profit.expr) - 68) <= 1E-2)
def test_external_grey_box_react_example_maximize_cb_outputs(self):
ex = import_file(
os.path.join(example_dir, 'external_grey_box', 'react_example',
'maximize_cb_outputs.py'))
m = ex.maximize_cb_outputs()
self.assertAlmostEqual(pyo.value(m.reactor.inputs['sv']),
1.34381,
places=3)
self.assertAlmostEqual(pyo.value(m.reactor.outputs['cb']),
1072.4372,
places=2)
def test_GLOA_ex_633_trespalacios(self):
"""Test LOA with Francisco thesis example."""
exfile = import_file(join(exdir, 'small_lit', 'ex_633_trespalacios.py'))
model = exfile.build_simple_nonconvex_gdp()
SolverFactory('gdpopt').solve(
model, strategy='GLOA',
mip_solver=mip_solver,
nlp_solver=global_nlp_solver,
nlp_solver_args=global_nlp_solver_args,
tee=False)
objective_value = value(model.obj.expr)
self.assertAlmostEqual(objective_value, 4.46, 2)
def test_GLOA_8PP(self):
"""Test the global logic-based outer approximation algorithm."""
exfile = import_file(
join(exdir, 'eight_process', 'eight_proc_model.py'))
eight_process = exfile.build_eight_process_flowsheet()
SolverFactory('gdpopt').solve(
eight_process, strategy='GLOA', tee=False,
mip_solver=mip_solver,
nlp_solver=global_nlp_solver,
nlp_solver_args=global_nlp_solver_args
)
self.assertTrue(fabs(value(eight_process.profit.expr) - 68) <= 1E-2)
def test_RIC_8PP_gams_solver(self):
# Make sure that the duals are still correct
exfile = import_file(
join(exdir, 'eight_process', 'eight_proc_model.py'))
eight_process = exfile.build_eight_process_flowsheet()
SolverFactory('gdpopt').solve(
eight_process, strategy='RIC',
mip_solver=mip_solver,
nlp_solver='gams',
max_slack=0,
tee=False)
self.assertTrue(fabs(value(eight_process.profit.expr) - 68) <= 1E-2)
def test_pyomo_react_example_maximize_with_obj(self):
ex = import_file(
os.path.join(example_dir, 'external_grey_box', 'react_example',
'maximize_cb_ratio_residuals.py'))
m = ex.maximize_cb_ratio_residuals_with_obj()
self.assertAlmostEqual(pyo.value(m.reactor.inputs['sv']),
1.26541996,
places=3)
self.assertAlmostEqual(pyo.value(m.reactor.inputs['cb']),
1071.7410089,
places=2)
self.assertAlmostEqual(pyo.value(m.obj), 0.15190409266, places=3)
def test_improper_basic_step(self):
model_builder = import_file(
join(exdir, 'two_rxn_lee', 'two_rxn_model.py'))
m = model_builder.build_model()
m.basic_step = apply_basic_step([m.reactor_choice, m.max_demand])
for disj in m.basic_step.disjuncts.values():
self.assertEqual(
disj.improper_constraints[1].body.polynomial_degree(), 2)
self.assertEqual(disj.improper_constraints[1].lower, None)
self.assertEqual(disj.improper_constraints[1].upper, 2)
self.assertEqual(len(disj.improper_constraints), 1)
self.assertFalse(m.max_demand.active)
def test_LBB_8PP(self):
"""Test the logic-based branch and bound algorithm."""
exfile = import_file(
join(exdir, 'eight_process', 'eight_proc_model.py'))
eight_process = exfile.build_eight_process_flowsheet()
SolverFactory('gdpopt').solve(
eight_process, tee=False,
strategy='LBB',
minlp_solver=minlp_solver,
minlp_solver_args=minlp_args,
)
self.assertTrue(fabs(value(eight_process.profit.expr) - 68) <= 1E-2)
