def test_capture_output_stack_error(self):
OUT1 = StringIO()
OUT2 = StringIO()
old = (sys.stdout, sys.stderr)
try:
a = tee.capture_output(OUT1)
a.setup()
b = tee.capture_output(OUT2)
b.setup()
with self.assertRaisesRegex(RuntimeError, 'Captured output does not match sys.stdout'):
a.reset()
b.tee = None
finally:
sys.stdout, sys.stderr = old
def test_parse(self):
f = FileDownloader()
f.parse_args([])
self.assertFalse(f.insecure)
self.assertIsNone(f.cacert)
self.assertIsNone(f.target)
f = FileDownloader()
f.parse_args(['--insecure'])
self.assertTrue(f.insecure)
self.assertIsNone(f.cacert)
self.assertIsNone(f.target)
f = FileDownloader()
f.parse_args(['--insecure', '--cacert', this_file()])
self.assertTrue(f.insecure)
self.assertEqual(f.cacert, this_file())
self.assertIsNone(f.target)
f = FileDownloader()
f.parse_args(['--insecure', 'bar', '--cacert', this_file()])
self.assertTrue(f.insecure)
self.assertEqual(f.cacert, this_file())
self.assertEqual(f.target, 'bar')
f = FileDownloader()
with capture_output() as io:
with self.assertRaises(SystemExit):
f.parse_args(['--cacert'])
self.assertIn('argument --cacert: expected one argument',
io.getvalue())
f = FileDownloader()
with capture_output() as io:
with self.assertRaises(SystemExit):
f.parse_args(['--cacert', '--insecure'])
self.assertIn('argument --cacert: expected one argument',
io.getvalue())
f = FileDownloader()
with self.assertRaisesRegex(
RuntimeError, "--cacert='nonexistant_file_name' does "
"not refer to a valid file"):
f.parse_args(['--cacert', 'nonexistant_file_name'])
f = FileDownloader()
with capture_output() as io:
with self.assertRaises(SystemExit):
f.parse_args(['--foo'])
self.assertIn('error: unrecognized arguments: --foo', io.getvalue())
def test_reals(self):
point = pyomo.opt.blackbox.RealVars()
point.vars = [1.0]
with capture_output(currdir + 'real_point.out'):
point.display()
self.assertFileEqualsBaseline(currdir + 'real_point.out',
currdir + 'real_point.txt')
def __enter__(self):
_pyomo = logging.getLogger('pyomo')
self.original = (_pyomo.level, _pyomo.handlers)
#
# Configure the logger
#
if self.options.runtime.logging == 'quiet':
_pyomo.setLevel(logging.ERROR)
elif self.options.runtime.logging == 'warning':
_pyomo.setLevel(logging.WARNING)
elif self.options.runtime.logging == 'info':
_pyomo.setLevel(logging.INFO)
elif self.options.runtime.logging == 'verbose':
_pyomo.setLevel(logging.DEBUG)
elif self.options.runtime.logging == 'debug':
_pyomo.setLevel(logging.DEBUG)
elif _pyomo.getEffectiveLevel() == logging.NOTSET:
_pyomo.setLevel(logging.WARNING)
if self.options.runtime.logfile:
_logfile = self.options.runtime.logfile
self.fileLogger = logging.FileHandler(_logfile, 'w')
_pyomo.handlers = []
_pyomo.addHandler(self.fileLogger)
# TBD: This seems dangerous in Windows, as the process will
# have multiple open file handles pointing to the same file.
self.capture = capture_output(_logfile)
self.capture.setup()
return self
def run_solverfactory_test():
skip_solvers = {
'py',
'xpress',
'_xpress_shell',
'_mock_xpress',
}
with LoggingIntercept() as LOG, capture_output(capture_fd=True) as OUT:
info = []
for solver in sorted(pyo.SolverFactory):
_doc = pyo.SolverFactory.doc(solver)
if _doc is not None and 'DEPRECATED' in _doc:
_avail = 'DEPR'
elif solver in skip_solvers:
_avail = 'SKIP'
else:
_avail = str(pyo.SolverFactory(solver).available(False))
info.append(" %s(%s): %s" % (solver, _avail, _doc))
#_check_log_and_out(LOG, OUT, 20, solver)
glpk = pyo.SolverFactory('glpk')
print("")
print("Pyomo Solvers")
print("-------------")
print("\n".join(info))
if type(glpk.available(False)) != bool:
print("Solver glpk.available() did not return bool")
sys.exit(3)
_check_log_and_out(LOG, OUT, 20)
def available(self, exception_flag=True):
if not gurobipy_available:
if exception_flag:
gurobipy.log_import_warning(logger=__name__)
raise ApplicationError(
"No Python bindings available for %s solver plugin" %
(type(self), ))
return False
if self._verified_license is None:
with capture_output(capture_fd=True) as OUT:
try:
# verify that we can get a Gurobi license
# Gurobipy writes out license file information when creating
# the environment
m = gurobipy.Model()
m.dispose()
GurobiDirect._verified_license = True
except Exception as e:
GurobiDirect._import_messages += \
"\nCould not create Model - gurobi message=%s\n" % (e,)
GurobiDirect._verified_license = False
if OUT.getvalue():
GurobiDirect._import_messages += "\n" + OUT.getvalue()
if exception_flag and not self._verified_license:
logger.warning(GurobiDirect._import_messages)
raise ApplicationError(
"Could not create a gurobipy Model for %s solver plugin" %
(type(self), ))
return self._verified_license
def solver_log(logger, level=logging.ERROR):
"""Context manager to send solver output to a logger. This uses a separate
thread to log solver output while the solver is running"""
# wait 3 seconds to join thread. Should be plenty of time. In case
# something goes horribly wrong though don't want to hang. The logging
# thread is daemonic, so it will shut down with the main process even if it
# stays around for some mysterious reason while the model is running.
join_timeout = 3
tee = logger.isEnabledFor(level)
if not solver_capture():
yield SolverLogInfo(tee=tee)
else:
with capture_output() as s:
lt = IOToLogTread(s, logger=logger, level=level)
lt.start()
try:
yield SolverLogInfo(tee=tee, thread=lt)
except:
lt.stop.set()
lt.join(timeout=join_timeout)
raise
# thread should end when s is closed, but setting stop makes sure
# the last of the output gets logged before closing s
lt.stop.set()
lt.join(timeout=join_timeout)
def test_help_solvers(self):
with capture_output() as OUT:
help_solvers()
OUT = OUT.getvalue()
self.assertTrue(re.search('Pyomo Solvers and Solver Managers', OUT))
self.assertTrue(re.search('Serial Solver', OUT))
# Test known solvers and metasolver flags
# ASL is a metasolver
self.assertTrue(re.search('\n \*asl ', OUT))
# PS is bundles with Pyomo so should always be available
self.assertTrue(re.search('\n \+ps ', OUT))
for solver in ('ipopt', 'cbc', 'glpk'):
s = SolverFactory(solver)
if s.available():
self.assertTrue(
re.search("\n \+%s " % solver, OUT),
"' +%s' not found in help --solvers" % solver)
else:
self.assertTrue(
re.search("\n %s " % solver, OUT),
"' %s' not found in help --solvers" % solver)
for solver in ('baron', ):
s = SolverFactory(solver)
if s.license_is_valid():
self.assertTrue(
re.search("\n \+%s " % solver, OUT),
"' +%s' not found in help --solvers" % solver)
elif s.available():
self.assertTrue(
re.search("\n \-%s " % solver, OUT),
"' -%s' not found in help --solvers" % solver)
else:
self.assertTrue(
re.search("\n %s " % solver, OUT),
"' %s' not found in help --solvers" % solver)
def pyomo(self, cmd, **kwds):
if 'root' in kwds:
OUTPUT = kwds['root'] + '.out'
results = kwds['root'] + '.jsn'
TempfileManager.add_tempfile(OUTPUT, exists=False)
TempfileManager.add_tempfile(results, exists=False)
else:
OUTPUT = StringIO()
results = 'results.jsn'
TempfileManager.create_tempfile(suffix='results.jsn')
with capture_output(OUTPUT):
try:
_dir = os.getcwd()
os.chdir(currdir)
args = ['solve', '--solver=glpk', '--results-format=json',
'--save-results=%s' % results]
if type(cmd) is list:
args.extend(cmd)
elif cmd.endswith('json') or cmd.endswith('yaml'):
args.append(cmd)
else:
args.extend(re.split('[ ]+', cmd))
output = main.main(args)
finally:
os.chdir(_dir)
if not 'root' in kwds:
return OUTPUT.getvalue()
return output
def test_warm_start(self):
m = ConcreteModel()
m.x = Var()
m.z = Var(domain=Integers)
m.w = Var(domain=Boolean)
m.c = Constraint(expr=m.x + m.z + m.w >= 0)
m.o = Objective(expr=m.x + m.z + m.w)
# Set some initial values for warm start.
m.x.set_value(10)
m.z.set_value(5)
m.w.set_value(1)
with SolverFactory("cbc") as opt:
with capture_output() as output:
opt.solve(m, tee=True, warmstart=True)
# Check if CBC loaded the warmstart file.
self.assertIn('opening mipstart file', output.getvalue())
# Only integer and binary variables are considered by CBC.
self.assertIn('MIPStart values read for 2 variables.',
output.getvalue())
# m.x is ignored because it is continuous, so cost should be 5+1
self.assertIn('MIPStart provided solution with cost 6',
output.getvalue())
def pyomo(self, cmd, **kwds):
if 'root' in kwds:
OUTPUT = kwds['root'] + '.out'
results = kwds['root'] + '.jsn'
self.ofile = OUTPUT
else:
OUTPUT = StringIO()
results = 'results.jsn'
with capture_output(OUTPUT):
os.chdir(currdir)
if type(cmd) is list:
output = main.main([
'solve', '--solver=glpk', '--results-format=json',
'--save-results=%s' % results
] + cmd)
elif cmd.endswith('json') or cmd.endswith('yaml'):
output = main.main([
'solve', '--results-format=json',
'--save-results=%s' % results
] + [cmd])
else:
args = re.split('[ ]+', cmd)
output = main.main([
'solve', '--solver=glpk', '--results-format=json',
'--save-results=%s' % results
] + list(args))
if not 'root' in kwds:
return OUTPUT.getvalue()
return output
def _runner(q, qualname):
"Utility wrapper for running functions, used by timeout()"
resultType = _RunnerResult.call
if q in _runner.data:
fcn, args, kwargs = _runner.data[q]
elif isinstance(qualname, str):
# Use unittest to instantiate the TestCase and run it
resultType = _RunnerResult.unittest
def fcn():
s = _unittest.TestLoader().loadTestsFromName(qualname)
r = _unittest.TestResult()
s.run(r)
return r.errors + r.failures, r.skipped
args = ()
kwargs = {}
else:
qualname, fcn, args, kwargs = qualname
_runner.data[qualname] = None
OUT = StringIO()
try:
with capture_output(OUT):
result = fcn(*args, **kwargs)
q.put((resultType, result, OUT.getvalue()))
except:
import traceback
etype, e, tb = sys.exc_info()
if not isinstance(e, AssertionError):
e = etype("%s\nOriginal traceback:\n%s" %
(e, ''.join(traceback.format_tb(tb))))
q.put((_RunnerResult.exception, e, OUT.getvalue()))
finally:
_runner.data.pop(qualname)
def _test_disc_first(self, tname):
ofile = join(currdir, tname + '.' + self.sim_mod + '.out')
bfile = join(currdir, tname + '.' + self.sim_mod + '.txt')
with capture_output(ofile):
# create model
exmod = import_file(join(exdir, tname + '.py'))
m = exmod.create_model()
# Discretize model
discretizer = TransformationFactory('dae.collocation')
discretizer.apply_to(m, nfe=10, ncp=5)
# Simulate model
sim = Simulator(m, package=self.sim_mod)
if hasattr(m, 'var_input'):
tsim, profiles = sim.simulate(numpoints=100,
varying_inputs=m.var_input)
else:
tsim, profiles = sim.simulate(numpoints=100)
# Initialize model
sim.initialize_model()
self._print(m, profiles)
if not os.path.exists(bfile):
os.rename(ofile, bfile)
# os.system('diff ' + ofile + ' ' + bfile)
self.assertFileEqualsBaseline(ofile, bfile, tolerance=0.01)
def _apply_solver(self):
StaleFlagManager.mark_all_as_stale()
self._solver_model.setlogfile(self._log_file)
if self._keepfiles:
print("Solver log file: " + self._log_file)
# Setting a log file in xpress disables all output
# in xpress versions less than 36.
# This callback prints all messages to stdout
# when using those xpress versions.
if self._tee and XpressDirect._version[0] < 36:
self._solver_model.addcbmessage(_print_message, None, 0)
# set xpress options
# if the user specifies a 'mipgap', set it, and
# set xpress's related options to 0.
if self.options.mipgap is not None:
self._solver_model.setControl('miprelstop',
float(self.options.mipgap))
self._solver_model.setControl('miprelcutoff', 0.0)
self._solver_model.setControl('mipaddcutoff', 0.0)
# xpress is picky about the type which is passed
# into a control. So we will infer and cast
# get the xpress valid controls
xp_controls = xpress.controls
for key, option in self.options.items():
if key == 'mipgap': # handled above
continue
try:
self._solver_model.setControl(key, option)
except XpressDirect.XpressException:
# take another try, converting to its type
# we'll wrap this in a function to raise the
# xpress error
contr_type = type(getattr(xp_controls, key))
if not _is_convertable(contr_type, option):
raise
self._solver_model.setControl(key, contr_type(option))
start_time = time.time()
if self._tee:
self._solver_model.solve()
else:
# In xpress versions greater than or equal 36,
# it seems difficult to completely suppress console
# output without disabling logging altogether.
# As a work around, we capature all screen output
# when tee is False.
with capture_output() as OUT:
self._solver_model.solve()
self._opt_time = time.time() - start_time
self._solver_model.setlogfile('')
if self._tee and XpressDirect._version[0] < 36:
self._solver_model.removecbmessage(_print_message, None)
# FIXME: can we get a return code indicating if XPRESS had a significant failure?
return Bunch(rc=None, log=None)
def test_help_transformations(self):
with capture_output() as OUT:
help_transformations()
OUT = OUT.getvalue()
self.assertTrue(re.search('Pyomo Model Transformations', OUT))
self.assertTrue(re.search('core.relax_integer_vars', OUT))
# test a transformation that we know is deprecated
self.assertTrue(re.search('duality.linear_dual\s+\[DEPRECATED\]', OUT))
def test_duplicate_capture_output(self):
out = StringIO()
capture = tee.capture_output(out)
capture.setup()
try:
with self.assertRaisesRegex(RuntimeError, 'Duplicate call to capture_output.setup'):
capture.setup()
finally:
capture.reset()
def test_mi(self):
point = pyomo.opt.blackbox.MixedIntVars()
point.reals = [1.0]
point.ints = [1.0]
point.bits = [0]
with capture_output(currdir + 'mi_point.out'):
point.display()
self.assertFileEqualsBaseline(currdir + 'mi_point.out',
currdir + 'mi_point.txt')
def test_capture_output_logfile_string(self):
with TempfileManager.new_context() as tempfile:
logfile = tempfile.create_tempfile()
self.assertTrue(isinstance(logfile, str))
with tee.capture_output(logfile):
print('HELLO WORLD')
with open(logfile, 'r') as f:
result = f.read()
self.assertEqual('HELLO WORLD\n', result)
def test_report_timing(self):
# Create a set to ensure that the global sets have already been
# constructed (this is an issue until the new set system is
# merged in and the GlobalSet objects are not automatically
# created by pyomo.core
m = ConcreteModel()
m.x = Var([1,2])
ref = r"""
(0(\.\d+)?) seconds to construct Block ConcreteModel; 1 index total
(0(\.\d+)?) seconds to construct RangeSet FiniteSimpleRangeSet; 1 index total
(0(\.\d+)?) seconds to construct Var x; 2 indices total
(0(\.\d+)?) seconds to construct Suffix Suffix; 1 index total
(0(\.\d+)?) seconds to apply Transformation RelaxIntegerVars \(in-place\)
""".strip()
xfrm = TransformationFactory('core.relax_integer_vars')
try:
with capture_output() as out:
report_timing()
m = ConcreteModel()
m.r = RangeSet(2)
m.x = Var(m.r)
xfrm.apply_to(m)
result = out.getvalue().strip()
self.maxDiff = None
for l, r in zip(result.splitlines(), ref.splitlines()):
self.assertRegex(str(l.strip()), str(r.strip()))
finally:
report_timing(False)
os = StringIO()
try:
report_timing(os)
m = ConcreteModel()
m.r = RangeSet(2)
m.x = Var(m.r)
xfrm.apply_to(m)
result = os.getvalue().strip()
self.maxDiff = None
for l, r in zip(result.splitlines(), ref.splitlines()):
self.assertRegex(str(l.strip()), str(r.strip()))
finally:
report_timing(False)
buf = StringIO()
with LoggingIntercept(buf, 'pyomo'):
m = ConcreteModel()
m.r = RangeSet(2)
m.x = Var(m.r)
xfrm.apply_to(m)
result = os.getvalue().strip()
self.maxDiff = None
for l, r in zip(result.splitlines(), ref.splitlines()):
self.assertRegex(str(l.strip()), str(r.strip()))
self.assertEqual(buf.getvalue().strip(), "")
def _test(self, tname, M):
ofile = currdir + tname + '_%s.out' % str(self.xfrm)
bfile = currdir + tname + '_%s.txt' % str(self.xfrm)
if self.xfrm is not None:
xfrm = TransformationFactory(self.xfrm)
xfrm.apply_to(M)
with capture_output(ofile):
self._print(M)
if not os.path.exists(bfile):
os.rename(ofile, bfile)
self.assertFileEqualsBaseline(ofile, bfile)
def test_capture_output_logfile_string(self):
logfile = TempfileManager.create_tempfile()
self.assertTrue(isinstance(logfile, str))
try:
with tee.capture_output(logfile):
print('HELLO WORLD')
with open(logfile, 'r') as f:
result = f.read()
self.assertEqual('HELLO WORLD\n', result)
finally:
TempfileManager.clear_tempfiles()
def pyomo ( self, cmd, **kwargs):
args = ['solve'] + re.split('[ ]+', cmd )
out = kwargs.get( 'file', None )
if out is None:
out = StringIO()
with capture_output(out):
os.chdir( currdir )
output = main( args )
if not 'file' in kwargs:
return output.getvalue()
return output
def test_test_functions(self):
with capture_output() as OUT:
self.assertTrue(self.compare_files(
os.path.join(currdir,'tests','ref1.txt'),
os.path.join(currdir,'tests','ref2.txt'),
))
self.assertTrue(self.compare_files(
os.path.join(currdir,'tests','ref2.txt'),
os.path.join(currdir,'tests','ref1.txt'),
))
self.assertEqual(OUT.getvalue(), "")
with self.assertRaises(self.failureException):
with capture_output() as OUT:
self.compare_files(
os.path.join(currdir,'tests','ref1.txt'),
os.path.join(currdir,'tests','ref2.txt'),
abstol=1e-10,
)
self.assertIn('BASELINE FILE', OUT.getvalue())
self.assertIn('TEST OUTPUT FILE', OUT.getvalue())
def test_display(self):
model = ConcreteModel()
model.e = Expression()
with capture_output() as out:
model.e.display()
self.assertEqual(out.getvalue().strip(), """
e : Size=1
Key : Value
None : Undefined
""".strip())
model.e.set_value(1.0)
with capture_output() as out:
model.e.display()
self.assertEqual(out.getvalue().strip(), """
e : Size=1
Key : Value
None : 1.0
""".strip())
out = StringIO()
with capture_output() as no_out:
model.e.display(ostream=out)
self.assertEqual(no_out.getvalue(), "")
self.assertEqual(out.getvalue().strip(), """
e : Size=1
Key : Value
None : 1.0
""".strip())
model.E = Expression([1,2])
with capture_output() as out:
model.E.display()
self.assertEqual(out.getvalue().strip(), """
E : Size=2
Key : Value
1 : Undefined
2 : Undefined
""".strip())
model.E[1].set_value(1.0)
with capture_output() as out:
model.E.display()
self.assertEqual(out.getvalue().strip(), """
E : Size=2
Key : Value
1 : 1.0
2 : Undefined
""".strip())
out = StringIO()
with capture_output() as no_out:
model.E.display(ostream=out)
self.assertEqual(no_out.getvalue(), "")
self.assertEqual(out.getvalue().strip(), """
E : Size=2
Key : Value
1 : 1.0
2 : Undefined
""".strip())
def __call__(self):
_cwd = os.getcwd()
try:
with capture_output() as OUT:
import xpress
finally:
# In some versions of XPRESS (notably 8.9.0), `import
# xpress` temporarily changes the CWD. If the import fails
# (e.g., due to an expired license), the CWD is not always
# restored. This block ensures that the CWD is preserved.
os.chdir(_cwd)
self.import_message += OUT.getvalue()
return xpress
def run_writer_test():
with LoggingIntercept() as LOG, capture_output(capture_fd=True) as OUT:
# Enumerate the writers...
from pyomo.opt import WriterFactory
info = []
for writer in sorted(WriterFactory):
info.append(" %s: %s" % (writer, WriterFactory.doc(writer)))
_check_log_and_out(LOG, OUT, 10, writer)
print("Pyomo Problem Writers")
print("---------------------")
print('\n'.join(info))
with LoggingIntercept() as LOG, capture_output(capture_fd=True) as OUT:
# Test a writer
m = pyo.ConcreteModel()
m.x = pyo.Var()
m.c = pyo.Constraint(expr=m.x >= 1)
m.o = pyo.Objective(expr=m.x**2)
from pyomo.common.tempfiles import TempfileManager
with TempfileManager:
fname = TempfileManager.create_tempfile(suffix='pyomo.lp')
m.write(fname)
with open(fname, 'r') as FILE:
data = FILE.read()
if not all(d.strip() == b.strip()
for d, b in zip(data.strip().splitlines(),
_baseline.strip().splitlines())):
print("Result did not match baseline.\nRESULT:\n%s\nBASELINE:\n%s" %
(data, _baseline))
print(data.strip().splitlines())
print(_baseline.strip().splitlines())
sys.exit(2)
_check_log_and_out(LOG, OUT, 10)
def convert(self, cmd, type, **kwds):
args = re.split('[ ]+', cmd)
args.append(
"--symbolic-solver-labels") # for readability / quick inspections
if 'file' in kwds:
OUTPUT = kwds['file']
else:
OUTPUT = StringIO()
with capture_output(OUTPUT):
os.chdir(currdir)
if type == 'lp':
output = main.pyomo2lp(list(args))
else:
output = main.pyomo2nl(list(args))
if not 'file' in kwds:
return OUTPUT.getvalue()
return output.retval, output.errorcode
def _test(self, tname, M):
ofile = currdir + tname + '_%s.out' % str(self.xfrm)
bfile = currdir + tname + '_%s.txt' % str(self.xfrm)
if self.xfrm is not None:
xfrm = TransformationFactory(self.xfrm)
xfrm.apply_to(M)
with capture_output(ofile):
self._print(M)
if not os.path.exists(bfile):
os.rename(ofile, bfile)
try:
self.assertTrue(cmp(ofile, bfile),
msg="Files %s and %s differ" % (ofile, bfile))
except:
with open(ofile, 'r') as f1, open(bfile, 'r') as f2:
f1_contents = list(filter(None, f1.read().split()))
f2_contents = list(filter(None, f2.read().split()))
self.assertEqual(f1_contents, f2_contents)
def test_caputure_output_fd(self):
r, w = os.pipe()
os.dup2(w, 1)
sys.stdout = os.fdopen(1, 'w', closefd=False)
with tee.capture_output(capture_fd=True) as OUT:
sys.stdout.write("to_stdout_1\n")
sys.stdout.flush()
with os.fdopen(1, 'w', closefd=False) as F:
F.write("to_fd1_1\n")
F.flush()
sys.stdout.write("to_stdout_2\n")
sys.stdout.flush()
with os.fdopen(1, 'w', closefd=False) as F:
F.write("to_fd1_2\n")
F.flush()
self.assertEqual(OUT.getvalue(), "to_stdout_1\nto_fd1_1\n")
with os.fdopen(r, 'r') as FILE:
os.close(1)
os.close(w)
self.assertEqual(FILE.read(), "to_stdout_2\nto_fd1_2\n")
def test_log_callback(self):
h = self.get_basic_model()
h.setOptionValue('log_to_console', True)
class Foo(object):
def __str__(self):
return 'an instance of Foo'
def __repr__(self):
return self.__str__()
def log_callback(log_type, message, data):
print('got a log message: ', log_type, data, message)
h.setLogCallback(log_callback, Foo())
out = StringIO()
with capture_output(out) as t:
h.run()
out = out.getvalue()
self.assertIn(
'got a log message: HighsLogType.kInfo an instance of Foo Presolving model',
out)