def _presolve(self, *args, **kwds):
"""
Peform presolves.
"""
TempfileManager.push()
self._keepfiles = kwds.pop("keepfiles", False)
self._define_signal_handlers = kwds.pop('use_signal_handling', None)
OptSolver._presolve(self, *args, **kwds)
#
# Verify that the input problems exists
#
for filename in self._problem_files:
if not os.path.exists(filename):
msg = 'Solver failed to locate input problem file: %s'
raise ValueError(msg % filename)
#
# Create command line
#
self._command = self.create_command_line(self.executable(),
self._problem_files)
self._log_file = self._command.log_file
#
# The pre-cleanup is probably unncessary, but also not harmful.
#
if (self._log_file is not None) and \
os.path.exists(self._log_file):
os.remove(self._log_file)
if (self._soln_file is not None) and \
os.path.exists(self._soln_file):
os.remove(self._soln_file)
def _postsolve(self):
# take care of the annoying GUROBI log file in the current
# directory. this approach doesn't seem overly efficient, but
# python os module functions doesn't accept regular expression
# directly.
filename_list = os.listdir(".")
for filename in filename_list:
# IMPT: trap the possible exception raised by the file not
# existing. this can occur in pyro environments where
# > 1 workers are running GUROBI, and were started
# from the same directory. these logs don't matter
# anyway (we redirect everything), and are largely an
# annoyance.
try:
if re.match('gurobi\.log', filename) != None:
os.remove(filename)
except OSError:
pass
# let the base class deal with returning results.
results = ILMLicensedSystemCallSolver._postsolve(self)
# finally, clean any temporary files registered with the temp file
# manager, created populated *directly* by this plugin. does not
# include, for example, the execution script. but does include
# the warm-start file.
TempfileManager.pop(remove=not self._keepfiles)
return results
def tearDown(self):
if os.path.exists("unknown.lp"):
os.unlink("unknown.lp")
TempfileManager.clear_tempfiles()
if os.path.exists(os.path.join(currdir,'result.yml')):
os.remove(os.path.join(currdir,'result.yml'))
self.model = None
def solve(self, sp, *args, **kwds):
"""
Solve a stochastic program.
See the 'solve' method on the base class for
additional keyword documentation.
Args:
sp: The stochastic program to solve.
keep_solver_files (bool): Retain temporary solver
input and output files after the solve completes.
*args: Passed to the derived solver class
(see the _solve_impl method).
**kwds: Passed to the derived solver class
(see the _solve_impl method).
Returns: A results object with information about the solution.
"""
self._files.clear()
assert self.executable is not None
keep_solver_files = kwds.pop("keep_solver_files", False)
TempfileManager.push()
try:
return super(SPSolverShellCommand, self).solve(sp, *args, **kwds)
finally:
# cleanup
TempfileManager.pop(remove=not keep_solver_files)
if keep_solver_files:
logger.info("Retaining the following solver files:\n" +
pprint.pformat(self.files))
def test_open_tempfile_windows(self):
TempfileManager.push()
fname = TempfileManager.create_tempfile()
f = open(fname)
try:
_orig = tempfiles.deletion_errors_are_fatal
tempfiles.deletion_errors_are_fatal = True
with self.assertRaisesRegex(WindowsError,
".*process cannot access the file"):
TempfileManager.pop()
finally:
tempfiles.deletion_errors_are_fatal = _orig
f.close()
os.remove(fname)
TempfileManager.push()
fname = TempfileManager.create_tempfile()
f = open(fname)
log = StringIO()
try:
_orig = tempfiles.deletion_errors_are_fatal
tempfiles.deletion_errors_are_fatal = False
with LoggingIntercept(log, 'pyomo.common'):
TempfileManager.pop()
self.assertIn("Unable to delete temporary file", log.getvalue())
finally:
tempfiles.deletion_errors_are_fatal = _orig
f.close()
os.remove(fname)
def tearDown(self):
global tempdir
TempfileManager.pop()
TempfileManager.tempdir = old_tempdir
if os.path.exists(tempdir):
shutil.rmtree(tempdir)
tempdir = None
def test_scip_solve_from_instance_options(self):
# Creating a dummy scip.set file in the cwd
# will cover the code that prints a warning
_cwd = os.getcwd()
tmpdir = TempfileManager.create_tempdir()
try:
os.chdir(tmpdir)
open(join(tmpdir, 'scip.set'), "w").close()
# Test scip solve from a pyomo instance and load the solution
with LoggingIntercept() as LOG:
results = self.scip.solve(self.model,
suffixes=['.*'],
options={"limits/softtime": 100})
self.assertRegex(
LOG.getvalue().replace("\n", " "),
r"A file named (.*) exists in the current working "
r"directory, but SCIP options are being "
r"set using a separate options file. The "
r"options file \1 will be ignored.")
finally:
os.chdir(_cwd)
# We don't want the test to care about which Scip version we are using
self.model.solutions.store_to(results)
results.Solution(0).Message = "Scip"
results.Solver.Message = "Scip"
results.Solver.Time = 0
_out = TempfileManager.create_tempfile(".txt")
results.write(filename=_out, times=False, format='json')
self.compare_json(
_out, join(currdir, "test_scip_solve_from_instance.baseline"))
def test_add1(self):
"""Test explicit adding of a file that is missing"""
try:
TempfileManager.add_tempfile(tempdir + 'add1')
self.fail("Expected IOError because file 'add1' does not exist")
except IOError:
pass
def _postsolve(self):
# take care of the annoying (and empty) CPLEX temporary files in the current directory.
# this approach doesn't seem overly efficient, but python os module functions don't
# accept regular expression directly.
filename_list = os.listdir(".")
for filename in filename_list:
# CPLEX temporary files come in two flavors - cplex.log and clone*.log.
# the latter is the case for multi-processor environments.
# IMPT: trap the possible exception raised by the file not existing.
# this can occur in pyro environments where > 1 workers are
# running CPLEX, and were started from the same directory.
# these logs don't matter anyway (we redirect everything),
# and are largely an annoyance.
try:
if re.match('cplex\.log', filename) != None:
os.remove(filename)
elif re.match('clone\d+\.log', filename) != None:
os.remove(filename)
except OSError:
pass
# let the base class deal with returning results.
results = ILMLicensedSystemCallSolver._postsolve(self)
# finally, clean any temporary files registered with the temp file
# manager, created populated *directly* by this plugin. does not
# include, for example, the execution script. but does include
# the warm-start file.
TempfileManager.pop(remove=not self._keepfiles)
return results
def apply(self, *args, **kwargs):
"""
Run the external pico_convert utility
"""
if len(args) != 3:
raise ConverterError("Cannot apply pico_convert with more than one filename or model")
_exe = pyomo.common.Executable("pico_convert")
if not _exe:
raise ConverterError("The 'pico_convert' application cannot be found")
pico_convert_cmd = _exe.path()
target=str(args[1])
if target=="cpxlp":
target="lp"
# NOTE: if you have an extra "." in the suffix, the pico_convert program fails to output to the correct filename.
output_filename = TempfileManager.create_tempfile(suffix = 'pico_convert.' + target)
if not isinstance(args[2], str):
fname= TempfileManager.create_tempfile(suffix= 'pico_convert.' +str(args[0]))
args[2].write(filename=fname, format=args[1])
cmd = pico_convert_cmd +" --output="+output_filename+" "+target+" "+fname
else:
cmd = pico_convert_cmd +" --output="+output_filename+" "+target
for item in args[2:]:
if not os.path.exists(item):
raise ConverterError("File "+item+" does not exist!")
cmd = cmd + " "+item
print("Running command: "+cmd)
subprocess.run(cmd, stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL)
if not os.path.exists(output_filename): #pragma:nocover
raise ApplicationError(\
"Problem launching 'pico_convert' to create "+output_filename)
return (output_filename,),None # no variable map at the moment
def create_command_line(self, executable, problem_files):
#
# Not the best place to catch this, but we want to make sure
# that we have done the version check somewhere
#
if executable not in self._known_versions:
self._known_versions[executable] = self._get_version(executable)
_ver = self._known_versions[executable]
if not _ver or _ver < (4, 58):
raise RuntimeError(
"Pyomo only supports versions of GLPK since 4.58; "
"found version %s. Please upgrade your installation "
"of GLPK" % ('.'.join(map(str, _ver)), ))
#
# Define log file
#
if self._log_file is None:
self._log_file = TempfileManager.create_tempfile(
suffix='.glpk.log')
#
# Define solution file
#
self._glpfile = TempfileManager.create_tempfile(suffix='.glpk.glp')
self._rawfile = TempfileManager.create_tempfile(suffix='.glpk.raw')
self._soln_file = self._rawfile
#
# Define command line
#
cmd = [executable]
if self._timer:
cmd.insert(0, self._timer)
for key in self.options:
opt = self.options[key]
if opt is None or (isinstance(opt, str) and opt.strip() == ''):
# Handle the case for options that must be
# specified without a value
cmd.append("--%s" % key)
else:
cmd.extend(["--%s" % key, str(opt)])
if self._timelimit is not None and self._timelimit > 0.0:
cmd.extend(['--tmlim', str(self._timelimit)])
cmd.extend(['--write', self._rawfile])
cmd.extend(['--wglp', self._glpfile])
if self._problem_format == ProblemFormat.cpxlp:
cmd.extend(['--cpxlp', problem_files[0]])
elif self._problem_format == ProblemFormat.mps:
cmd.extend(['--freemps', problem_files[0]])
elif self._problem_format == ProblemFormat.mod:
cmd.extend(['--math', problem_files[0]])
for fname in problem_files[1:]:
cmd.extend(['--data', fname])
return Bunch(cmd=cmd, log_file=self._log_file, env=None)
def do_setup(self, flag):
TempfileManager.push()
if flag:
if not cplexamp_available:
self.skipTest("The 'cplexamp' command is not available")
self.asl = SolverFactory('asl:cplexamp')
else:
self.asl = SolverFactory('_mock_asl:cplexamp')
def _run_ipopt_with_stats(model, solver, max_iter=500, max_cpu_time=120):
"""
Run the solver (must be ipopt) and return the convergence statistics
Parameters
----------
model : Pyomo model
The pyomo model to be solved
solver : Pyomo solver
The pyomo solver to use - it must be ipopt, but with whichever options
are preferred
max_iter : int
The maximum number of iterations to allow for ipopt
max_cpu_time : int
The maximum cpu time to allow for ipopt (in seconds)
Returns
-------
Returns a tuple with (solve status object, bool (solve successful or
not), number of iters, solve time)
"""
# ToDo: Check that the "solver" is, in fact, IPOPT
TempfileManager.push()
tempfile = TempfileManager.create_tempfile(suffix='ipopt_out', text=True)
opts = {
'output_file': tempfile,
'max_iter': max_iter,
'max_cpu_time': max_cpu_time
}
status_obj = solver.solve(model, options=opts, tee=True)
solved = True
if status_obj.solver.termination_condition != TerminationCondition.optimal:
solved = False
iters = 0
time = 0
# parse the output file to get the iteration count, solver times, etc.
with open(tempfile, 'r') as f:
for line in f:
if line.startswith('Number of Iterations....:'):
tokens = line.split()
iters = int(tokens[3])
elif line.startswith(
'Total CPU secs in IPOPT (w/o function evaluations) ='):
tokens = line.split()
time += float(tokens[9])
elif line.startswith(
'Total CPU secs in NLP function evaluations ='):
tokens = line.split()
time += float(tokens[8])
TempfileManager.pop(remove=True)
return status_obj, solved, iters, time
def _presolve(self, *args, **kwds):
# create a context in the temporary file manager for
# this plugin - is "pop"ed in the _postsolve method.
TempfileManager.push()
# if the first argument is a string (representing a filename),
# then we don't have an instance => the solver is being applied
# to a file.
self._warm_start_solve = kwds.pop('warmstart', False)
self._warm_start_file_name = kwds.pop('warmstart_file', None)
user_warmstart = False
if self._warm_start_file_name is not None:
user_warmstart = True
# the input argument can currently be one of two things: an
# instance or a filename. if a filename is provided and a
# warm-start is indicated, we go ahead and create the temporary
# file - assuming that the user has already, via some external
# mechanism, invoked warm_start() with a instance to create the
# warm start file.
if self._warm_start_solve and \
isinstance(args[0], str):
# we assume the user knows what they are doing...
pass
elif self._warm_start_solve and \
(not isinstance(args[0], str)):
# assign the name of the warm start file *before* calling
# the base class presolve - the base class method ends up
# creating the command line, and the warm start file-name is
# (obviously) needed there.
if self._warm_start_file_name is None:
assert not user_warmstart
self._warm_start_file_name = TempfileManager.create_tempfile(
suffix='.gurobi.mst')
# let the base class handle any remaining keywords/actions.
ILMLicensedSystemCallSolver._presolve(self, *args, **kwds)
# NB: we must let the base class presolve run first so that the
# symbol_map is actually constructed!
if (len(args) > 0) and (not isinstance(args[0], str)):
if len(args) != 1:
raise ValueError(
"GUROBI _presolve method can only handle a single "
"problem instance - %s were supplied" % (len(args), ))
# write the warm-start file - currently only supports MIPs.
# we only know how to deal with a single problem instance.
if self._warm_start_solve and (not user_warmstart):
start_time = time.time()
self._warm_start(args[0])
end_time = time.time()
if self._report_timing is True:
print("Warm start write time=%.2f seconds" %
(end_time - start_time))
def run_pyomo(self, cmd, root):
results = root + '.jsn'
TempfileManager.add_tempfile(results, exists=False)
output = root + '.out'
TempfileManager.add_tempfile(output, exists=False)
cmd = ['pyomo', 'solve', '--solver=glpk', '--results-format=json',
'--save-results=%s' % results] + cmd
with open(output, 'w') as f:
result = subprocess.run(cmd, stdout=f, stderr=f)
return result
def solve(self, model, timer: HierarchicalTimer = None):
avail = self.available()
if not avail:
raise PyomoException(
f'Solver {self.__class__} is not available ({avail}).')
if self._last_results_object is not None:
self._last_results_object.solution_loader.invalidate()
if timer is None:
timer = HierarchicalTimer()
try:
TempfileManager.push()
if self.config.filename is None:
self._filename = TempfileManager.create_tempfile()
else:
self._filename = self.config.filename
TempfileManager.add_tempfile(self._filename + '.lp', exists=False)
TempfileManager.add_tempfile(self._filename + '.log', exists=False)
timer.start('write lp file')
self._writer.write(model, self._filename + '.lp', timer=timer)
timer.stop('write lp file')
res = self._apply_solver(timer)
self._last_results_object = res
if self.config.report_timing:
logger.info('\n' + str(timer))
return res
finally:
# finally, clean any temporary files registered with the
# temp file manager, created/populated *directly* by this
# plugin.
TempfileManager.pop(remove=not self.config.keepfiles)
if not self.config.keepfiles:
self._filename = None
def solve(self, model, timer: HierarchicalTimer = None):
self.available(exception_flag=True)
if timer is None:
timer = HierarchicalTimer()
try:
TempfileManager.push()
if self.config.filename is None:
self._filename = TempfileManager.create_tempfile()
else:
self._filename = self.config.filename
TempfileManager.add_tempfile(self._filename + '.lp', exists=False)
TempfileManager.add_tempfile(self._filename + '.log', exists=False)
timer.start('write lp file')
self._writer.write(model, self._filename + '.lp', timer=timer)
timer.stop('write lp file')
res = self._apply_solver(timer)
if self.config.report_timing:
logger.info('\n' + str(timer))
return res
finally:
# finally, clean any temporary files registered with the
# temp file manager, created/populated *directly* by this
# plugin.
TempfileManager.pop(remove=not self.config.keepfiles)
if not self.config.keepfiles:
self._filename = None
def setUp(self):
if not scip_available:
self.skipTest("The 'scipampl' command is not available")
TempfileManager.push()
self.scip = SolverFactory('scip', solver_io='nl')
m = self.model = ConcreteModel()
m.v = Var()
m.o = Objective(expr=m.v)
m.c = Constraint(expr=m.v >= 1)
def __enter__(self):
self._cwd = os.getcwd()
# Add a new context
TempfileManager.push()
# Create a new tempdir in this context
self._tempdir = TempfileManager.create_tempdir(
suffix=self._suffix,
prefix=self._prefix,
dir=self._dir,
)
os.chdir(self._tempdir)
def do_setup(self, flag):
global tmpdir
tmpdir = os.getcwd()
os.chdir(currdir)
TempfileManager.sequential_files(0)
if flag:
if not cplexamp_available:
self.skipTest("The 'cplexamp' command is not available")
self.asl = SolverFactory('asl:cplexamp')
else:
self.asl = SolverFactory('_mock_asl:cplexamp')
def test_capture_output_logfile_string(self):
logfile = TempfileManager.create_tempfile()
self.assertTrue(isinstance(logfile, str))
try:
with tee.capture_output(logfile):
print('HELLO WORLD')
with open(logfile, 'r') as f:
result = f.read()
self.assertEqual('HELLO WORLD\n', result)
finally:
TempfileManager.clear_tempfiles()
def test_ipopt_solve_from_nl(self):
# Test ipopt solve from nl file
_log = TempfileManager.create_tempfile(".test_ipopt.log")
results = self.ipopt.solve(join(currdir, "sisser.pyomo.nl"),
logfile=_log,
suffixes=['.*'])
# We don't want the test to care about which Ipopt version we are using
results.Solution(0).Message = "Ipopt"
results.Solver.Message = "Ipopt"
_out = TempfileManager.create_tempfile(".test_ipopt.txt")
results.write(filename=_out, times=False, format='json')
self.compare_json(_out, join(currdir, "test_solve_from_nl.baseline"))
def do_setup(self):
global tmpdir
tmpdir = os.getcwd()
os.chdir(currdir)
TempfileManager.sequential_files(0)
self.scip = SolverFactory('scip', solver_io='nl')
m = self.model = ConcreteModel()
m.v = Var()
m.o = Objective(expr=m.v)
m.c = Constraint(expr=m.v >= 1)
def _postsolve(self):
# let the base class deal with returning results.
results = super(CBCSHELL, self)._postsolve()
# finally, clean any temporary files registered with the temp file
# manager, created populated *directly* by this plugin. does not
# include, for example, the execution script. but does include
# the warm-start file.
TempfileManager.pop(remove=not self._keepfiles)
return results
def create_command_line(self, executable, problem_files):
#
# Define log file
#
if self._log_file is None:
self._log_file = TempfileManager.create_tempfile(
suffix='.glpk.log')
#
# Define solution file
#
self._glpfile = TempfileManager.create_tempfile(suffix='.glpk.glp')
self._rawfile = TempfileManager.create_tempfile(suffix='.glpk.raw')
self._soln_file = self._rawfile
#
# Define command line
#
cmd = [executable]
if self._timer:
cmd.insert(0, self._timer)
for key in self.options:
opt = self.options[key]
if opt is None or (isinstance(opt, string_types)
and opt.strip() == ''):
# Handle the case for options that must be
# specified without a value
cmd.append("--%s" % key)
else:
cmd.extend(["--%s" % key, str(opt)])
#if isinstance(opt, basestring) and ' ' in opt:
# cmd.append('--%s "%s"' % (key, str(opt)))
#else:
# cmd.append('--%s %s' % (key, str(opt)))
if self._timelimit is not None and self._timelimit > 0.0:
cmd.extend(['--tmlim', str(self._timelimit)])
cmd.extend(['--write', self._rawfile])
cmd.extend(['--wglp', self._glpfile])
if self._problem_format == ProblemFormat.cpxlp:
cmd.extend(['--cpxlp', problem_files[0]])
elif self._problem_format == ProblemFormat.mps:
cmd.extend(['--freemps', problem_files[0]])
elif self._problem_format == ProblemFormat.mod:
cmd.extend(['--math', problem_files[0]])
for fname in problem_files[1:]:
cmd.extend(['--data', fname])
return Bunch(cmd=cmd, log_file=self._log_file, env=None)
def test_solve4(self):
""" Test ASL - test4.nl """
_log = TempfileManager.create_tempfile(".test_solve4.log")
_out = TempfileManager.create_tempfile(".test_solve4.txt")
results = self.asl.solve(join(currdir, "test4.nl"),
logfile=_log,
suffixes=['.*'])
results.write(filename=_out, times=False, format='json')
_baseline = join(currdir, "test4_asl.txt")
with open(_out, 'r') as out, open(_baseline, 'r') as txt:
self.assertStructuredAlmostEqual(json.load(txt), json.load(out),
abstol=1e-4,
allow_second_superset=True)
def _presolve(self, **kwds):
warmstart_flag = kwds.pop('warmstart', False)
self._keepfiles = kwds.pop('keepfiles', False)
self._save_results = kwds.pop('save_results', True)
self._integer_only_warmstarts = kwds.pop('integer_only_warmstarts',
False)
# create a context in the temporary file manager for
# this plugin - is "pop"ed in the _postsolve method.
TempfileManager.push()
self.results = None
model = self._pyomo_model
# this implies we have a custom solution "parser",
# preventing the OptSolver _presolve method from
# creating one
self._results_format = ResultsFormat.soln
# use the base class _presolve to consume the
# important keywords
OptSolver._presolve(self, **kwds)
# ***********************************************************
# The following code is only needed for backwards compatability of load_solutions=False.
# If we ever only want to support the load_vars, load_duals, etc. methods, then this can be deleted.
if self._save_results:
self._smap_id = id(self._symbol_map)
if isinstance(self._pyomo_model, IBlock):
# BIG HACK (see pyomo.core.kernel write function)
if not hasattr(self._pyomo_model, "._symbol_maps"):
setattr(self._pyomo_model, "._symbol_maps", {})
getattr(self._pyomo_model,
"._symbol_maps")[self._smap_id] = self._symbol_map
else:
self._pyomo_model.solutions.add_symbol_map(self._symbol_map)
# ***********************************************************
if warmstart_flag:
if self.warm_start_capable():
self._warm_start()
else:
raise ValueError(
'{0} solver plugin is not capable of warmstart.'.format(
type(self)))
if self._log_file is None:
self._log_file = TempfileManager.create_tempfile(suffix='.log')
def test_load_local_asl_library(self):
DLL = find_GSL()
if not DLL:
self.skipTest("Could not find the amplgsl.dll library")
LIB = 'test_pyomo_external_gsl.dll'
model = ConcreteModel()
model.gamma = ExternalFunction(library=LIB, function="gsl_sf_gamma")
model.x = Var(initialize=3, bounds=(1e-5, None))
model.o = Objective(expr=model.gamma(model.x))
with TempfileManager.new_context() as tempfile:
dname = tempfile.mkdtemp()
shutil.copyfile(DLL, os.path.join(dname, LIB))
# Without changing directories, the load should fail
with self.assertRaises(OSError):
value(model.o)
# Changing directories should pick up the library
try:
orig_dir = os.getcwd()
os.chdir(dname)
self.assertAlmostEqual(value(model.o), 2.0, 7)
finally:
os.chdir(orig_dir)
def generate_scenario_tree_image(options):
with ScenarioTreeInstanceFactory(
options.model_location, options.scenario_tree_location) as factory:
scenario_tree = factory.generate_scenario_tree(
downsample_fraction=options.scenario_tree_downsample_fraction,
bundles=options.scenario_bundle_specification,
random_bundles=options.create_random_bundles,
random_seed=options.scenario_tree_random_seed,
verbose=options.verbose)
with TempfileManager.push():
tmpdotfile = TempfileManager.create_tempfile(suffix=".dot")
scenario_tree.save_to_dot(tmpdotfile)
os.system('dot -Tpdf -o %s %s' % (options.output_file, tmpdotfile))
print("Output Saved To: %s" % (options.output_file))
def test3_write_lp(self):
""" Convert from AMPL to LP """
self.model = pyomo.opt.AmplModel(join(currdir, 'test3.mod'))
_test = TempfileManager.create_tempfile(suffix='test3.lp')
try:
self.model.write(_test)
except ApplicationError:
err = sys.exc_info()[1]
if pyomo.common.Executable("glpsol"):
self.fail("Unexpected ApplicationError - glpsol is enabled "
"but not available: '%s'" % str(err))
return
except pyomo.opt.ConverterError:
err = sys.exc_info()[1]
if pyomo.common.Executable("glpsol"):
self.fail("Unexpected ConverterError - glpsol is enabled "
"but not available: '%s'" % str(err))
return
_base = join(currdir, 'test3.baseline.lp')
with open(_test, 'r') as run, open(_base, 'r') as baseline:
for line1, line2 in zip_longest(run, baseline):
for _pattern in ('Problem:', ):
if line1.find(_pattern) >= 0:
line1 = line1[:line1.find(_pattern) + len(_pattern)]
line2 = line2[:line2.find(_pattern) + len(_pattern)]
self.assertEqual(line1,
line2,
msg="Files %s and %s differ" % (_test, _base))
