def _execute_command(self,command):
"""
Execute the command
"""
start_time = time.time()
try:
if 'script' in command:
_input = command.script
else:
_input = None
[rc, log] = run(
command.cmd,
stdin = _input,
timelimit = self._timelimit if self._timelimit is None else self._timelimit + max(1, 0.01*self._timelimit),
env = command.env,
tee = self._tee
)
except WindowsError:
err = sys.exc_info()[1]
msg = 'Could not execute the command: %s\tError message: %s'
raise ApplicationError(msg % (command.cmd, err))
sys.stdout.flush()
self._last_solve_time = time.time() - start_time
return [rc,log]
def available(self, exception_flag=True):
if not (FD_available and OO_available):
if exception_flag:
raise ApplicationError(
'Cannot execute solver without FuncDesigner and OpenOpt installed'
)
return False
return OptSolver.available(self, exception_flag)
def available(self, exception_flag=False):
""" True if the solver is available """
if self._assert_available:
return True
if not OptSolver.available(self, exception_flag):
return False
ans = self.executable()
if ans is None:
if exception_flag:
msg = "No executable found for solver '%s'"
raise ApplicationError(msg % self.name)
return False
return True
def set_callback(self, name, callback_fn=None):
"""
Set the callback function for a named callback.
A call-back function has the form:
def fn(solver, model):
pass
where 'solver' is the native solver interface object and 'model' is
a Pyomo model instance object.
"""
if not self._allow_callbacks:
raise ApplicationError("Callbacks disabled for solver %s" %
self.name)
if callback_fn is None:
if name in self._callback:
del self._callback[name]
else:
self._callback[name] = callback_fn
def run_command(cmd,
outfile=None,
cwd=None,
ostream=None,
stdin=None,
stdout=None,
stderr=None,
valgrind=False,
valgrind_log=None,
valgrind_options=None,
memmon=False,
env=None,
define_signal_handlers=None,
debug=False,
verbose=True,
timelimit=None,
tee=None,
ignore_output=False,
shell=False,
thread_reader=None):
#
# Set the define_signal_handlers based on the global default flag.
#
if define_signal_handlers is None:
define_signal_handlers = GlobalData.DEFINE_SIGNAL_HANDLERS_DEFAULT
#
# Move to the specified working directory
#
if cwd is not None:
oldpwd = os.getcwd()
os.chdir(cwd)
cmd_type = type(cmd)
if cmd_type is list:
# make a private copy of the list
_cmd = cmd[:]
elif cmd_type is tuple:
_cmd = list(cmd)
else:
_cmd = quote_split(cmd.strip())
#
# Setup memmoon
#
if memmon:
memmon = pyutilib.services.registered_executable("memmon")
if memmon is None:
raise IOError("Unable to find the 'memmon' executable")
_cmd.insert(0, memmon.get_path())
#
# Setup valgrind
#
if valgrind:
#
# The valgrind_log option specifies a logfile that is used to store
# valgrind output.
#
valgrind_cmd = pyutilib.services.registered_executable("valgrind")
if valgrind_cmd is None:
raise IOError("Unable to find the 'valgrind' executable")
valgrind_cmd = [valgrind_cmd.get_path()]
if valgrind_options is None:
valgrind_cmd.extend(
("-v", "--tool=memcheck", "--trace-children=yes"))
elif type(valgrind_options) in (list, tuple):
valgrind_cmd.extend(valgrind_options)
else:
valgrind_cmd.extend(quote_split(valgrind_options.strip()))
if valgrind_log is not None:
valgrind_cmd.append("--log-file-exactly=" + valgrind_log.strip())
_cmd = valgrind_cmd + _cmd
#
# Redirect stdout and stderr
#
tmpfile = None
if ostream is not None:
stdout_arg = stderr_arg = ostream
if outfile is not None or stdout is not None or stderr is not None:
raise ValueError("subprocess.run_command(): ostream, outfile, and "
"{stdout, stderr} options are mutually exclusive")
output = "Output printed to specified ostream"
elif outfile is not None:
stdout_arg = stderr_arg = open(outfile, "w")
if stdout is not None or stderr is not None:
raise ValueError("subprocess.run_command(): outfile and "
"{stdout, stderr} options are mutually exclusive")
output = "Output printed to file '%s'" % outfile
elif not (stdout is None and stderr is None):
stdout_arg = stdout
stderr_arg = stderr
output = "Output printed to specified stdout and stderr streams"
else:
# Create a temporary file. The mode is w+, which means that we
# can read and write.
# NOTE: the default mode is w+b, but writing to the binary mode
# seems to cause problems in the _stream_reader function on Python
# 3.x.
stdout_arg = stderr_arg = tmpfile = tempfile.TemporaryFile(mode='w+')
output = ""
if stdout_arg is stderr_arg:
try:
if not tee or (not tee[0] and not tee[1]):
stderr_arg = STDOUT
except:
pass
#
# Setup the default environment
#
if env is None:
env = os.environ.copy()
#
# Setup signal handler
#
if define_signal_handlers:
handler = verbose_signal_handler if verbose else signal_handler
if sys.platform[0:3] != "win" and sys.platform[0:4] != 'java':
GlobalData.original_signal_handlers[signal.SIGHUP] \
= signal.signal(signal.SIGHUP, handler)
GlobalData.original_signal_handlers[signal.SIGINT] \
= signal.signal(signal.SIGINT, handler)
GlobalData.original_signal_handlers[signal.SIGTERM] \
= signal.signal(signal.SIGTERM, handler)
rc = -1
if debug:
print("Executing command %s" % (_cmd, ))
try:
try:
simpleCase = not tee
if stdout_arg is not None:
stdout_arg.fileno()
if stderr_arg is not None:
stderr_arg.fileno()
except:
simpleCase = False
out_th = []
th = None
GlobalData.signal_handler_busy = False
if simpleCase:
#
# Redirect IO to the stdout_arg/stderr_arg files
#
process = SubprocessMngr(_cmd,
stdin=stdin,
stdout=stdout_arg,
stderr=stderr_arg,
env=env,
shell=shell)
GlobalData.current_process = process.process
rc = process.wait(timelimit)
GlobalData.current_process = None
else:
#
# Aggressively wait for output from the process, and
# send this to both the stdout/stdarg value, as well
# as doing a normal 'print'
#
out_fd = []
for fid in (0, 1):
if fid == 0:
s, raw = stdout_arg, sys.stdout
else:
s, raw = stderr_arg, sys.stderr
try:
tee_fid = tee[fid]
except:
tee_fid = tee
if s is None or s is STDOUT:
out_fd.append(s)
elif not tee_fid:
# This catches using StringIO as an output buffer:
# Python's subprocess requires the stream objects to
# have a "fileno()" attribute, which StringIO does
# not have. We will mock things up by putting a
# pipe in between the subprocess and the StringIO
# buffer. <sigh>
#
#if hasattr(s, 'fileno'):
#
# Update: in Python 3, StringIO declares a fileno()
# method, but that method throws an exception. So,
# we can't just check for the attribute: we *must*
# call the method and see if we get an exception.
try:
s.fileno()
out_fd.append(s)
except:
r, w = os.pipe()
out_fd.append(w)
out_th.append(((fid, r, s), r, w))
#th = Thread(target=thread_reader, args=(r,None,s,fid))
#out_th.append((th, r, w))
else:
r, w = os.pipe()
out_fd.append(w)
out_th.append(((fid, r, raw, s), r, w))
#th = Thread( target=thread_reader, args=(r,raw,s,fid) )
#out_th.append((th, r, w))
#
process = SubprocessMngr(_cmd,
stdin=stdin,
stdout=out_fd[0],
stderr=out_fd[1],
env=env,
shell=shell)
GlobalData.current_process = process.process
GlobalData.signal_handler_busy = False
#
# Create a thread to read in stdout and stderr data
#
if out_th:
if thread_reader is not None:
reader = thread_reader
elif len(out_th) == 1:
reader = _stream_reader
elif _peek_available:
reader = _merged_reader
else:
reader = _pseudo_merged_reader
th = Thread(target=reader, args=[x[0] for x in out_th])
th.daemon = True
th.start()
#
# Wait for process to finish
#
rc = process.wait(timelimit)
GlobalData.current_process = None
out_fd = None
except _WindowsError:
err = sys.exc_info()[1]
raise ApplicationError(
"Could not execute the command: '%s'\n\tError message: %s" %
(' '.join(_cmd), err))
except OSError:
#
# Ignore IOErrors, which are caused by interupts
#
pass
finally:
# restore the previous signal handlers, if necessary
for _sig in list(GlobalData.original_signal_handlers):
signal.signal(_sig, GlobalData.original_signal_handlers.pop(_sig))
#
# Flush stdout/stderr. Some platforms (notably Matlab, which
# replaces stdout with a MexPrinter) have stdout/stderr that do not
# implement flush() See https://github.com/Pyomo/pyomo/issues/156
#
try:
sys.stdout.flush()
except AttributeError:
pass
try:
sys.stderr.flush()
except AttributeError:
pass
if out_th:
#
# 'Closing' the PIPE to send EOF to the reader.
#
for p in out_th:
os.close(p[2])
if th is not None:
# Note, there is a condition where the subprocess can die
# very quickly (raising an OSError) before the reader
# threads have a chance to be set up. Testing for None
# avoids joining a thread that doesn't exist.
th.join()
for p in out_th:
os.close(p[1])
if th is not None:
del th
if outfile is not None:
stdout_arg.close()
elif tmpfile is not None and not ignore_output:
tmpfile.seek(0)
output = "".join(tmpfile.readlines())
tmpfile.close()
#
# Move back from the specified working directory
#
if cwd is not None:
os.chdir(oldpwd)
#
# Return the output
#
return [rc, output]
def available(self, exception_flag=True):
"""Determine if this optimizer is available."""
if exception_flag:
raise ApplicationError("Solver (%s) not available" %
str(self.name))
return False
def solve(self, *args, **kwds):
""" Solve the problem """
self.available(exception_flag=True)
#
# If the inputs are models, then validate that they have been
# constructed! Collect suffix names to try and import from solution.
#
from pyomo.core.base.block import _BlockData
import pyomo.core.base.suffix
from pyomo.core.kernel.block import IBlock
import pyomo.core.kernel.suffix
_model = None
for arg in args:
if isinstance(arg, (_BlockData, IBlock)):
if isinstance(arg, _BlockData):
if not arg.is_constructed():
raise RuntimeError(
"Attempting to solve model=%s with unconstructed "
"component(s)" % (arg.name, ))
_model = arg
# import suffixes must be on the top-level model
if isinstance(arg, _BlockData):
model_suffixes = list(name for (name,comp) \
in pyomo.core.base.suffix.\
active_import_suffix_generator(arg))
else:
assert isinstance(arg, IBlock)
model_suffixes = list(comp.storage_key for comp
in pyomo.core.kernel.suffix.\
import_suffix_generator(arg,
active=True,
descend_into=False))
if len(model_suffixes) > 0:
kwds_suffixes = kwds.setdefault('suffixes', [])
for name in model_suffixes:
if name not in kwds_suffixes:
kwds_suffixes.append(name)
#
# Handle ephemeral solvers options here. These
# will override whatever is currently in the options
# dictionary, but we will reset these options to
# their original value at the end of this method.
#
orig_options = self.options
self.options = Options()
self.options.update(orig_options)
self.options.update(kwds.pop('options', {}))
self.options.update(
self._options_string_to_dict(kwds.pop('options_string', '')))
try:
# we're good to go.
initial_time = time.time()
self._presolve(*args, **kwds)
presolve_completion_time = time.time()
if self._report_timing:
print(" %6.2f seconds required for presolve" %
(presolve_completion_time - initial_time))
if not _model is None:
self._initialize_callbacks(_model)
_status = self._apply_solver()
if hasattr(self, '_transformation_data'):
del self._transformation_data
if not hasattr(_status, 'rc'):
logger.warning(
"Solver (%s) did not return a solver status code.\n"
"This is indicative of an internal solver plugin error.\n"
"Please report this to the Pyomo developers.")
elif _status.rc:
logger.error("Solver (%s) returned non-zero return code (%s)" %
(
self.name,
_status.rc,
))
if self._tee:
logger.error(
"See the solver log above for diagnostic information.")
elif hasattr(_status, 'log') and _status.log:
logger.error("Solver log:\n" + str(_status.log))
raise ApplicationError("Solver (%s) did not exit normally" %
self.name)
solve_completion_time = time.time()
if self._report_timing:
print(" %6.2f seconds required for solver" %
(solve_completion_time - presolve_completion_time))
result = self._postsolve()
result._smap_id = self._smap_id
result._smap = None
if _model:
if isinstance(_model, IBlock):
if len(result.solution) == 1:
result.solution(0).symbol_map = \
getattr(_model, "._symbol_maps")[result._smap_id]
result.solution(0).default_variable_value = \
self._default_variable_value
if self._load_solutions:
_model.load_solution(result.solution(0))
else:
assert len(result.solution) == 0
# see the hack in the write method
# we don't want this to stick around on the model
# after the solve
assert len(getattr(_model, "._symbol_maps")) == 1
delattr(_model, "._symbol_maps")
del result._smap_id
if self._load_solutions and \
(len(result.solution) == 0):
logger.error("No solution is available")
else:
if self._load_solutions:
_model.solutions.load_from(result,
select=self._select_index,
default_variable_value=self.
_default_variable_value)
result._smap_id = None
result.solution.clear()
else:
result._smap = _model.solutions.symbol_map[
self._smap_id]
_model.solutions.delete_symbol_map(self._smap_id)
postsolve_completion_time = time.time()
if self._report_timing:
print(" %6.2f seconds required for postsolve" %
(postsolve_completion_time - solve_completion_time))
finally:
#
# Reset the options dict
#
self.options = orig_options
return result
def solve(self, *args, **kwds):
"""
Solve the model.
Keyword Arguments
-----------------
suffixes: list of str
The strings should represnt suffixes support by the solver. Examples include 'dual', 'slack', and 'rc'.
options: dict
Dictionary of solver options. See the solver documentation for possible solver options.
warmstart: bool
If True, the solver will be warmstarted.
keepfiles: bool
If True, the solver log file will be saved.
logfile: str
Name to use for the solver log file.
load_solutions: bool
If True and a solution exists, the solution will be loaded into the Pyomo model.
report_timing: bool
If True, then timing information will be printed.
tee: bool
If True, then the solver log will be printed.
"""
if self._pyomo_model is None:
msg = 'Please use set_instance to set the instance before calling solve with the persistent'
msg += ' solver interface.'
raise RuntimeError(msg)
if len(args) != 0:
if self._pyomo_model is not args[0]:
msg = 'The problem instance provided to the solve method is not the same as the instance provided'
msg += ' to the set_instance method in the persistent solver interface. '
raise ValueError(msg)
self.available(exception_flag=True)
# Collect suffix names to try and import from solution.
if isinstance(self._pyomo_model, _BlockData):
model_suffixes = list(name for (
name,
comp) in active_import_suffix_generator(self._pyomo_model))
else:
assert isinstance(self._pyomo_model, IBlock)
model_suffixes = list(
comp.storage_key for comp in import_suffix_generator(
self._pyomo_model, active=True, descend_into=False))
if len(model_suffixes) > 0:
kwds_suffixes = kwds.setdefault('suffixes', [])
for name in model_suffixes:
if name not in kwds_suffixes:
kwds_suffixes.append(name)
#
# Handle ephemeral solvers options here. These
# will override whatever is currently in the options
# dictionary, but we will reset these options to
# their original value at the end of this method.
#
orig_options = self.options
self.options = Options()
self.options.update(orig_options)
self.options.update(kwds.pop('options', {}))
self.options.update(
self._options_string_to_dict(kwds.pop('options_string', '')))
try:
# we're good to go.
initial_time = time.time()
self._presolve(**kwds)
presolve_completion_time = time.time()
if self._report_timing:
print(" %6.2f seconds required for presolve" %
(presolve_completion_time - initial_time))
if self._pyomo_model is not None:
self._initialize_callbacks(self._pyomo_model)
_status = self._apply_solver()
if hasattr(self, '_transformation_data'):
del self._transformation_data
if not hasattr(_status, 'rc'):
logger.warning(
"Solver (%s) did not return a solver status code.\n"
"This is indicative of an internal solver plugin error.\n"
"Please report this to the Pyomo developers.")
elif _status.rc:
logger.error("Solver (%s) returned non-zero return code (%s)" %
(
self.name,
_status.rc,
))
if self._tee:
logger.error(
"See the solver log above for diagnostic information.")
elif hasattr(_status, 'log') and _status.log:
logger.error("Solver log:\n" + str(_status.log))
raise ApplicationError("Solver (%s) did not exit normally" %
self.name)
solve_completion_time = time.time()
if self._report_timing:
print(" %6.2f seconds required for solver" %
(solve_completion_time - presolve_completion_time))
result = self._postsolve()
# ***********************************************************
# The following code is only needed for backwards compatability of load_solutions=False.
# If we ever only want to support the load_vars, load_duals, etc. methods, then this can be deleted.
if self._save_results:
result._smap_id = self._smap_id
result._smap = None
_model = self._pyomo_model
if _model:
if isinstance(_model, IBlock):
if len(result.solution) == 1:
result.solution(0).symbol_map = \
getattr(_model, "._symbol_maps")[result._smap_id]
result.solution(0).default_variable_value = \
self._default_variable_value
if self._load_solutions:
_model.load_solution(result.solution(0))
else:
assert len(result.solution) == 0
# see the hack in the write method
# we don't want this to stick around on the model
# after the solve
assert len(getattr(_model, "._symbol_maps")) == 1
delattr(_model, "._symbol_maps")
del result._smap_id
if self._load_solutions and \
(len(result.solution) == 0):
logger.error("No solution is available")
else:
if self._load_solutions:
_model.solutions.load_from(
result,
select=self._select_index,
default_variable_value=self.
_default_variable_value)
result._smap_id = None
result.solution.clear()
else:
result._smap = _model.solutions.symbol_map[
self._smap_id]
_model.solutions.delete_symbol_map(self._smap_id)
# ********************************************************
postsolve_completion_time = time.time()
if self._report_timing:
print(" %6.2f seconds required for postsolve" %
(postsolve_completion_time - solve_completion_time))
finally:
#
# Reset the options dict
#
self.options = orig_options
return result
def _postsolve(self):
results = SolverResults()
#print 'ANS', dir(self._ans),
#print self._ans.evals
#print self._ans.ff
#print self._ans.rf
#print self._ans.xf
solv = results.solver
solv.name = self.options.subsolver
#solv.status = self._glpk_get_solver_status()
#solv.memory_used = "%d bytes, (%d KiB)" % (peak_mem, peak_mem/1024)
solv.wallclock_time = self._ans.elapsed['solver_time']
solv.cpu_time = self._ans.elapsed['solver_cputime']
solv.termination_message = self._ans.msg
istop = self._ans.istop
if istop == openopt.kernel.setDefaultIterFuncs.SMALL_DF:
solv.termination_condition = TerminationCondition.other
sstatus = SolutionStatus.locallyOptimal
elif istop == openopt.kernel.setDefaultIterFuncs.SMALL_DELTA_X:
solv.termination_condition = TerminationCondition.minStepLength
sstatus = SolutionStatus.stoppedByLimit
elif istop == openopt.kernel.setDefaultIterFuncs.SMALL_DELTA_F:
solv.termination_condition = TerminationCondition.other
sstatus = SolutionStatus.stoppedByLimit
elif istop == openopt.kernel.setDefaultIterFuncs.FVAL_IS_ENOUGH:
solv.termination_condition = TerminationCondition.minFunctionValue
sstatus = SolutionStatus.stoppedByLimit
elif istop == openopt.kernel.setDefaultIterFuncs.MAX_NON_SUCCESS:
solv.termination_condition = TerminationCondition.other
sstatus = SolutionStatus.unsure
elif istop == openopt.kernel.setDefaultIterFuncs.USER_DEMAND_STOP:
solv.termination_condition = TerminationCondition.userInterrupt
sstatus = SolutionStatus.bestSoFar
elif istop == openopt.kernel.setDefaultIterFuncs.BUTTON_ENOUGH_HAS_BEEN_PRESSED:
solv.termination_condition = TerminationCondition.userInterrupt
sstatus = SolutionStatus.bestSoFar
elif istop == openopt.kernel.setDefaultIterFuncs.SOLVED_WITH_UNIMPLEMENTED_OR_UNKNOWN_REASON:
solv.termination_condition = TerminationCondition.other
sstatus = SolutionStatus.unsure
elif istop == openopt.kernel.setDefaultIterFuncs.UNDEFINED:
solv.termination_condition = TerminationCondition.unknown
sstatus = SolutionStatus.unsure
elif istop == openopt.kernel.setDefaultIterFuncs.IS_NAN_IN_X:
solv.termination_condition = TerminationCondition.other
sstatus = SolutionStatus.unknown
elif istop == openopt.kernel.setDefaultIterFuncs.IS_LINE_SEARCH_FAILED:
solv.termination_condition = TerminationCondition.other
sstatus = SolutionStatus.error
elif istop == openopt.kernel.setDefaultIterFuncs.IS_MAX_ITER_REACHED:
solv.termination_condition = TerminationCondition.maxIterations
sstatus = SolutionStatus.stoppedByLimit
elif istop == openopt.kernel.setDefaultIterFuncs.IS_MAX_CPU_TIME_REACHED:
solv.termination_condition = TerminationCondition.maxTimeLimit
sstatus = SolutionStatus.stoppedByLimit
elif istop == openopt.kernel.setDefaultIterFuncs.IS_MAX_TIME_REACHED:
solv.termination_condition = TerminationCondition.maxTimeLimit
sstatus = SolutionStatus.stoppedByLimit
elif istop == openopt.kernel.setDefaultIterFuncs.IS_MAX_FUN_EVALS_REACHED:
solv.termination_condition = TerminationCondition.maxEvaluations
sstatus = SolutionStatus.stoppedByLimit
elif istop == openopt.kernel.setDefaultIterFuncs.IS_ALL_VARS_FIXED:
solv.termination_condition = TerminationCondition.other
sstatus = SolutionStatus.unknown
elif istop == openopt.kernel.setDefaultIterFuncs.FAILED_TO_OBTAIN_MOVE_DIRECTION:
solv.termination_condition = TerminationCondition.other
sstatus = SolutionStatus.error
elif istop == openopt.kernel.setDefaultIterFuncs.USER_DEMAND_EXIT:
solv.termination_condition = TerminationCondition.userInterrupt
sstatus = SolutionStatus.bestSoFar
elif istop == -100:
#solv.termination_condition = TerminationCondition.other
sstatus = SolutionStatus.error
else:
raise ApplicationError(
"Unexpected OpenOpt termination code: '%d'" % istop)
prob = results.problem
prob.name = self._instance.name
prob.number_of_constraints = self._instance.statistics.number_of_constraints
prob.number_of_variables = self._instance.statistics.number_of_variables
prob.number_of_binary_variables = self._instance.statistics.number_of_binary_variables
prob.number_of_integer_variables = self._instance.statistics.number_of_integer_variables
prob.number_of_continuous_variables = self._instance.statistics.number_of_continuous_variables
prob.number_of_objectives = self._instance.statistics.number_of_objectives
from pyomo.core import maximize
if self._problem.sense == maximize:
prob.sense = ProblemSense.maximize
else:
prob.sense = ProblemSense.minimize
if not sstatus in (SolutionStatus.error, ):
soln = Solution()
soln.status = sstatus
if type(self._ans.ff) in (list, tuple):
oval = float(self._ans.ff[0])
else:
oval = float(self._ans.ff)
if self._problem.sense == maximize:
soln.objective[self._problem._f_name[0]] = {'Value': -oval}
else:
soln.objective[self._problem._f_name[0]] = {'Value': oval}
for var_label in self._ans.xf.keys():
if self._ans.xf[var_label].is_integer():
soln.variable[var_label.name] = {
'Value': int(self._ans.xf[var_label])
}
else:
soln.variable[var_label.name] = {
'Value': float(self._ans.xf[var_label])
}
results.solution.insert(soln)
self._instance.solutions.add_symbol_map(self._symbol_map)
self._smap_id = id(self._symbol_map)
self._instance = None
self._symbol_map = None
self._problem = None
return results