def _presolve(self, *args, **kwds):
"""
Peform presolves.
"""
TempfileManager.push()
self._keepfiles = kwds.pop("keepfiles", False)
self._define_signal_handlers = kwds.pop('use_signal_handling',None)
OptSolver._presolve(self, *args, **kwds)
#
# Verify that the input problems exists
#
for filename in self._problem_files:
if not os.path.exists(filename):
msg = 'Solver failed to locate input problem file: %s'
raise ValueError(msg % filename)
#
# Create command line
#
self._command = self.create_command_line(
self.executable(), self._problem_files)
self._log_file=self._command.log_file
#
# The pre-cleanup is probably unncessary, but also not harmful.
#
if (self._log_file is not None) and \
os.path.exists(self._log_file):
os.remove(self._log_file)
if (self._soln_file is not None) and \
os.path.exists(self._soln_file):
os.remove(self._soln_file)
def _presolve(self, *args, **kwds):
"""
Peform presolves.
"""
TempfileManager.push()
self._keepfiles = kwds.pop("keepfiles", False)
OptSolver._presolve(self, *args, **kwds)
#
# Verify that the input problems exists
#
for filename in self._problem_files:
if not os.path.exists(filename):
msg = 'Solver failed to locate input problem file: %s'
raise ValueError(msg % filename)
#
# Create command line
#
self._command = self.create_command_line(
self.executable(), self._problem_files)
self._log_file=self._command.log_file
#
# The pre-cleanup is probably unncessary, but also not harmful.
#
if (self._log_file is not None) and \
os.path.exists(self._log_file):
os.remove(self._log_file)
if (self._soln_file is not None) and \
os.path.exists(self._soln_file):
os.remove(self._soln_file)
def solve(self, model, timer: HierarchicalTimer = None):
self.available(exception_flag=True)
if timer is None:
timer = HierarchicalTimer()
try:
TempfileManager.push()
if self.config.filename is None:
self._filename = TempfileManager.create_tempfile()
else:
self._filename = self.config.filename
TempfileManager.add_tempfile(self._filename + '.lp', exists=False)
TempfileManager.add_tempfile(self._filename + '.soln', exists=False)
TempfileManager.add_tempfile(self._filename + '.log', exists=False)
timer.start('write lp file')
self._writer.write(model, self._filename+'.lp', timer=timer)
timer.stop('write lp file')
res = self._apply_solver(timer)
if self.config.report_timing:
logger.info('\n' + str(timer))
return res
finally:
# finally, clean any temporary files registered with the
# temp file manager, created/populated *directly* by this
# plugin.
TempfileManager.pop(remove=not self.config.keepfiles)
if not self.config.keepfiles:
self._filename = None
if self.config.report_timing:
print(timer)
def _presolve(self, *args, **kwds):
# create a context in the temporary file manager for
# this plugin - is "pop"ed in the _postsolve method.
TempfileManager.push()
# if the first argument is a string (representing a filename),
# then we don't have an instance => the solver is being applied
# to a file.
self._warm_start_solve = kwds.pop('warmstart', False)
self._warm_start_file_name = kwds.pop('warmstart_file', None)
user_warmstart = False
if self._warm_start_file_name is not None:
user_warmstart = True
# the input argument can currently be one of two things: an instance or a filename.
# if a filename is provided and a warm-start is indicated, we go ahead and
# create the temporary file - assuming that the user has already, via some external
# mechanism, invoked warm_start() with a instance to create the warm start file.
if self._warm_start_solve and \
isinstance(args[0], string_types):
# we assume the user knows what they are doing...
pass
elif self._warm_start_solve and \
(not isinstance(args[0], string_types)):
# assign the name of the warm start file *before* calling the base class
# presolve - the base class method ends up creating the command line,
# and the warm start file-name is (obviously) needed there.
if self._warm_start_file_name is None:
assert not user_warmstart
self._warm_start_file_name = TempfileManager.\
create_tempfile(suffix = '.cbc.soln')
# let the base class handle any remaining keywords/actions.
# let the base class handle any remaining keywords/actions.
super(CBCSHELL, self)._presolve(*args, **kwds)
# NB: we must let the base class presolve run first so that the
# symbol_map is actually constructed!
if (len(args) > 0) and (not isinstance(args[0], string_types)):
if len(args) != 1:
raise ValueError(
"CBCplugin _presolve method can only handle a single "
"problem instance - %s were supplied" % (len(args), ))
# write the warm-start file - currently only supports MIPs.
# we only know how to deal with a single problem instance.
if self._warm_start_solve and (not user_warmstart):
start_time = time.time()
self._warm_start(args[0])
end_time = time.time()
if self._report_timing is True:
print("Warm start write time=%.2f seconds" %
(end_time - start_time))
def generate_scenario_tree_image(options):
with ScenarioTreeInstanceFactory(
options.model_location, options.scenario_tree_location) as factory:
scenario_tree = factory.generate_scenario_tree(
downsample_fraction=options.scenario_tree_downsample_fraction,
bundles=options.scenario_bundle_specification,
random_bundles=options.create_random_bundles,
random_seed=options.scenario_tree_random_seed,
verbose=options.verbose)
with TempfileManager.push():
tmpdotfile = TempfileManager.create_tempfile(suffix=".dot")
scenario_tree.save_to_dot(tmpdotfile)
os.system('dot -Tpdf -o %s %s' % (options.output_file, tmpdotfile))
print("Output Saved To: %s" % (options.output_file))
def generate_scenario_tree_image(options):
with ScenarioTreeInstanceFactory(options.model_location, options.scenario_tree_location) as factory:
scenario_tree = factory.generate_scenario_tree(
downsample_fraction=options.scenario_tree_downsample_fraction,
bundles=options.scenario_bundle_specification,
random_bundles=options.create_random_bundles,
random_seed=options.scenario_tree_random_seed,
verbose=options.verbose,
)
with TempfileManager.push():
tmpdotfile = TempfileManager.create_tempfile(suffix=".dot")
scenario_tree.save_to_dot(tmpdotfile)
os.system("dot -Tpdf -o %s %s" % (options.output_file, tmpdotfile))
print("Output Saved To: %s" % (options.output_file))
def run_command(command=None,
parser=None,
args=None,
name='unknown',
data=None,
options=None):
"""
Execute a function that processes command-line arguments and
then calls a command-line driver.
This function provides a generic facility for executing a command
function is rather generic. This function is segregated from
the driver to enable profiling of the command-line execution.
Required:
command: The name of a function that will be executed to perform process the command-line
options with a parser object.
parser: The parser object that is used by the command-line function.
Optional:
options: If this is not None, then ignore the args option and use
this to specify command options.
args: Command-line arguments that are parsed. If this value is `None`, then the
arguments in `sys.argv` are used to parse the command-line.
name: Specifying the name of the command-line (for error messages).
data: A container of labeled data.
Returned:
retval: Return values from the command-line execution.
errorcode: 0 if Pyomo ran successfully
"""
#
#
# Parse command-line options
#
#
retval = None
errorcode = 0
if options is None:
try:
if type(args) is argparse.Namespace:
_options = args
else:
_options = parser.parse_args(args=args)
# Replace the parser options object with a pyutilib.misc.Options object
options = pyutilib.misc.Options()
for key in dir(_options):
if key[0] != '_':
val = getattr(_options, key)
if not isinstance(val, types.MethodType):
options[key] = val
except SystemExit:
# the parser throws a system exit if "-h" is specified - catch
# it to exit gracefully.
return Container(retval=retval, errorcode=errorcode)
#
# Configure loggers
#
configure_loggers(options=options)
#
# Setup I/O redirect to a file
#
logfile = options.runtime.logfile
if not logfile is None:
pyutilib.misc.setup_redirect(logfile)
#
# Call the main Pyomo runner with profiling
#
TempfileManager.push()
pcount = options.runtime.profile_count
if pcount > 0:
if not pstats_available:
if not logfile is None:
pyutilib.misc.reset_redirect()
msg = "Cannot use the 'profile' option. The Python 'pstats' " \
'package cannot be imported!'
raise ValueError(msg)
tfile = TempfileManager.create_tempfile(suffix=".profile")
tmp = profile.runctx(
command.__name__ + '(options=options,parser=parser)',
command.__globals__, locals(), tfile)
p = pstats.Stats(tfile).strip_dirs()
p.sort_stats('time', 'cumulative')
p = p.print_stats(pcount)
p.print_callers(pcount)
p.print_callees(pcount)
p = p.sort_stats('cumulative', 'calls')
p.print_stats(pcount)
p.print_callers(pcount)
p.print_callees(pcount)
p = p.sort_stats('calls')
p.print_stats(pcount)
p.print_callers(pcount)
p.print_callees(pcount)
retval = tmp
else:
#
# Call the main Pyomo runner without profiling
#
TempfileManager.push()
try:
retval = command(options=options, parser=parser)
except SystemExit:
err = sys.exc_info()[1]
#
# If debugging is enabled or the 'catch' option is specified, then
# exit. Otherwise, print an "Exiting..." message.
#
if __debug__ and (options.runtime.logging == 'debug'
or options.runtime.catch_errors):
sys.exit(0)
print('Exiting %s: %s' % (name, str(err)))
errorcode = err.code
except Exception:
err = sys.exc_info()[1]
#
# If debugging is enabled or the 'catch' option is specified, then
# pass the exception up the chain (to pyomo_excepthook)
#
if __debug__ and (options.runtime.logging == 'debug'
or options.runtime.catch_errors):
if not logfile is None:
pyutilib.misc.reset_redirect()
TempfileManager.pop(remove=not options.runtime.keep_files)
raise
if not options.model is None and not options.model.save_file is None:
model = "model " + options.model.save_file
else:
model = "model"
global filter_excepthook
if filter_excepthook:
action = "loading"
else:
action = "running"
msg = "Unexpected exception while %s %s:\n" % (action, model)
#
# This handles the case where the error is propagated by a KeyError.
# KeyError likes to pass raw strings that don't handle newlines
# (they translate "\n" to "\\n"), as well as tacking on single
# quotes at either end of the error message. This undoes all that.
#
errStr = str(err)
if type(err) == KeyError and errStr != "None":
errStr = str(err).replace(r"\n", "\n")[1:-1]
logging.getLogger('pyomo.core').error(msg + errStr)
errorcode = 1
if not logfile is None:
pyutilib.misc.reset_redirect()
if options.runtime.disable_gc:
gc.enable()
TempfileManager.pop(remove=not options.runtime.keep_files)
return Container(retval=retval, errorcode=errorcode)
def _solve_impl(self,
sp,
keep_solver_files=False,
output_solver_log=False,
symbolic_solver_labels=False):
if len(sp.scenario_tree.stages) > 2:
raise ValueError("SD solver does not handle more "
"than 2 time-stages")
if sp.objective_sense == maximize:
raise ValueError("SD solver does not yet handle "
"maximization problems")
TempfileManager.push()
try:
#
# Setup the SD working directory
#
working_directory = TempfileManager.create_tempdir()
config_filename = os.path.join(working_directory,
"config.sd")
sdinput_directory = os.path.join(working_directory,
"sdinput",
"pysp_model")
sdoutput_directory = os.path.join(working_directory,
"sdoutput",
"pysp_model")
logfile = os.path.join(working_directory, "sd.log")
os.makedirs(sdinput_directory)
assert os.path.exists(sdinput_directory)
assert not os.path.exists(sdoutput_directory)
self._write_config(config_filename)
if self.get_option('single_replication'):
solution_filename = os.path.join(
sdoutput_directory,
"pysp_model.detailed_rep_soln.out")
else:
solution_filename = os.path.join(
sdoutput_directory,
"pysp_model.detailed_soln.out")
#
# Create the SD input files
#
io_options = {'symbolic_solver_labels':
symbolic_solver_labels}
symbol_map = None
if isinstance(sp, ImplicitSP):
symbol_map = pyomo.pysp.smps.smpsutils.\
convert_implicit(
sdinput_directory,
"pysp_model",
sp,
core_format='mps',
io_options=io_options)
else:
pyomo.pysp.smps.smpsutils.\
convert_explicit(
sdinput_directory,
"pysp_model",
sp,
core_format='mps',
io_options=io_options)
#
# Launch SD
#
if keep_solver_files:
print("Solver working directory: '%s'"
% (working_directory))
print("Solver log file: '%s'"
% (logfile))
start = time.time()
rc, log = pyutilib.subprocess.run(
self.get_option("sd_executable"),
cwd=working_directory,
stdin="pysp_model",
outfile=logfile,
tee=output_solver_log)
stop = time.time()
assert os.path.exists(sdoutput_directory)
#
# Parse the SD solution
#
xhat, results = self._read_solution(solution_filename)
results.solver_time = stop - start
if symbol_map is not None:
# load the first stage variable solution into
# the reference model
for symbol, varvalue in xhat.items():
symbol_map.bySymbol[symbol]().value = varvalue
else:
# TODO: this is a hack for the non-implicit SP case
# so that this solver can still be run using
# the explicit scenario intput representation
results.xhat = xhat
finally:
#
# cleanup
#
TempfileManager.pop(
remove=not keep_solver_files)
return results
def run_command(command=None, parser=None, args=None, name='unknown', data=None, options=None):
"""
Execute a function that processes command-line arguments and
then calls a command-line driver.
This function provides a generic facility for executing a command
function is rather generic. This function is segregated from
the driver to enable profiling of the command-line execution.
Required:
command: The name of a function that will be executed to perform process the command-line
options with a parser object.
parser: The parser object that is used by the command-line function.
Optional:
options: If this is not None, then ignore the args option and use
this to specify command options.
args: Command-line arguments that are parsed. If this value is `None`, then the
arguments in `sys.argv` are used to parse the command-line.
name: Specifying the name of the command-line (for error messages).
data: A container of labeled data.
Returned:
retval: Return values from the command-line execution.
errorcode: 0 if Pyomo ran successfully
"""
#
#
# Parse command-line options
#
#
retval = None
errorcode = 0
if options is None:
try:
if type(args) is argparse.Namespace:
_options = args
else:
_options = parser.parse_args(args=args)
# Replace the parser options object with a pyutilib.misc.Options object
options = pyutilib.misc.Options()
for key in dir(_options):
if key[0] != '_':
val = getattr(_options, key)
if not isinstance(val, types.MethodType):
options[key] = val
except SystemExit:
# the parser throws a system exit if "-h" is specified - catch
# it to exit gracefully.
return Container(retval=retval, errorcode=errorcode)
#
# Configure loggers
#
configure_loggers(options=options)
#
# Setup I/O redirect to a file
#
logfile = options.runtime.logfile
if not logfile is None:
pyutilib.misc.setup_redirect(logfile)
#
# Call the main Pyomo runner with profiling
#
TempfileManager.push()
pcount = options.runtime.profile_count
if pcount > 0:
if not pstats_available:
if not logfile is None:
pyutilib.misc.reset_redirect()
msg = "Cannot use the 'profile' option. The Python 'pstats' " \
'package cannot be imported!'
raise ValueError(msg)
tfile = TempfileManager.create_tempfile(suffix=".profile")
tmp = profile.runctx(
command.__name__ + '(options=options,parser=parser)', command.__globals__, locals(), tfile
)
p = pstats.Stats(tfile).strip_dirs()
p.sort_stats('time', 'cumulative')
p = p.print_stats(pcount)
p.print_callers(pcount)
p.print_callees(pcount)
p = p.sort_stats('cumulative','calls')
p.print_stats(pcount)
p.print_callers(pcount)
p.print_callees(pcount)
p = p.sort_stats('calls')
p.print_stats(pcount)
p.print_callers(pcount)
p.print_callees(pcount)
retval = tmp
else:
#
# Call the main Pyomo runner without profiling
#
TempfileManager.push()
try:
retval = command(options=options, parser=parser)
except SystemExit:
err = sys.exc_info()[1]
#
# If debugging is enabled or the 'catch' option is specified, then
# exit. Otherwise, print an "Exiting..." message.
#
if __debug__ and (options.runtime.logging == 'debug' or options.runtime.catch_errors):
sys.exit(0)
print('Exiting %s: %s' % (name, str(err)))
errorcode = err.code
except Exception:
err = sys.exc_info()[1]
#
# If debugging is enabled or the 'catch' option is specified, then
# pass the exception up the chain (to pyomo_excepthook)
#
if __debug__ and (options.runtime.logging == 'debug' or options.runtime.catch_errors):
if not logfile is None:
pyutilib.misc.reset_redirect()
TempfileManager.pop(remove=not options.runtime.keep_files)
raise
if not options.model is None and not options.model.save_file is None:
model = "model " + options.model.save_file
else:
model = "model"
global filter_excepthook
if filter_excepthook:
action = "loading"
else:
action = "running"
msg = "Unexpected exception while %s %s:\n" % (action, model)
#
# This handles the case where the error is propagated by a KeyError.
# KeyError likes to pass raw strings that don't handle newlines
# (they translate "\n" to "\\n"), as well as tacking on single
# quotes at either end of the error message. This undoes all that.
#
errStr = str(err)
if type(err) == KeyError and errStr != "None":
errStr = str(err).replace(r"\n","\n")[1:-1]
logging.getLogger('pyomo.core').error(msg+errStr)
errorcode = 1
if not logfile is None:
pyutilib.misc.reset_redirect()
if options.runtime.disable_gc:
gc.enable()
TempfileManager.pop(remove=not options.runtime.keep_files)
return Container(retval=retval, errorcode=errorcode)
def _solve_impl(self,
sp,
keep_solver_files=False,
output_solver_log=False,
symbolic_solver_labels=False):
if len(sp.scenario_tree.stages) > 2:
raise ValueError("SD solver does not handle more "
"than 2 time-stages")
if sp.objective_sense == maximize:
raise ValueError("SD solver does not yet handle "
"maximization problems")
TempfileManager.push()
try:
#
# Setup the SD working directory
#
working_directory = TempfileManager.create_tempdir()
config_filename = os.path.join(working_directory, "config.sd")
sdinput_directory = os.path.join(working_directory, "sdinput",
"pysp_model")
sdoutput_directory = os.path.join(working_directory, "sdoutput",
"pysp_model")
logfile = os.path.join(working_directory, "sd.log")
os.makedirs(sdinput_directory)
assert os.path.exists(sdinput_directory)
assert not os.path.exists(sdoutput_directory)
self._write_config(config_filename)
if self.get_option('single_replication'):
solution_filename = os.path.join(
sdoutput_directory, "pysp_model.detailed_rep_soln.out")
else:
solution_filename = os.path.join(
sdoutput_directory, "pysp_model.detailed_soln.out")
#
# Create the SD input files
#
io_options = {'symbolic_solver_labels': symbolic_solver_labels}
symbol_map = None
if isinstance(sp, ImplicitSP):
symbol_map = pyomo.pysp.smps.smpsutils.\
convert_implicit(
sdinput_directory,
"pysp_model",
sp,
core_format='mps',
io_options=io_options)
else:
pyomo.pysp.smps.smpsutils.\
convert_explicit(
sdinput_directory,
"pysp_model",
sp,
core_format='mps',
io_options=io_options)
#
# Launch SD
#
if keep_solver_files:
print("Solver working directory: '%s'" % (working_directory))
print("Solver log file: '%s'" % (logfile))
start = time.time()
rc, log = pyutilib.subprocess.run(self.get_option("sd_executable"),
cwd=working_directory,
stdin="pysp_model",
outfile=logfile,
tee=output_solver_log)
stop = time.time()
assert os.path.exists(sdoutput_directory)
#
# Parse the SD solution
#
xhat, results = self._read_solution(solution_filename)
results.solver_time = stop - start
if symbol_map is not None:
# load the first stage variable solution into
# the reference model
for symbol, varvalue in xhat.items():
symbol_map.bySymbol[symbol]().value = varvalue
else:
# TODO: this is a hack for the non-implicit SP case
# so that this solver can still be run using
# the explicit scenario intput representation
results.xhat = xhat
finally:
#
# cleanup
#
TempfileManager.pop(remove=not keep_solver_files)
return results
def __init__(self, pyomo_model):
"""
Pyomo nonlinear program interface
Parameters
----------
pyomo_model: pyomo.environ.ConcreteModel
Pyomo concrete model
"""
TempfileManager.push()
try:
# get the temp file names for the nl file
nl_file = TempfileManager.create_tempfile(suffix='pynumero.nl')
# The current AmplInterface code only supports a single
# objective function Therefore, we throw an error if there
# is not one (and only one) active objective function. This
# is better than adding a dummy objective that the user does
# not know about (since we do not have a good place to
# remove this objective later)
#
# TODO: extend the AmplInterface and the AslNLP to correctly
# handle this
#
# This currently addresses issue #1217
objectives = list(
pyomo_model.component_data_objects(ctype=pyo.Objective,
active=True,
descend_into=True))
if len(objectives) != 1:
raise NotImplementedError(
'The ASL interface and PyomoNLP in PyNumero currently '
'only support single objective problems. Deactivate '
'any extra objectives you may have, or add a dummy '
'objective (f(x)=0) if you have a square problem.')
self._objective = objectives[0]
# write the nl file for the Pyomo model and get the symbolMap
fname, symbolMap = WriterFactory('nl')(pyomo_model, nl_file,
lambda x: True, {})
# create component maps from vardata to idx and condata to idx
self._vardata_to_idx = vdidx = ComponentMap()
self._condata_to_idx = cdidx = ComponentMap()
# TODO: Are these names totally consistent?
for name, obj in six.iteritems(symbolMap.bySymbol):
if name[0] == 'v':
vdidx[obj()] = int(name[1:])
elif name[0] == 'c':
cdidx[obj()] = int(name[1:])
# The NL writer advertises the external function libraries
# through the PYOMO_AMPLFUNC environment variable; merge it
# with any preexisting AMPLFUNC definitions
amplfunc = "\n".join(val for val in (
os.environ.get('AMPLFUNC', ''),
os.environ.get('PYOMO_AMPLFUNC', ''),
) if val)
with CtypesEnviron(AMPLFUNC=amplfunc):
super(PyomoNLP, self).__init__(nl_file)
# keep pyomo model in cache
self._pyomo_model = pyomo_model
finally:
# delete the nl file
TempfileManager.pop()