def solve(self, model, timer: HierarchicalTimer = None):
StaleFlagManager.mark_all_as_stale()
avail = self.available()
if not avail:
raise PyomoException(
f'Solver {self.__class__} is not available ({avail}).')
if self._last_results_object is not None:
self._last_results_object.solution_loader.invalidate()
if timer is None:
timer = HierarchicalTimer()
try:
TempfileManager.push()
if self.config.filename is None:
self._filename = TempfileManager.create_tempfile()
else:
self._filename = self.config.filename
TempfileManager.add_tempfile(self._filename + '.lp', exists=False)
TempfileManager.add_tempfile(self._filename + '.log', exists=False)
timer.start('write lp file')
self._writer.write(model, self._filename + '.lp', timer=timer)
timer.stop('write lp file')
res = self._apply_solver(timer)
self._last_results_object = res
if self.config.report_timing:
logger.info('\n' + str(timer))
return res
finally:
# finally, clean any temporary files registered with the
# temp file manager, created/populated *directly* by this
# plugin.
TempfileManager.pop(remove=not self.config.keepfiles)
if not self.config.keepfiles:
self._filename = None
def _apply_solver(self):
StaleFlagManager.mark_all_as_stale()
self._solver_model.setlogfile(self._log_file)
if self._keepfiles:
print("Solver log file: " + self._log_file)
# Setting a log file in xpress disables all output
# in xpress versions less than 36.
# This callback prints all messages to stdout
# when using those xpress versions.
if self._tee and XpressDirect._version[0] < 36:
self._solver_model.addcbmessage(_print_message, None, 0)
# set xpress options
# if the user specifies a 'mipgap', set it, and
# set xpress's related options to 0.
if self.options.mipgap is not None:
self._solver_model.setControl('miprelstop',
float(self.options.mipgap))
self._solver_model.setControl('miprelcutoff', 0.0)
self._solver_model.setControl('mipaddcutoff', 0.0)
# xpress is picky about the type which is passed
# into a control. So we will infer and cast
# get the xpress valid controls
xp_controls = xpress.controls
for key, option in self.options.items():
if key == 'mipgap': # handled above
continue
try:
self._solver_model.setControl(key, option)
except XpressDirect.XpressException:
# take another try, converting to its type
# we'll wrap this in a function to raise the
# xpress error
contr_type = type(getattr(xp_controls, key))
if not _is_convertable(contr_type, option):
raise
self._solver_model.setControl(key, contr_type(option))
start_time = time.time()
if self._tee:
self._solver_model.solve()
else:
# In xpress versions greater than or equal 36,
# it seems difficult to completely suppress console
# output without disabling logging altogether.
# As a work around, we capature all screen output
# when tee is False.
with capture_output() as OUT:
self._solver_model.solve()
self._opt_time = time.time() - start_time
self._solver_model.setlogfile('')
if self._tee and XpressDirect._version[0] < 36:
self._solver_model.removecbmessage(_print_message, None)
# FIXME: can we get a return code indicating if XPRESS had a significant failure?
return Bunch(rc=None, log=None)
def load_vars(self, vars_to_load=None):
"""
Load the values from the solver's variables into the corresponding pyomo variables.
Parameters
----------
vars_to_load: list of Var
"""
self._load_vars(vars_to_load)
StaleFlagManager.mark_all_as_stale(delayed=True)
def cbGetSolution(self, vars):
"""
Parameters
----------
vars: iterable of vars
"""
StaleFlagManager.mark_all_as_stale()
if not isinstance(vars, Iterable):
vars = [vars]
gurobi_vars = [self._pyomo_var_to_solver_var_map[i] for i in vars]
var_values = self._solver_model.cbGetSolution(gurobi_vars)
for i, v in enumerate(vars):
v.set_value(var_values[i], skip_validation=True)
def __getstate__(self):
state = super().__getstate__()
for i in _GeneralBooleanVarData.__slots__:
state[i] = getattr(self, i)
if isinstance(self._associated_binary, ReferenceType):
state['_associated_binary'] = self._associated_binary()
state['_stale'] = StaleFlagManager.is_stale(self._stale)
return state
def _apply_solver(self):
StaleFlagManager.mark_all_as_stale()
if self._tee:
def _process_stream(msg):
sys.stdout.write(msg)
sys.stdout.flush()
self._solver_model.set_Stream(self._mosek.streamtype.log,
_process_stream)
if self._keepfiles:
logger.info("Solver log file: {}".format(self._log_file))
for key, option in self.options.items():
try:
param = key.split('.')
if param[0] == 'mosek':
param.pop(0)
param = getattr(self._mosek, param[0])(param[1])
if 'sparam' in key.split('.'):
self._solver_model.putstrparam(param, option)
elif 'dparam' in key.split('.'):
self._solver_model.putdouparam(param, option)
elif 'iparam' in key.split('.'):
if isinstance(option, str):
option = option.split('.')
if option[0] == 'mosek':
option.pop('mosek')
option = getattr(self._mosek, option[0])(option[1])
else:
self._solver_model.putintparam(param, option)
except (TypeError, AttributeError):
raise
try:
self._termcode = self._solver_model.optimize()
self._solver_model.solutionsummary(self._mosek.streamtype.msg)
except self._mosek.Error as e:
logger.error(e)
raise
return Bunch(rc=None, log=None)
def set_value(self, val, skip_validation=False):
"""Set the current variable value.
Set the value of this variable. The incoming value is converted
to a numeric value (i.e., expressions are evaluated). If the
variable has units, the incoming value is converted to the
correct units before storing the value. The final value is
checked against both the variable domain and bounds, and an
exception is raised if the value is not valid. Domain and
bounds checking can be bypassed by setting the ``skip_validation``
argument to :const:`True`.
"""
# Special case: setting a variable to None "clears" the variable.
if val is None:
self._value = None
self._stale = 0 # True
return
# TODO: generate a warning/error:
#
# Check if this Var has units: assigning dimensionless
# values to a variable with units should be an error
if type(val) not in native_numeric_types:
if self.parent_component()._units is not None:
_src_magnitude = value(val)
_src_units = units.get_units(val)
val = units.convert_value(
num_value=_src_magnitude,
from_units=_src_units,
to_units=self.parent_component()._units)
else:
val = value(val)
if not skip_validation:
if val not in self.domain:
logger.warning(
"Setting Var '%s' to a value `%s` (%s) not in domain %s." %
(self.name, val, type(val).__name__, self.domain),
extra={'id': 'W1001'},
)
elif (self._lb is not None
and val < value(self._lb)) or (self._ub is not None
and val > value(self._ub)):
logger.warning(
"Setting Var '%s' to a numeric value `%s` "
"outside the bounds %s." % (self.name, val, self.bounds),
extra={'id': 'W1002'},
)
self._value = val
self._stale = StaleFlagManager.get_flag(self._stale)
def __setstate__(self, state):
"""Restore a picked state into this instance.
Note: adapted from class ComponentData in pyomo.core.base.component
"""
if state.pop('_stale', True):
state['_stale'] = 0
else:
state['_stale'] = StaleFlagManager.get_flag(0)
super().__setstate__(state)
if self._associated_binary is not None and \
type(self._associated_binary) is not \
_DeprecatedImplicitAssociatedBinaryVariable:
self._associated_binary = weakref_ref(self._associated_binary)
def set_value(self, val, skip_validation=False):
"""
Set the value of this numeric object, after
validating its value. If the 'valid' flag is True,
then the validation step is skipped.
"""
# Note that it is basically as fast to check the type as it is
# to check the skip_validation flag. Considering that we expect
# the flag to always be False, we will just ignore it in the
# name of efficiency.
if val.__class__ not in _logical_var_types:
if not skip_validation:
logger.warning("implicitly casting '%s' value %s to bool" %
(self.name, val))
val = bool(val)
self._value = val
self._stale = StaleFlagManager.get_flag(self._stale)
def stale(self, val):
if val:
self._stale = 0 # True
else:
self._stale = StaleFlagManager.get_flag(0)
def stale(self):
return StaleFlagManager.is_stale(self._stale)
def __setstate__(self, state):
if state.pop('_stale', True):
state['_stale'] = 0
else:
state['_stale'] = StaleFlagManager.get_flag(0)
super().__setstate__(state)
def __getstate__(self):
state = super(_GeneralVarData, self).__getstate__()
for i in _GeneralVarData.__slots__:
state[i] = getattr(self, i)
state['_stale'] = StaleFlagManager.is_stale(self._stale)
return state
def select(self,
index=0,
allow_consistent_values_for_fixed_vars=False,
comparison_tolerance_for_fixed_vars=1e-5,
ignore_invalid_labels=False,
ignore_fixed_vars=True):
"""
Select a solution from the model's solutions.
allow_consistent_values_for_fixed_vars: a flag that
indicates whether a solution can specify consistent
values for variables in the model that are fixed.
ignore_invalid_labels: a flag that indicates whether
labels in the solution that don't appear in the model
yield an error. This allows for loading a results object
generated from one model into another related, but not
identical, model.
"""
instance = self._instance()
#
# Set the "stale" flag of each variable in the model prior to
# loading the solution, so you known which variables have "real"
# values and which ones don't.
#
StaleFlagManager.mark_all_as_stale()
if index is not None:
self.index = index
soln = self.solutions[self.index]
#
# Generate the list of active import suffixes on this top level model
#
valid_import_suffixes = dict(active_import_suffix_generator(instance))
#
# To ensure that import suffix data gets properly overwritten (e.g.,
# the case where nonzero dual values exist on the suffix and but only
# sparse dual values exist in the results object) we clear all active
# import suffixes.
#
for suffix in valid_import_suffixes.values():
suffix.clear_all_values()
#
# Load problem (model) level suffixes. These would only come from ampl
# interfaced solution suffixes at this point in time.
#
for id_, (pobj, entry) in soln._entry['problem'].items():
for _attr_key, attr_value in entry.items():
attr_key = _attr_key[0].lower() + _attr_key[1:]
if attr_key in valid_import_suffixes:
valid_import_suffixes[attr_key][pobj] = attr_value
#
# Load objective data (suffixes)
#
for id_, (odata, entry) in soln._entry['objective'].items():
odata = odata()
for _attr_key, attr_value in entry.items():
attr_key = _attr_key[0].lower() + _attr_key[1:]
if attr_key in valid_import_suffixes:
valid_import_suffixes[attr_key][odata] = attr_value
#
# Load variable data (suffixes and values)
#
for id_, (vdata, entry) in soln._entry['variable'].items():
vdata = vdata()
val = entry['Value']
if vdata.fixed is True:
if ignore_fixed_vars:
continue
if not allow_consistent_values_for_fixed_vars:
msg = "Variable '%s' in model '%s' is currently fixed - new" \
' value is not expected in solution'
raise TypeError(msg % (vdata.name, instance.name))
if math.fabs(val - vdata.value
) > comparison_tolerance_for_fixed_vars:
raise TypeError("Variable '%s' in model '%s' is currently "
"fixed - a value of '%s' in solution is "
"not within tolerance=%s of the current "
"value of '%s'" %
(vdata.name, instance.name, str(val),
str(comparison_tolerance_for_fixed_vars),
str(vdata.value)))
vdata.set_value(val, skip_validation=True)
for _attr_key, attr_value in entry.items():
attr_key = _attr_key[0].lower() + _attr_key[1:]
if attr_key == 'value':
continue
elif attr_key in valid_import_suffixes:
valid_import_suffixes[attr_key][vdata] = attr_value
#
# Load constraint data (suffixes)
#
for id_, (cdata, entry) in soln._entry['constraint'].items():
cdata = cdata()
for _attr_key, attr_value in entry.items():
attr_key = _attr_key[0].lower() + _attr_key[1:]
if attr_key in valid_import_suffixes:
valid_import_suffixes[attr_key][cdata] = attr_value
# Set the state flag to "delayed advance": it will auto-advance
# if a non-stale variable is updated (causing all non-stale
# variables to be marked as stale).
StaleFlagManager.mark_all_as_stale(delayed=True)
def load_solution(self,
solution,
allow_consistent_values_for_fixed_vars=False,
comparison_tolerance_for_fixed_vars=1e-5):
"""
Load a solution.
Args:
solution: A :class:`pyomo.opt.Solution` object with a
symbol map. Optionally, the solution can be tagged
with a default variable value (e.g., 0) that will be
applied to those variables in the symbol map that do
not have a value in the solution.
allow_consistent_values_for_fixed_vars:
Indicates whether a solution can specify
consistent values for variables that are
fixed.
comparison_tolerance_for_fixed_vars: The
tolerance used to define whether or not a
value in the solution is consistent with the
value of a fixed variable.
"""
from pyomo.core.kernel.suffix import \
import_suffix_generator
symbol_map = solution.symbol_map
default_variable_value = getattr(solution, "default_variable_value",
None)
# Generate the list of active import suffixes on
# this top level model
valid_import_suffixes = \
{obj.storage_key:obj
for obj in import_suffix_generator(self)}
# To ensure that import suffix data gets properly
# overwritten (e.g., the case where nonzero dual
# values exist on the suffix and but only sparse
# dual values exist in the results object) we clear
# all active import suffixes.
for suffix in valid_import_suffixes.values():
suffix.clear()
# Load problem (model) level suffixes. These would
# only come from ampl interfaced solution suffixes
# at this point in time.
for _attr_key, attr_value in solution.problem.items():
attr_key = _attr_key[0].lower() + _attr_key[1:]
if attr_key in valid_import_suffixes:
valid_import_suffixes[attr_key][self] = attr_value
#
# Set the "stale" flag of each variable in the model prior to
# loading the solution, so you known which variables have "real"
# values and which ones don't.
#
StaleFlagManager.mark_all_as_stale()
#
# Load variable data
#
from pyomo.core.kernel.variable import IVariable
var_skip_attrs = ['id', 'canonical_label']
seen_var_ids = set()
for label, entry in solution.variable.items():
var = symbol_map.getObject(label)
if (var is None) or \
(var is SymbolMap.UnknownSymbol):
# NOTE: the following is a hack, to handle
# the ONE_VAR_CONSTANT variable that is
# necessary for the objective
# constant-offset terms. probably should
# create a dummy variable in the model
# map at the same time the objective
# expression is being constructed.
if "ONE_VAR_CONST" in label:
continue
else:
raise KeyError("Variable associated with symbol '%s' "
"is not found on this block" % (label))
seen_var_ids.add(id(var))
if (not allow_consistent_values_for_fixed_vars) and \
var.fixed:
raise ValueError("Variable '%s' is currently fixed. "
"A new value is not expected "
"in solution" % (var.name))
for _attr_key, attr_value in entry.items():
attr_key = _attr_key[0].lower() + _attr_key[1:]
if attr_key == 'value':
if allow_consistent_values_for_fixed_vars and \
var.fixed and \
(math.fabs(attr_value - var.value) > \
comparison_tolerance_for_fixed_vars):
raise ValueError(
"Variable %s is currently fixed. "
"A value of '%s' in solution is "
"not within tolerance=%s of the current "
"value of '%s'" %
(var.name, attr_value,
comparison_tolerance_for_fixed_vars, var.value))
var.set_value(attr_value, skip_validation=True)
elif attr_key in valid_import_suffixes:
valid_import_suffixes[attr_key][var] = attr_value
# start to build up the set of unseen variable ids
unseen_var_ids = set(symbol_map.byObject.keys())
# at this point it contains ids for non-variable types
unseen_var_ids.difference_update(seen_var_ids)
#
# Load objective solution (should simply be suffixes if
# they exist)
#
objective_skip_attrs = ['id', 'canonical_label', 'value']
for label, entry in solution.objective.items():
obj = symbol_map.getObject(label)
if (obj is None) or \
(obj is SymbolMap.UnknownSymbol):
raise KeyError("Objective associated with symbol '%s' "
"is not found on this block" % (label))
# Because of __default_objective__, an objective might
# appear twice in the objective dictionary.
unseen_var_ids.discard(id(obj))
for _attr_key, attr_value in entry.items():
attr_key = _attr_key[0].lower() + _attr_key[1:]
if attr_key in valid_import_suffixes:
valid_import_suffixes[attr_key][obj] = \
attr_value
#
# Load constraint solution
#
con_skip_attrs = ['id', 'canonical_label']
for label, entry in solution.constraint.items():
con = symbol_map.getObject(label)
if con is SymbolMap.UnknownSymbol:
#
# This is a hack - see above.
#
if "ONE_VAR_CONST" in label:
continue
else:
raise KeyError("Constraint associated with symbol '%s' "
"is not found on this block" % (label))
unseen_var_ids.remove(id(con))
for _attr_key, attr_value in entry.items():
attr_key = _attr_key[0].lower() + _attr_key[1:]
if attr_key in valid_import_suffixes:
valid_import_suffixes[attr_key][con] = \
attr_value
#
# Load sparse variable solution
#
if default_variable_value is not None:
for var_id in unseen_var_ids:
var = symbol_map.getObject(symbol_map.byObject[var_id])
if var.ctype is not IVariable:
continue
if (not allow_consistent_values_for_fixed_vars) and \
var.fixed:
raise ValueError("Variable '%s' is currently fixed. "
"A new value is not expected "
"in solution" % (var.name))
if allow_consistent_values_for_fixed_vars and \
var.fixed and \
(math.fabs(default_variable_value - var.value) > \
comparison_tolerance_for_fixed_vars):
raise ValueError(
"Variable %s is currently fixed. "
"A value of '%s' in solution is "
"not within tolerance=%s of the current "
"value of '%s'" %
(var.name, default_variable_value,
comparison_tolerance_for_fixed_vars, var.value))
var.set_value(default_variable_value, skip_validation=True)
# Set the state flag to "delayed advance": it will auto-advance
# if a non-stale variable is updated (causing all non-stale
# variables to be marked as stale).
StaleFlagManager.mark_all_as_stale(delayed=True)
def _apply_solver(self):
StaleFlagManager.mark_all_as_stale()
if self._tee:
self._solver_model.setParam('OutputFlag', 1)
else:
self._solver_model.setParam('OutputFlag', 0)
if self._keepfiles:
# Only save log file when the user wants to keep it.
self._solver_model.setParam('LogFile', self._log_file)
print("Solver log file: " + self._log_file)
# Options accepted by gurobi (case insensitive):
# ['Cutoff', 'IterationLimit', 'NodeLimit', 'SolutionLimit', 'TimeLimit',
# 'FeasibilityTol', 'IntFeasTol', 'MarkowitzTol', 'MIPGap', 'MIPGapAbs',
# 'OptimalityTol', 'PSDTol', 'Method', 'PerturbValue', 'ObjScale', 'ScaleFlag',
# 'SimplexPricing', 'Quad', 'NormAdjust', 'BarIterLimit', 'BarConvTol',
# 'BarCorrectors', 'BarOrder', 'Crossover', 'CrossoverBasis', 'BranchDir',
# 'Heuristics', 'MinRelNodes', 'MIPFocus', 'NodefileStart', 'NodefileDir',
# 'NodeMethod', 'PumpPasses', 'RINS', 'SolutionNumber', 'SubMIPNodes', 'Symmetry',
# 'VarBranch', 'Cuts', 'CutPasses', 'CliqueCuts', 'CoverCuts', 'CutAggPasses',
# 'FlowCoverCuts', 'FlowPathCuts', 'GomoryPasses', 'GUBCoverCuts', 'ImpliedCuts',
# 'MIPSepCuts', 'MIRCuts', 'NetworkCuts', 'SubMIPCuts', 'ZeroHalfCuts', 'ModKCuts',
# 'Aggregate', 'AggFill', 'PreDual', 'DisplayInterval', 'IISMethod', 'InfUnbdInfo',
# 'LogFile', 'PreCrush', 'PreDepRow', 'PreMIQPMethod', 'PrePasses', 'Presolve',
# 'ResultFile', 'ImproveStartTime', 'ImproveStartGap', 'Threads', 'Dummy', 'OutputFlag']
for key, option in self.options.items():
# When options come from the pyomo command, all
# values are string types, so we try to cast
# them to a numeric value in the event that
# setting the parameter fails.
try:
self._solver_model.setParam(key, option)
except TypeError:
# we place the exception handling for
# checking the cast of option to a float in
# another function so that we can simply
# call raise here instead of except
# TypeError as e / raise e, because the
# latter does not preserve the Gurobi stack
# trace
if not _is_numeric(option):
raise
self._solver_model.setParam(key, float(option))
if self._version_major >= 5:
for suffix in self._suffixes:
if re.match(suffix, "dual"):
self._solver_model.setParam(gurobipy.GRB.Param.QCPDual, 1)
self._solver_model.optimize(self._callback)
self._needs_updated = False
if self._keepfiles:
# Change LogFile to make Gurobi close the original log file.
# May not work for all Gurobi versions, like ver. 9.5.0.
self._solver_model.setParam('LogFile', 'default')
# FIXME: can we get a return code indicating if Gurobi had a significant failure?
return Bunch(rc=None, log=None)
def _apply_solver(self):
StaleFlagManager.mark_all_as_stale()
# In recent versions of CPLEX it is helpful to manually open the
# log file and then explicitly close it after CPLEX is finished.
# This ensures that the file is closed (and unlocked) on Windows
# before the TempfileManager (or user) attempts to delete the
# log file. Passing in an opened file object is supported at
# least as far back as CPLEX 12.5.1 [the oldest version
# supported by IBM as of 1 Oct 2020]
if self.version() >= (12, 5, 1) \
and isinstance(self._log_file, str):
_log_file = (open(self._log_file, 'a'),)
_close_log_file = True
else:
_log_file = (self._log_file,)
_close_log_file = False
if self._tee:
def _process_stream(arg):
sys.stdout.write(arg)
return arg
_log_file += (_process_stream,)
try:
self._solver_model.set_results_stream(*_log_file)
if self._keepfiles:
print("Solver log file: "+self._log_file)
obj_degree = self._objective.expr.polynomial_degree()
if obj_degree is None or obj_degree > 2:
raise DegreeError('CPLEXDirect does not support expressions of degree {0}.'\
.format(obj_degree))
elif obj_degree == 2:
quadratic_objective = True
else:
quadratic_objective = False
num_integer_vars = self._solver_model.variables.get_num_integer()
num_binary_vars = self._solver_model.variables.get_num_binary()
num_sos = self._solver_model.SOS.get_num()
if self._solver_model.quadratic_constraints.get_num() != 0:
quadratic_cons = True
else:
quadratic_cons = False
if (num_integer_vars + num_binary_vars + num_sos) > 0:
integer = True
else:
integer = False
if integer:
if quadratic_cons:
self._solver_model.set_problem_type(self._solver_model.problem_type.MIQCP)
elif quadratic_objective:
self._solver_model.set_problem_type(self._solver_model.problem_type.MIQP)
else:
self._solver_model.set_problem_type(self._solver_model.problem_type.MILP)
else:
if quadratic_cons:
self._solver_model.set_problem_type(self._solver_model.problem_type.QCP)
elif quadratic_objective:
self._solver_model.set_problem_type(self._solver_model.problem_type.QP)
else:
self._solver_model.set_problem_type(self._solver_model.problem_type.LP)
# if the user specifies a 'mipgap'
# set cplex's mip.tolerances.mipgap
if self.options.mipgap is not None:
self._solver_model.parameters.mip.tolerances.mipgap.set(float(self.options.mipgap))
for key, option in self.options.items():
if key == 'mipgap': # handled above
continue
opt_cmd = self._solver_model.parameters
key_pieces = key.split('_')
for key_piece in key_pieces:
opt_cmd = getattr(opt_cmd, key_piece)
# When options come from the pyomo command, all
# values are string types, so we try to cast
# them to a numeric value in the event that
# setting the parameter fails.
try:
opt_cmd.set(option)
except self._cplex.exceptions.CplexError:
# we place the exception handling for
# checking the cast of option to a float in
# another function so that we can simply
# call raise here instead of except
# TypeError as e / raise e, because the
# latter does not preserve the Cplex stack
# trace
if not _is_numeric(option):
raise
opt_cmd.set(float(option))
t0 = time.time()
self._solver_model.solve()
t1 = time.time()
self._wallclock_time = t1 - t0
finally:
self._solver_model.set_results_stream(None)
if _close_log_file:
_log_file[0].close()
# FIXME: can we get a return code indicating if CPLEX had a significant failure?
return Bunch(rc=None, log=None)
def value(self, value):
self._value = value
self._stale = StaleFlagManager.get_flag(self._stale)
def stale(self, stale):
if stale:
self._stale = 0
else:
self._stale = StaleFlagManager.get_flag(0)
def stale(self):
"""The stale status of the variable"""
return StaleFlagManager.is_stale(self._stale)