def _add_optimality_conditions(self, instance, submodel):
"""
Add optimality conditions for the submodel
This assumes that the original model has the form:
min c1*x + d1*y
A3*x <= b3
A1*x + B1*y <= b1
min c2*x + d2*y + x'*Q*y
A2*x + B2*y + x'*E2*y <= b2
y >= 0
NOTE THE VARIABLE BOUNDS!
"""
#
# Populate the block with the linear constraints.
# Note that we don't simply clone the current block.
# We need to collect a single set of equations that
# can be easily expressed.
#
d2 = {}
B2 = {}
vtmp = {}
utmp = {}
sids_set = set()
sids_list = []
#
block = Block(concrete=True)
block.u = VarList()
block.v = VarList()
block.c1 = ConstraintList()
block.c2 = ComplementarityList()
block.c3 = ComplementarityList()
#
# Collect submodel objective terms
#
# TODO: detect fixed variables
#
for odata in submodel.component_data_objects(Objective, active=True):
if odata.sense == maximize:
d_sense = -1
else:
d_sense = 1
#
# Iterate through the variables in the representation
#
o_terms = generate_standard_repn(odata.expr, compute_values=False)
#
# Linear terms
#
for i, var in enumerate(o_terms.linear_vars):
if var.parent_component().local_name in self._fixed_upper_vars:
#
# Skip fixed upper variables
#
continue
#
# Store the coefficient for the variable. The coefficient is
# negated if the objective is maximized.
#
id_ = id(var)
d2[id_] = d_sense * o_terms.linear_coefs[i]
if not id_ in sids_set:
sids_set.add(id_)
sids_list.append(id_)
#
# Quadratic terms
#
for i, var in enumerate(o_terms.quadratic_vars):
if var[0].parent_component().local_name in self._fixed_upper_vars:
if var[1].parent_component().local_name in self._fixed_upper_vars:
#
# Skip fixed upper variables
#
continue
#
# Add the linear term
#
id_ = id(var[1])
d2[id_] = d2.get(id_,0) + d_sense * o_terms.quadratic_coefs[i] * var[0]
if not id_ in sids_set:
sids_set.add(id_)
sids_list.append(id_)
elif var[1].parent_component().local_name in self._fixed_upper_vars:
#
# Add the linear term
#
id_ = id(var[0])
d2[id_] = d2.get(id_,0) + d_sense * o_terms.quadratic_coefs[i] * var[1]
if not id_ in sids_set:
sids_set.add(id_)
sids_list.append(id_)
else:
raise RuntimeError("Cannot apply this transformation to a problem with quadratic terms where both variables are in the lower level.")
#
# Stop after the first objective
#
break
#
# Iterate through all lower level variables, adding dual variables
# and complementarity slackness conditions for y bound constraints
#
for vcomponent in instance.component_objects(Var, active=True):
if vcomponent.local_name in self._fixed_upper_vars:
#
# Skip fixed upper variables
#
continue
for ndx in vcomponent:
#
# For each index, get the bounds for the variable
#
lb, ub = vcomponent[ndx].bounds
if not lb is None:
#
# Add the complementarity slackness condition for a lower bound
#
v = block.v.add()
block.c3.add( complements(vcomponent[ndx] >= lb, v >= 0) )
else:
v = None
if not ub is None:
#
# Add the complementarity slackness condition for an upper bound
#
w = block.v.add()
vtmp[id(vcomponent[ndx])] = w
block.c3.add( complements(vcomponent[ndx] <= ub, w >= 0) )
else:
w = None
if not (v is None and w is None):
#
# Record the variables for which complementarity slackness conditions
# were created.
#
id_ = id(vcomponent[ndx])
vtmp[id_] = (v,w)
if not id_ in sids_set:
sids_set.add(id_)
sids_list.append(id_)
#
# Iterate through all constraints, adding dual variables and
# complementary slackness conditions (for inequality constraints)
#
for cdata in submodel.component_data_objects(Constraint, active=True):
if cdata.equality:
# Don't add a complementary slackness condition for an equality constraint
u = block.u.add()
utmp[id(cdata)] = (None,u)
else:
if not cdata.lower is None:
#
# Add the complementarity slackness condition for a greater-than inequality
#
u = block.u.add()
block.c2.add( complements(- cdata.body <= - cdata.lower, u >= 0) )
else:
u = None
if not cdata.upper is None:
#
# Add the complementarity slackness condition for a less-than inequality
#
w = block.u.add()
block.c2.add( complements(cdata.body <= cdata.upper, w >= 0) )
else:
w = None
if not (u is None and w is None):
utmp[id(cdata)] = (u,w)
#
# Store the coefficients for the constraint variables that are not fixed
#
c_terms = generate_standard_repn(cdata.body, compute_values=False)
#
# Linear terms
#
for i, var in enumerate(c_terms.linear_vars):
if var.parent_component().local_name in self._fixed_upper_vars:
continue
id_ = id(var)
B2.setdefault(id_,{}).setdefault(id(cdata),c_terms.linear_coefs[i])
if not id_ in sids_set:
sids_set.add(id_)
sids_list.append(id_)
#
# Quadratic terms
#
for i, var in enumerate(c_terms.quadratic_vars):
if var[0].parent_component().local_name in self._fixed_upper_vars:
if var[1].parent_component().local_name in self._fixed_upper_vars:
continue
id_ = id(var[1])
if id_ in B2:
B2[id_][id(cdata)] = c_terms.quadratic_coefs[i] * var[0]
else:
B2.setdefault(id_,{}).setdefault(id(cdata),c_terms.quadratic_coefs[i] * var[0])
if not id_ in sids_set:
sids_set.add(id_)
sids_list.append(id_)
elif var[1].parent_component().local_name in self._fixed_upper_vars:
id_ = id(var[0])
if id_ in B2:
B2[id_][id(cdata)] = c_terms.quadratic_coefs[i] * var[1]
else:
B2.setdefault(id_,{}).setdefault(id(cdata),c_terms.quadratic_coefs[i] * var[1])
if not id_ in sids_set:
sids_set.add(id_)
sids_list.append(id_)
else:
raise RuntimeError("Cannot apply this transformation to a problem with quadratic terms where both variables are in the lower level.")
#
# Generate stationarity equations
#
tmp__ = (None, None)
for vid in sids_list:
exp = d2.get(vid,0)
#
lb_dual, ub_dual = vtmp.get(vid, tmp__)
if vid in vtmp:
if not lb_dual is None:
exp -= lb_dual # dual for variable lower bound
if not ub_dual is None:
exp += ub_dual # dual for variable upper bound
#
B2_ = B2.get(vid,{})
utmp_keys = list(utmp.keys())
if self._deterministic:
utmp_keys.sort(key=lambda x:utmp[x][0].local_name if utmp[x][1] is None else utmp[x][1].local_name)
for uid in utmp_keys:
if uid in B2_:
lb_dual, ub_dual = utmp[uid]
if not lb_dual is None:
exp -= B2_[uid] * lb_dual
if not ub_dual is None:
exp += B2_[uid] * ub_dual
if type(exp) in six.integer_types or type(exp) is float:
# TODO: Annotate the model as unbounded
raise IOError("Unbounded variable without side constraints")
else:
block.c1.add( exp == 0 )
#
# Return block
#
return block
def _apply_to(self, model, detect_fixed_vars=True):
"""Apply the transformation to the given model."""
# Generate the equality sets
eq_var_map = _build_equality_set(model)
# Detect and process fixed variables.
if detect_fixed_vars:
_fix_equality_fixed_variables(model)
# Generate aggregation infrastructure
model._var_aggregator_info = Block(
doc="Holds information for the variable aggregation "
"transformation system.")
z = model._var_aggregator_info.z = VarList(doc="Aggregated variables.")
# Map of the aggregate var to the equalty set (ComponentSet)
z_to_vars = model._var_aggregator_info.z_to_vars = ComponentMap()
# Map of variables to their corresponding aggregate var
var_to_z = model._var_aggregator_info.var_to_z = ComponentMap()
processed_vars = ComponentSet()
# TODO This iteritems is sorted by the variable name of the key in
# order to preserve determinism. Unfortunately, var.name() is an
# expensive operation right now.
for var, eq_set in sorted(eq_var_map.items(),
key=lambda tup: tup[0].name):
if var in processed_vars:
continue # Skip already-process variables
# This would be weird. The variable hasn't been processed, but is
# in the map. Raise an exception.
assert var_to_z.get(var, None) is None
z_agg = z.add()
z_to_vars[z_agg] = eq_set
var_to_z.update(ComponentMap((v, z_agg) for v in eq_set))
# Set the bounds of the aggregate variable based on the bounds of
# the variables in its equality set.
z_agg.setlb(max_if_not_None(v.lb for v in eq_set if v.has_lb()))
z_agg.setub(min_if_not_None(v.ub for v in eq_set if v.has_ub()))
# Set the fixed status of the aggregate var
fixed_vars = [v for v in eq_set if v.fixed]
if fixed_vars:
# Check to make sure all the fixed values are the same.
if any(var.value != fixed_vars[0].value
for var in fixed_vars[1:]):
raise ValueError(
"Aggregate variable for equality set is fixed to "
"multiple different values: %s" % (fixed_vars, ))
z_agg.fix(fixed_vars[0].value)
# Check that the fixed value lies within bounds.
if z_agg.has_lb() and z_agg.value < value(z_agg.lb):
raise ValueError(
"Aggregate variable for equality set is fixed to "
"a value less than its lower bound: %s < LB %s" %
(z_agg.value, value(z_agg.lb)))
if z_agg.has_ub() and z_agg.value > value(z_agg.ub):
raise ValueError(
"Aggregate variable for equality set is fixed to "
"a value greater than its upper bound: %s > UB %s" %
(z_agg.value, value(z_agg.ub)))
else:
# Set the value to be the average of the values within the
# bounds only if the value is not already fixed.
values_within_bounds = [
v.value for v in eq_set if (v.value is not None and (
(z_agg.has_lb() and v.value >= value(z_agg.lb))
or not z_agg.has_lb()) and (
(z_agg.has_ub() and v.value <= value(z_agg.ub))
or not z_agg.has_ub()))
]
num_vals = len(values_within_bounds)
z_agg.value = (
sum(val for val in values_within_bounds) / num_vals) \
if num_vals > 0 else None
processed_vars.update(eq_set)
# Do the substitution
substitution_map = {id(var): z_var for var, z_var in var_to_z.items()}
for constr in model.component_data_objects(ctype=Constraint,
active=True):
new_body = ExpressionReplacementVisitor(
substitute=substitution_map).dfs_postorder_stack(constr.body)
constr.set_value((constr.lower, new_body, constr.upper))
for objective in model.component_data_objects(ctype=Objective,
active=True):
new_expr = ExpressionReplacementVisitor(
substitute=substitution_map).dfs_postorder_stack(
objective.expr)
objective.set_value(new_expr)
def _dualize(self, block, unfixed=[]):
"""
Generate the dual of a block
"""
#
# Collect linear terms from the block
#
A, b_coef, c_rhs, c_sense, d_sense, vnames, cnames, v_domain = collect_linear_terms(
block, unfixed)
##print(A)
##print(vnames)
##print(cnames)
##print(list(A.keys()))
##print("---")
##print(A.keys())
##print(c_sense)
##print(c_rhs)
#
# Construct the block
#
if isinstance(block, Model):
dual = ConcreteModel()
else:
dual = Block()
dual.construct()
_vars = {}
def getvar(name, ndx=None):
v = _vars.get((name, ndx), None)
if v is None:
v = Var()
if ndx is None:
v_name = name
elif type(ndx) is tuple:
v_name = "%s[%s]" % (name, ','.join(map(str, ndx)))
else:
v_name = "%s[%s]" % (name, str(ndx))
setattr(dual, v_name, v)
_vars[name, ndx] = v
return v
#
# Construct the objective
#
if d_sense == minimize:
dual.o = Objective(expr=sum(-b_coef[name, ndx] * getvar(name, ndx)
for name, ndx in b_coef),
sense=d_sense)
else:
dual.o = Objective(expr=sum(b_coef[name, ndx] * getvar(name, ndx)
for name, ndx in b_coef),
sense=d_sense)
#
# Construct the constraints
#
for cname in A:
for ndx, terms in iteritems(A[cname]):
expr = 0
for term in terms:
expr += term.coef * getvar(term.var, term.ndx)
if not (cname, ndx) in c_rhs:
c_rhs[cname, ndx] = 0.0
if c_sense[cname, ndx] == 'e':
e = expr - c_rhs[cname, ndx] == 0
elif c_sense[cname, ndx] == 'l':
e = expr - c_rhs[cname, ndx] <= 0
else:
e = expr - c_rhs[cname, ndx] >= 0
c = Constraint(expr=e)
if ndx is None:
c_name = cname
elif type(ndx) is tuple:
c_name = "%s[%s]" % (cname, ','.join(map(str, ndx)))
else:
c_name = "%s[%s]" % (cname, str(ndx))
setattr(dual, c_name, c)
#
for (name, ndx), domain in iteritems(v_domain):
v = getvar(name, ndx)
flag = type(ndx) is tuple and (ndx[-1] == 'lb'
or ndx[-1] == 'ub')
if domain == 1:
v.domain = NonNegativeReals
elif domain == -1:
v.domain = NonPositiveReals
else:
# TODO: verify that this case is possible
v.domain = Reals
return dual
def solve(self, model, **kwds):
"""Solve the model.
Warning: this solver is still in beta. Keyword arguments subject to
change. Undocumented keyword arguments definitely subject to change.
This function performs all of the GDPopt solver setup and problem
validation. It then calls upon helper functions to construct the
initial master approximation and iteration loop.
Args:
model (Block): a Pyomo model or block to be solved
"""
config = self.CONFIG(kwds.pop('options', {}))
config.set_value(kwds)
solve_data = GDPoptSolveData()
created_GDPopt_block = False
old_logger_level = config.logger.getEffectiveLevel()
try:
if config.tee and old_logger_level > logging.INFO:
# If the logger does not already include INFO, include it.
config.logger.setLevel(logging.INFO)
config.logger.info("---Starting GDPopt---")
# Create a model block on which to store GDPopt-specific utility
# modeling objects.
if hasattr(model, 'GDPopt_utils'):
raise RuntimeError(
"GDPopt needs to create a Block named GDPopt_utils "
"on the model object, but an attribute with that name "
"already exists.")
else:
created_GDPopt_block = True
model.GDPopt_utils = Block(
doc="Container for GDPopt solver utility modeling objects")
solve_data.original_model = model
solve_data.working_model = clone_orig_model_with_lists(model)
GDPopt = solve_data.working_model.GDPopt_utils
record_original_model_statistics(solve_data, config)
solve_data.current_strategy = config.strategy
# Reformulate integer variables to binary
reformulate_integer_variables(solve_data.working_model, config)
# Save ordered lists of main modeling components, so that data can
# be easily transferred between future model clones.
build_ordered_component_lists(solve_data.working_model)
record_working_model_statistics(solve_data, config)
solve_data.results.solver.name = 'GDPopt ' + str(self.version())
# Save model initial values. These are used later to initialize NLP
# subproblems.
GDPopt.initial_var_values = list(v.value
for v in GDPopt.working_var_list)
# Store the initial model state as the best solution found. If we
# find no better solution, then we will restore from this copy.
solve_data.best_solution_found = list(GDPopt.initial_var_values)
# Validate the model to ensure that GDPopt is able to solve it.
if not model_is_valid(solve_data, config):
return
# Maps in order to keep track of certain generated constraints
GDPopt.oa_cut_map = ComponentMap()
# Integer cuts exclude particular discrete decisions
GDPopt.integer_cuts = ConstraintList(doc='integer cuts')
# Feasible integer cuts exclude discrete realizations that have
# been explored via an NLP subproblem. Depending on model
# characteristics, the user may wish to revisit NLP subproblems
# (with a different initialization, for example). Therefore, these
# cuts are not enabled by default, unless the initial model has no
# discrete decisions.
# Note: these cuts will only exclude integer realizations that are
# not already in the primary GDPopt_integer_cuts ConstraintList.
GDPopt.no_backtracking = ConstraintList(
doc='explored integer cuts')
# Set up iteration counters
solve_data.master_iteration = 0
solve_data.mip_iteration = 0
solve_data.nlp_iteration = 0
# set up bounds
solve_data.LB = float('-inf')
solve_data.UB = float('inf')
solve_data.iteration_log = {}
# Flag indicating whether the solution improved in the past
# iteration or not
solve_data.feasible_solution_improved = False
# Initialize the master problem
self._GDPopt_initialize_master(solve_data, config)
# Algorithm main loop
self._GDPopt_iteration_loop(solve_data, config)
# Update values in working model
copy_var_list_values(from_list=solve_data.best_solution_found,
to_list=GDPopt.working_var_list,
config=config)
GDPopt.objective_value.set_value(
value(solve_data.working_objective_expr))
# Update values in original model
copy_var_list_values(
GDPopt.orig_var_list,
solve_data.original_model.GDPopt_utils.orig_var_list, config)
solve_data.results.problem.lower_bound = solve_data.LB
solve_data.results.problem.upper_bound = solve_data.UB
finally:
config.logger.setLevel(old_logger_level)
if created_GDPopt_block:
model.del_component('GDPopt_utils')
def _create_using(self, model, **kwds):
precision = kwds.pop('precision', 8)
user_discretize = kwds.pop('discretize', set())
verbose = kwds.pop('verbose', False)
M = model.clone()
# TODO: if discretize is not empty, we must translate those
# components over to the components on the cloned instance
_discretize = {}
if user_discretize:
for _var in user_discretize:
_v = M.find_component(_var.name)
if _v.component() is _v:
for _vv in _v.itervalues():
_discretize.setdefault(id(_vv), len(_discretize))
else:
_discretize.setdefault(id(_v), len(_discretize))
# Iterate over all Constraints and identify the bilinear and
# quadratic terms
bilinear_terms = []
quadratic_terms = []
for constraint in M.component_map(Constraint,
active=True).itervalues():
for cname, c in constraint._data.iteritems():
if c.body.polynomial_degree() != 2:
continue
self._collect_bilinear(c.body, bilinear_terms, quadratic_terms)
# We want to find the (minimum?) number of variables to
# discretize so that we cover all the bilinearities -- without
# discretizing both sides of any single bilinear expression.
# First step: figure out how many expressions each term appears
# in
_counts = {}
for q in quadratic_terms:
if not q[1].is_continuous():
continue
_id = id(q[1])
if _id not in _counts:
_counts[_id] = (q[1], set())
_counts[_id][1].add(_id)
for bi in bilinear_terms:
for i in (0, 1):
if not bi[i + 1].is_continuous():
continue
_id = id(bi[i + 1])
if _id not in _counts:
_counts[_id] = (bi[i + 1], set())
_counts[_id][1].add(id(bi[2 - i]))
_tmp_counts = dict(_counts)
# First, remove the variables that the user wants to have discretized
for _id in _discretize:
for _i in _tmp_counts[_id][1]:
if _i == _id:
continue
_tmp_counts[_i][1].remove(_id)
del _tmp_counts[_id]
# All quadratic terms must be discretized (?)
#for q in quadratic_terms:
# _id = id(q[1])
# if _id not in _tmp_counts:
# continue
# _discretize.setdefault(_id, len(_discretize))
# for _i in _tmp_counts[_id][1]:
# if _i == _id:
# continue
# _tmp_counts[_i][1].remove(_id)
# del _tmp_counts[_id]
# Now pick a (minimal) subset of the terms in bilinear expressions
while _tmp_counts:
_ct, _id = max((len(_tmp_counts[i][1]), i) for i in _tmp_counts)
if not _ct:
break
_discretize.setdefault(_id, len(_discretize))
for _i in list(_tmp_counts[_id][1]):
if _i == _id:
continue
_tmp_counts[_i][1].remove(_id)
del _tmp_counts[_id]
#
# Discretize things
#
# Define a block (namespace) for holding the disaggregated
# variables and new constraints
if False: # Set to true when the LP writer is fixed
M._radix_linearization = Block()
_block = M._radix_linearization
else:
_block = M
_block.DISCRETIZATION = RangeSet(precision)
_block.DISCRETIZED_VARIABLES = RangeSet(0, len(_discretize) - 1)
_block.z = Var(_block.DISCRETIZED_VARIABLES,
_block.DISCRETIZATION,
within=Binary)
_block.dv = Var(_block.DISCRETIZED_VARIABLES,
bounds=(0, 2**-precision))
# Actually discretize the terms we have marked for discretization
for _id, _idx in iteritems(_discretize):
if verbose:
logger.info("Discretizing variable %s as %s" %
(_counts[_id][0].name, _idx))
self._discretize_variable(_block, _counts[_id][0], _idx)
_known_bilinear = {}
# For each quadratic term, if it hasn't been discretized /
# generated, do so, and remember the resulting W term for later
# use...
#for _expr, _x1 in quadratic_terms:
# self._discretize_term( _expr, _x1, _x1,
# _block, _discretize, _known_bilinear )
# For each bilinear term, if it hasn't been discretized /
# generated, do so, and remember the resulting W term for later
# use...
for _expr, _x1, _x2 in bilinear_terms:
self._discretize_term(_expr, _x1, _x2, _block, _discretize,
_known_bilinear)
# Return the discretized instance!
return M
def _dualize(self, block, unfixed=[]):
"""
Generate the dual of a block
"""
#
# Collect linear terms from the block
#
A, b_coef, c_rhs, c_sense, d_sense, vnames, cnames, v_domain = collect_linear_terms(
block, unfixed)
#
# Construct the block
#
if isinstance(block, Model):
dual = ConcreteModel()
else:
dual = Block()
for v, is_indexed in vnames:
if is_indexed:
setattr(dual, v + '_Index', Set(dimen=None))
setattr(dual, v, Var(getattr(dual, v + '_Index')))
else:
setattr(dual, v, Var())
for cname, is_indexed in cnames:
if is_indexed:
setattr(dual, cname + '_Index', Set(dimen=None))
setattr(dual, cname,
Constraint(getattr(dual, cname + '_Index')))
setattr(dual, cname + '_lower_',
Var(getattr(dual, cname + '_Index')))
setattr(dual, cname + '_upper_',
Var(getattr(dual, cname + '_Index')))
else:
setattr(dual, cname, Constraint())
setattr(dual, cname + '_lower_', Var())
setattr(dual, cname + '_upper_', Var())
dual.construct()
#
# Construct the objective
#
if d_sense == minimize:
dual.o = Objective(expr=sum(-b_coef[name, ndx] *
getattr(dual, name)[ndx]
for name, ndx in b_coef),
sense=d_sense)
else:
dual.o = Objective(expr=sum(b_coef[name, ndx] *
getattr(dual, name)[ndx]
for name, ndx in b_coef),
sense=d_sense)
#
# Construct the constraints
#
for cname in A:
c = getattr(dual, cname)
c_index = getattr(dual, cname +
"_Index") if c.is_indexed() else None
for ndx, terms in iteritems(A[cname]):
if not c_index is None and not ndx in c_index:
c_index.add(ndx)
expr = 0
for term in terms:
v = getattr(dual, term.var)
if not term.ndx in v:
v.add(term.ndx)
expr += term.coef * v[term.ndx]
if not (cname, ndx) in c_rhs:
c_rhs[cname, ndx] = 0.0
if c_sense[cname, ndx] == 'e':
c.add(ndx, expr - c_rhs[cname, ndx] == 0)
elif c_sense[cname, ndx] == 'l':
c.add(ndx, expr - c_rhs[cname, ndx] <= 0)
else:
c.add(ndx, expr - c_rhs[cname, ndx] >= 0)
for (name, ndx), domain in iteritems(v_domain):
v = getattr(dual, name)
flag = type(ndx) is tuple and (ndx[-1] == 'lb'
or ndx[-1] == 'ub')
if domain == 1:
if flag:
v[ndx].domain = NonNegativeReals
else:
v.domain = NonNegativeReals
elif domain == -1:
if flag:
v[ndx].domain = NonPositiveReals
else:
v.domain = NonPositiveReals
else:
if flag:
# TODO: verify that this case is possible
v[ndx].domain = Reals
else:
v.domain = Reals
return dual