def _add_optimality_conditions(self, instance, submodel): """ Add optimality conditions for the submodel This assumes that the original model has the form: min c1*x + d1*y A3*x <= b3 A1*x + B1*y <= b1 min c2*x + d2*y + x'*Q*y A2*x + B2*y + x'*E2*y <= b2 y >= 0 NOTE THE VARIABLE BOUNDS! """ # # Populate the block with the linear constraints. # Note that we don't simply clone the current block. # We need to collect a single set of equations that # can be easily expressed. # d2 = {} B2 = {} vtmp = {} utmp = {} sids_set = set() sids_list = [] # block = Block(concrete=True) block.u = VarList() block.v = VarList() block.c1 = ConstraintList() block.c2 = ComplementarityList() block.c3 = ComplementarityList() # # Collect submodel objective terms # # TODO: detect fixed variables # for odata in submodel.component_data_objects(Objective, active=True): if odata.sense == maximize: d_sense = -1 else: d_sense = 1 # # Iterate through the variables in the representation # o_terms = generate_standard_repn(odata.expr, compute_values=False) # # Linear terms # for i, var in enumerate(o_terms.linear_vars): if var.parent_component().local_name in self._fixed_upper_vars: # # Skip fixed upper variables # continue # # Store the coefficient for the variable. The coefficient is # negated if the objective is maximized. # id_ = id(var) d2[id_] = d_sense * o_terms.linear_coefs[i] if not id_ in sids_set: sids_set.add(id_) sids_list.append(id_) # # Quadratic terms # for i, var in enumerate(o_terms.quadratic_vars): if var[0].parent_component().local_name in self._fixed_upper_vars: if var[1].parent_component().local_name in self._fixed_upper_vars: # # Skip fixed upper variables # continue # # Add the linear term # id_ = id(var[1]) d2[id_] = d2.get(id_,0) + d_sense * o_terms.quadratic_coefs[i] * var[0] if not id_ in sids_set: sids_set.add(id_) sids_list.append(id_) elif var[1].parent_component().local_name in self._fixed_upper_vars: # # Add the linear term # id_ = id(var[0]) d2[id_] = d2.get(id_,0) + d_sense * o_terms.quadratic_coefs[i] * var[1] if not id_ in sids_set: sids_set.add(id_) sids_list.append(id_) else: raise RuntimeError("Cannot apply this transformation to a problem with quadratic terms where both variables are in the lower level.") # # Stop after the first objective # break # # Iterate through all lower level variables, adding dual variables # and complementarity slackness conditions for y bound constraints # for vcomponent in instance.component_objects(Var, active=True): if vcomponent.local_name in self._fixed_upper_vars: # # Skip fixed upper variables # continue for ndx in vcomponent: # # For each index, get the bounds for the variable # lb, ub = vcomponent[ndx].bounds if not lb is None: # # Add the complementarity slackness condition for a lower bound # v = block.v.add() block.c3.add( complements(vcomponent[ndx] >= lb, v >= 0) ) else: v = None if not ub is None: # # Add the complementarity slackness condition for an upper bound # w = block.v.add() vtmp[id(vcomponent[ndx])] = w block.c3.add( complements(vcomponent[ndx] <= ub, w >= 0) ) else: w = None if not (v is None and w is None): # # Record the variables for which complementarity slackness conditions # were created. # id_ = id(vcomponent[ndx]) vtmp[id_] = (v,w) if not id_ in sids_set: sids_set.add(id_) sids_list.append(id_) # # Iterate through all constraints, adding dual variables and # complementary slackness conditions (for inequality constraints) # for cdata in submodel.component_data_objects(Constraint, active=True): if cdata.equality: # Don't add a complementary slackness condition for an equality constraint u = block.u.add() utmp[id(cdata)] = (None,u) else: if not cdata.lower is None: # # Add the complementarity slackness condition for a greater-than inequality # u = block.u.add() block.c2.add( complements(- cdata.body <= - cdata.lower, u >= 0) ) else: u = None if not cdata.upper is None: # # Add the complementarity slackness condition for a less-than inequality # w = block.u.add() block.c2.add( complements(cdata.body <= cdata.upper, w >= 0) ) else: w = None if not (u is None and w is None): utmp[id(cdata)] = (u,w) # # Store the coefficients for the constraint variables that are not fixed # c_terms = generate_standard_repn(cdata.body, compute_values=False) # # Linear terms # for i, var in enumerate(c_terms.linear_vars): if var.parent_component().local_name in self._fixed_upper_vars: continue id_ = id(var) B2.setdefault(id_,{}).setdefault(id(cdata),c_terms.linear_coefs[i]) if not id_ in sids_set: sids_set.add(id_) sids_list.append(id_) # # Quadratic terms # for i, var in enumerate(c_terms.quadratic_vars): if var[0].parent_component().local_name in self._fixed_upper_vars: if var[1].parent_component().local_name in self._fixed_upper_vars: continue id_ = id(var[1]) if id_ in B2: B2[id_][id(cdata)] = c_terms.quadratic_coefs[i] * var[0] else: B2.setdefault(id_,{}).setdefault(id(cdata),c_terms.quadratic_coefs[i] * var[0]) if not id_ in sids_set: sids_set.add(id_) sids_list.append(id_) elif var[1].parent_component().local_name in self._fixed_upper_vars: id_ = id(var[0]) if id_ in B2: B2[id_][id(cdata)] = c_terms.quadratic_coefs[i] * var[1] else: B2.setdefault(id_,{}).setdefault(id(cdata),c_terms.quadratic_coefs[i] * var[1]) if not id_ in sids_set: sids_set.add(id_) sids_list.append(id_) else: raise RuntimeError("Cannot apply this transformation to a problem with quadratic terms where both variables are in the lower level.") # # Generate stationarity equations # tmp__ = (None, None) for vid in sids_list: exp = d2.get(vid,0) # lb_dual, ub_dual = vtmp.get(vid, tmp__) if vid in vtmp: if not lb_dual is None: exp -= lb_dual # dual for variable lower bound if not ub_dual is None: exp += ub_dual # dual for variable upper bound # B2_ = B2.get(vid,{}) utmp_keys = list(utmp.keys()) if self._deterministic: utmp_keys.sort(key=lambda x:utmp[x][0].local_name if utmp[x][1] is None else utmp[x][1].local_name) for uid in utmp_keys: if uid in B2_: lb_dual, ub_dual = utmp[uid] if not lb_dual is None: exp -= B2_[uid] * lb_dual if not ub_dual is None: exp += B2_[uid] * ub_dual if type(exp) in six.integer_types or type(exp) is float: # TODO: Annotate the model as unbounded raise IOError("Unbounded variable without side constraints") else: block.c1.add( exp == 0 ) # # Return block # return block
def _apply_to(self, model, detect_fixed_vars=True): """Apply the transformation to the given model.""" # Generate the equality sets eq_var_map = _build_equality_set(model) # Detect and process fixed variables. if detect_fixed_vars: _fix_equality_fixed_variables(model) # Generate aggregation infrastructure model._var_aggregator_info = Block( doc="Holds information for the variable aggregation " "transformation system.") z = model._var_aggregator_info.z = VarList(doc="Aggregated variables.") # Map of the aggregate var to the equalty set (ComponentSet) z_to_vars = model._var_aggregator_info.z_to_vars = ComponentMap() # Map of variables to their corresponding aggregate var var_to_z = model._var_aggregator_info.var_to_z = ComponentMap() processed_vars = ComponentSet() # TODO This iteritems is sorted by the variable name of the key in # order to preserve determinism. Unfortunately, var.name() is an # expensive operation right now. for var, eq_set in sorted(eq_var_map.items(), key=lambda tup: tup[0].name): if var in processed_vars: continue # Skip already-process variables # This would be weird. The variable hasn't been processed, but is # in the map. Raise an exception. assert var_to_z.get(var, None) is None z_agg = z.add() z_to_vars[z_agg] = eq_set var_to_z.update(ComponentMap((v, z_agg) for v in eq_set)) # Set the bounds of the aggregate variable based on the bounds of # the variables in its equality set. z_agg.setlb(max_if_not_None(v.lb for v in eq_set if v.has_lb())) z_agg.setub(min_if_not_None(v.ub for v in eq_set if v.has_ub())) # Set the fixed status of the aggregate var fixed_vars = [v for v in eq_set if v.fixed] if fixed_vars: # Check to make sure all the fixed values are the same. if any(var.value != fixed_vars[0].value for var in fixed_vars[1:]): raise ValueError( "Aggregate variable for equality set is fixed to " "multiple different values: %s" % (fixed_vars, )) z_agg.fix(fixed_vars[0].value) # Check that the fixed value lies within bounds. if z_agg.has_lb() and z_agg.value < value(z_agg.lb): raise ValueError( "Aggregate variable for equality set is fixed to " "a value less than its lower bound: %s < LB %s" % (z_agg.value, value(z_agg.lb))) if z_agg.has_ub() and z_agg.value > value(z_agg.ub): raise ValueError( "Aggregate variable for equality set is fixed to " "a value greater than its upper bound: %s > UB %s" % (z_agg.value, value(z_agg.ub))) else: # Set the value to be the average of the values within the # bounds only if the value is not already fixed. values_within_bounds = [ v.value for v in eq_set if (v.value is not None and ( (z_agg.has_lb() and v.value >= value(z_agg.lb)) or not z_agg.has_lb()) and ( (z_agg.has_ub() and v.value <= value(z_agg.ub)) or not z_agg.has_ub())) ] num_vals = len(values_within_bounds) z_agg.value = ( sum(val for val in values_within_bounds) / num_vals) \ if num_vals > 0 else None processed_vars.update(eq_set) # Do the substitution substitution_map = {id(var): z_var for var, z_var in var_to_z.items()} for constr in model.component_data_objects(ctype=Constraint, active=True): new_body = ExpressionReplacementVisitor( substitute=substitution_map).dfs_postorder_stack(constr.body) constr.set_value((constr.lower, new_body, constr.upper)) for objective in model.component_data_objects(ctype=Objective, active=True): new_expr = ExpressionReplacementVisitor( substitute=substitution_map).dfs_postorder_stack( objective.expr) objective.set_value(new_expr)
def _dualize(self, block, unfixed=[]): """ Generate the dual of a block """ # # Collect linear terms from the block # A, b_coef, c_rhs, c_sense, d_sense, vnames, cnames, v_domain = collect_linear_terms( block, unfixed) ##print(A) ##print(vnames) ##print(cnames) ##print(list(A.keys())) ##print("---") ##print(A.keys()) ##print(c_sense) ##print(c_rhs) # # Construct the block # if isinstance(block, Model): dual = ConcreteModel() else: dual = Block() dual.construct() _vars = {} def getvar(name, ndx=None): v = _vars.get((name, ndx), None) if v is None: v = Var() if ndx is None: v_name = name elif type(ndx) is tuple: v_name = "%s[%s]" % (name, ','.join(map(str, ndx))) else: v_name = "%s[%s]" % (name, str(ndx)) setattr(dual, v_name, v) _vars[name, ndx] = v return v # # Construct the objective # if d_sense == minimize: dual.o = Objective(expr=sum(-b_coef[name, ndx] * getvar(name, ndx) for name, ndx in b_coef), sense=d_sense) else: dual.o = Objective(expr=sum(b_coef[name, ndx] * getvar(name, ndx) for name, ndx in b_coef), sense=d_sense) # # Construct the constraints # for cname in A: for ndx, terms in iteritems(A[cname]): expr = 0 for term in terms: expr += term.coef * getvar(term.var, term.ndx) if not (cname, ndx) in c_rhs: c_rhs[cname, ndx] = 0.0 if c_sense[cname, ndx] == 'e': e = expr - c_rhs[cname, ndx] == 0 elif c_sense[cname, ndx] == 'l': e = expr - c_rhs[cname, ndx] <= 0 else: e = expr - c_rhs[cname, ndx] >= 0 c = Constraint(expr=e) if ndx is None: c_name = cname elif type(ndx) is tuple: c_name = "%s[%s]" % (cname, ','.join(map(str, ndx))) else: c_name = "%s[%s]" % (cname, str(ndx)) setattr(dual, c_name, c) # for (name, ndx), domain in iteritems(v_domain): v = getvar(name, ndx) flag = type(ndx) is tuple and (ndx[-1] == 'lb' or ndx[-1] == 'ub') if domain == 1: v.domain = NonNegativeReals elif domain == -1: v.domain = NonPositiveReals else: # TODO: verify that this case is possible v.domain = Reals return dual
def solve(self, model, **kwds): """Solve the model. Warning: this solver is still in beta. Keyword arguments subject to change. Undocumented keyword arguments definitely subject to change. This function performs all of the GDPopt solver setup and problem validation. It then calls upon helper functions to construct the initial master approximation and iteration loop. Args: model (Block): a Pyomo model or block to be solved """ config = self.CONFIG(kwds.pop('options', {})) config.set_value(kwds) solve_data = GDPoptSolveData() created_GDPopt_block = False old_logger_level = config.logger.getEffectiveLevel() try: if config.tee and old_logger_level > logging.INFO: # If the logger does not already include INFO, include it. config.logger.setLevel(logging.INFO) config.logger.info("---Starting GDPopt---") # Create a model block on which to store GDPopt-specific utility # modeling objects. if hasattr(model, 'GDPopt_utils'): raise RuntimeError( "GDPopt needs to create a Block named GDPopt_utils " "on the model object, but an attribute with that name " "already exists.") else: created_GDPopt_block = True model.GDPopt_utils = Block( doc="Container for GDPopt solver utility modeling objects") solve_data.original_model = model solve_data.working_model = clone_orig_model_with_lists(model) GDPopt = solve_data.working_model.GDPopt_utils record_original_model_statistics(solve_data, config) solve_data.current_strategy = config.strategy # Reformulate integer variables to binary reformulate_integer_variables(solve_data.working_model, config) # Save ordered lists of main modeling components, so that data can # be easily transferred between future model clones. build_ordered_component_lists(solve_data.working_model) record_working_model_statistics(solve_data, config) solve_data.results.solver.name = 'GDPopt ' + str(self.version()) # Save model initial values. These are used later to initialize NLP # subproblems. GDPopt.initial_var_values = list(v.value for v in GDPopt.working_var_list) # Store the initial model state as the best solution found. If we # find no better solution, then we will restore from this copy. solve_data.best_solution_found = list(GDPopt.initial_var_values) # Validate the model to ensure that GDPopt is able to solve it. if not model_is_valid(solve_data, config): return # Maps in order to keep track of certain generated constraints GDPopt.oa_cut_map = ComponentMap() # Integer cuts exclude particular discrete decisions GDPopt.integer_cuts = ConstraintList(doc='integer cuts') # Feasible integer cuts exclude discrete realizations that have # been explored via an NLP subproblem. Depending on model # characteristics, the user may wish to revisit NLP subproblems # (with a different initialization, for example). Therefore, these # cuts are not enabled by default, unless the initial model has no # discrete decisions. # Note: these cuts will only exclude integer realizations that are # not already in the primary GDPopt_integer_cuts ConstraintList. GDPopt.no_backtracking = ConstraintList( doc='explored integer cuts') # Set up iteration counters solve_data.master_iteration = 0 solve_data.mip_iteration = 0 solve_data.nlp_iteration = 0 # set up bounds solve_data.LB = float('-inf') solve_data.UB = float('inf') solve_data.iteration_log = {} # Flag indicating whether the solution improved in the past # iteration or not solve_data.feasible_solution_improved = False # Initialize the master problem self._GDPopt_initialize_master(solve_data, config) # Algorithm main loop self._GDPopt_iteration_loop(solve_data, config) # Update values in working model copy_var_list_values(from_list=solve_data.best_solution_found, to_list=GDPopt.working_var_list, config=config) GDPopt.objective_value.set_value( value(solve_data.working_objective_expr)) # Update values in original model copy_var_list_values( GDPopt.orig_var_list, solve_data.original_model.GDPopt_utils.orig_var_list, config) solve_data.results.problem.lower_bound = solve_data.LB solve_data.results.problem.upper_bound = solve_data.UB finally: config.logger.setLevel(old_logger_level) if created_GDPopt_block: model.del_component('GDPopt_utils')
def _create_using(self, model, **kwds): precision = kwds.pop('precision', 8) user_discretize = kwds.pop('discretize', set()) verbose = kwds.pop('verbose', False) M = model.clone() # TODO: if discretize is not empty, we must translate those # components over to the components on the cloned instance _discretize = {} if user_discretize: for _var in user_discretize: _v = M.find_component(_var.name) if _v.component() is _v: for _vv in _v.itervalues(): _discretize.setdefault(id(_vv), len(_discretize)) else: _discretize.setdefault(id(_v), len(_discretize)) # Iterate over all Constraints and identify the bilinear and # quadratic terms bilinear_terms = [] quadratic_terms = [] for constraint in M.component_map(Constraint, active=True).itervalues(): for cname, c in constraint._data.iteritems(): if c.body.polynomial_degree() != 2: continue self._collect_bilinear(c.body, bilinear_terms, quadratic_terms) # We want to find the (minimum?) number of variables to # discretize so that we cover all the bilinearities -- without # discretizing both sides of any single bilinear expression. # First step: figure out how many expressions each term appears # in _counts = {} for q in quadratic_terms: if not q[1].is_continuous(): continue _id = id(q[1]) if _id not in _counts: _counts[_id] = (q[1], set()) _counts[_id][1].add(_id) for bi in bilinear_terms: for i in (0, 1): if not bi[i + 1].is_continuous(): continue _id = id(bi[i + 1]) if _id not in _counts: _counts[_id] = (bi[i + 1], set()) _counts[_id][1].add(id(bi[2 - i])) _tmp_counts = dict(_counts) # First, remove the variables that the user wants to have discretized for _id in _discretize: for _i in _tmp_counts[_id][1]: if _i == _id: continue _tmp_counts[_i][1].remove(_id) del _tmp_counts[_id] # All quadratic terms must be discretized (?) #for q in quadratic_terms: # _id = id(q[1]) # if _id not in _tmp_counts: # continue # _discretize.setdefault(_id, len(_discretize)) # for _i in _tmp_counts[_id][1]: # if _i == _id: # continue # _tmp_counts[_i][1].remove(_id) # del _tmp_counts[_id] # Now pick a (minimal) subset of the terms in bilinear expressions while _tmp_counts: _ct, _id = max((len(_tmp_counts[i][1]), i) for i in _tmp_counts) if not _ct: break _discretize.setdefault(_id, len(_discretize)) for _i in list(_tmp_counts[_id][1]): if _i == _id: continue _tmp_counts[_i][1].remove(_id) del _tmp_counts[_id] # # Discretize things # # Define a block (namespace) for holding the disaggregated # variables and new constraints if False: # Set to true when the LP writer is fixed M._radix_linearization = Block() _block = M._radix_linearization else: _block = M _block.DISCRETIZATION = RangeSet(precision) _block.DISCRETIZED_VARIABLES = RangeSet(0, len(_discretize) - 1) _block.z = Var(_block.DISCRETIZED_VARIABLES, _block.DISCRETIZATION, within=Binary) _block.dv = Var(_block.DISCRETIZED_VARIABLES, bounds=(0, 2**-precision)) # Actually discretize the terms we have marked for discretization for _id, _idx in iteritems(_discretize): if verbose: logger.info("Discretizing variable %s as %s" % (_counts[_id][0].name, _idx)) self._discretize_variable(_block, _counts[_id][0], _idx) _known_bilinear = {} # For each quadratic term, if it hasn't been discretized / # generated, do so, and remember the resulting W term for later # use... #for _expr, _x1 in quadratic_terms: # self._discretize_term( _expr, _x1, _x1, # _block, _discretize, _known_bilinear ) # For each bilinear term, if it hasn't been discretized / # generated, do so, and remember the resulting W term for later # use... for _expr, _x1, _x2 in bilinear_terms: self._discretize_term(_expr, _x1, _x2, _block, _discretize, _known_bilinear) # Return the discretized instance! return M
def _dualize(self, block, unfixed=[]): """ Generate the dual of a block """ # # Collect linear terms from the block # A, b_coef, c_rhs, c_sense, d_sense, vnames, cnames, v_domain = collect_linear_terms( block, unfixed) # # Construct the block # if isinstance(block, Model): dual = ConcreteModel() else: dual = Block() for v, is_indexed in vnames: if is_indexed: setattr(dual, v + '_Index', Set(dimen=None)) setattr(dual, v, Var(getattr(dual, v + '_Index'))) else: setattr(dual, v, Var()) for cname, is_indexed in cnames: if is_indexed: setattr(dual, cname + '_Index', Set(dimen=None)) setattr(dual, cname, Constraint(getattr(dual, cname + '_Index'))) setattr(dual, cname + '_lower_', Var(getattr(dual, cname + '_Index'))) setattr(dual, cname + '_upper_', Var(getattr(dual, cname + '_Index'))) else: setattr(dual, cname, Constraint()) setattr(dual, cname + '_lower_', Var()) setattr(dual, cname + '_upper_', Var()) dual.construct() # # Construct the objective # if d_sense == minimize: dual.o = Objective(expr=sum(-b_coef[name, ndx] * getattr(dual, name)[ndx] for name, ndx in b_coef), sense=d_sense) else: dual.o = Objective(expr=sum(b_coef[name, ndx] * getattr(dual, name)[ndx] for name, ndx in b_coef), sense=d_sense) # # Construct the constraints # for cname in A: c = getattr(dual, cname) c_index = getattr(dual, cname + "_Index") if c.is_indexed() else None for ndx, terms in iteritems(A[cname]): if not c_index is None and not ndx in c_index: c_index.add(ndx) expr = 0 for term in terms: v = getattr(dual, term.var) if not term.ndx in v: v.add(term.ndx) expr += term.coef * v[term.ndx] if not (cname, ndx) in c_rhs: c_rhs[cname, ndx] = 0.0 if c_sense[cname, ndx] == 'e': c.add(ndx, expr - c_rhs[cname, ndx] == 0) elif c_sense[cname, ndx] == 'l': c.add(ndx, expr - c_rhs[cname, ndx] <= 0) else: c.add(ndx, expr - c_rhs[cname, ndx] >= 0) for (name, ndx), domain in iteritems(v_domain): v = getattr(dual, name) flag = type(ndx) is tuple and (ndx[-1] == 'lb' or ndx[-1] == 'ub') if domain == 1: if flag: v[ndx].domain = NonNegativeReals else: v.domain = NonNegativeReals elif domain == -1: if flag: v[ndx].domain = NonPositiveReals else: v.domain = NonPositiveReals else: if flag: # TODO: verify that this case is possible v[ndx].domain = Reals else: v.domain = Reals return dual