def _print_model_LP(self, model, output_file, solver_capability, labeler, output_fixed_variable_bounds=False, file_determinism=1, row_order=None, column_order=None, skip_trivial_constraints=False, force_objective_constant=False, include_all_variable_bounds=False): symbol_map = SymbolMap() variable_symbol_map = SymbolMap() # NOTE: we use createSymbol instead of getSymbol because we # know whether or not the symbol exists, and don't want # to the overhead of error/duplicate checking. # cache frequently called functions create_symbol_func = SymbolMap.createSymbol create_symbols_func = SymbolMap.createSymbols alias_symbol_func = SymbolMap.alias variable_label_pairs = [] # populate the symbol map in a single pass. #objective_list, constraint_list, sosconstraint_list, variable_list \ # = self._populate_symbol_map(model, # symbol_map, # labeler, # variable_symbol_map, # file_determinism=file_determinism) sortOrder = SortComponents.unsorted if file_determinism >= 1: sortOrder = sortOrder | SortComponents.indices if file_determinism >= 2: sortOrder = sortOrder | SortComponents.alphabetical # # Create variable symbols (and cache the block list) # all_blocks = [] variable_list = [] for block in model.block_data_objects(active=True, sort=sortOrder): all_blocks.append(block) for vardata in block.component_data_objects(Var, active=True, sort=sortOrder, descend_into=False): variable_list.append(vardata) variable_label_pairs.append( (vardata, create_symbol_func(symbol_map, vardata, labeler))) # # WEH - TODO: See if this is faster # #all_blocks = list( model.block_data_objects( # active=True, sort=sortOrder) ) #variable_list = list( model.component_data_objects( # Var, sort=sortOrder) ) #variable_label_pairs = list( # (vardata, create_symbol_func(symbol_map, vardata, labeler)) # for vardata in variable_list ) variable_symbol_map.addSymbols(variable_label_pairs) # and extract the information we'll need for rapid labeling. object_symbol_dictionary = symbol_map.byObject variable_symbol_dictionary = variable_symbol_map.byObject # cache - these are called all the time. print_expr_canonical = self._print_expr_canonical # print the model name and the source, so we know roughly where # it came from. # # NOTE: this *must* use the "\* ... *\" comment format: the GLPK # LP parser does not correctly handle other formats (notably, "%"). output_file.write("\\* Source Pyomo model name=%s *\\\n\n" % (model.name, )) # # Objective # supports_quadratic_objective = solver_capability('quadratic_objective') numObj = 0 onames = [] for block in all_blocks: gen_obj_repn = getattr(block, "_gen_obj_repn", True) # Get/Create the ComponentMap for the repn if not hasattr(block, '_repn'): block._repn = ComponentMap() block_repn = block._repn for objective_data in block.component_data_objects( Objective, active=True, sort=sortOrder, descend_into=False): numObj += 1 onames.append(objective_data.name) if numObj > 1: raise ValueError( "More than one active objective defined for input " "model '%s'; Cannot write legal LP file\n" "Objectives: %s" % (model.name, ' '.join(onames))) create_symbol_func(symbol_map, objective_data, labeler) symbol_map.alias(objective_data, '__default_objective__') if objective_data.is_minimizing(): output_file.write("min \n") else: output_file.write("max \n") if gen_obj_repn: repn = generate_standard_repn(objective_data.expr) block_repn[objective_data] = repn else: repn = block_repn[objective_data] degree = repn.polynomial_degree() if degree == 0: logger.warning( "Constant objective detected, replacing " "with a placeholder to prevent solver failure.") force_objective_constant = True elif degree == 2: if not supports_quadratic_objective: raise RuntimeError( "Selected solver is unable to handle " "objective functions with quadratic terms. " "Objective at issue: %s." % objective_data.name) elif degree is None: raise RuntimeError( "Cannot write legal LP file. Objective '%s' " "has nonlinear terms that are not quadratic." % objective_data.name) output_file.write( object_symbol_dictionary[id(objective_data)] + ':\n') offset = print_expr_canonical( repn, output_file, object_symbol_dictionary, variable_symbol_dictionary, True, column_order, force_objective_constant=force_objective_constant) if numObj == 0: raise ValueError("ERROR: No objectives defined for input model. " "Cannot write legal LP file.") # Constraints # # If there are no non-trivial constraints, you'll end up with an empty # constraint block. CPLEX is OK with this, but GLPK isn't. And # eliminating the constraint block (i.e., the "s.t." line) causes GLPK # to whine elsewhere. Output a warning if the constraint block is empty, # so users can quickly determine the cause of the solve failure. output_file.write("\n") output_file.write("s.t.\n") output_file.write("\n") have_nontrivial = False supports_quadratic_constraint = solver_capability( 'quadratic_constraint') def constraint_generator(): for block in all_blocks: gen_con_repn = getattr(block, "_gen_con_repn", True) # Get/Create the ComponentMap for the repn if not hasattr(block, '_repn'): block._repn = ComponentMap() block_repn = block._repn for constraint_data in block.component_data_objects( Constraint, active=True, sort=sortOrder, descend_into=False): if (not constraint_data.has_lb()) and \ (not constraint_data.has_ub()): assert not constraint_data.equality continue # non-binding, so skip if constraint_data._linear_canonical_form: repn = constraint_data.canonical_form() elif gen_con_repn: repn = generate_standard_repn(constraint_data.body) block_repn[constraint_data] = repn else: repn = block_repn[constraint_data] yield constraint_data, repn if row_order is not None: sorted_constraint_list = list(constraint_generator()) sorted_constraint_list.sort(key=lambda x: row_order[x[0]]) def yield_all_constraints(): for data, repn in sorted_constraint_list: yield data, repn else: yield_all_constraints = constraint_generator # FIXME: This is a hack to get nested blocks working... eq_string_template = "= %" + self._precision_string + '\n' geq_string_template = ">= %" + self._precision_string + '\n\n' leq_string_template = "<= %" + self._precision_string + '\n\n' for constraint_data, repn in yield_all_constraints(): have_nontrivial = True degree = repn.polynomial_degree() # # Write constraint # # There are conditions, e.g., when fixing variables, under which # a constraint block might be empty. Ignore these, for both # practical reasons and the fact that the CPLEX LP format # requires a variable in the constraint body. It is also # possible that the body of the constraint consists of only a # constant, in which case the "variable" of if degree == 0: if skip_trivial_constraints: continue elif degree == 2: if not supports_quadratic_constraint: raise ValueError( "Solver unable to handle quadratic expressions. Constraint" " at issue: '%s'" % (constraint_data.name)) elif degree is None: raise ValueError( "Cannot write legal LP file. Constraint '%s' has a body " "with nonlinear terms." % (constraint_data.name)) # Create symbol con_symbol = create_symbol_func(symbol_map, constraint_data, labeler) if constraint_data.equality: assert value(constraint_data.lower) == \ value(constraint_data.upper) label = 'c_e_' + con_symbol + '_' alias_symbol_func(symbol_map, constraint_data, label) output_file.write(label + ':\n') offset = print_expr_canonical(repn, output_file, object_symbol_dictionary, variable_symbol_dictionary, False, column_order) bound = constraint_data.lower bound = _get_bound(bound) - offset output_file.write(eq_string_template % (_no_negative_zero(bound))) output_file.write("\n") else: if constraint_data.has_lb(): if constraint_data.has_ub(): label = 'r_l_' + con_symbol + '_' else: label = 'c_l_' + con_symbol + '_' alias_symbol_func(symbol_map, constraint_data, label) output_file.write(label + ':\n') offset = print_expr_canonical(repn, output_file, object_symbol_dictionary, variable_symbol_dictionary, False, column_order) bound = constraint_data.lower bound = _get_bound(bound) - offset output_file.write(geq_string_template % (_no_negative_zero(bound))) else: assert constraint_data.has_ub() if constraint_data.has_ub(): if constraint_data.has_lb(): label = 'r_u_' + con_symbol + '_' else: label = 'c_u_' + con_symbol + '_' alias_symbol_func(symbol_map, constraint_data, label) output_file.write(label + ':\n') offset = print_expr_canonical(repn, output_file, object_symbol_dictionary, variable_symbol_dictionary, False, column_order) bound = constraint_data.upper bound = _get_bound(bound) - offset output_file.write(leq_string_template % (_no_negative_zero(bound))) else: assert constraint_data.has_lb() if not have_nontrivial: logger.warning('Empty constraint block written in LP format ' \ '- solver may error') # the CPLEX LP format doesn't allow constants in the objective (or # constraint body), which is a bit silly. To avoid painful # book-keeping, we introduce the following "variable", constrained # to the value 1. This is used when quadratic terms are present. # worst-case, if not used, is that CPLEX easily pre-processes it out. prefix = "" output_file.write('%sc_e_ONE_VAR_CONSTANT: \n' % prefix) output_file.write('%sONE_VAR_CONSTANT = 1.0\n' % prefix) output_file.write("\n") # SOS constraints # # For now, we write out SOS1 and SOS2 constraints in the cplex format # # All Component objects are stored in model._component, which is a # dictionary of {class: {objName: object}}. # # Consider the variable X, # # model.X = Var(...) # # We print X to CPLEX format as X(i,j,k,...) where i, j, k, ... are the # indices of X. # SOSlines = StringIO() sos1 = solver_capability("sos1") sos2 = solver_capability("sos2") writtenSOS = False for block in all_blocks: for soscondata in block.component_data_objects(SOSConstraint, active=True, sort=sortOrder, descend_into=False): create_symbol_func(symbol_map, soscondata, labeler) level = soscondata.level if (level == 1 and not sos1) or \ (level == 2 and not sos2) or \ (level > 2): raise ValueError( "Solver does not support SOS level %s constraints" % (level)) if writtenSOS == False: SOSlines.write("SOS\n") writtenSOS = True # This updates the referenced_variable_ids, just in case # there is a variable that only appears in an # SOSConstraint, in which case this needs to be known # before we write the "bounds" section (Cplex does not # handle this correctly, Gurobi does) self.printSOS(symbol_map, labeler, variable_symbol_map, soscondata, SOSlines) # # Bounds # output_file.write("bounds\n") # Scan all variables even if we're only writing a subset of them. # required because we don't store maps by variable type currently. # FIXME: This is a hack to get nested blocks working... lb_string_template = "%" + self._precision_string + " <= " ub_string_template = " <= %" + self._precision_string + "\n" # Track the number of integer and binary variables, so you can # output their status later. integer_vars = [] binary_vars = [] for vardata in variable_list: # TODO: We could just loop over the set of items in # self._referenced_variable_ids, except this is # a dictionary that is hashed by id(vardata) # which would make the bounds section # nondeterministic (bad for unit testing) if (not include_all_variable_bounds) and \ (id(vardata) not in self._referenced_variable_ids): continue if vardata.fixed: if not output_fixed_variable_bounds: raise ValueError( "Encountered a fixed variable (%s) inside an active " "objective or constraint expression on model %s, which is " "usually indicative of a preprocessing error. Use the " "IO-option 'output_fixed_variable_bounds=True' to suppress " "this error and fix the variable by overwriting its bounds " "in the LP file." % (vardata.name, model.name)) if vardata.value is None: raise ValueError( "Variable cannot be fixed to a value of None.") vardata_lb = value(vardata.value) vardata_ub = value(vardata.value) else: vardata_lb = _get_bound(vardata.lb) vardata_ub = _get_bound(vardata.ub) name_to_output = variable_symbol_dictionary[id(vardata)] # track the number of integer and binary variables, so we know whether # to output the general / binary sections below. if vardata.is_binary(): binary_vars.append(name_to_output) elif vardata.is_integer(): integer_vars.append(name_to_output) elif not vardata.is_continuous(): raise TypeError( "Invalid domain type for variable with name '%s'. " "Variable is not continuous, integer, or binary." % (vardata.name)) # in the CPLEX LP file format, the default variable # bounds are 0 and +inf. These bounds are in # conflict with Pyomo, which assumes -inf and +inf # (which we would argue is more rational). output_file.write(" ") if vardata.has_lb(): output_file.write(lb_string_template % (_no_negative_zero(vardata_lb))) else: output_file.write(" -inf <= ") if name_to_output == "e": raise ValueError( "Attempting to write variable with name 'e' in a CPLEX LP " "formatted file will cause a parse failure due to confusion with " "numeric values expressed in scientific notation") output_file.write(name_to_output) if vardata.has_ub(): output_file.write(ub_string_template % (_no_negative_zero(vardata_ub))) else: output_file.write(" <= +inf\n") if len(integer_vars) > 0: output_file.write("general\n") for var_name in integer_vars: output_file.write(' %s\n' % var_name) if len(binary_vars) > 0: output_file.write("binary\n") for var_name in binary_vars: output_file.write(' %s\n' % var_name) # Write the SOS section output_file.write(SOSlines.getvalue()) # # wrap-up # output_file.write("end\n") # Clean up the symbol map to only contain variables referenced # in the active constraints **Note**: warm start method may # rely on this for choosing the set of potential warm start # variables vars_to_delete = set(variable_symbol_map.byObject.keys()) - \ set(self._referenced_variable_ids.keys()) sm_byObject = symbol_map.byObject sm_bySymbol = symbol_map.bySymbol var_sm_byObject = variable_symbol_map.byObject for varid in vars_to_delete: symbol = var_sm_byObject[varid] del sm_byObject[varid] del sm_bySymbol[symbol] del variable_symbol_map return symbol_map
class CPLEXPersistent(CPLEXDirect, PersistentSolver): """The CPLEX LP/MIP solver """ pyomo.util.plugin.alias('_cplex_persistent', doc='Persistent Python interface to the CPLEX LP/MIP solver') def __init__(self, **kwds): # # Call base class constructor # kwds['type'] = 'cplexpersistent' CPLEXDirect.__init__(self, **kwds) # maps pyomo var data labels to the corresponding CPLEX variable id. self._cplex_variable_ids = {} self._cplex_variable_names = None # # updates all variable bounds in the compiled model - handles # fixed variables and related issues. re-does everything from # scratch by default, ignoring whatever was specified # previously. if the value associated with the keyword # vars_to_update is a non-empty list (assumed to be variable name # / index pairs), then only the bounds for those variables are # updated. this function assumes that the variables themselves # already exist in the compiled model. # def compile_variable_bounds(self, pyomo_instance, vars_to_update): from pyomo.core.base import Var if self._active_cplex_instance is None: raise RuntimeError("***The CPLEXPersistent solver plugin " "cannot compile variable bounds - no " "instance is presently compiled") # the bound update entries should be name-value pairs new_lower_bounds = [] new_upper_bounds = [] # operates through side effects on the above lists! def update_bounds_lists(var_name): var_lb = None var_ub = None if var_data.fixed and self._output_fixed_variable_bounds: var_lb = var_ub = var_data.value elif var_data.fixed: # if we've been directed to not deal with fixed # variables, then skip - they should have been # compiled out of any description of the constraints return else: if not var_data.has_lb(): var_lb = -CPLEXDirect._cplex_module.infinity else: var_lb = value(var_data.lb) if not var_data.has_ub(): var_ub = CPLEXDirect._cplex_module.infinity else: var_ub= value(var_data.ub) var_cplex_id = self._cplex_variable_ids[var_name] new_lower_bounds.append((var_cplex_id, var_lb)) new_upper_bounds.append((var_cplex_id, var_ub)) if len(vars_to_update) == 0: for var_data in pyomo_instance.component_data_objects(Var, active=True): var_name = self._symbol_map.getSymbol(var_data, self._labeler) update_bounds_lists(var_name) else: for var_name, var_index in vars_to_update: var = pyomo_instance.find_component(var_name) # TBD - do some error checking! var_data = var[var_index] var_name = self._symbol_map.getSymbol(var_data, self._labeler) update_bounds_lists(var_name) self._active_cplex_instance.variables.set_lower_bounds(new_lower_bounds) self._active_cplex_instance.variables.set_upper_bounds(new_upper_bounds) # # method to compile objective of the input pyomo instance. # TBD: # it may be smarter just to track the associated pyomo instance, # and re-compile it automatically from a cached local attribute. # this would ensure consistency, among other things! # def compile_objective(self, pyomo_instance): from pyomo.core.base import Objective from pyomo.repn import canonical_is_constant, LinearCanonicalRepn, canonical_degree if self._active_cplex_instance is None: raise RuntimeError("***The CPLEXPersistent solver plugin " "cannot compile objective - no " "instance is presently compiled") cplex_instance = self._active_cplex_instance self._has_quadratic_objective = False cntr = 0 for block in pyomo_instance.block_data_objects(active=True): gen_obj_canonical_repn = \ getattr(block, "_gen_obj_canonical_repn", True) # Get/Create the ComponentMap for the repn if not hasattr(block,'_canonical_repn'): block._canonical_repn = ComponentMap() block_canonical_repn = block._canonical_repn for obj_data in block.component_data_objects(Objective, active=True, descend_into=False): cntr += 1 if cntr > 1: raise ValueError( "Multiple active objectives found on Pyomo instance '%s'. " "Solver '%s' will only handle a single active objective" \ % (pyomo_instance.name, self.type)) if obj_data.is_minimizing(): cplex_instance.objective.set_sense( cplex_instance.objective.sense.minimize) else: cplex_instance.objective.set_sense( cplex_instance.objective.sense.maximize) cplex_instance.objective.set_name( self._symbol_map.getSymbol(obj_data, self._labeler)) if gen_obj_canonical_repn: obj_repn = generate_canonical_repn(obj_data.expr) block_canonical_repn[obj_data] = obj_repn else: obj_repn = block_canonical_repn[obj_data] if (isinstance(obj_repn, LinearCanonicalRepn) and \ ((obj_repn.linear == None) or \ (len(obj_repn.linear) == 0))) or \ canonical_is_constant(obj_repn): print("Warning: Constant objective detected, replacing " "with a placeholder to prevent solver failure.") offset = obj_repn.constant if offset is None: offset = 0.0 objective_expression = [("ONE_VAR_CONSTANT",offset)] cplex_instance.objective.set_linear(objective_expression) else: if isinstance(obj_repn, LinearCanonicalRepn): objective_expression, offset = \ self._encode_constraint_body_linear_specialized( obj_repn, self._labeler, use_variable_names=False, cplex_variable_name_index_map=self._cplex_variable_ids, as_pairs=True) if offset != 0.0: objective_expression.append((self._cplex_variable_ids["ONE_VAR_CONSTANT"],offset)) cplex_instance.objective.set_linear(objective_expression) else: #Linear terms if 1 in obj_repn: objective_expression, offset = \ self._encode_constraint_body_linear( obj_repn, self._labeler, as_pairs=True) if offset != 0.0: objective_expression.append(("ONE_VAR_CONSTANT",offset)) cplex_instance.objective.set_linear(objective_expression) #Quadratic terms if 2 in obj_repn: self._has_quadratic_objective = True objective_expression = \ self._encode_constraint_body_quadratic(obj_repn, self._labeler, as_triples=True, is_obj=2.0) cplex_instance.objective.\ set_quadratic_coefficients(objective_expression) degree = canonical_degree(obj_repn) if (degree is None) or (degree > 2): raise ValueError( "CPLEXPersistent plugin does not support general nonlinear " "objective expressions (only linear or quadratic).\n" "Objective: %s" % (obj_data.name)) # # method to populate the CPLEX problem instance (interface) from # the supplied Pyomo problem instance. # def compile_instance(self, pyomo_instance, symbolic_solver_labels=False, output_fixed_variable_bounds=False, skip_trivial_constraints=False): from pyomo.core.base import Var, Constraint, SOSConstraint from pyomo.repn import canonical_is_constant, LinearCanonicalRepn, canonical_degree self._symbolic_solver_labels = symbolic_solver_labels self._output_fixed_variable_bounds = output_fixed_variable_bounds self._skip_trivial_constraints = skip_trivial_constraints self._has_quadratic_constraints = False self._has_quadratic_objective = False self._active_cplex_instance = CPLEXDirect._cplex_module.Cplex() if self._symbolic_solver_labels: labeler = self._labeler = TextLabeler() else: labeler = self._labeler = NumericLabeler('x') self._symbol_map = SymbolMap() self._instance = pyomo_instance if isinstance(pyomo_instance, IBlockStorage): # BIG HACK if not hasattr(pyomo_instance, "._symbol_maps"): setattr(pyomo_instance, "._symbol_maps", {}) getattr(pyomo_instance, "._symbol_maps")[id(self._symbol_map)] = \ self._symbol_map else: pyomo_instance.solutions.add_symbol_map(self._symbol_map) self._smap_id = id(self._symbol_map) # we use this when iterating over the constraints because it # will have a much smaller hash table, we also use this for # the warm start code after it is cleaned to only contain # variables referenced in the constraints self._variable_symbol_map = SymbolMap() # cplex wants the caller to set the problem type, which is (for # current purposes) strictly based on variable type counts. self._num_binary_variables = 0 self._num_integer_variables = 0 self._num_continuous_variables = 0 self._used_sos_constraints = False ############################################# # populate the variables in the cplex model # ############################################# var_names = [] var_lbs = [] var_ubs = [] var_types = [] self._referenced_variable_ids.clear() # maps pyomo var data labels to the corresponding CPLEX variable id. self._cplex_variable_ids.clear() # cached in the loop below - used to update the symbol map # immediately following loop termination. var_label_pairs = [] for var_data in pyomo_instance.component_data_objects(Var, active=True): if var_data.fixed and not self._output_fixed_variable_bounds: # if a variable is fixed, and we're preprocessing # fixed variables (as in not outputting them), there # is no need to add them to the compiled model. continue var_name = self._symbol_map.getSymbol(var_data, labeler) var_names.append(var_name) var_label_pairs.append((var_data, var_name)) self._cplex_variable_ids[var_name] = len(self._cplex_variable_ids) if not var_data.has_lb(): var_lbs.append(-CPLEXDirect._cplex_module.infinity) else: var_lbs.append(value(var_data.lb)) if not var_data.has_ub(): var_ubs.append(CPLEXDirect._cplex_module.infinity) else: var_ubs.append(value(var_data.ub)) if var_data.is_integer(): var_types.append(self._active_cplex_instance.variables.type.integer) self._num_integer_variables += 1 elif var_data.is_binary(): var_types.append(self._active_cplex_instance.variables.type.binary) self._num_binary_variables += 1 elif var_data.is_continuous(): var_types.append(self._active_cplex_instance.variables.type.continuous) self._num_continuous_variables += 1 else: raise TypeError("Invalid domain type for variable with name '%s'. " "Variable is not continuous, integer, or binary.") self._active_cplex_instance.variables.add(names=var_names, lb=var_lbs, ub=var_ubs, types=var_types) self._active_cplex_instance.variables.add(lb=[1], ub=[1], names=["ONE_VAR_CONSTANT"]) self._cplex_variable_ids["ONE_VAR_CONSTANT"] = len(self._cplex_variable_ids) self._variable_symbol_map.addSymbols(var_label_pairs) self._cplex_variable_names = self._active_cplex_instance.variables.get_names() ######################################################## # populate the standard constraints in the cplex model # ######################################################## expressions = [] senses = [] rhss = [] range_values = [] names = [] qexpressions = [] qlinears = [] qsenses = [] qrhss = [] qnames = [] for block in pyomo_instance.block_data_objects(active=True): gen_con_canonical_repn = \ getattr(block, "_gen_con_canonical_repn", True) # Get/Create the ComponentMap for the repn if not hasattr(block,'_canonical_repn'): block._canonical_repn = ComponentMap() block_canonical_repn = block._canonical_repn for con in block.component_data_objects(Constraint, active=True, descend_into=False): if (not con.has_lb()) and \ (not con.has_ub()): assert not con.equality continue # not binding at all, don't bother con_repn = None if con._linear_canonical_form: con_repn = con.canonical_form() elif isinstance(con, LinearCanonicalRepn): con_repn = con else: if gen_con_canonical_repn: con_repn = generate_canonical_repn(con.body) block_canonical_repn[con] = con_repn else: con_repn = block_canonical_repn[con] # There are conditions, e.g., when fixing variables, under which # a constraint block might be empty. Ignore these, for both # practical reasons and the fact that the CPLEX LP format # requires a variable in the constraint body. It is also # possible that the body of the constraint consists of only a # constant, in which case the "variable" of if isinstance(con_repn, LinearCanonicalRepn): if self._skip_trivial_constraints and \ ((con_repn.linear is None) or \ (len(con_repn.linear) == 0)): continue else: # we shouldn't come across a constant canonical repn # that is not LinearCanonicalRepn assert not canonical_is_constant(con_repn) name = self._symbol_map.getSymbol(con, labeler) expr = None qexpr = None quadratic = False if isinstance(con_repn, LinearCanonicalRepn): expr, offset = \ self._encode_constraint_body_linear_specialized(con_repn, labeler, use_variable_names=False, cplex_variable_name_index_map=self._cplex_variable_ids) else: degree = canonical_degree(con_repn) if degree == 2: quadratic = True elif (degree != 0) or (degree != 1): raise ValueError( "CPLEXPersistent plugin does not support general nonlinear " "constraint expression (only linear or quadratic).\n" "Constraint: %s" % (con.name)) expr, offset = self._encode_constraint_body_linear(con_repn, labeler) if quadratic: if expr is None: expr = CPLEXDirect._cplex_module.SparsePair(ind=[0],val=[0.0]) self._has_quadratic_constraints = True qexpr = self._encode_constraint_body_quadratic(con_repn,labeler) qnames.append(name) if con.equality: # equality constraint. qsenses.append('E') qrhss.append(self._get_bound(con.lower) - offset) elif con.has_lb() and con.has_ub(): raise RuntimeError( "The CPLEXDirect plugin can not translate range " "constraints containing quadratic expressions.") elif con.has_lb(): assert not con.has_ub() qsenses.append('G') qrhss.append(self._get_bound(con.lower) - offset) else: assert con.has_ub() qsenses.append('L') qrhss.append(self._get_bound(con.upper) - offset) qlinears.append(expr) qexpressions.append(qexpr) else: names.append(name) expressions.append(expr) if con.equality: # equality constraint. senses.append('E') rhss.append(self._get_bound(con.lower) - offset) range_values.append(0.0) elif con.has_lb() and con.has_ub(): # ranged constraint. senses.append('R') lower_bound = self._get_bound(con.lower) - offset upper_bound = self._get_bound(con.upper) - offset rhss.append(lower_bound) range_values.append(upper_bound - lower_bound) elif con.has_lb(): senses.append('G') rhss.append(self._get_bound(con.lower) - offset) range_values.append(0.0) else: assert con.has_ub() senses.append('L') rhss.append(self._get_bound(con.upper) - offset) range_values.append(0.0) ################################################### # populate the SOS constraints in the cplex model # ################################################### # SOS constraints - largely taken from cpxlp.py so updates there, # should be applied here # TODO: Allow users to specify the variables coefficients for custom # branching/set orders - refer to cpxlp.py sosn = self._capabilities.sosn sos1 = self._capabilities.sos1 sos2 = self._capabilities.sos2 modelSOS = ModelSOS() for soscondata in pyomo_instance.component_data_objects(SOSConstraint, active=True): level = soscondata.level if (level == 1 and not sos1) or \ (level == 2 and not sos2) or \ (level > 2 and not sosn): raise Exception("Solver does not support SOS level %s constraints" % (level,)) modelSOS.count_constraint(self._symbol_map, labeler, self._variable_symbol_map, soscondata) if modelSOS.sosType: for key in modelSOS.sosType: self._active_cplex_instance.SOS.add(type = modelSOS.sosType[key], name = modelSOS.sosName[key], SOS = [modelSOS.varnames[key], modelSOS.weights[key]]) self._referenced_variable_ids.update(modelSOS.varids[key]) self._used_sos_constraints = True self._active_cplex_instance.linear_constraints.add( lin_expr=expressions, senses=senses, rhs=rhss, range_values=range_values, names=names) for index in xrange(len(qexpressions)): self._active_cplex_instance.quadratic_constraints.add( lin_expr=qlinears[index], quad_expr=qexpressions[index], sense=qsenses[index], rhs=qrhss[index], name=qnames[index]) ############################################# # populate the objective in the cplex model # ############################################# self.compile_objective(pyomo_instance) # # simple method to query whether a Pyomo instance has already been # compiled. # def instance_compiled(self): return self._active_cplex_instance is not None # # Override base class method to check for compiled instance # def _warm_start(self, instance): if self._active_cplex_instance is None: raise RuntimeError("***The CPLEXPersistent solver plugin " "cannot warm start - no instance is " "presently compiled") # clear any existing warm starts. self._active_cplex_instance.MIP_starts.delete() # the iteration order is identical to that used in generating # the cplex instance, so all should be well. variable_ids = [] variable_values = [] # IMPT: the var_data returned is a weak ref! for label, var_data in iteritems(self._variable_symbol_map.bySymbol): cplex_id = self._cplex_variable_ids[label] if var_data().fixed and not self._output_fixed_variable_bounds: continue elif var_data().value is not None: variable_ids.append(cplex_id) variable_values.append(var_data().value) if len(variable_ids): self._active_cplex_instance.MIP_starts.add( [variable_ids, variable_values], self._active_cplex_instance.MIP_starts.effort_level.auto) # # Override base class method to check for compiled instance # def _populate_cplex_instance(self, model): assert model == self._instance def _presolve(self, *args, **kwds): if self._active_cplex_instance is None: raise RuntimeError("***The CPLEXPersistent solver plugin" " cannot presolve - no instance is " "presently compiled") # These must be passed in to the compile_instance method, # but assert that any values here match those already supplied if 'symbolic_solver_labels' in kwds: assert self._symbolic_solver_labels == \ kwds['symbolic_solver_labels'] if 'output_fixed_variable_bounds' in kwds: assert self._output_fixed_variable_bounds == \ kwds['output_fixed_variable_bounds'] if 'skip_trivial_constraints' in kwds: assert self._skip_trivial_constraints == \ kwds["skip_trivial_constraints"] if isinstance(self._instance, IBlockStorage): # BIG HACK if not hasattr(self._instance, "._symbol_maps"): setattr(self._instance, "._symbol_maps", {}) getattr(self._instance, "._symbol_maps")[id(self._symbol_map)] = \ self._symbol_map else: if self._smap_id not in self._instance.solutions.symbol_map: self._instance.solutions.add_symbol_map(self._symbol_map) ################################################ # populate the problem type in the cplex model # ################################################ # This gets rid of the annoying "Freeing MIP data." message. def _filter_freeing_mip_data(val): if val.strip() == 'Freeing MIP data.': return "" return val self._active_cplex_instance.set_warning_stream(sys.stderr, fn=_filter_freeing_mip_data) if (self._has_quadratic_objective is True) or \ (self._has_quadratic_constraints is True): if (self._num_integer_variables > 0) or \ (self._num_binary_variables > 0) or \ (self._used_sos_constraints): if self._has_quadratic_constraints is True: self._active_cplex_instance.set_problem_type( self._active_cplex_instance.problem_type.MIQCP) else: self._active_cplex_instance.set_problem_type( self._active_cplex_instance.problem_type.MIQP) else: if self._has_quadratic_constraints is True: self._active_cplex_instance.set_problem_type( self._active_cplex_instance.problem_type.QCP) else: self._active_cplex_instance.set_problem_type( self._active_cplex_instance.problem_type.QP) elif (self._num_integer_variables > 0) or \ (self._num_binary_variables > 0) or \ (self._used_sos_constraints): self._active_cplex_instance.set_problem_type( self._active_cplex_instance.problem_type.MILP) else: self._active_cplex_instance.set_problem_type( self._active_cplex_instance.problem_type.LP) # restore the warning stream without our filter function self._active_cplex_instance.set_warning_stream(sys.stderr) CPLEXDirect._presolve(self, *args, **kwds) # like other solver plugins, persistent solver plugins can # take an instance as an input argument. the only context in # which this instance is used, however, is for warm-starting. if len(args) > 2: raise ValueError("The CPLEXPersistent plugin method " "'_presolve' can be supplied at most " "one problem instance - %s were " "supplied" % len(args)) # Re-add the symbol map id if it was cleared # after a previous solution load if id(self._symbol_map) not in args[0].solutions.symbol_map: args[0].solutions.add_symbol_map(self._symbol_map) self._smap_id = id(self._symbol_map) # # invoke the solver on the currently compiled instance!!! # def _apply_solver(self): if self._active_cplex_instance is None: raise RuntimeError("***The CPLEXPersistent solver plugin cannot " "apply solver - no instance is presently compiled") # NOTE: # CPLEX maintains the pool of feasible solutions from the # prior solve as the set of mip starts for the next solve. # and evaluating multiple mip starts (and there can be many) # is expensive. so if the warm_start method is not invoked, # there will potentially be a lot of time wasted. return CPLEXDirect._apply_solver(self) def _postsolve(self): if self._active_cplex_instance is None: raise RuntimeError("***The CPLEXPersistent solver plugin " "cannot postsolve - no instance is " "presently compiled") active_cplex_instance = self._active_cplex_instance variable_symbol_map = self._variable_symbol_map instance = self._instance ret = CPLEXDirect._postsolve(self) # # These get reset to None by the base class method # self._active_cplex_instance = active_cplex_instance self._variable_symbol_map = variable_symbol_map self._instance = instance return ret
def _print_model_MPS(self, model, output_file, solver_capability, labeler, output_fixed_variable_bounds=False, file_determinism=1, row_order=None, column_order=None, skip_trivial_constraints=False, force_objective_constant=False, include_all_variable_bounds=False, skip_objective_sense=False): symbol_map = SymbolMap() variable_symbol_map = SymbolMap() # NOTE: we use createSymbol instead of getSymbol because we # know whether or not the symbol exists, and don't want # to the overhead of error/duplicate checking. # cache frequently called functions extract_variable_coefficients = self._extract_variable_coefficients create_symbol_func = SymbolMap.createSymbol create_symbols_func = SymbolMap.createSymbols alias_symbol_func = SymbolMap.alias variable_label_pairs = [] sortOrder = SortComponents.unsorted if file_determinism >= 1: sortOrder = sortOrder | SortComponents.indices if file_determinism >= 2: sortOrder = sortOrder | SortComponents.alphabetical # # Create variable symbols (and cache the block list) # all_blocks = [] variable_list = [] for block in model.block_data_objects(active=True, sort=sortOrder): all_blocks.append(block) for vardata in block.component_data_objects(Var, active=True, sort=sortOrder, descend_into=False): variable_list.append(vardata) variable_label_pairs.append( (vardata, create_symbol_func(symbol_map, vardata, labeler))) variable_symbol_map.addSymbols(variable_label_pairs) # and extract the information we'll need for rapid labeling. object_symbol_dictionary = symbol_map.byObject variable_symbol_dictionary = variable_symbol_map.byObject # sort the variable ordering by the user # column_order ComponentMap if column_order is not None: variable_list.sort(key=lambda _x: column_order[_x]) # prepare to hold the sparse columns variable_to_column = ComponentMap( (vardata, i) for i, vardata in enumerate(variable_list)) # add one position for ONE_VAR_CONSTANT column_data = [[] for i in xrange(len(variable_list) + 1)] quadobj_data = [] quadmatrix_data = [] # constraint rhs rhs_data = [] # print the model name and the source, so we know # roughly where output_file.write("* Source: Pyomo MPS Writer\n") output_file.write("* Format: Free MPS\n") output_file.write("*\n") output_file.write("NAME %s\n" % (model.name, )) # # ROWS section # objective_label = None numObj = 0 onames = [] for block in all_blocks: gen_obj_repn = \ getattr(block, "_gen_obj_repn", True) # Get/Create the ComponentMap for the repn if not hasattr(block, '_repn'): block._repn = ComponentMap() block_repn = block._repn for objective_data in block.component_data_objects( Objective, active=True, sort=sortOrder, descend_into=False): numObj += 1 onames.append(objective_data.name) if numObj > 1: raise ValueError( "More than one active objective defined for input " "model '%s'; Cannot write legal MPS file\n" "Objectives: %s" % (model.name, ' '.join(onames))) objective_label = create_symbol_func(symbol_map, objective_data, labeler) symbol_map.alias(objective_data, '__default_objective__') if not skip_objective_sense: output_file.write("OBJSENSE\n") if objective_data.is_minimizing(): output_file.write(" MIN\n") else: output_file.write(" MAX\n") # This section is not recognized by the COIN-OR # MPS reader #output_file.write("OBJNAME\n") #output_file.write(" %s\n" % (objective_label)) output_file.write("ROWS\n") output_file.write(" N %s\n" % (objective_label)) if gen_obj_repn: repn = \ generate_standard_repn(objective_data.expr) block_repn[objective_data] = repn else: repn = block_repn[objective_data] degree = repn.polynomial_degree() if degree == 0: logger.warning( "Constant objective detected, replacing " "with a placeholder to prevent solver failure.") force_objective_constant = True elif degree is None: raise RuntimeError( "Cannot write legal MPS file. Objective '%s' " "has nonlinear terms that are not quadratic." % objective_data.name) constant = extract_variable_coefficients( objective_label, repn, column_data, quadobj_data, variable_to_column) if force_objective_constant or (constant != 0.0): # ONE_VAR_CONSTANT column_data[-1].append((objective_label, constant)) if numObj == 0: raise ValueError( "Cannot write legal MPS file: No objective defined " "for input model '%s'." % str(model)) assert objective_label is not None # Constraints def constraint_generator(): for block in all_blocks: gen_con_repn = \ getattr(block, "_gen_con_repn", True) # Get/Create the ComponentMap for the repn if not hasattr(block, '_repn'): block._repn = ComponentMap() block_repn = block._repn for constraint_data in block.component_data_objects( Constraint, active=True, sort=sortOrder, descend_into=False): if (not constraint_data.has_lb()) and \ (not constraint_data.has_ub()): assert not constraint_data.equality continue # non-binding, so skip if constraint_data._linear_canonical_form: repn = constraint_data.canonical_form() elif gen_con_repn: repn = generate_standard_repn(constraint_data.body) block_repn[constraint_data] = repn else: repn = block_repn[constraint_data] yield constraint_data, repn if row_order is not None: sorted_constraint_list = list(constraint_generator()) sorted_constraint_list.sort(key=lambda x: row_order[x[0]]) def yield_all_constraints(): for constraint_data, repn in sorted_constraint_list: yield constraint_data, repn else: yield_all_constraints = constraint_generator for constraint_data, repn in yield_all_constraints(): degree = repn.polynomial_degree() # Write constraint if degree == 0: if skip_trivial_constraints: continue elif degree is None: raise RuntimeError( "Cannot write legal MPS file. Constraint '%s' " "has nonlinear terms that are not quadratic." % constraint_data.name) # Create symbol con_symbol = create_symbol_func(symbol_map, constraint_data, labeler) if constraint_data.equality: assert value(constraint_data.lower) == \ value(constraint_data.upper) label = 'c_e_' + con_symbol + '_' alias_symbol_func(symbol_map, constraint_data, label) output_file.write(" E %s\n" % (label)) offset = extract_variable_coefficients(label, repn, column_data, quadmatrix_data, variable_to_column) bound = constraint_data.lower bound = _get_bound(bound) - offset rhs_data.append((label, _no_negative_zero(bound))) else: if constraint_data.has_lb(): if constraint_data.has_ub(): label = 'r_l_' + con_symbol + '_' else: label = 'c_l_' + con_symbol + '_' alias_symbol_func(symbol_map, constraint_data, label) output_file.write(" G %s\n" % (label)) offset = extract_variable_coefficients( label, repn, column_data, quadmatrix_data, variable_to_column) bound = constraint_data.lower bound = _get_bound(bound) - offset rhs_data.append((label, _no_negative_zero(bound))) else: assert constraint_data.has_ub() if constraint_data.has_ub(): if constraint_data.has_lb(): label = 'r_u_' + con_symbol + '_' else: label = 'c_u_' + con_symbol + '_' alias_symbol_func(symbol_map, constraint_data, label) output_file.write(" L %s\n" % (label)) offset = extract_variable_coefficients( label, repn, column_data, quadmatrix_data, variable_to_column) bound = constraint_data.upper bound = _get_bound(bound) - offset rhs_data.append((label, _no_negative_zero(bound))) else: assert constraint_data.has_lb() if len(column_data[-1]) > 0: # ONE_VAR_CONSTANT = 1 output_file.write(" E c_e_ONE_VAR_CONSTANT\n") column_data[-1].append(("c_e_ONE_VAR_CONSTANT", 1)) rhs_data.append(("c_e_ONE_VAR_CONSTANT", 1)) # # COLUMNS section # column_template = " %s %s %" + self._precision_string + "\n" output_file.write("COLUMNS\n") cnt = 0 for vardata in variable_list: col_entries = column_data[variable_to_column[vardata]] cnt += 1 if len(col_entries) > 0: var_label = variable_symbol_dictionary[id(vardata)] for i, (row_label, coef) in enumerate(col_entries): output_file.write( column_template % (var_label, row_label, _no_negative_zero(coef))) elif include_all_variable_bounds: # the column is empty, so add a (0 * var) # term to the objective # * Note that some solvers (e.g., Gurobi) # will accept an empty column as a line # with just the column name. This doesn't # seem to work for CPLEX 12.6, so I am # doing it this way so that it will work for both var_label = variable_symbol_dictionary[id(vardata)] output_file.write(column_template % (var_label, objective_label, 0)) assert cnt == len(column_data) - 1 if len(column_data[-1]) > 0: col_entries = column_data[-1] var_label = "ONE_VAR_CONSTANT" for i, (row_label, coef) in enumerate(col_entries): output_file.write( column_template % (var_label, row_label, _no_negative_zero(coef))) # # RHS section # rhs_template = " RHS %s %" + self._precision_string + "\n" output_file.write("RHS\n") for i, (row_label, rhs) in enumerate(rhs_data): # note: we have already converted any -0 to 0 by this point output_file.write(rhs_template % (row_label, rhs)) # SOS constraints SOSlines = StringIO() sos1 = solver_capability("sos1") sos2 = solver_capability("sos2") for block in all_blocks: for soscondata in block.component_data_objects(SOSConstraint, active=True, sort=sortOrder, descend_into=False): create_symbol_func(symbol_map, soscondata, labeler) level = soscondata.level if (level == 1 and not sos1) or \ (level == 2 and not sos2) or \ (level > 2): raise ValueError( "Solver does not support SOS level %s constraints" % (level)) # This updates the referenced_variable_ids, just in case # there is a variable that only appears in an # SOSConstraint, in which case this needs to be known # before we write the "bounds" section (Cplex does not # handle this correctly, Gurobi does) self._printSOS(symbol_map, labeler, variable_symbol_map, soscondata, SOSlines) # # BOUNDS section # entry_template = "%s %" + self._precision_string + "\n" output_file.write("BOUNDS\n") for vardata in variable_list: if include_all_variable_bounds or \ (id(vardata) in self._referenced_variable_ids): var_label = variable_symbol_dictionary[id(vardata)] if vardata.fixed: if not output_fixed_variable_bounds: raise ValueError( "Encountered a fixed variable (%s) inside an active " "objective or constraint expression on model %s, which is " "usually indicative of a preprocessing error. Use the " "IO-option 'output_fixed_variable_bounds=True' to suppress " "this error and fix the variable by overwriting its bounds " "in the MPS file." % (vardata.name, model.name)) if vardata.value is None: raise ValueError( "Variable cannot be fixed to a value of None.") output_file.write( (" FX BOUND " + entry_template) % (var_label, _no_negative_zero(value(vardata.value)))) continue # convert any -0 to 0 to make baseline diffing easier vardata_lb = _no_negative_zero(_get_bound(vardata.lb)) vardata_ub = _no_negative_zero(_get_bound(vardata.ub)) unbounded_lb = not vardata.has_lb() unbounded_ub = not vardata.has_ub() treat_as_integer = False if vardata.is_binary(): if (vardata_lb == 0) and (vardata_ub == 1): output_file.write(" BV BOUND %s\n" % (var_label)) continue else: # so we can add bounds treat_as_integer = True if treat_as_integer or vardata.is_integer(): # Indicating unbounded integers is tricky because # the only way to indicate a variable is integer # is using the bounds section. Thus, we signify # infinity with a large number (10E20) # * Note: Gurobi allows values like inf and -inf # but CPLEX 12.6 does not, so I am just # using a large value if not unbounded_lb: output_file.write((" LI BOUND " + entry_template) % (var_label, vardata_lb)) else: output_file.write(" LI BOUND %s -10E20\n" % (var_label)) if not unbounded_ub: output_file.write((" UI BOUND " + entry_template) % (var_label, vardata_ub)) else: output_file.write(" UI BOUND %s 10E20\n" % (var_label)) else: assert vardata.is_continuous() if unbounded_lb and unbounded_ub: output_file.write(" FR BOUND %s\n" % (var_label)) else: if not unbounded_lb: output_file.write((" LO BOUND " + entry_template) % (var_label, vardata_lb)) else: output_file.write(" MI BOUND %s\n" % (var_label)) if not unbounded_ub: output_file.write((" UP BOUND " + entry_template) % (var_label, vardata_ub)) # # SOS section # output_file.write(SOSlines.getvalue()) # Formatting of the next two sections comes from looking # at Gurobi and Cplex output # # QUADOBJ section # if len(quadobj_data) > 0: assert len(quadobj_data) == 1 # it looks like the COIN-OR MPS Reader only # recognizes QUADOBJ (Gurobi and Cplex seem to # be okay with this) output_file.write("QUADOBJ\n") #output_file.write("QMATRIX\n") label, quad_terms = quadobj_data[0] assert label == objective_label # sort by the sorted tuple of symbols (or column assignments) # for the variables appearing in the term quad_terms = sorted(quad_terms, key=lambda _x: \ sorted((variable_to_column[_x[0][0]], variable_to_column[_x[0][1]]))) for term, coef in quad_terms: # sort the term for consistent output var1, var2 = sorted(term, key=lambda _x: variable_to_column[_x]) var1_label = variable_symbol_dictionary[id(var1)] var2_label = variable_symbol_dictionary[id(var2)] # Don't forget that a quadratic objective is always # assumed to be divided by 2 if var1_label == var2_label: output_file.write( column_template % (var1_label, var2_label, _no_negative_zero(coef * 2))) else: # the matrix needs to be symmetric so split # the coefficient (but remember it is divided by 2) output_file.write( column_template % (var1_label, var2_label, _no_negative_zero(coef))) output_file.write( column_template % (var2_label, var1_label, _no_negative_zero(coef))) # # QCMATRIX section # if len(quadmatrix_data) > 0: for row_label, quad_terms in quadmatrix_data: output_file.write("QCMATRIX %s\n" % (row_label)) # sort by the sorted tuple of symbols (or # column assignments) for the variables # appearing in the term quad_terms = sorted(quad_terms, key=lambda _x: \ sorted((variable_to_column[_x[0][0]], variable_to_column[_x[0][1]]))) for term, coef in quad_terms: # sort the term for consistent output var1, var2 = sorted(term, key=lambda _x: variable_to_column[_x]) var1_label = variable_symbol_dictionary[id(var1)] var2_label = variable_symbol_dictionary[id(var2)] if var1_label == var2_label: output_file.write( column_template % (var1_label, var2_label, _no_negative_zero(coef))) else: # the matrix needs to be symmetric so split # the coefficient output_file.write(column_template % (var1_label, var2_label, _no_negative_zero(coef * 0.5))) output_file.write(column_template % (var2_label, var1_label, coef * 0.5)) output_file.write("ENDATA\n") # Clean up the symbol map to only contain variables referenced # in the active constraints **Note**: warm start method may # rely on this for choosing the set of potential warm start # variables vars_to_delete = set(variable_symbol_map.byObject.keys()) - \ set(self._referenced_variable_ids.keys()) sm_byObject = symbol_map.byObject sm_bySymbol = symbol_map.bySymbol var_sm_byObject = variable_symbol_map.byObject for varid in vars_to_delete: symbol = var_sm_byObject[varid] del sm_byObject[varid] del sm_bySymbol[symbol] del variable_symbol_map return symbol_map
def _print_model_MPS(self, model, output_file, solver_capability, labeler, output_fixed_variable_bounds=False, file_determinism=1, row_order=None, column_order=None, skip_trivial_constraints=False, force_objective_constant=False, include_all_variable_bounds=False, skip_objective_sense=False): symbol_map = SymbolMap() variable_symbol_map = SymbolMap() # NOTE: we use createSymbol instead of getSymbol because we # know whether or not the symbol exists, and don't want # to the overhead of error/duplicate checking. # cache frequently called functions extract_variable_coefficients = self._extract_variable_coefficients create_symbol_func = SymbolMap.createSymbol create_symbols_func = SymbolMap.createSymbols alias_symbol_func = SymbolMap.alias variable_label_pairs = [] sortOrder = SortComponents.unsorted if file_determinism >= 1: sortOrder = sortOrder | SortComponents.indices if file_determinism >= 2: sortOrder = sortOrder | SortComponents.alphabetical # # Create variable symbols (and cache the block list) # all_blocks = [] variable_list = [] for block in model.block_data_objects(active=True, sort=sortOrder): all_blocks.append(block) for vardata in block.component_data_objects( Var, active=True, sort=sortOrder, descend_into=False): variable_list.append(vardata) variable_label_pairs.append( (vardata,create_symbol_func(symbol_map, vardata, labeler))) variable_symbol_map.addSymbols(variable_label_pairs) # and extract the information we'll need for rapid labeling. object_symbol_dictionary = symbol_map.byObject variable_symbol_dictionary = variable_symbol_map.byObject # sort the variable ordering by the user # column_order ComponentMap if column_order is not None: variable_list.sort(key=lambda _x: column_order[_x]) # prepare to hold the sparse columns variable_to_column = ComponentMap( (vardata, i) for i, vardata in enumerate(variable_list)) # add one position for ONE_VAR_CONSTANT column_data = [[] for i in xrange(len(variable_list)+1)] quadobj_data = [] quadmatrix_data = [] # constraint rhs rhs_data = [] # print the model name and the source, so we know # roughly where output_file.write("* Source: Pyomo MPS Writer\n") output_file.write("* Format: Free MPS\n") output_file.write("*\n") output_file.write("NAME %s\n" % (model.name,)) # # ROWS section # objective_label = None numObj = 0 onames = [] for block in all_blocks: gen_obj_canonical_repn = \ getattr(block, "_gen_obj_canonical_repn", True) # Get/Create the ComponentMap for the repn if not hasattr(block,'_canonical_repn'): block._canonical_repn = ComponentMap() block_canonical_repn = block._canonical_repn for objective_data in block.component_data_objects( Objective, active=True, sort=sortOrder, descend_into=False): numObj += 1 onames.append(objective_data.cname()) if numObj > 1: raise ValueError( "More than one active objective defined for input " "model '%s'; Cannot write legal MPS file\n" "Objectives: %s" % (model.cname(True), ' '.join(onames))) objective_label = create_symbol_func(symbol_map, objective_data, labeler) symbol_map.alias(objective_data, '__default_objective__') if not skip_objective_sense: output_file.write("OBJSENSE\n") if objective_data.is_minimizing(): output_file.write(" MIN\n") else: output_file.write(" MAX\n") # This section is not recognized by the COIN-OR # MPS reader #output_file.write("OBJNAME\n") #output_file.write(" %s\n" % (objective_label)) output_file.write("ROWS\n") output_file.write(" N %s\n" % (objective_label)) if gen_obj_canonical_repn: canonical_repn = \ generate_canonical_repn(objective_data.expr) block_canonical_repn[objective_data] = canonical_repn else: canonical_repn = block_canonical_repn[objective_data] degree = canonical_degree(canonical_repn) if degree == 0: print("Warning: Constant objective detected, replacing " "with a placeholder to prevent solver failure.") force_objective_constant = True elif (degree != 1) and (degree != 2): raise RuntimeError( "Cannot write legal MPS file. Objective '%s' " "has nonlinear terms that are not quadratic." % objective_data.cname(True)) constant = extract_variable_coefficients( objective_label, canonical_repn, column_data, quadobj_data, variable_to_column) if force_objective_constant or (constant != 0.0): # ONE_VAR_CONSTANT column_data[-1].append((objective_label, constant)) if numObj == 0: raise ValueError( "Cannot write legal MPS file: No objective defined " "for input model '%s'." % str(model)) assert objective_label is not None # Constraints def constraint_generator(): for block in all_blocks: gen_con_canonical_repn = \ getattr(block, "_gen_con_canonical_repn", True) # Get/Create the ComponentMap for the repn if not hasattr(block,'_canonical_repn'): block._canonical_repn = ComponentMap() block_canonical_repn = block._canonical_repn for constraint_data in block.component_data_objects( Constraint, active=True, sort=sortOrder, descend_into=False): if isinstance(constraint_data, LinearCanonicalRepn): canonical_repn = constraint_data else: if gen_con_canonical_repn: canonical_repn = generate_canonical_repn( constraint_data.body) block_canonical_repn[constraint_data] = canonical_repn else: canonical_repn = block_canonical_repn[constraint_data] yield constraint_data, canonical_repn if row_order is not None: sorted_constraint_list = list(constraint_generator()) sorted_constraint_list.sort(key=lambda x: row_order[x[0]]) def yield_all_constraints(): for constraint_data, canonical_repn in sorted_constraint_list: yield constraint_data, canonical_repn else: yield_all_constraints = constraint_generator for constraint_data, canonical_repn in yield_all_constraints(): degree = canonical_degree(canonical_repn) # Write constraint if degree == 0: if skip_trivial_constraints: continue elif (degree != 1) and (degree != 2): raise RuntimeError( "Cannot write legal MPS file. Constraint '%s' " "has nonlinear terms that are not quadratic." % constraint_data.cname(True)) # Create symbol con_symbol = create_symbol_func(symbol_map, constraint_data, labeler) if constraint_data.equality: label = 'c_e_' + con_symbol + '_' alias_symbol_func(symbol_map, constraint_data, label) output_file.write(" E %s\n" % (label)) offset = extract_variable_coefficients( label, canonical_repn, column_data, quadmatrix_data, variable_to_column) bound = constraint_data.lower bound = self._get_bound(bound) - offset rhs_data.append((label, bound)) else: if constraint_data.lower is not None: if constraint_data.upper is not None: label = 'r_l_' + con_symbol + '_' else: label = 'c_l_' + con_symbol + '_' alias_symbol_func(symbol_map, constraint_data, label) output_file.write(" G %s\n" % (label)) offset = extract_variable_coefficients( label, canonical_repn, column_data, quadmatrix_data, variable_to_column) bound = constraint_data.lower bound = self._get_bound(bound) - offset rhs_data.append((label, bound)) if constraint_data.upper is not None: if constraint_data.lower is not None: label = 'r_u_' + con_symbol + '_' else: label = 'c_u_' + con_symbol + '_' alias_symbol_func(symbol_map, constraint_data, label) output_file.write(" L %s\n" % (label)) offset = extract_variable_coefficients( label, canonical_repn, column_data, quadmatrix_data, variable_to_column) bound = constraint_data.upper bound = self._get_bound(bound) - offset rhs_data.append((label, bound)) if len(column_data[-1]) > 0: # ONE_VAR_CONSTANT = 1 output_file.write(" E c_e_ONE_VAR_CONSTANT\n") column_data[-1].append(("c_e_ONE_VAR_CONSTANT",1)) rhs_data.append(("c_e_ONE_VAR_CONSTANT",1)) # # COLUMNS section # column_template = " %s %s %"+self._precision_string+"\n" output_file.write("COLUMNS\n") cnt = 0 for vardata in variable_list: col_entries = column_data[variable_to_column[vardata]] cnt += 1 if len(col_entries) > 0: var_label = variable_symbol_dictionary[id(vardata)] for i, (row_label, coef) in enumerate(col_entries): output_file.write(column_template % (var_label, row_label, coef)) elif include_all_variable_bounds: # the column is empty, so add a (0 * var) # term to the objective # * Note that some solvers (e.g., Gurobi) # will accept an empty column as a line # with just the column name. This doesn't # seem to work for CPLEX 12.6, so I am # doing it this way so that it will work for both var_label = variable_symbol_dictionary[id(vardata)] output_file.write(column_template % (var_label, objective_label, 0)) assert cnt == len(column_data)-1 if len(column_data[-1]) > 0: col_entries = column_data[-1] var_label = "ONE_VAR_CONSTANT" for i, (row_label, coef) in enumerate(col_entries): output_file.write(column_template % (var_label, row_label, coef)) # # RHS section # rhs_template = " RHS %s %"+self._precision_string+"\n" output_file.write("RHS\n") for i, (row_label, rhs) in enumerate(rhs_data): output_file.write(rhs_template % (row_label, rhs)) # SOS constraints SOSlines = StringIO() sos1 = solver_capability("sos1") sos2 = solver_capability("sos2") for block in all_blocks: for soscondata in block.component_data_objects( SOSConstraint, active=True, sort=sortOrder, descend_into=False): create_symbol_func(symbol_map, soscondata, labeler) level = soscondata.level if (level == 1 and not sos1) or \ (level == 2 and not sos2) or \ (level > 2): raise ValueError( "Solver does not support SOS level %s constraints" % (level)) # This updates the referenced_variable_ids, just in case # there is a variable that only appears in an # SOSConstraint, in which case this needs to be known # before we write the "bounds" section (Cplex does not # handle this correctly, Gurobi does) self._printSOS(symbol_map, labeler, variable_symbol_map, soscondata, SOSlines) # # BOUNDS section # entry_template = "%s %"+self._precision_string+"\n" output_file.write("BOUNDS\n") for vardata in variable_list: if include_all_variable_bounds or \ (id(vardata) in self._referenced_variable_ids): var_label = variable_symbol_dictionary[id(vardata)] if vardata.fixed: if not output_fixed_variable_bounds: raise ValueError( "Encountered a fixed variable (%s) inside an active " "objective or constraint expression on model %s, which is " "usually indicative of a preprocessing error. Use the " "IO-option 'output_fixed_variable_bounds=True' to suppress " "this error and fix the variable by overwriting its bounds " "in the MPS file." % (vardata.cname(True), model.cname(True))) if vardata.value is None: raise ValueError("Variable cannot be fixed to a value of None.") output_file.write((" FX BOUND "+entry_template) % (var_label, value(vardata.value))) continue vardata_lb = self._get_bound(vardata.lb) vardata_ub = self._get_bound(vardata.ub) # Make it harder for -0 to show up in # the output. This makes file diffing # for test baselines slightly less # annoying if vardata_lb == 0: vardata_lb = 0 if vardata_ub == 0: vardata_ub = 0 unbounded_lb = (vardata_lb is None) or (vardata_lb == -infinity) unbounded_ub = (vardata_ub is None) or (vardata_ub == infinity) treat_as_integer = False if vardata.is_binary(): if (vardata_lb == 0) and (vardata_ub == 1): output_file.write(" BV BOUND %s\n" % (var_label)) continue else: # so we can add bounds treat_as_integer = True if treat_as_integer or vardata.is_integer(): # Indicating unbounded integers is tricky because # the only way to indicate a variable is integer # is using the bounds section. Thus, we signify # infinity with a large number (10E20) # * Note: Gurobi allows values like inf and -inf # but CPLEX 12.6 does not, so I am just # using a large value if not unbounded_lb: output_file.write((" LI BOUND "+entry_template) % (var_label, vardata_lb)) else: output_file.write(" LI BOUND %s -10E20\n" % (var_label)) if not unbounded_ub: output_file.write((" UI BOUND "+entry_template) % (var_label, vardata_ub)) else: output_file.write(" UI BOUND %s 10E20\n" % (var_label)) else: assert vardata.is_continuous() if unbounded_lb and unbounded_ub: output_file.write(" FR BOUND %s\n" % (var_label)) else: if not unbounded_lb: output_file.write((" LO BOUND "+entry_template) % (var_label, vardata_lb)) else: output_file.write(" MI BOUND %s\n" % (var_label)) if not unbounded_ub: output_file.write((" UP BOUND "+entry_template) % (var_label, vardata_ub)) # # SOS section # output_file.write(SOSlines.getvalue()) # Formatting of the next two sections comes from looking # at Gurobi and Cplex output # # QUADOBJ section # if len(quadobj_data) > 0: assert len(quadobj_data) == 1 # it looks like the COIN-OR MPS Reader only # recognizes QUADOBJ (Gurobi and Cplex seem to # be okay with this) output_file.write("QUADOBJ\n") #output_file.write("QMATRIX\n") label, quad_terms = quadobj_data[0] assert label == objective_label for (var1, var2), coef in sorted(quad_terms, key=lambda _x: (variable_to_column[_x[0][0]], variable_to_column[_x[0][1]])): var1_label = variable_symbol_dictionary[id(var1)] var2_label = variable_symbol_dictionary[id(var2)] # Don't forget that a quadratic objective is always # assumed to be divided by 2 if var1_label == var2_label: output_file.write(column_template % (var1_label, var2_label, coef * 2)) else: # the matrix needs to be symmetric so split # the coefficient (but remember it is divided by 2) output_file.write(column_template % (var1_label, var2_label, coef)) output_file.write(column_template % (var2_label, var1_label, coef)) # # QCMATRIX section # if len(quadmatrix_data) > 0: for row_label, quad_terms in quadmatrix_data: output_file.write("QCMATRIX %s\n" % (row_label)) for (var1, var2), coef in sorted(quad_terms, key=lambda _x: (variable_to_column[_x[0][0]], variable_to_column[_x[0][1]])): var1_label = variable_symbol_dictionary[id(var1)] var2_label = variable_symbol_dictionary[id(var2)] if var1_label == var2_label: output_file.write(column_template % (var1_label, var2_label, coef)) else: # the matrix needs to be symmetric so split # the coefficient output_file.write(column_template % (var1_label, var2_label, coef * 0.5)) output_file.write(column_template % (var2_label, var1_label, coef * 0.5)) output_file.write("ENDATA\n") # Clean up the symbol map to only contain variables referenced # in the active constraints **Note**: warm start method may # rely on this for choosing the set of potential warm start # variables vars_to_delete = set(variable_symbol_map.byObject.keys()) - \ set(self._referenced_variable_ids.keys()) sm_byObject = symbol_map.byObject sm_bySymbol = symbol_map.bySymbol var_sm_byObject = variable_symbol_map.byObject for varid in vars_to_delete: symbol = var_sm_byObject[varid] del sm_byObject[varid] del sm_bySymbol[symbol] del variable_symbol_map return symbol_map
def _print_model_LP(self, model, output_file, solver_capability, labeler, output_fixed_variable_bounds=False, file_determinism=1, row_order=None, column_order=None, skip_trivial_constraints=False, force_objective_constant=False, include_all_variable_bounds=False): symbol_map = SymbolMap() variable_symbol_map = SymbolMap() # NOTE: we use createSymbol instead of getSymbol because we # know whether or not the symbol exists, and don't want # to the overhead of error/duplicate checking. # cache frequently called functions create_symbol_func = SymbolMap.createSymbol create_symbols_func = SymbolMap.createSymbols alias_symbol_func = SymbolMap.alias variable_label_pairs = [] # populate the symbol map in a single pass. #objective_list, constraint_list, sosconstraint_list, variable_list \ # = self._populate_symbol_map(model, # symbol_map, # labeler, # variable_symbol_map, # file_determinism=file_determinism) sortOrder = SortComponents.unsorted if file_determinism >= 1: sortOrder = sortOrder | SortComponents.indices if file_determinism >= 2: sortOrder = sortOrder | SortComponents.alphabetical # # Create variable symbols (and cache the block list) # all_blocks = [] variable_list = [] for block in model.block_data_objects(active=True, sort=sortOrder): all_blocks.append(block) for vardata in block.component_data_objects( Var, active=True, sort=sortOrder, descend_into=False): variable_list.append(vardata) variable_label_pairs.append( (vardata,create_symbol_func(symbol_map, vardata, labeler))) variable_symbol_map.addSymbols(variable_label_pairs) # and extract the information we'll need for rapid labeling. object_symbol_dictionary = symbol_map.byObject variable_symbol_dictionary = variable_symbol_map.byObject # cache - these are called all the time. print_expr_canonical = self._print_expr_canonical # print the model name and the source, so we know roughly where # it came from. # # NOTE: this *must* use the "\* ... *\" comment format: the GLPK # LP parser does not correctly handle other formats (notably, "%"). output_file.write( "\\* Source Pyomo model name=%s *\\\n\n" % (model.name,) ) # # Objective # supports_quadratic_objective = \ solver_capability('quadratic_objective') numObj = 0 onames = [] for block in all_blocks: gen_obj_canonical_repn = \ getattr(block, "_gen_obj_canonical_repn", True) # Get/Create the ComponentMap for the repn if not hasattr(block,'_canonical_repn'): block._canonical_repn = ComponentMap() block_canonical_repn = block._canonical_repn for objective_data in block.component_data_objects( Objective, active=True, sort=sortOrder, descend_into=False): numObj += 1 onames.append(objective_data.name) if numObj > 1: raise ValueError( "More than one active objective defined for input " "model '%s'; Cannot write legal LP file\n" "Objectives: %s" % (model.name, ' '.join(onames))) create_symbol_func(symbol_map, objective_data, labeler) symbol_map.alias(objective_data, '__default_objective__') if objective_data.is_minimizing(): output_file.write("min \n") else: output_file.write("max \n") if gen_obj_canonical_repn: canonical_repn = \ generate_canonical_repn(objective_data.expr) block_canonical_repn[objective_data] = canonical_repn else: canonical_repn = block_canonical_repn[objective_data] degree = canonical_degree(canonical_repn) if degree == 0: logger.warning("Constant objective detected, replacing " "with a placeholder to prevent solver failure.") force_objective_constant = True elif degree == 2: if not supports_quadratic_objective: raise RuntimeError( "Selected solver is unable to handle " "objective functions with quadratic terms. " "Objective at issue: %s." % objective_data.name) elif degree != 1: raise RuntimeError( "Cannot write legal LP file. Objective '%s' " "has nonlinear terms that are not quadratic." % objective_data.name) output_file.write( object_symbol_dictionary[id(objective_data)]+':\n') offset = print_expr_canonical( canonical_repn, output_file, object_symbol_dictionary, variable_symbol_dictionary, True, column_order, force_objective_constant=force_objective_constant) if numObj == 0: raise ValueError( "ERROR: No objectives defined for input model '%s'; " " cannot write legal LP file" % str(model.name)) # Constraints # # If there are no non-trivial constraints, you'll end up with an empty # constraint block. CPLEX is OK with this, but GLPK isn't. And # eliminating the constraint block (i.e., the "s.t." line) causes GLPK # to whine elsewhere. Output a warning if the constraint block is empty, # so users can quickly determine the cause of the solve failure. output_file.write("\n") output_file.write("s.t.\n") output_file.write("\n") have_nontrivial = False supports_quadratic_constraint = solver_capability('quadratic_constraint') def constraint_generator(): for block in all_blocks: gen_con_canonical_repn = \ getattr(block, "_gen_con_canonical_repn", True) # Get/Create the ComponentMap for the repn if not hasattr(block,'_canonical_repn'): block._canonical_repn = ComponentMap() block_canonical_repn = block._canonical_repn for constraint_data in block.component_data_objects( Constraint, active=True, sort=sortOrder, descend_into=False): if isinstance(constraint_data, LinearCanonicalRepn): canonical_repn = constraint_data else: if gen_con_canonical_repn: canonical_repn = generate_canonical_repn(constraint_data.body) block_canonical_repn[constraint_data] = canonical_repn else: canonical_repn = block_canonical_repn[constraint_data] yield constraint_data, canonical_repn if row_order is not None: sorted_constraint_list = list(constraint_generator()) sorted_constraint_list.sort(key=lambda x: row_order[x[0]]) def yield_all_constraints(): for constraint_data, canonical_repn in sorted_constraint_list: yield constraint_data, canonical_repn else: yield_all_constraints = constraint_generator # FIXME: This is a hack to get nested blocks working... eq_string_template = "= %"+self._precision_string+'\n' geq_string_template = ">= %"+self._precision_string+'\n\n' leq_string_template = "<= %"+self._precision_string+'\n\n' for constraint_data, canonical_repn in yield_all_constraints(): have_nontrivial = True degree = canonical_degree(canonical_repn) # # Write constraint # # There are conditions, e.g., when fixing variables, under which # a constraint block might be empty. Ignore these, for both # practical reasons and the fact that the CPLEX LP format # requires a variable in the constraint body. It is also # possible that the body of the constraint consists of only a # constant, in which case the "variable" of if degree == 0: if skip_trivial_constraints: continue elif degree == 2: if not supports_quadratic_constraint: raise ValueError( "Solver unable to handle quadratic expressions. Constraint" " at issue: '%s'" % (constraint_data.name)) elif degree != 1: raise ValueError( "Cannot write legal LP file. Constraint '%s' has a body " "with nonlinear terms." % (constraint_data.name)) # Create symbol con_symbol = create_symbol_func(symbol_map, constraint_data, labeler) if constraint_data.equality: label = 'c_e_' + con_symbol + '_' alias_symbol_func(symbol_map, constraint_data, label) output_file.write(label+':\n') offset = print_expr_canonical(canonical_repn, output_file, object_symbol_dictionary, variable_symbol_dictionary, False, column_order) bound = constraint_data.lower bound = self._get_bound(bound) - offset output_file.write(eq_string_template % (_no_negative_zero(bound))) output_file.write("\n") else: if constraint_data.lower is not None: if constraint_data.upper is not None: label = 'r_l_' + con_symbol + '_' else: label = 'c_l_' + con_symbol + '_' alias_symbol_func(symbol_map, constraint_data, label) output_file.write(label+':\n') offset = print_expr_canonical(canonical_repn, output_file, object_symbol_dictionary, variable_symbol_dictionary, False, column_order) bound = constraint_data.lower bound = self._get_bound(bound) - offset output_file.write(geq_string_template % (_no_negative_zero(bound))) if constraint_data.upper is not None: if constraint_data.lower is not None: label = 'r_u_' + con_symbol + '_' else: label = 'c_u_' + con_symbol + '_' alias_symbol_func(symbol_map, constraint_data, label) output_file.write(label+':\n') offset = print_expr_canonical(canonical_repn, output_file, object_symbol_dictionary, variable_symbol_dictionary, False, column_order) bound = constraint_data.upper bound = self._get_bound(bound) - offset output_file.write(leq_string_template % (_no_negative_zero(bound))) if not have_nontrivial: logger.warning('Empty constraint block written in LP format ' \ '- solver may error') # the CPLEX LP format doesn't allow constants in the objective (or # constraint body), which is a bit silly. To avoid painful # book-keeping, we introduce the following "variable", constrained # to the value 1. This is used when quadratic terms are present. # worst-case, if not used, is that CPLEX easily pre-processes it out. prefix = "" output_file.write('%sc_e_ONE_VAR_CONSTANT: \n' % prefix) output_file.write('%sONE_VAR_CONSTANT = 1.0\n' % prefix) output_file.write("\n") # SOS constraints # # For now, we write out SOS1 and SOS2 constraints in the cplex format # # All Component objects are stored in model._component, which is a # dictionary of {class: {objName: object}}. # # Consider the variable X, # # model.X = Var(...) # # We print X to CPLEX format as X(i,j,k,...) where i, j, k, ... are the # indices of X. # SOSlines = StringIO() sos1 = solver_capability("sos1") sos2 = solver_capability("sos2") writtenSOS = False for block in all_blocks: for soscondata in block.component_data_objects( SOSConstraint, active=True, sort=sortOrder, descend_into=False): create_symbol_func(symbol_map, soscondata, labeler) level = soscondata.level if (level == 1 and not sos1) or \ (level == 2 and not sos2) or \ (level > 2): raise ValueError( "Solver does not support SOS level %s constraints" % (level)) if writtenSOS == False: SOSlines.write("SOS\n") writtenSOS = True # This updates the referenced_variable_ids, just in case # there is a variable that only appears in an # SOSConstraint, in which case this needs to be known # before we write the "bounds" section (Cplex does not # handle this correctly, Gurobi does) self.printSOS(symbol_map, labeler, variable_symbol_map, soscondata, SOSlines) # # Bounds # output_file.write("bounds\n") # Scan all variables even if we're only writing a subset of them. # required because we don't store maps by variable type currently. # FIXME: This is a hack to get nested blocks working... lb_string_template = "%"+self._precision_string+" <= " ub_string_template = " <= %"+self._precision_string+"\n" # Track the number of integer and binary variables, so you can # output their status later. integer_vars = [] binary_vars = [] for vardata in variable_list: # TODO: We could just loop over the set of items in # self._referenced_variable_ids, except this is # a dictionary that is hashed by id(vardata) # which would make the bounds section # nondeterministic (bad for unit testing) if (not include_all_variable_bounds) and \ (id(vardata) not in self._referenced_variable_ids): continue if vardata.fixed: if not output_fixed_variable_bounds: raise ValueError( "Encountered a fixed variable (%s) inside an active " "objective or constraint expression on model %s, which is " "usually indicative of a preprocessing error. Use the " "IO-option 'output_fixed_variable_bounds=True' to suppress " "this error and fix the variable by overwriting its bounds " "in the LP file." % (vardata.name, model.name)) if vardata.value is None: raise ValueError("Variable cannot be fixed to a value of None.") vardata_lb = value(vardata.value) vardata_ub = value(vardata.value) else: vardata_lb = self._get_bound(vardata.lb) vardata_ub = self._get_bound(vardata.ub) name_to_output = variable_symbol_dictionary[id(vardata)] # track the number of integer and binary variables, so we know whether # to output the general / binary sections below. if vardata.is_integer(): integer_vars.append(name_to_output) elif vardata.is_binary(): binary_vars.append(name_to_output) elif not vardata.is_continuous(): raise TypeError("Invalid domain type for variable with name '%s'. " "Variable is not continuous, integer, or binary." % (vardata.name)) # in the CPLEX LP file format, the default variable # bounds are 0 and +inf. These bounds are in # conflict with Pyomo, which assumes -inf and +inf # (which we would argue is more rational). output_file.write(" ") if (vardata_lb is not None) and (vardata_lb != -infinity): output_file.write(lb_string_template % (_no_negative_zero(vardata_lb))) else: output_file.write(" -inf <= ") if name_to_output == "e": raise ValueError( "Attempting to write variable with name 'e' in a CPLEX LP " "formatted file will cause a parse failure due to confusion with " "numeric values expressed in scientific notation") output_file.write(name_to_output) if (vardata_ub is not None) and (vardata_ub != infinity): output_file.write(ub_string_template % (_no_negative_zero(vardata_ub))) else: output_file.write(" <= +inf\n") if len(integer_vars) > 0: output_file.write("general\n") for var_name in integer_vars: output_file.write(' %s\n' % var_name) if len(binary_vars) > 0: output_file.write("binary\n") for var_name in binary_vars: output_file.write(' %s\n' % var_name) # Write the SOS section output_file.write(SOSlines.getvalue()) # # wrap-up # output_file.write("end\n") # Clean up the symbol map to only contain variables referenced # in the active constraints **Note**: warm start method may # rely on this for choosing the set of potential warm start # variables vars_to_delete = set(variable_symbol_map.byObject.keys()) - \ set(self._referenced_variable_ids.keys()) sm_byObject = symbol_map.byObject sm_bySymbol = symbol_map.bySymbol var_sm_byObject = variable_symbol_map.byObject for varid in vars_to_delete: symbol = var_sm_byObject[varid] del sm_byObject[varid] del sm_bySymbol[symbol] del variable_symbol_map return symbol_map
class CPLEXPersistent(CPLEXDirect, PersistentSolver): """The CPLEX LP/MIP solver """ pyomo.util.plugin.alias('_cplex_persistent', doc='Persistent Python interface to the CPLEX LP/MIP solver') def __init__(self, **kwds): # # Call base class constructor # kwds['type'] = 'cplexpersistent' CPLEXDirect.__init__(self, **kwds) # maps pyomo var data labels to the corresponding CPLEX variable id. self._cplex_variable_ids = {} self._cplex_variable_names = None # # updates all variable bounds in the compiled model - handles # fixed variables and related issues. re-does everything from # scratch by default, ignoring whatever was specified # previously. if the value associated with the keyword # vars_to_update is a non-empty list (assumed to be variable name # / index pairs), then only the bounds for those variables are # updated. this function assumes that the variables themselves # already exist in the compiled model. # def compile_variable_bounds(self, pyomo_instance, vars_to_update): from pyomo.core.base import Var if self._active_cplex_instance is None: raise RuntimeError("***The CPLEXPersistent solver plugin " "cannot compile variable bounds - no " "instance is presently compiled") # the bound update entries should be name-value pairs new_lower_bounds = [] new_upper_bounds = [] # operates through side effects on the above lists! def update_bounds_lists(var_name): var_lb = None var_ub = None if var_data.fixed and self._output_fixed_variable_bounds: var_lb = var_ub = var_data.value elif var_data.fixed: # if we've been directed to not deal with fixed # variables, then skip - they should have been # compiled out of any description of the constraints return else: if var_data.lb is None: var_lb = -cplex.infinity else: var_lb = value(var_data.lb) if var_data.ub is None: var_ub = cplex.infinity else: var_ub= value(var_data.ub) var_cplex_id = self._cplex_variable_ids[var_name] new_lower_bounds.append((var_cplex_id, var_lb)) new_upper_bounds.append((var_cplex_id, var_ub)) if len(vars_to_update) == 0: for var_data in pyomo_instance.component_data_objects(Var, active=True): var_name = self._symbol_map.getSymbol(var_data, self._labeler) update_bounds_lists(var_name) else: for var_name, var_index in vars_to_update: var = pyomo_instance.find_component(var_name) # TBD - do some error checking! var_data = var[var_index] var_name = self._symbol_map.getSymbol(var_data, self._labeler) update_bounds_lists(var_name) self._active_cplex_instance.variables.set_lower_bounds(new_lower_bounds) self._active_cplex_instance.variables.set_upper_bounds(new_upper_bounds) # # method to compile objective of the input pyomo instance. # TBD: # it may be smarter just to track the associated pyomo instance, # and re-compile it automatically from a cached local attribute. # this would ensure consistency, among other things! # def compile_objective(self, pyomo_instance): from pyomo.core.base import Objective from pyomo.repn import canonical_is_constant, LinearCanonicalRepn, canonical_degree if self._active_cplex_instance is None: raise RuntimeError("***The CPLEXPersistent solver plugin " "cannot compile objective - no " "instance is presently compiled") cplex_instance = self._active_cplex_instance cntr = 0 for block in pyomo_instance.block_data_objects(active=True): gen_obj_canonical_repn = \ getattr(block, "_gen_obj_canonical_repn", True) # Get/Create the ComponentMap for the repn if not hasattr(block,'_canonical_repn'): block._canonical_repn = ComponentMap() block_canonical_repn = block._canonical_repn for obj_data in block.component_data_objects(Objective, active=True, descend_into=False): cntr += 1 if cntr > 1: raise ValueError( "Multiple active objectives found on Pyomo instance '%s'. " "Solver '%s' will only handle a single active objective" \ % (pyomo_instance.cname(True), self.type)) if obj_data.is_minimizing(): cplex_instance.objective.set_sense( cplex_instance.objective.sense.minimize) else: cplex_instance.objective.set_sense( cplex_instance.objective.sense.maximize) cplex_instance.objective.set_name( self._symbol_map.getSymbol(obj_data, self._labeler)) if gen_obj_canonical_repn: obj_repn = generate_canonical_repn(obj_data.expr) block_canonical_repn[obj_data] = obj_repn else: obj_repn = block_canonical_repn[obj_data] if (isinstance(obj_repn, LinearCanonicalRepn) and \ (obj_repn.linear == None)) or \ canonical_is_constant(obj_repn): print("Warning: Constant objective detected, replacing " "with a placeholder to prevent solver failure.") offset = obj_repn.constant if offset is None: offset = 0.0 objective_expression = [("ONE_VAR_CONSTANT",offset)] cplex_instance.objective.set_linear(objective_expression) else: if isinstance(obj_repn, LinearCanonicalRepn): objective_expression, offset = \ self._encode_constraint_body_linear_specialized( obj_repn, self._labeler, use_variable_names=False, cplex_variable_name_index_map=self._cplex_variable_ids, as_pairs=True) if offset != 0.0: objective_expression.append((self._cplex_variable_ids["ONE_VAR_CONSTANT"],offset)) cplex_instance.objective.set_linear(objective_expression) else: #Linear terms if 1 in obj_repn: objective_expression, offset = \ self._encode_constraint_body_linear( obj_repn, self._labeler, as_pairs=True) if offset != 0.0: objective_expression.append(("ONE_VAR_CONSTANT",offset)) cplex_instance.objective.set_linear(objective_expression) #Quadratic terms if 2 in obj_repn: self._has_quadratic_objective = True objective_expression = \ self._encode_constraint_body_quadratic(obj_repn, self._labeler, as_triples=True, is_obj=2.0) cplex_instance.objective.\ set_quadratic_coefficients(objective_expression) degree = canonical_degree(obj_repn) if (degree is None) or (degree > 2): raise ValueError( "CPLEXPersistent plugin does not support general nonlinear " "objective expressions (only linear or quadratic).\n" "Objective: %s" % (obj_data.cname(True))) # # method to populate the CPLEX problem instance (interface) from # the supplied Pyomo problem instance. # def compile_instance(self, pyomo_instance, symbolic_solver_labels=False, output_fixed_variable_bounds=False, skip_trivial_constraints=False): from pyomo.core.base import Var, Constraint, SOSConstraint from pyomo.repn import canonical_is_constant, LinearCanonicalRepn, canonical_degree self._symbolic_solver_labels = symbolic_solver_labels self._output_fixed_variable_bounds = output_fixed_variable_bounds self._skip_trivial_constraints = skip_trivial_constraints self._has_quadratic_constraints = False self._has_quadratic_objective = False used_sos_constraints = False self._active_cplex_instance = cplex.Cplex() if self._symbolic_solver_labels: labeler = self._labeler = TextLabeler() else: labeler = self._labeler = NumericLabeler('x') self._symbol_map = SymbolMap() self._instance = pyomo_instance pyomo_instance.solutions.add_symbol_map(self._symbol_map) self._smap_id = id(self._symbol_map) # we use this when iterating over the constraints because it # will have a much smaller hash table, we also use this for # the warm start code after it is cleaned to only contain # variables referenced in the constraints self._variable_symbol_map = SymbolMap() # cplex wants the caller to set the problem type, which is (for # current purposes) strictly based on variable type counts. num_binary_variables = 0 num_integer_variables = 0 num_continuous_variables = 0 ############################################# # populate the variables in the cplex model # ############################################# var_names = [] var_lbs = [] var_ubs = [] var_types = [] self._referenced_variable_ids.clear() # maps pyomo var data labels to the corresponding CPLEX variable id. self._cplex_variable_ids.clear() # cached in the loop below - used to update the symbol map # immediately following loop termination. var_label_pairs = [] for var_data in pyomo_instance.component_data_objects(Var, active=True): if var_data.fixed and not self._output_fixed_variable_bounds: # if a variable is fixed, and we're preprocessing # fixed variables (as in not outputting them), there # is no need to add them to the compiled model. continue var_name = self._symbol_map.getSymbol(var_data, labeler) var_names.append(var_name) var_label_pairs.append((var_data, var_name)) self._cplex_variable_ids[var_name] = len(self._cplex_variable_ids) if (var_data.lb is None) or (var_data.lb == -infinity): var_lbs.append(-cplex.infinity) else: var_lbs.append(value(var_data.lb)) if (var_data.ub is None) or (var_data.ub == infinity): var_ubs.append(cplex.infinity) else: var_ubs.append(value(var_data.ub)) if var_data.is_integer(): var_types.append(self._active_cplex_instance.variables.type.integer) num_integer_variables += 1 elif var_data.is_binary(): var_types.append(self._active_cplex_instance.variables.type.binary) num_binary_variables += 1 elif var_data.is_continuous(): var_types.append(self._active_cplex_instance.variables.type.continuous) num_continuous_variables += 1 else: raise TypeError("Invalid domain type for variable with name '%s'. " "Variable is not continuous, integer, or binary.") self._active_cplex_instance.variables.add(names=var_names, lb=var_lbs, ub=var_ubs, types=var_types) self._active_cplex_instance.variables.add(lb=[1], ub=[1], names=["ONE_VAR_CONSTANT"]) self._cplex_variable_ids["ONE_VAR_CONSTANT"] = len(self._cplex_variable_ids) self._variable_symbol_map.addSymbols(var_label_pairs) self._cplex_variable_names = self._active_cplex_instance.variables.get_names() ######################################################## # populate the standard constraints in the cplex model # ######################################################## expressions = [] senses = [] rhss = [] range_values = [] names = [] qexpressions = [] qlinears = [] qsenses = [] qrhss = [] qnames = [] for block in pyomo_instance.block_data_objects(active=True): gen_con_canonical_repn = \ getattr(block, "_gen_con_canonical_repn", True) # Get/Create the ComponentMap for the repn if not hasattr(block,'_canonical_repn'): block._canonical_repn = ComponentMap() block_canonical_repn = block._canonical_repn for con in block.component_data_objects(Constraint, active=True, descend_into=False): if (con.lower is None) and \ (con.upper is None): continue # not binding at all, don't bother con_repn = None if isinstance(con, LinearCanonicalRepn): con_repn = con else: if gen_con_canonical_repn: con_repn = generate_canonical_repn(con.body) block_canonical_repn[con] = con_repn else: con_repn = block_canonical_repn[con] # There are conditions, e.g., when fixing variables, under which # a constraint block might be empty. Ignore these, for both # practical reasons and the fact that the CPLEX LP format # requires a variable in the constraint body. It is also # possible that the body of the constraint consists of only a # constant, in which case the "variable" of if isinstance(con_repn, LinearCanonicalRepn): if (con_repn.linear is None) and \ self._skip_trivial_constraints: continue else: # we shouldn't come across a constant canonical repn # that is not LinearCanonicalRepn assert not canonical_is_constant(con_repn) name = self._symbol_map.getSymbol(con, labeler) expr = None qexpr = None quadratic = False if isinstance(con_repn, LinearCanonicalRepn): expr, offset = \ self._encode_constraint_body_linear_specialized(con_repn, labeler, use_variable_names=False, cplex_variable_name_index_map=self._cplex_variable_ids) else: degree = canonical_degree(con_repn) if degree == 2: quadratic = True elif (degree != 0) or (degree != 1): raise ValueError( "CPLEXPersistent plugin does not support general nonlinear " "constraint expression (only linear or quadratic).\n" "Constraint: %s" % (con.cname(True))) expr, offset = self._encode_constraint_body_linear(con_repn, labeler) if quadratic: if expr is None: expr = cplex.SparsePair(ind=[0],val=[0.0]) self._has_quadratic_constraints = True qexpr = self._encode_constraint_body_quadratic(con_repn,labeler) qnames.append(name) if con.equality: # equality constraint. qsenses.append('E') qrhss.append(self._get_bound(con.lower) - offset) elif (con.lower is not None) and (con.upper is not None): raise RuntimeError( "The CPLEXDirect plugin can not translate range " "constraints containing quadratic expressions.") elif con.lower is not None: assert con.upper is None qsenses.append('G') qrhss.append(self._get_bound(con.lower) - offset) else: qsenses.append('L') qrhss.append(self._get_bound(con.upper) - offset) qlinears.append(expr) qexpressions.append(qexpr) else: names.append(name) expressions.append(expr) if con.equality: # equality constraint. senses.append('E') rhss.append(self._get_bound(con.lower) - offset) range_values.append(0.0) elif (con.lower is not None) and (con.upper is not None): # ranged constraint. senses.append('R') lower_bound = self._get_bound(con.lower) - offset upper_bound = self._get_bound(con.upper) - offset rhss.append(lower_bound) range_values.append(upper_bound - lower_bound) elif con.lower is not None: senses.append('G') rhss.append(self._get_bound(con.lower) - offset) range_values.append(0.0) else: senses.append('L') rhss.append(self._get_bound(con.upper) - offset) range_values.append(0.0) ################################################### # populate the SOS constraints in the cplex model # ################################################### # SOS constraints - largely taken from cpxlp.py so updates there, # should be applied here # TODO: Allow users to specify the variables coefficients for custom # branching/set orders - refer to cpxlp.py sosn = self._capabilities.sosn sos1 = self._capabilities.sos1 sos2 = self._capabilities.sos2 modelSOS = ModelSOS() for soscondata in pyomo_instance.component_data_objects(SOSConstraint, active=True): level = soscondata.level if (level == 1 and not sos1) or \ (level == 2 and not sos2) or \ (level > 2 and not sosn): raise Exception("Solver does not support SOS level %s constraints" % (level,)) modelSOS.count_constraint(self._symbol_map, labeler, self._variable_symbol_map, soscondata) if modelSOS.sosType: for key in modelSOS.sosType: self._active_cplex_instance.SOS.add(type = modelSOS.sosType[key], name = modelSOS.sosName[key], SOS = [modelSOS.varnames[key], modelSOS.weights[key]]) self._referenced_variable_ids.update(modelSOS.varids[key]) used_sos_constraints = True self._active_cplex_instance.linear_constraints.add( lin_expr=expressions, senses=senses, rhs=rhss, range_values=range_values, names=names) for index in xrange(len(qexpressions)): self._active_cplex_instance.quadratic_constraints.add( lin_expr=qlinears[index], quad_expr=qexpressions[index], sense=qsenses[index], rhs=qrhss[index], name=qnames[index]) ############################################# # populate the objective in the cplex model # ############################################# self.compile_objective(pyomo_instance) ################################################ # populate the problem type in the cplex model # ################################################ # This gets rid of the annoying "Freeing MIP data." message. def _filter_freeing_mip_data(val): if val.strip() == 'Freeing MIP data.': return "" return val self._active_cplex_instance.set_warning_stream(sys.stderr, fn=_filter_freeing_mip_data) if (self._has_quadratic_objective is True) or \ (self._has_quadratic_constraints is True): if (num_integer_variables > 0) or \ (num_binary_variables > 0) or \ (used_sos_constraints): if self._has_quadratic_constraints is True: self._active_cplex_instance.set_problem_type( self._active_cplex_instance.problem_type.MIQCP) else: self._active_cplex_instance.set_problem_type( self._active_cplex_instance.problem_type.MIQP) else: if self._has_quadratic_constraints is True: self._active_cplex_instance.set_problem_type( self._active_cplex_instance.problem_type.QCP) else: self._active_cplex_instance.set_problem_type( self._active_cplex_instance.problem_type.QP) elif (num_integer_variables > 0) or \ (num_binary_variables > 0) or \ (used_sos_constraints): self._active_cplex_instance.set_problem_type( self._active_cplex_instance.problem_type.MILP) else: self._active_cplex_instance.set_problem_type( self._active_cplex_instance.problem_type.LP) # restore the warning stream without our filter function self._active_cplex_instance.set_warning_stream(sys.stderr) # # simple method to query whether a Pyomo instance has already been # compiled. # def instance_compiled(self): return self._active_cplex_instance is not None # # Override base class method to check for compiled instance # def _warm_start(self, instance): if self._active_cplex_instance is None: raise RuntimeError("***The CPLEXPersistent solver plugin " "cannot warm start - no instance is " "presently compiled") # clear any existing warm starts. self._active_cplex_instance.MIP_starts.delete() # the iteration order is identical to that used in generating # the cplex instance, so all should be well. variable_ids = [] variable_values = [] # IMPT: the var_data returned is a weak ref! for label, var_data in iteritems(self._variable_symbol_map.bySymbol): cplex_id = self._cplex_variable_ids[label] if var_data().fixed and not self._output_fixed_variable_bounds: continue elif var_data().value is not None: variable_ids.append(cplex_id) variable_values.append(var_data().value) if len(variable_ids): self._active_cplex_instance.MIP_starts.add( [variable_ids, variable_values], self._active_cplex_instance.MIP_starts.effort_level.auto) # # Override base class method to check for compiled instance # def _populate_cplex_instance(self, model): assert model == self._instance def _presolve(self, *args, **kwds): if self._active_cplex_instance is None: raise RuntimeError("***The CPLEXPersistent solver plugin" " cannot presolve - no instance is " "presently compiled") # These must be passed in to the compile_instance method, # but assert that any values here match those already supplied if 'symbolic_solver_labels' in kwds: assert self._symbolic_solver_labels == \ kwds['symbolic_solver_labels'] if 'output_fixed_variable_bounds' in kwds: assert self._output_fixed_variable_bounds == \ kwds['output_fixed_variable_bounds'] if 'skip_trivial_constraints' in kwds: assert self._skip_trivial_constraints == \ kwds["skip_trivial_constraints"] if self._smap_id not in self._instance.solutions.symbol_map: self._instance.solutions.add_symbol_map(self._symbol_map) CPLEXDirect._presolve(self, *args, **kwds) # like other solver plugins, persistent solver plugins can # take an instance as an input argument. the only context in # which this instance is used, however, is for warm-starting. if len(args) > 2: raise ValueError("The CPLEXPersistent plugin method " "'_presolve' can be supplied at most " "one problem instance - %s were " "supplied" % len(args)) # Re-add the symbol map id if it was cleared # after a previous solution load if id(self._symbol_map) not in args[0].solutions.symbol_map: args[0].solutions.add_symbol_map(self._symbol_map) self._smap_id = id(self._symbol_map) # # invoke the solver on the currently compiled instance!!! # def _apply_solver(self): if self._active_cplex_instance is None: raise RuntimeError("***The CPLEXPersistent solver plugin cannot " "apply solver - no instance is presently compiled") # NOTE: # CPLEX maintains the pool of feasible solutions from the # prior solve as the set of mip starts for the next solve. # and evaluating multiple mip starts (and there can be many) # is expensive. so if the warm_start method is not invoked, # there will potentially be a lot of time wasted. return CPLEXDirect._apply_solver(self) def _postsolve(self): if self._active_cplex_instance is None: raise RuntimeError("***The CPLEXPersistent solver plugin " "cannot postsolve - no instance is " "presently compiled") active_cplex_instance = self._active_cplex_instance variable_symbol_map = self._variable_symbol_map instance = self._instance ret = CPLEXDirect._postsolve(self) # # These get reset to None by the base class method # self._active_cplex_instance = active_cplex_instance self._variable_symbol_map = variable_symbol_map self._instance = instance return ret