class TrustRegionSolver(plugin.Plugin): """ A trust region filter method for black box / glass box optimizaiton Solves nonlinear optimization problems containing external function calls through automatic construction of reduced models (ROM), also known as surrogate models. Currently implements linear and quadratic reduced models. See Eason, Biegler (2016) AIChE Journal for more details Arguments: """ # + param.CONFIG.generte_yaml_template() plugin.implements(IOptSolver) plugin.alias( 'trustregion', doc='Trust region filter method for black box/glass box optimization') def available(self, exception_flag=True): """Check if solver is available. TODO: For now, it is always available. However, sub-solvers may not always be available, and so this should reflect that possibility. """ return True def version(self): """Return a 3-tuple describing the solver version.""" return __version__ def solve(self, model, eflist, **kwds): assert not kwds #config = param.CONFIG(kwds) return TRF(model, eflist) #, config)
class JSONSolutionSaverExtension(PySPConfiguredExtension, PySPConfiguredObject, SingletonPlugin): implements(IPySPSolutionSaverExtension) _declared_options = \ PySPConfigBlock("Options declared for the " "JSONSolutionSaverExtension class") safe_declare_common_option(_declared_options, "output_name") safe_declare_common_option(_declared_options, "save_stages") _default_prefix = "jsonsaver_" # # Note: Do not try to user super() or access the # class name inside the __init__ method when # a class derives from a SingletonPlugin. Due to # how Pyutilib implements its Singleton type, # the __class__ cell will be empty. # (See: https://stackoverflow.com/questions/ # 13126727/how-is-super-in-python-3-implemented) # def __init__(self): PySPConfiguredExtension.__init__(self) def save(self, manager): if self.get_option("output_name") is not None: stage_solutions = [] # Do NOT open file in 'binary' mode when dumping JSON # (produces an error in Python3) with open(self.get_option('output_name'), 'w') as f: cntr = 0 for stage in manager.scenario_tree.stages: if (self.get_option('save_stages') <= 0) or \ (cntr+1 <= self.get_option('save_stages')): cntr += 1 node_solutions = {} for tree_node in stage.nodes: _node_solution = extract_node_solution(tree_node) if _node_solution is None: print( "No solution appears to be stored in node with " "name %s. No solution will be saved." % (tree_node.name)) return False node_solutions[tree_node.name] = _node_solution stage_solutions.append(node_solutions) else: break json.dump(stage_solutions, f, indent=2, sort_keys=True) print("Saved scenario tree solution for %s time stages " "to file %s" % (cntr, self.get_option('output_name'))) return True print("No value was set for %s option 'output_name'. " "Nothing will be saved." % (type(self).__name__)) return False
class ConnectorExpander(Plugin): implements(IPyomoScriptModifyInstance) def apply(self, **kwds): instance = kwds.pop('instance') xform = TransformationFactory('core.expand_connectors') xform.apply_to(instance, **kwds) return instance
class TMP(Plugin): implements(modelapi[key], service=True) def __init__(self): self.fn = getattr(data.local.usermodel, key) def apply(self, **kwds): return self.fn(**kwds)
class ConvexHull_Transformation_Plugin(Plugin): implements(IPyomoScriptModifyInstance, service=True) def apply(self, **kwds): instance = kwds.pop('instance') xform = TransformationFactory('gdp.chull') return xform.apply(instance, **kwds)
class BigM_Transformation_PyomoScript_Plugin(Plugin): implements(IPyomoScriptModifyInstance, service=True) def apply(self, **kwds): instance = kwds.pop('instance') # Not sure why the ModifyInstance callback started passing the # model along with the instance. We will ignore it. model = kwds.pop('model', None) xform = TransformationFactory('gdp.bigm') return xform.apply_to(instance, **kwds)
class Transformation(Plugin): """ Base class for all model transformations. """ implements(IModelTransformation, service=False) def __init__(self, **kwds): kwds["name"] = kwds.get("name", "transformation") super(Transformation, self).__init__(**kwds) @deprecated( "Transformation.apply() has been deprecated. Please use either " "Transformation.apply_to() for in-place transformations or " "Transformation.create_using() for transformations that create a " "new, independent transformed model instance.") def apply(self, model, **kwds): inplace = kwds.pop('inplace', True) if inplace: self.apply_to(model, **kwds) else: return self.create_using(model, **kwds) def apply_to(self, model, **kwds): """ Apply the transformation to the given model. """ timer = TransformationTimer(self, 'in-place') if not hasattr(model, '_transformation_data'): model._transformation_data = TransformationData() self._apply_to(model, **kwds) timer.report() def create_using(self, model, **kwds): """ Create a new model with this transformation """ timer = TransformationTimer(self, 'out-of-place') if not hasattr(model, '_transformation_data'): model._transformation_data = TransformationData() new_model = self._create_using(model, **kwds) timer.report() return new_model def _apply_to(self, model, **kwds): raise RuntimeError( "The Transformation.apply_to method is not implemented.") def _create_using(self, model, **kwds): instance = model.clone() self._apply_to(instance, **kwds) return instance
class UnknownDataManager(Plugin): implements(IDataManager) def __init__(self, *args, **kwds): Plugin.__init__(self, **kwds) # # The 'type' is the class type of the solver instance # self.type = kwds["type"] def available(self): return False
class PyomoDataCommands(Plugin): alias("dat", "Pyomo data command file interface") implements(IDataManager, service=False) def __init__(self): self._info = [] self.options = Options() def available(self): return True def initialize(self, **kwds): self.filename = kwds.pop('filename') self.add_options(**kwds) def add_options(self, **kwds): self.options.update(kwds) def open(self): if self.filename is None: #pragma:nocover raise IOError("No filename specified") if not os.path.exists(self.filename): #pragma:nocover raise IOError("Cannot find file '%s'" % self.filename) def close(self): pass def read(self): """ This function does nothing, since executing Pyomo data commands both reads and processes the data all at once. """ pass def write(self, data): #pragma:nocover """ This function does nothing, because we cannot write to a *.dat file. """ pass def process(self, model, data, default): """ Read Pyomo data commands and process the data. """ _process_include(['include', self.filename], model, data, default, self.options) def clear(self): self._info = []
class ConvexHull_Transformation_PyomoScript_Plugin(Plugin): """Plugin to automatically call the GDP Convex Hull relaxation within the Pyomo script. """ implements(IPyomoScriptModifyInstance, service=True) def apply(self, **kwds): instance = kwds.pop('instance') # Not sure why the ModifyInstance callback started passing the # model along with the instance. We will ignore it. model = kwds.pop('model', None) xform = TransformationFactory('gdp.chull') return xform.apply_to(instance, **kwds)
class ExpressionRegistration(Plugin): implements(IPyomoExpression, service=False) def __init__(self, type, cls, swap=False): self._type = type self._cls = cls self._swap = swap def type(self): return self._type def create(self, args): if self._swap: args = list(args) args.reverse() return self._cls(args)
class OptimalityPHExtension(SingletonPlugin): implements(phextension.IPHExtension) def reset(self, ph): pass def pre_ph_initialization(self, ph): print "Adding OptimalityGapConvergence to list of convergers, with target={}.".format( convergence_threshold) # code here is similar to _converger-related code in pyomo.pysp.ph # note: this is applied in addition to the other convergers, so those # should be set to strict values if we want this to be the main criterion # TODO: find a way to specify the threshold on the command-line # (a crude way would be to write several different versions of this file # with different hard-coded thresholds.) ph._convergers.append( OptimalityGapConvergence( convergence_threshold=convergence_threshold)) def post_instance_creation(self, ph): pass def post_ph_initialization(self, ph): pass def post_iteration_0_solves(self, ph): pass def post_iteration_0(self, ph): pass def pre_iteration_k_solves(self, ph): pass def post_iteration_k_solves(self, ph): pass def post_iteration_k(self, ph): pass def post_ph_execution(self, ph): pass
class TMP(Plugin): implements(IModelComponent, service=False) alias(cls.__name__, description) component = cls
class TableData(Plugin): """ An object that imports data from a table in an external data source. """ implements(IDataManager, service=False) def __init__(self): """ Constructor """ self._info = None self._data = None self.options = Options() self.options.ncolumns = 1 def available(self): return True def initialize(self, **kwds): self.filename = kwds.pop('filename') self.add_options(**kwds) def add_options(self, **kwds): self.options.update(kwds) def open(self): #pragma:nocover """ Open the table """ pass def read(self): #pragma:nocover """ Read data from the table """ return False def write(self, data): #pragma:nocover """ Write data from the table """ return False def close(self): #pragma:nocover """ Close the table """ pass def process(self, model, data, default): """ Return the data that was extracted from this table """ if model is None: model = self.options.model if not self.options.namespace in data: data[self.options.namespace] = {} return _process_data(self._info, model, data[self.options.namespace], default, self.filename, index=self.options.index, set=self.options.set, param=self.options.param, ncolumns=self.options.ncolumns) def clear(self): """ Clear the data that was extracted from this table """ self._info = None def _set_data(self, headers, rows): header_index = [] if self.options.select is None: for i in xrange(len(headers)): header_index.append(i) else: for i in self.options.select: header_index.append(headers.index(str(i))) self.options.ncolumns = len(headers) if not self.options.param is None: if not type(self.options.param) in (list, tuple): self.options.param = (self.options.param, ) _params = [] for p in self.options.param: if isinstance(p, Param): self.options.model = p.model() _params.append(p.name) else: _params.append(p) self.options.param = tuple(_params) if isinstance(self.options.set, Set): self.options.model = self.options.set.model() self.options.set = self.options.set.name if isinstance(self.options.index, Set): self.options.model = self.options.index.model() self.options.index = self.options.index.name if self.options.format is None: if not self.options.set is None: self.options.format = 'set' elif not self.options.param is None: self.options.format = 'table' if self.options.format is None: raise ValueError("Unspecified format and data option") elif self.options.set is None and self.options.param is None: msg = "Must specify the set or parameter option for data" raise IOError(msg) if self.options.format == 'set': if not self.options.index is None: msg = "Cannot specify index for data with the 'set' format: %s" raise IOError(msg % str(self.options.index)) self._info = ["set", self.options.set, ":="] for row in rows: if self.options.ncolumns > 1: self._info.append(tuple(row)) else: self._info.extend(row) elif self.options.format == 'set_array': if not self.options.index is None: msg = "Cannot specify index for data with the 'set_array' " \ 'format: %s' raise IOError(msg % str(self.options.index)) self._info = ["set", self.options.set, ":"] self._info.extend(headers[1:]) self._info.append(":=") for row in rows: self._info.extend(row) elif self.options.format == 'transposed_array': self._info = ["param", self.options.param[0], "(tr)", ":"] self._info.extend(headers[1:]) self._info.append(":=") for row in rows: self._info.extend(row) elif self.options.format == 'array': self._info = ["param", self.options.param[0], ":"] self._info.extend(headers[1:]) self._info.append(":=") for row in rows: self._info.extend(row) elif self.options.format == 'table': if self.options.index is not None: self._info = ["param", ":", self.options.index, ":"] else: self._info = ["param", ":"] for param in self.options.param: self._info.append(param) self._info.append(":=") for row in rows: for i in header_index: self._info.append(row[i]) self.options.ncolumns = len(header_index) else: msg = "Unknown parameter format: '%s'" raise ValueError(msg % self.options.format) def get_table(self): tmp = [] if not self.options.columns is None: tmp.append(self.options.columns) if not self.options.set is None: # Create column names if self.options.columns is None: cols = [] for i in xrange(self.options.set.dimen): cols.append(self.options.set.name + str(i)) tmp.append(cols) # Get rows for data in self.options.set: if self.options.set.dimen > 1: tmp.append(list(data)) else: tmp.append([data]) elif not self.options.param is None: if type(self.options.param) in (list, tuple): _param = self.options.param else: _param = [self.options.param] tmp = [] # Collect data for index in _param[0]: if index is None: row = [] elif type(index) in (list, tuple): row = list(index) else: row = [index] for param in _param: row.append(value(param[index])) tmp.append(row) # Create column names if self.options.columns is None: cols = [] for i in xrange(len(tmp[0]) - len(_param)): cols.append('I' + str(i)) for param in _param: cols.append(param) tmp = [cols] + tmp return tmp
class ConnectorExpander(Plugin): implements(IPyomoScriptModifyInstance) def apply(self, **kwds): if __debug__ and logger.isEnabledFor(logging.DEBUG): #pragma:nocover logger.debug("Calling ConnectorExpander") instance = kwds['instance'] blockList = list(instance.block_data_objects(active=True)) noConnectors = True for b in blockList: if b.component_map(Connector): noConnectors = False break if noConnectors: return if __debug__ and logger.isEnabledFor(logging.DEBUG): #pragma:nocover logger.debug(" Connectors found!") # # At this point, there are connectors in the model, so we must # look for constraints that involve connectors and expand them. # #options = kwds['options'] #model = kwds['model'] # In general, blocks should be relatively self-contained, so we # should build the connectors from the "bottom up": blockList.reverse() # Expand each constraint involving a connector for block in blockList: if __debug__ and logger.isEnabledFor(logging.DEBUG): #pragma:nocover logger.debug(" block: " + block.name) CCC = {} for name, constraint in itertools.chain\ ( iteritems(block.component_map(Constraint)), iteritems(block.component_map(ConstraintList)) ): cList = [] CCC[name+'.expanded'] = cList for idx, c in iteritems(constraint._data): if __debug__ and logger.isEnabledFor(logging.DEBUG): #pragma:nocover logger.debug(" (looking at constraint %s[%s])", name, idx) connectors = [] self._gather_connectors(c.body, connectors) if len(connectors) == 0: continue if __debug__ and logger.isEnabledFor(logging.DEBUG): #pragma:nocover logger.debug(" (found connectors in constraint)") # Validate that all connectors match errors, ref, skip = self._validate_connectors(connectors) if errors: logger.error( ( "Connector mismatch: errors detected when " "constructing constraint %s\n " % (name + (idx and '[%s]' % idx or '')) ) + '\n '.join(reversed(errors)) ) raise ValueError( "Connector mismatch in constraint %s" % \ name + (idx and '[%s]' % idx or '')) if __debug__ and logger.isEnabledFor(logging.DEBUG): #pragma:nocover logger.debug(" (connectors valid)") # Fill in any empty connectors for conn in connectors: if conn.vars: continue for var in ref.vars: if var in skip: continue v = Var() block.add_component(conn.local_name + '.auto.' + var, v) conn.vars[var] = v v.construct() # OK - expand this constraint self._expand_constraint(block, name, idx, c, ref, skip, cList) # Now deactivate the original constraint c.deactivate() for name, exprs in iteritems(CCC): cList = ConstraintList() block.add_component( name, cList ) cList.construct() for expr in exprs: cList.add(expr) # Now, go back and implement VarList aggregators for block in blockList: for conn in itervalues(block.component_map(Connector)): for var, aggregator in iteritems(conn.aggregators): c = Constraint(expr=aggregator(block, var)) block.add_component( conn.local_name + '.' + var.local_name + '.aggregate', c) c.construct() def _gather_connectors(self, expr, connectors): if expr.is_expression(): if expr.__class__ is _ProductExpression: for e in expr._numerator: self._gather_connectors(e, connectors) for e in expr._denominator: self._gather_connectors(e, connectors) else: for e in expr._args: self._gather_connectors(e, connectors) elif isinstance(expr, _ConnectorValue): connectors.append(expr) def _validate_connectors(self, connectors): errors = [] ref = None skip = set() for idx in xrange(len(connectors)): if connectors[idx].vars.keys(): ref = connectors.pop(idx) break if ref is None: errors.append( "Cannot identify a reference connector: no connectors " "have assigned variables" ) return errors, ref, skip a = set(ref.vars.keys()) for key, val in iteritems(ref.vars): if val is None: skip.add(key) for tmp in connectors: b = set(tmp.vars.keys()) if not b: continue for key, val in iteritems(tmp.vars): if val is None: skip.add(key) for var in a - b: # TODO: add a fq_name so we can easily get # the full model.block.connector name errors.append( "Connector '%s' missing variable '%s' " "(appearing in reference connector '%s')" % ( tmp.name, var, ref.name ) ) for var in b - a: errors.append( "Reference connector '%s' missing variable '%s' " "(appearing in connector '%s')" % ( ref.name, var, tmp.name ) ) return errors, ref, skip def _expand_constraint(self, block, name, idx, constraint, ref, skip, cList): def _substitute_var(arg, var): if arg.is_expression(): if arg.__class__ is _ProductExpression: _substitute_vars(arg._numerator, var) _substitute_vars(arg._denominator, var) else: _substitute_vars(arg._args, var) return arg elif isinstance(arg, _ConnectorValue): v = arg.vars[var] if v.is_expression(): v = v.clone() return _substitute_var(v, var) elif isinstance(arg, VarList): return arg.add() return arg def _substitute_vars(args, var): for idx, arg in enumerate(args): if arg.is_expression(): if arg.__class__ is _ProductExpression: _substitute_vars(arg._numerator, var) _substitute_vars(arg._denominator, var) else: _substitute_vars(arg._args, var) elif isinstance(arg, _ConnectorValue): v = arg.vars[var] if v.is_expression(): v = v.clone() args[idx] = _substitute_var(v, var) elif isinstance(arg, VarList): args[idx] = arg.add() for var in ref.vars.iterkeys(): if var in skip: continue if constraint.body.is_expression(): c = _substitute_var(constraint.body.clone(), var) else: c = _substitute_var(constraint.body, var) if constraint.equality: cList.append( ( c, constraint.upper ) ) else: cList.append( ( constraint.lower, c, constraint.upper ) )
class InterScenarioPlugin(SingletonPlugin): implements(phextension.IPHExtension) def __init__(self): self.enableRhoUpdates = True self.enableFeasibilityCuts = True self.enableIncumbentCuts = True self.epsilon = 1e-7 self.cut_scale = 0 #1e-4 self.allow_variable_slack = False # Force this plugin to run every N iterations self.iterationInterval = 100 # Alternative methods to trigger the plugin: # # If the convergence metric degrades by either a relative or # absolute amount self.convergenceRelativeDegredation = 10.33 self.convergenceAbsoluteDegredation = 10.001 # If at least recutThreshold fraction of all-to-all scenario # tests produced feasibility cuts self.recutThreshold = 0.33 # If at least this fraction of unique solutions are preserved # from one iteration to the next self.repeated_solution_threshhold = 0.90 # multiplier on computed rho values self.rhoScale = 0.75 # How quickly rho moves to new values [0..1] # 0: no damping (jump to calculated rho) # 1: complete damping (do not change current value of rho) self.rhoDamping = 0.1 # Minimum difference in objective to include a cut, and minimum # difference in variable values to include that term in a cut self.cutThreshold_minDiff = 0.0001 # Fraction of the cut library to use for cross-scenario # (all-to-all) cuts self.cutThreshold_crossCut = 0 # Force the InterScenario plugin to re-run while the improvement # in the Lagrangean bound is at least this much: self.iteration0RecutBoundImprovement = 0.0025 def reset(self, ph): self.incumbent = None self.rho = None self.x_deviation = None self.lastConvergenceMetric = None self.feasibility_cuts = [] self.incumbent_cuts = [] self.lastRun = 0 self.average_solution = None self.converger = NormalizedTermDiffConvergence() self.unique_scenario_solutions = [] def pre_ph_initialization(self, ph): self.reset(ph) pass def post_instance_creation(self, ph): if self.enableRhoUpdates: rootNode = ph._scenario_tree.findRootNode() for v in rootNode._xbars: ph.setRhoAllScenarios(rootNode, v, 0) pass def post_ph_initialization(self, ph): if len(ph._scenario_tree._stages) > 2: raise RuntimeError( "InterScenario plugin only works with 2-stage problems") self._sense_to_min = 1 if ph._objective_sense == minimize else -1 # We are going to manage RHO here. So, we want to turn it off # until we finish the initial round of interscenario feasibility # cuts. if self.enableRhoUpdates: rootNode = ph._scenario_tree.findRootNode() for v in rootNode._xbars: ph.setRhoAllScenarios(rootNode, v, 0) #self.rho = dict((v,ph._rho) for v in ph._scenario_tree.findRootNode()._xbars) def post_iteration_0_solves(self, ph): self._collect_unique_scenario_solutions(ph) self._interscenario_plugin(ph) count = 0 while self.rho is None and self.feasibility_cuts: count += 1 toc("InterScenario plugin: PH iteration 0 re-solve pass %s" % (count, )) _stale_scenarios = [] for _id, _soln in enumerate(self.unique_scenario_solutions): _was_cut = sum(1 for c in self.feasibility_cuts if type(c[_id]) is tuple) if _was_cut: _stale_scenarios.extend(_soln[1]) self._distribute_cuts(ph, True) toc("InterScenario plugin: distributed cuts to scenarios") self._collect_unique_scenario_solutions(ph) self._interscenario_plugin(ph) self.lastRun = 0 def post_iteration_0(self, ph): self.converger.update(ph._current_iteration, ph, ph._scenario_tree, ph._instances) self.lastConvergenceMetric = self.converger.lastMetric() pass def pre_iteration_k_solves(self, ph): if self.feasibility_cuts or self.incumbent_cuts: self._distribute_cuts(ph) pass def post_iteration_k_solves(self, ph): self.converger.update(ph._current_iteration, ph, ph._scenario_tree, ph._instances) curr = self.converger.lastMetric() last = self.lastConvergenceMetric delta = curr - last #print("InterScenario convergence:", last, curr, delta) run = False if (self._collect_unique_scenario_solutions(ph) >= self.repeated_solution_threshhold): print("InterScenario plugin: triggered by no change in " "scenario solutions") run = True if (delta > last * self.convergenceRelativeDegredation and delta > self.convergenceAbsoluteDegredation): print("InterScenario plugin: triggered by convergence degredation " "(%0.4f; %+0.4f)" % (curr, delta)) run = True if ph._current_iteration - self.lastRun >= self.iterationInterval: print("InterScenario plugin: triggered by iteration limit") run = True if self.rho is None: print("InterScenario plugin: triggered to initialize rho") run = True elif self.enableRhoUpdates: rootNode = ph._scenario_tree.findRootNode() for _id, rho in iteritems(self.rho): _max = rootNode._maximums[_id] _min = rootNode._minimums[_id] if rho < self.epsilon and _max - _min > self.epsilon: print("InterScenario plugin: triggered by variable " "divergence with rho==0 (%s: %s; [%s, %s])" % (_id, rho, _max, _min)) run = True break if run: self.lastRun = ph._current_iteration self._interscenario_plugin(ph) self.lastConvergenceMetric = curr pass def post_iteration_k(self, ph): pass def post_ph_execution(self, ph): self._collect_unique_scenario_solutions(ph) self._interscenario_plugin(ph) pass def _interscenario_plugin(self, ph): toc("InterScenario plugin: analyzing scenario dual information") # (1) Collect all scenario (first) stage variables #self._collect_unique_scenario_solutions(ph) # (2) Filter them to find a set we want to distribute pass # (3) Distribute (some) of the variable sets out to the # scenarios, fix, and resolve; Collect and return the # objectives, duals, and any cuts partial_obj_values, dual_values, cuts, probability \ = self._solve_interscenario_solutions( ph ) # Compute the non-anticipative objective values for each # scenario solution self.feasible_objectives = self._compute_objective( partial_obj_values, probability) for _id, soln in enumerate(self.unique_scenario_solutions): _scenarios = [ph._scenario_tree.get_scenario(x) for x in soln[1]] print( " Solution %2d: generated %2d cuts, " "cut by %2d other scenarios; objective %10s, " "scenario cost [%s], cut obj [%s] [generated by %s]" % (_id, sum(1 for c in cuts[_id] if type(c) is tuple), sum(1 for c in cuts if type(c[_id]) is tuple), "None" if self.feasible_objectives[_id] is None else "%10.2f" % self.feasible_objectives[_id], ", ".join("%10.2f" % x._cost for x in _scenarios), " ".join("%5.2f" % x[0] if type(x) is tuple else "%5s" % x for x in cuts[_id]), ','.join(soln[1]))) scenarioCosts = [ ph._scenario_tree.get_scenario(x)._cost for s in self.unique_scenario_solutions for x in s[1] ] scenarioProb = [ ph._scenario_tree.get_scenario(x)._probability for s in self.unique_scenario_solutions for x in s[1] ] _avg = sum(scenarioProb[i] * c for i, c in enumerate(scenarioCosts)) _max = max(scenarioCosts) _min = min(scenarioCosts) if self.average_solution is None: _del_avg = None _del_avg_str = "-----%" else: _prev = self.average_solution _del_avg = (_avg - _prev) / max(abs(_avg), abs(_prev)) _del_avg_str = "%+.2f%%" % (100 * _del_avg, ) self.average_solution = _avg print(" Average scenario cost: %f (%s) Max-min: %f (%0.2f%%)" % (_avg, _del_avg_str, _max - _min, abs(100. * (_max - _min) / _avg))) # (4) save any cuts for distribution before the next solve #self.feasibility_cuts = [] #for c in cuts: # self.feasibility_cuts.extend( # x for x in c if type(x) is tuple and x[0] > self.cutThreshold ) #cutCount = len(self.feasibility_cuts) if self.enableFeasibilityCuts: self.feasibility_cuts = cuts cutCount = sum( sum(1 for x in c if type(x) is tuple and x[0] > self.cutThreshold_minDiff) for c in cuts) subProblemCount = sum(len(c) for c in cuts) # (5) compute and publish the new incumbent self._update_incumbent(ph) # (6a) If this is iteration 0, and we have feasibility cuts, and # they are (sufficiently) helping the Lagrangean bound, then # skip setting rho and do another round oc cuts if ph._current_iteration == 0: # Tell ph that we may have a good opter bound ph._update_reported_bounds(outer=self.average_solution) if (cutCount > self.recutThreshold * (subProblemCount - len(cuts)) and (_del_avg is None or _del_avg > self.iteration0RecutBoundImprovement)): # Bypass RHO updates and check for more cuts #self.lastRun = ph._current_iteration - self.iterationInterval return # (6b) compute updated rho estimates new_rho, loginfo = self._process_dual_information( ph, dual_values, probability) _scale = self.rhoScale if self.rho is None: print("InterScenario plugin: initializing rho") self.rho = {} for v, r in iteritems(new_rho): self.rho[v] = _scale * r else: _damping = self.rhoDamping for v, r in iteritems(new_rho): if self.rho[v]: self.rho[v] += (1 - _damping) * (_scale * r - self.rho[v]) #self.rho[v] = max(_scale*r, self.rho[v]) - \ # _damping*abs(_scale*r - self.rho[v]) else: self.rho[v] = _scale * r for v, l in sorted(iteritems(loginfo)): if v is None: print(l) else: print(l % (self.rho[v], )) #print("SETTING SELF.RHO", self.rho) rootNode = ph._scenario_tree.findRootNode() if self.enableRhoUpdates: for v, r in iteritems(self.rho): ph.setRhoAllScenarios(rootNode, v, r) def _collect_unique_scenario_solutions(self, ph): # list of (varmap, scenario_list) tuples _old_unique_scenario_solutions = self.unique_scenario_solutions self.unique_scenario_solutions = [] # See ph.py:update_variable_statistics for a multistage version... rootNode = ph._scenario_tree.findRootNode() for scenario in rootNode._scenarios: _this_sol = dict(scenario._x[rootNode._name]) for _id, _val in iteritems(scenario._x[rootNode._name]): #if rootNode.is_variable_fixed(_id): # continue if rootNode.is_variable_binary(_id) or \ rootNode.is_variable_integer(_id): _this_sol[_id] = int(round(_val)) found = False # Note: because we are looking for unique variable values, # then if the user is bundling, this will implicitly re-form # the bundles for _sol in self.unique_scenario_solutions: if _this_sol == _sol[0]: _sol[1].append(scenario._name) found = True break if not found: self.unique_scenario_solutions.append( (_this_sol, [scenario._name])) _unchanged = 0 for _old_soln, _old_scen in _old_unique_scenario_solutions: for _soln, _scen in self.unique_scenario_solutions: if _old_soln == _soln: _unchanged += 1 break print("Interscenario plugin: %s unchanged scenario solutions " "(out of %s)" % (_unchanged, len(self.unique_scenario_solutions))) return float(_unchanged) / len(self.unique_scenario_solutions) def _solve_interscenario_solutions(self, ph): results = ( [], [], [], ) probability = [] #cutlist = [] distributed = isinstance(ph._solver_manager, SolverManager_PHPyro) action_handles = [] if ph._scenario_tree.contains_bundles(): subproblems = ph._scenario_tree._scenario_bundles else: subproblems = ph._scenario_tree._scenarios for problem in subproblems: probability.append(problem._probability) options = (self.unique_scenario_solutions, ) kwd_options = { 'epsilon': self.epsilon, 'cut_scale': self.cut_scale, 'allow_slack': self.allow_variable_slack, 'enable_rho': self.enableRhoUpdates, 'enable_cuts': self.enableFeasibilityCuts } if distributed: action_handles.append( ph._solver_manager.queue( action="invoke_external_function", name=problem._name, queue_name=ph._phpyro_job_worker_map[problem._name], invocation_type=InvocationType.SingleInvocation.key, generateResponse=True, module_name='pyomo.pysp.plugins.interscenario', function_name='solve_fixed_scenario_solutions', function_kwds=kwd_options, function_args=options, )) else: _tmp = solve_fixed_scenario_solutions(ph, ph._scenario_tree, problem, *options, **kwd_options) for i, r in enumerate(results): r.append(_tmp[i]) #cutlist.extend(_tmp[-1]) if distributed: num_results_so_far = 0 num_results = len(action_handles) for r in results: r.extend([None] * num_results) while (num_results_so_far < num_results): _ah = ph._solver_manager.wait_any() _ah_id = action_handles.index(_ah) _tmp = ph._solver_manager.get_results(_ah) for i, r in enumerate(results): r[_ah_id] = _tmp[i] #cutlist.extend(_tmp[-1]) num_results_so_far += 1 return results + (probability, ) # + (cutlist,) def _distribute_cuts(self, ph, resolve=False): totalCuts = 0 cutObj = sorted( c[0] for x in self.feasibility_cuts for c in x if type(c) is tuple and c[0] > self.cutThreshold_minDiff) if cutObj: allCutThreshold = cutObj[min( int((1 - self.cutThreshold_crossCut) * len(cutObj)), len(cutObj) - 1)] else: allCutThreshold = 1 distributed = isinstance(ph._solver_manager, SolverManager_PHPyro) if ph._scenario_tree.contains_bundles(): subproblems = ph._scenario_tree._scenario_bundles get_scenarios = lambda x: x._scenario_names else: subproblems = ph._scenario_tree._scenarios get_scenarios = lambda x: [x] resolves = [] for problem in subproblems: cuts = [] for id, (x, s) in enumerate(self.unique_scenario_solutions): found = False for scenario in get_scenarios(problem): if scenario._name in s: found = True break if found: cuts.extend(c[id] for c in self.feasibility_cuts if type(c[id]) is tuple and c[id][0] > self.cutThreshold_minDiff) elif self.feasible_objectives[id] is None: # We only add cuts generated by other scenarios to # scenarios that are not currently feasible (as # these are feassibility cuts, they should not # impact feasible scenarios) cuts.extend( c[id] for c in self.feasibility_cuts if type(c[id]) is tuple and c[id][0] > allCutThreshold) if not cuts and not self.incumbent_cuts: resolves.append(None) continue totalCuts += len(cuts) if distributed: resolves.append( ph._solver_manager.queue( action="invoke_external_function", name=problem._name, queue_name=ph._phpyro_job_worker_map[problem._name], invocation_type=InvocationType.SingleInvocation.key, generateResponse=True, module_name='pyomo.pysp.plugins.interscenario', function_name='add_new_cuts', function_kwds=None, function_args=(cuts, self.incumbent_cuts, resolve), )) else: ans = add_new_cuts(ph, ph._scenario_tree, problem, cuts, self.incumbent_cuts, resolve) resolves.append(ans) toc("distributed cuts to scenario %s%s" % (problem._name, ' and resolved scenario' if resolve else '')) toc("InterScenario plugin: added %d feasibility cuts from a " "library of %s cuts" % (totalCuts, len(cutObj))) self.feasibility_cuts = [] if self.incumbent_cuts: print("InterScenario plugin: added %d incumbent cuts" % (len(self.incumbent_cuts), )) self.incumbent_cuts = [] if distributed: num_results_so_far = sum(1 for x in resolves if x is None) num_results = len(resolves) while (num_results_so_far < num_results): _ah = ph._solver_manager.wait_any() _ah_idx = resolves.index(_ah) resolves[_ah_idx] = ph._solver_manager.get_results(_ah) num_results_so_far += 1 if resolve: # Transfer the first stage values and cost back to PH and # recompute xbar rootNode = ph._scenario_tree.findRootNode() for _id, problem in enumerate(subproblems): ans = resolves[_id] if ans is None: continue for scenario in get_scenarios(problem): scenario._cost = ans[1] assert (sorted(ans[0]) == sorted( scenario._x[rootNode._name])) scenario._x[rootNode._name] = ans[0] #[_vid] = _vval ph.update_variable_statistics() def _compute_objective(self, partial_obj_values, probability): obj_values = [] for soln_id in xrange(len(self.unique_scenario_solutions)): obj = 0. for scen_or_bundle_id, p in enumerate(probability): if partial_obj_values[scen_or_bundle_id][soln_id] is None: obj = None break obj += p * partial_obj_values[scen_or_bundle_id][soln_id] obj_values.append(obj) return obj_values def _update_incumbent(self, ph): feasible_obj = [ o for o in enumerate(self.feasible_objectives) if o[1] is not None ] if not feasible_obj: print("InterScenario plugin: No scenario solutions are " "globally feasible") return print("InterScenario plugin: Feasible objectives: %s" % (sorted(o[1] for o in feasible_obj), )) best_id, best_obj = min( ((x[0], self._sense_to_min * x[1]) for x in feasible_obj), key=operator.itemgetter(1)) binary_vars = [] integer_vars = [] continuous_vars = [] rootNode = ph._scenario_tree.findRootNode() for _id in rootNode._scenarios[0]._x[rootNode._name]: if rootNode.is_variable_fixed(_id): continue if rootNode.is_variable_binary(_id): binary_vars.append(_id) elif rootNode.is_variable_integer(_id): integer_vars.append(_id) elif rootNode.is_variable_semicontinuous(_id): assert False, "FIXME" else: # we can not add incumbent cuts for continuous domains continuous_vars.append(_id) if self.incumbent is None or \ self.incumbent[0] * self._sense_to_min > best_obj + self.epsilon: # Cut the old incumbent if self.enableIncumbentCuts and self.incumbent and not continuous_vars: _x = self.incumbent[1][0] self.incumbent_cuts.append(( dict((vid, round(_x[vid])) for vid in binary_vars), dict((vid, round(_x[vid])) for vid in integer_vars), )) # New incumbent! self.incumbent = (best_obj * self._sense_to_min, self.unique_scenario_solutions[best_id], best_id) # Tell PH (that we have a good inner bound) ph._update_reported_bounds(inner=best_obj) msg = "InterScenario plugin: NEW incumbent: %s = %s, %s" \ % self.incumbent print(msg) logger.info(msg) elif self.incumbent[0] * self._sense_to_min < best_obj - self.epsilon: # Keep existing incumbent... so the best thing here can be cut msg = "InterScenario plugin: incumbent: %s = %s, %s" \ % self.incumbent print(msg) best_id = -1 if continuous_vars or not self.enableIncumbentCuts: return for _id, obj in feasible_obj: if _id == best_id: continue _x = self.unique_scenario_solutions[_id][0] self.incumbent_cuts.append(( dict((vid, round(_x[vid])) for vid in binary_vars), dict((vid, round(_x[vid])) for vid in integer_vars), )) def _process_dual_information(self, ph, dual_values, probability): # Notes: # dual_values: [ [ { var_id: dual } ] ] # - list of list of maps of variable id to dual value. The # outer list is returned by each subproblem (corresponds to # a bundle or scenario). The order in this list matches # the order in the probability list. The inner list holds # the dual values for each solution the scenario/bundle was # asked to evaluate. This inner list is in the same order # as the solutions list. # probability: [ scenario/bundle probility ] # - list of the scenario or bundle probability for the # submodel that returned the corresponding objective/dual # values # unique_scenario_solutions: [ {var_id:var_value}, [ scenario_names ] ] # - list of candidate solutions holding the 1st stage # variable values (in a map) and the list of scenarios # that had that solution as the optimal solution in this # iteration # soln_prob: the total probability of all scenarios that have # this solution as their locally-optimal solution soln_prob = [0.] * len(self.unique_scenario_solutions) for soln_id, soln_info in enumerate(self.unique_scenario_solutions): for src_scen_name in soln_info[1]: src_scen = ph._scenario_tree.get_scenario(src_scen_name) soln_prob[soln_id] += src_scen._probability total_soln_prob = sum(soln_prob) # xbar: { var_id : xbar } # - this has the average first stage variable values. We # should really get this from the scenario tree, as we # cannot guarantee that we will see all the current values # here (they can be filtered) #xbar = dict( ( # k, # sum(v*soln_prob[i] for i,v in enumerate(vv))/total_soln_prob ) # for k, vv in iteritems(var_info) ) xbar = ph._scenario_tree.findRootNode()._xbars if self.x_deviation is None: self.x_deviation = dict( (v, max(s[0][v] for s in self.unique_scenario_solutions) - min(s[0][v] for s in self.unique_scenario_solutions)) for v in xbar) max_dual = dict((v, 0.) for v in xbar) weighted_rho = dict((v, 0.) for v in xbar) for soln_id, soln_p in enumerate(soln_prob): x = self.unique_scenario_solutions[soln_id][0] avg_dual = dict((v, 0.) for v in xbar) p_total = 0. for scen_id, p in enumerate(probability): if dual_values[scen_id][soln_id] is None: continue for v, d in iteritems(dual_values[scen_id][soln_id]): avg_dual[v] += math.copysign(d, xbar[v] - x[v]) * p max_dual[v] = max(max_dual[v], abs(d)) p_total += p if p_total: for v in avg_dual: avg_dual[v] /= p_total #x_deviation = dict( (v, abs(xbar[v]-self.unique_scenario_solutions[soln_id][0][v])) # for v in xbar ) for v, x_dev in iteritems(self.x_deviation): weighted_rho[v] += soln_prob[soln_id] * avg_dual[v] / (x_dev + 1.) if False: # MAX dual (not average) for v, x_dev in iteritems(self.x_deviation): weighted_rho[v] += max_dual[v] / (x_dev + 1.) # var_info: { var_id : [ scenario values ] } # - this has the list of all values for a single 1st stage # variable, in the same order as the solutions list (and the # soln_prob list) var_info = {} for soln_id, soln_info in enumerate(self.unique_scenario_solutions): for k, v in iteritems(soln_info[0]): try: var_info[k].append(v) except: var_info[k] = [v] dual_info = {} for sid, scenario_results in enumerate(dual_values): for solution in scenario_results: if solution is None: continue for k, v in iteritems(solution): try: dual_info[k].append(v) except: dual_info[k] = [v] # (optionally) scale the weighted rho #for v in xbar: # weighted_rho[v] = weighted_rho[v] / total_soln_prob # Check for rho == 0 _min_rho = min(_rho for _rho in weighted_rho if _rho > 0) for v in xbar: if weighted_rho[v] <= 0: # If there is variable disagreement, but no objective # pressure to price the disagreement, the best thing we # can do is guess and let later iterations sort it out. # #if max(var_info[v]) - min(var_info[v]) > 0: # weighted_rho[v] = 1. # # Actually, we will just set all 0 rho values to the # smallest non-zero dual weighted_rho[v] = _min_rho loginfo = { None: "%4s: %6s [%7s, %7s] %7s; " "%6s [%6s, %6s] %6s; RHO %7s : %7s" % ('---', 'Dual', 'min', 'max', 'stdev', 'Var', 'min', 'max', 'stdev', 'computed', 'final') } for k, duals in iteritems(dual_info): # DISABLE! #break d_min = min(duals) d_max = max(duals) _sum = sum(abs(x) for x in duals) _sumsq = sum(x**2 for x in duals) n = float(len(duals)) d_avg = _sum / n d_stdev = math.sqrt(abs(_sumsq / n - d_avg**2)) x_min = min(var_info[k]) x_max = max(var_info[k]) _sum = sum(abs(x) for x in var_info[k]) _sumsq = sum(x**2 for x in var_info[k]) n = float(len(var_info[k])) x_avg = _sum / n x_stdev = math.sqrt(abs(_sumsq / n - x_avg**2 + 1e-6)) loginfo[k] = \ "%4d: %6.1f [%7.1f, %7.1f] %7.1f; " \ "%6.1f [%6.1f, %6.1f] %6.1f; RHO %7.2f : %%7.2f" % ( k, d_avg, d_min, d_max, d_stdev, x_avg, x_min, x_max, x_stdev, weighted_rho[k] ) return weighted_rho, loginfo
class JSONSolutionLoaderExtension(PySPConfiguredExtension, PySPConfiguredObject, SingletonPlugin): implements(IPySPSolutionLoaderExtension) _declared_options = \ PySPConfigBlock("Options declared for the " "JSONSolutionLoaderExtension class") safe_declare_common_option(_declared_options, "input_name") safe_declare_common_option(_declared_options, "load_stages") _default_prefix = "jsonloader_" # # Note: Do not try to user super() or access the # class name inside the __init__ method when # a class derives from a SingletonPlugin. Due to # how Pyutilib implements its Singleton type, # the __class__ cell will be empty. # (See: https://stackoverflow.com/questions/ # 13126727/how-is-super-in-python-3-implemented) # def __init__(self): PySPConfiguredExtension.__init__(self) def load(self, manager): if self.get_option("input_name") is not None: stage_solutions = None # Do NOT open file in 'binary' mode when loading JSON # (produces an error in Python3) with open(self.get_option("input_name"), 'r') as f: stage_solutions = json.load(f) cntr = 0 if self.get_option('load_stages') > len( manager.scenario_tree.stages): raise ValueError( "The value of the %s option (%s) can not be greater than " "the number of time stages in the local scenario tree (%s)" % (self.get_full_option_name('load_stages'), self.get_option('load_stages'), len(manager.scenario_tree.stages))) if self.get_option('load_stages') > len(stage_solutions): raise ValueError( "The value of the %s option (%s) can not be greater than " "the number of time stages in the scenario tree solution " "stored in %s (%s)" % (self.get_full_option_name('load_stages'), self.get_option('load_stages'), self.get_option('input_name'), len(stage_solutions))) for stage, stage_solution in zip_longest( manager.scenario_tree.stages, stage_solutions): if stage_solution is None: break if (self.get_option('load_stages') <= 0) or \ (cntr+1 <= self.get_option('load_stages')): if stage is None: raise RuntimeError( "Local scenario tree has fewer stages (%s) than what is " "held by the solution loaded from file %s. Use the " "option %s to limit the number of stages that " "are loaded." % (cntr, self.get_option('input_name'), self.get_full_option_name('load_stages'))) cntr += 1 for tree_node in stage.nodes: try: node_solution = stage_solution[tree_node.name] except KeyError: raise KeyError( "Local scenario tree contains a tree node " "that was not found in the solution at time" "-stage %s: %s" % (cntr, tree_node.name)) load_node_solution(tree_node, node_solution) else: break print("Loaded scenario tree solution for %s time stages " "from file %s" % (cntr, self.get_option('input_name'))) return True print("No value was set for %s option 'input_name'. " "Nothing will be saved." % (type(self).__name__)) return False
class PyomoTask_tmp(with_metaclass(TaskMeta,PyomoTask)): plugin.alias(_alias) plugin.implements(IPyomoTask, service=False) def __init__(self, *args, **kwargs): kwargs['fn'] = fn PyomoTask.__init__(self, *args, **kwargs) if not fn is None: if len(argspec.args) is 0: nargs = 0 elif argspec.defaults is None: nargs = len(argspec.args) else: nargs = len(argspec.args) - len(argspec.defaults) self._kwargs = argspec.args[nargs:] if nargs != 1 and 'data' not in self._kwargs: logger.error("A Pyomo functor '%s' must have a 'data argument" % _alias) if argspec.defaults is None: _defaults = {} else: _defaults = dict(list(zip(argspec.args[nargs:], argspec.defaults))) # docinfo = parse_docstring(fn) # if 'data' in docinfo['optional']: self.inputs.declare('data', doc='A container of labeled data.', optional=True) else: self.inputs.declare('data', doc='A container of labeled data.') for name in argspec.args[nargs:]: if name in docinfo['optional']: self.inputs.declare(name, optional=True, default=_defaults[name], doc=docinfo['optional'][name]) elif name in docinfo['required']: self.inputs.declare(name, doc=docinfo['required'][name]) elif name != 'data': #print docinfo logger.error("Argument '%s' is not specified in the docstring!" % name) # self.outputs.declare('data', doc='A container of labeled data.') if outputs is None: _outputs = list(docinfo['return'].keys()) else: _outputs = outputs for name in _outputs: if name in docinfo['return']: self.outputs.declare(name, doc=docinfo['return'][name]) else: logger.error("Return value '%s' is not specified in the docstring!" % name) # self._nested_requirements = [] for name in docinfo['required']: if '.' in name: self._nested_requirements.append(name) # # Error check keys for docinfo # for name in docinfo['required']: if '.' in name: continue if not name in self.inputs: logger.error("Unexpected name '%s' in list of required inputs for functor '%s'" % (name,_alias)) for name in docinfo['optional']: if not name in self.inputs: logger.error("Unexpected name '%s' in list of optional inputs for functor '%s'" % (name,_alias)) for name in docinfo['return']: if not name in self.outputs: logger.error("Unexpected name '%s' in list of outputs for functor '%s'" % (name,_alias)) # self.__help__ = fn.__doc__ self.__doc__ = fn.__doc__ self.__short_doc__ = docinfo['short_doc'].strip() self.__long_doc__ = docinfo['long_doc'].strip() self.__namespace__ = namespace
class JSONDictionary(Plugin): alias("json", "JSON file interface") implements(IDataManager, service=False) def __init__(self): self._info = {} self.options = Options() def available(self): return True def initialize(self, **kwds): self.filename = kwds.pop('filename') self.add_options(**kwds) def add_options(self, **kwds): self.options.update(kwds) def open(self): if self.filename is None: raise IOError("No filename specified") def close(self): pass def read(self): """ This function loads data from a JSON file and tuplizes the nested dictionaries and lists of lists. """ if not os.path.exists(self.filename): raise IOError("Cannot find file '%s'" % self.filename) INPUT = open(self.filename, 'r') jdata = json.load(INPUT) INPUT.close() if jdata is None or len(jdata) == 0: raise IOError("Empty JSON data file") self._info = {} for k, v in jdata.items(): self._info[k] = tuplize(v) def write(self, data): """ This function creates a JSON file for the specified data. """ with open(self.filename, 'w') as OUTPUT: jdata = {} if self.options.data is None: for k, v in data.items(): jdata[k] = detuplize(v) elif type(self.options.data) in (list, tuple): for k in self.options.data: jdata[k] = detuplize(data[k]) else: k = self.options.data jdata[k] = detuplize(data[k]) json.dump(jdata, OUTPUT) def process(self, model, data, default): """ Set the data for the selected components """ if not self.options.namespace in data: data[self.options.namespace] = {} # try: if self.options.data is None: for key in self._info: self._set_data(data, self.options.namespace, key, self._info[key]) elif type(self.options.data) in (list, tuple): for key in self.options.data: self._set_data(data, self.options.namespace, key, self._info[key]) else: key = self.options.data self._set_data(data, self.options.namespace, key, self._info[key]) except KeyError: raise IOError( "Data value for '%s' is not available in JSON file '%s'" % (key, self.filename)) def _set_data(self, data, namespace, name, value): if type(value) is dict: data[namespace][name] = value else: data[namespace][name] = {None: value} def clear(self): self._info = {}