def load_ph_warmstart(ph, scenariotree_solution): scenario_tree = ph._scenario_tree scenario_solutions = scenariotree_solution['scenario solutions'] for scenario in scenario_tree._scenarios: scenario_name = scenario._name scenario_sol = scenario_solutions[scenario_name] variable_sol = scenario_sol['variables'] for tree_node in scenario._node_list: isNotLeafNode = not tree_node.is_leaf_node() if isNotLeafNode: scenario._w[tree_node.name].clear() scenario_w = scenario._w[tree_node._name] for variable_id, (var_name, index) in \ iteritems(tree_node._variable_ids): name_label = str(var_name)+str(indexToString(index)) varsol = variable_sol[name_label] if variable_id in tree_node._standard_variable_ids: if 'weight' in varsol: scenario_w[variable_id] = varsol['weight'] node_solutions = scenariotree_solution['node solutions'] for stage in scenario_tree._stages[:-1]: for tree_node in stage._tree_nodes: variable_sol = node_solutions[tree_node._name]['variables'] for variable_id in tree_node._standard_variable_ids: var_name, index = tree_node._variable_ids[variable_id] sol = variable_sol[str(var_name)+str(indexToString(index))] tree_node._xbars[variable_id] = sol['xbar']
def _w_printing(self, ofile, ph=None): # print the w values in a useful way to the open file ofile # if ph is None, just write the header if ph is None: ofile.write("iteration; tree node; scenario; variable; ID; W\n") else: root_node_name = ph._scenario_tree.findRootNode()._name for stage, tree_node, variable_id, variable_values, is_fixed, is_stale in \ scenario_tree_node_variables_generator_noinstances(ph._scenario_tree, includeDerivedVariables=False, includeLastStage=False): if is_stale is False \ and (OnlyRootNode is False or tree_node._name == root_node_name): for scenario in tree_node._scenarios: scen_name = scenario._name weight_value = scenario._w[ tree_node._name][variable_id] variable_name, index = tree_node._variable_ids[ variable_id] full_variable_name = variable_name + indexToString( index) ofile.write( str(ph._current_iteration) + ';' + tree_node._name + ';' + scen_name + ';' + full_variable_name + ';' + str(variable_id) + ';' + str(weight_value) + '\n')
def _inspect_variable_convergence(self, ph, ph_iter): # collect termdiff by node and variable so we can # report and possibly sort term_diff = dict((tree_node._name, {}) \ for stage in ph._scenario_tree._stages[:-1] for tree_node in stage._tree_nodes) # Track these for reporting purposes node_fixed_cnt = dict((tree_node._name, 0) \ for stage in ph._scenario_tree._stages[:-1] for tree_node in stage._tree_nodes) total_fixed_cnt = 0 for stage, tree_node, variable_id, variable_values, is_fixed, is_stale \ in scenario_tree_node_variables_generator_noinstances( ph._scenario_tree, includeDerivedVariables=False, includeLastStage=False): if is_fixed: node_fixed_cnt[tree_node._name] += 1 total_fixed_cnt += 1 # Depending on preprocessing options, stale may indicate # fixed or unused in the model, either way we can skip it if (not is_stale): var_node_avg = 0.0 for var_value, scenario_probability in variable_values: var_node_avg += scenario_probability * var_value var_term_diff = 0.0 for var_value, scenario_probability in variable_values: var_term_diff += \ scenario_probability * \ fabs(var_value - var_node_avg) term_diff[tree_node._name][variable_id] = var_term_diff # Print individual variable term diffs by node # and sorted highest to lowest # skip the leaf stage ofile = open(self.wonly_file, 'a') for stage in ph._scenario_tree._stages[:-1]: for tree_node in stage._tree_nodes: for variable_id, var_term_diff in sorted( term_diff[tree_node._name].items(), key=itemgetter(1), reverse=True): variable_name, index = tree_node._variable_ids[variable_id] ofile.write( str(ph_iter) + "; " + tree_node._name + "; " + variable_name + indexToString(index) + "; " + str(var_term_diff) + '\n') ofile.close()
def initialize_algorithm_data(self, rho_init=1.0, y_init=0.0, z_init=0.0): # used to check dual-feasibility of initial y y_sum = {} for stage in self._manager.scenario_tree.stages[:-1]: for tree_node in stage.nodes: y_sum_node = y_sum[tree_node.name] = \ dict((id_, 0.0) for id_ in tree_node._standard_variable_ids) x = {} y = {} for scenario in self._manager.scenario_tree.scenarios: x_scenario = x[scenario.name] = {} y_scenario = y[scenario.name] = {} for tree_node in scenario.node_list[:-1]: assert not tree_node.is_leaf_node() x_node = x_scenario[tree_node.name] = {} y_node = y_scenario[tree_node.name] = {} y_sum_node = y_sum[tree_node.name] for id_ in tree_node._standard_variable_ids: x_node[id_] = None if type(y_init) is dict: y_node[id_] = y_init[scenario.name][ tree_node.name][id_] else: y_node[id_] = y_init y_sum_node[id_] += y_node[id_] # check dual-feasibility of y for stage in self._manager.scenario_tree.stages[:-1]: for tree_node in stage.nodes: y_sum_node = y_sum[tree_node.name] for id_ in tree_node._standard_variable_ids: if abs(y_sum_node[id_]) > 1e-6: name, index = tree_node._variable_ids[id_] raise ValueError( "Initial lagrange multipler estimates for non-" "anticipative variable %s do not sum to zero: %s" % (name + indexToString(index), repr( y_sum_node[id_]))) rho = {} z = {} for stage in self._manager.scenario_tree.stages[:-1]: for tree_node in stage.nodes: z_node = z[tree_node.name] = {} rho_node = rho[tree_node.name] = {} for id_ in tree_node._standard_variable_ids: if type(rho_init) is dict: rho_node[id_] = rho_init[tree_node.name][id_] else: rho_node[id_] = rho_init if type(z_init) is dict: z_node[id_] = z_init[tree_node.name][id_] else: z_node[id_] = z_init return rho, x, y, z
def evaluate_current_node_solution(sp, sp_solver, **solve_kwds): scenario_tree = sp.scenario_tree # Save the current fixed state and fix queue, then clear the fix queue fixed = {} fix_queue = {} for tree_node in scenario_tree.nodes: fixed[tree_node.name] = copy.deepcopy(tree_node._fixed) fix_queue[tree_node.name] = copy.deepcopy(tree_node._fix_queue) tree_node.clear_fix_queue() # Fix all non-anticipative variables to their # current value in the node solution for stage in scenario_tree.stages[:-1]: for tree_node in stage.nodes: for variable_id in tree_node._standard_variable_ids: if variable_id in tree_node._solution: tree_node.fix_variable(variable_id, tree_node._solution[variable_id]) else: from pysp.phutils import indexToString name, index = tree_node._variable_ids[variable_id] raise ValueError( "Scenario tree variable with name %s (scenario_tree_id=%s) " "does not have a solution stored on scenario tree node %s. " "Unable to evaluate solution." % (name + indexToString(index), variable_id, tree_node.name)) # Push fixed variable statuses on instances (or # transmit to the phsolverservers) sp.push_fix_queue_to_instances() failures = sp_solver.solve_subproblems(**solve_kwds) # Free all non-anticipative variables for stage in scenario_tree._stages[:-1]: for tree_node in stage.nodes: for variable_id in tree_node._standard_variable_ids: tree_node.free_variable(variable_id) # Refix all previously fixed variables for tree_node in scenario_tree.nodes: node_fixed = fixed[tree_node.name] for variable_id in node_fixed: tree_node.fix_variable(variable_id, node_fixed[variable_id]) sp.push_fix_queue_to_instances() # Restore the fix_queue for tree_node in scenario_tree.nodes: tree_node._fix_queue.update(fix_queue[tree_node.name]) return failures
def extract_scenario_solutions(scenario_tree, include_ph_objective_parameters=False, include_leaf_stage_vars=True): scenario_solutions = {} for scenario in scenario_tree._scenarios: scenario_name = scenario._name scenario_sol = scenario_solutions[scenario_name] = {} variable_sol = scenario_sol['variables'] = {} for tree_node in scenario._node_list: isNotLeafNode = not tree_node.is_leaf_node() if isNotLeafNode or include_leaf_stage_vars: if isNotLeafNode and include_ph_objective_parameters: weight_values = scenario._w[tree_node._name] rho_values = scenario._rho[tree_node._name] x_values = scenario._x[tree_node._name] for variable_id, (var_name, index) in \ iteritems(tree_node._variable_ids): name_label = str(var_name)+str(indexToString(index)) varsol = variable_sol[name_label] = {} varsol['value'] = x_values.get(variable_id) varsol['fixed'] = scenario.is_variable_fixed(tree_node, variable_id) varsol['stale'] = scenario.is_variable_stale(tree_node, variable_id) if include_ph_objective_parameters: if isNotLeafNode and \ (variable_id in tree_node._standard_variable_ids): varsol['rho'] = rho_values[variable_id] \ if (isNotLeafNode) \ else None varsol['weight'] = weight_values[variable_id] \ if (isNotLeafNode) \ else None else: varsol['rho'] = None varsol['weight'] = None scenario_sol['objective'] = scenario._objective scenario_sol['cost'] = scenario._cost if include_ph_objective_parameters: scenario_sol['ph weight term'] = scenario._weight_term_cost scenario_sol['ph proximal term'] = scenario._proximal_term_cost scenario_sol['stage costs'] = copy.deepcopy(scenario._stage_costs) return scenario_solutions
def extract_node_solutions(scenario_tree, include_ph_objective_parameters=False, include_variable_statistics=False, include_leaf_stage_vars=True): scenario_tree.snapshotSolutionFromScenarios() node_solutions = {} stages = None if include_leaf_stage_vars: stages = scenario_tree._stages else: stages = scenario_tree._stages[:-1] for stage in stages: for tree_node in stage._tree_nodes: isNotLeafNode = not tree_node.is_leaf_node() node_sol = node_solutions[tree_node._name] = {} variable_sol = node_sol['variables'] = {} for variable_id, (var_name, index) in \ iteritems(tree_node._variable_ids): name_label = str(var_name)+str(indexToString(index)) sol = variable_sol[name_label] = {} sol['solution'] = tree_node._solution[variable_id] sol['fixed'] = tree_node.is_variable_fixed(variable_id) sol['derived'] = \ bool(variable_id in tree_node._derived_variable_ids) if include_variable_statistics: if isNotLeafNode: sol['minimum'] = tree_node._minimums[variable_id] sol['average'] = tree_node._averages[variable_id] sol['maximum'] = tree_node._maximums[variable_id] else: sol['minimum'] = None sol['average'] = None sol['maximum'] = None if include_ph_objective_parameters: if isNotLeafNode and \ (variable_id in tree_node._standard_variable_ids): sol['xbar'] = tree_node._xbars[variable_id] sol['wbar'] = tree_node._wbars[variable_id] else: sol['xbar'] = None sol['wbar'] = None node_sol['expected cost'] = tree_node.computeExpectedNodeCost() return node_solutions
def extract_node_solution(tree_node): solution = {} for variable_id in tree_node._standard_variable_ids: varname, index = tree_node._variable_ids[variable_id] # store variable solution data as a list of (index, value) # tuples We avoid nesting another dictionary mapping index -> # value because (a) its cheaper and more lightweight as a list # and (b) because json serializes all dictionary keys as # strings (meaning an index of None is not recoverable) if varname not in solution: solution[varname] = [] if variable_id in tree_node._solution: solution[varname].append((index, tree_node._solution[variable_id])) else: name, index = tree_node._variable_ids[variable_id] full_name = name + indexToString(index) print("%s: node solution missing for variable with scenario tree " "id %s (%s)" % (tree_node.name, variable_id, full_name)) return None for varname in list(solution.keys()): solution[varname] = sorted(solution[varname], key=lambda x: x[0]) return solution
def update_rho(self, manager, x, y, z, rho): first_line = ("Updating Rho Values:\n%21s %25s %16s %16s %16s" % ("Action", "Variable", "Primal Residual", "Dual Residual", "New Rho")) first_print = True self.compute_primal_residual_norm(manager, x, z) self.compute_dual_residual_norm(manager, z, rho) verbose = self.get_option("verbose") for stage in manager.scenario_tree.stages[:-1]: for tree_node in stage.nodes: prnorm_node = self._primal_residual_norm[tree_node.name] drnorm_node = self._dual_residual_norm[tree_node.name] rho_node = rho[tree_node.name] for id_ in tree_node._standard_variable_ids: name, index = tree_node._variable_ids[id_] prnorm_var = prnorm_node[id_] drnorm_var = drnorm_node[id_] action = None if (prnorm_var > self._mu * drnorm_var) and \ (prnorm_var > self._tol): rho_node[id_] *= self._rho_increase_factor action = "Increasing" elif (drnorm_var > self._mu * prnorm_var) and \ (drnorm_var > self._tol): rho_node[id_] *= self._rho_decrease_factor action = "Decreasing" if verbose: if action is not None: if first_print: first_print = False print(first_line) print("%21s %25s %16g %16g %16g" % (action, name + indexToString(index), prnorm_var, drnorm_var, rho_node[id_])) self.snapshot_z(manager, z)
def ExtractInternalNodeSolutionsWithSlamming(ph): from pysp.plugins.wwphextension import _parse_yaml_file # Since it was a file, # assume that the argument was a json file with slamming instructions. # This will ignore suffixes we don't care about. # If there are no instructions use xbar. # Note: there is an implicit pecking order. print("For x-hat, using slamming suffixes in", ph._xhat_method) slamdict = {} for suffix_name, suffix_value, variable_ids in \ _parse_yaml_file(ph, ph._xhat_method): for node_name, node_variable_ids in iteritems(variable_ids): for variable_id in node_variable_ids: if variable_id not in slamdict: slamdict[variable_id] = {} slamdict[variable_id][suffix_name] = suffix_value verbose = ph._verbose node_solutions = {} for stage in ph._scenario_tree._stages[:-1]: for tree_node in stage._tree_nodes: this_node_sol = node_solutions[tree_node._name] = {} xbars = tree_node._xbars mins = tree_node._minimums maxs = tree_node._maximums warnb = False # did the user do something less than cool? for variable_id in tree_node._standard_variable_ids: if verbose: variable_name, index = tree_node._variable_ids[variable_id] full_variable_name = variable_name + indexToString(index) print("Setting x-hat for", full_variable_name) if variable_id not in slamdict or slamdict[variable_id][ 'CanSlamToAnywhere']: if not tree_node.is_variable_discrete(variable_id): this_node_sol[variable_id] = xbars[variable_id] if verbose: print(" x-bar", this_node_sol[variable_id]) else: this_node_sol[variable_id] = int( round(xbars[variable_id])) if verbose: print(" rounded x-bar", this_node_sol[variable_id]) elif slamdict[variable_id]['CanSlamToMin']: this_node_sol[variable_id] = mins[variable_id] if verbose: print(" min over scenarios", this_node_sol[variable_id]) elif slamdict[variable_id]['CanSlamToMax']: this_node_sol[variable_id] = maxs[variable_id] if verbose: print(" max over scenarios", this_node_sol[variable_id]) elif slamdict[variable_id]['CanSlamToLB']: warnb = True this_node_sol[variable_id] = mins[variable_id] if verbose: print(" Lower Bound", this_node_sol[variable_id]) elif slamdict[variable_id]['CanSlamToUB']: warnb = True this_node_sol[variable_id] = maxs[variable_id] if verbose: print(" Upper Bound", this_node_sol[variable_id]) if warnb: print( "Warning: for xhat determination from file %s, some variables had an upper or lower bound slam but not a corresponding min or max", ph._xhat_method) return node_solutions
def Compute_and_Write_it_all(self, W_Traces, ph): VarsOfInterest = set() fname = self.wsummary_filename print("sorgw.py is writing the semi-colon separated values file " + fname) ofile = open(fname, "w") ofile.write( "var; scen; WZeroCrossing; DiffsRatio; DiffZeroCrossings; w values...\n" ) for varid in W_Traces: assert (OnlyRootNode) variable_name, index = ph._scenario_tree.findRootNode( )._variable_ids[varid] varname = variable_name + indexToString(index) for scenname in W_Traces[varid]: WZeroCrossings, DiffsRatio, DiffZeroCrossings = self.Score_a_Trace( W_Traces[varid][scenname]) if self.Of_Interest(WZeroCrossings, DiffsRatio, DiffZeroCrossings): VarsOfInterest.add(varid) ofile.write(varname + ';' + scenname + ';' + str(WZeroCrossings) + ';' + str(DiffsRatio) + ';' + str(DiffZeroCrossings)) for w in W_Traces[varid][scenname]: ofile.write(';' + str(w)) ofile.write('\n') ofile.close # now processing interesting vars BiggestLoser = None LoserRange = 0 fname = self.winterest_filename print("sorgw.py is writing the semi-colon separated values file " + fname) ofile = open(fname, "w") ofile.write( "var; scen; WZeroCrossing; DiffsRatio; DiffZeroCrossings; w values...\n" ) for varid in VarsOfInterest: vwmax = vwmin = 0 assert (OnlyRootNode) variable_name, index = ph._scenario_tree.findRootNode( )._variable_ids[varid] varname = variable_name + indexToString(index) for scenname in W_Traces[varid]: WZeroCrossings, DiffsRatio, DiffZeroCrossings = self.Score_a_Trace( W_Traces[varid][scenname]) ofile.write(varname + ';' + scenname + ';' + str(WZeroCrossings) + ';' + str(DiffsRatio) + ';' + str(DiffZeroCrossings)) for w in W_Traces[varid][scenname]: ofile.write(';' + str(w)) if w > vwmax: vwmax = w if w < vwmin: vwmin = w ofile.write('\n') vwrange = vwmax - vwmin if vwrange > LoserRange: LoserRange = vwrange BiggestLoser = varid ofile.close if BiggestLoser is not None: ph._sorgw_BiggestLoser = BiggestLoser print("sorgw.py complete: RootNodeOnly=" + str(OnlyRootNode) + " BiggestLoser=" + str(BiggestLoser) + " with vwrange=" + str(vwrange))
def form_linearized_objective_constraints(instance_name, instance, scenario_tree, linearize_nonbinary_penalty_terms, breakpoint_strategy, tolerance): # keep track and return what was added to the instance, so # it can be cleaned up if necessary. new_instance_attributes = [] linearization_index_set_name = "PH_LINEARIZATION_INDEX_SET" linearization_index_set = instance.find_component( linearization_index_set_name) if linearization_index_set is None: linearization_index_set = Set(initialize=range( 0, linearize_nonbinary_penalty_terms * 2), dimen=1, name=linearization_index_set_name) instance.add_component(linearization_index_set_name, linearization_index_set) scenario = scenario_tree.get_scenario(instance_name) nodeid_to_vardata_map = instance._ScenarioTreeSymbolMap.bySymbol for tree_node in scenario._node_list[:-1]: xbar_dict = tree_node._xbars # if linearizing, then we have previously defined a variable # associated with the result of the linearized approximation # of the penalty term - this is simply added to the objective # function. linearized_cost_variable_name = "PHQUADPENALTY_" + str(tree_node._name) linearized_cost_variable = instance.find_component( linearized_cost_variable_name) # grab the linearization constraint associated with the # linearized cost variable, if it exists. otherwise, create it # - but an empty variety. the constraints are stage-specific - # we could index by constraint, but we don't know if that is # really worth the additional effort. linearization_constraint_name = "PH_LINEARIZATION_" + str( tree_node._name) linearization_constraint = instance.find_component( linearization_constraint_name) if linearization_constraint is not None: # clear whatever constraint components are there - there # may be fewer breakpoints, due to tolerances, and we # don't want to the old pieces laying around. linearization_constraint.clear() else: # this is the first time the constraint is being added - # add it to the list of PH-specific constraints for this # instance. new_instance_attributes.append(linearization_constraint_name) nodal_index_set_name = "PHINDEX_" + str(tree_node._name) nodal_index_set = instance.find_component(nodal_index_set_name) assert nodal_index_set is not None linearization_constraint = \ Constraint(nodal_index_set, linearization_index_set, name=linearization_constraint_name) linearization_constraint.construct() instance.add_component(linearization_constraint_name, linearization_constraint) for variable_id in tree_node._variable_ids: # don't add weight terms for derived variables at the tree # node. if variable_id in tree_node._derived_variable_ids: continue if variable_id not in tree_node._minimums: variable_name, index = tree_node._variable_ids[variable_id] raise RuntimeError( "No minimum value statistic found for variable=%s " "on tree node=%s; cannot form linearized PH objective" % (variable_name + indexToString(index), tree_node._name)) if variable_id not in tree_node._maximums: variable_name, index = tree_node._variable_ids[variable_id] raise RuntimeError( "No maximums value statistic found for " "variable=%s on tree node=%s; cannot " "form linearized PH objective" % (variable_name + indexToString(index), tree_node._name)) xbar = xbar_dict[variable_id] node_min = tree_node._minimums[variable_id] node_max = tree_node._maximums[variable_id] instance_vardata = nodeid_to_vardata_map[variable_id] if (instance_vardata.stale is False) and (instance_vardata.fixed is False): # binaries have already been dealt with in the process of PH objective function formation. if isinstance(instance_vardata.domain, BooleanSet) is False: x = instance_vardata if x.lb is None or x.ub is None: msg = "Missing bound for variable '%s'\n" \ 'Both lower and upper bounds required when' \ ' piece-wise approximating quadratic ' \ 'penalty terms' raise ValueError(msg % instance_vardata.name) lb = value(x.lb) ub = value(x.ub) # compute the breakpoint sequence according to the specified strategy. try: strategy = ( compute_uniform_breakpoints, compute_uniform_between_nodestat_breakpoints, compute_uniform_between_woodruff_breakpoints, compute_exponential_from_mean_breakpoints, )[breakpoint_strategy] args = ( lb, node_min, xbar, node_max, ub, \ linearize_nonbinary_penalty_terms, tolerance ) breakpoints = strategy(*args) except ValueError: e = sys.exc_info()[1] msg = 'A breakpoint distribution strategy (%s) ' \ 'is currently not supported within PH!' raise ValueError(msg % breakpoint_strategy) for i in xrange(len(breakpoints) - 1): this_lb = breakpoints[i] this_ub = breakpoints[i + 1] segment_tuple = create_piecewise_constraint_tuple( this_lb, this_ub, x, xbar, linearized_cost_variable[variable_id], tolerance) linearization_constraint.add((variable_id, i), segment_tuple) return new_instance_attributes
def pre_iteration_k_solves(self, ph): if (ph._current_iteration > self._stop_iter_rho_update) and \ all(not _converger.isConverged(ph) for _converger in ph._convergers): return converged = any( _converger.isConverged(ph) for _converger in ph._convergers) rho_updated = False adjust_rho = 0 if self._prev_avg is None: self._snapshot_avg(ph) else: self._primal_residual_history.append( self._compute_primal_residual_norm(ph)) self._dual_residual_history.append( self._compute_dual_residual_norm(ph)) self._snapshot_avg(ph) first_line = ("Updating Rho Values:\n%21s %25s %16s %16s %16s" % ("Action", "Variable", "Primal Residual", "Dual Residual", "New Rho")) first = True for stage in ph._scenario_tree._stages[:-1]: for tree_node in stage._tree_nodes: primal_resid = \ math.sqrt(sum(self._primal_residual_history[-1]\ [tree_node._name].values())) dual_resid = \ math.sqrt(sum(self._dual_residual_history[-1]\ [tree_node._name].values())) for variable_id in tree_node._standard_variable_ids: name, index = tree_node._variable_ids[variable_id] primal_resid = \ math.sqrt(self._primal_residual_history[-1]\ [tree_node._name][variable_id]) dual_resid = \ math.sqrt(self._dual_residual_history[-1]\ [tree_node._name][variable_id]) action = None rho = tree_node._scenarios[0]._rho[ tree_node._name][variable_id] if (primal_resid > 10 * dual_resid) and (primal_resid > self._tol): rho *= self._rho_increase action = "Increasing" elif ((dual_resid > 10 * primal_resid) and (dual_resid > self._tol)): if self._converged_count >= self._required_converged_before_decrease: rho /= self._rho_decrease action = "Decreasing" elif converged: rho /= self._rho_feasible_decrease action = "Feasible, Decreasing" elif (primal_resid < self._tol) and (dual_resid < self._tol): rho /= self._rho_converged_residual_decrease action = "Converged, Decreasing" if action is not None: if first: first = False print(first_line) print("%21s %25s %16g %16g %16g" % (action, name + indexToString(index), primal_resid, dual_resid, rho)) for scenario in tree_node._scenarios: scenario._rho[ tree_node._name][variable_id] = rho self._rho_norm_history.append(self._compute_rho_norm(ph)) if rho_updated: print("log(|rho|) = " + repr(math.log(self._rho_norm_history[-1])))