def reset(self, ph):
self.incumbent = None
self.rho = None
self.x_deviation = None
self.lastConvergenceMetric = None
self.feasibility_cuts = []
self.incumbent_cuts = []
self.lastRun = 0
self.average_solution = None
self.converger = NormalizedTermDiffConvergence()
self.unique_scenario_solutions = []
def reset(self, ph):
self.incumbent = None
self.rho = None
self.x_deviation = None
self.lastConvergenceMetric = None
self.feasibility_cuts = []
self.incumbent_cuts = []
self.lastRun = 0
self.average_solution = None
self.converger = NormalizedTermDiffConvergence()
class InterScenarioPlugin(SingletonPlugin):
implements(phextension.IPHExtension)
def __init__(self):
self.enableRhoUpdates = True
self.enableFeasibilityCuts = True
self.enableIncumbentCuts = True
self.epsilon = 1e-7
self.cut_scale = 0#1e-4
self.allow_variable_slack = False
# Force this plugin to run every N iterations
self.iterationInterval = 100
# Alternative methods to trigger the plugin:
#
# If the convergence metric degrades by either a relative or
# absolute amount
self.convergenceRelativeDegredation = 10.33
self.convergenceAbsoluteDegredation = 10.001
# If at least recutThreshold fraction of all-to-all scenario
# tests produced feasibility cuts
self.recutThreshold = 0.33
# If at least this fraction of unique solutions are preserved
# from one iteration to the next
self.repeated_solution_threshhold = 0.90
# multiplier on computed rho values
self.rhoScale = 0.75
# How quickly rho moves to new values [0..1]
# 0: no damping (jump to calculated rho)
# 1: complete damping (do not change current value of rho)
self.rhoDamping = 0.1
# Minimum difference in objective to include a cut, and minimum
# difference in variable values to include that term in a cut
self.cutThreshold_minDiff = 0.0001
# Fraction of the cut library to use for cross-scenario
# (all-to-all) cuts
self.cutThreshold_crossCut = 0
# Force the InterScenario plugin to re-run while the improvement
# in the Lagrangean bound is at least this much:
self.iteration0RecutBoundImprovement = 0.0025
def reset(self, ph):
self.incumbent = None
self.rho = None
self.x_deviation = None
self.lastConvergenceMetric = None
self.feasibility_cuts = []
self.incumbent_cuts = []
self.lastRun = 0
self.average_solution = None
self.converger = NormalizedTermDiffConvergence()
self.unique_scenario_solutions = []
def pre_ph_initialization(self,ph):
self.reset(ph)
pass
def post_instance_creation(self,ph):
if self.enableRhoUpdates:
rootNode = ph._scenario_tree.findRootNode()
for v in rootNode._xbars:
ph.setRhoAllScenarios(rootNode, v, 0)
pass
def post_ph_initialization(self, ph):
if len(ph._scenario_tree._stages) > 2:
raise RuntimeError(
"InterScenario plugin only works with 2-stage problems" )
self._sense_to_min = 1 if ph._objective_sense == minimize else -1
# We are going to manage RHO here. So, we want to turn it off
# until we finish the initial round of interscenario feasibility
# cuts.
if self.enableRhoUpdates:
rootNode = ph._scenario_tree.findRootNode()
for v in rootNode._xbars:
ph.setRhoAllScenarios(rootNode, v, 0)
#self.rho = dict((v,ph._rho) for v in ph._scenario_tree.findRootNode()._xbars)
def post_iteration_0_solves(self, ph):
self._collect_unique_scenario_solutions(ph)
self._interscenario_plugin(ph)
count = 0
while self.rho is None and self.feasibility_cuts:
count += 1
toc( "InterScenario plugin: PH iteration 0 re-solve pass %s"
% (count,) )
_stale_scenarios = []
for _id, _soln in enumerate(self.unique_scenario_solutions):
_was_cut = sum(
1 for c in self.feasibility_cuts if type(c[_id]) is tuple)
if _was_cut:
_stale_scenarios.extend(_soln[1])
self._distribute_cuts(ph, True)
toc("InterScenario plugin: distributed cuts to scenarios")
self._collect_unique_scenario_solutions(ph)
self._interscenario_plugin(ph)
self.lastRun = 0
def post_iteration_0(self, ph):
self.converger.update( ph._current_iteration,
ph,
ph._scenario_tree,
ph._instances )
self.lastConvergenceMetric = self.converger.lastMetric()
pass
def pre_iteration_k_solves(self, ph):
if self.feasibility_cuts or self.incumbent_cuts:
self._distribute_cuts(ph)
pass
def post_iteration_k_solves(self, ph):
self.converger.update( ph._current_iteration,
ph,
ph._scenario_tree,
ph._instances )
curr = self.converger.lastMetric()
last = self.lastConvergenceMetric
delta = curr - last
#print("InterScenario convergence:", last, curr, delta)
run = False
if ( self._collect_unique_scenario_solutions(ph) >=
self.repeated_solution_threshhold ):
print("InterScenario plugin: triggered by no change in "
"scenario solutions")
run = True
if ( delta > last * self.convergenceRelativeDegredation and
delta > self.convergenceAbsoluteDegredation ):
print( "InterScenario plugin: triggered by convergence degredation "
"(%0.4f; %+0.4f)" % (curr, delta) )
run = True
if ph._current_iteration-self.lastRun >= self.iterationInterval:
print("InterScenario plugin: triggered by iteration limit")
run = True
if self.rho is None:
print( "InterScenario plugin: triggered to initialize rho")
run = True
elif self.enableRhoUpdates:
rootNode = ph._scenario_tree.findRootNode()
for _id, rho in iteritems(self.rho):
_max = rootNode._maximums[_id]
_min = rootNode._minimums[_id]
if rho < self.epsilon and _max - _min > self.epsilon:
print( "InterScenario plugin: triggered by variable "
"divergence with rho==0 (%s: %s; [%s, %s])"
% (_id, rho, _max, _min))
run = True
break
if run:
self.lastRun = ph._current_iteration
self._interscenario_plugin(ph)
self.lastConvergenceMetric = curr
pass
def post_iteration_k(self, ph):
pass
def post_ph_execution(self, ph):
self._collect_unique_scenario_solutions(ph)
self._interscenario_plugin(ph)
pass
def _interscenario_plugin(self,ph):
toc("InterScenario plugin: analyzing scenario dual information")
# (1) Collect all scenario (first) stage variables
#self._collect_unique_scenario_solutions(ph)
# (2) Filter them to find a set we want to distribute
pass
# (3) Distribute (some) of the variable sets out to the
# scenarios, fix, and resolve; Collect and return the
# objectives, duals, and any cuts
partial_obj_values, dual_values, cuts, probability \
= self._solve_interscenario_solutions( ph )
# Compute the non-anticipative objective values for each
# scenario solution
self.feasible_objectives = self._compute_objective(
partial_obj_values, probability )
for _id, soln in enumerate(self.unique_scenario_solutions):
_scenarios = [ph._scenario_tree.get_scenario(x) for x in soln[1]]
print(
" Solution %2d: generated %2d cuts, "
"cut by %2d other scenarios; objective %10s, "
"scenario cost [%s], cut obj [%s] [generated by %s]" % (
_id,
sum(1 for c in cuts[_id] if type(c) is tuple),
sum(1 for c in cuts if type(c[_id]) is tuple),
"None" if self.feasible_objectives[_id] is None
else "%10.2f" % self.feasible_objectives[_id],
", ".join("%10.2f" % x._cost for x in _scenarios),
" ".join("%5.2f" % x[0] if type(x) is tuple else "%5s" % x
for x in cuts[_id]),
','.join(soln[1])
))
scenarioCosts = [ ph._scenario_tree.get_scenario(x)._cost
for s in self.unique_scenario_solutions
for x in s[1] ]
scenarioProb = [ ph._scenario_tree.get_scenario(x)._probability
for s in self.unique_scenario_solutions
for x in s[1] ]
_avg = sum( scenarioProb[i]*c for i,c in enumerate(scenarioCosts) )
_max = max( scenarioCosts )
_min = min( scenarioCosts )
if self.average_solution is None:
_del_avg = None
_del_avg_str = "-----%"
else:
_prev = self.average_solution
_del_avg = (_avg-_prev) / max(abs(_avg),abs(_prev))
_del_avg_str = "%+.2f%%" % ( 100*_del_avg, )
self.average_solution = _avg
print(" Average scenario cost: %f (%s) Max-min: %f (%0.2f%%)" % (
_avg, _del_avg_str, _max-_min, abs(100.*(_max-_min)/_avg) ))
# (4) save any cuts for distribution before the next solve
#self.feasibility_cuts = []
#for c in cuts:
# self.feasibility_cuts.extend(
# x for x in c if type(x) is tuple and x[0] > self.cutThreshold )
#cutCount = len(self.feasibility_cuts)
if self.enableFeasibilityCuts:
self.feasibility_cuts = cuts
cutCount = sum( sum( 1 for x in c if type(x) is tuple
and x[0]>self.cutThreshold_minDiff )
for c in cuts )
subProblemCount = sum(len(c) for c in cuts)
# (5) compute and publish the new incumbent
self._update_incumbent(ph)
# (6a) If this is iteration 0, and we have feasibility cuts, and
# they are (sufficiently) helping the Lagrangean bound, then
# skip setting rho and do another round oc cuts
if ph._current_iteration == 0:
# Tell ph that we may have a good opter bound
ph._update_reported_bounds(outer=self.average_solution)
if ( cutCount > self.recutThreshold*(subProblemCount-len(cuts))
and ( _del_avg is None or
_del_avg > self.iteration0RecutBoundImprovement )):
# Bypass RHO updates and check for more cuts
#self.lastRun = ph._current_iteration - self.iterationInterval
return
# (6b) compute updated rho estimates
new_rho, loginfo = self._process_dual_information(
ph, dual_values, probability )
_scale = self.rhoScale
if self.rho is None:
print("InterScenario plugin: initializing rho")
self.rho = {}
for v,r in iteritems(new_rho):
self.rho[v] = _scale*r
else:
_damping = self.rhoDamping
for v,r in iteritems(new_rho):
if self.rho[v]:
self.rho[v] += (1-_damping)*(_scale*r - self.rho[v])
#self.rho[v] = max(_scale*r, self.rho[v]) - \
# _damping*abs(_scale*r - self.rho[v])
else:
self.rho[v] = _scale*r
for v,l in sorted(iteritems(loginfo)):
if v is None:
print(l)
else:
print(l % (self.rho[v],))
#print("SETTING SELF.RHO", self.rho)
rootNode = ph._scenario_tree.findRootNode()
if self.enableRhoUpdates:
for v, r in iteritems(self.rho):
ph.setRhoAllScenarios(rootNode, v, r)
def _collect_unique_scenario_solutions(self, ph):
# list of (varmap, scenario_list) tuples
_old_unique_scenario_solutions = self.unique_scenario_solutions
self.unique_scenario_solutions = []
# See ph.py:update_variable_statistics for a multistage version...
rootNode = ph._scenario_tree.findRootNode()
for scenario in rootNode._scenarios:
_this_sol = dict(scenario._x[rootNode._name])
for _id, _val in iteritems(scenario._x[rootNode._name]):
#if rootNode.is_variable_fixed(_id):
# continue
if rootNode.is_variable_binary(_id) or \
rootNode.is_variable_integer(_id):
_this_sol[_id] = int(round(_val))
found = False
# Note: because we are looking for unique variable values,
# then if the user is bundling, this will implicitly re-form
# the bundles
for _sol in self.unique_scenario_solutions:
if _this_sol == _sol[0]:
_sol[1].append(scenario._name)
found = True
break
if not found:
self.unique_scenario_solutions.append(
( _this_sol, [scenario._name] ) )
_unchanged = 0
for _old_soln, _old_scen in _old_unique_scenario_solutions:
for _soln, _scen in self.unique_scenario_solutions:
if _old_soln == _soln:
_unchanged += 1
break
print( "Interscenario plugin: %s unchanged scenario solutions "
"(out of %s)" %
( _unchanged, len(self.unique_scenario_solutions) ))
return float(_unchanged) / len(self.unique_scenario_solutions)
def _solve_interscenario_solutions(self, ph):
results = ([],[],[],)
probability = []
#cutlist = []
distributed = isinstance( ph._solver_manager, SolverManager_PHPyro )
action_handles = []
if ph._scenario_tree.contains_bundles():
subproblems = ph._scenario_tree._scenario_bundles
else:
subproblems = ph._scenario_tree._scenarios
for problem in subproblems:
probability.append(problem._probability)
options=( self.unique_scenario_solutions, )
kwd_options = {
'epsilon': self.epsilon,
'cut_scale': self.cut_scale,
'allow_slack': self.allow_variable_slack,
'enable_rho': self.enableRhoUpdates,
'enable_cuts': self.enableFeasibilityCuts
}
if distributed:
action_handles.append(
ph._solver_manager.queue(
action="invoke_external_function",
name=problem._name,
queue_name=ph._phpyro_job_worker_map[problem._name],
invocation_type=InvocationType.SingleInvocation.key,
generateResponse=True,
module_name='pyomo.pysp.plugins.interscenario',
function_name='solve_fixed_scenario_solutions',
function_kwds=kwd_options,
function_args=options,
) )
else:
_tmp = solve_fixed_scenario_solutions(
ph, ph._scenario_tree, problem,
*options, **kwd_options )
for i,r in enumerate(results):
r.append(_tmp[i])
#cutlist.extend(_tmp[-1])
if distributed:
num_results_so_far = 0
num_results = len(action_handles)
for r in results:
r.extend([None]*num_results)
while (num_results_so_far < num_results):
_ah = ph._solver_manager.wait_any()
_ah_id = action_handles.index(_ah)
_tmp = ph._solver_manager.get_results(_ah)
for i,r in enumerate(results):
r[_ah_id] = _tmp[i]
#cutlist.extend(_tmp[-1])
num_results_so_far += 1
return results + (probability,) # + (cutlist,)
def _distribute_cuts(self, ph, resolve=False):
totalCuts = 0
cutObj = sorted( c[0] for x in self.feasibility_cuts for c in x
if type(c) is tuple
and c[0] > self.cutThreshold_minDiff )
if cutObj:
allCutThreshold = cutObj[
min( int((1-self.cutThreshold_crossCut)*len(cutObj)),
len(cutObj)-1 ) ]
else:
allCutThreshold = 1
distributed = isinstance( ph._solver_manager, SolverManager_PHPyro )
if ph._scenario_tree.contains_bundles():
subproblems = ph._scenario_tree._scenario_bundles
get_scenarios = lambda x: x._scenario_names
else:
subproblems = ph._scenario_tree._scenarios
get_scenarios = lambda x: [x]
resolves = []
for problem in subproblems:
cuts = []
for id, (x, s) in enumerate(self.unique_scenario_solutions):
found = False
for scenario in get_scenarios(problem):
if scenario._name in s:
found = True
break
if found:
cuts.extend( c[id] for c in self.feasibility_cuts
if type(c[id]) is tuple
and c[id][0] > self.cutThreshold_minDiff )
elif self.feasible_objectives[id] is None:
# We only add cuts generated by other scenarios to
# scenarios that are not currently feasible (as
# these are feassibility cuts, they should not
# impact feasible scenarios)
cuts.extend( c[id] for c in self.feasibility_cuts
if type(c[id]) is tuple
and c[id][0] > allCutThreshold )
if not cuts and not self.incumbent_cuts:
resolves.append(None)
continue
totalCuts += len(cuts)
if distributed:
resolves.append(
ph._solver_manager.queue(
action="invoke_external_function",
name=problem._name,
queue_name=ph._phpyro_job_worker_map[problem._name],
invocation_type=InvocationType.SingleInvocation.key,
generateResponse=True,
module_name='pyomo.pysp.plugins.interscenario',
function_name='add_new_cuts',
function_kwds=None,
function_args=( cuts,
self.incumbent_cuts,
resolve ),
) )
else:
ans = add_new_cuts( ph, ph._scenario_tree, problem,
cuts, self.incumbent_cuts, resolve )
resolves.append(ans)
toc("distributed cuts to scenario %s%s" %
( problem._name,
' and resolved scenario' if resolve else '' ))
toc( "InterScenario plugin: added %d feasibility cuts from a "
"library of %s cuts" % (totalCuts, len(cutObj)) )
self.feasibility_cuts = []
if self.incumbent_cuts:
print( "InterScenario plugin: added %d incumbent cuts" %
(len(self.incumbent_cuts), ) )
self.incumbent_cuts = []
if distributed:
num_results_so_far = sum(1 for x in resolves if x is None)
num_results = len(resolves)
while (num_results_so_far < num_results):
_ah = ph._solver_manager.wait_any()
_ah_idx = resolves.index(_ah)
resolves[_ah_idx] = ph._solver_manager.get_results(_ah)
num_results_so_far += 1
if resolve:
# Transfer the first stage values and cost back to PH and
# recompute xbar
rootNode = ph._scenario_tree.findRootNode()
for _id, problem in enumerate(subproblems):
ans = resolves[_id]
if ans is None:
continue
for scenario in get_scenarios(problem):
scenario._cost = ans[1]
assert( sorted(ans[0]) ==
sorted(scenario._x[rootNode._name]) )
scenario._x[rootNode._name] = ans[0] #[_vid] = _vval
ph.update_variable_statistics()
def _compute_objective(self, partial_obj_values, probability):
obj_values = []
for soln_id in xrange(len( self.unique_scenario_solutions )):
obj = 0.
for scen_or_bundle_id, p in enumerate(probability):
if partial_obj_values[scen_or_bundle_id][soln_id] is None:
obj = None
break
obj += p * partial_obj_values[scen_or_bundle_id][soln_id]
obj_values.append(obj)
return obj_values
def _update_incumbent(self, ph):
feasible_obj = [ o for o in enumerate(self.feasible_objectives)
if o[1] is not None ]
if not feasible_obj:
print( "InterScenario plugin: No scenario solutions are "
"globally feasible" )
return
print( "InterScenario plugin: Feasible objectives: %s" %
( sorted(o[1] for o in feasible_obj), ) )
best_id, best_obj = min(
((x[0], self._sense_to_min*x[1]) for x in feasible_obj),
key=operator.itemgetter(1) )
binary_vars = []
integer_vars = []
continuous_vars = []
rootNode = ph._scenario_tree.findRootNode()
for _id in rootNode._scenarios[0]._x[rootNode._name]:
if rootNode.is_variable_fixed(_id):
continue
if rootNode.is_variable_binary(_id):
binary_vars.append(_id)
elif rootNode.is_variable_integer(_id):
integer_vars.append(_id)
elif rootNode.is_variable_semicontinuous(_id):
assert False, "FIXME"
else:
# we can not add incumbent cuts for continuous domains
continuous_vars.append(_id)
if self.incumbent is None or \
self.incumbent[0] * self._sense_to_min > best_obj + self.epsilon:
# Cut the old incumbent
if self.enableIncumbentCuts and self.incumbent and not continuous_vars:
_x = self.incumbent[1][0]
self.incumbent_cuts.append(
( dict((vid, round(_x[vid])) for vid in binary_vars),
dict((vid, round(_x[vid])) for vid in integer_vars),
) )
# New incumbent!
self.incumbent = ( best_obj*self._sense_to_min,
self.unique_scenario_solutions[best_id],
best_id )
# Tell PH (that we have a good inner bound)
ph._update_reported_bounds(inner=best_obj)
msg = "InterScenario plugin: NEW incumbent: %s = %s, %s" \
% self.incumbent
print(msg)
logger.info(msg)
elif self.incumbent[0]*self._sense_to_min < best_obj - self.epsilon:
# Keep existing incumbent... so the best thing here can be cut
msg = "InterScenario plugin: incumbent: %s = %s, %s" \
% self.incumbent
print(msg)
best_id = -1
if continuous_vars or not self.enableIncumbentCuts:
return
for _id, obj in feasible_obj:
if _id == best_id:
continue
_x = self.unique_scenario_solutions[_id][0]
self.incumbent_cuts.append(
( dict((vid, round(_x[vid])) for vid in binary_vars),
dict((vid, round(_x[vid])) for vid in integer_vars),
) )
def _process_dual_information(self, ph, dual_values, probability):
# Notes:
# dual_values: [ [ { var_id: dual } ] ]
# - list of list of maps of variable id to dual value. The
# outer list is returned by each subproblem (corresponds to
# a bundle or scenario). The order in this list matches
# the order in the probability list. The inner list holds
# the dual values for each solution the scenario/bundle was
# asked to evaluate. This inner list is in the same order
# as the solutions list.
# probability: [ scenario/bundle probility ]
# - list of the scenario or bundle probability for the
# submodel that returned the corresponding objective/dual
# values
# unique_scenario_solutions: [ {var_id:var_value}, [ scenario_names ] ]
# - list of candidate solutions holding the 1st stage
# variable values (in a map) and the list of scenarios
# that had that solution as the optimal solution in this
# iteration
# soln_prob: the total probability of all scenarios that have
# this solution as their locally-optimal solution
soln_prob = [0.] * len(self.unique_scenario_solutions)
for soln_id, soln_info in enumerate(self.unique_scenario_solutions):
for src_scen_name in soln_info[1]:
src_scen = ph._scenario_tree.get_scenario(src_scen_name)
soln_prob[soln_id] += src_scen._probability
total_soln_prob = sum(soln_prob)
# xbar: { var_id : xbar }
# - this has the average first stage variable values. We
# should really get this from the scenario tree, as we
# cannot guarantee that we will see all the current values
# here (they can be filtered)
#xbar = dict( (
# k,
# sum(v*soln_prob[i] for i,v in enumerate(vv))/total_soln_prob )
# for k, vv in iteritems(var_info) )
xbar = ph._scenario_tree.findRootNode()._xbars
if self.x_deviation is None:
self.x_deviation = dict(
( v,
max(s[0][v] for s in self.unique_scenario_solutions)
- min(s[0][v] for s in self.unique_scenario_solutions) )
for v in xbar )
max_dual = dict((v,0.) for v in xbar)
weighted_rho = dict((v,0.) for v in xbar)
for soln_id, soln_p in enumerate(soln_prob):
x = self.unique_scenario_solutions[soln_id][0]
avg_dual = dict((v,0.) for v in xbar)
p_total = 0.
for scen_id, p in enumerate(probability):
if dual_values[scen_id][soln_id] is None:
continue
for v,d in iteritems(dual_values[scen_id][soln_id]):
avg_dual[v] += math.copysign(d, xbar[v]-x[v]) * p
max_dual[v] = max(max_dual[v], abs(d))
p_total += p
if p_total:
for v in avg_dual:
avg_dual[v] /= p_total
#x_deviation = dict( (v, abs(xbar[v]-self.unique_scenario_solutions[soln_id][0][v]))
# for v in xbar )
for v,x_dev in iteritems(self.x_deviation):
weighted_rho[v] += soln_prob[soln_id]*avg_dual[v]/(x_dev+1.)
if False: # MAX dual (not average)
for v,x_dev in iteritems(self.x_deviation):
weighted_rho[v] += max_dual[v]/(x_dev+1.)
# var_info: { var_id : [ scenario values ] }
# - this has the list of all values for a single 1st stage
# variable, in the same order as the solutions list (and the
# soln_prob list)
var_info = {}
for soln_id, soln_info in enumerate(self.unique_scenario_solutions):
for k,v in iteritems(soln_info[0]):
try:
var_info[k].append(v)
except:
var_info[k] = [v]
dual_info = {}
for sid, scenario_results in enumerate(dual_values):
for solution in scenario_results:
if solution is None:
continue
for k,v in iteritems(solution):
try:
dual_info[k].append(v)
except:
dual_info[k] = [v]
# (optionally) scale the weighted rho
#for v in xbar:
# weighted_rho[v] = weighted_rho[v] / total_soln_prob
# Check for rho == 0
_min_rho = min(_rho for _rho in weighted_rho if _rho > 0)
for v in xbar:
if weighted_rho[v] <= 0:
# If there is variable disagreement, but no objective
# pressure to price the disagreement, the best thing we
# can do is guess and let later iterations sort it out.
#
#if max(var_info[v]) - min(var_info[v]) > 0:
# weighted_rho[v] = 1.
#
# Actually, we will just set all 0 rho values to the
# smallest non-zero dual
weighted_rho[v] = _min_rho
loginfo = {
None:
"%4s: %6s [%7s, %7s] %7s; "
"%6s [%6s, %6s] %6s; RHO %7s : %7s" % (
'---',
'Dual', 'min', 'max', 'stdev',
'Var', 'min', 'max', 'stdev',
'computed', 'final' )
}
for k, duals in iteritems(dual_info):
# DISABLE!
#break
d_min = min(duals)
d_max = max(duals)
_sum = sum(abs(x) for x in duals)
_sumsq = sum(x**2 for x in duals)
n = float(len(duals))
d_avg = _sum/n
d_stdev = math.sqrt(abs(_sumsq/n - d_avg**2))
x_min = min(var_info[k])
x_max = max(var_info[k])
_sum = sum(abs(x) for x in var_info[k])
_sumsq = sum(x**2 for x in var_info[k])
n = float(len(var_info[k]))
x_avg = _sum/n
x_stdev = math.sqrt(abs(_sumsq/n - x_avg**2 + 1e-6))
loginfo[k] = \
"%4d: %6.1f [%7.1f, %7.1f] %7.1f; " \
"%6.1f [%6.1f, %6.1f] %6.1f; RHO %7.2f : %%7.2f" % (
k,
d_avg, d_min, d_max, d_stdev,
x_avg, x_min, x_max, x_stdev,
weighted_rho[k] )
return weighted_rho, loginfo
class InterScenarioPlugin(SingletonPlugin):
implements(phextension.IPHExtension)
def __init__(self):
self.enableRhoUpdates = True
self.enableFeasibilityCuts = True
self.enableIncumbentCuts = True
self.epsilon = 1e-7
self.cut_scale = 0 #1e-4
self.allow_variable_slack = False
# Force this plugin to run every N iterations
self.iterationInterval = 100
# Alternative methods to trigger the plugin:
#
# If the convergence metric degrades by either a relative or
# absolute amount
self.convergenceRelativeDegredation = 10.33
self.convergenceAbsoluteDegredation = 10.001
# If at least recutThreshold fraction of all-to-all scenario
# tests produced feasibility cuts
self.recutThreshold = 0.33
# If at least this fraction of unique solutions are preserved
# from one iteration to the next
self.repeated_solution_threshhold = 0.90
# multiplier on computed rho values
self.rhoScale = 0.75
# How quickly rho moves to new values [0..1]
# 0: no damping (jump to calculated rho)
# 1: complete damping (do not change current value of rho)
self.rhoDamping = 0.1
# Minimum difference in objective to include a cut, and minimum
# difference in variable values to include that term in a cut
self.cutThreshold_minDiff = 0.0001
# Fraction of the cut library to use for cross-scenario
# (all-to-all) cuts
self.cutThreshold_crossCut = 0
# Force the InterScenario plugin to re-run while the improvement
# in the Lagrangean bound is at least this much:
self.iteration0RecutBoundImprovement = 0.0025
def reset(self, ph):
self.incumbent = None
self.rho = None
self.x_deviation = None
self.lastConvergenceMetric = None
self.feasibility_cuts = []
self.incumbent_cuts = []
self.lastRun = 0
self.average_solution = None
self.converger = NormalizedTermDiffConvergence()
self.unique_scenario_solutions = []
def pre_ph_initialization(self, ph):
self.reset(ph)
pass
def post_instance_creation(self, ph):
if self.enableRhoUpdates:
rootNode = ph._scenario_tree.findRootNode()
for v in rootNode._xbars:
ph.setRhoAllScenarios(rootNode, v, 0)
pass
def post_ph_initialization(self, ph):
if len(ph._scenario_tree._stages) > 2:
raise RuntimeError(
"InterScenario plugin only works with 2-stage problems")
self._sense_to_min = 1 if ph._objective_sense == minimize else -1
# We are going to manage RHO here. So, we want to turn it off
# until we finish the initial round of interscenario feasibility
# cuts.
if self.enableRhoUpdates:
rootNode = ph._scenario_tree.findRootNode()
for v in rootNode._xbars:
ph.setRhoAllScenarios(rootNode, v, 0)
#self.rho = dict((v,ph._rho) for v in ph._scenario_tree.findRootNode()._xbars)
def post_iteration_0_solves(self, ph):
self._collect_unique_scenario_solutions(ph)
self._interscenario_plugin(ph)
count = 0
while self.rho is None and self.feasibility_cuts:
count += 1
toc("InterScenario plugin: PH iteration 0 re-solve pass %s" %
(count, ))
_stale_scenarios = []
for _id, _soln in enumerate(self.unique_scenario_solutions):
_was_cut = sum(1 for c in self.feasibility_cuts
if type(c[_id]) is tuple)
if _was_cut:
_stale_scenarios.extend(_soln[1])
self._distribute_cuts(ph, True)
toc("InterScenario plugin: distributed cuts to scenarios")
self._collect_unique_scenario_solutions(ph)
self._interscenario_plugin(ph)
self.lastRun = 0
def post_iteration_0(self, ph):
self.converger.update(ph._current_iteration, ph, ph._scenario_tree,
ph._instances)
self.lastConvergenceMetric = self.converger.lastMetric()
pass
def pre_iteration_k_solves(self, ph):
if self.feasibility_cuts or self.incumbent_cuts:
self._distribute_cuts(ph)
pass
def post_iteration_k_solves(self, ph):
self.converger.update(ph._current_iteration, ph, ph._scenario_tree,
ph._instances)
curr = self.converger.lastMetric()
last = self.lastConvergenceMetric
delta = curr - last
#print("InterScenario convergence:", last, curr, delta)
run = False
if (self._collect_unique_scenario_solutions(ph) >=
self.repeated_solution_threshhold):
print("InterScenario plugin: triggered by no change in "
"scenario solutions")
run = True
if (delta > last * self.convergenceRelativeDegredation
and delta > self.convergenceAbsoluteDegredation):
print("InterScenario plugin: triggered by convergence degredation "
"(%0.4f; %+0.4f)" % (curr, delta))
run = True
if ph._current_iteration - self.lastRun >= self.iterationInterval:
print("InterScenario plugin: triggered by iteration limit")
run = True
if self.rho is None:
print("InterScenario plugin: triggered to initialize rho")
run = True
elif self.enableRhoUpdates:
rootNode = ph._scenario_tree.findRootNode()
for _id, rho in iteritems(self.rho):
_max = rootNode._maximums[_id]
_min = rootNode._minimums[_id]
if rho < self.epsilon and _max - _min > self.epsilon:
print("InterScenario plugin: triggered by variable "
"divergence with rho==0 (%s: %s; [%s, %s])" %
(_id, rho, _max, _min))
run = True
break
if run:
self.lastRun = ph._current_iteration
self._interscenario_plugin(ph)
self.lastConvergenceMetric = curr
pass
def post_iteration_k(self, ph):
pass
def post_ph_execution(self, ph):
self._collect_unique_scenario_solutions(ph)
self._interscenario_plugin(ph)
pass
def _interscenario_plugin(self, ph):
toc("InterScenario plugin: analyzing scenario dual information")
# (1) Collect all scenario (first) stage variables
#self._collect_unique_scenario_solutions(ph)
# (2) Filter them to find a set we want to distribute
pass
# (3) Distribute (some) of the variable sets out to the
# scenarios, fix, and resolve; Collect and return the
# objectives, duals, and any cuts
partial_obj_values, dual_values, cuts, probability \
= self._solve_interscenario_solutions( ph )
# Compute the non-anticipative objective values for each
# scenario solution
self.feasible_objectives = self._compute_objective(
partial_obj_values, probability)
for _id, soln in enumerate(self.unique_scenario_solutions):
_scenarios = [ph._scenario_tree.get_scenario(x) for x in soln[1]]
print(
" Solution %2d: generated %2d cuts, "
"cut by %2d other scenarios; objective %10s, "
"scenario cost [%s], cut obj [%s] [generated by %s]" %
(_id, sum(1 for c in cuts[_id] if type(c) is tuple),
sum(1 for c in cuts if type(c[_id]) is tuple), "None"
if self.feasible_objectives[_id] is None else "%10.2f" %
self.feasible_objectives[_id], ", ".join("%10.2f" % x._cost
for x in _scenarios),
" ".join("%5.2f" % x[0] if type(x) is tuple else "%5s" % x
for x in cuts[_id]), ','.join(soln[1])))
scenarioCosts = [
ph._scenario_tree.get_scenario(x)._cost
for s in self.unique_scenario_solutions for x in s[1]
]
scenarioProb = [
ph._scenario_tree.get_scenario(x)._probability
for s in self.unique_scenario_solutions for x in s[1]
]
_avg = sum(scenarioProb[i] * c for i, c in enumerate(scenarioCosts))
_max = max(scenarioCosts)
_min = min(scenarioCosts)
if self.average_solution is None:
_del_avg = None
_del_avg_str = "-----%"
else:
_prev = self.average_solution
_del_avg = (_avg - _prev) / max(abs(_avg), abs(_prev))
_del_avg_str = "%+.2f%%" % (100 * _del_avg, )
self.average_solution = _avg
print(" Average scenario cost: %f (%s) Max-min: %f (%0.2f%%)" %
(_avg, _del_avg_str, _max - _min, abs(100. *
(_max - _min) / _avg)))
# (4) save any cuts for distribution before the next solve
#self.feasibility_cuts = []
#for c in cuts:
# self.feasibility_cuts.extend(
# x for x in c if type(x) is tuple and x[0] > self.cutThreshold )
#cutCount = len(self.feasibility_cuts)
if self.enableFeasibilityCuts:
self.feasibility_cuts = cuts
cutCount = sum(
sum(1 for x in c
if type(x) is tuple and x[0] > self.cutThreshold_minDiff)
for c in cuts)
subProblemCount = sum(len(c) for c in cuts)
# (5) compute and publish the new incumbent
self._update_incumbent(ph)
# (6a) If this is iteration 0, and we have feasibility cuts, and
# they are (sufficiently) helping the Lagrangean bound, then
# skip setting rho and do another round oc cuts
if ph._current_iteration == 0:
# Tell ph that we may have a good opter bound
ph._update_reported_bounds(outer=self.average_solution)
if (cutCount > self.recutThreshold * (subProblemCount - len(cuts))
and (_del_avg is None
or _del_avg > self.iteration0RecutBoundImprovement)):
# Bypass RHO updates and check for more cuts
#self.lastRun = ph._current_iteration - self.iterationInterval
return
# (6b) compute updated rho estimates
new_rho, loginfo = self._process_dual_information(
ph, dual_values, probability)
_scale = self.rhoScale
if self.rho is None:
print("InterScenario plugin: initializing rho")
self.rho = {}
for v, r in iteritems(new_rho):
self.rho[v] = _scale * r
else:
_damping = self.rhoDamping
for v, r in iteritems(new_rho):
if self.rho[v]:
self.rho[v] += (1 - _damping) * (_scale * r - self.rho[v])
#self.rho[v] = max(_scale*r, self.rho[v]) - \
# _damping*abs(_scale*r - self.rho[v])
else:
self.rho[v] = _scale * r
for v, l in sorted(iteritems(loginfo)):
if v is None:
print(l)
else:
print(l % (self.rho[v], ))
#print("SETTING SELF.RHO", self.rho)
rootNode = ph._scenario_tree.findRootNode()
if self.enableRhoUpdates:
for v, r in iteritems(self.rho):
ph.setRhoAllScenarios(rootNode, v, r)
def _collect_unique_scenario_solutions(self, ph):
# list of (varmap, scenario_list) tuples
_old_unique_scenario_solutions = self.unique_scenario_solutions
self.unique_scenario_solutions = []
# See ph.py:update_variable_statistics for a multistage version...
rootNode = ph._scenario_tree.findRootNode()
for scenario in rootNode._scenarios:
_this_sol = dict(scenario._x[rootNode._name])
for _id, _val in iteritems(scenario._x[rootNode._name]):
#if rootNode.is_variable_fixed(_id):
# continue
if rootNode.is_variable_binary(_id) or \
rootNode.is_variable_integer(_id):
_this_sol[_id] = int(round(_val))
found = False
# Note: because we are looking for unique variable values,
# then if the user is bundling, this will implicitly re-form
# the bundles
for _sol in self.unique_scenario_solutions:
if _this_sol == _sol[0]:
_sol[1].append(scenario._name)
found = True
break
if not found:
self.unique_scenario_solutions.append(
(_this_sol, [scenario._name]))
_unchanged = 0
for _old_soln, _old_scen in _old_unique_scenario_solutions:
for _soln, _scen in self.unique_scenario_solutions:
if _old_soln == _soln:
_unchanged += 1
break
print("Interscenario plugin: %s unchanged scenario solutions "
"(out of %s)" %
(_unchanged, len(self.unique_scenario_solutions)))
return float(_unchanged) / len(self.unique_scenario_solutions)
def _solve_interscenario_solutions(self, ph):
results = (
[],
[],
[],
)
probability = []
#cutlist = []
distributed = isinstance(ph._solver_manager, SolverManager_PHPyro)
action_handles = []
if ph._scenario_tree.contains_bundles():
subproblems = ph._scenario_tree._scenario_bundles
else:
subproblems = ph._scenario_tree._scenarios
for problem in subproblems:
probability.append(problem._probability)
options = (self.unique_scenario_solutions, )
kwd_options = {
'epsilon': self.epsilon,
'cut_scale': self.cut_scale,
'allow_slack': self.allow_variable_slack,
'enable_rho': self.enableRhoUpdates,
'enable_cuts': self.enableFeasibilityCuts
}
if distributed:
action_handles.append(
ph._solver_manager.queue(
action="invoke_external_function",
name=problem._name,
queue_name=ph._phpyro_job_worker_map[problem._name],
invocation_type=InvocationType.SingleInvocation.key,
generateResponse=True,
module_name='pyomo.pysp.plugins.interscenario',
function_name='solve_fixed_scenario_solutions',
function_kwds=kwd_options,
function_args=options,
))
else:
_tmp = solve_fixed_scenario_solutions(ph, ph._scenario_tree,
problem, *options,
**kwd_options)
for i, r in enumerate(results):
r.append(_tmp[i])
#cutlist.extend(_tmp[-1])
if distributed:
num_results_so_far = 0
num_results = len(action_handles)
for r in results:
r.extend([None] * num_results)
while (num_results_so_far < num_results):
_ah = ph._solver_manager.wait_any()
_ah_id = action_handles.index(_ah)
_tmp = ph._solver_manager.get_results(_ah)
for i, r in enumerate(results):
r[_ah_id] = _tmp[i]
#cutlist.extend(_tmp[-1])
num_results_so_far += 1
return results + (probability, ) # + (cutlist,)
def _distribute_cuts(self, ph, resolve=False):
totalCuts = 0
cutObj = sorted(
c[0] for x in self.feasibility_cuts for c in x
if type(c) is tuple and c[0] > self.cutThreshold_minDiff)
if cutObj:
allCutThreshold = cutObj[min(
int((1 - self.cutThreshold_crossCut) * len(cutObj)),
len(cutObj) - 1)]
else:
allCutThreshold = 1
distributed = isinstance(ph._solver_manager, SolverManager_PHPyro)
if ph._scenario_tree.contains_bundles():
subproblems = ph._scenario_tree._scenario_bundles
get_scenarios = lambda x: x._scenario_names
else:
subproblems = ph._scenario_tree._scenarios
get_scenarios = lambda x: [x]
resolves = []
for problem in subproblems:
cuts = []
for id, (x, s) in enumerate(self.unique_scenario_solutions):
found = False
for scenario in get_scenarios(problem):
if scenario._name in s:
found = True
break
if found:
cuts.extend(c[id] for c in self.feasibility_cuts
if type(c[id]) is tuple
and c[id][0] > self.cutThreshold_minDiff)
elif self.feasible_objectives[id] is None:
# We only add cuts generated by other scenarios to
# scenarios that are not currently feasible (as
# these are feassibility cuts, they should not
# impact feasible scenarios)
cuts.extend(
c[id] for c in self.feasibility_cuts
if type(c[id]) is tuple and c[id][0] > allCutThreshold)
if not cuts and not self.incumbent_cuts:
resolves.append(None)
continue
totalCuts += len(cuts)
if distributed:
resolves.append(
ph._solver_manager.queue(
action="invoke_external_function",
name=problem._name,
queue_name=ph._phpyro_job_worker_map[problem._name],
invocation_type=InvocationType.SingleInvocation.key,
generateResponse=True,
module_name='pyomo.pysp.plugins.interscenario',
function_name='add_new_cuts',
function_kwds=None,
function_args=(cuts, self.incumbent_cuts, resolve),
))
else:
ans = add_new_cuts(ph, ph._scenario_tree, problem, cuts,
self.incumbent_cuts, resolve)
resolves.append(ans)
toc("distributed cuts to scenario %s%s" %
(problem._name,
' and resolved scenario' if resolve else ''))
toc("InterScenario plugin: added %d feasibility cuts from a "
"library of %s cuts" % (totalCuts, len(cutObj)))
self.feasibility_cuts = []
if self.incumbent_cuts:
print("InterScenario plugin: added %d incumbent cuts" %
(len(self.incumbent_cuts), ))
self.incumbent_cuts = []
if distributed:
num_results_so_far = sum(1 for x in resolves if x is None)
num_results = len(resolves)
while (num_results_so_far < num_results):
_ah = ph._solver_manager.wait_any()
_ah_idx = resolves.index(_ah)
resolves[_ah_idx] = ph._solver_manager.get_results(_ah)
num_results_so_far += 1
if resolve:
# Transfer the first stage values and cost back to PH and
# recompute xbar
rootNode = ph._scenario_tree.findRootNode()
for _id, problem in enumerate(subproblems):
ans = resolves[_id]
if ans is None:
continue
for scenario in get_scenarios(problem):
scenario._cost = ans[1]
assert (sorted(ans[0]) == sorted(
scenario._x[rootNode._name]))
scenario._x[rootNode._name] = ans[0] #[_vid] = _vval
ph.update_variable_statistics()
def _compute_objective(self, partial_obj_values, probability):
obj_values = []
for soln_id in xrange(len(self.unique_scenario_solutions)):
obj = 0.
for scen_or_bundle_id, p in enumerate(probability):
if partial_obj_values[scen_or_bundle_id][soln_id] is None:
obj = None
break
obj += p * partial_obj_values[scen_or_bundle_id][soln_id]
obj_values.append(obj)
return obj_values
def _update_incumbent(self, ph):
feasible_obj = [
o for o in enumerate(self.feasible_objectives) if o[1] is not None
]
if not feasible_obj:
print("InterScenario plugin: No scenario solutions are "
"globally feasible")
return
print("InterScenario plugin: Feasible objectives: %s" %
(sorted(o[1] for o in feasible_obj), ))
best_id, best_obj = min(
((x[0], self._sense_to_min * x[1]) for x in feasible_obj),
key=operator.itemgetter(1))
binary_vars = []
integer_vars = []
continuous_vars = []
rootNode = ph._scenario_tree.findRootNode()
for _id in rootNode._scenarios[0]._x[rootNode._name]:
if rootNode.is_variable_fixed(_id):
continue
if rootNode.is_variable_binary(_id):
binary_vars.append(_id)
elif rootNode.is_variable_integer(_id):
integer_vars.append(_id)
elif rootNode.is_variable_semicontinuous(_id):
assert False, "FIXME"
else:
# we can not add incumbent cuts for continuous domains
continuous_vars.append(_id)
if self.incumbent is None or \
self.incumbent[0] * self._sense_to_min > best_obj + self.epsilon:
# Cut the old incumbent
if self.enableIncumbentCuts and self.incumbent and not continuous_vars:
_x = self.incumbent[1][0]
self.incumbent_cuts.append((
dict((vid, round(_x[vid])) for vid in binary_vars),
dict((vid, round(_x[vid])) for vid in integer_vars),
))
# New incumbent!
self.incumbent = (best_obj * self._sense_to_min,
self.unique_scenario_solutions[best_id], best_id)
# Tell PH (that we have a good inner bound)
ph._update_reported_bounds(inner=best_obj)
msg = "InterScenario plugin: NEW incumbent: %s = %s, %s" \
% self.incumbent
print(msg)
logger.info(msg)
elif self.incumbent[0] * self._sense_to_min < best_obj - self.epsilon:
# Keep existing incumbent... so the best thing here can be cut
msg = "InterScenario plugin: incumbent: %s = %s, %s" \
% self.incumbent
print(msg)
best_id = -1
if continuous_vars or not self.enableIncumbentCuts:
return
for _id, obj in feasible_obj:
if _id == best_id:
continue
_x = self.unique_scenario_solutions[_id][0]
self.incumbent_cuts.append((
dict((vid, round(_x[vid])) for vid in binary_vars),
dict((vid, round(_x[vid])) for vid in integer_vars),
))
def _process_dual_information(self, ph, dual_values, probability):
# Notes:
# dual_values: [ [ { var_id: dual } ] ]
# - list of list of maps of variable id to dual value. The
# outer list is returned by each subproblem (corresponds to
# a bundle or scenario). The order in this list matches
# the order in the probability list. The inner list holds
# the dual values for each solution the scenario/bundle was
# asked to evaluate. This inner list is in the same order
# as the solutions list.
# probability: [ scenario/bundle probility ]
# - list of the scenario or bundle probability for the
# submodel that returned the corresponding objective/dual
# values
# unique_scenario_solutions: [ {var_id:var_value}, [ scenario_names ] ]
# - list of candidate solutions holding the 1st stage
# variable values (in a map) and the list of scenarios
# that had that solution as the optimal solution in this
# iteration
# soln_prob: the total probability of all scenarios that have
# this solution as their locally-optimal solution
soln_prob = [0.] * len(self.unique_scenario_solutions)
for soln_id, soln_info in enumerate(self.unique_scenario_solutions):
for src_scen_name in soln_info[1]:
src_scen = ph._scenario_tree.get_scenario(src_scen_name)
soln_prob[soln_id] += src_scen._probability
total_soln_prob = sum(soln_prob)
# xbar: { var_id : xbar }
# - this has the average first stage variable values. We
# should really get this from the scenario tree, as we
# cannot guarantee that we will see all the current values
# here (they can be filtered)
#xbar = dict( (
# k,
# sum(v*soln_prob[i] for i,v in enumerate(vv))/total_soln_prob )
# for k, vv in iteritems(var_info) )
xbar = ph._scenario_tree.findRootNode()._xbars
if self.x_deviation is None:
self.x_deviation = dict(
(v, max(s[0][v] for s in self.unique_scenario_solutions) -
min(s[0][v] for s in self.unique_scenario_solutions))
for v in xbar)
max_dual = dict((v, 0.) for v in xbar)
weighted_rho = dict((v, 0.) for v in xbar)
for soln_id, soln_p in enumerate(soln_prob):
x = self.unique_scenario_solutions[soln_id][0]
avg_dual = dict((v, 0.) for v in xbar)
p_total = 0.
for scen_id, p in enumerate(probability):
if dual_values[scen_id][soln_id] is None:
continue
for v, d in iteritems(dual_values[scen_id][soln_id]):
avg_dual[v] += math.copysign(d, xbar[v] - x[v]) * p
max_dual[v] = max(max_dual[v], abs(d))
p_total += p
if p_total:
for v in avg_dual:
avg_dual[v] /= p_total
#x_deviation = dict( (v, abs(xbar[v]-self.unique_scenario_solutions[soln_id][0][v]))
# for v in xbar )
for v, x_dev in iteritems(self.x_deviation):
weighted_rho[v] += soln_prob[soln_id] * avg_dual[v] / (x_dev +
1.)
if False: # MAX dual (not average)
for v, x_dev in iteritems(self.x_deviation):
weighted_rho[v] += max_dual[v] / (x_dev + 1.)
# var_info: { var_id : [ scenario values ] }
# - this has the list of all values for a single 1st stage
# variable, in the same order as the solutions list (and the
# soln_prob list)
var_info = {}
for soln_id, soln_info in enumerate(self.unique_scenario_solutions):
for k, v in iteritems(soln_info[0]):
try:
var_info[k].append(v)
except:
var_info[k] = [v]
dual_info = {}
for sid, scenario_results in enumerate(dual_values):
for solution in scenario_results:
if solution is None:
continue
for k, v in iteritems(solution):
try:
dual_info[k].append(v)
except:
dual_info[k] = [v]
# (optionally) scale the weighted rho
#for v in xbar:
# weighted_rho[v] = weighted_rho[v] / total_soln_prob
# Check for rho == 0
_min_rho = min(_rho for _rho in weighted_rho if _rho > 0)
for v in xbar:
if weighted_rho[v] <= 0:
# If there is variable disagreement, but no objective
# pressure to price the disagreement, the best thing we
# can do is guess and let later iterations sort it out.
#
#if max(var_info[v]) - min(var_info[v]) > 0:
# weighted_rho[v] = 1.
#
# Actually, we will just set all 0 rho values to the
# smallest non-zero dual
weighted_rho[v] = _min_rho
loginfo = {
None:
"%4s: %6s [%7s, %7s] %7s; "
"%6s [%6s, %6s] %6s; RHO %7s : %7s" %
('---', 'Dual', 'min', 'max', 'stdev', 'Var', 'min', 'max',
'stdev', 'computed', 'final')
}
for k, duals in iteritems(dual_info):
# DISABLE!
#break
d_min = min(duals)
d_max = max(duals)
_sum = sum(abs(x) for x in duals)
_sumsq = sum(x**2 for x in duals)
n = float(len(duals))
d_avg = _sum / n
d_stdev = math.sqrt(abs(_sumsq / n - d_avg**2))
x_min = min(var_info[k])
x_max = max(var_info[k])
_sum = sum(abs(x) for x in var_info[k])
_sumsq = sum(x**2 for x in var_info[k])
n = float(len(var_info[k]))
x_avg = _sum / n
x_stdev = math.sqrt(abs(_sumsq / n - x_avg**2 + 1e-6))
loginfo[k] = \
"%4d: %6.1f [%7.1f, %7.1f] %7.1f; " \
"%6.1f [%6.1f, %6.1f] %6.1f; RHO %7.2f : %%7.2f" % (
k,
d_avg, d_min, d_max, d_stdev,
x_avg, x_min, x_max, x_stdev,
weighted_rho[k] )
return weighted_rho, loginfo
class InterScenarioPlugin(SingletonPlugin):
implements(phextension.IPHExtension)
def __init__(self):
self.enableRhoUpdates = True
self.enableFeasibilityCuts = True
self.enableIncumbentCuts = True
self.epsilon = 1e-7
self.cut_scale = 0#1e-4
self.allow_variable_slack = False
self.convergenceRelativeDegredation = 0.33
self.convergenceAbsoluteDegredation = 0.001
# Force this plugin to run every N iterations
self.iterationInterval = 1
# multiplier on computed rho values
self.rhoScale = 0.75
# How quickly rho moves to new values [0..1]
# 0: no damping (jump to calculated rho)
# 1: complete damping (do not change current value of rho)
self.rhoDamping = 0.1
# Minimum difference in objective to include a cut, and minimum
# difference in variable values to include that term in a cut
self.cutThreshold_minDiff = 0.0001
# Fraction of the cut library to use for cross-scenario
# (all-to-all) cuts
self.cutThreshold_crossCut = 0
# Force the InterScenario plugin to re-run the next iteration if
# at least recutThreshold fraction of all-to-all scenario tests
# produced feasibility cuts
self.recutThreshold = 0.33
# Force the InterScenario plugin to re-run while the improvement
# in the Lagrangean bound is at least this much:
self.recutBoundImprovement = 0.0025
def reset(self, ph):
self.incumbent = None
self.rho = None
self.x_deviation = None
self.lastConvergenceMetric = None
self.feasibility_cuts = []
self.incumbent_cuts = []
self.lastRun = 0
self.average_solution = None
self.converger = NormalizedTermDiffConvergence()
def pre_ph_initialization(self,ph):
self.reset(ph)
pass
def post_instance_creation(self,ph):
if self.enableRhoUpdates:
rootNode = ph._scenario_tree.findRootNode()
for v in rootNode._xbars:
ph.setRhoAllScenarios(rootNode, v, 0)
pass
def post_ph_initialization(self, ph):
if len(ph._scenario_tree._stages) > 2:
raise RuntimeError(
"InterScenario plugin only works with 2-stage problems" )
self._sense_to_min = 1 if ph._objective_sense == minimize else -1
# We are going to manage RHO here. So, we want to turn it off
# until we finish the initial round of interscenario feasibility
# cuts.
if self.enableRhoUpdates:
rootNode = ph._scenario_tree.findRootNode()
for v in rootNode._xbars:
ph.setRhoAllScenarios(rootNode, v, 0)
#self.rho = dict((v,ph._rho) for v in ph._scenario_tree.findRootNode()._xbars)
def post_iteration_0_solves(self, ph):
self._interscenario_plugin(ph)
count = 0
while self.rho is None and self.feasibility_cuts:
count += 1
toc( "InterScenario plugin: PH iteration 0 re-solve pass %s"
% (count,) )
_stale_scenarios = []
for _id, _soln in enumerate(self.unique_scenario_solutions):
_was_cut = sum(
1 for c in self.feasibility_cuts if type(c[_id]) is tuple)
if _was_cut:
_stale_scenarios.extend(_soln[1])
self._distribute_cuts(ph, True)
toc("InterScenario: distributed cuts to scenarios")
self._interscenario_plugin(ph)
self.lastRun = 0
def post_iteration_0(self, ph):
self.converger.update( ph._current_iteration,
ph,
ph._scenario_tree,
ph._instances )
self.lastConvergenceMetric = self.converger.lastMetric()
pass
def pre_iteration_k_solves(self, ph):
if self.feasibility_cuts or self.incumbent_cuts:
self._distribute_cuts(ph)
pass
def post_iteration_k_solves(self, ph):
self.converger.update( ph._current_iteration,
ph,
ph._scenario_tree,
ph._instances )
curr = self.converger.lastMetric()
last = self.lastConvergenceMetric
delta = curr - last
#print("InterScenario convergence:", last, curr, delta)
run = False
if ( delta > last * self.convergenceRelativeDegredation and
delta > self.convergenceAbsoluteDegredation ):
print( "InterScenario plugin: triggered by convergence degredation "
"(%0.4f; %+0.4f)" % (curr, delta) )
run = True
if ph._current_iteration-self.lastRun >= self.iterationInterval:
print("InterScenario plugin: triggered by iteration limit")
run = True
if self.rho is None:
print( "InterScenario plugin: triggered to initialize rho")
run = True
elif self.enableRhoUpdates:
rootNode = ph._scenario_tree.findRootNode()
for _id, rho in iteritems(self.rho):
_max = rootNode._maximums[_id]
_min = rootNode._minimums[_id]
if rho < self.epsilon and _max - _min > self.epsilon:
print( "InterScenario plugin: triggered by variable "
"divergence with rho==0 (%s: %s; [%s, %s])"
% (_id, rho, _max, _min))
run = True
break
if run:
self.lastRun = ph._current_iteration
self._interscenario_plugin(ph)
self.lastConvergenceMetric = curr
pass
def post_iteration_k(self, ph):
pass
def post_ph_execution(self, ph):
self._interscenario_plugin(ph)
pass
def _interscenario_plugin(self,ph):
toc("InterScenario plugin: analyzing scenario dual information")
# (1) Collect all scenario (first) stage variables
self._collect_unique_scenario_solutions(ph)
# (2) Filter them to find a set we want to distribute
pass
# (3) Distribute (some) of the variable sets out to the
# scenarios, fix, and resolve; Collect and return the
# objectives, duals, and any cuts
partial_obj_values, dual_values, cuts, probability \
= self._solve_interscenario_solutions( ph )
# Compute the non-anticipative objective values for each
# scenario solution
self.feasible_objectives = self._compute_objective(
partial_obj_values, probability )
for _id, soln in enumerate(self.unique_scenario_solutions):
_scenarios = [ph._scenario_tree.get_scenario(x) for x in soln[1]]
print(
" Solution %2d: generated %2d cuts, "
"cut by %2d other scenarios; objective %10s, "
"scenario cost [%s], cut obj [%s] [generated by %s]" % (
_id,
sum(1 for c in cuts[_id] if type(c) is tuple),
sum(1 for c in cuts if type(c[_id]) is tuple),
"None" if self.feasible_objectives[_id] is None
else "%10.2f" % self.feasible_objectives[_id],
", ".join("%10.2f" % x._cost for x in _scenarios),
" ".join("%5.2f" % x[0] if type(x) is tuple else "%5s" % x
for x in cuts[_id]),
','.join(soln[1])
))
)
scenarioCosts = [ ph._scenario_tree.get_scenario(x)._cost
for s in self.unique_scenario_solutions
for x in s[1] ]
scenarioProb = [ ph._scenario_tree.get_scenario(x)._probability
for s in self.unique_scenario_solutions
for x in s[1] ]
_avg = sum( scenarioProb[i]*c for i,c in enumerate(scenarioCosts) )
_max = max( scenarioCosts )
_min = min( scenarioCosts )
if self.average_solution is None:
_del_avg = None
_del_avg_str = "-----%"
else:
_prev = self.average_solution
_del_avg = (_avg-_prev) / max(abs(_avg),abs(_prev))
_del_avg_str = "%+.2f%%" % ( 100*_del_avg, )
self.average_solution = _avg
print(" Average scenario cost: %f (%s) Max-min: %f (%0.2f%%)" % (
_avg, _del_avg_str, _max-_min, abs(100.*(_max-_min)/_avg) ))
# (4) save any cuts for distribution before the next solve
#self.feasibility_cuts = []
#for c in cuts:
# self.feasibility_cuts.extend(
# x for x in c if type(x) is tuple and x[0] > self.cutThreshold )
#cutCount = len(self.feasibility_cuts)
if self.enableFeasibilityCuts:
self.feasibility_cuts = cuts
cutCount = sum( sum( 1 for x in c if type(x) is tuple
and x[0]>self.cutThreshold_minDiff )
for c in cuts )
subProblemCount = sum(len(c) for c in cuts)
# (5) compute and publish the new incumbent
self._update_incumbent(ph)
# (6) set the new rho values
if ph._current_iteration == 0 and \
cutCount > self.recutThreshold*(subProblemCount-len(cuts)) and\
( _del_avg is None or _del_avg > self.recutBoundImprovement ):
# Bypass RHO updates and check for more cuts
#self.lastRun = ph._current_iteration - self.iterationInterval
return
# (7) compute updated rho estimates
new_rho, loginfo = self._process_dual_information(
ph, dual_values, probability )