def post_iteration_0_solves(self, ph):
self._interscenario_plugin(ph)
count = 0
while self.rho is None and self.feasibility_cuts:
count += 1
toc( "InterScenario plugin: PH iteration 0 re-solve pass %s"
% (count,) )
_stale_scenarios = []
for _id, _soln in enumerate(self.unique_scenario_solutions):
_was_cut = sum(
1 for c in self.feasibility_cuts if type(c[_id]) is tuple)
if _was_cut:
_stale_scenarios.extend(_soln[1])
self._distribute_cuts(ph, True)
toc("InterScenario: distributed cuts to scenarios")
self._interscenario_plugin(ph)
self.lastRun = 0
def post_iteration_0_solves(self, ph):
self._collect_unique_scenario_solutions(ph)
self._interscenario_plugin(ph)
count = 0
while self.rho is None and self.feasibility_cuts:
count += 1
toc("InterScenario plugin: PH iteration 0 re-solve pass %s" %
(count, ))
_stale_scenarios = []
for _id, _soln in enumerate(self.unique_scenario_solutions):
_was_cut = sum(1 for c in self.feasibility_cuts
if type(c[_id]) is tuple)
if _was_cut:
_stale_scenarios.extend(_soln[1])
self._distribute_cuts(ph, True)
toc("InterScenario plugin: distributed cuts to scenarios")
self._collect_unique_scenario_solutions(ph)
self._interscenario_plugin(ph)
self.lastRun = 0
def _distribute_cuts(self, ph, resolve=False):
totalCuts = 0
cutObj = sorted( c[0] for x in self.feasibility_cuts for c in x
if type(c) is tuple
and c[0] > self.cutThreshold_minDiff )
if cutObj:
allCutThreshold = cutObj[
min( int((1-self.cutThreshold_crossCut)*len(cutObj)),
len(cutObj)-1 ) ]
else:
allCutThreshold = 1
distributed = isinstance( ph._solver_manager, SolverManager_PHPyro )
if ph._scenario_tree.contains_bundles():
subproblems = ph._scenario_tree._scenario_bundles
get_scenarios = lambda x: x._scenario_names
else:
subproblems = ph._scenario_tree._scenarios
get_scenarios = lambda x: [x]
resolves = []
for problem in subproblems:
cuts = []
for id, (x, s) in enumerate(self.unique_scenario_solutions):
found = False
for scenario in get_scenarios(problem):
if scenario._name in s:
found = True
break
if found:
cuts.extend( c[id] for c in self.feasibility_cuts
if type(c[id]) is tuple
and c[id][0] > self.cutThreshold_minDiff )
elif self.feasible_objectives[id] is None:
# We only add cuts generated by other scenarios to
# scenarios that are not currently feasible (as
# these are feassibility cuts, they should not
# impact feasible scenarios)
cuts.extend( c[id] for c in self.feasibility_cuts
if type(c[id]) is tuple
and c[id][0] > allCutThreshold )
if not cuts and not self.incumbent_cuts:
resolves.append(None)
continue
totalCuts += len(cuts)
if distributed:
resolves.append(
ph._solver_manager.queue(
action="invoke_external_function",
name=problem._name,
queue_name=ph._phpyro_job_worker_map[problem._name],
invocation_type=InvocationType.SingleInvocation.key,
generateResponse=True,
module_name='pyomo.pysp.plugins.interscenario',
function_name='add_new_cuts',
function_kwds=None,
function_args=( cuts,
self.incumbent_cuts,
resolve ),
) )
else:
ans = add_new_cuts( ph, ph._scenario_tree, problem,
cuts, self.incumbent_cuts, resolve )
resolves.append(ans)
toc("distributed cuts to scenario %s%s" %
( problem._name,
' and resolved scenario' if resolve else '' ))
toc( "InterScenario plugin: added %d feasibility cuts from a "
"library of %s cuts" % (totalCuts, len(cutObj)) )
self.feasibility_cuts = []
if self.incumbent_cuts:
print( "InterScenario plugin: added %d incumbent cuts" %
(len(self.incumbent_cuts), ) )
self.incumbent_cuts = []
if distributed:
num_results_so_far = sum(1 for x in resolves if x is None)
num_results = len(resolves)
while (num_results_so_far < num_results):
_ah = ph._solver_manager.wait_any()
_ah_idx = resolves.index(_ah)
resolves[_ah_idx] = ph._solver_manager.get_results(_ah)
num_results_so_far += 1
if resolve:
# Transfer the first stage values and cost back to PH and
# recompute xbar
rootNode = ph._scenario_tree.findRootNode()
for _id, problem in enumerate(subproblems):
ans = resolves[_id]
if ans is None:
continue
for scenario in get_scenarios(problem):
scenario._cost = ans[1]
assert( sorted(ans[0]) ==
sorted(scenario._x[rootNode._name]) )
scenario._x[rootNode._name] = ans[0] #[_vid] = _vval
ph.update_variable_statistics()
def _interscenario_plugin(self,ph):
toc("InterScenario plugin: analyzing scenario dual information")
# (1) Collect all scenario (first) stage variables
#self._collect_unique_scenario_solutions(ph)
# (2) Filter them to find a set we want to distribute
pass
# (3) Distribute (some) of the variable sets out to the
# scenarios, fix, and resolve; Collect and return the
# objectives, duals, and any cuts
partial_obj_values, dual_values, cuts, probability \
= self._solve_interscenario_solutions( ph )
# Compute the non-anticipative objective values for each
# scenario solution
self.feasible_objectives = self._compute_objective(
partial_obj_values, probability )
for _id, soln in enumerate(self.unique_scenario_solutions):
_scenarios = [ph._scenario_tree.get_scenario(x) for x in soln[1]]
print(
" Solution %2d: generated %2d cuts, "
"cut by %2d other scenarios; objective %10s, "
"scenario cost [%s], cut obj [%s] [generated by %s]" % (
_id,
sum(1 for c in cuts[_id] if type(c) is tuple),
sum(1 for c in cuts if type(c[_id]) is tuple),
"None" if self.feasible_objectives[_id] is None
else "%10.2f" % self.feasible_objectives[_id],
", ".join("%10.2f" % x._cost for x in _scenarios),
" ".join("%5.2f" % x[0] if type(x) is tuple else "%5s" % x
for x in cuts[_id]),
','.join(soln[1])
))
scenarioCosts = [ ph._scenario_tree.get_scenario(x)._cost
for s in self.unique_scenario_solutions
for x in s[1] ]
scenarioProb = [ ph._scenario_tree.get_scenario(x)._probability
for s in self.unique_scenario_solutions
for x in s[1] ]
_avg = sum( scenarioProb[i]*c for i,c in enumerate(scenarioCosts) )
_max = max( scenarioCosts )
_min = min( scenarioCosts )
if self.average_solution is None:
_del_avg = None
_del_avg_str = "-----%"
else:
_prev = self.average_solution
_del_avg = (_avg-_prev) / max(abs(_avg),abs(_prev))
_del_avg_str = "%+.2f%%" % ( 100*_del_avg, )
self.average_solution = _avg
print(" Average scenario cost: %f (%s) Max-min: %f (%0.2f%%)" % (
_avg, _del_avg_str, _max-_min, abs(100.*(_max-_min)/_avg) ))
# (4) save any cuts for distribution before the next solve
#self.feasibility_cuts = []
#for c in cuts:
# self.feasibility_cuts.extend(
# x for x in c if type(x) is tuple and x[0] > self.cutThreshold )
#cutCount = len(self.feasibility_cuts)
if self.enableFeasibilityCuts:
self.feasibility_cuts = cuts
cutCount = sum( sum( 1 for x in c if type(x) is tuple
and x[0]>self.cutThreshold_minDiff )
for c in cuts )
subProblemCount = sum(len(c) for c in cuts)
# (5) compute and publish the new incumbent
self._update_incumbent(ph)
# (6a) If this is iteration 0, and we have feasibility cuts, and
# they are (sufficiently) helping the Lagrangean bound, then
# skip setting rho and do another round oc cuts
if ph._current_iteration == 0:
# Tell ph that we may have a good opter bound
ph._update_reported_bounds(outer=self.average_solution)
if ( cutCount > self.recutThreshold*(subProblemCount-len(cuts))
and ( _del_avg is None or
_del_avg > self.iteration0RecutBoundImprovement )):
# Bypass RHO updates and check for more cuts
#self.lastRun = ph._current_iteration - self.iterationInterval
return
# (6b) compute updated rho estimates
new_rho, loginfo = self._process_dual_information(
ph, dual_values, probability )
_scale = self.rhoScale
if self.rho is None:
print("InterScenario plugin: initializing rho")
self.rho = {}
for v,r in iteritems(new_rho):
self.rho[v] = _scale*r
else:
_damping = self.rhoDamping
for v,r in iteritems(new_rho):
if self.rho[v]:
self.rho[v] += (1-_damping)*(_scale*r - self.rho[v])
#self.rho[v] = max(_scale*r, self.rho[v]) - \
# _damping*abs(_scale*r - self.rho[v])
else:
self.rho[v] = _scale*r
for v,l in sorted(iteritems(loginfo)):
if v is None:
print(l)
else:
print(l % (self.rho[v],))
#print("SETTING SELF.RHO", self.rho)
rootNode = ph._scenario_tree.findRootNode()
if self.enableRhoUpdates:
for v, r in iteritems(self.rho):
ph.setRhoAllScenarios(rootNode, v, r)
def get_modified_instance( ph, scenario_tree, scenario_or_bundle, **options):
# Find the model
if scenario_tree.contains_bundles():
model = ph._bundle_binding_instance_map[scenario_or_bundle._name]
else:
model = ph._instances[scenario_or_bundle._name]
b = model.component('_interscenario_plugin')
if b is not None:
return model
#
# We need to add the interscenario information to this model
#
model._interscenario_plugin = b = Block()
# Save our options
#
b.epsilon = options.pop('epsilon')
b.cut_scale = options.pop('cut_scale')
b.allow_slack = options.pop('allow_slack')
b.enable_rho = options.pop('enable_rho')
b.enable_cuts = options.pop('enable_cuts')
assert( len(options) == 0 )
# Information for generating cuts
#
b.cutlist = ConstraintList()
b.abs_int_vars = VarList(within=NonNegativeIntegers)
b.abs_binary_vars = VarList(within=Binary)
# Note: the var_ids are on the ORIGINAL scenario models
rootNode = scenario_tree.findRootNode()
var_ids = list(iterkeys(rootNode._variable_datas))
# Right now, this is hard-coded for 2-stage problems - so we only
# need to worry about the variables from the root node. These
# variables should exist on all scenarios. Set up a (trivial)
# equality constraint for each variable:
# var == current_value{param} + separation_variable{var, fixed=0}
b.STAGE1VAR = _S1V = Set(initialize=var_ids)
b.separation_variables = _sep = Var( _S1V, dense=True )
b.fixed_variable_values = _param = Param(_S1V, mutable=True, initialize=0)
b.rho = weakref.ref(model.component('PHRHO_%s' % rootNode._name))
b.weights = weakref.ref(model.component('PHWEIGHT_%s' % rootNode._name))
if b.allow_slack:
for idx in _sep:
_sep[idx].setlb(-b.epsilon)
_sep[idx].setub(b.epsilon)
else:
_sep.fix(0)
_cuidBuffer = {}
_src = b.local_stage1_varmap = {}
for i in _S1V:
# Note indexing: for each 1st stage var, pick an arbitrary
# (first) scenario and return the variable (and not it's
# probability)
_cuid = ComponentUID(rootNode._variable_datas[i][0][0], _cuidBuffer)
_src[i] = weakref.ref(_cuid.find_component_on(model))
#_base_src[i] = weakref.ref(_cuid.find_component_on(base_model))
def _set_var_value(b, i):
return _param[i] + _sep[i] - _src[i]() == 0
b.fixed_variables_constraint \
= _con = Constraint( _S1V, rule=_set_var_value )
#
# TODO: When we get the duals of the first-stage variables, do we
# want the dual WRT the original objective, or the dual WRT the
# augmented objective?
#
# Move the objective to a standardized place so we can easily find it later
if PYOMO_4_0:
_orig_objective = list( x[2] for x in model.all_component_data(
Objective, active=True, descend_into=True ) )
else:
_orig_objective = list( model.component_data_objects(
Objective, active=True, descend_into=True ) )
assert(len(_orig_objective) == 1)
_orig_objective = _orig_objective[0]
b.original_obj = weakref.ref(_orig_objective)
# add (and deactivate) the objective for the infeasibility
# separation problem.
b.separation_obj = Objective(
expr= sum( _sep[i]**2 for i in var_ids ),
sense = minimize )
# Make sure we get dual information
if 'dual' not in model:
# Export and import floating point data
model.dual = Suffix(direction=Suffix.IMPORT_EXPORT)
#if 'rc' not in model:
# model.rc = Suffix(direction=Suffix.IMPORT_EXPORT)
if FALLBACK_ON_BRUTE_FORCE_PREPROCESS:
model.preprocess()
else:
_map = {}
preprocess_block_constraints(b, idMap=_map)
# Note: we wait to deactivate the objective until after we
# preprocess so that the obective is correctly processed.
b.separation_obj.deactivate()
# (temporarily) deactivate the fixed stage-1 variables
_con.deactivate()
toc("InterScenario plugin: generated modified problem instance")
return model
def solve_fixed_scenario_solutions(
ph, scenario_tree, scenario_or_bundle,
scenario_solutions, **model_options ):
model = get_modified_instance(
ph, scenario_tree, scenario_or_bundle, **model_options )
_block = model._interscenario_plugin
_param = _block.fixed_variable_values
_sep = _block.separation_variables
_con = _block.fixed_variables_constraint
# We need to know which scenarios are local to this instance ... so
# we don't waste time repeating work.
if scenario_tree.contains_bundles():
local_scenarios = scenario_or_bundle._scenario_names
else:
local_scenarios = [ scenario_or_bundle._name ]
ipopt = SolverFactory("ipopt")
#
# Turn off RHO!
#
_saved_rho_values = _block.rho().extract_values()
_block.rho().store_values(0)
# Enable the constraints to fix the Stage 1 variables:
_con.activate()
# Solve each solution here and cache the resulting objective
cutlist = []
obj_values = []
dual_values = []
for var_values, scenario_name_list in scenario_solutions:
local = False
for scenario in local_scenarios:
if scenario in scenario_name_list:
local = True
break
if local:
# Here is where we could save some time and not repeat work
# ... for now I am being lazy and re-solving so that we get
# the dual values, etc for this scenario as well. If nothing
# else, it makes averaging easier.
pass
assert( len(var_values) == len(_param) )
for var_id, var_value in iteritems(var_values):
_param[var_id] = var_value
# TODO: We only need to update the StandardRepn for the binding
# constraints ... so we could save a LOT of time by not
# preprocessing the whole model.
#
if FALLBACK_ON_BRUTE_FORCE_PREPROCESS:
model.preprocess()
else:
var_id_map = {}
preprocess_block_constraints(_block, idMap=var_id_map)
toc("preprocessed scenario %s" % ( scenario_or_bundle._name, ))
output_buffer = StringIO()
pyutilib.misc.setup_redirect(output_buffer)
try:
results = ph._solver.solve(model, tee=True) # warmstart=True)
except:
logger.warning("Exception raised solving the interscenario "
"evaluation subproblem")
logger.warning("Solver log:\n%s" % output_buffer.getvalue())
raise
finally:
pyutilib.misc.reset_redirect()
toc("solved solution from scenario set %s on scenario %s" %
( scenario_name_list, scenario_or_bundle._name, ))
ss = results.solver.status
tc = results.solver.termination_condition
#self.timeInSolver += results['Solver'][0]['Time']
if ss == SolverStatus.ok and tc in _acceptable_termination_conditions:
state = 0 #'FEASIBLE'
if PYOMO_4_0:
model.load(results)
else:
model.solutions.load_from(results)
#
# Turn off W, recompute the objective
#
_saved_w_values = _block.weights().extract_values()
_block.weights().store_values(0)
obj_values.append(value(_block.original_obj()))
_block.weights().store_values(_saved_w_values)
# NOTE: Getting the dual values resolves the model
# (potentially relaxing second state variables.
if _block.enable_rho:
dual_values.append( get_dual_values(ph._solver, model) )
else:
dual_values.append(None)
cutlist.append(". ")
elif True or tc in _infeasible_termination_conditions:
state = 1 #'INFEASIBLE'
obj_values.append(None)
dual_values.append(None)
if _block.enable_cuts:
cut = solve_separation_problem(ph._solver, model, True)
if cut == '????':
if ph._solver.problem_format() != ProblemFormat.nl:
model.preprocess()
#preprocess_block_objectives(_block)
#preprocess_block_constraints(_block)
cut = solve_separation_problem(ipopt, model, False)
else:
cut = "X "
cutlist.append( cut )
toc("solved separation problem for solution from scenario set "
"%s on scenario %s" %
( scenario_name_list, scenario_or_bundle._name, ))
else:
state = 2 #'NONOPTIMAL'
obj_values.append(None)
dual_values.append(None)
cutlist.append("? ")
logger.warning("Solving the interscenario evaluation "
"subproblem failed (%s)." % (state,) )
logger.warning("Solver log:\n%s" % output_buffer.getvalue())
#
# Turn RHO, W back on!
#
_block.weights().store_values(_saved_w_values)
_block.rho().store_values(_saved_rho_values)
# Disable the constraints to fix the Stage 1 variables:
_con.deactivate()
return obj_values, dual_values, cutlist
def _distribute_cuts(self, ph, resolve=False):
totalCuts = 0
cutObj = sorted(
c[0] for x in self.feasibility_cuts for c in x
if type(c) is tuple and c[0] > self.cutThreshold_minDiff)
if cutObj:
allCutThreshold = cutObj[min(
int((1 - self.cutThreshold_crossCut) * len(cutObj)),
len(cutObj) - 1)]
else:
allCutThreshold = 1
distributed = isinstance(ph._solver_manager, SolverManager_PHPyro)
if ph._scenario_tree.contains_bundles():
subproblems = ph._scenario_tree._scenario_bundles
get_scenarios = lambda x: x._scenario_names
else:
subproblems = ph._scenario_tree._scenarios
get_scenarios = lambda x: [x]
resolves = []
for problem in subproblems:
cuts = []
for id, (x, s) in enumerate(self.unique_scenario_solutions):
found = False
for scenario in get_scenarios(problem):
if scenario._name in s:
found = True
break
if found:
cuts.extend(c[id] for c in self.feasibility_cuts
if type(c[id]) is tuple
and c[id][0] > self.cutThreshold_minDiff)
elif self.feasible_objectives[id] is None:
# We only add cuts generated by other scenarios to
# scenarios that are not currently feasible (as
# these are feassibility cuts, they should not
# impact feasible scenarios)
cuts.extend(
c[id] for c in self.feasibility_cuts
if type(c[id]) is tuple and c[id][0] > allCutThreshold)
if not cuts and not self.incumbent_cuts:
resolves.append(None)
continue
totalCuts += len(cuts)
if distributed:
resolves.append(
ph._solver_manager.queue(
action="invoke_external_function",
name=problem._name,
queue_name=ph._phpyro_job_worker_map[problem._name],
invocation_type=InvocationType.SingleInvocation.key,
generateResponse=True,
module_name='pyomo.pysp.plugins.interscenario',
function_name='add_new_cuts',
function_kwds=None,
function_args=(cuts, self.incumbent_cuts, resolve),
))
else:
ans = add_new_cuts(ph, ph._scenario_tree, problem, cuts,
self.incumbent_cuts, resolve)
resolves.append(ans)
toc("distributed cuts to scenario %s%s" %
(problem._name,
' and resolved scenario' if resolve else ''))
toc("InterScenario plugin: added %d feasibility cuts from a "
"library of %s cuts" % (totalCuts, len(cutObj)))
self.feasibility_cuts = []
if self.incumbent_cuts:
print("InterScenario plugin: added %d incumbent cuts" %
(len(self.incumbent_cuts), ))
self.incumbent_cuts = []
if distributed:
num_results_so_far = sum(1 for x in resolves if x is None)
num_results = len(resolves)
while (num_results_so_far < num_results):
_ah = ph._solver_manager.wait_any()
_ah_idx = resolves.index(_ah)
resolves[_ah_idx] = ph._solver_manager.get_results(_ah)
num_results_so_far += 1
if resolve:
# Transfer the first stage values and cost back to PH and
# recompute xbar
rootNode = ph._scenario_tree.findRootNode()
for _id, problem in enumerate(subproblems):
ans = resolves[_id]
if ans is None:
continue
for scenario in get_scenarios(problem):
scenario._cost = ans[1]
assert (sorted(ans[0]) == sorted(
scenario._x[rootNode._name]))
scenario._x[rootNode._name] = ans[0] #[_vid] = _vval
ph.update_variable_statistics()
def _interscenario_plugin(self, ph):
toc("InterScenario plugin: analyzing scenario dual information")
# (1) Collect all scenario (first) stage variables
#self._collect_unique_scenario_solutions(ph)
# (2) Filter them to find a set we want to distribute
pass
# (3) Distribute (some) of the variable sets out to the
# scenarios, fix, and resolve; Collect and return the
# objectives, duals, and any cuts
partial_obj_values, dual_values, cuts, probability \
= self._solve_interscenario_solutions( ph )
# Compute the non-anticipative objective values for each
# scenario solution
self.feasible_objectives = self._compute_objective(
partial_obj_values, probability)
for _id, soln in enumerate(self.unique_scenario_solutions):
_scenarios = [ph._scenario_tree.get_scenario(x) for x in soln[1]]
print(
" Solution %2d: generated %2d cuts, "
"cut by %2d other scenarios; objective %10s, "
"scenario cost [%s], cut obj [%s] [generated by %s]" %
(_id, sum(1 for c in cuts[_id] if type(c) is tuple),
sum(1 for c in cuts if type(c[_id]) is tuple), "None"
if self.feasible_objectives[_id] is None else "%10.2f" %
self.feasible_objectives[_id], ", ".join("%10.2f" % x._cost
for x in _scenarios),
" ".join("%5.2f" % x[0] if type(x) is tuple else "%5s" % x
for x in cuts[_id]), ','.join(soln[1])))
scenarioCosts = [
ph._scenario_tree.get_scenario(x)._cost
for s in self.unique_scenario_solutions for x in s[1]
]
scenarioProb = [
ph._scenario_tree.get_scenario(x)._probability
for s in self.unique_scenario_solutions for x in s[1]
]
_avg = sum(scenarioProb[i] * c for i, c in enumerate(scenarioCosts))
_max = max(scenarioCosts)
_min = min(scenarioCosts)
if self.average_solution is None:
_del_avg = None
_del_avg_str = "-----%"
else:
_prev = self.average_solution
_del_avg = (_avg - _prev) / max(abs(_avg), abs(_prev))
_del_avg_str = "%+.2f%%" % (100 * _del_avg, )
self.average_solution = _avg
print(" Average scenario cost: %f (%s) Max-min: %f (%0.2f%%)" %
(_avg, _del_avg_str, _max - _min, abs(100. *
(_max - _min) / _avg)))
# (4) save any cuts for distribution before the next solve
#self.feasibility_cuts = []
#for c in cuts:
# self.feasibility_cuts.extend(
# x for x in c if type(x) is tuple and x[0] > self.cutThreshold )
#cutCount = len(self.feasibility_cuts)
if self.enableFeasibilityCuts:
self.feasibility_cuts = cuts
cutCount = sum(
sum(1 for x in c
if type(x) is tuple and x[0] > self.cutThreshold_minDiff)
for c in cuts)
subProblemCount = sum(len(c) for c in cuts)
# (5) compute and publish the new incumbent
self._update_incumbent(ph)
# (6a) If this is iteration 0, and we have feasibility cuts, and
# they are (sufficiently) helping the Lagrangean bound, then
# skip setting rho and do another round oc cuts
if ph._current_iteration == 0:
# Tell ph that we may have a good opter bound
ph._update_reported_bounds(outer=self.average_solution)
if (cutCount > self.recutThreshold * (subProblemCount - len(cuts))
and (_del_avg is None
or _del_avg > self.iteration0RecutBoundImprovement)):
# Bypass RHO updates and check for more cuts
#self.lastRun = ph._current_iteration - self.iterationInterval
return
# (6b) compute updated rho estimates
new_rho, loginfo = self._process_dual_information(
ph, dual_values, probability)
_scale = self.rhoScale
if self.rho is None:
print("InterScenario plugin: initializing rho")
self.rho = {}
for v, r in iteritems(new_rho):
self.rho[v] = _scale * r
else:
_damping = self.rhoDamping
for v, r in iteritems(new_rho):
if self.rho[v]:
self.rho[v] += (1 - _damping) * (_scale * r - self.rho[v])
#self.rho[v] = max(_scale*r, self.rho[v]) - \
# _damping*abs(_scale*r - self.rho[v])
else:
self.rho[v] = _scale * r
for v, l in sorted(iteritems(loginfo)):
if v is None:
print(l)
else:
print(l % (self.rho[v], ))
#print("SETTING SELF.RHO", self.rho)
rootNode = ph._scenario_tree.findRootNode()
if self.enableRhoUpdates:
for v, r in iteritems(self.rho):
ph.setRhoAllScenarios(rootNode, v, r)
def get_modified_instance(ph, scenario_tree, scenario_or_bundle, **options):
# Find the model
if scenario_tree.contains_bundles():
model = ph._bundle_binding_instance_map[scenario_or_bundle._name]
else:
model = ph._instances[scenario_or_bundle._name]
b = model.component('_interscenario_plugin')
if b is not None:
return model
#
# We need to add the interscenario information to this model
#
model._interscenario_plugin = b = Block()
# Save our options
#
b.epsilon = options.pop('epsilon')
b.cut_scale = options.pop('cut_scale')
b.allow_slack = options.pop('allow_slack')
b.enable_rho = options.pop('enable_rho')
b.enable_cuts = options.pop('enable_cuts')
assert (len(options) == 0)
# Information for generating cuts
#
b.cutlist = ConstraintList()
b.abs_int_vars = VarList(within=NonNegativeIntegers)
b.abs_binary_vars = VarList(within=Binary)
# Note: the var_ids are on the ORIGINAL scenario models
rootNode = scenario_tree.findRootNode()
var_ids = list(iterkeys(rootNode._variable_datas))
# Right now, this is hard-coded for 2-stage problems - so we only
# need to worry about the variables from the root node. These
# variables should exist on all scenarios. Set up a (trivial)
# equality constraint for each variable:
# var == current_value{param} + separation_variable{var, fixed=0}
b.STAGE1VAR = _S1V = Set(initialize=var_ids)
b.separation_variables = _sep = Var(_S1V, dense=True)
b.fixed_variable_values = _param = Param(_S1V, mutable=True, initialize=0)
b.rho = weakref.ref(model.component('PHRHO_%s' % rootNode._name))
b.weights = weakref.ref(model.component('PHWEIGHT_%s' % rootNode._name))
if b.allow_slack:
for idx in _sep:
_sep[idx].setlb(-b.epsilon)
_sep[idx].setub(b.epsilon)
else:
_sep.fix(0)
_cuidBuffer = {}
_src = b.local_stage1_varmap = {}
for i in _S1V:
# Note indexing: for each 1st stage var, pick an arbitrary
# (first) scenario and return the variable (and not it's
# probability)
_cuid = ComponentUID(rootNode._variable_datas[i][0][0], _cuidBuffer)
_src[i] = weakref.ref(_cuid.find_component_on(model))
#_base_src[i] = weakref.ref(_cuid.find_component_on(base_model))
def _set_var_value(b, i):
return _param[i] + _sep[i] - _src[i]() == 0
b.fixed_variables_constraint \
= _con = Constraint( _S1V, rule=_set_var_value )
#
# TODO: When we get the duals of the first-stage variables, do we
# want the dual WRT the original objective, or the dual WRT the
# augmented objective?
#
# Move the objective to a standardized place so we can easily find it later
if PYOMO_4_0:
_orig_objective = list(x[2] for x in model.all_component_data(
Objective, active=True, descend_into=True))
else:
_orig_objective = list(
model.component_data_objects(Objective,
active=True,
descend_into=True))
assert (len(_orig_objective) == 1)
_orig_objective = _orig_objective[0]
b.original_obj = weakref.ref(_orig_objective)
# add (and deactivate) the objective for the infeasibility
# separation problem.
b.separation_obj = Objective(expr=sum(_sep[i]**2 for i in var_ids),
sense=minimize)
# Make sure we get dual information
if 'dual' not in model:
# Export and import floating point data
model.dual = Suffix(direction=Suffix.IMPORT_EXPORT)
#if 'rc' not in model:
# model.rc = Suffix(direction=Suffix.IMPORT_EXPORT)
if FALLBACK_ON_BRUTE_FORCE_PREPROCESS:
model.preprocess()
else:
_map = {}
preprocess_block_constraints(b, idMap=_map)
# Note: we wait to deactivate the objective until after we
# preprocess so that the obective is correctly processed.
b.separation_obj.deactivate()
# (temporarily) deactivate the fixed stage-1 variables
_con.deactivate()
toc("InterScenario plugin: generated modified problem instance")
return model
def solve_fixed_scenario_solutions(ph, scenario_tree, scenario_or_bundle,
scenario_solutions, **model_options):
model = get_modified_instance(ph, scenario_tree, scenario_or_bundle,
**model_options)
_block = model._interscenario_plugin
_param = _block.fixed_variable_values
_sep = _block.separation_variables
_con = _block.fixed_variables_constraint
# We need to know which scenarios are local to this instance ... so
# we don't waste time repeating work.
if scenario_tree.contains_bundles():
local_scenarios = scenario_or_bundle._scenario_names
else:
local_scenarios = [scenario_or_bundle._name]
ipopt = SolverFactory("ipopt")
#
# Turn off RHO!
#
_saved_rho_values = _block.rho().extract_values()
_block.rho().store_values(0)
# Enable the constraints to fix the Stage 1 variables:
_con.activate()
# Solve each solution here and cache the resulting objective
cutlist = []
obj_values = []
dual_values = []
for var_values, scenario_name_list in scenario_solutions:
local = False
for scenario in local_scenarios:
if scenario in scenario_name_list:
local = True
break
if local:
# Here is where we could save some time and not repeat work
# ... for now I am being lazy and re-solving so that we get
# the dual values, etc for this scenario as well. If nothing
# else, it makes averaging easier.
pass
assert (len(var_values) == len(_param))
for var_id, var_value in iteritems(var_values):
_param[var_id] = var_value
# TODO: We only need to update the StandardRepn for the binding
# constraints ... so we could save a LOT of time by not
# preprocessing the whole model.
#
if FALLBACK_ON_BRUTE_FORCE_PREPROCESS:
model.preprocess()
else:
var_id_map = {}
preprocess_block_constraints(_block, idMap=var_id_map)
toc("preprocessed scenario %s" % (scenario_or_bundle._name, ))
output_buffer = StringIO()
setup_redirect(output_buffer)
try:
results = ph._solver.solve(model, tee=True) # warmstart=True)
except:
logger.warning("Exception raised solving the interscenario "
"evaluation subproblem")
logger.warning("Solver log:\n%s" % output_buffer.getvalue())
raise
finally:
reset_redirect()
toc("solved solution from scenario set %s on scenario %s" % (
scenario_name_list,
scenario_or_bundle._name,
))
ss = results.solver.status
tc = results.solver.termination_condition
#self.timeInSolver += results['Solver'][0]['Time']
if ss == SolverStatus.ok and tc in _acceptable_termination_conditions:
state = 0 #'FEASIBLE'
if PYOMO_4_0:
model.load(results)
else:
model.solutions.load_from(results)
#
# Turn off W, recompute the objective
#
_saved_w_values = _block.weights().extract_values()
_block.weights().store_values(0)
obj_values.append(value(_block.original_obj()))
_block.weights().store_values(_saved_w_values)
# NOTE: Getting the dual values resolves the model
# (potentially relaxing second state variables.
if _block.enable_rho:
dual_values.append(get_dual_values(ph._solver, model))
else:
dual_values.append(None)
cutlist.append(". ")
elif True or tc in _infeasible_termination_conditions:
state = 1 #'INFEASIBLE'
obj_values.append(None)
dual_values.append(None)
if _block.enable_cuts:
cut = solve_separation_problem(ph._solver, model, True)
if cut == '????':
if ph._solver.problem_format() != ProblemFormat.nl:
model.preprocess()
#preprocess_block_objectives(_block)
#preprocess_block_constraints(_block)
cut = solve_separation_problem(ipopt, model, False)
else:
cut = "X "
cutlist.append(cut)
toc("solved separation problem for solution from scenario set "
"%s on scenario %s" % (
scenario_name_list,
scenario_or_bundle._name,
))
else:
state = 2 #'NONOPTIMAL'
obj_values.append(None)
dual_values.append(None)
cutlist.append("? ")
logger.warning("Solving the interscenario evaluation "
"subproblem failed (%s)." % (state, ))
logger.warning("Solver log:\n%s" % output_buffer.getvalue())
#
# Turn RHO, W back on!
#
_block.weights().store_values(_saved_w_values)
_block.rho().store_values(_saved_rho_values)
# Disable the constraints to fix the Stage 1 variables:
_con.deactivate()
return obj_values, dual_values, cutlist
def _interscenario_plugin(self,ph):
toc("InterScenario plugin: analyzing scenario dual information")
# (1) Collect all scenario (first) stage variables
self._collect_unique_scenario_solutions(ph)
# (2) Filter them to find a set we want to distribute
pass
# (3) Distribute (some) of the variable sets out to the
# scenarios, fix, and resolve; Collect and return the
# objectives, duals, and any cuts
partial_obj_values, dual_values, cuts, probability \
= self._solve_interscenario_solutions( ph )
# Compute the non-anticipative objective values for each
# scenario solution
self.feasible_objectives = self._compute_objective(
partial_obj_values, probability )
for _id, soln in enumerate(self.unique_scenario_solutions):
_scenarios = [ph._scenario_tree.get_scenario(x) for x in soln[1]]
print(
" Solution %2d: generated %2d cuts, "
"cut by %2d other scenarios; objective %10s, "
"scenario cost [%s], cut obj [%s] [generated by %s]" % (
_id,
sum(1 for c in cuts[_id] if type(c) is tuple),
sum(1 for c in cuts if type(c[_id]) is tuple),
"None" if self.feasible_objectives[_id] is None
else "%10.2f" % self.feasible_objectives[_id],
", ".join("%10.2f" % x._cost for x in _scenarios),
" ".join("%5.2f" % x[0] if type(x) is tuple else "%5s" % x
for x in cuts[_id]),
','.join(soln[1])
))
)
scenarioCosts = [ ph._scenario_tree.get_scenario(x)._cost
for s in self.unique_scenario_solutions
for x in s[1] ]
scenarioProb = [ ph._scenario_tree.get_scenario(x)._probability
for s in self.unique_scenario_solutions
for x in s[1] ]
_avg = sum( scenarioProb[i]*c for i,c in enumerate(scenarioCosts) )
_max = max( scenarioCosts )
_min = min( scenarioCosts )
if self.average_solution is None:
_del_avg = None
_del_avg_str = "-----%"
else:
_prev = self.average_solution
_del_avg = (_avg-_prev) / max(abs(_avg),abs(_prev))
_del_avg_str = "%+.2f%%" % ( 100*_del_avg, )
self.average_solution = _avg
print(" Average scenario cost: %f (%s) Max-min: %f (%0.2f%%)" % (
_avg, _del_avg_str, _max-_min, abs(100.*(_max-_min)/_avg) ))
# (4) save any cuts for distribution before the next solve
#self.feasibility_cuts = []
#for c in cuts:
# self.feasibility_cuts.extend(
# x for x in c if type(x) is tuple and x[0] > self.cutThreshold )
#cutCount = len(self.feasibility_cuts)
if self.enableFeasibilityCuts:
self.feasibility_cuts = cuts
cutCount = sum( sum( 1 for x in c if type(x) is tuple
and x[0]>self.cutThreshold_minDiff )
for c in cuts )
name=problem._name,
queue_name=ph._phpyro_job_worker_map[problem._name],
invocation_type=InvocationType.SingleInvocation.key,
generateResponse=True,
module_name='pyomo.pysp.plugins.interscenario',
function_name='add_new_cuts',
function_kwds=None,
function_args=( cuts,
self.incumbent_cuts,
resolve ),
) )
else:
ans = add_new_cuts( ph, ph._scenario_tree, problem,
cuts, self.incumbent_cuts, resolve )
resolves.append(ans)
toc("distributed cuts to scenario %s%s" %
( problem._name,
' and resolved scenario' if resolve else '' ))
toc( "InterScenario plugin: added %d feasibility cuts from a "
"library of %s cuts" % (totalCuts, len(cutObj)) )
self.feasibility_cuts = []
if self.incumbent_cuts:
print( "InterScenario plugin: added %d incumbent cuts" %
(len(self.incumbent_cuts), ) )
self.incumbent_cuts = []
if distributed:
num_results_so_far = sum(1 for x in resolves if x is None)
