def load_ph_warmstart(ph,
scenariotree_solution):
scenario_tree = ph._scenario_tree
scenario_solutions = scenariotree_solution['scenario solutions']
for scenario in scenario_tree._scenarios:
scenario_name = scenario._name
scenario_sol = scenario_solutions[scenario_name]
variable_sol = scenario_sol['variables']
for tree_node in scenario._node_list:
isNotLeafNode = not tree_node.is_leaf_node()
if isNotLeafNode:
scenario._w[tree_node.name].clear()
scenario_w = scenario._w[tree_node._name]
for variable_id, (var_name, index) in \
iteritems(tree_node._variable_ids):
name_label = str(var_name)+str(indexToString(index))
varsol = variable_sol[name_label]
if variable_id in tree_node._standard_variable_ids:
if 'weight' in varsol:
scenario_w[variable_id] = varsol['weight']
node_solutions = scenariotree_solution['node solutions']
for stage in scenario_tree._stages[:-1]:
for tree_node in stage._tree_nodes:
variable_sol = node_solutions[tree_node._name]['variables']
for variable_id in tree_node._standard_variable_ids:
var_name, index = tree_node._variable_ids[variable_id]
sol = variable_sol[str(var_name)+str(indexToString(index))]
tree_node._xbars[variable_id] = sol['xbar']
def load_ph_warmstart(ph,
scenariotree_solution):
scenario_tree = ph._scenario_tree
scenario_solutions = scenariotree_solution['scenario solutions']
for scenario in scenario_tree._scenarios:
scenario_name = scenario._name
scenario_sol = scenario_solutions[scenario_name]
variable_sol = scenario_sol['variables']
for tree_node in scenario._node_list:
isNotLeafNode = not tree_node.is_leaf_node()
if isNotLeafNode:
scenario._w[tree_node.name].clear()
scenario_w = scenario._w[tree_node._name]
for variable_id, (var_name, index) in \
iteritems(tree_node._variable_ids):
name_label = str(var_name)+str(indexToString(index))
varsol = variable_sol[name_label]
if variable_id in tree_node._standard_variable_ids:
if 'weight' in varsol:
scenario_w[variable_id] = varsol['weight']
node_solutions = scenariotree_solution['node solutions']
for stage in scenario_tree._stages[:-1]:
for tree_node in stage._tree_nodes:
variable_sol = node_solutions[tree_node._name]['variables']
for variable_id in tree_node._standard_variable_ids:
var_name, index = tree_node._variable_ids[variable_id]
sol = variable_sol[str(var_name)+str(indexToString(index))]
tree_node._xbars[variable_id] = sol['xbar']
def _w_printing(self, ofile, ph=None):
# print the w values in a useful way to the open file ofile
# if ph is None, just write the header
if ph is None:
ofile.write("iteration; tree node; scenario; variable; ID; W\n")
else:
root_node_name = ph._scenario_tree.findRootNode()._name
for stage, tree_node, variable_id, variable_values, is_fixed, is_stale in \
scenario_tree_node_variables_generator_noinstances(ph._scenario_tree,
includeDerivedVariables=False,
includeLastStage=False):
if is_stale is False \
and (OnlyRootNode is False or tree_node._name == root_node_name):
for scenario in tree_node._scenarios:
scen_name = scenario._name
weight_value = scenario._w[
tree_node._name][variable_id]
variable_name, index = tree_node._variable_ids[
variable_id]
full_variable_name = variable_name + indexToString(
index)
ofile.write(
str(ph._current_iteration) + ';' +
tree_node._name + ';' + scen_name + ';' +
full_variable_name + ';' + str(variable_id) + ';' +
str(weight_value) + '\n')
def write_build_vars_table(ph, solution, output_file):
# root_node = ph.get_scenario_tree().findRootNode()
# this works on runph and solverserver
root_node = ph._scenario_tree.findRootNode()
variable_names = sorted(root_node._variable_indices.keys())
# alternatively, this could be (as in ph.pprint)
# variable_names = sorted(ph.get_scenario_tree().stages[0]._variables.keys())
# useful elements for getting variable values:
# node._variable_indices = {varname: [(key1a, key1b), (key2a, key2b)]}
# node._variable_ids = {varid: (varname, (key1a, key1b))}
# node._name_index_to_id = {(varname, (key1a, key1b)): varid}
variable_data = [
(
v + indexToString(k), # var name with index
solution[root_node._name_index_to_id[(v, k)]] # current value
) for v in variable_names
for k in sorted(root_node._variable_indices[v])
]
print "writing {}...".format(output_file)
# print "variables to write:"
# print variable_data
with open(output_file, 'w') as f:
f.writelines("\t".join(map(str, r)) + "\n"
for r in [("variable", "value")] + variable_data)
def initialize_algorithm_data(self,
rho_init=1.0,
y_init=0.0,
z_init=0.0):
# used to check dual-feasibility of initial y
y_sum = {}
for stage in self._manager.scenario_tree.stages[:-1]:
for tree_node in stage.nodes:
y_sum_node = y_sum[tree_node.name] = \
dict((id_, 0.0) for id_ in tree_node._standard_variable_ids)
x = {}
y = {}
for scenario in self._manager.scenario_tree.scenarios:
x_scenario = x[scenario.name] = {}
y_scenario = y[scenario.name] = {}
for tree_node in scenario.node_list[:-1]:
assert not tree_node.is_leaf_node()
x_node = x_scenario[tree_node.name] = {}
y_node = y_scenario[tree_node.name] = {}
y_sum_node = y_sum[tree_node.name]
for id_ in tree_node._standard_variable_ids:
x_node[id_] = None
if type(y_init) is dict:
y_node[id_] = y_init[scenario.name][tree_node.name][id_]
else:
y_node[id_] = y_init
y_sum_node[id_] += y_node[id_]
# check dual-feasibility of y
for stage in self._manager.scenario_tree.stages[:-1]:
for tree_node in stage.nodes:
y_sum_node = y_sum[tree_node.name]
for id_ in tree_node._standard_variable_ids:
if abs(y_sum_node[id_]) > 1e-6:
name, index = tree_node._variable_ids[id_]
raise ValueError(
"Initial lagrange multipler estimates for non-"
"anticipative variable %s do not sum to zero: %s"
% (name+indexToString(index), repr(y_sum_node[id_])))
rho = {}
z = {}
for stage in self._manager.scenario_tree.stages[:-1]:
for tree_node in stage.nodes:
z_node = z[tree_node.name] = {}
rho_node = rho[tree_node.name] = {}
for id_ in tree_node._standard_variable_ids:
if type(rho_init) is dict:
rho_node[id_] = rho_init[tree_node.name][id_]
else:
rho_node[id_] = rho_init
if type(z_init) is dict:
z_node[id_] = z_init[tree_node.name][id_]
else:
z_node[id_] = z_init
return rho, x, y, z
def initialize_algorithm_data(self,
rho_init=1.0,
y_init=0.0,
z_init=0.0):
# used to check dual-feasibility of initial y
y_sum = {}
for stage in self._manager.scenario_tree.stages[:-1]:
for tree_node in stage.nodes:
y_sum_node = y_sum[tree_node.name] = \
dict((id_, 0.0) for id_ in tree_node._standard_variable_ids)
x = {}
y = {}
for scenario in self._manager.scenario_tree.scenarios:
x_scenario = x[scenario.name] = {}
y_scenario = y[scenario.name] = {}
for tree_node in scenario.node_list[:-1]:
assert not tree_node.is_leaf_node()
x_node = x_scenario[tree_node.name] = {}
y_node = y_scenario[tree_node.name] = {}
y_sum_node = y_sum[tree_node.name]
for id_ in tree_node._standard_variable_ids:
x_node[id_] = None
if type(y_init) is dict:
y_node[id_] = y_init[scenario.name][tree_node.name][id_]
else:
y_node[id_] = y_init
y_sum_node[id_] += y_node[id_]
# check dual-feasibility of y
for stage in self._manager.scenario_tree.stages[:-1]:
for tree_node in stage.nodes:
y_sum_node = y_sum[tree_node.name]
for id_ in tree_node._standard_variable_ids:
if abs(y_sum_node[id_]) > 1e-6:
name, index = tree_node._variable_ids[id_]
raise ValueError(
"Initial lagrange multipler estimates for non-"
"anticipative variable %s do not sum to zero: %s"
% (name+indexToString(index), repr(y_sum_node[id_])))
rho = {}
z = {}
for stage in self._manager.scenario_tree.stages[:-1]:
for tree_node in stage.nodes:
z_node = z[tree_node.name] = {}
rho_node = rho[tree_node.name] = {}
for id_ in tree_node._standard_variable_ids:
if type(rho_init) is dict:
rho_node[id_] = rho_init[tree_node.name][id_]
else:
rho_node[id_] = rho_init
if type(z_init) is dict:
z_node[id_] = z_init[tree_node.name][id_]
else:
z_node[id_] = z_init
return rho, x, y, z
def _inspect_variable_convergence(self, ph, ph_iter):
# collect termdiff by node and variable so we can
# report and possibly sort
term_diff = dict((tree_node._name, {}) \
for stage in ph._scenario_tree._stages[:-1]
for tree_node in stage._tree_nodes)
# Track these for reporting purposes
node_fixed_cnt = dict((tree_node._name, 0) \
for stage in ph._scenario_tree._stages[:-1]
for tree_node in stage._tree_nodes)
total_fixed_cnt = 0
for stage, tree_node, variable_id, variable_values, is_fixed, is_stale \
in scenario_tree_node_variables_generator_noinstances(
ph._scenario_tree,
includeDerivedVariables=False,
includeLastStage=False):
if is_fixed:
node_fixed_cnt[tree_node._name] += 1
total_fixed_cnt += 1
# Depending on preprocessing options, stale may indicate
# fixed or unused in the model, either way we can skip it
if (not is_stale):
var_node_avg = 0.0
for var_value, scenario_probability in variable_values:
var_node_avg += scenario_probability * var_value
var_term_diff = 0.0
for var_value, scenario_probability in variable_values:
var_term_diff += \
scenario_probability * \
fabs(var_value - var_node_avg)
term_diff[tree_node._name][variable_id] = var_term_diff
# Print individual variable term diffs by node
# and sorted highest to lowest
# skip the leaf stage
ofile = open(self.wonly_file, 'a')
for stage in ph._scenario_tree._stages[:-1]:
for tree_node in stage._tree_nodes:
for variable_id, var_term_diff in sorted(
term_diff[tree_node._name].items(),
key=itemgetter(1),
reverse=True):
variable_name, index = tree_node._variable_ids[variable_id]
ofile.write(
str(ph_iter) + "; " + tree_node._name + "; " +
variable_name + indexToString(index) + "; " +
str(var_term_diff) + '\n')
ofile.close()
def post_ph_execution(self, ph):
#import pdb; pdb.Pdb(stdout=sys.__stdout__).set_trace() # have to grab original stdout, or this crashes
# note: when solved in serial mode, ph.get_scenario_tree().get_arbitrary_scenario()._instance
# contains a solved instance that can (maybe) be used for reporting at this point,
# as done in old_post_ph_execution below. However, when using pyro, that is not available.
# Also, at this point with the parallel or serial solver, ph.get_scenario_tree()._solution
# and ph._scenario_tree.findRootNode().get_variable_value(name, index) (which reads it) have no values.
# So we take a different course, using the best available solution directly from the scenario tree.
# The code below is modeled on pyomo.pysp.ph.ProgressiveHedging.pprint()
# note: at this point, we might be able to use ph.get_scenario_tree().get_arbitrary_scenario()._x[root_node.name]
# to get a solution for one of the scenarios (the first), but ExtractInternalNodeSolutionsforInner()
# seems designed to give a definitive solution (i.e., the best admissable solution) (see
# compute_and_report_inner_bound_using_xhat which is calculated a few lines below the point
# where this is called in pyomo.pysp.ph)
# NOTE: this actually gets called by the phsolverserver (on the remote node), not runph.
# It's not clear what information is in the local scenario tree about the solutions from
# other scenarios. So ExtractInternalNodeSolutionsforInner() may actually not be valid
# at this point. THIS NEEDS FURTHER INVESTIGATION.
# useful elements for getting variable values:
# node._variable_indices = {varname: [(key1a, key1b), (key2a, key2b)]}
# node._variable_ids = {varid: (varname, (key1a, key1b))}
# node._name_index_to_id = {(varname, (key1a, key1b)): varid}
root_node = ph.get_scenario_tree().findRootNode()
solution = ExtractInternalNodeSolutionsforInner(ph)[
root_node.name] # {id: value}
variable_names = sorted(root_node._variable_indices.keys())
# alternatively, this could be (as in ph.pprint)
# variable_names = sorted(ph.get_scenario_tree().stages[0]._variables.keys())
variable_data = [
(
v + indexToString(k), # var name with index
solution[root_node._name_index_to_id[(v, k)]] # current value
) for v in variable_names
for k in sorted(root_node._variable_indices[v])
]
jobid = os.environ.get('SLURM_JOBID')
if jobid is None: # not running under slurm
jobid = datetime.datetime.now().isoformat("_").replace(":", ".")
output_file = os.path.join("outputs", "build_{}.tsv".format(jobid))
print "writing {}...".format(output_file)
# print "variables to write:"
# print variable_data
with open(output_file, 'w') as f:
f.writelines("\t".join(map(str, r)) + "\n"
for r in [("variable", "value")] + variable_data)
def _inspect_variable_convergence(self, ph, ph_iter):
# collect termdiff by node and variable so we can
# report and possibly sort
term_diff = dict((tree_node._name, {}) \
for stage in ph._scenario_tree._stages[:-1]
for tree_node in stage._tree_nodes)
# Track these for reporting purposes
node_fixed_cnt = dict((tree_node._name, 0) \
for stage in ph._scenario_tree._stages[:-1]
for tree_node in stage._tree_nodes)
total_fixed_cnt = 0
for stage, tree_node, variable_id, variable_values, is_fixed, is_stale \
in scenario_tree_node_variables_generator_noinstances(
ph._scenario_tree,
includeDerivedVariables=False,
includeLastStage=False):
if is_fixed:
node_fixed_cnt[tree_node._name] += 1
total_fixed_cnt += 1
# Depending on preprocessing options, stale may indicate
# fixed or unused in the model, either way we can skip it
if (not is_stale):
var_node_avg = 0.0
for var_value, scenario_probability in variable_values:
var_node_avg += scenario_probability * var_value
var_term_diff = 0.0
for var_value, scenario_probability in variable_values:
var_term_diff += \
scenario_probability * \
fabs(var_value - var_node_avg)
term_diff[tree_node._name][variable_id] = var_term_diff
# Print individual variable term diffs by node
# and sorted highest to lowest
# skip the leaf stage
ofile = open(self.wonly_file, 'a')
for stage in ph._scenario_tree._stages[:-1]:
for tree_node in stage._tree_nodes:
for variable_id, var_term_diff in sorted(term_diff[tree_node._name].items(),
key=itemgetter(1),
reverse=True):
variable_name, index = tree_node._variable_ids[variable_id]
ofile.write(str(ph_iter)+"; "+tree_node._name+"; "+variable_name+indexToString(index)+"; "+str(var_term_diff)+'\n')
ofile.close()
def evaluate_current_node_solution(sp, sp_solver, **solve_kwds):
scenario_tree = sp.scenario_tree
# Save the current fixed state and fix queue, then clear the fix queue
fixed = {}
fix_queue = {}
for tree_node in scenario_tree.nodes:
fixed[tree_node.name] = copy.deepcopy(tree_node._fixed)
fix_queue[tree_node.name] = copy.deepcopy(tree_node._fix_queue)
tree_node.clear_fix_queue()
# Fix all non-anticipative variables to their
# current value in the node solution
for stage in scenario_tree.stages[:-1]:
for tree_node in stage.nodes:
for variable_id in tree_node._standard_variable_ids:
if variable_id in tree_node._solution:
tree_node.fix_variable(variable_id,
tree_node._solution[variable_id])
else:
from pyomo.pysp.phutils import indexToString
name, index = tree_node._variable_ids[variable_id]
raise ValueError(
"Scenario tree variable with name %s (scenario_tree_id=%s) "
"does not have a solution stored on scenario tree node %s. "
"Unable to evaluate solution." % (name+indexToString(index),
variable_id,
tree_node.name))
# Push fixed variable statuses on instances (or
# transmit to the phsolverservers)
sp.push_fix_queue_to_instances()
failures = sp_solver.solve_subproblems(**solve_kwds)
# Free all non-anticipative variables
for stage in scenario_tree._stages[:-1]:
for tree_node in stage.nodes:
for variable_id in tree_node._standard_variable_ids:
tree_node.free_variable(variable_id)
# Refix all previously fixed variables
for tree_node in scenario_tree.nodes:
node_fixed = fixed[tree_node.name]
for variable_id in node_fixed:
tree_node.fix_variable(variable_id, node_fixed[variable_id])
sp.push_fix_queue_to_instances()
# Restore the fix_queue
for tree_node in scenario_tree.nodes:
tree_node._fix_queue.update(fix_queue[tree_node.name])
return failures
def evaluate_current_node_solution(sp, sp_solver, **solve_kwds):
scenario_tree = sp.scenario_tree
# Save the current fixed state and fix queue, then clear the fix queue
fixed = {}
fix_queue = {}
for tree_node in scenario_tree.nodes:
fixed[tree_node.name] = copy.deepcopy(tree_node._fixed)
fix_queue[tree_node.name] = copy.deepcopy(tree_node._fix_queue)
tree_node.clear_fix_queue()
# Fix all non-anticipative variables to their
# current value in the node solution
for stage in scenario_tree.stages[:-1]:
for tree_node in stage.nodes:
for variable_id in tree_node._standard_variable_ids:
if variable_id in tree_node._solution:
tree_node.fix_variable(variable_id,
tree_node._solution[variable_id])
else:
from pyomo.pysp.phutils import indexToString
name, index = tree_node._variable_ids[variable_id]
raise ValueError(
"Scenario tree variable with name %s (scenario_tree_id=%s) "
"does not have a solution stored on scenario tree node %s. "
"Unable to evaluate solution." %
(name + indexToString(index), variable_id,
tree_node.name))
# Push fixed variable statuses on instances (or
# transmit to the phsolverservers)
sp.push_fix_queue_to_instances()
failures = sp_solver.solve_subproblems(**solve_kwds)
# Free all non-anticipative variables
for stage in scenario_tree._stages[:-1]:
for tree_node in stage.nodes:
for variable_id in tree_node._standard_variable_ids:
tree_node.free_variable(variable_id)
# Refix all previously fixed variables
for tree_node in scenario_tree.nodes:
node_fixed = fixed[tree_node.name]
for variable_id in node_fixed:
tree_node.fix_variable(variable_id, node_fixed[variable_id])
sp.push_fix_queue_to_instances()
# Restore the fix_queue
for tree_node in scenario_tree.nodes:
tree_node._fix_queue.update(fix_queue[tree_node.name])
return failures
def extract_scenario_solutions(scenario_tree,
include_ph_objective_parameters=False,
include_leaf_stage_vars=True):
scenario_solutions = {}
for scenario in scenario_tree._scenarios:
scenario_name = scenario._name
scenario_sol = scenario_solutions[scenario_name] = {}
variable_sol = scenario_sol['variables'] = {}
for tree_node in scenario._node_list:
isNotLeafNode = not tree_node.is_leaf_node()
if isNotLeafNode or include_leaf_stage_vars:
if isNotLeafNode and include_ph_objective_parameters:
weight_values = scenario._w[tree_node._name]
rho_values = scenario._rho[tree_node._name]
x_values = scenario._x[tree_node._name]
for variable_id, (var_name, index) in \
iteritems(tree_node._variable_ids):
name_label = str(var_name)+str(indexToString(index))
varsol = variable_sol[name_label] = {}
varsol['value'] = x_values.get(variable_id)
varsol['fixed'] = scenario.is_variable_fixed(tree_node,
variable_id)
varsol['stale'] = scenario.is_variable_stale(tree_node,
variable_id)
if include_ph_objective_parameters:
if isNotLeafNode and \
(variable_id in tree_node._standard_variable_ids):
varsol['rho'] = rho_values[variable_id] \
if (isNotLeafNode) \
else None
varsol['weight'] = weight_values[variable_id] \
if (isNotLeafNode) \
else None
else:
varsol['rho'] = None
varsol['weight'] = None
scenario_sol['objective'] = scenario._objective
scenario_sol['cost'] = scenario._cost
if include_ph_objective_parameters:
scenario_sol['ph weight term'] = scenario._weight_term_cost
scenario_sol['ph proximal term'] = scenario._proximal_term_cost
scenario_sol['stage costs'] = copy.deepcopy(scenario._stage_costs)
return scenario_solutions
def extract_scenario_solutions(scenario_tree,
include_ph_objective_parameters=False,
include_leaf_stage_vars=True):
scenario_solutions = {}
for scenario in scenario_tree._scenarios:
scenario_name = scenario._name
scenario_sol = scenario_solutions[scenario_name] = {}
variable_sol = scenario_sol['variables'] = {}
for tree_node in scenario._node_list:
isNotLeafNode = not tree_node.is_leaf_node()
if isNotLeafNode or include_leaf_stage_vars:
if isNotLeafNode and include_ph_objective_parameters:
weight_values = scenario._w[tree_node._name]
rho_values = scenario._rho[tree_node._name]
x_values = scenario._x[tree_node._name]
for variable_id, (var_name, index) in \
iteritems(tree_node._variable_ids):
name_label = str(var_name)+str(indexToString(index))
varsol = variable_sol[name_label] = {}
varsol['value'] = x_values.get(variable_id)
varsol['fixed'] = scenario.is_variable_fixed(tree_node,
variable_id)
varsol['stale'] = scenario.is_variable_stale(tree_node,
variable_id)
if include_ph_objective_parameters:
if isNotLeafNode and \
(variable_id in tree_node._standard_variable_ids):
varsol['rho'] = rho_values[variable_id] \
if (isNotLeafNode) \
else None
varsol['weight'] = weight_values[variable_id] \
if (isNotLeafNode) \
else None
else:
varsol['rho'] = None
varsol['weight'] = None
scenario_sol['objective'] = scenario._objective
scenario_sol['cost'] = scenario._cost
if include_ph_objective_parameters:
scenario_sol['ph weight term'] = scenario._weight_term_cost
scenario_sol['ph proximal term'] = scenario._proximal_term_cost
scenario_sol['stage costs'] = copy.deepcopy(scenario._stage_costs)
return scenario_solutions
def extract_node_solutions(scenario_tree,
include_ph_objective_parameters=False,
include_variable_statistics=False,
include_leaf_stage_vars=True):
scenario_tree.snapshotSolutionFromScenarios()
node_solutions = {}
stages = None
if include_leaf_stage_vars:
stages = scenario_tree._stages
else:
stages = scenario_tree._stages[:-1]
for stage in stages:
for tree_node in stage._tree_nodes:
isNotLeafNode = not tree_node.is_leaf_node()
node_sol = node_solutions[tree_node._name] = {}
variable_sol = node_sol['variables'] = {}
for variable_id, (var_name, index) in \
iteritems(tree_node._variable_ids):
name_label = str(var_name)+str(indexToString(index))
sol = variable_sol[name_label] = {}
sol['solution'] = tree_node._solution[variable_id]
sol['fixed'] = tree_node.is_variable_fixed(variable_id)
sol['derived'] = \
bool(variable_id in tree_node._derived_variable_ids)
if include_variable_statistics:
if isNotLeafNode:
sol['minimum'] = tree_node._minimums[variable_id]
sol['average'] = tree_node._averages[variable_id]
sol['maximum'] = tree_node._maximums[variable_id]
else:
sol['minimum'] = None
sol['average'] = None
sol['maximum'] = None
if include_ph_objective_parameters:
if isNotLeafNode and \
(variable_id in tree_node._standard_variable_ids):
sol['xbar'] = tree_node._xbars[variable_id]
sol['wbar'] = tree_node._wbars[variable_id]
else:
sol['xbar'] = None
sol['wbar'] = None
node_sol['expected cost'] = tree_node.computeExpectedNodeCost()
return node_solutions
def extract_node_solutions(scenario_tree,
include_ph_objective_parameters=False,
include_variable_statistics=False,
include_leaf_stage_vars=True):
scenario_tree.snapshotSolutionFromScenarios()
node_solutions = {}
stages = None
if include_leaf_stage_vars:
stages = scenario_tree._stages
else:
stages = scenario_tree._stages[:-1]
for stage in stages:
for tree_node in stage._tree_nodes:
isNotLeafNode = not tree_node.is_leaf_node()
node_sol = node_solutions[tree_node._name] = {}
variable_sol = node_sol['variables'] = {}
for variable_id, (var_name, index) in \
iteritems(tree_node._variable_ids):
name_label = str(var_name)+str(indexToString(index))
sol = variable_sol[name_label] = {}
sol['solution'] = tree_node._solution[variable_id]
sol['fixed'] = tree_node.is_variable_fixed(variable_id)
sol['derived'] = \
bool(variable_id in tree_node._derived_variable_ids)
if include_variable_statistics:
if isNotLeafNode:
sol['minimum'] = tree_node._minimums[variable_id]
sol['average'] = tree_node._averages[variable_id]
sol['maximum'] = tree_node._maximums[variable_id]
else:
sol['minimum'] = None
sol['average'] = None
sol['maximum'] = None
if include_ph_objective_parameters:
if isNotLeafNode and \
(variable_id in tree_node._standard_variable_ids):
sol['xbar'] = tree_node._xbars[variable_id]
sol['wbar'] = tree_node._wbars[variable_id]
else:
sol['xbar'] = None
sol['wbar'] = None
node_sol['expected cost'] = tree_node.computeExpectedNodeCost()
return node_solutions
def update_rho(self, manager, x, y, z, rho):
first_line = ("Updating Rho Values:\n%21s %25s %16s %16s %16s"
% ("Action",
"Variable",
"Primal Residual",
"Dual Residual",
"New Rho"))
first_print = True
self.compute_primal_residual_norm(manager, x, z)
self.compute_dual_residual_norm(manager, z, rho)
verbose = self.get_option("verbose")
for stage in manager.scenario_tree.stages[:-1]:
for tree_node in stage.nodes:
prnorm_node = self._primal_residual_norm[tree_node.name]
drnorm_node = self._dual_residual_norm[tree_node.name]
rho_node = rho[tree_node.name]
for id_ in tree_node._standard_variable_ids:
name, index = tree_node._variable_ids[id_]
prnorm_var = prnorm_node[id_]
drnorm_var = drnorm_node[id_]
action = None
if (prnorm_var > self._mu * drnorm_var) and \
(prnorm_var > self._tol):
rho_node[id_] *= self._rho_increase_factor
action = "Increasing"
elif (drnorm_var > self._mu * prnorm_var) and \
(drnorm_var > self._tol):
rho_node[id_] *= self._rho_decrease_factor
action = "Decreasing"
if verbose:
if action is not None:
if first_print:
first_print = False
print(first_line)
print("%21s %25s %16g %16g %16g"
% (action,
name+indexToString(index),
prnorm_var,
drnorm_var,
rho_node[id_]))
self.snapshot_z(manager, z)
def update_rho(self, manager, x, y, z, rho):
first_line = ("Updating Rho Values:\n%21s %25s %16s %16s %16s"
% ("Action",
"Variable",
"Primal Residual",
"Dual Residual",
"New Rho"))
first_print = True
self.compute_primal_residual_norm(manager, x, z)
self.compute_dual_residual_norm(manager, z, rho)
verbose = self.get_option("verbose")
for stage in manager.scenario_tree.stages[:-1]:
for tree_node in stage.nodes:
prnorm_node = self._primal_residual_norm[tree_node.name]
drnorm_node = self._dual_residual_norm[tree_node.name]
rho_node = rho[tree_node.name]
for id_ in tree_node._standard_variable_ids:
name, index = tree_node._variable_ids[id_]
prnorm_var = prnorm_node[id_]
drnorm_var = drnorm_node[id_]
action = None
if (prnorm_var > self._mu * drnorm_var) and \
(prnorm_var > self._tol):
rho_node[id_] *= self._rho_increase_factor
action = "Increasing"
elif (drnorm_var > self._mu * prnorm_var) and \
(drnorm_var > self._tol):
rho_node[id_] *= self._rho_decrease_factor
action = "Decreasing"
if verbose:
if action is not None:
if first_print:
first_print = False
print(first_line)
print("%21s %25s %16g %16g %16g"
% (action,
name+indexToString(index),
prnorm_var,
drnorm_var,
rho_node[id_]))
self.snapshot_z(manager, z)
def extract_node_solution(tree_node):
solution = {}
for variable_id in tree_node._standard_variable_ids:
varname, index = tree_node._variable_ids[variable_id]
# store variable solution data as a list of (index, value)
# tuples We avoid nesting another dictionary mapping index ->
# value because (a) its cheaper and more lightweight as a list
# and (b) because json serializes all dictionary keys as
# strings (meaning an index of None is not recoverable)
if varname not in solution:
solution[varname] = []
if variable_id in tree_node._solution:
solution[varname].append((index, tree_node._solution[variable_id]))
else:
name, index = tree_node._variable_ids[variable_id]
full_name = name + indexToString(index)
print("%s: node solution missing for variable with scenario tree "
"id %s (%s)" % (tree_node.name, variable_id, full_name))
return None
for varname in list(solution.keys()):
solution[varname] = sorted(solution[varname], key=lambda x: x[0])
return solution
def extract_node_solution(tree_node):
solution = {}
for variable_id in tree_node._standard_variable_ids:
varname, index = tree_node._variable_ids[variable_id]
# store variable solution data as a list of (index, value)
# tuples We avoid nesting another dictionary mapping index ->
# value because (a) its cheaper and more lightweight as a list
# and (b) because json serializes all dictionary keys as
# strings (meaning an index of None is not recoverable)
if varname not in solution:
solution[varname] = []
if variable_id in tree_node._solution:
solution[varname].append((index, tree_node._solution[variable_id]))
else:
name, index = tree_node._variable_ids[variable_id]
full_name = name+indexToString(index)
print("%s: node solution missing for variable with scenario tree "
"id %s (%s)"
% (tree_node.name, variable_id, full_name))
return None
for varname in list(solution.keys()):
solution[varname] = sorted(solution[varname], key=lambda x: x[0])
return solution
def form_linearized_objective_constraints(instance_name,
instance,
scenario_tree,
linearize_nonbinary_penalty_terms,
breakpoint_strategy,
tolerance):
# keep track and return what was added to the instance, so
# it can be cleaned up if necessary.
new_instance_attributes = []
linearization_index_set_name = "PH_LINEARIZATION_INDEX_SET"
linearization_index_set = instance.find_component(linearization_index_set_name)
if linearization_index_set is None:
linearization_index_set = Set(initialize=range(0, linearize_nonbinary_penalty_terms*2), dimen=1, name=linearization_index_set_name)
instance.add_component(linearization_index_set_name, linearization_index_set)
scenario = scenario_tree.get_scenario(instance_name)
nodeid_to_vardata_map = instance._ScenarioTreeSymbolMap.bySymbol
for tree_node in scenario._node_list[:-1]:
xbar_dict = tree_node._xbars
# if linearizing, then we have previously defined a variable
# associated with the result of the linearized approximation
# of the penalty term - this is simply added to the objective
# function.
linearized_cost_variable_name = "PHQUADPENALTY_"+str(tree_node._name)
linearized_cost_variable = instance.find_component(linearized_cost_variable_name)
# grab the linearization constraint associated with the
# linearized cost variable, if it exists. otherwise, create it
# - but an empty variety. the constraints are stage-specific -
# we could index by constraint, but we don't know if that is
# really worth the additional effort.
linearization_constraint_name = "PH_LINEARIZATION_"+str(tree_node._name)
linearization_constraint = instance.find_component(linearization_constraint_name)
if linearization_constraint is not None:
# clear whatever constraint components are there - there
# may be fewer breakpoints, due to tolerances, and we
# don't want to the old pieces laying around.
linearization_constraint.clear()
else:
# this is the first time the constraint is being added -
# add it to the list of PH-specific constraints for this
# instance.
new_instance_attributes.append(linearization_constraint_name)
nodal_index_set_name = "PHINDEX_"+str(tree_node._name)
nodal_index_set = instance.find_component(nodal_index_set_name)
assert nodal_index_set is not None
linearization_constraint = \
Constraint(nodal_index_set,
linearization_index_set,
name=linearization_constraint_name)
linearization_constraint.construct()
instance.add_component(linearization_constraint_name, linearization_constraint)
for variable_id in tree_node._variable_ids:
# don't add weight terms for derived variables at the tree
# node.
if variable_id in tree_node._derived_variable_ids:
continue
if variable_id not in tree_node._minimums:
variable_name, index = tree_node._variable_ids[variable_id]
raise RuntimeError("No minimum value statistic found for variable=%s "
"on tree node=%s; cannot form linearized PH objective"
% (variable_name+indexToString(index),
tree_node._name))
if variable_id not in tree_node._maximums:
variable_name, index = tree_node._variable_ids[variable_id]
raise RuntimeError("No maximums value statistic found for "
"variable=%s on tree node=%s; cannot "
"form linearized PH objective"
% (variable_name+indexToString(index),
tree_node._name))
xbar = xbar_dict[variable_id]
node_min = tree_node._minimums[variable_id]
node_max = tree_node._maximums[variable_id]
instance_vardata = nodeid_to_vardata_map[variable_id]
if (instance_vardata.stale is False) and (instance_vardata.fixed is False):
# binaries have already been dealt with in the process of PH objective function formation.
if isinstance(instance_vardata.domain, BooleanSet) is False:
x = instance_vardata
if x.lb is None or x.ub is None:
msg = "Missing bound for variable '%s'\n" \
'Both lower and upper bounds required when' \
' piece-wise approximating quadratic ' \
'penalty terms'
raise ValueError(msg % instance_vardata.name)
lb = value(x.lb)
ub = value(x.ub)
# compute the breakpoint sequence according to the specified strategy.
try:
strategy = (compute_uniform_breakpoints,
compute_uniform_between_nodestat_breakpoints,
compute_uniform_between_woodruff_breakpoints,
compute_exponential_from_mean_breakpoints,
)[ breakpoint_strategy ]
args = ( lb, node_min, xbar, node_max, ub, \
linearize_nonbinary_penalty_terms, tolerance )
breakpoints = strategy( *args )
except ValueError:
e = sys.exc_info()[1]
msg = 'A breakpoint distribution strategy (%s) ' \
'is currently not supported within PH!'
raise ValueError(msg % breakpoint_strategy)
for i in xrange(len(breakpoints)-1):
this_lb = breakpoints[i]
this_ub = breakpoints[i+1]
segment_tuple = create_piecewise_constraint_tuple(this_lb,
this_ub,
x,
xbar,
linearized_cost_variable[variable_id],
tolerance)
linearization_constraint.add((variable_id,i), segment_tuple)
return new_instance_attributes
def Compute_and_Write_it_all(self, W_Traces, ph):
VarsOfInterest = set()
fname = self.wsummary_filename
print("sorgw.py is writing the semi-colon separated values file " +
fname)
ofile = open(fname, "w")
ofile.write(
"var; scen; WZeroCrossing; DiffsRatio; DiffZeroCrossings; w values...\n"
)
for varid in W_Traces:
assert (OnlyRootNode)
variable_name, index = ph._scenario_tree.findRootNode(
)._variable_ids[varid]
varname = variable_name + indexToString(index)
for scenname in W_Traces[varid]:
WZeroCrossings, DiffsRatio, DiffZeroCrossings = self.Score_a_Trace(
W_Traces[varid][scenname])
if self.Of_Interest(WZeroCrossings, DiffsRatio,
DiffZeroCrossings):
VarsOfInterest.add(varid)
ofile.write(varname + ';' + scenname + ';' +
str(WZeroCrossings) + ';' + str(DiffsRatio) + ';' +
str(DiffZeroCrossings))
for w in W_Traces[varid][scenname]:
ofile.write(';' + str(w))
ofile.write('\n')
ofile.close
# now processing interesting vars
BiggestLoser = None
LoserRange = 0
fname = self.winterest_filename
print("sorgw.py is writing the semi-colon separated values file " +
fname)
ofile = open(fname, "w")
ofile.write(
"var; scen; WZeroCrossing; DiffsRatio; DiffZeroCrossings; w values...\n"
)
for varid in VarsOfInterest:
vwmax = vwmin = 0
assert (OnlyRootNode)
variable_name, index = ph._scenario_tree.findRootNode(
)._variable_ids[varid]
varname = variable_name + indexToString(index)
for scenname in W_Traces[varid]:
WZeroCrossings, DiffsRatio, DiffZeroCrossings = self.Score_a_Trace(
W_Traces[varid][scenname])
ofile.write(varname + ';' + scenname + ';' +
str(WZeroCrossings) + ';' + str(DiffsRatio) + ';' +
str(DiffZeroCrossings))
for w in W_Traces[varid][scenname]:
ofile.write(';' + str(w))
if w > vwmax:
vwmax = w
if w < vwmin:
vwmin = w
ofile.write('\n')
vwrange = vwmax - vwmin
if vwrange > LoserRange:
LoserRange = vwrange
BiggestLoser = varid
ofile.close
if BiggestLoser is not None:
ph._sorgw_BiggestLoser = BiggestLoser
print("sorgw.py complete: RootNodeOnly=" + str(OnlyRootNode) +
" BiggestLoser=" + str(BiggestLoser) + " with vwrange=" +
str(vwrange))
def pre_iteration_k_solves(self, ph):
if (ph._current_iteration > self._stop_iter_rho_update) and \
all(not _converger.isConverged(ph) for _converger in ph._convergers):
return
converged = any(_converger.isConverged(ph) for _converger in ph._convergers)
rho_updated = False
adjust_rho = 0
if self._prev_avg is None:
self._snapshot_avg(ph)
else:
self._primal_residual_history.append(
self._compute_primal_residual_norm(ph))
self._dual_residual_history.append(
self._compute_dual_residual_norm(ph))
self._snapshot_avg(ph)
first_line = ("Updating Rho Values:\n%21s %25s %16s %16s %16s"
% ("Action",
"Variable",
"Primal Residual",
"Dual Residual",
"New Rho"))
first = True
for stage in ph._scenario_tree._stages[:-1]:
for tree_node in stage._tree_nodes:
primal_resid = \
math.sqrt(sum(self._primal_residual_history[-1]\
[tree_node._name].values()))
dual_resid = \
math.sqrt(sum(self._dual_residual_history[-1]\
[tree_node._name].values()))
for variable_id in tree_node._standard_variable_ids:
name, index = tree_node._variable_ids[variable_id]
primal_resid = \
math.sqrt(self._primal_residual_history[-1]\
[tree_node._name][variable_id])
dual_resid = \
math.sqrt(self._dual_residual_history[-1]\
[tree_node._name][variable_id])
action = None
rho = tree_node._scenarios[0]._rho[tree_node._name][variable_id]
if (primal_resid > 10*dual_resid) and (primal_resid > self._tol):
rho *= self._rho_increase
action = "Increasing"
elif ((dual_resid > 10*primal_resid) and (dual_resid > self._tol)):
if self._converged_count >= self._required_converged_before_decrease:
rho /= self._rho_decrease
action = "Decreasing"
elif converged:
rho /= self._rho_feasible_decrease
action = "Feasible, Decreasing"
elif (primal_resid < self._tol) and (dual_resid < self._tol):
rho /= self._rho_converged_residual_decrease
action = "Converged, Decreasing"
if action is not None:
if first:
first = False
print(first_line)
print("%21s %25s %16g %16g %16g"
% (action, name+indexToString(index),
primal_resid, dual_resid, rho))
for scenario in tree_node._scenarios:
scenario._rho[tree_node._name][variable_id] = rho
self._rho_norm_history.append(self._compute_rho_norm(ph))
if rho_updated:
print("log(|rho|) = "+repr(math.log(self._rho_norm_history[-1])))
def form_linearized_objective_constraints(instance_name,
instance,
scenario_tree,
linearize_nonbinary_penalty_terms,
breakpoint_strategy,
tolerance):
# keep track and return what was added to the instance, so
# it can be cleaned up if necessary.
new_instance_attributes = []
linearization_index_set_name = "PH_LINEARIZATION_INDEX_SET"
linearization_index_set = instance.find_component(linearization_index_set_name)
if linearization_index_set is None:
linearization_index_set = Set(initialize=range(0, linearize_nonbinary_penalty_terms*2), dimen=1, name=linearization_index_set_name)
instance.add_component(linearization_index_set_name, linearization_index_set)
scenario = scenario_tree.get_scenario(instance_name)
nodeid_to_vardata_map = instance._ScenarioTreeSymbolMap.bySymbol
for tree_node in scenario._node_list[:-1]:
xbar_dict = tree_node._xbars
# if linearizing, then we have previously defined a variable
# associated with the result of the linearized approximation
# of the penalty term - this is simply added to the objective
# function.
linearized_cost_variable_name = "PHQUADPENALTY_"+str(tree_node._name)
linearized_cost_variable = instance.find_component(linearized_cost_variable_name)
# grab the linearization constraint associated with the
# linearized cost variable, if it exists. otherwise, create it
# - but an empty variety. the constraints are stage-specific -
# we could index by constraint, but we don't know if that is
# really worth the additional effort.
linearization_constraint_name = "PH_LINEARIZATION_"+str(tree_node._name)
linearization_constraint = instance.find_component(linearization_constraint_name)
if linearization_constraint is not None:
# clear whatever constraint components are there - there
# may be fewer breakpoints, due to tolerances, and we
# don't want to the old pieces laying around.
linearization_constraint.clear()
else:
# this is the first time the constraint is being added -
# add it to the list of PH-specific constraints for this
# instance.
new_instance_attributes.append(linearization_constraint_name)
nodal_index_set_name = "PHINDEX_"+str(tree_node._name)
nodal_index_set = instance.find_component(nodal_index_set_name)
assert nodal_index_set is not None
linearization_constraint = \
Constraint(nodal_index_set,
linearization_index_set,
name=linearization_constraint_name)
linearization_constraint.construct()
instance.add_component(linearization_constraint_name, linearization_constraint)
for variable_id in tree_node._variable_ids:
# don't add weight terms for derived variables at the tree
# node.
if variable_id in tree_node._derived_variable_ids:
continue
if variable_id not in tree_node._minimums:
variable_name, index = tree_node._variable_ids[variable_id]
raise RuntimeError("No minimum value statistic found for variable=%s "
"on tree node=%s; cannot form linearized PH objective"
% (variable_name+indexToString(index),
tree_node._name))
if variable_id not in tree_node._maximums:
variable_name, index = tree_node._variable_ids[variable_id]
raise RuntimeError("No maximums value statistic found for "
"variable=%s on tree node=%s; cannot "
"form linearized PH objective"
% (variable_name+indexToString(index),
tree_node._name))
xbar = xbar_dict[variable_id]
node_min = tree_node._minimums[variable_id]
node_max = tree_node._maximums[variable_id]
instance_vardata = nodeid_to_vardata_map[variable_id]
if (instance_vardata.stale is False) and (instance_vardata.fixed is False):
# binaries have already been dealt with in the process of PH objective function formation.
if isinstance(instance_vardata.domain, BooleanSet) is False:
x = instance_vardata
if x.lb is None or x.ub is None:
msg = "Missing bound for variable '%s'\n" \
'Both lower and upper bounds required when' \
' piece-wise approximating quadratic ' \
'penalty terms'
raise ValueError(msg % instance_vardata.name)
lb = value(x.lb)
ub = value(x.ub)
# compute the breakpoint sequence according to the specified strategy.
try:
strategy = (compute_uniform_breakpoints,
compute_uniform_between_nodestat_breakpoints,
compute_uniform_between_woodruff_breakpoints,
compute_exponential_from_mean_breakpoints,
)[ breakpoint_strategy ]
args = ( lb, node_min, xbar, node_max, ub, \
linearize_nonbinary_penalty_terms, tolerance )
breakpoints = strategy( *args )
except ValueError:
e = sys.exc_info()[1]
msg = 'A breakpoint distribution strategy (%s) ' \
'is currently not supported within PH!'
raise ValueError(msg % breakpoint_strategy)
for i in xrange(len(breakpoints)-1):
this_lb = breakpoints[i]
this_ub = breakpoints[i+1]
segment_tuple = create_piecewise_constraint_tuple(this_lb,
this_ub,
x,
xbar,
linearized_cost_variable[variable_id],
tolerance)
linearization_constraint.add((variable_id,i), segment_tuple)
return new_instance_attributes
def pre_iteration_k_solves(self, ph):
if (ph._current_iteration > self._stop_iter_rho_update) and \
all(not _converger.isConverged(ph) for _converger in ph._convergers):
return
converged = any(
_converger.isConverged(ph) for _converger in ph._convergers)
rho_updated = False
adjust_rho = 0
if self._prev_avg is None:
self._snapshot_avg(ph)
else:
self._primal_residual_history.append(
self._compute_primal_residual_norm(ph))
self._dual_residual_history.append(
self._compute_dual_residual_norm(ph))
self._snapshot_avg(ph)
first_line = ("Updating Rho Values:\n%21s %25s %16s %16s %16s" %
("Action", "Variable", "Primal Residual",
"Dual Residual", "New Rho"))
first = True
for stage in ph._scenario_tree._stages[:-1]:
for tree_node in stage._tree_nodes:
primal_resid = \
math.sqrt(sum(self._primal_residual_history[-1]\
[tree_node._name].values()))
dual_resid = \
math.sqrt(sum(self._dual_residual_history[-1]\
[tree_node._name].values()))
for variable_id in tree_node._standard_variable_ids:
name, index = tree_node._variable_ids[variable_id]
primal_resid = \
math.sqrt(self._primal_residual_history[-1]\
[tree_node._name][variable_id])
dual_resid = \
math.sqrt(self._dual_residual_history[-1]\
[tree_node._name][variable_id])
action = None
rho = tree_node._scenarios[0]._rho[
tree_node._name][variable_id]
if (primal_resid > 10 * dual_resid) and (primal_resid >
self._tol):
rho *= self._rho_increase
action = "Increasing"
elif ((dual_resid > 10 * primal_resid)
and (dual_resid > self._tol)):
if self._converged_count >= self._required_converged_before_decrease:
rho /= self._rho_decrease
action = "Decreasing"
elif converged:
rho /= self._rho_feasible_decrease
action = "Feasible, Decreasing"
elif (primal_resid < self._tol) and (dual_resid <
self._tol):
rho /= self._rho_converged_residual_decrease
action = "Converged, Decreasing"
if action is not None:
if first:
first = False
print(first_line)
print("%21s %25s %16g %16g %16g" %
(action, name + indexToString(index),
primal_resid, dual_resid, rho))
for scenario in tree_node._scenarios:
scenario._rho[
tree_node._name][variable_id] = rho
self._rho_norm_history.append(self._compute_rho_norm(ph))
if rho_updated:
print("log(|rho|) = " + repr(math.log(self._rho_norm_history[-1])))
def ExtractInternalNodeSolutionsWithSlamming(ph):
from pyomo.pysp.plugins.wwphextension import _parse_yaml_file
# Since it was a file,
# assume that the argument was a json file with slamming instructions.
# This will ignore suffixes we don't care about.
# If there are no instructions use xbar.
# Note: there is an implicit pecking order.
print ("For x-hat, using slamming suffixes in",ph._xhat_method)
slamdict = {}
for suffix_name, suffix_value, variable_ids in \
_parse_yaml_file(ph, ph._xhat_method):
for node_name, node_variable_ids in iteritems(variable_ids):
for variable_id in node_variable_ids:
if variable_id not in slamdict:
slamdict[variable_id] = {}
slamdict[variable_id][suffix_name] = suffix_value
verbose = ph._verbose
node_solutions = {}
for stage in ph._scenario_tree._stages[:-1]:
for tree_node in stage._tree_nodes:
this_node_sol = node_solutions[tree_node._name] = {}
xbars = tree_node._xbars
mins = tree_node._minimums
maxs = tree_node._maximums
warnb = False # did the user do something less than cool?
for variable_id in tree_node._standard_variable_ids:
if verbose:
variable_name, index = tree_node._variable_ids[variable_id]
full_variable_name = variable_name+indexToString(index)
print ("Setting x-hat for",full_variable_name)
if variable_id not in slamdict or slamdict[variable_id]['CanSlamToAnywhere']:
if not tree_node.is_variable_discrete(variable_id):
this_node_sol[variable_id] = xbars[variable_id]
if verbose:
print (" x-bar", this_node_sol[variable_id])
else:
this_node_sol[variable_id] = int(round(xbars[variable_id]))
if verbose:
print (" rounded x-bar", this_node_sol[variable_id])
elif slamdict[variable_id]['CanSlamToMin']:
this_node_sol[variable_id] = mins[variable_id]
if verbose:
print (" min over scenarios", this_node_sol[variable_id])
elif slamdict[variable_id]['CanSlamToMax']:
this_node_sol[variable_id] = maxs[variable_id]
if verbose:
print (" max over scenarios", this_node_sol[variable_id])
elif slamdict[variable_id]['CanSlamToLB']:
warnb = True
this_node_sol[variable_id] = mins[variable_id]
if verbose:
print (" Lower Bound", this_node_sol[variable_id])
elif slamdict[variable_id]['CanSlamToUB']:
warnb = True
this_node_sol[variable_id] = maxs[variable_id]
if verbose:
print (" Upper Bound", this_node_sol[variable_id])
if warnb:
print ("Warning: for xhat determination from file %s, some variables had an upper or lower bound slam but not a corresponding min or max", ph._xhat_method)
return node_solutions
