def _solve_impl(self,
sp,
output_solver_log=False,
verbose=False,
logfile=None,
**kwds):
"""
Solve a stochastic program with the SchurIpopt solver.
See the 'solve' method on the base class for
additional keyword documentation.
Args:
sp: The stochastic program to solve.
output_solver_log (bool): Stream the solver
output during the solve.
logfile: The name of the logfile to save the
solver output into.
verbose: Report verbose status information to
aid debugging.
**kwds: Passed to the DDSIP file writer as I/O
options (e.g., symbolic_solver_labels=True).
Returns: A results object with information about the solution.
"""
#
# Setup the SchurIpopt working directory
#
problem_list_filename = "PySP_Subproblems.txt"
working_directory = self._create_tempdir("workdir",
dir=os.getcwd())
if logfile is None:
logfile = os.path.join(working_directory,
"schuripopt.log")
self._add_tempfile("logfile", logfile)
else:
self._files["logfile"] = logfile
if verbose:
print("Schuripopt solver working directory: %s"
% (working_directory))
print("Schuripopt solver logfile: %s"
% (logfile))
#
# Launch SchurIpopt from the worker processes
# (assumed to be launched together using mpirun)
#
status = self._launch_solver(
sp,
working_directory,
logfile=logfile,
output_solver_log=output_solver_log,
verbose=verbose,
io_options=kwds)
objective = 0.0
solver_status = set()
solver_message = set()
termination_condition = set()
solution_status = set()
if status.solve_type == "bundles":
assert sp.scenario_tree.contains_bundles()
assert len(status.objective) == \
len(sp.scenario_tree.bundles)
for bundle in sp.scenario_tree.bundles:
if objective is not None:
if isinstance(status.objective[bundle.name], UndefinedData):
objective = None
else:
objective += bundle.probability * \
status.objective[bundle.name]
solver_status.add(status.solver_status[bundle.name])
solver_message.add(status.solver_message[bundle.name])
termination_condition.add(status.termination_condition[bundle.name])
if isinstance(status.solution_status[bundle.name], UndefinedData):
solution_status.add(None)
else:
solution_status.add(status.solution_status[bundle.name])
else:
assert status.solve_type == "scenarios"
assert len(status.objective) == \
len(sp.scenario_tree.scenarios)
for scenario in sp.scenario_tree.scenarios:
if objective is not None:
if isinstance(status.objective[scenario.name], UndefinedData):
objective = None
else:
objective += scenario.probability * \
status.objective[scenario.name]
solver_status.add(status.solver_status[scenario.name])
solver_message.add(status.solver_message[scenario.name])
termination_condition.add(status.termination_condition[scenario.name])
if isinstance(status.solution_status[scenario.name], UndefinedData):
solution_status.add(None)
else:
solution_status.add(status.solution_status[scenario.name])
assert len(solver_status) == 1
assert len(solver_message) == 1
assert len(termination_condition) == 1
assert len(solution_status) == 1
results = SPSolverResults()
results.objective = None
results.bound = None
results.status = solution_status.pop()
results.solver.status = solver_status.pop()
results.solver.termination_condition = termination_condition.pop()
results.solver.message = solver_message.pop()
results.solver.time = max(status.solve_time.values())
results.solver.pyomo_time = \
max(status.pyomo_solve_time.values())
results.xhat = None
if str(results.solver.status) == "ok" and \
str(results.solver.termination_condition) == "optimal":
results.objective = objective
xhat = results.xhat = {}
for stage in sp.scenario_tree.stages[:-1]:
for node in stage.nodes:
worker_name = sp.get_worker_for_scenario(
node.scenarios[0].name)
node_solution = sp.invoke_function_on_worker(
worker_name,
"EXTERNAL_collect_solution",
thisfile,
invocation_type=InvocationType.Single,
function_args=(node.name,))
node_xhat = xhat[node.name] = {}
for id_ in node_solution:
node_xhat[repr(id_)] = node_solution[id_][0]
return results
def _solve_impl(self,
sp,
output_solver_log=False,
keep_solver_files=False,
symbolic_solver_labels=False):
"""
Solve a stochastic program by building the extensive
form and calling and a Pyomo solver.
See the 'solve' method on the base class for
additional keyword documentation.
Args:
sp: The stochastic program to solve. Pyro based
managers are not accepted. All scenario
models must be managed locally.
output_solver_log (bool): Stream the solver
output during the solve.
keep_solver_files (bool): Retain temporary solver
input and output files after the solve completes.
symbolic_solver_labels (bool): Generate solver
input files using human-readable symbols
(makes debugging easier).
Returns: A results object with information about the solution.
"""
if isinstance(sp,
(ScenarioTreeManagerClientPyro,
ScenarioTreeManagerSolverClientPyro)):
raise TypeError("The EF solver does not handle "
"Pyro-based scenario tree managers")
orig_parents = {}
if sp.scenario_tree.contains_bundles():
for scenario in sp.scenario_tree.scenarios:
if scenario._instance._parent is not None:
orig_parents[scenario] = scenario._instance._parent
scenario._instance._parent = None
assert not scenario._instance_objective.active
scenario._instance_objective.activate()
try:
with ExtensiveFormAlgorithm(sp, self._options) as ef:
ef.build_ef()
ef.solve(
output_solver_log=output_solver_log,
keep_solver_files=keep_solver_files,
symbolic_solver_labels=symbolic_solver_labels,
check_status=False)
finally:
for scenario, parent in orig_parents.items():
scenario._instance._parent = parent
assert scenario._instance_objective.active
scenario._instance_objective.deactivate()
results = SPSolverResults()
results.objective = ef.objective
results.bound = ef.bound
results.status = ef.solution_status
results.solver.status = ef.solver_status
results.solver.termination_condition = ef.termination_condition
results.solver.message = ef.termination_message
results.solver.time = ef.time
results.solver.pyomo_time = ef.pyomo_time
results.xhat = None
if ef.solution_status is not None:
xhat = results.xhat = {}
for stage in sp.scenario_tree.stages[:-1]:
for node in stage.nodes:
node_xhat = xhat[node.name] = {}
reference_scenario = node.scenarios[0]
node_x = reference_scenario._x[node.name]
for id_ in node._variable_ids:
node_xhat[id_] = node_x[id_]
# TODO: stage costs
#for stagenum, stage in enumerate(sp.scenario_tree.stages[:-1]):
# cost_variable_name, cost_variable_index = \
# stage._cost_variable
# stage_cost_obj = \
# instance.find_component(cost_variable_name)[cost_variable_index]
# if not stage_cost_obj.is_expression_type():
# refvar = ComponentUID(stage_cost_obj,cuid_buffer=tmp).\
# find_component(reference_model)
# refvar.value = stage_cost_obj.value
# refvar.stale = stage_cost_obj.stale
return results
def _read_solution(self, sp, symbols_filename, info_filename,
solution_filename):
"""Parses a DDSIP solution file."""
# parse the symbol map
symbol_map = {}
with open(symbols_filename) as f:
for line in f:
lp_symbol, scenario_tree_id = line.strip().split()
symbol_map[lp_symbol] = scenario_tree_id
#
# Xhat
#
try:
xhat = {}
with open(solution_filename, 'r') as f:
line = f.readline()
while line.strip() != "1. Best Solution":
line = f.readline()
line = f.readline()
assert line.startswith("Variable name Value")
line = f.readline()
assert line.startswith("-----------------------------------")
line = f.readline().strip()
while line != "":
line = line.split()
varlabel, varsol = line
xhat[symbol_map[varlabel]] = float(varsol)
line = f.readline().strip()
except (IOError, OSError):
logger.warn("Exception encountered while parsing ddsip "
"solution file '%s':\n%s'" %
(solution_filename, traceback.format_exc()))
xhat = None
#
# Objective, bound, status, etc.
#
results = SPSolverResults()
results.solver.status_code = None
results.status = None
results.solver.status = None
results.solver.termination_condition = None
results.solver.message = None
results.solver.time = None
try:
with open(info_filename, 'r') as f:
line = f.readline()
while True:
if line.startswith("Total CPU time:"):
break
line = f.readline()
if line == '':
# Unexpected file format or the solve failed
logger.warn("Unexpected ddsip info file format. No "
"status information will be returned")
return xhat, results
line = f.readline().strip()
while (line == "") or \
(line == "----------------------------------------------------------------------------------------"):
line = f.readline().strip()
if line.startswith("EEV"):
results.eev = float(line.split()[1])
line = f.readline().strip()
if line.startswith("VSS"):
results.vss = float(line.split()[1])
line = f.readline().strip()
assert line.startswith("EVPI")
line = line.split()
results.evpi = float(line[1])
line = f.readline().strip()
assert line == ""
line = f.readline().strip()
assert line == "----------------------------------------------------------------------------------------"
line = f.readline().strip()
line = line.split()
assert len(line) == 4
assert line[0] == "Status"
results.solver.status_code = int(line[1])
assert line[2] == "Time"
results.solver.time = float(line[3])
(results.status,
results.solver.status,
results.solver.termination_condition,
results.solver.message) = \
_ddsip_status_map.get(results.solver.status_code,
(SolutionStatus.unknown,
SolverStatus.unknown,
TerminationCondition.unknown,
None))
line = f.readline().strip()
line = line.split()
assert len(line) == 6
assert line[0] == "Upper"
assert line[3] == "Tree"
results.tree_depth = int(line[5])
line = f.readline().strip()
line = line.split()
assert len(line) == 2
assert line[0] == "Nodes"
results.nodes = int(line[1])
line = f.readline().strip()
while line != "----------------------------------------------------------------------------------------":
line = f.readline().strip()
line = f.readline().strip()
assert line.startswith("Best Value")
results.objective = float(line.split()[2])
# NOTE: I think DDSIP refers to the "bound" as
# "Lower Bound", even when the objective
# is being maximized.
line = f.readline().strip()
assert line.startswith("Lower Bound")
results.bound = float(line.split()[2])
except (IOError, OSError):
logger.warn("Exception encountered while parsing ddsip "
"info file '%s':\n%s'" %
(info_filename, traceback.format_exc()))
return xhat, results
def _solve_impl(self,
sp,
output_solver_log=False,
keep_solver_files=False,
symbolic_solver_labels=False):
"""
Solve a stochastic program by building the extensive
form and calling and a Pyomo solver.
See the 'solve' method on the base class for
additional keyword documentation.
Args:
sp: The stochastic program to solve. Pyro based
managers are not accepted. All scenario
models must be managed locally.
output_solver_log (bool): Stream the solver
output during the solve.
keep_solver_files (bool): Retain temporary solver
input and output files after the solve completes.
symbolic_solver_labels (bool): Generate solver
input files using human-readable symbols
(makes debugging easier).
Returns: A results object with information about the solution.
"""
if isinstance(sp, (ScenarioTreeManagerClientPyro,
ScenarioTreeManagerSolverClientPyro)):
raise TypeError("The EF solver does not handle "
"Pyro-based scenario tree managers")
orig_parents = {}
if sp.scenario_tree.contains_bundles():
for scenario in sp.scenario_tree.scenarios:
if scenario._instance._parent is not None:
orig_parents[scenario] = scenario._instance._parent
scenario._instance._parent = None
assert not scenario._instance_objective.active
scenario._instance_objective.activate()
try:
with ExtensiveFormAlgorithm(sp, self._options) as ef:
ef.build_ef()
ef.solve(output_solver_log=output_solver_log,
keep_solver_files=keep_solver_files,
symbolic_solver_labels=symbolic_solver_labels,
check_status=False)
finally:
for scenario, parent in orig_parents.items():
scenario._instance._parent = parent
assert scenario._instance_objective.active
scenario._instance_objective.deactivate()
results = SPSolverResults()
results.objective = ef.objective
results.bound = ef.bound
results.status = ef.solution_status
results.solver.status = ef.solver_status
results.solver.termination_condition = ef.termination_condition
results.solver.message = ef.termination_message
results.solver.time = ef.time
results.solver.pyomo_time = ef.pyomo_time
results.xhat = None
if ef.solution_status is not None:
xhat = results.xhat = {}
for stage in sp.scenario_tree.stages[:-1]:
for node in stage.nodes:
node_xhat = xhat[node.name] = {}
reference_scenario = node.scenarios[0]
node_x = reference_scenario._x[node.name]
for id_ in node._variable_ids:
node_xhat[id_] = node_x[id_]
# TODO: stage costs
#for stagenum, stage in enumerate(sp.scenario_tree.stages[:-1]):
# cost_variable_name, cost_variable_index = \
# stage._cost_variable
# stage_cost_obj = \
# instance.find_component(cost_variable_name)[cost_variable_index]
# if not stage_cost_obj.is_expression_type():
# refvar = ComponentUID(stage_cost_obj,cuid_buffer=tmp).\
# find_component(reference_model)
# refvar.value = stage_cost_obj.value
# refvar.stale = stage_cost_obj.stale
return results
def _solve_impl(self,
sp,
output_solver_log=False,
verbose=False,
logfile=None,
**kwds):
"""
Solve a stochastic program with the SchurIpopt solver.
See the 'solve' method on the base class for
additional keyword documentation.
Args:
sp: The stochastic program to solve.
output_solver_log (bool): Stream the solver
output during the solve.
logfile: The name of the logfile to save the
solver output into.
verbose: Report verbose status information to
aid debugging.
**kwds: Passed to the DDSIP file writer as I/O
options (e.g., symbolic_solver_labels=True).
Returns: A results object with information about the solution.
"""
#
# Setup the SchurIpopt working directory
#
problem_list_filename = "PySP_Subproblems.txt"
working_directory = self._create_tempdir("workdir",
dir=os.getcwd())
if logfile is None:
logfile = os.path.join(working_directory,
"schuripopt.log")
self._add_tempfile("logfile", logfile)
else:
self._files["logfile"] = logfile
if verbose:
print("Schuripopt solver working directory: %s"
% (working_directory))
print("Schuripopt solver logfile: %s"
% (logfile))
#
# Launch SchurIpopt from the worker processes
# (assumed to be launched together using mpirun)
#
status = self._launch_solver(
sp,
working_directory,
logfile=logfile,
output_solver_log=output_solver_log,
verbose=verbose,
io_options=kwds)
objective = 0.0
solver_status = set()
solver_message = set()
termination_condition = set()
solution_status = set()
if status.solve_type == "bundles":
assert sp.scenario_tree.contains_bundles()
assert len(status.objective) == \
len(sp.scenario_tree.bundles)
for bundle in sp.scenario_tree.bundles:
if objective is not None:
if isinstance(status.objective[bundle.name], UndefinedData):
objective = None
else:
objective += bundle.probability * \
status.objective[bundle.name]
solver_status.add(status.solver_status[bundle.name])
solver_message.add(status.solver_message[bundle.name])
termination_condition.add(status.termination_condition[bundle.name])
if isinstance(status.solution_status[bundle.name], UndefinedData):
solution_status.add(None)
else:
solution_status.add(status.solution_status[bundle.name])
else:
assert status.solve_type == "scenarios"
assert len(status.objective) == \
len(sp.scenario_tree.scenarios)
for scenario in sp.scenario_tree.scenarios:
if objective is not None:
if isinstance(status.objective[scenario.name], UndefinedData):
objective = None
else:
objective += scenario.probability * \
status.objective[scenario.name]
solver_status.add(status.solver_status[scenario.name])
solver_message.add(status.solver_message[scenario.name])
termination_condition.add(status.termination_condition[scenario.name])
if isinstance(status.solution_status[scenario.name], UndefinedData):
solution_status.add(None)
else:
solution_status.add(status.solution_status[scenario.name])
assert len(solver_status) == 1
assert len(solver_message) == 1
assert len(termination_condition) == 1
assert len(solution_status) == 1
results = SPSolverResults()
results.objective = None
results.bound = None
results.status = solution_status.pop()
results.solver.status = solver_status.pop()
results.solver.termination_condition = termination_condition.pop()
results.solver.message = solver_message.pop()
results.solver.time = max(status.solve_time.values())
results.solver.pyomo_time = \
max(status.pyomo_solve_time.values())
results.xhat = None
if str(results.solver.status) == "ok" and \
str(results.solver.termination_condition) == "optimal":
results.objective = objective
xhat = results.xhat = {}
for stage in sp.scenario_tree.stages[:-1]:
for node in stage.nodes:
worker_name = sp.get_worker_for_scenario(
node.scenarios[0].name)
node_solution = sp.invoke_function_on_worker(
worker_name,
"EXTERNAL_collect_solution",
thisfile,
invocation_type=InvocationType.Single,
function_args=(node.name,))
node_xhat = xhat[node.name] = {}
for id_ in node_solution:
node_xhat[repr(id_)] = node_solution[id_][0]
return results
def _read_solution(self, symbols_filename, solution_filename):
""" Parses an SD solution file """
# parse the symbol map
symbol_map = {}
with open(symbols_filename) as f:
for line in f:
lp_symbol, scenario_tree_id = line.strip().split()
symbol_map[lp_symbol] = scenario_tree_id
results = SPSolverResults()
results.status = None
results.solver.status = None
results.solver.termination_condition = None
results.solver.message = None
xhat = {}
try:
with open(solution_filename, 'r') as f:
line = f.readline()
assert line.startswith("Problem:")
assert line.split()[1].strip() == "pysp_model"
line = f.readline()
assert line.startswith("First Stage Rows:")
line = f.readline()
assert line.startswith("First Stage Columns:")
line = f.readline()
assert line.startswith("First Stage Non-zeros:")
line = f.readline()
assert line.startswith("Replication No.") or \
line.startswith("Number of replications:")
line = f.readline()
assert line.startswith("Status:")
results.solver.message = line.split(":")[1].strip()
(results.status,
results.solver.status,
results.solver.termination_condition) = \
_sd_status_map.get(results.solver.message,
(SolutionStatus.unknown,
SolverStatus.unknown,
TerminationCondition.unknown))
#
# Objective and Bound
#
line = f.readline()
assert line.startswith("Total Objective Function Upper Bound:")
line = line.split(':')
if line[1].strip() == '':
pass
else:
assert len(line) == 4
line = line[1]
if "half-width" in line:
# we are given confidence intervals on the objective
line = line.split(',')
assert len(line) == 4
results.objective = float(line[0])
assert line[1].startswith('[')
assert line[2].endswith(']')
results.objective_interval = (float(line[1][1:]),
float(line[2][:-1]))
else:
results.objective = float(line[1])
line = f.readline()
assert line.startswith("Total Objective Function Lower Bound:")
line = line.split(':')
if line[1].strip() == '':
pass
else:
if "half-width" in line[1]:
# we are given confidence intervals on the bound
line = line[1].split(',')
assert len(line) == 4
results.bound = float(line[0])
assert line[1].startswith('[')
assert line[2].endswith(']')
results.bound_interval = (float(line[1][1:]),
float(line[2][:-1]))
else:
results.bound = float(line[1].strip())
#
# Xhat
#
line = f.readline()
assert line.strip() == ''
line = f.readline()
assert line.startswith('First Stage Solutions:')
line = f.readline()
assert line.startswith(
' No. Row name Activity Lower bound Upper bound Dual Dual STDEV'
)
line = f.readline()
assert line.startswith(
'------ ------------ ------------- ------------- ------------- ------------- -------------'
)
xhat_start_line = ' No. Column name Activity Lower bound Upper bound Reduced Cost RC STDEV'
line = f.readline()
while not line.startswith(xhat_start_line):
line = f.readline()
line = f.readline()
assert line.startswith(
'------ ------------ ------------- ------------- ------------- ------------- -------------'
)
line = f.readline().strip().split()
while line:
varlabel, varvalue = line[1:3]
varlabel = varlabel.strip()
varvalue = float(varvalue)
xhat[symbol_map[varlabel]] = varvalue
line = f.readline().strip().split()
except (IOError, OSError):
logger.warn("Exception encountered while parsing sd "
"solution file '%s':\n%s'" %
(solution_filename, traceback.format_exc()))
xhat = None
return xhat, results
def _read_solution(self,
sp,
symbols_filename,
info_filename,
solution_filename):
"""Parses a DDSIP solution file."""
# parse the symbol map
symbol_map = {}
with open(symbols_filename) as f:
for line in f:
lp_symbol, scenario_tree_id = line.strip().split()
symbol_map[lp_symbol] = scenario_tree_id
#
# Xhat
#
try:
xhat = {}
with open(solution_filename, 'r') as f:
line = f.readline()
while line.strip() != "1. Best Solution":
line = f.readline()
line = f.readline()
assert line.startswith("Variable name Value")
line = f.readline()
assert line.startswith("-----------------------------------")
line = f.readline().strip()
while line != "":
line = line.split()
varlabel, varsol = line
xhat[symbol_map[varlabel]] = float(varsol)
line = f.readline().strip()
except (IOError, OSError):
logger.warn(
"Exception encountered while parsing ddsip "
"solution file '%s':\n%s'"
% (solution_filename, traceback.format_exc()))
xhat = None
#
# Objective, bound, status, etc.
#
results = SPSolverResults()
results.solver.status_code = None
results.status = None
results.solver.status = None
results.solver.termination_condition = None
results.solver.message = None
results.solver.time = None
try:
with open(info_filename, 'r') as f:
line = f.readline()
while True:
if line.startswith("Total CPU time:"):
break
line = f.readline()
if line == '':
# Unexpected file format or the solve failed
logger.warn(
"Unexpected ddsip info file format. No "
"status information will be returned")
return xhat, results
line = f.readline().strip()
while (line == "") or \
(line == "----------------------------------------------------------------------------------------"):
line = f.readline().strip()
if line.startswith("EEV"):
results.eev = float(line.split()[1])
line = f.readline().strip()
if line.startswith("VSS"):
results.vss = float(line.split()[1])
line = f.readline().strip()
assert line.startswith("EVPI")
line = line.split()
results.evpi = float(line[1])
line = f.readline().strip()
assert line == ""
line = f.readline().strip()
assert line == "----------------------------------------------------------------------------------------"
line = f.readline().strip()
line = line.split()
assert len(line) == 4
assert line[0] == "Status"
results.solver.status_code = int(line[1])
assert line[2] == "Time"
results.solver.time = float(line[3])
(results.status,
results.solver.status,
results.solver.termination_condition,
results.solver.message) = \
_ddsip_status_map.get(results.solver.status_code,
(SolutionStatus.unknown,
SolverStatus.unknown,
TerminationCondition.unknown,
None))
line = f.readline().strip()
line = line.split()
assert len(line) == 6
assert line[0] == "Upper"
assert line[3] == "Tree"
results.tree_depth = int(line[5])
line = f.readline().strip()
line = line.split()
assert len(line) == 2
assert line[0] == "Nodes"
results.nodes = int(line[1])
line = f.readline().strip()
while line != "----------------------------------------------------------------------------------------":
line = f.readline().strip()
line = f.readline().strip()
assert line.startswith("Best Value")
results.objective = float(line.split()[2])
# NOTE: I think DDSIP refers to the "bound" as
# "Lower Bound", even when the objective
# is being maximized.
line = f.readline().strip()
assert line.startswith("Lower Bound")
results.bound = float(line.split()[2])
except (IOError, OSError):
logger.warn(
"Exception encountered while parsing ddsip "
"info file '%s':\n%s'"
% (info_filename, traceback.format_exc()))
return xhat, results
