def __init__(self, manager, *args, **kwds):
import pyomo.solvers.plugins.smanager.pyro
super(ExtensiveFormAlgorithm, self).__init__(*args, **kwds)
# TODO: after PH moves over to the new code
#if not isinstance(manager, ScenarioTreeManager):
# raise TypeError("ExtensiveFormAlgorithm requires an instance of the "
# "ScenarioTreeManager interface as the "
# "second argument")
if not manager.initialized:
raise ValueError("ExtensiveFormAlgorithm requires a scenario tree "
"manager that has been fully initialized")
self._manager = manager
self.instance = None
self._solver_manager = None
self._solver = None
# The following attributes will be modified by the
# solve() method. For users that are scripting, these
# can be accessed after the solve() method returns.
# They will be reset each time solve() is called.
############################################
self.objective = None
self.gap = None
self.termination_condition = None
self.termination_message = None
self.solver_status = None
self.solution_status = None
self.solver_results = None
self.time = None
self.pyomo_time = None
############################################
# apparently the SolverFactory does not have sane
# behavior when the solver name is None
if self.get_option("solver") is None:
raise ValueError("The 'solver' option can not be None")
self._solver = SolverFactory(self.get_option("solver"),
solver_io=self.get_option("solver_io"))
if isinstance(self._solver, UnknownSolver):
raise ValueError("Failed to create solver of type=" +
self.get_option("solver") +
" for use in extensive form solve")
solver_manager_type = self.get_option("solver_manager")
if solver_manager_type == "phpyro":
print("*** WARNING ***: PHPyro is not a supported solver "
"manager type for the extensive-form solver. "
"Falling back to serial.")
solver_manager_type = 'serial'
self._solver_manager = SolverManagerFactory(
solver_manager_type,
host=self.get_option("solver_manager_pyro_host"),
port=self.get_option("solver_manager_pyro_port"))
if self._solver_manager is None:
raise ValueError("Failed to create solver manager of type=" +
self.get_option("solver") +
" for use in extensive form solve")
def __init__(self, *args, **kwds):
if self.__class__ is _ScenarioTreeManagerSolverWorker:
raise NotImplementedError(
"%s is an abstract class for subclassing" % self.__class__)
super(_ScenarioTreeManagerSolverWorker, self).__init__(*args, **kwds)
assert self.manager is not None
# solver related objects
self._scenario_solvers = {}
self._bundle_solvers = {}
self._preprocessor = None
self._solver_manager = None
# there are situations in which it is valuable to snapshot /
# store the solutions associated with the scenario
# instances. for example, when one wants to use a warm-start
# from a particular iteration solve, following a modification
# and re-solve of the problem instances in a user-defined
# callback. the following nested dictionary is intended to
# serve that purpose. The nesting is dependent on whether
# bundling and or phpyro is in use
self._cached_solutions = {}
self._cached_scenariotree_solutions = {}
#
# initialize the preprocessor
#
self._preprocessor = None
if not self.get_option("disable_advanced_preprocessing"):
self._preprocessor = ScenarioTreePreprocessor(
self._options, options_prefix=self._options_prefix)
assert self._manager.preprocessor is None
self._manager.preprocessor = self._preprocessor
#
# initialize the solver manager
#
self._solver_manager = SolverManagerFactory(
self.get_option("solver_manager"),
host=self.get_option('solver_manager_pyro_host'),
port=self.get_option('solver_manager_pyro_port'))
for scenario in self.manager.scenario_tree._scenarios:
assert scenario._instance is not None
solver = self._scenario_solvers[scenario.name] = \
SolverFactory(self.get_option("solver"),
solver_io=self.get_option("solver_io"))
if self._preprocessor is not None:
self._preprocessor.add_scenario(scenario, scenario._instance,
solver)
for bundle in self.manager.scenario_tree._scenario_bundles:
solver = self._bundle_solvers[bundle.name] = \
SolverFactory(self.get_option("solver"),
solver_io=self.get_option("solver_io"))
bundle_instance = \
self.manager._bundle_binding_instance_map[bundle.name]
if self._preprocessor is not None:
self._preprocessor.add_bundle(bundle, bundle_instance, solver)
def __init__(self, manager, *args, **kwds):
import pyomo.solvers.plugins.smanager.pyro
super(ExtensiveFormAlgorithm, self).__init__(*args, **kwds)
# TODO: after PH moves over to the new code
#if not isinstance(manager, ScenarioTreeManager):
# raise TypeError("ExtensiveFormAlgorithm requires an instance of the "
# "ScenarioTreeManager interface as the "
# "second argument")
if not manager.initialized:
raise ValueError("ExtensiveFormAlgorithm requires a scenario tree "
"manager that has been fully initialized")
self._manager = manager
self.instance = None
self._solver_manager = None
self._solver = None
# The following attributes will be modified by the
# solve() method. For users that are scripting, these
# can be accessed after the solve() method returns.
# They will be reset each time solve() is called.
############################################
self.objective = None
self.gap = None
self.termination_condition = None
self.termination_message = None
self.solver_status = None
self.solution_status = None
self.solver_results = None
self.time = None
self.pyomo_time = None
############################################
# apparently the SolverFactory does not have sane
# behavior when the solver name is None
if self.get_option("solver") is None:
raise ValueError("The 'solver' option can not be None")
self._solver = SolverFactory(self.get_option("solver"),
solver_io=self.get_option("solver_io"))
if isinstance(self._solver, UnknownSolver):
raise ValueError("Failed to create solver of type="+
self.get_option("solver")+
" for use in extensive form solve")
solver_manager_type = self.get_option("solver_manager")
if solver_manager_type == "phpyro":
print("*** WARNING ***: PHPyro is not a supported solver "
"manager type for the extensive-form solver. "
"Falling back to serial.")
solver_manager_type = 'serial'
self._solver_manager = SolverManagerFactory(
solver_manager_type,
host=self.get_option("solver_manager_pyro_host"),
port=self.get_option("solver_manager_pyro_port"))
if self._solver_manager is None:
raise ValueError("Failed to create solver manager of type="
+self.get_option("solver")+
" for use in extensive form solve")
def pyomo_solver(model, solver, config, neos=False):
"""Solve the continuous NLP by discretizing it using backward finite difference and pyomo solvers
Arguments:
model {pyomo model} -- a concrete model build using Pyomo
solver {string} -- Solver used for the optimization problem. For e.g. ipopt
config{yaml} -- configurations of the setting
neos {bool} -- Use the neos solver if True
"""
# time variables
if model.tf < 1.0:
N = config['freq']
else:
N = int(round(model.tf * config['freq']))
# finite difference discretization
# pyo.TransformationFactory('dae.finite_difference').apply_to(model, wrt=model.t, nfe=N, scheme='BACKWARD')
# direct collocation
pyo.TransformationFactory('dae.collocation').apply_to(model,
wrt=model.t,
nfe=N,
ncp=6)
# if a particular variable has to be piecewise constant then uncomment the following and replace the var parameter in 'reduce_collocation_points'
# pyo.TransformationFactory('dae.collocation').apply_to(model, wrt=model.t, nf=10, ncp=6).reduce_collocation_points(model, var=model.u, ncp=1, contset=model.t)
if neos:
print("Using NEOS with Pyomo")
SolverManagerFactory('neos').solve(model, opt=solver).write()
else:
pyo.SolverFactory(solver).solve(model).write()
return model
def __init__(self, params):
self.numScen = params.numScen
self.mstX = params.x
self.mstSita = params.sita
sb_mdl = import_file("sub.py").model
sub_insts = []
global directory
for s in range(self.numScen):
sub_path = directory + "\\data\\ScenNode" + str(s + 1) + ".dat"
sub_insts.append(
sb_mdl.create_instance(name="sub"+str(s+1), \
filename=sub_path))
solver_manager = SolverManagerFactory("serial")
self.model = sb_mdl
self.instance = sub_insts
self.solver_manager = solver_manager
def run_path(data):
#call solver
opt = SolverFactory('cplex')
solver_manager = SolverManagerFactory('pyro')
#report_timing = True to see time taken for each model component
inst = pmod.model.create_instance(data, report_timing=True)
#fix variable I to be 0
for (h, v, t) in inst.aux:
inst.I[h, v, t].fix(0)
#tee=True to trace the cplex process
result = opt.solve(inst, tee=True, warmstart=True)
inst.solutions.load_from(result)
'''
for (s,e) in inst.OD:
for p in inst.P:
for j in inst.N:
for t in inst.TS:
if inst.q[s,e,p,j,t]() > 0:
print 'queue', s,e,p,j,t,inst.q[s,e,p,j,t]()
'''
for (s, e) in inst.OD:
for p in inst.P:
for j in inst.N:
for t in inst.TS:
if inst.f[s, e, p, j, t]() > 0:
print >> f, 'outflow', s, e, p, j, t, inst.f[s, e, p,
j, t]()
'''
for (s,e) in inst.OD:
for p in inst.P:
for j in inst.N:
for t in inst.TS:
for v in inst.VS:
if inst.b[s,e,p,v,t,j]() > 0:
print 'board', s,e,p,v,t,j,inst.b[s,e,p,v,t,j]()
'''
print >> f, 'objective value', inst.obj()
#result.write()
inst.display()
exec_time = time.time() - start_time
print "execution time", exec_time, 's'
class ExtensiveFormAlgorithm(PySPConfiguredObject):
@classmethod
def _declare_options(cls, options=None):
if options is None:
options = PySPConfigBlock()
safe_declare_unique_option(
options,
"cvar_weight",
PySPConfigValue(
1.0,
domain=_domain_nonnegative,
description=("The weight associated with the CVaR term in "
"the risk-weighted objective "
"formulation. If the weight is 0, then "
"*only* a non-weighted CVaR cost will appear "
"in the EF objective - the expected cost "
"component will be dropped. Default is 1.0."),
doc=None,
visibility=0),
ap_group=_ef_group_label)
safe_declare_unique_option(
options,
"generate_weighted_cvar",
PySPConfigValue(
False,
domain=bool,
description=("Add a weighted CVaR term to the "
"primary objective. Default is False."),
doc=None,
visibility=0),
ap_group=_ef_group_label)
safe_declare_unique_option(
options,
"risk_alpha",
PySPConfigValue(
0.95,
domain=_domain_unit_interval,
description=("The probability threshold associated with "
"CVaR (or any future) risk-oriented "
"performance metrics. Default is 0.95."),
doc=None,
visibility=0),
ap_group=_ef_group_label)
safe_declare_unique_option(
options,
"cc_alpha",
PySPConfigValue(
0.0,
domain=_domain_unit_interval,
description=("The probability threshold associated with a "
"chance constraint. The RHS will be one "
"minus this value. Default is 0."),
doc=None,
visibility=0),
ap_group=_ef_group_label)
safe_declare_unique_option(
options,
"cc_indicator_var",
PySPConfigValue(
None,
domain=_domain_must_be_str,
description=("The name of the binary variable to be used "
"to construct a chance constraint. Default "
"is None, which indicates no chance "
"constraint."),
doc=None,
visibility=0),
ap_group=_ef_group_label)
safe_declare_common_option(options, "solver")
safe_declare_common_option(options, "solver_io")
safe_declare_common_option(options, "solver_manager")
safe_declare_common_option(options, "solver_options")
safe_declare_common_option(options, "disable_warmstart")
safe_declare_common_option(options, "solver_manager_pyro_host")
safe_declare_common_option(options, "solver_manager_pyro_port")
safe_declare_common_option(options, "solver_manager_pyro_shutdown")
safe_declare_common_option(options,
"verbose",
ap_group=_ef_group_label)
safe_declare_common_option(options,
"output_times",
ap_group=_ef_group_label)
safe_declare_common_option(options,
"output_solver_results",
ap_group=_ef_group_label)
return options
def __enter__(self):
return self
def __exit__(self, *args):
self.close()
def close(self):
self.destroy_ef()
if self._solver_manager is not None:
if isinstance(self._solver_manager,
pyomo.solvers.plugins.smanager.\
pyro.SolverManager_Pyro):
if self.get_option("pyro_shutdown_workers"):
self._solver_manager.shutdown_workers()
self._solver_manager = None
self._manager = None
def __init__(self, manager, *args, **kwds):
import pyomo.solvers.plugins.smanager.pyro
super(ExtensiveFormAlgorithm, self).__init__(*args, **kwds)
# TODO: after PH moves over to the new code
#if not isinstance(manager, ScenarioTreeManager):
# raise TypeError("ExtensiveFormAlgorithm requires an instance of the "
# "ScenarioTreeManager interface as the "
# "second argument")
if not manager.initialized:
raise ValueError("ExtensiveFormAlgorithm requires a scenario tree "
"manager that has been fully initialized")
self._manager = manager
self.instance = None
self._solver_manager = None
self._solver = None
# The following attributes will be modified by the
# solve() method. For users that are scripting, these
# can be accessed after the solve() method returns.
# They will be reset each time solve() is called.
############################################
self.objective = None
self.gap = None
self.termination_condition = None
self.termination_message = None
self.solver_status = None
self.solution_status = None
self.solver_results = None
self.time = None
self.pyomo_time = None
############################################
# apparently the SolverFactory does not have sane
# behavior when the solver name is None
if self.get_option("solver") is None:
raise ValueError("The 'solver' option can not be None")
self._solver = SolverFactory(self.get_option("solver"),
solver_io=self.get_option("solver_io"))
if isinstance(self._solver, UnknownSolver):
raise ValueError("Failed to create solver of type=" +
self.get_option("solver") +
" for use in extensive form solve")
solver_manager_type = self.get_option("solver_manager")
if solver_manager_type == "phpyro":
print("*** WARNING ***: PHPyro is not a supported solver "
"manager type for the extensive-form solver. "
"Falling back to serial.")
solver_manager_type = 'serial'
self._solver_manager = SolverManagerFactory(
solver_manager_type,
host=self.get_option("solver_manager_pyro_host"),
port=self.get_option("solver_manager_pyro_port"))
if self._solver_manager is None:
raise ValueError("Failed to create solver manager of type=" +
self.get_option("solver") +
" for use in extensive form solve")
def build_ef(self):
self.destroy_ef()
if self.get_option("verbose"):
print("Creating extensive form instance")
start_time = time.time()
# then validate the associated parameters.
generate_weighted_cvar = False
cvar_weight = None
risk_alpha = None
if self.get_option("generate_weighted_cvar"):
generate_weighted_cvar = True
cvar_weight = self.get_option("cvar_weight")
risk_alpha = self.get_option("risk_alpha")
self.instance = create_ef_instance(
self._manager.scenario_tree,
verbose_output=self.get_option("verbose"),
generate_weighted_cvar=generate_weighted_cvar,
cvar_weight=cvar_weight,
risk_alpha=risk_alpha,
cc_indicator_var_name=self.get_option("cc_indicator_var"),
cc_alpha=self.get_option("cc_alpha"))
if self.get_option("verbose") or self.get_option("output_times"):
print("Time to construct extensive form instance=%.2f seconds" %
(time.time() - start_time))
def destroy_ef(self):
if self.instance is not None:
for scenario in self._manager.scenario_tree.scenarios:
self.instance.del_component(scenario.name)
scenario._instance_objective.activate()
self.instance = None
def write(self, filename):
if self.instance is None:
raise RuntimeError(
"The extensive form instance has not been constructed."
"Call the build_ef() method to construct it.")
suf = os.path.splitext(filename)[1]
if suf not in ['.nl', '.lp', '.mps']:
if self._solver.problem_format() == ProblemFormat.cpxlp:
filename += '.lp'
elif self._solver.problem_format() == ProblemFormat.nl:
filename += '.nl'
elif self._solver.problem_format() == ProblemFormat.mps:
filename += '.mps'
else:
raise ValueError("Could not determine output file format. "
"No recognized ending suffix was provided "
"and no format was indicated was by the "
"--solver-io option.")
start_time = time.time()
print("Writing extensive form to file=" + filename)
smap_id = write_ef(self.instance, filename,
self.get_option("symbolic_solver_labels"))
if self.get_option("verbose") or self.get_option("output_times"):
print("Time to write output file=%.2f seconds" %
(time.time() - start_time))
return filename, smap_id
def solve(self,
check_status=True,
exception_on_failure=True,
output_solver_log=False,
symbolic_solver_labels=False,
keep_solver_files=False,
io_options=None):
# TODO: Does this import need to be delayed because
# it is in a plugins subdirectory
from pyomo.solvers.plugins.solvers.persistent_solver import \
PersistentSolver
if self.instance is None:
raise RuntimeError(
"The extensive form instance has not been constructed."
"Call the build_ef() method to construct it.")
start_time = time.time()
if self.get_option("verbose"):
print("Queuing extensive form solve")
self.objective = None
self.gap = None
self.bound = None
self.termination_condition = None
self.termination_message = None
self.solver_status = None
self.solution_status = None
self.solver_results = None
self.time = None
self.pyomo_time = None
if isinstance(self._solver, PersistentSolver):
self._solver.compile_instance(
self.instance, symbolic_solver_labels=symbolic_solver_labels)
solve_kwds = {}
solve_kwds['load_solutions'] = False
if keep_solver_files:
solve_kwds['keepfiles'] = True
if symbolic_solver_labels:
solve_kwds['symbolic_solver_labels'] = True
if output_solver_log:
solve_kwds['tee'] = True
solver_options = self.get_option("solver_options")
if len(solver_options) > 0:
if type(solver_options) is tuple:
solve_kwds["options"] = {}
for name_val in solver_options:
assert "=" in name_val
name, val = name_val.split("=")
solve_kwds["options"][name.strip()] = val.strip()
else:
solve_kwds["options"] = solver_options
if io_options is not None:
solve_kwds.update(io_options)
self.objective_sense = \
find_active_objective(self.instance).sense
if (not self.get_option("disable_warmstart")) and \
(self._solver.warm_start_capable()):
action_handle = self._solver_manager.queue(self.instance,
opt=self._solver,
warmstart=True,
**solve_kwds)
else:
action_handle = self._solver_manager.queue(self.instance,
opt=self._solver,
**solve_kwds)
if self.get_option("verbose"):
print("Waiting for extensive form solve")
results = self._solver_manager.wait_for(action_handle)
if self.get_option("verbose"):
print("Done with extensive form solve - loading results")
if self.get_option("output_solver_results"):
print("Results for ef:")
results.write(num=1)
self.solver_results = results
if hasattr(results.solver,"user_time") and \
(not isinstance(results.solver.user_time,
UndefinedData)) and \
(results.solver.user_time is not None):
# the solve time might be a string, or might
# not be - we eventually would like more
# consistency on this front from the solver
# plugins.
self.time = \
float(results.solver.user_time)
elif hasattr(results.solver, "time"):
self.time = \
float(results.solver.time)
else:
self.time = None
if hasattr(results, "pyomo_solve_time"):
self.pyomo_time = \
results.pyomo_solve_time
else:
self.pyomo_time = None
self.termination_condition = \
results.solver.termination_condition
self.termination_message = None
if hasattr(results.solver, "termination_message"):
self.termination_message = results.solver.termination_message
elif hasattr(results.solver, "message"):
self.termination_message = results.solver.message
self.solver_status = \
results.solver.status
if len(results.solution) > 0:
assert len(results.solution) == 1
results_sm = results._smap
self.instance.solutions.load_from(results)
solution0 = results.solution(0)
if hasattr(solution0, "gap") and \
(solution0.gap is not None) and \
(not isinstance(solution0.gap,
UndefinedData)):
self.gap = float(solution0.gap)
else:
self.gap = None
self.solution_status = solution0.status
if self.get_option("verbose"):
print("Storing solution in scenario tree")
for scenario in self._manager.scenario_tree.scenarios:
scenario.update_solution_from_instance()
self._manager.scenario_tree.snapshotSolutionFromScenarios()
self.objective = self._manager.scenario_tree.\
findRootNode().\
computeExpectedNodeCost()
if self.gap is not None:
if self.objective_sense == pyomo.core.base.minimize:
self.bound = self.objective - self.gap
else:
self.bound = self.objective + self.gap
else:
self.objective = None
self.gap = None
self.bound = None
self.solution_status = None
failure = False
if check_status:
if not ((self.solution_status == SolutionStatus.optimal) or \
(self.solution_status == SolutionStatus.feasible)):
failure = True
if self.get_option("verbose") or \
exception_on_failure:
msg = ("EF solve failed solution status check:\n"
"Solver Status: %s\n"
"Termination Condition: %s\n"
"Solution Status: %s\n" %
(self.solver_status, self.termination_condition,
self.solution_status))
if self.get_option("verbose"):
print(msg)
if exception_on_failure:
raise RuntimeError(msg)
else:
if self.get_option("verbose"):
print("EF solve completed. Skipping status check.")
if self.get_option("verbose") or self.get_option("output_times"):
print("Time to solve and load results for the "
"extensive form=%.2f seconds" % (time.time() - start_time))
return failure
# parallel.py
from __future__ import division
from pyomo.environ import *
from pyomo.opt import SolverFactory
from pyomo.opt.parallel import SolverManagerFactory
import sys
action_handle_map = {} # maps action handles to instances
# Create a solver
optsolver = SolverFactory('cplex')
# create a solver manager
# 'pyro' could be replaced with 'serial'
solver_manager = SolverManagerFactory('pyro')
if solver_manager is None:
print "Failed to create solver manager."
sys.exit(1)
#
# A simple model with binary variables and
# an empty constraint list.
#
model = AbstractModel()
model.n = Param(default=4)
model.x = Var(RangeSet(model.n), within=Binary)
def o_rule(model):
return sum_product(model.x)
model.o = Objective(rule=o_rule)
model.c = ConstraintList()
def __init__(self, manager, *args, **kwds):
import pysp.pyro.smanager_pyro
super(ExtensiveFormAlgorithm, self).__init__(*args, **kwds)
# TODO: after PH moves over to the new code
#if not isinstance(manager, ScenarioTreeManager):
# raise TypeError("ExtensiveFormAlgorithm requires an instance of the "
# "ScenarioTreeManager interface as the "
# "second argument")
if not manager.initialized:
raise ValueError("ExtensiveFormAlgorithm requires a scenario tree "
"manager that has been fully initialized")
self._manager = manager
self.instance = None
self._solver_manager = None
self._solver = None
# The following attributes will be modified by the
# solve() method. For users that are scripting, these
# can be accessed after the solve() method returns.
# They will be reset each time solve() is called.
############################################
self.objective = undefined
self.gap = undefined
self.termination_condition = undefined
self.solver_status = undefined
self.solution_status = undefined
self.solver_results = undefined
self.pyomo_solve_time = undefined
self.solve_time = undefined
############################################
self._solver = SolverFactory(self.get_option("solver"),
solver_io=self.get_option("solver_io"))
if isinstance(self._solver, UnknownSolver):
raise ValueError("Failed to create solver of type="+
self.get_option("solver")+
" for use in extensive form solve")
if len(self.get_option("solver_options")) > 0:
if self.get_option("verbose"):
print("Initializing ef solver with options="
+str(list(self.get_option("solver_options"))))
self._solver.set_options("".join(self.get_option("solver_options")))
if self.get_option("mipgap") is not None:
if (self.get_option("mipgap") < 0.0) or \
(self.get_option("mipgap") > 1.0):
raise ValueError("Value of the mipgap parameter for the EF "
"solve must be on the unit interval; "
"value specified="+str(self.get_option("mipgap")))
self._solver.options.mipgap = float(self.get_option("mipgap"))
solver_manager_type = self.get_option("solver_manager")
if solver_manager_type == "phpyro":
print("*** WARNING ***: PHPyro is not a supported solver "
"manager type for the extensive-form solver. "
"Falling back to serial.")
solver_manager_type = 'serial'
self._solver_manager = SolverManagerFactory(
solver_manager_type,
host=self.get_option("pyro_host"),
port=self.get_option("pyro_port"))
if self._solver_manager is None:
raise ValueError("Failed to create solver manager of type="
+self.get_option("solver")+
" for use in extensive form solve")
class ExtensiveFormAlgorithm(PySPConfiguredObject):
_declared_options = \
PySPConfigBlock("Options declared for the "
"ExtensiveFormAlgorithm class")
safe_declare_unique_option(
_declared_options,
"cvar_weight",
PySPConfigValue(
1.0,
domain=_domain_nonnegative,
description=(
"The weight associated with the CVaR term in "
"the risk-weighted objective "
"formulation. If the weight is 0, then "
"*only* a non-weighted CVaR cost will appear "
"in the EF objective - the expected cost "
"component will be dropped. Default is 1.0."
),
doc=None,
visibility=0),
ap_group=_ef_group_label)
safe_declare_unique_option(
_declared_options,
"generate_weighted_cvar",
PySPConfigValue(
False,
domain=bool,
description=(
"Add a weighted CVaR term to the "
"primary objective. Default is False."
),
doc=None,
visibility=0),
ap_group=_ef_group_label)
safe_declare_unique_option(
_declared_options,
"risk_alpha",
PySPConfigValue(
0.95,
domain=_domain_unit_interval,
description=(
"The probability threshold associated with "
"CVaR (or any future) risk-oriented "
"performance metrics. Default is 0.95."
),
doc=None,
visibility=0),
ap_group=_ef_group_label)
safe_declare_unique_option(
_declared_options,
"cc_alpha",
PySPConfigValue(
0.0,
domain=_domain_unit_interval,
description=(
"The probability threshold associated with a "
"chance constraint. The RHS will be one "
"minus this value. Default is 0."
),
doc=None,
visibility=0),
ap_group=_ef_group_label)
safe_declare_unique_option(
_declared_options,
"cc_indicator_var",
PySPConfigValue(
None,
domain=_domain_must_be_str,
description=(
"The name of the binary variable to be used "
"to construct a chance constraint. Default "
"is None, which indicates no chance "
"constraint."
),
doc=None,
visibility=0),
ap_group=_ef_group_label)
safe_declare_unique_option(
_declared_options,
"mipgap",
PySPConfigValue(
None,
domain=_domain_unit_interval,
description=(
"Specifies the mipgap for the EF solve."
),
doc=None,
visibility=0),
ap_group=_ef_group_label)
safe_declare_unique_option(
_declared_options,
"solver",
PySPConfigValue(
"cplex",
domain=_domain_must_be_str,
description=(
"Specifies the solver used to solve the "
"extensive form model. Default is cplex."
),
doc=None,
visibility=0),
ap_group=_ef_group_label)
safe_declare_unique_option(
_declared_options,
"solver_io",
PySPConfigValue(
None,
domain=_domain_must_be_str,
description=(
"The type of IO used to execute the "
"solver. Different solvers support different "
"types of IO, but the following are common "
"options: lp - generate LP files, nl - "
"generate NL files, python - direct Python "
"interface, os - generate OSiL XML files."
),
doc=None,
visibility=0),
ap_group=_ef_group_label)
safe_declare_unique_option(
_declared_options,
"solver_manager",
PySPConfigValue(
'serial',
domain=_domain_must_be_str,
description=(
"The type of solver manager used to "
"coordinate solves. Default is serial."
),
doc=None,
visibility=0),
ap_group=_ef_group_label)
safe_declare_unique_option(
_declared_options,
"solver_options",
PySPConfigValue(
(),
domain=_domain_tuple_of_str_or_dict,
description=(
"Persistent solver options used when "
"solving the extensive form model. This "
"option can be used multiple times from "
"the command line to specify more than "
"one solver option."
),
doc=None,
visibility=0),
ap_kwds={'action': 'append'},
ap_group=_ef_group_label)
safe_declare_unique_option(
_declared_options,
"disable_warmstart",
PySPConfigValue(
False,
domain=bool,
description=(
"Disable warm-start of EF solves. "
"Default is False."
),
doc=None,
visibility=0),
ap_group=_ef_group_label)
safe_declare_unique_option(
_declared_options,
"pyro_host",
PySPConfigValue(
None,
domain=_domain_must_be_str,
description=(
"The hostname to bind on when searching "
"for a Pyro nameserver."
),
doc=None,
visibility=0),
ap_group=_ef_group_label)
safe_declare_unique_option(
_declared_options,
"pyro_port",
PySPConfigValue(
None,
domain=_domain_nonnegative_integer,
description=(
"The port to bind on when searching for "
"a Pyro nameserver."
),
doc=None,
visibility=0),
ap_group=_ef_group_label)
safe_declare_unique_option(
_declared_options,
"pyro_shutdown",
PySPConfigValue(
False,
domain=bool,
description=(
"Attempt to shut down all Pyro-related components "
"associated with the Pyro name server used by any scenario "
"tree manager or solver manager. Components to shutdown "
"include the name server, dispatch server, and any "
"scenariotreeserver or pyro_mip_server processes. Note "
"that if Pyro4 is in use the nameserver will always "
"ignore this request."
),
doc=None,
visibility=0),
ap_group=_ef_group_label)
safe_declare_unique_option(
_declared_options,
"pyro_shutdown_workers",
PySPConfigValue(
False,
domain=bool,
description=(
"Upon exit, send shutdown requests to all worker "
"processes that were acquired through the dispatcher. "
"This typically includes scenariotreeserver processes "
"(used by the Pyro scenario tree manager) and pyro_mip_server "
"processes (used by the Pyro solver manager). This leaves "
"any dispatchers and namservers running as well as any "
"processes registered with the dispather that were not "
"acquired for work by this client."
),
doc=None,
visibility=0),
ap_group=_ef_group_label)
safe_declare_common_option(_declared_options,
"symbolic_solver_labels",
ap_group=_ef_group_label)
safe_declare_common_option(_declared_options,
"output_solver_log",
ap_group=_ef_group_label)
safe_declare_common_option(_declared_options,
"verbose",
ap_group=_ef_group_label)
safe_declare_common_option(_declared_options,
"output_times",
ap_group=_ef_group_label)
safe_declare_common_option(_declared_options,
"keep_solver_files",
ap_group=_ef_group_label)
safe_declare_common_option(_declared_options,
"output_solver_results",
ap_group=_ef_group_label)
def __enter__(self):
return self
def __exit__(self, *args):
self.close()
def close(self):
self.destroy_ef()
if self._solver is not None:
self._solver.deactivate()
if self._solver_manager is not None:
if isinstance(self._solver_manager,
pyomo.solvers.plugins.smanager.\
pyro.SolverManager_Pyro):
if self.get_option("pyro_shutdown_workers"):
self._solver_manager.shutdown_workers()
self._solver_manager.deactivate()
self._solver_manager = None
self._manager = None
self.objective = undefined
self.objective_sense = undefined
self.gap = undefined
self.termination_condition = undefined
self.solver_status = undefined
self.solution_status = undefined
self.solver_results = undefined
self.pyomo_solve_time = undefined
self.solve_time = undefined
def __init__(self, manager, *args, **kwds):
import pyomo.solvers.plugins.smanager.pyro
super(ExtensiveFormAlgorithm, self).__init__(*args, **kwds)
# TODO: after PH moves over to the new code
#if not isinstance(manager, ScenarioTreeManager):
# raise TypeError("ExtensiveFormAlgorithm requires an instance of the "
# "ScenarioTreeManager interface as the "
# "second argument")
if not manager.initialized:
raise ValueError("ExtensiveFormAlgorithm requires a scenario tree "
"manager that has been fully initialized")
self._manager = manager
self.instance = None
self._solver_manager = None
self._solver = None
# The following attributes will be modified by the
# solve() method. For users that are scripting, these
# can be accessed after the solve() method returns.
# They will be reset each time solve() is called.
############################################
self.objective = undefined
self.gap = undefined
self.termination_condition = undefined
self.solver_status = undefined
self.solution_status = undefined
self.solver_results = undefined
self.pyomo_solve_time = undefined
self.solve_time = undefined
############################################
self._solver = SolverFactory(self.get_option("solver"),
solver_io=self.get_option("solver_io"))
if isinstance(self._solver, UnknownSolver):
raise ValueError("Failed to create solver of type="+
self.get_option("solver")+
" for use in extensive form solve")
if len(self.get_option("solver_options")) > 0:
if self.get_option("verbose"):
print("Initializing ef solver with options="
+str(list(self.get_option("solver_options"))))
self._solver.set_options("".join(self.get_option("solver_options")))
if self.get_option("mipgap") is not None:
if (self.get_option("mipgap") < 0.0) or \
(self.get_option("mipgap") > 1.0):
raise ValueError("Value of the mipgap parameter for the EF "
"solve must be on the unit interval; "
"value specified="+str(self.get_option("mipgap")))
self._solver.options.mipgap = float(self.get_option("mipgap"))
solver_manager_type = self.get_option("solver_manager")
if solver_manager_type == "phpyro":
print("*** WARNING ***: PHPyro is not a supported solver "
"manager type for the extensive-form solver. "
"Falling back to serial.")
solver_manager_type = 'serial'
self._solver_manager = SolverManagerFactory(
solver_manager_type,
host=self.get_option("pyro_host"),
port=self.get_option("pyro_port"))
if self._solver_manager is None:
raise ValueError("Failed to create solver manager of type="
+self.get_option("solver")+
" for use in extensive form solve")
def build_ef(self):
self.destroy_ef()
if self.get_option("verbose"):
print("Creating extensive form instance")
start_time = time.time()
# then validate the associated parameters.
generate_weighted_cvar = False
cvar_weight = None
risk_alpha = None
if self.get_option("generate_weighted_cvar"):
generate_weighted_cvar = True
cvar_weight = self.get_option("cvar_weight")
risk_alpha = self.get_option("risk_alpha")
self.instance = create_ef_instance(
self._manager.scenario_tree,
verbose_output=self.get_option("verbose"),
generate_weighted_cvar=generate_weighted_cvar,
cvar_weight=cvar_weight,
risk_alpha=risk_alpha,
cc_indicator_var_name=self.get_option("cc_indicator_var"),
cc_alpha=self.get_option("cc_alpha"))
if self.get_option("verbose") or self.get_option("output_times"):
print("Time to construct extensive form instance=%.2f seconds"
%(time.time() - start_time))
def destroy_ef(self):
if self.instance is not None:
for scenario in self._manager.scenario_tree.scenarios:
self.instance.del_component(scenario.name)
scenario._instance_objective.activate()
self.instance = None
def write(self, filename):
if self.instance is None:
raise RuntimeError(
"The extensive form instance has not been constructed."
"Call the build_ef() method to construct it.")
suf = os.path.splitext(filename)[1]
if suf not in ['.nl','.lp','.mps']:
if self._solver.problem_format() == ProblemFormat.cpxlp:
filename += '.lp'
elif self._solver.problem_format() == ProblemFormat.nl:
filename += '.nl'
elif self._solver.problem_format() == ProblemFormat.mps:
filename += '.mps'
else:
raise ValueError("Could not determine output file format. "
"No recognized ending suffix was provided "
"and no format was indicated was by the "
"--solver-io option.")
start_time = time.time()
if self.get_option("verbose"):
print("Starting to write extensive form")
smap_id = write_ef(self.instance,
filename,
self.get_option("symbolic_solver_labels"))
print("Extensive form written to file="+filename)
if self.get_option("verbose") or self.get_option("output_times"):
print("Time to write output file=%.2f seconds"
% (time.time() - start_time))
return filename, smap_id
def solve(self,
check_status=True,
exception_on_failure=True,
io_options=None):
if self.instance is None:
raise RuntimeError(
"The extensive form instance has not been constructed."
"Call the build_ef() method to construct it.")
start_time = time.time()
if self.get_option("verbose"):
print("Queuing extensive form solve")
self.objective = undefined
self.gap = undefined
self.bound = undefined
self.pyomo_solve_time = undefined
self.solve_time = undefined
self.termination_condition = undefined
self.solver_status = undefined
self.solution_status = undefined
self.solver_results = undefined
if isinstance(self._solver, PersistentSolver):
self._solver.compile_instance(
self.instance,
symbolic_solver_labels=self.get_option("symbolic_solver_labels"))
solve_kwds = {}
solve_kwds['load_solutions'] = False
if self.get_option("keep_solver_files"):
solve_kwds['keepfiles'] = True
if self.get_option("symbolic_solver_labels"):
solve_kwds['symbolic_solver_labels'] = True
if self.get_option("output_solver_log"):
solve_kwds['tee'] = True
if io_options is not None:
solve_kwds.update(io_options)
self.objective_sense = \
find_active_objective(self.instance).sense
if (not self.get_option("disable_warmstart")) and \
(self._solver.warm_start_capable()):
action_handle = self._solver_manager.queue(self.instance,
opt=self._solver,
warmstart=True,
**solve_kwds)
else:
action_handle = self._solver_manager.queue(self.instance,
opt=self._solver,
**solve_kwds)
if self.get_option("verbose"):
print("Waiting for extensive form solve")
results = self._solver_manager.wait_for(action_handle)
if self.get_option("verbose"):
print("Done with extensive form solve - loading results")
if self.get_option("output_solver_results"):
print("Results for ef:")
results.write(num=1)
self.solver_results = results
if hasattr(results.solver,"user_time") and \
(not isinstance(results.solver.user_time,
UndefinedData)) and \
(results.solver.user_time is not None):
# the solve time might be a string, or might
# not be - we eventually would like more
# consistency on this front from the solver
# plugins.
self.solve_time = \
float(results.solver.user_time)
elif hasattr(results.solver,"time"):
self.solve_time = \
float(results.solver.time)
else:
self.solve_time = undefined
if hasattr(results,"pyomo_solve_time"):
self.pyomo_solve_time = \
results.pyomo_solve_time
else:
self.pyomo_solve_times = undefined
self.termination_condition = \
results.solver.termination_condition
self.solver_status = \
results.solver.status
if len(results.solution) > 0:
assert len(results.solution) == 1
results_sm = results._smap
self.instance.solutions.load_from(results)
solution0 = results.solution(0)
if hasattr(solution0, "gap") and \
(solution0.gap is not None):
self.gap = solution0.gap
else:
self.gap = undefined
self.solution_status = solution0.status
if self.get_option("verbose"):
print("Storing solution in scenario tree")
for scenario in self._manager.scenario_tree.scenarios:
scenario.update_solution_from_instance()
self._manager.scenario_tree.snapshotSolutionFromScenarios()
self.objective = self._manager.scenario_tree.\
findRootNode().\
computeExpectedNodeCost()
if self.gap is not undefined:
if self.objective_sense == pyomo.core.base.minimize:
self.bound = self.objective - self.gap
else:
self.bound = self.objective + self.gap
else:
self.objective = undefined
self.gap = undefined
self.bound = undefined
self.solution_status = undefined
failure = False
if check_status:
if not ((self.solution_status == SolutionStatus.optimal) or \
(self.solution_status == SolutionStatus.feasible)):
failure = True
if self.get_option("verbose") or \
exception_on_failure:
msg = ("EF solve failed solution status check:\n"
"Solver Status: %s\n"
"Termination Condition: %s\n"
"Solution Status: %s\n"
% (self.solver_status,
self.termination_condition,
self.solution_status))
if self.get_option("verbose"):
print(msg)
if exception_on_failure:
raise RuntimeError(msg)
else:
if self.get_option("verbose"):
print("EF solve completed. Skipping status check.")
if self.get_option("verbose") or self.get_option("output_times"):
print("Time to solve and load results for the "
"extensive form=%.2f seconds"
% (time.time()-start_time))
return failure
def __init__(self, manager, *args, **kwds):
import pyomo.solvers.plugins.smanager.pyro
super(ExtensiveFormAlgorithm, self).__init__(*args, **kwds)
# TODO: after PH moves over to the new code
#if not isinstance(manager, ScenarioTreeManager):
# raise TypeError("ExtensiveFormAlgorithm requires an instance of the "
# "ScenarioTreeManager interface as the "
# "second argument")
if not manager.initialized:
raise ValueError("ExtensiveFormAlgorithm requires a scenario tree "
"manager that has been fully initialized")
self._manager = manager
self.instance = None
self._solver_manager = None
self._solver = None
# The following attributes will be modified by the
# solve() method. For users that are scripting, these
# can be accessed after the solve() method returns.
# They will be reset each time solve() is called.
############################################
self.objective = undefined
self.gap = undefined
self.termination_condition = undefined
self.solver_status = undefined
self.solution_status = undefined
self.solver_results = undefined
self.pyomo_solve_time = undefined
self.solve_time = undefined
############################################
self._solver = SolverFactory(self.get_option("solver"),
solver_io=self.get_option("solver_io"))
if isinstance(self._solver, UnknownSolver):
raise ValueError("Failed to create solver of type="+
self.get_option("solver")+
" for use in extensive form solve")
if len(self.get_option("solver_options")) > 0:
if self.get_option("verbose"):
print("Initializing ef solver with options="
+str(list(self.get_option("solver_options"))))
self._solver.set_options("".join(self.get_option("solver_options")))
if self.get_option("mipgap") is not None:
if (self.get_option("mipgap") < 0.0) or \
(self.get_option("mipgap") > 1.0):
raise ValueError("Value of the mipgap parameter for the EF "
"solve must be on the unit interval; "
"value specified="+str(self.get_option("mipgap")))
self._solver.options.mipgap = float(self.get_option("mipgap"))
solver_manager_type = self.get_option("solver_manager")
if solver_manager_type == "phpyro":
print("*** WARNING ***: PHPyro is not a supported solver "
"manager type for the extensive-form solver. "
"Falling back to serial.")
solver_manager_type = 'serial'
self._solver_manager = SolverManagerFactory(
solver_manager_type,
host=self.get_option("pyro_host"),
port=self.get_option("pyro_port"))
if self._solver_manager is None:
raise ValueError("Failed to create solver manager of type="
+self.get_option("solver")+
" for use in extensive form solve")
def __init__(self, *args, **kwds):
if self.__class__ is _ScenarioTreeManagerSolverWorker:
raise NotImplementedError(
"%s is an abstract class for subclassing" % self.__class__)
super(_ScenarioTreeManagerSolverWorker, self).__init__(*args, **kwds)
# TODO: Does this import need to be delayed because
# it is in a plugins subdirectory?
from pyomo.solvers.plugins.solvers.persistent_solver import \
PersistentSolver
assert self.manager is not None
# solver related objects
self._scenario_solvers = {}
self._bundle_solvers = {}
self._preprocessor = None
self._solver_manager = None
# there are situations in which it is valuable to snapshot /
# store the solutions associated with the scenario
# instances. for example, when one wants to use a warm-start
# from a particular iteration solve, following a modification
# and re-solve of the problem instances in a user-defined
# callback. the following nested dictionary is intended to
# serve that purpose. The nesting is dependent on whether
# bundling and or phpyro is in use
self._cached_solutions = {}
self._cached_scenariotree_solutions = {}
#
# initialize the preprocessor
#
self._preprocessor = None
if not self.get_option("disable_advanced_preprocessing"):
self._preprocessor = ScenarioTreePreprocessor(self._options,
options_prefix=self._options_prefix)
assert self._manager.preprocessor is None
self._manager.preprocessor = self._preprocessor
#
# initialize the solver manager
#
self._solver_manager = SolverManagerFactory(
self.get_option("solver_manager"),
host=self.get_option('solver_manager_pyro_host'),
port=self.get_option('solver_manager_pyro_port'))
for scenario in self.manager.scenario_tree._scenarios:
assert scenario._instance is not None
solver = self._scenario_solvers[scenario.name] = \
SolverFactory(self.get_option("solver"),
solver_io=self.get_option("solver_io"))
if isinstance(solver, PersistentSolver) and \
self.get_option("disable_advanced_preprocessing"):
raise ValueError("Advanced preprocessing can not be disabled "
"when persistent solvers are used")
if self._preprocessor is not None:
self._preprocessor.add_scenario(scenario,
scenario._instance,
solver)
for bundle in self.manager.scenario_tree._scenario_bundles:
solver = self._bundle_solvers[bundle.name] = \
SolverFactory(self.get_option("solver"),
solver_io=self.get_option("solver_io"))
if isinstance(solver, PersistentSolver) and \
self.get_option("disable_advanced_preprocessing"):
raise ValueError("Advanced preprocessing can not be disabled "
"when persistent solvers are used")
bundle_instance = \
self.manager._bundle_binding_instance_map[bundle.name]
if self._preprocessor is not None:
self._preprocessor.add_bundle(bundle,
bundle_instance,
solver)
def EFAlgorithmBuilder(options, scenario_tree):
solution_writer_plugins = ExtensionPoint(ISolutionWriterExtension)
for plugin in solution_writer_plugins:
plugin.disable()
solution_plugins = []
if len(options.solution_writer) > 0:
for this_extension in options.solution_writer:
if this_extension in sys.modules:
print("User-defined EF solution writer module="
+this_extension+" already imported - skipping")
else:
print("Trying to import user-defined EF "
"solution writer module="+this_extension)
# make sure "." is in the PATH.
original_path = list(sys.path)
sys.path.insert(0,'.')
pyutilib.misc.import_file(this_extension)
print("Module successfully loaded")
sys.path[:] = original_path # restore to what it was
# now that we're sure the module is loaded, re-enable this
# specific plugin. recall that all plugins are disabled
# by default in phinit.py, for various reasons. if we want
# them to be picked up, we need to enable them explicitly.
import inspect
module_to_find = this_extension
if module_to_find.rfind(".py"):
module_to_find = module_to_find.rstrip(".py")
if module_to_find.find("/") != -1:
module_to_find = module_to_find.split("/")[-1]
for name, obj in inspect.getmembers(sys.modules[module_to_find], inspect.isclass):
import pyomo.util
# the second condition gets around goofyness related to issubclass returning
# True when the obj is the same as the test class.
if issubclass(obj, pyomo.util.plugin.SingletonPlugin) and name != "SingletonPlugin":
for plugin in solution_writer_plugins(all=True):
if isinstance(plugin, obj):
plugin.enable()
solution_plugins.append(plugin)
ef_solver = SolverFactory(options.solver_type,
solver_io=options.solver_io)
if isinstance(ef_solver, UnknownSolver):
raise ValueError("Failed to create solver of type="+
options.solver_type+
" for use in extensive form solve")
if len(options.solver_options) > 0:
print("Initializing ef solver with options="
+str(options.solver_options))
ef_solver.set_options("".join(options.solver_options))
if options.mipgap is not None:
if (options.mipgap < 0.0) or (options.mipgap > 1.0):
raise ValueError("Value of the mipgap parameter for the EF "
"solve must be on the unit interval; "
"value specified="+str(options.mipgap))
ef_solver.options.mipgap = float(options.mipgap)
ef_solver_manager = SolverManagerFactory(options.solver_manager_type,
host=options.pyro_host,
port=options.pyro_port)
if ef_solver_manager is None:
raise ValueError("Failed to create solver manager of type="
+options.solver_type+
" for use in extensive form solve")
binding_instance = CreateExtensiveFormInstance(options, scenario_tree)
ef = ExtensiveFormAlgorithm(options,
binding_instance,
scenario_tree,
ef_solver_manager,
ef_solver,
solution_plugins=solution_plugins)
return ef
class ExtensiveFormAlgorithm(PySPConfiguredObject):
@classmethod
def _declare_options(cls, options=None):
if options is None:
options = PySPConfigBlock()
safe_declare_unique_option(
options,
"cvar_weight",
PySPConfigValue(
1.0,
domain=_domain_nonnegative,
description=(
"The weight associated with the CVaR term in "
"the risk-weighted objective "
"formulation. If the weight is 0, then "
"*only* a non-weighted CVaR cost will appear "
"in the EF objective - the expected cost "
"component will be dropped. Default is 1.0."
),
doc=None,
visibility=0),
ap_group=_ef_group_label)
safe_declare_unique_option(
options,
"generate_weighted_cvar",
PySPConfigValue(
False,
domain=bool,
description=(
"Add a weighted CVaR term to the "
"primary objective. Default is False."
),
doc=None,
visibility=0),
ap_group=_ef_group_label)
safe_declare_unique_option(
options,
"risk_alpha",
PySPConfigValue(
0.95,
domain=_domain_unit_interval,
description=(
"The probability threshold associated with "
"CVaR (or any future) risk-oriented "
"performance metrics. Default is 0.95."
),
doc=None,
visibility=0),
ap_group=_ef_group_label)
safe_declare_unique_option(
options,
"cc_alpha",
PySPConfigValue(
0.0,
domain=_domain_unit_interval,
description=(
"The probability threshold associated with a "
"chance constraint. The RHS will be one "
"minus this value. Default is 0."
),
doc=None,
visibility=0),
ap_group=_ef_group_label)
safe_declare_unique_option(
options,
"cc_indicator_var",
PySPConfigValue(
None,
domain=_domain_must_be_str,
description=(
"The name of the binary variable to be used "
"to construct a chance constraint. Default "
"is None, which indicates no chance "
"constraint."
),
doc=None,
visibility=0),
ap_group=_ef_group_label)
safe_declare_common_option(options,
"solver")
safe_declare_common_option(options,
"solver_io")
safe_declare_common_option(options,
"solver_manager")
safe_declare_common_option(options,
"solver_options")
safe_declare_common_option(options,
"disable_warmstart")
safe_declare_common_option(options,
"solver_manager_pyro_host")
safe_declare_common_option(options,
"solver_manager_pyro_port")
safe_declare_common_option(options,
"solver_manager_pyro_shutdown")
safe_declare_common_option(options,
"verbose",
ap_group=_ef_group_label)
safe_declare_common_option(options,
"output_times",
ap_group=_ef_group_label)
safe_declare_common_option(options,
"output_solver_results",
ap_group=_ef_group_label)
return options
def __enter__(self):
return self
def __exit__(self, *args):
self.close()
def close(self):
self.destroy_ef()
if self._solver_manager is not None:
if isinstance(self._solver_manager,
pyomo.solvers.plugins.smanager.\
pyro.SolverManager_Pyro):
if self.get_option("pyro_shutdown_workers"):
self._solver_manager.shutdown_workers()
self._solver_manager = None
self._manager = None
def __init__(self, manager, *args, **kwds):
import pyomo.solvers.plugins.smanager.pyro
super(ExtensiveFormAlgorithm, self).__init__(*args, **kwds)
# TODO: after PH moves over to the new code
#if not isinstance(manager, ScenarioTreeManager):
# raise TypeError("ExtensiveFormAlgorithm requires an instance of the "
# "ScenarioTreeManager interface as the "
# "second argument")
if not manager.initialized:
raise ValueError("ExtensiveFormAlgorithm requires a scenario tree "
"manager that has been fully initialized")
self._manager = manager
self.instance = None
self._solver_manager = None
self._solver = None
# The following attributes will be modified by the
# solve() method. For users that are scripting, these
# can be accessed after the solve() method returns.
# They will be reset each time solve() is called.
############################################
self.objective = None
self.gap = None
self.termination_condition = None
self.termination_message = None
self.solver_status = None
self.solution_status = None
self.solver_results = None
self.time = None
self.pyomo_time = None
############################################
# apparently the SolverFactory does not have sane
# behavior when the solver name is None
if self.get_option("solver") is None:
raise ValueError("The 'solver' option can not be None")
self._solver = SolverFactory(self.get_option("solver"),
solver_io=self.get_option("solver_io"))
if isinstance(self._solver, UnknownSolver):
raise ValueError("Failed to create solver of type="+
self.get_option("solver")+
" for use in extensive form solve")
solver_manager_type = self.get_option("solver_manager")
if solver_manager_type == "phpyro":
print("*** WARNING ***: PHPyro is not a supported solver "
"manager type for the extensive-form solver. "
"Falling back to serial.")
solver_manager_type = 'serial'
self._solver_manager = SolverManagerFactory(
solver_manager_type,
host=self.get_option("solver_manager_pyro_host"),
port=self.get_option("solver_manager_pyro_port"))
if self._solver_manager is None:
raise ValueError("Failed to create solver manager of type="
+self.get_option("solver")+
" for use in extensive form solve")
def build_ef(self):
self.destroy_ef()
if self.get_option("verbose"):
print("Creating extensive form instance")
start_time = time.time()
# then validate the associated parameters.
generate_weighted_cvar = False
cvar_weight = None
risk_alpha = None
if self.get_option("generate_weighted_cvar"):
generate_weighted_cvar = True
cvar_weight = self.get_option("cvar_weight")
risk_alpha = self.get_option("risk_alpha")
self.instance = create_ef_instance(
self._manager.scenario_tree,
verbose_output=self.get_option("verbose"),
generate_weighted_cvar=generate_weighted_cvar,
cvar_weight=cvar_weight,
risk_alpha=risk_alpha,
cc_indicator_var_name=self.get_option("cc_indicator_var"),
cc_alpha=self.get_option("cc_alpha"))
if self.get_option("verbose") or self.get_option("output_times"):
print("Time to construct extensive form instance=%.2f seconds"
%(time.time() - start_time))
def destroy_ef(self):
if self.instance is not None:
for scenario in self._manager.scenario_tree.scenarios:
self.instance.del_component(scenario.name)
scenario._instance_objective.activate()
self.instance = None
def write(self, filename):
if self.instance is None:
raise RuntimeError(
"The extensive form instance has not been constructed."
"Call the build_ef() method to construct it.")
suf = os.path.splitext(filename)[1]
if suf not in ['.nl','.lp','.mps']:
if self._solver.problem_format() == ProblemFormat.cpxlp:
filename += '.lp'
elif self._solver.problem_format() == ProblemFormat.nl:
filename += '.nl'
elif self._solver.problem_format() == ProblemFormat.mps:
filename += '.mps'
else:
raise ValueError("Could not determine output file format. "
"No recognized ending suffix was provided "
"and no format was indicated was by the "
"--solver-io option.")
start_time = time.time()
print("Writing extensive form to file="+filename)
smap_id = write_ef(self.instance,
filename,
self.get_option("symbolic_solver_labels"))
if self.get_option("verbose") or self.get_option("output_times"):
print("Time to write output file=%.2f seconds"
% (time.time() - start_time))
return filename, smap_id
def solve(self,
check_status=True,
exception_on_failure=True,
output_solver_log=False,
symbolic_solver_labels=False,
keep_solver_files=False,
io_options=None):
# TODO: Does this import need to be delayed because
# it is in a plugins subdirectory
from pyomo.solvers.plugins.solvers.persistent_solver import \
PersistentSolver
if self.instance is None:
raise RuntimeError(
"The extensive form instance has not been constructed."
"Call the build_ef() method to construct it.")
start_time = time.time()
if self.get_option("verbose"):
print("Queuing extensive form solve")
self.objective = None
self.gap = None
self.bound = None
self.termination_condition = None
self.termination_message = None
self.solver_status = None
self.solution_status = None
self.solver_results = None
self.time = None
self.pyomo_time = None
if isinstance(self._solver, PersistentSolver):
self._solver.compile_instance(
self.instance,
symbolic_solver_labels=symbolic_solver_labels)
solve_kwds = {}
solve_kwds['load_solutions'] = False
if keep_solver_files:
solve_kwds['keepfiles'] = True
if symbolic_solver_labels:
solve_kwds['symbolic_solver_labels'] = True
if output_solver_log:
solve_kwds['tee'] = True
solver_options = self.get_option("solver_options")
if len(solver_options) > 0:
if type(solver_options) is tuple:
solve_kwds["options"] = {}
for name_val in solver_options:
assert "=" in name_val
name, val = name_val.split("=")
solve_kwds["options"][name.strip()] = val.strip()
else:
solve_kwds["options"] = solver_options
if io_options is not None:
solve_kwds.update(io_options)
self.objective_sense = \
find_active_objective(self.instance).sense
if (not self.get_option("disable_warmstart")) and \
(self._solver.warm_start_capable()):
action_handle = self._solver_manager.queue(self.instance,
opt=self._solver,
warmstart=True,
**solve_kwds)
else:
action_handle = self._solver_manager.queue(self.instance,
opt=self._solver,
**solve_kwds)
if self.get_option("verbose"):
print("Waiting for extensive form solve")
results = self._solver_manager.wait_for(action_handle)
if self.get_option("verbose"):
print("Done with extensive form solve - loading results")
if self.get_option("output_solver_results"):
print("Results for ef:")
results.write(num=1)
self.solver_results = results
if hasattr(results.solver,"user_time") and \
(not isinstance(results.solver.user_time,
UndefinedData)) and \
(results.solver.user_time is not None):
# the solve time might be a string, or might
# not be - we eventually would like more
# consistency on this front from the solver
# plugins.
self.time = \
float(results.solver.user_time)
elif hasattr(results.solver,"time"):
self.time = \
float(results.solver.time)
else:
self.time = None
if hasattr(results,"pyomo_solve_time"):
self.pyomo_time = \
results.pyomo_solve_time
else:
self.pyomo_time = None
self.termination_condition = \
results.solver.termination_condition
self.termination_message = None
if hasattr(results.solver,"termination_message"):
self.termination_message = results.solver.termination_message
elif hasattr(results.solver,"message"):
self.termination_message = results.solver.message
self.solver_status = \
results.solver.status
if len(results.solution) > 0:
assert len(results.solution) == 1
results_sm = results._smap
self.instance.solutions.load_from(results)
solution0 = results.solution(0)
if hasattr(solution0, "gap") and \
(solution0.gap is not None) and \
(not isinstance(solution0.gap,
UndefinedData)):
self.gap = float(solution0.gap)
else:
self.gap = None
self.solution_status = solution0.status
if self.get_option("verbose"):
print("Storing solution in scenario tree")
for scenario in self._manager.scenario_tree.scenarios:
scenario.update_solution_from_instance()
self._manager.scenario_tree.snapshotSolutionFromScenarios()
self.objective = self._manager.scenario_tree.\
findRootNode().\
computeExpectedNodeCost()
if self.gap is not None:
if self.objective_sense == pyomo.core.base.minimize:
self.bound = self.objective - self.gap
else:
self.bound = self.objective + self.gap
else:
self.objective = None
self.gap = None
self.bound = None
self.solution_status = None
failure = False
if check_status:
if not ((self.solution_status == SolutionStatus.optimal) or \
(self.solution_status == SolutionStatus.feasible)):
failure = True
if self.get_option("verbose") or \
exception_on_failure:
msg = ("EF solve failed solution status check:\n"
"Solver Status: %s\n"
"Termination Condition: %s\n"
"Solution Status: %s\n"
% (self.solver_status,
self.termination_condition,
self.solution_status))
if self.get_option("verbose"):
print(msg)
if exception_on_failure:
raise RuntimeError(msg)
else:
if self.get_option("verbose"):
print("EF solve completed. Skipping status check.")
if self.get_option("verbose") or self.get_option("output_times"):
print("Time to solve and load results for the "
"extensive form=%.2f seconds"
% (time.time()-start_time))
return failure
class _ScenarioTreeManagerSolverWorker(ScenarioTreeManagerSolver,
PySPConfiguredObject):
@classmethod
def _declare_options(cls, options=None):
if options is None:
options = PySPConfigBlock()
# options for controlling the solver manager
# (not the scenario tree manager)
safe_declare_common_option(options,
"solver_manager_pyro_host")
safe_declare_common_option(options,
"solver_manager_pyro_port")
safe_declare_common_option(options,
"solver_manager_pyro_shutdown")
ScenarioTreePreprocessor._declare_options(options)
return options
@property
def preprocessor(self):
return self._preprocessor
def __init__(self, *args, **kwds):
if self.__class__ is _ScenarioTreeManagerSolverWorker:
raise NotImplementedError(
"%s is an abstract class for subclassing" % self.__class__)
super(_ScenarioTreeManagerSolverWorker, self).__init__(*args, **kwds)
# TODO: Does this import need to be delayed because
# it is in a plugins subdirectory?
from pyomo.solvers.plugins.solvers.persistent_solver import \
PersistentSolver
assert self.manager is not None
# solver related objects
self._scenario_solvers = {}
self._bundle_solvers = {}
self._preprocessor = None
self._solver_manager = None
# there are situations in which it is valuable to snapshot /
# store the solutions associated with the scenario
# instances. for example, when one wants to use a warm-start
# from a particular iteration solve, following a modification
# and re-solve of the problem instances in a user-defined
# callback. the following nested dictionary is intended to
# serve that purpose. The nesting is dependent on whether
# bundling and or phpyro is in use
self._cached_solutions = {}
self._cached_scenariotree_solutions = {}
#
# initialize the preprocessor
#
self._preprocessor = None
if not self.get_option("disable_advanced_preprocessing"):
self._preprocessor = ScenarioTreePreprocessor(self._options,
options_prefix=self._options_prefix)
assert self._manager.preprocessor is None
self._manager.preprocessor = self._preprocessor
#
# initialize the solver manager
#
self._solver_manager = SolverManagerFactory(
self.get_option("solver_manager"),
host=self.get_option('solver_manager_pyro_host'),
port=self.get_option('solver_manager_pyro_port'))
for scenario in self.manager.scenario_tree._scenarios:
assert scenario._instance is not None
solver = self._scenario_solvers[scenario.name] = \
SolverFactory(self.get_option("solver"),
solver_io=self.get_option("solver_io"))
if isinstance(solver, PersistentSolver) and \
self.get_option("disable_advanced_preprocessing"):
raise ValueError("Advanced preprocessing can not be disabled "
"when persistent solvers are used")
if self._preprocessor is not None:
self._preprocessor.add_scenario(scenario,
scenario._instance,
solver)
for bundle in self.manager.scenario_tree._scenario_bundles:
solver = self._bundle_solvers[bundle.name] = \
SolverFactory(self.get_option("solver"),
solver_io=self.get_option("solver_io"))
if isinstance(solver, PersistentSolver) and \
self.get_option("disable_advanced_preprocessing"):
raise ValueError("Advanced preprocessing can not be disabled "
"when persistent solvers are used")
bundle_instance = \
self.manager._bundle_binding_instance_map[bundle.name]
if self._preprocessor is not None:
self._preprocessor.add_bundle(bundle,
bundle_instance,
solver)
#
# Override some methods for ScenarioTreeManager that
# were implemented by _ScenarioTreeManagerWorker:
#
def _close_impl(self):
if (self._manager is not None) and \
(self._manager.preprocessor is not None):
assert self.preprocessor is self._manager.preprocessor
for bundle in self.manager.scenario_tree._scenario_bundles:
self._preprocessor.remove_bundle(bundle)
for scenario in self.manager.scenario_tree._scenarios:
assert scenario._instance is not None
self._preprocessor.remove_scenario(scenario)
self._manager.preprocessor = None
self._preprocessor = None
else:
assert self._preprocessor is None
if self._solver_manager is not None:
self._solver_manager.deactivate()
self._solver_manager = None
if self.get_option("solver_manager_pyro_shutdown"):
print("Shutting down Pyro components for solver manager.")
shutdown_pyro_components(
host=self.get_option("solver_manager_pyro_host"),
port=self.get_option("solver_manager_pyro_port"),
num_retries=0,
caller_name=self.__class__.__name__)
for solver in self._scenario_solvers.values():
solver.deactivate()
self._scenario_solvers = {}
for solver in self._bundle_solvers.values():
solver.deactivate()
self._bundle_solvers = {}
self._preprocessor = None
self._objective_sense = None
#
# Abstract methods for ScenarioTreeManagerSolver:
#
def _queue_object_solves(self,
object_type,
objects,
ephemeral_solver_options,
disable_warmstart):
if self.get_option("verbose"):
print("Queuing %s solves" % (object_type[:-1]))
assert object_type in ('bundles', 'scenarios')
solver_dict = None
instance_dict = None
modify_kwds_func = None
if object_type == 'bundles':
if objects is None:
objects = self.manager.scenario_tree._scenario_bundle_map
solver_dict = self._bundle_solvers
instance_dict = self.manager._bundle_binding_instance_map
for bundle_name in objects:
for scenario_name in self.manager.scenario_tree.\
get_bundle(bundle_name).\
scenario_names:
self.manager.scenario_tree.get_scenario(scenario_name).\
_instance_objective.deactivate()
if self.preprocessor is not None:
self.preprocessor.preprocess_bundles(bundles=objects)
modify_kwds_func = self.preprocessor.modify_bundle_solver_keywords
else:
if objects is None:
objects = self.manager.scenario_tree._scenario_map
solver_dict = self._scenario_solvers
instance_dict = self.manager._instances
if self.manager.scenario_tree.contains_bundles():
for scenario_name in objects:
self.manager.scenario_tree.get_scenario(scenario_name).\
_instance_objective.activate()
if self.preprocessor is not None:
self.preprocessor.preprocess_scenarios(scenarios=objects)
modify_kwds_func = self.preprocessor.modify_scenario_solver_keywords
assert solver_dict is not None
assert instance_dict is not None
# setup common solve keywords
common_kwds = {}
common_kwds['tee'] = self.get_option("output_solver_log")
common_kwds['keepfiles'] = self.get_option("keep_solver_files")
common_kwds['symbolic_solver_labels'] = \
self.get_option("symbolic_solver_labels")
# we always manually load solutions, so we can
# control error reporting and such
common_kwds['load_solutions'] = False
# Load solver options
solver_options = {}
if type(self.get_option("solver_options")) is tuple:
solver_options.update(
OptSolver._options_string_to_dict(
"".join(self.get_option("solver_options"))))
else:
solver_options.update(self.get_option("solver_options"))
common_kwds['options'] = solver_options
#
# override "persistent" values that are included from this
# classes registered options
#
if ephemeral_solver_options is not None:
common_kwds['options'].update(ephemeral_solver_options)
# maps action handles to subproblem names
action_handle_data = {}
for object_name in objects:
if modify_kwds_func is not None:
# be sure to modify a copy of the kwds
solve_kwds = modify_kwds_func(object_name, dict(common_kwds))
else:
solve_kwds = common_kwds
opt = solver_dict[object_name]
instance = instance_dict[object_name]
if (not self.get_option("disable_warmstart")) and \
(not disable_warmstart) and \
opt.warm_start_capable():
new_action_handle = \
self._solver_manager.queue(instance,
opt=opt,
warmstart=True,
**solve_kwds)
else:
new_action_handle = \
self._solver_manager.queue(instance,
opt=opt,
**solve_kwds)
action_handle_data[new_action_handle] = object_name
return self.manager.AsyncResult(
self._solver_manager,
action_handle_data=action_handle_data)
class ExtensiveFormAlgorithm(PySPConfiguredObject):
_declared_options = \
PySPConfigBlock("Options declared for the "
"ExtensiveFormAlgorithm class")
safe_declare_unique_option(
_declared_options,
"cvar_weight",
PySPConfigValue(
1.0,
domain=_domain_nonnegative,
description=("The weight associated with the CVaR term in "
"the risk-weighted objective "
"formulation. If the weight is 0, then "
"*only* a non-weighted CVaR cost will appear "
"in the EF objective - the expected cost "
"component will be dropped. Default is 1.0."),
doc=None,
visibility=0),
ap_group=_ef_group_label)
safe_declare_unique_option(
_declared_options,
"generate_weighted_cvar",
PySPConfigValue(False,
domain=bool,
description=("Add a weighted CVaR term to the "
"primary objective. Default is False."),
doc=None,
visibility=0),
ap_group=_ef_group_label)
safe_declare_unique_option(
_declared_options,
"risk_alpha",
PySPConfigValue(
0.95,
domain=_domain_unit_interval,
description=("The probability threshold associated with "
"CVaR (or any future) risk-oriented "
"performance metrics. Default is 0.95."),
doc=None,
visibility=0),
ap_group=_ef_group_label)
safe_declare_unique_option(
_declared_options,
"cc_alpha",
PySPConfigValue(
0.0,
domain=_domain_unit_interval,
description=("The probability threshold associated with a "
"chance constraint. The RHS will be one "
"minus this value. Default is 0."),
doc=None,
visibility=0),
ap_group=_ef_group_label)
safe_declare_unique_option(
_declared_options,
"cc_indicator_var",
PySPConfigValue(
None,
domain=_domain_must_be_str,
description=("The name of the binary variable to be used "
"to construct a chance constraint. Default "
"is None, which indicates no chance "
"constraint."),
doc=None,
visibility=0),
ap_group=_ef_group_label)
safe_declare_unique_option(
_declared_options,
"mipgap",
PySPConfigValue(None,
domain=_domain_unit_interval,
description=("Specifies the mipgap for the EF solve."),
doc=None,
visibility=0),
ap_group=_ef_group_label)
safe_declare_unique_option(
_declared_options,
"solver",
PySPConfigValue(
"cplex",
domain=_domain_must_be_str,
description=("Specifies the solver used to solve the "
"extensive form model. Default is cplex."),
doc=None,
visibility=0),
ap_group=_ef_group_label)
safe_declare_unique_option(
_declared_options,
"solver_io",
PySPConfigValue(
None,
domain=_domain_must_be_str,
description=("The type of IO used to execute the "
"solver. Different solvers support different "
"types of IO, but the following are common "
"options: lp - generate LP files, nl - "
"generate NL files, python - direct Python "
"interface, os - generate OSiL XML files."),
doc=None,
visibility=0),
ap_group=_ef_group_label)
safe_declare_unique_option(
_declared_options,
"solver_manager",
PySPConfigValue('serial',
domain=_domain_must_be_str,
description=("The type of solver manager used to "
"coordinate solves. Default is serial."),
doc=None,
visibility=0),
ap_group=_ef_group_label)
safe_declare_unique_option(
_declared_options,
"solver_options",
PySPConfigValue((),
domain=_domain_tuple_of_str_or_dict,
description=("Persistent solver options used when "
"solving the extensive form model. This "
"option can be used multiple times from "
"the command line to specify more than "
"one solver option."),
doc=None,
visibility=0),
ap_kwds={'action': 'append'},
ap_group=_ef_group_label)
safe_declare_unique_option(
_declared_options,
"disable_warmstart",
PySPConfigValue(False,
domain=bool,
description=("Disable warm-start of EF solves. "
"Default is False."),
doc=None,
visibility=0),
ap_group=_ef_group_label)
safe_declare_unique_option(
_declared_options,
"pyro_host",
PySPConfigValue(None,
domain=_domain_must_be_str,
description=("The hostname to bind on when searching "
"for a Pyro nameserver."),
doc=None,
visibility=0),
ap_group=_ef_group_label)
safe_declare_unique_option(
_declared_options,
"pyro_port",
PySPConfigValue(None,
domain=_domain_nonnegative_integer,
description=("The port to bind on when searching for "
"a Pyro nameserver."),
doc=None,
visibility=0),
ap_group=_ef_group_label)
safe_declare_unique_option(
_declared_options,
"pyro_shutdown",
PySPConfigValue(
False,
domain=bool,
description=(
"Attempt to shut down all Pyro-related components "
"associated with the Pyro name server used by any scenario "
"tree manager or solver manager. Components to shutdown "
"include the name server, dispatch server, and any "
"scenariotreeserver or pyro_mip_server processes. Note "
"that if Pyro4 is in use the nameserver will always "
"ignore this request."),
doc=None,
visibility=0),
ap_group=_ef_group_label)
safe_declare_unique_option(
_declared_options,
"pyro_shutdown_workers",
PySPConfigValue(
False,
domain=bool,
description=(
"Upon exit, send shutdown requests to all worker "
"processes that were acquired through the dispatcher. "
"This typically includes scenariotreeserver processes "
"(used by the Pyro scenario tree manager) and pyro_mip_server "
"processes (used by the Pyro solver manager). This leaves "
"any dispatchers and namservers running as well as any "
"processes registered with the dispather that were not "
"acquired for work by this client."),
doc=None,
visibility=0),
ap_group=_ef_group_label)
safe_declare_common_option(_declared_options,
"symbolic_solver_labels",
ap_group=_ef_group_label)
safe_declare_common_option(_declared_options,
"output_solver_log",
ap_group=_ef_group_label)
safe_declare_common_option(_declared_options,
"verbose",
ap_group=_ef_group_label)
safe_declare_common_option(_declared_options,
"output_times",
ap_group=_ef_group_label)
safe_declare_common_option(_declared_options,
"keep_solver_files",
ap_group=_ef_group_label)
safe_declare_common_option(_declared_options,
"output_solver_results",
ap_group=_ef_group_label)
def __enter__(self):
return self
def __exit__(self, *args):
self.close()
def close(self):
self.destroy_ef()
if self._solver is not None:
self._solver.deactivate()
if self._solver_manager is not None:
if isinstance(self._solver_manager,
pyomo.solvers.plugins.smanager.\
pyro.SolverManager_Pyro):
if self.get_option("pyro_shutdown_workers"):
self._solver_manager.shutdown_workers()
self._solver_manager.deactivate()
self._solver_manager = None
self._manager = None
self.objective = undefined
self.objective_sense = undefined
self.gap = undefined
self.termination_condition = undefined
self.solver_status = undefined
self.solution_status = undefined
self.solver_results = undefined
self.pyomo_solve_time = undefined
self.solve_time = undefined
def __init__(self, manager, *args, **kwds):
import pyomo.solvers.plugins.smanager.pyro
super(ExtensiveFormAlgorithm, self).__init__(*args, **kwds)
# TODO: after PH moves over to the new code
#if not isinstance(manager, ScenarioTreeManager):
# raise TypeError("ExtensiveFormAlgorithm requires an instance of the "
# "ScenarioTreeManager interface as the "
# "second argument")
if not manager.initialized:
raise ValueError("ExtensiveFormAlgorithm requires a scenario tree "
"manager that has been fully initialized")
self._manager = manager
self.instance = None
self._solver_manager = None
self._solver = None
# The following attributes will be modified by the
# solve() method. For users that are scripting, these
# can be accessed after the solve() method returns.
# They will be reset each time solve() is called.
############################################
self.objective = undefined
self.gap = undefined
self.termination_condition = undefined
self.solver_status = undefined
self.solution_status = undefined
self.solver_results = undefined
self.pyomo_solve_time = undefined
self.solve_time = undefined
############################################
self._solver = SolverFactory(self.get_option("solver"),
solver_io=self.get_option("solver_io"))
if isinstance(self._solver, UnknownSolver):
raise ValueError("Failed to create solver of type=" +
self.get_option("solver") +
" for use in extensive form solve")
if len(self.get_option("solver_options")) > 0:
if self.get_option("verbose"):
print("Initializing ef solver with options=" +
str(list(self.get_option("solver_options"))))
self._solver.set_options("".join(
self.get_option("solver_options")))
if self.get_option("mipgap") is not None:
if (self.get_option("mipgap") < 0.0) or \
(self.get_option("mipgap") > 1.0):
raise ValueError("Value of the mipgap parameter for the EF "
"solve must be on the unit interval; "
"value specified=" +
str(self.get_option("mipgap")))
self._solver.options.mipgap = float(self.get_option("mipgap"))
solver_manager_type = self.get_option("solver_manager")
if solver_manager_type == "phpyro":
print("*** WARNING ***: PHPyro is not a supported solver "
"manager type for the extensive-form solver. "
"Falling back to serial.")
solver_manager_type = 'serial'
self._solver_manager = SolverManagerFactory(
solver_manager_type,
host=self.get_option("pyro_host"),
port=self.get_option("pyro_port"))
if self._solver_manager is None:
raise ValueError("Failed to create solver manager of type=" +
self.get_option("solver") +
" for use in extensive form solve")
def build_ef(self):
self.destroy_ef()
if self.get_option("verbose"):
print("Creating extensive form instance")
start_time = time.time()
# then validate the associated parameters.
generate_weighted_cvar = False
cvar_weight = None
risk_alpha = None
if self.get_option("generate_weighted_cvar"):
generate_weighted_cvar = True
cvar_weight = self.get_option("cvar_weight")
risk_alpha = self.get_option("risk_alpha")
self.instance = create_ef_instance(
self._manager.scenario_tree,
verbose_output=self.get_option("verbose"),
generate_weighted_cvar=generate_weighted_cvar,
cvar_weight=cvar_weight,
risk_alpha=risk_alpha,
cc_indicator_var_name=self.get_option("cc_indicator_var"),
cc_alpha=self.get_option("cc_alpha"))
if self.get_option("verbose") or self.get_option("output_times"):
print("Time to construct extensive form instance=%.2f seconds" %
(time.time() - start_time))
def destroy_ef(self):
if self.instance is not None:
for scenario in self._manager.scenario_tree.scenarios:
self.instance.del_component(scenario.name)
scenario._instance_objective.activate()
self.instance = None
def write(self, filename):
if self.instance is None:
raise RuntimeError(
"The extensive form instance has not been constructed."
"Call the build_ef() method to construct it.")
suf = os.path.splitext(filename)[1]
if suf not in ['.nl', '.lp', '.mps']:
if self._solver.problem_format() == ProblemFormat.cpxlp:
filename += '.lp'
elif self._solver.problem_format() == ProblemFormat.nl:
filename += '.nl'
elif self._solver.problem_format() == ProblemFormat.mps:
filename += '.mps'
else:
raise ValueError("Could not determine output file format. "
"No recognized ending suffix was provided "
"and no format was indicated was by the "
"--solver-io option.")
start_time = time.time()
if self.get_option("verbose"):
print("Starting to write extensive form")
smap_id = write_ef(self.instance, filename,
self.get_option("symbolic_solver_labels"))
print("Extensive form written to file=" + filename)
if self.get_option("verbose") or self.get_option("output_times"):
print("Time to write output file=%.2f seconds" %
(time.time() - start_time))
return filename, smap_id
def solve(self,
check_status=True,
exception_on_failure=True,
io_options=None):
if self.instance is None:
raise RuntimeError(
"The extensive form instance has not been constructed."
"Call the build_ef() method to construct it.")
start_time = time.time()
if self.get_option("verbose"):
print("Queuing extensive form solve")
self.objective = undefined
self.gap = undefined
self.bound = undefined
self.pyomo_solve_time = undefined
self.solve_time = undefined
self.termination_condition = undefined
self.solver_status = undefined
self.solution_status = undefined
self.solver_results = undefined
if isinstance(self._solver, PersistentSolver):
self._solver.set_instance(self.instance,
symbolic_solver_labels=self.get_option(
"symbolic_solver_labels"))
solve_kwds = {}
solve_kwds['load_solutions'] = False
if self.get_option("keep_solver_files"):
solve_kwds['keepfiles'] = True
if self.get_option("symbolic_solver_labels"):
solve_kwds['symbolic_solver_labels'] = True
if self.get_option("output_solver_log"):
solve_kwds['tee'] = True
if io_options is not None:
solve_kwds.update(io_options)
self.objective_sense = \
find_active_objective(self.instance).sense
if (not self.get_option("disable_warmstart")) and \
(self._solver.warm_start_capable()):
action_handle = self._solver_manager.queue(self.instance,
opt=self._solver,
warmstart=True,
**solve_kwds)
else:
action_handle = self._solver_manager.queue(self.instance,
opt=self._solver,
**solve_kwds)
if self.get_option("verbose"):
print("Waiting for extensive form solve")
results = self._solver_manager.wait_for(action_handle)
if self.get_option("verbose"):
print("Done with extensive form solve - loading results")
if self.get_option("output_solver_results"):
print("Results for ef:")
results.write(num=1)
self.solver_results = results
if hasattr(results.solver,"user_time") and \
(not isinstance(results.solver.user_time,
UndefinedData)) and \
(results.solver.user_time is not None):
# the solve time might be a string, or might
# not be - we eventually would like more
# consistency on this front from the solver
# plugins.
self.solve_time = \
float(results.solver.user_time)
elif hasattr(results.solver, "time"):
self.solve_time = \
float(results.solver.time)
else:
self.solve_time = undefined
if hasattr(results, "pyomo_solve_time"):
self.pyomo_solve_time = \
results.pyomo_solve_time
else:
self.pyomo_solve_times = undefined
self.termination_condition = \
results.solver.termination_condition
self.solver_status = \
results.solver.status
if len(results.solution) > 0:
assert len(results.solution) == 1
results_sm = results._smap
self.instance.solutions.load_from(results)
solution0 = results.solution(0)
if hasattr(solution0, "gap") and \
(solution0.gap is not None):
self.gap = solution0.gap
else:
self.gap = undefined
self.solution_status = solution0.status
if self.get_option("verbose"):
print("Storing solution in scenario tree")
for scenario in self._manager.scenario_tree.scenarios:
scenario.update_solution_from_instance()
self._manager.scenario_tree.snapshotSolutionFromScenarios()
self.objective = self._manager.scenario_tree.\
findRootNode().\
computeExpectedNodeCost()
if self.gap is not undefined:
if self.objective_sense == pyomo.core.base.minimize:
self.bound = self.objective - self.gap
else:
self.bound = self.objective + self.gap
else:
self.objective = undefined
self.gap = undefined
self.bound = undefined
self.solution_status = undefined
failure = False
if check_status:
if not ((self.solution_status == SolutionStatus.optimal) or \
(self.solution_status == SolutionStatus.feasible)):
failure = True
if self.get_option("verbose") or \
exception_on_failure:
msg = ("EF solve failed solution status check:\n"
"Solver Status: %s\n"
"Termination Condition: %s\n"
"Solution Status: %s\n" %
(self.solver_status, self.termination_condition,
self.solution_status))
if self.get_option("verbose"):
print(msg)
if exception_on_failure:
raise RuntimeError(msg)
else:
if self.get_option("verbose"):
print("EF solve completed. Skipping status check.")
if self.get_option("verbose") or self.get_option("output_times"):
print("Time to solve and load results for the "
"extensive form=%.2f seconds" % (time.time() - start_time))
return failure
def BenderAlgorithmBuilder(options, scenario_tree):
import pyomo.environ
import pyomo.solvers.plugins.smanager.phpyro
import pyomo.solvers.plugins.smanager.pyro
solution_writer_plugins = ExtensionPoint(ISolutionWriterExtension)
for plugin in solution_writer_plugins:
plugin.disable()
solution_plugins = []
if len(options.solution_writer) > 0:
for this_extension in options.solution_writer:
if this_extension in sys.modules:
print("User-defined PH solution writer module=" +
this_extension + " already imported - skipping")
else:
print("Trying to import user-defined PH "
"solution writer module=" + this_extension)
# make sure "." is in the PATH.
original_path = list(sys.path)
sys.path.insert(0, '.')
import_file(this_extension)
print("Module successfully loaded")
sys.path[:] = original_path # restore to what it was
# now that we're sure the module is loaded, re-enable this
# specific plugin. recall that all plugins are disabled
# by default in phinit.py, for various reasons. if we want
# them to be picked up, we need to enable them explicitly.
import inspect
module_to_find = this_extension
if module_to_find.rfind(".py"):
module_to_find = module_to_find.rstrip(".py")
if module_to_find.find("/") != -1:
module_to_find = module_to_find.split("/")[-1]
for name, obj in inspect.getmembers(sys.modules[module_to_find],
inspect.isclass):
import pyomo.common
# the second condition gets around goofyness related
# to issubclass returning True when the obj is the
# same as the test class.
if issubclass(obj, pyomo.common.plugin.SingletonPlugin
) and name != "SingletonPlugin":
for plugin in solution_writer_plugins(all=True):
if isinstance(plugin, obj):
plugin.enable()
solution_plugins.append(plugin)
#
# if any of the ww extension configuration options are specified
# without the ww extension itself being enabled, halt and warn the
# user - this has led to confusion in the past, and will save user
# support time.
#
if (len(options.ww_extension_cfgfile) > 0) and \
(options.enable_ww_extensions is False):
raise ValueError("A configuration file was specified "
"for the WW extension module, but the WW extensions "
"are not enabled!")
if (len(options.ww_extension_suffixfile) > 0) and \
(options.enable_ww_extensions is False):
raise ValueError("A suffix file was specified for the WW "
"extension module, but the WW extensions are not "
"enabled!")
if (len(options.ww_extension_annotationfile) > 0) and \
(options.enable_ww_extensions is False):
raise ValueError("A annotation file was specified for the "
"WW extension module, but the WW extensions are not "
"enabled!")
#
# disable all plugins up-front. then, enable them on an as-needed
# basis later in this function. the reason that plugins should be
# disabled is that they may have been programmatically enabled in
# a previous run of PH, and we want to start from a clean slate.
#
ph_extension_point = ExtensionPoint(IPHExtension)
for plugin in ph_extension_point:
plugin.disable()
ph_plugins = []
#
# deal with any plugins. ww extension comes first currently,
# followed by an option user-defined plugin. order only matters
# if both are specified.
#
if options.enable_ww_extensions:
import pyomo.pysp.plugins.wwphextension
# explicitly enable the WW extension plugin - it may have been
# previously loaded and/or enabled.
ph_extension_point = ExtensionPoint(IPHExtension)
for plugin in ph_extension_point(all=True):
if isinstance(plugin,
pyomo.pysp.plugins.wwphextension.wwphextension):
plugin.enable()
ph_plugins.append(plugin)
# there is no reset-style method for plugins in general,
# or the ww ph extension in plugin in particular. if no
# configuration or suffix filename is specified, set to
# None so that remnants from the previous use of the
# plugin aren't picked up.
if len(options.ww_extension_cfgfile) > 0:
plugin._configuration_filename = options.ww_extension_cfgfile
else:
plugin._configuration_filename = None
if len(options.ww_extension_suffixfile) > 0:
plugin._suffix_filename = options.ww_extension_suffixfile
else:
plugin._suffix_filename = None
if len(options.ww_extension_annotationfile) > 0:
plugin._annotation_filename = options.ww_extension_annotationfile
else:
plugin._annotation_filename = None
if len(options.user_defined_extensions) > 0:
for this_extension in options.user_defined_extensions:
if this_extension in sys.modules:
print("User-defined PH extension module=" + this_extension +
" already imported - skipping")
else:
print("Trying to import user-defined PH extension module=" +
this_extension)
# make sure "." is in the PATH.
original_path = list(sys.path)
sys.path.insert(0, '.')
import_file(this_extension)
print("Module successfully loaded")
# restore to what it was
sys.path[:] = original_path
# now that we're sure the module is loaded, re-enable this
# specific plugin. recall that all plugins are disabled
# by default in phinit.py, for various reasons. if we want
# them to be picked up, we need to enable them explicitly.
import inspect
module_to_find = this_extension
if module_to_find.rfind(".py"):
module_to_find = module_to_find.rstrip(".py")
if module_to_find.find("/") != -1:
module_to_find = module_to_find.split("/")[-1]
for name, obj in inspect.getmembers(sys.modules[module_to_find],
inspect.isclass):
import pyomo.common
# the second condition gets around goofyness related
# to issubclass returning True when the obj is the
# same as the test class.
if issubclass(obj, pyomo.common.plugin.SingletonPlugin
) and name != "SingletonPlugin":
ph_extension_point = ExtensionPoint(IPHExtension)
for plugin in ph_extension_point(all=True):
if isinstance(plugin, obj):
plugin.enable()
ph_plugins.append(plugin)
ph = None
solver_manager = None
try:
# construct the solver manager.
if options.verbose:
print("Constructing solver manager of type=" +
options.solver_manager_type)
solver_manager = SolverManagerFactory(options.solver_manager_type,
host=options.pyro_host,
port=options.pyro_port)
if solver_manager is None:
raise ValueError("Failed to create solver manager of "
"type=" + options.solver_manager_type +
" specified in call to PH constructor")
ph = ProgressiveHedging(options)
if isinstance(
solver_manager,
pyomo.solvers.plugins.smanager.phpyro.SolverManager_PHPyro):
if scenario_tree.contains_bundles():
num_jobs = len(scenario_tree._scenario_bundles)
if not _OLD_OUTPUT:
print("Bundle solver jobs available: " + str(num_jobs))
else:
num_jobs = len(scenario_tree._scenarios)
if not _OLD_OUTPUT:
print("Scenario solver jobs available: " + str(num_jobs))
servers_expected = options.phpyro_required_workers
if (servers_expected is None):
servers_expected = num_jobs
timeout = options.phpyro_workers_timeout if \
(options.phpyro_required_workers is None) else \
None
solver_manager.acquire_servers(servers_expected, timeout)
ph.initialize(scenario_tree=scenario_tree,
solver_manager=solver_manager,
ph_plugins=ph_plugins,
solution_plugins=solution_plugins)
except:
if ph is not None:
ph.release_components()
if solver_manager is not None:
if isinstance(
solver_manager, pyomo.solvers.plugins.smanager.phpyro.
SolverManager_PHPyro):
solver_manager.release_servers(
shutdown=ph._shutdown_pyro_workers)
elif isinstance(
solver_manager,
pyomo.solvers.plugins.smanager.pyro.SolverManager_Pyro):
if ph._shutdown_pyro_workers:
solver_manager.shutdown_workers()
print("Failed to initialize progressive hedging algorithm")
raise
return ph
def apply_optimizer(data, instance=None):
"""
Perform optimization with a concrete instance
Required:
instance: Problem instance.
Returned:
results: Optimization results.
opt: Optimizer object.
"""
#
if not data.options.runtime.logging == 'quiet':
sys.stdout.write('[%8.2f] Applying solver\n' % (time.time()-start_time))
sys.stdout.flush()
#
#
# Create Solver and Perform Optimization
#
solver = data.options.solvers[0].solver_name
if solver is None:
raise ValueError("Problem constructing solver: no solver specified")
if len(data.options.solvers[0].suffixes) > 0:
for suffix_name in data.options.solvers[0].suffixes:
if suffix_name[0] in ['"',"'"]:
suffix_name = suffix[1:-1]
# Don't redeclare the suffix if it already exists
suffix = getattr(instance, suffix_name, None)
if suffix is None:
setattr(instance, suffix_name, Suffix(direction=Suffix.IMPORT))
else:
raise ValueError("Problem declaring solver suffix %s. A component "\
"with that name already exists on model %s."
% (suffix_name, instance.name))
if getattr(data.options.solvers[0].options, 'timelimit', 0) == 0:
data.options.solvers[0].options.timelimit = None
#
# Default results
#
results = None
#
# Figure out the type of solver manager
#
solver_mngr_name = None
if data.options.solvers[0].manager is None:
solver_mngr_name = 'serial'
elif not data.options.solvers[0].manager in SolverManagerFactory.services():
raise ValueError("Unknown solver manager %s"
% data.options.solvers[0].manager)
else:
solver_mngr_name = data.options.solvers[0].manager
#
# Create the solver manager
#
solver_mngr_kwds = {}
if data.options.solvers[0].pyro_host is not None:
solver_mngr_kwds['host'] = data.options.solvers[0].pyro_host
if data.options.solvers[0].pyro_port is not None:
solver_mngr_kwds['port'] = data.options.solvers[0].pyro_port
with SolverManagerFactory(solver_mngr_name, **solver_mngr_kwds) as solver_mngr:
if solver_mngr is None:
msg = "Problem constructing solver manager '%s'"
raise ValueError(msg % str(data.options.solvers[0].manager))
#
# Setup keywords for the solve
#
keywords = {}
if (data.options.runtime.keep_files or \
data.options.postsolve.print_logfile):
keywords['keepfiles'] = True
if data.options.model.symbolic_solver_labels:
keywords['symbolic_solver_labels'] = True
if data.options.model.file_determinism != 1:
keywords['file_determinism'] = data.options.model.file_determinism
keywords['tee'] = data.options.runtime.stream_output
keywords['timelimit'] = getattr(data.options.solvers[0].options, 'timelimit', 0)
#
# Call the solver
#
if solver_mngr_name == 'serial':
#
# If we're running locally, then we create the optimizer and pass it into the
# solver manager.
#
with SolverFactory(solver, solver_io=data.options.solvers[0].io_format) as opt:
if opt is None:
raise ValueError("Problem constructing solver `%s`" % str(solver))
from pyomo.core.base.plugin import registered_callback
for name in registered_callback:
opt.set_callback(name, registered_callback[name])
if len(data.options.solvers[0].options) > 0:
opt.set_options(data.options.solvers[0].options)
#opt.set_options(" ".join("%s=%s" % (key, value)
# for key, value in data.options.solvers[0].options.iteritems()
# if not key == 'timelimit'))
if not data.options.solvers[0].options_string is None:
opt.set_options(data.options.solvers[0].options_string)
#
# Use the solver manager to call the optimizer
#
results = solver_mngr.solve(instance, opt=opt, **keywords)
else:
#
# Get the solver option arguments
#
if len(data.options.solvers[0].options) > 0 and not data.options.solvers[0].options_string is None:
# If both 'options' and 'options_string' were specified, then create a
# single options string that is passed to the solver.
ostring = " ".join("%s=%s" % (key, value)
for key, value in data.options.solvers[0].options.iteritems()
if not value is None)
keywords['options'] = ostring + ' ' + data.options.solvers[0].options_string
elif len(data.options.solvers[0].options) > 0:
keywords['options'] = data.options.solvers[0].options
else:
keywords['options'] = data.options.solvers[0].options_string
#
# If we're running remotely, then we pass the optimizer name to the solver
# manager.
#
results = solver_mngr.solve(instance, opt=solver, **keywords)
if (pympler_available is True) and \
(data.options.runtime.profile_memory >= 1):
global memory_data
mem_used = muppy.get_size(muppy.get_objects())
if mem_used > data.local.max_memory:
data.local.max_memory = mem_used
print(" Total memory = %d bytes following optimization" % mem_used)
return pyutilib.misc.Options(results=results, opt=solver, local=data.local)
def apply_optimizer(data, instance=None):
"""
Perform optimization with a concrete instance
Required:
instance: Problem instance.
Returned:
results: Optimization results.
opt: Optimizer object.
"""
#
if not data.options.runtime.logging == 'quiet':
sys.stdout.write('[%8.2f] Applying solver\n' %
(time.time() - start_time))
sys.stdout.flush()
#
#
# Create Solver and Perform Optimization
#
solver = data.options.solvers[0].solver_name
if solver is None:
raise ValueError("Problem constructing solver: no solver specified")
if len(data.options.solvers[0].suffixes) > 0:
for suffix_name in data.options.solvers[0].suffixes:
if suffix_name[0] in ['"', "'"]:
suffix_name = suffix_name[1:-1]
# Don't redeclare the suffix if it already exists
suffix = getattr(instance, suffix_name, None)
if suffix is None:
setattr(instance, suffix_name, Suffix(direction=Suffix.IMPORT))
else:
raise ValueError("Problem declaring solver suffix %s. A component "\
"with that name already exists on model %s."
% (suffix_name, instance.name))
if getattr(data.options.solvers[0].options, 'timelimit', 0) == 0:
data.options.solvers[0].options.timelimit = None
#
# Default results
#
results = None
#
# Figure out the type of solver manager
#
solver_mngr_name = None
if data.options.solvers[0].manager is None:
solver_mngr_name = 'serial'
elif not data.options.solvers[0].manager in SolverManagerFactory:
raise ValueError("Unknown solver manager %s" %
data.options.solvers[0].manager)
else:
solver_mngr_name = data.options.solvers[0].manager
#
# Create the solver manager
#
solver_mngr_kwds = {}
with SolverManagerFactory(solver_mngr_name,
**solver_mngr_kwds) as solver_mngr:
if solver_mngr is None:
msg = "Problem constructing solver manager '%s'"
raise ValueError(msg % str(data.options.solvers[0].manager))
#
# Setup keywords for the solve
#
keywords = {}
if (data.options.runtime.keep_files or \
data.options.postsolve.print_logfile):
keywords['keepfiles'] = True
if data.options.model.symbolic_solver_labels:
keywords['symbolic_solver_labels'] = True
if data.options.model.file_determinism != 1:
keywords['file_determinism'] = data.options.model.file_determinism
keywords['tee'] = data.options.runtime.stream_output
keywords['timelimit'] = getattr(data.options.solvers[0].options,
'timelimit', 0)
keywords['report_timing'] = data.options.runtime.report_timing
# FIXME: solver_io and executable are not being used
# in the case of a non-serial solver manager
#
# Call the solver
#
if solver_mngr_name == 'serial':
#
# If we're running locally, then we create the
# optimizer and pass it into the solver manager.
#
sf_kwds = {}
sf_kwds['solver_io'] = data.options.solvers[0].io_format
if data.options.solvers[0].solver_executable is not None:
sf_kwds['executable'] = data.options.solvers[
0].solver_executable
with SolverFactory(solver, **sf_kwds) as opt:
if opt is None:
raise ValueError("Problem constructing solver `%s`" %
str(solver))
for name in registered_callback:
opt.set_callback(name, registered_callback[name])
if len(data.options.solvers[0].options) > 0:
opt.set_options(data.options.solvers[0].options)
#opt.set_options(" ".join("%s=%s" % (key, value)
# for key, value in data.options.solvers[0].options.iteritems()
# if not key == 'timelimit'))
if not data.options.solvers[0].options_string is None:
opt.set_options(data.options.solvers[0].options_string)
#
# Use the solver manager to call the optimizer
#
results = solver_mngr.solve(instance, opt=opt, **keywords)
else:
#
# Get the solver option arguments
#
if len(
data.options.solvers[0].options
) > 0 and not data.options.solvers[0].options_string is None:
# If both 'options' and 'options_string' were specified, then create a
# single options string that is passed to the solver.
ostring = " ".join("%s=%s" % (key, value) for key, value in
data.options.solvers[0].options.iteritems()
if not value is None)
keywords['options'] = ostring + ' ' + data.options.solvers[
0].options_string
elif len(data.options.solvers[0].options) > 0:
keywords['options'] = data.options.solvers[0].options
else:
keywords['options'] = data.options.solvers[0].options_string
#
# If we're running remotely, then we pass the optimizer name to the solver
# manager.
#
results = solver_mngr.solve(instance, opt=solver, **keywords)
if data.options.runtime.profile_memory >= 1 and pympler_available:
global memory_data
mem_used = pympler.muppy.get_size(pympler.muppy.get_objects())
if mem_used > data.local.max_memory:
data.local.max_memory = mem_used
print(" Total memory = %d bytes following optimization" % mem_used)
return Bunch(results=results, opt=solver, local=data.local)
# import the master a
# initialize the master instance.
mstr_mdl = import_file("master.py").model
mstr_inst = mstr_mdl.create_instance("master.dat")
# initialize the sub-problem instances.
sb_mdl = import_file("sub.py").model
sub_insts = []
sub_insts.append(
sb_mdl.create_instance(name="sub1", \
filename="sub1.dat"))
sub_insts.append(
sb_mdl.create_instance(name="sub2", \
filename="sub2.dat"))
# initialize the solver manager.
solver_manager = SolverManagerFactory("serial")
# miscellaneous initialization.
mstr_inst.Min_Stage2_cost = float("-Inf")
gap = float("Inf")
max_iterations = 50
# main benders loop.
for i in range(1, max_iterations + 1):
print("\nIteration=%d" % (i))
#solve the subproblem
solve_all_instances(solver_manager, 'cplex', sub_insts)
for instance in sub_insts:
print("cost for scenario="+instance.name+" is"+\
class _ScenarioTreeManagerSolverWorker(ScenarioTreeManagerSolver,
PySPConfiguredObject):
@classmethod
def _declare_options(cls, options=None):
if options is None:
options = PySPConfigBlock()
# options for controlling the solver manager
# (not the scenario tree manager)
safe_declare_common_option(options,
"solver_manager_pyro_host")
safe_declare_common_option(options,
"solver_manager_pyro_port")
safe_declare_common_option(options,
"solver_manager_pyro_shutdown")
ScenarioTreePreprocessor._declare_options(options)
return options
@property
def preprocessor(self):
return self._preprocessor
def __init__(self, *args, **kwds):
if self.__class__ is _ScenarioTreeManagerSolverWorker:
raise NotImplementedError(
"%s is an abstract class for subclassing" % self.__class__)
super(_ScenarioTreeManagerSolverWorker, self).__init__(*args, **kwds)
# TODO: Does this import need to be delayed because
# it is in a plugins subdirectory?
from pyomo.solvers.plugins.solvers.persistent_solver import \
PersistentSolver
assert self.manager is not None
# solver related objects
self._scenario_solvers = {}
self._bundle_solvers = {}
self._preprocessor = None
self._solver_manager = None
# there are situations in which it is valuable to snapshot /
# store the solutions associated with the scenario
# instances. for example, when one wants to use a warm-start
# from a particular iteration solve, following a modification
# and re-solve of the problem instances in a user-defined
# callback. the following nested dictionary is intended to
# serve that purpose. The nesting is dependent on whether
# bundling and or phpyro is in use
self._cached_solutions = {}
self._cached_scenariotree_solutions = {}
#
# initialize the preprocessor
#
self._preprocessor = None
if not self.get_option("disable_advanced_preprocessing"):
self._preprocessor = ScenarioTreePreprocessor(self._options,
options_prefix=self._options_prefix)
assert self._manager.preprocessor is None
self._manager.preprocessor = self._preprocessor
#
# initialize the solver manager
#
self._solver_manager = SolverManagerFactory(
self.get_option("solver_manager"),
host=self.get_option('solver_manager_pyro_host'),
port=self.get_option('solver_manager_pyro_port'))
for scenario in self.manager.scenario_tree._scenarios:
assert scenario._instance is not None
solver = self._scenario_solvers[scenario.name] = \
SolverFactory(self.get_option("solver"),
solver_io=self.get_option("solver_io"))
if isinstance(solver, PersistentSolver) and \
self.get_option("disable_advanced_preprocessing"):
raise ValueError("Advanced preprocessing can not be disabled "
"when persistent solvers are used")
if self._preprocessor is not None:
self._preprocessor.add_scenario(scenario,
scenario._instance,
solver)
for bundle in self.manager.scenario_tree._scenario_bundles:
solver = self._bundle_solvers[bundle.name] = \
SolverFactory(self.get_option("solver"),
solver_io=self.get_option("solver_io"))
if isinstance(solver, PersistentSolver) and \
self.get_option("disable_advanced_preprocessing"):
raise ValueError("Advanced preprocessing can not be disabled "
"when persistent solvers are used")
bundle_instance = \
self.manager._bundle_binding_instance_map[bundle.name]
if self._preprocessor is not None:
self._preprocessor.add_bundle(bundle,
bundle_instance,
solver)
#
# Override some methods for ScenarioTreeManager that
# were implemented by _ScenarioTreeManagerWorker:
#
def _close_impl(self):
if (self._manager is not None) and \
(self._manager.preprocessor is not None):
assert self.preprocessor is self._manager.preprocessor
for bundle in self.manager.scenario_tree._scenario_bundles:
self._preprocessor.remove_bundle(bundle)
for scenario in self.manager.scenario_tree._scenarios:
assert scenario._instance is not None
self._preprocessor.remove_scenario(scenario)
self._manager.preprocessor = None
self._preprocessor = None
else:
assert self._preprocessor is None
if self._solver_manager is not None:
#self._solver_manager.deactivate()
self._solver_manager = None
if self.get_option("solver_manager_pyro_shutdown"):
print("Shutting down Pyro components for solver manager.")
shutdown_pyro_components(
host=self.get_option("solver_manager_pyro_host"),
port=self.get_option("solver_manager_pyro_port"),
num_retries=0,
caller_name=self.__class__.__name__)
#for solver in self._scenario_solvers.values():
# solver.deactivate()
self._scenario_solvers = {}
#for solver in self._bundle_solvers.values():
# solver.deactivate()
self._bundle_solvers = {}
self._preprocessor = None
self._objective_sense = None
#
# Abstract methods for ScenarioTreeManagerSolver:
#
def _queue_object_solves(self,
object_type,
objects,
ephemeral_solver_options,
disable_warmstart):
if self.get_option("verbose"):
print("Queuing %s solves" % (object_type[:-1]))
assert object_type in ('bundles', 'scenarios')
solver_dict = None
instance_dict = None
modify_kwds_func = None
if object_type == 'bundles':
if objects is None:
objects = self.manager.scenario_tree._scenario_bundle_map
solver_dict = self._bundle_solvers
instance_dict = self.manager._bundle_binding_instance_map
for bundle_name in objects:
for scenario_name in self.manager.scenario_tree.\
get_bundle(bundle_name).\
scenario_names:
self.manager.scenario_tree.get_scenario(scenario_name).\
_instance_objective.deactivate()
if self.preprocessor is not None:
self.preprocessor.preprocess_bundles(bundles=objects)
modify_kwds_func = self.preprocessor.modify_bundle_solver_keywords
else:
if objects is None:
objects = self.manager.scenario_tree._scenario_map
solver_dict = self._scenario_solvers
instance_dict = self.manager._instances
if self.manager.scenario_tree.contains_bundles():
for scenario_name in objects:
self.manager.scenario_tree.get_scenario(scenario_name).\
_instance_objective.activate()
if self.preprocessor is not None:
self.preprocessor.preprocess_scenarios(scenarios=objects)
modify_kwds_func = self.preprocessor.modify_scenario_solver_keywords
assert solver_dict is not None
assert instance_dict is not None
# setup common solve keywords
common_kwds = {}
common_kwds['tee'] = self.get_option("output_solver_log")
common_kwds['keepfiles'] = self.get_option("keep_solver_files")
common_kwds['symbolic_solver_labels'] = \
self.get_option("symbolic_solver_labels")
# we always manually load solutions, so we can
# control error reporting and such
common_kwds['load_solutions'] = False
# Load solver options
solver_options = {}
if type(self.get_option("solver_options")) is tuple:
solver_options.update(
OptSolver._options_string_to_dict(
"".join(self.get_option("solver_options"))))
else:
solver_options.update(self.get_option("solver_options"))
common_kwds['options'] = solver_options
#
# override "persistent" values that are included from this
# classes registered options
#
if ephemeral_solver_options is not None:
common_kwds['options'].update(ephemeral_solver_options)
# maps action handles to subproblem names
action_handle_data = {}
for object_name in objects:
if modify_kwds_func is not None:
# be sure to modify a copy of the kwds
solve_kwds = modify_kwds_func(object_name, dict(common_kwds))
else:
solve_kwds = common_kwds
opt = solver_dict[object_name]
instance = instance_dict[object_name]
if (not self.get_option("disable_warmstart")) and \
(not disable_warmstart) and \
opt.warm_start_capable():
new_action_handle = \
self._solver_manager.queue(instance,
opt=opt,
warmstart=True,
**solve_kwds)
else:
new_action_handle = \
self._solver_manager.queue(instance,
opt=opt,
**solve_kwds)
action_handle_data[new_action_handle] = object_name
return self.manager.AsyncResult(
self._solver_manager,
action_handle_data=action_handle_data)
def initialize(self, scenario_tree):
import pyomo.environ
import pyomo.solvers.plugins.smanager.phpyro
init_start_time = time.time()
print("Initializing Scenario Tree Manager")
print("")
if scenario_tree is None:
raise ValueError("A scenario tree must be supplied to the "
"ScenarioTreeManager initialize() method")
self._scenario_tree = scenario_tree
# construct the solver manager.
if self._options.verbose:
print("Constructing solver manager of type=" +
self._options.solver_manager_type)
self._solver_manager = \
SolverManagerFactory(self._options.solver_manager_type,
host=self._options.pyro_manager_hostname)
if self._solver_manager is None:
raise ValueError("Failed to create solver manager of "
"type=" + self._options.solver_manager_type)
isPHPyro = isinstance(self._solver_manager,
pyomo.solvers.plugins.\
smanager.phpyro.SolverManager_PHPyro)
if isPHPyro:
if self._scenario_tree.contains_bundles():
num_jobs = len(self._scenario_tree._scenario_bundles)
if not _OLD_OUTPUT:
print("Bundle solver jobs available: " + str(num_jobs))
else:
num_jobs = len(self._scenario_tree._scenarios)
if not _OLD_OUTPUT:
print("Scenario solver jobs available: " + str(num_jobs))
workers_expected = self._options.phpyro_required_workers
if (workers_expected is None):
workers_expected = num_jobs
timeout = self._options.phpyro_workers_timeout if \
(self._options.phpyro_required_workers is None) else \
None
self._solver_manager.acquire_workers(workers_expected, timeout)
initialization_action_handles = []
if isPHPyro:
if self._options.verbose:
print("Broadcasting requests to initialize "
"distributed scenario tree workers")
initialization_action_handles.extend(
scenariotreeserverutils.\
initialize_scenariotree_workers(self))
if self._options.verbose:
print("Distributed scenario tree initialization "
"requests successfully transmitted")
else:
build_start_time = time.time()
if not _OLD_OUTPUT:
print("Constructing scenario tree instances")
self._instances = \
self._scenario_tree._scenario_instance_factory.\
construct_instances_for_scenario_tree(
scenario_tree,
flatten_expressions=self._options.flatten_expressions,
report_timing=self._options.output_times,
preprocess=False)
if self._options.verbose or self._options.output_times:
print("Time to construct scenario instances=%.2f seconds" %
(time.time() - build_start_time))
if not _OLD_OUTPUT:
print("Linking instances into scenario tree")
build_start_time = time.time()
# with the scenario instances now available, link the
# referenced objects directly into the scenario tree.
self._scenario_tree.linkInInstances(
self._instances,
objective_sense=self._options.objective_sense,
create_variable_ids=True)
if self._options.verbose or self._options.output_times:
print("Time link scenario tree with instances=%.2f seconds" %
(time.time() - build_start_time))
if self._scenario_tree.contains_bundles():
build_start_time = time.time()
if self._options.verbose:
print("Forming binding instances for all scenario bundles")
self._bundle_binding_instance_map.clear()
self._bundle_scenario_instance_map.clear()
if not self._scenario_tree.contains_bundles():
raise RuntimeError(
"Failed to create binding instances for scenario "
"bundles - no scenario bundles are defined!")
for scenario_bundle in self._scenario_tree._scenario_bundles:
if self._options.verbose:
print(
"Creating binding instance for scenario bundle=%s"
% (scenario_bundle._name))
self._bundle_scenario_instance_map[
scenario_bundle._name] = {}
for scenario_name in scenario_bundle._scenario_names:
self._bundle_scenario_instance_map[scenario_bundle._name]\
[scenario_name] = self._instances[scenario_name]
# IMPORTANT: The bundle variable IDs must be idential to
# those in the parent scenario tree - this is
# critical for storing results, which occurs at
# the full-scale scenario tree.
scenario_bundle._scenario_tree.linkInInstances(
self._instances,
create_variable_ids=False,
master_scenario_tree=self._scenario_tree,
initialize_solution_data=False)
bundle_ef_instance = create_ef_instance(
scenario_bundle._scenario_tree,
ef_instance_name=scenario_bundle._name,
verbose_output=self._verbose)
self._bundle_binding_instance_map[scenario_bundle._name] = \
bundle_ef_instance
if self._output_times:
print("Scenario bundle construction time=%.2f seconds" %
(time.time() - build_start_time))
# If specified, run the user script to collect aggregate
# scenario data. This can slow down PH initialization as
# syncronization across all phsolverservers is required
if self._options.aggregate_cfgfile is not None:
callback_name = "pysp_aggregategetter_callback"
if isPHPyro:
# Transmit invocation to phsolverservers
print("Transmitting user aggregate callback invocations "
"to phsolverservers")
if self._scenario_tree.contains_bundles():
for scenario_bundle in self._scenario_tree._scenario_bundles:
ah = scenariotreeserverutils.\
transmit_external_function_invocation_to_worker(
self,
scenario_bundle._name,
self._callback_module_name[callback_name],
callback_name,
invocation_type=(scenariotreeserverutils.InvocationType.\
PerScenarioChainedInvocation),
return_action_handle=True,
function_args=(self._aggregate_user_data,))
while (1):
action_handle = self._solver_manager.wait_any()
if action_handle in initialization_action_handles:
initialization_action_handles.remove(
action_handle)
self._solver_manager.get_results(action_handle)
elif action_handle == ah:
result = self._solver_manager.get_results(
action_handle)
break
assert len(result) == 1
self._aggregate_user_data = result[0]
else:
for scenario in self._scenario_tree._scenarios:
ah = scenariotreeserverutils.\
transmit_external_function_invocation_to_worker(
self,
scenario._name,
self._callback_module_name[callback_name],
callback_name,
invocation_type=(scenariotreeserverutils.InvocationType.\
SingleInvocation),
return_action_handle=True,
function_args=(self._aggregate_user_data,))
while (1):
action_handle = self._solver_manager.wait_any()
if action_handle in initialization_action_handles:
initialization_action_handles.remove(
action_handle)
self._solver_manager.get_results(action_handle)
elif action_handle == ah:
result = self._solver_manager.get_results(
action_handle)
break
assert len(result) == 1
self._aggregate_user_data = result[0]
# Transmit final aggregate state to phsolverservers
print("Broadcasting final aggregate data to phsolverservers")
initialization_action_handles.extend(
scenariotreeserverutils.transmit_external_function_invocation(
self,
"pyomo.pysp.ph",
"assign_aggregate_data",
invocation_type=(scenariotreeserverutils.InvocationType.\
SingleInvocation),
return_action_handles=True,
function_args=(self._aggregate_user_data,)))
else:
print("Executing user aggregate getter callback function")
for scenario in self._scenario_tree._scenarios:
result = self._callback_function[callback_name](
self, self._scenario_tree, scenario,
self._aggregate_user_data)
assert len(result) == 1
self._aggregate_user_data = result[0]
# if specified, run the user script to initialize variable
# bounds at their whim.
if self._options.bounds_cfgfile is not None:
callback_name = "pysp_boundsetter_callback"
if isPHPyro:
# Transmit invocation to phsolverservers
print("Transmitting user bound callback invocations to "
"phsolverservers")
if self._scenario_tree.contains_bundles():
for scenario_bundle in self._scenario_tree._scenario_bundles:
initialization_action_handles.append(
scenariotreeserverutils.\
transmit_external_function_invocation_to_worker(
self,
scenario_bundle._name,
self._callback_module_name[callback_name],
callback_name,
invocation_type=(scenariotreeserverutils.InvocationType.\
PerScenarioInvocation),
return_action_handle=True))
else:
for scenario in self._scenario_tree._scenarios:
initialization_action_handles.append(
scenariotreeserverutils.\
transmit_external_function_invocation_to_worker(
self,
scenario._name,
self._callback_module_name[callback_name],
callback_name,
invocation_type=(scenariotreeserverutils.InvocationType.\
SingleInvocation),
return_action_handle=True))
else:
print("Executing user bound setter callback function")
for scenario in self._scenario_tree._scenarios:
self._callback_function[callback_name](self,
self._scenario_tree,
scenario)
# gather scenario tree data if not local
if isPHPyro:
if self._options.verbose:
print("Broadcasting requests to collect scenario tree "
"instance data from PH solver servers")
scenariotreeserverutils.\
gather_scenario_tree_data(self,
initialization_action_handles)
assert len(initialization_action_handles) == 0
if self._options.verbose:
print("Scenario tree instance data successfully collected")
if self._options.verbose:
print("Broadcasting scenario tree id mapping"
"to PH solver servers")
scenariotreeserverutils.transmit_scenario_tree_ids(self)
if self._options.verbose:
print("Scenario tree ids successfully sent")
self._objective_sense = \
self._scenario_tree._scenarios[0]._objective_sense
if self._options.verbose:
print("Scenario tree manager is successfully initialized")
if self._options.output_times:
print("Overall initialization time=%.2f seconds" %
(time.time() - init_start_time))
# gather and report memory statistics (for leak
# detection purposes) if specified.
if (guppy_available) and (self._options.profile_memory >= 1):
print(hpy().heap())
# indicate that we're ready to run.
self._initialized = True
def initialize(self, scenario_tree):
import pyomo.environ
import pyomo.solvers.plugins.smanager.phpyro
init_start_time = time.time()
print("Initializing Scenario Tree Manager")
print("")
if scenario_tree is None:
raise ValueError("A scenario tree must be supplied to the "
"ScenarioTreeManager initialize() method")
self._scenario_tree = scenario_tree
# construct the solver manager.
if self._options.verbose:
print("Constructing solver manager of type="
+self._options.solver_manager_type)
self._solver_manager = \
SolverManagerFactory(self._options.solver_manager_type,
host=self._options.pyro_manager_hostname)
if self._solver_manager is None:
raise ValueError("Failed to create solver manager of "
"type="+self._options.solver_manager_type)
isPHPyro = isinstance(self._solver_manager,
pyomo.solvers.plugins.\
smanager.phpyro.SolverManager_PHPyro)
if isPHPyro:
if self._scenario_tree.contains_bundles():
num_jobs = len(self._scenario_tree._scenario_bundles)
if not _OLD_OUTPUT:
print("Bundle solver jobs available: "+str(num_jobs))
else:
num_jobs = len(self._scenario_tree._scenarios)
if not _OLD_OUTPUT:
print("Scenario solver jobs available: "+str(num_jobs))
workers_expected = self._options.phpyro_required_workers
if (workers_expected is None):
workers_expected = num_jobs
timeout = self._options.phpyro_workers_timeout if \
(self._options.phpyro_required_workers is None) else \
None
self._solver_manager.acquire_workers(workers_expected,
timeout)
initialization_action_handles = []
if isPHPyro:
if self._options.verbose:
print("Broadcasting requests to initialize "
"distributed scenario tree workers")
initialization_action_handles.extend(
scenariotreeserverutils.\
initialize_scenariotree_workers(self))
if self._options.verbose:
print("Distributed scenario tree initialization "
"requests successfully transmitted")
else:
build_start_time = time.time()
if not _OLD_OUTPUT:
print("Constructing scenario tree instances")
self._instances = \
self._scenario_tree._scenario_instance_factory.\
construct_instances_for_scenario_tree(
scenario_tree,
flatten_expressions=self._options.flatten_expressions,
report_timing=self._options.output_times,
preprocess=False)
if self._options.verbose or self._options.output_times:
print("Time to construct scenario instances=%.2f seconds"
% (time.time() - build_start_time))
if not _OLD_OUTPUT:
print("Linking instances into scenario tree")
build_start_time = time.time()
# with the scenario instances now available, link the
# referenced objects directly into the scenario tree.
self._scenario_tree.linkInInstances(
self._instances,
objective_sense=self._options.objective_sense,
create_variable_ids=True)
if self._options.verbose or self._options.output_times:
print("Time link scenario tree with instances=%.2f seconds"
% (time.time() - build_start_time))
if self._scenario_tree.contains_bundles():
build_start_time = time.time()
if self._options.verbose:
print("Forming binding instances for all scenario bundles")
self._bundle_binding_instance_map.clear()
self._bundle_scenario_instance_map.clear()
if not self._scenario_tree.contains_bundles():
raise RuntimeError("Failed to create binding instances for scenario "
"bundles - no scenario bundles are defined!")
for scenario_bundle in self._scenario_tree._scenario_bundles:
if self._options.verbose:
print("Creating binding instance for scenario bundle=%s"
% (scenario_bundle._name))
self._bundle_scenario_instance_map[scenario_bundle._name] = {}
for scenario_name in scenario_bundle._scenario_names:
self._bundle_scenario_instance_map[scenario_bundle._name]\
[scenario_name] = self._instances[scenario_name]
# IMPORTANT: The bundle variable IDs must be idential to
# those in the parent scenario tree - this is
# critical for storing results, which occurs at
# the full-scale scenario tree.
scenario_bundle._scenario_tree.linkInInstances(
self._instances,
create_variable_ids=False,
master_scenario_tree=self._scenario_tree,
initialize_solution_data=False)
bundle_ef_instance = create_ef_instance(
scenario_bundle._scenario_tree,
ef_instance_name=scenario_bundle._name,
verbose_output=self._verbose)
self._bundle_binding_instance_map[scenario_bundle._name] = \
bundle_ef_instance
if self._output_times:
print("Scenario bundle construction time=%.2f seconds"
% (time.time() - build_start_time))
# If specified, run the user script to collect aggregate
# scenario data. This can slow down PH initialization as
# syncronization across all phsolverservers is required
if self._options.aggregate_cfgfile is not None:
callback_name = "pysp_aggregategetter_callback"
if isPHPyro:
# Transmit invocation to phsolverservers
print("Transmitting user aggregate callback invocations "
"to phsolverservers")
if self._scenario_tree.contains_bundles():
for scenario_bundle in self._scenario_tree._scenario_bundles:
ah = scenariotreeserverutils.\
transmit_external_function_invocation_to_worker(
self,
scenario_bundle._name,
self._callback_module_name[callback_name],
callback_name,
invocation_type=(scenariotreeserverutils.InvocationType.\
PerScenarioChainedInvocation),
return_action_handle=True,
function_args=(self._aggregate_user_data,))
while(1):
action_handle = self._solver_manager.wait_any()
if action_handle in initialization_action_handles:
initialization_action_handles.remove(action_handle)
self._solver_manager.get_results(action_handle)
elif action_handle == ah:
result = self._solver_manager.get_results(action_handle)
break
assert len(result) == 1
self._aggregate_user_data = result[0]
else:
for scenario in self._scenario_tree._scenarios:
ah = scenariotreeserverutils.\
transmit_external_function_invocation_to_worker(
self,
scenario._name,
self._callback_module_name[callback_name],
callback_name,
invocation_type=(scenariotreeserverutils.InvocationType.\
SingleInvocation),
return_action_handle=True,
function_args=(self._aggregate_user_data,))
while(1):
action_handle = self._solver_manager.wait_any()
if action_handle in initialization_action_handles:
initialization_action_handles.remove(action_handle)
self._solver_manager.get_results(action_handle)
elif action_handle == ah:
result = self._solver_manager.get_results(action_handle)
break
assert len(result) == 1
self._aggregate_user_data = result[0]
# Transmit final aggregate state to phsolverservers
print("Broadcasting final aggregate data to phsolverservers")
initialization_action_handles.extend(
scenariotreeserverutils.transmit_external_function_invocation(
self,
"pyomo.pysp.ph",
"assign_aggregate_data",
invocation_type=(scenariotreeserverutils.InvocationType.\
SingleInvocation),
return_action_handles=True,
function_args=(self._aggregate_user_data,)))
else:
print("Executing user aggregate getter callback function")
for scenario in self._scenario_tree._scenarios:
result = self._callback_function[callback_name](
self,
self._scenario_tree,
scenario,
self._aggregate_user_data)
assert len(result) == 1
self._aggregate_user_data = result[0]
# if specified, run the user script to initialize variable
# bounds at their whim.
if self._options.bounds_cfgfile is not None:
callback_name = "pysp_boundsetter_callback"
if isPHPyro:
# Transmit invocation to phsolverservers
print("Transmitting user bound callback invocations to "
"phsolverservers")
if self._scenario_tree.contains_bundles():
for scenario_bundle in self._scenario_tree._scenario_bundles:
initialization_action_handles.append(
scenariotreeserverutils.\
transmit_external_function_invocation_to_worker(
self,
scenario_bundle._name,
self._callback_module_name[callback_name],
callback_name,
invocation_type=(scenariotreeserverutils.InvocationType.\
PerScenarioInvocation),
return_action_handle=True))
else:
for scenario in self._scenario_tree._scenarios:
initialization_action_handles.append(
scenariotreeserverutils.\
transmit_external_function_invocation_to_worker(
self,
scenario._name,
self._callback_module_name[callback_name],
callback_name,
invocation_type=(scenariotreeserverutils.InvocationType.\
SingleInvocation),
return_action_handle=True))
else:
print("Executing user bound setter callback function")
for scenario in self._scenario_tree._scenarios:
self._callback_function[callback_name](
self,
self._scenario_tree,
scenario)
# gather scenario tree data if not local
if isPHPyro:
if self._options.verbose:
print("Broadcasting requests to collect scenario tree "
"instance data from PH solver servers")
scenariotreeserverutils.\
gather_scenario_tree_data(self,
initialization_action_handles)
assert len(initialization_action_handles) == 0
if self._options.verbose:
print("Scenario tree instance data successfully collected")
if self._options.verbose:
print("Broadcasting scenario tree id mapping"
"to PH solver servers")
scenariotreeserverutils.transmit_scenario_tree_ids(self)
if self._options.verbose:
print("Scenario tree ids successfully sent")
self._objective_sense = \
self._scenario_tree._scenarios[0]._objective_sense
if self._options.verbose:
print("Scenario tree manager is successfully initialized")
if self._options.output_times:
print("Overall initialization time=%.2f seconds"
% (time.time() - init_start_time))
# gather and report memory statistics (for leak
# detection purposes) if specified.
if (guppy_available) and (self._options.profile_memory >= 1):
print(hpy().heap())
# indicate that we're ready to run.
self._initialized = True