def __init__(self, fsfile, fsfct=None, tree_model=None, phopts=None):
"""Initialize a StochSolver object.
"""
if fsfct is None:
# Changed in October 2018: None implies AbstractModel
args_list = _optiondict_2_list(phopts)
parser = phinit.construct_ph_options_parser("")
options = parser.parse_args(args_list)
scenario_instance_factory = \
ScenarioTreeInstanceFactory(fsfile, tree_model)
try:
self.scenario_tree = \
phinit.GenerateScenarioTreeForPH(options,
scenario_instance_factory)
except:
print("ERROR in StochSolver called from",
inspect.stack()[1][3])
raise RuntimeError(
"fsfct is None, so assuming",
"AbstractModel but could not find all ingredients.")
else: # concrete model
if callable(fsfct):
scen_function = fsfct
else: # better be a string
fsfile = fsfile.replace('.py', '') # import does not like .py
# __import__ only gives the top level module
# probably need to be dealing with modules installed via setup.py
m = __import__(fsfile)
for n in fsfile.split(".")[1:]:
m = getattr(m, n)
scen_function = getattr(m, fsfct)
if tree_model is None:
treecbname = "pysp_scenario_tree_model_callback"
tree_maker = getattr(m, treecbname)
tree = tree_maker()
scenario_instance_factory = ScenarioTreeInstanceFactory(
scen_function, tree_model)
else:
# DLW March 21: still not correct
scenario_instance_factory = \
ScenarioTreeInstanceFactory(scen_function, tree_model)
kwargs = _kwfromphopts(phopts)
self.scenario_tree = \
scenario_instance_factory.generate_scenario_tree(**kwargs) #verbose = True)
instances = scenario_instance_factory. \
construct_instances_for_scenario_tree(self.scenario_tree)
self.scenario_tree.linkInInstances(instances)
def test_random_bundles(self):
self.assertTrue("ReferenceModel" not in sys.modules)
scenario_instance_factory = \
ScenarioTreeInstanceFactory(farmer_model_dir,
farmer_scenariotree_dir)
scenario_tree = \
scenario_instance_factory.generate_scenario_tree(random_bundles=2)
self.assertEqual(scenario_tree.contains_bundles(), True)
self.assertEqual(len(scenario_tree._scenario_bundles), 2)
self.assertTrue("ReferenceModel" in sys.modules)
del sys.modules["ReferenceModel"]
def test_random_bundles(self):
self.assertTrue("ReferenceModel" not in sys.modules)
scenario_instance_factory = \
ScenarioTreeInstanceFactory(farmer_model_dir,
farmer_scenariotree_dir)
scenario_tree = \
scenario_instance_factory.generate_scenario_tree(random_bundles=2)
self.assertEqual(scenario_tree.contains_bundles(), True)
self.assertEqual(len(scenario_tree._scenario_bundles), 2)
self.assertTrue("ReferenceModel" in sys.modules)
del sys.modules["ReferenceModel"]
def __init__(self,
fsfile,
fsfct="pysp_instance_creation_callback",
tree_model=None,
phopts=None):
"""Initialize a StochSolver object.
"""
fsfile = fsfile.replace('.py', '') # import does not like .py
# __import__ only gives the top level module
# probably need to be dealing with modules installed via setup.py
m = __import__(fsfile)
for n in fsfile.split(".")[1:]:
m = getattr(m, n)
scen_function = getattr(m, fsfct)
if tree_model is None:
tree_maker = getattr(m, \
"pysp_scenario_tree_model_callback")
tree = tree_maker()
# tree_model = tree.as_concrete_model()
tree_model = tree
# DLW March 21: still not correct
scenario_instance_factory = \
ScenarioTreeInstanceFactory(scen_function, tree_model)
# ScenarioTreeInstanceFactory("ReferenceModel.py", tree_model)
else:
# DLW March 21: still not correct
scenario_instance_factory = \
ScenarioTreeInstanceFactory(scen_function, tree_model)
kwargs = _kwfromphopts(phopts)
self.scenario_tree = \
scenario_instance_factory.generate_scenario_tree(**kwargs) # verbose = True)
instances = scenario_instance_factory. \
construct_instances_for_scenario_tree(self.scenario_tree)
self.scenario_tree.linkInInstances(instances)
def run(args=None):
AllInOne = False
# The value of AllInOne will be set to True for the "old"
# computeconf (with fraction_for_solve) and will stay False for
# the "new" computeconf (with MRP_directory_basename)
try:
conf_options_parser = construct_ph_options_parser(
"computeconf [options]")
conf_options_parser.add_argument(
"--fraction-scenarios-for-solve",
help=
"The fraction of scenarios that are allocated to finding a solution. Default is None.",
action="store",
dest="fraction_for_solve",
type=float,
default=None)
conf_options_parser.add_argument(
"--number-samples-for-confidence-interval",
help=
"The number of samples of scenarios that are allocated to the confidence inteval (n_g). Default is None.",
action="store",
dest="n_g",
type=int,
default=None)
conf_options_parser.add_argument(
"--confidence-interval-alpha",
help="The alpha level for the confidence interval. Default is 0.05",
action="store",
dest="confidence_interval_alpha",
type=float,
default=0.05)
conf_options_parser.add_argument(
"--solve-xhat-with-ph",
help=
"Perform xhat solve via PH rather than an EF solve. Default is False",
action="store_true",
dest="solve_xhat_with_ph",
default=False)
conf_options_parser.add_argument(
"--random-seed",
help=
"Seed the random number generator used to select samples. Defaults to 0, indicating time seed will be used.",
action="store",
dest="random_seed",
type=int,
default=0)
conf_options_parser.add_argument(
"--append-file",
help=
"File to which summary run information is appended, for output tracking purposes.",
action="store",
dest="append_file",
type=str,
default=None)
conf_options_parser.add_argument(
"--write-xhat-solution",
help=
"Write xhat solutions (first stage variables only) to the append file? Defaults to False.",
action="store_true",
dest="write_xhat_solution",
default=False)
conf_options_parser.add_argument(
"--generate-weighted-cvar",
help="Add a weighted CVaR term to the primary objective",
action="store_true",
dest="generate_weighted_cvar",
default=False)
conf_options_parser.add_argument(
"--cvar-weight",
help=
"The weight associated with the CVaR term in the risk-weighted objective formulation. Default is 1.0. If the weight is 0, then *only* a non-weighted CVaR cost will appear in the EF objective - the expected cost component will be dropped.",
action="store",
dest="cvar_weight",
type=float,
default=1.0)
conf_options_parser.add_argument(
"--risk-alpha",
help=
"The probability threshold associated with cvar (or any future) risk-oriented performance metrics. Default is 0.95.",
action="store",
dest="risk_alpha",
type=float,
default=0.95)
conf_options_parser.add_argument(
"--MRP-directory-basename",
help=
"The basename for the replicate directories. It will be appended by the number of the group (loop over n_g). Default is None",
action="store",
dest="MRP_directory_basename",
type=str,
default=None)
options = conf_options_parser.parse_args(args=args)
# temporary hack
options._ef_options = conf_options_parser._ef_options
options._ef_options.import_argparse(options)
except SystemExit as _exc:
# the parser throws a system exit if "-h" is specified - catch
# it to exit gracefully.
return _exc.code
# seed the generator if a user-supplied seed is
# provided. otherwise, python will seed from the current system
# time.
if options.random_seed > 0:
random.seed(options.random_seed)
start_time = time.time()
if options.verbose:
print("Importing model and scenario tree files")
scenario_instance_factory = \
ScenarioTreeInstanceFactory(options.model_directory,
options.instance_directory,
options.verbose)
if _OLD_OUTPUT:
print("Loading reference model and scenario tree")
if options.verbose or options.output_times:
print("Time to import model and scenario "
"tree structure files=%.2f seconds" % (time.time() - start_time))
try:
scenario_tree = \
scenario_instance_factory.generate_scenario_tree(
downsample_fraction=options.scenario_tree_downsample_fraction,
bundles=options.scenario_bundle_specification,
random_bundles=options.create_random_bundles,
random_seed=options.scenario_tree_random_seed)
#
# print the input tree for validation/information purposes.
#
if options.verbose:
scenario_tree.pprint()
#
# validate the tree prior to doing anything serious
#
if not scenario_tree.validate():
raise RuntimeError("Scenario tree is invalid")
else:
if options.verbose:
print("Scenario tree is valid!")
index_list, num_scenarios_for_solution, num_scenarios_per_sample = \
partition_scenario_space(scenario_tree,
options)
#index_list = [0,3,5,7,1,4,6,8,2,9]
#for ndx in index_list:
# print("%d: %s" % (ndx, scenario_tree._scenarios[ndx]._name))
xhat_ph = find_candidate(scenario_instance_factory, index_list,
num_scenarios_for_solution, scenario_tree,
options)
run_conf(scenario_instance_factory, index_list,
num_scenarios_for_solution, num_scenarios_per_sample,
scenario_tree, xhat_ph, options)
finally:
# delete temporary unarchived directories
if scenario_instance_factory is not None:
scenario_instance_factory.close()
def run(args=None):
AllInOne = False
# The value of AllInOne will be set to True for the "old"
# computeconf (with fraction_for_solve) and will stay False for
# the "new" computeconf (with MRP_directory_basename)
try:
conf_options_parser = construct_ph_options_parser("computeconf [options]")
conf_options_parser.add_argument(
"--fraction-scenarios-for-solve",
help="The fraction of scenarios that are allocated to finding a solution. Default is None.",
action="store",
dest="fraction_for_solve",
type=float,
default=None,
)
conf_options_parser.add_argument(
"--number-samples-for-confidence-interval",
help="The number of samples of scenarios that are allocated to the confidence inteval (n_g). Default is None.",
action="store",
dest="n_g",
type=int,
default=None,
)
conf_options_parser.add_argument(
"--confidence-interval-alpha",
help="The alpha level for the confidence interval. Default is 0.05",
action="store",
dest="confidence_interval_alpha",
type=float,
default=0.05,
)
conf_options_parser.add_argument(
"--solve-xhat-with-ph",
help="Perform xhat solve via PH rather than an EF solve. Default is False",
action="store_true",
dest="solve_xhat_with_ph",
default=False,
)
conf_options_parser.add_argument(
"--random-seed",
help="Seed the random number generator used to select samples. Defaults to 0, indicating time seed will be used.",
action="store",
dest="random_seed",
type=int,
default=0,
)
conf_options_parser.add_argument(
"--append-file",
help="File to which summary run information is appended, for output tracking purposes.",
action="store",
dest="append_file",
type=str,
default=None,
)
conf_options_parser.add_argument(
"--write-xhat-solution",
help="Write xhat solutions (first stage variables only) to the append file? Defaults to False.",
action="store_true",
dest="write_xhat_solution",
default=False,
)
conf_options_parser.add_argument(
"--generate-weighted-cvar",
help="Add a weighted CVaR term to the primary objective",
action="store_true",
dest="generate_weighted_cvar",
default=False,
)
conf_options_parser.add_argument(
"--cvar-weight",
help="The weight associated with the CVaR term in the risk-weighted objective formulation. Default is 1.0. If the weight is 0, then *only* a non-weighted CVaR cost will appear in the EF objective - the expected cost component will be dropped.",
action="store",
dest="cvar_weight",
type=float,
default=1.0,
)
conf_options_parser.add_argument(
"--risk-alpha",
help="The probability threshold associated with cvar (or any future) risk-oriented performance metrics. Default is 0.95.",
action="store",
dest="risk_alpha",
type=float,
default=0.95,
)
conf_options_parser.add_argument(
"--MRP-directory-basename",
help="The basename for the replicate directories. It will be appended by the number of the group (loop over n_g). Default is None",
action="store",
dest="MRP_directory_basename",
type=str,
default=None,
)
options = conf_options_parser.parse_args(args=args)
# temporary hack
options._ef_options = conf_options_parser._ef_options
options._ef_options.import_argparse(options)
except SystemExit as _exc:
# the parser throws a system exit if "-h" is specified - catch
# it to exit gracefully.
return _exc.code
# seed the generator if a user-supplied seed is
# provided. otherwise, python will seed from the current system
# time.
if options.random_seed > 0:
random.seed(options.random_seed)
start_time = time.time()
if options.verbose:
print("Importing model and scenario tree files")
scenario_instance_factory = ScenarioTreeInstanceFactory(options.model_directory, options.instance_directory)
if _OLD_OUTPUT:
print("Loading reference model and scenario tree")
if options.verbose or options.output_times:
print("Time to import model and scenario " "tree structure files=%.2f seconds" % (time.time() - start_time))
try:
scenario_tree = scenario_instance_factory.generate_scenario_tree(
downsample_fraction=options.scenario_tree_downsample_fraction,
bundles=options.scenario_bundle_specification,
random_bundles=options.create_random_bundles,
random_seed=options.scenario_tree_random_seed,
verbose=options.verbose,
)
#
# print the input tree for validation/information purposes.
#
if options.verbose:
scenario_tree.pprint()
#
# validate the tree prior to doing anything serious
#
if not scenario_tree.validate():
raise RuntimeError("Scenario tree is invalid")
else:
if options.verbose:
print("Scenario tree is valid!")
index_list, num_scenarios_for_solution, num_scenarios_per_sample = partition_scenario_space(
scenario_tree, options
)
# index_list = [0,3,5,7,1,4,6,8,2,9]
# for ndx in index_list:
# print("%d: %s" % (ndx, scenario_tree._scenarios[ndx]._name))
xhat_ph = find_candidate(
scenario_instance_factory, index_list, num_scenarios_for_solution, scenario_tree, options
)
run_conf(
scenario_instance_factory,
index_list,
num_scenarios_for_solution,
num_scenarios_per_sample,
scenario_tree,
xhat_ph,
options,
)
finally:
# delete temporary unarchived directories
if scenario_instance_factory is not None:
scenario_instance_factory.close()
class ScenarioTreeServerPyro(pyu_pyro.TaskWorker):
# Maps name to a registered worker class to instantiate
_registered_workers = {}
@classmethod
def get_registered_worker_type(cls, name):
if name in cls._registered_workers:
return cls._registered_workers[name]
raise KeyError("No worker type has been registered under the name "
"'%s' for ScenarioTreeServerPyro" % (name))
def __init__(self, *args, **kwds):
mpi = kwds.pop('mpi', None)
# add for purposes of diagnostic output.
kwds["name"] = ("ScenarioTreeServerPyro_%d@%s" %
(os.getpid(), socket.gethostname()))
if mpi is not None:
assert len(mpi) == 2
kwds["name"] += "_MPIRank_" + str(mpi[1].rank)
kwds["caller_name"] = kwds["name"]
self._modules_imported = kwds.pop('modules_imported', {})
pyu_pyro.TaskWorker.__init__(self, **kwds)
assert hasattr(self, "_bulk_task_collection")
self._bulk_task_collection = True
self._contiguous_task_processing = False
self.type = self.WORKERNAME
self.block = True
self.timeout = None
self._worker_map = {}
self._init_verbose = self._verbose
# A reference to the mpi4py.MPI namespace
self.MPI = None
# The communicator and group associated with all processors
self.mpi_comm_world = None
self.mpi_group_world = None
# The communicator associated with the workers assigned
# to the current current client
self.mpi_comm_workers = None
if mpi is not None:
assert len(mpi) == 2
self.MPI = mpi[0]
self.mpi_comm_world = mpi[1]
self.mpi_group_world = self.mpi_comm_world.Get_group()
#
# These will be used by all subsequent workers created
# by this server. Their creation can eat up a nontrivial
# amount of initialization time when a large number of
# workers are created on this server, so we only create
# them once.
#
self._scenario_instance_factory = None
self._full_scenario_tree = None
def reset(self):
if self._scenario_instance_factory is not None:
self._scenario_instance_factory.close()
self._scenario_instance_factory = None
self._full_scenario_tree = None
for worker_name in list(self._worker_map):
self.remove_worker(worker_name)
if self.mpi_comm_workers is not None:
self.mpi_comm_workers.Free()
self.mpi_comm_workers = None
self._verbose = self._init_verbose
def remove_worker(self, name):
self._worker_map[name].close()
del self._worker_map[name]
def process(self, data):
self._worker_task_return_queue = self._current_task_client
try:
# The only reason we are go through this much
# effort to deal with the serpent serializer
# is because it is the default in Pyro4.
if pyu_pyro.using_pyro4 and \
(Pyro4.config.SERIALIZER == 'serpent'):
if six.PY3:
assert type(data) is dict
assert data['encoding'] == 'base64'
data = base64.b64decode(data['data'])
else:
assert type(data) is unicode
data = str(data)
return pickle.dumps(self._process(pickle.loads(data)))
except:
logger.error("Scenario tree server %s caught an exception of type "
"%s while processing a task. Going idle." %
(self.WORKERNAME, sys.exc_info()[0].__name__))
traceback.print_exception(*sys.exc_info())
self._worker_error = True
return pickle.dumps(
pyu_pyro.TaskProcessingError(traceback.format_exc()))
def _process(self, data):
data = Bunch(**data)
result = None
if not data.action.startswith('ScenarioTreeServerPyro_'):
#with PauseGC() as pgc:
result = getattr(self._worker_map[data.worker_name], data.action)\
(*data.args, **data.kwds)
elif data.action == 'ScenarioTreeServerPyro_setup':
model_input = data.options.pop('model', None)
if model_input is None:
model_input = data.options.pop('model_callback')
assert dill_available
model_input = dill.loads(model_input)
scenario_tree_input = data.options.pop('scenario_tree')
data_input = data.options.pop('data')
mpi_group = data.options.pop("mpi_group", None)
verbose = data.options.pop("verbose", False)
assert len(data.options) == 0
self._verbose |= verbose
assert self._scenario_instance_factory is None
assert self._full_scenario_tree is None
if self._verbose:
print("Server %s received setup request." % (self.WORKERNAME))
# Make sure these are not archives
assert (not isinstance(model_input, six.string_types)) or \
os.path.exists(model_input)
assert isinstance(scenario_tree_input, ScenarioTree)
self._scenario_instance_factory = \
ScenarioTreeInstanceFactory(
model_input,
scenario_tree_input,
data=data_input)
#
# Try to prevent unnecessarily re-importing the model module
# if other callbacks are in the same location. Doing so might
# have serious consequences.
#
if self._scenario_instance_factory._model_module is not None:
self._modules_imported[self._scenario_instance_factory.\
_model_filename] = \
self._scenario_instance_factory._model_module
assert self._scenario_instance_factory._scenario_tree_module is None
self._full_scenario_tree = \
self._scenario_instance_factory.generate_scenario_tree()
assert self.mpi_comm_workers is None
if self.mpi_comm_world is not None:
assert self.mpi_group_world is not None
assert mpi_group is not None
mpi_group = self.mpi_group_world.Incl(mpi_group)
self.mpi_comm_workers = \
self.mpi_comm_world.Create_group(mpi_group)
else:
assert mpi_group is None
if self._full_scenario_tree is None:
raise RuntimeError("Unable to launch scenario tree worker - "
"scenario tree construction failed.")
result = True
elif data.action == "ScenarioTreeServerPyro_initialize":
worker_name = data.worker_name
if self._verbose:
print("Server %s received request to initialize "
"scenario tree worker with name %s." %
(self.WORKERNAME, worker_name))
assert self._scenario_instance_factory is not None
assert self._full_scenario_tree is not None
if worker_name in self._worker_map:
raise RuntimeError(
"Server %s Cannot initialize worker with name '%s' "
"because a worker already exists with that name." %
(self.WORKERNAME, worker_name))
worker_type = self._registered_workers[data.worker_type]
self._worker_map[worker_name] = worker_type(
self, worker_name, *data.init_args, **data.init_kwds)
result = True
elif data.action == "ScenarioTreeServerPyro_release":
if self._verbose:
print("Server %s releasing worker: %s" %
(self.WORKERNAME, data.worker_name))
self.remove_worker(data.worker_name)
result = True
elif data.action == "ScenarioTreeServerPyro_reset":
if self._verbose:
print("Server %s received reset request" % (self.WORKERNAME))
self.reset()
result = True
elif data.action == "ScenarioTreeServerPyro_shutdown":
if self._verbose:
print("Server %s received shutdown request" %
(self.WORKERNAME))
self.reset()
self._worker_shutdown = True
result = True
else:
raise ValueError("Server %s: Invalid command: %s" %
(self.WORKERNAME, data.action))
return result
class ScenarioTreeServerPyro(TaskWorker, PySPConfiguredObject):
# Maps name to a registered worker class to instantiate
_registered_workers = {}
_declared_options = \
PySPConfigBlock("Options declared for the "
"ScenarioTreeServerPyro class")
#
# scenario instance construction
#
safe_declare_common_option(_declared_options,
"model_location")
safe_declare_common_option(_declared_options,
"scenario_tree_location")
#
# scenario tree generation
#
safe_declare_common_option(_declared_options,
"scenario_tree_random_seed")
safe_declare_common_option(_declared_options,
"scenario_tree_downsample_fraction")
#
# various
#
safe_declare_common_option(_declared_options,
"verbose")
@classmethod
def get_registered_worker_type(cls, name):
if name in cls._registered_workers:
return cls._registered_workers[name]
raise KeyError("No worker type has been registered under the name "
"'%s' for ScenarioTreeServerPyro" % (name))
def __init__(self, *args, **kwds):
# add for purposes of diagnostic output.
kwds["name"] = ("ScenarioTreeServerPyro_%d@%s"
% (os.getpid(), socket.gethostname()))
kwds["caller_name"] = kwds["name"]
self._modules_imported = kwds.pop('modules_imported', {})
TaskWorker.__init__(self, **kwds)
# This classes options get updated during the "setup" phase
options = self.register_options()
PySPConfiguredObject.__init__(self, options)
self.type = self.WORKERNAME
self.block = True
self.timeout = None
self._worker_map = {}
#
# These will be used by all subsequent workers created
# by this server. Their creation can eat up a nontrivial
# amount of initialization time when a large number of
# workers are created on this server, so we only create
# them once.
#
self._scenario_instance_factory = None
self._full_scenario_tree = None
def reset(self):
if self._scenario_instance_factory is not None:
self._scenario_instance_factory.close()
self._scenario_instance_factory = None
self._full_scenario_tree = None
for worker_name in list(self._worker_map):
self.remove_worker(worker_name)
def remove_worker(self, name):
self._worker_map[name].close()
del self._worker_map[name]
def process(self, data):
self._worker_task_return_queue = self._current_task_client
try:
return self._process(data)
except:
logger.error(
"Scenario tree server %s caught an exception of type "
"%s while processing a task. Going idle."
% (self.WORKERNAME, sys.exc_info()[0].__name__))
traceback.print_exception(*sys.exc_info())
self._worker_error = True
return TaskProcessingError(traceback.format_exc())
def _process(self, data):
data = pyutilib.misc.Bunch(**data)
result = None
if not data.action.startswith('ScenarioTreeServerPyro_'):
#with PauseGC() as pgc:
result = getattr(self._worker_map[data.worker_name], data.action)\
(*data.args, **data.kwds)
elif data.action == 'ScenarioTreeServerPyro_setup':
options = self.register_options()
for name, val in iteritems(data.options):
options.get(name).set_value(val)
self.set_options(options)
self._options.verbose = self._options.verbose | self._verbose
assert self._scenario_instance_factory is None
assert self._full_scenario_tree is None
if self._options.verbose:
print("Server %s received setup request."
% (self.WORKERNAME))
print("Options:")
self.display_options()
# Make sure these are not archives
assert os.path.exists(self._options.model_location)
assert (self._options.scenario_tree_location is None) or \
os.path.exists(self._options.scenario_tree_location)
self._scenario_instance_factory = \
ScenarioTreeInstanceFactory(
self._options.model_location,
scenario_tree_location=self._options.scenario_tree_location,
verbose=self._options.verbose)
#
# Try prevent unnecessarily re-imported the model module
# if other callbacks are in the same location
#
self._modules_imported[
self._scenario_instance_factory._model_location] = \
self._scenario_instance_factory._model_module
self._modules_imported[
self._scenario_instance_factory._model_filename] = \
self._scenario_instance_factory._model_module
self._full_scenario_tree = \
self._scenario_instance_factory.generate_scenario_tree(
downsample_fraction=self._options.scenario_tree_downsample_fraction,
random_seed=self._options.scenario_tree_random_seed)
if self._full_scenario_tree is None:
raise RuntimeError("Unable to launch scenario tree worker - "
"scenario tree construction failed.")
result = True
elif data.action == "ScenarioTreeServerPyro_initialize":
worker_name = data.worker_name
if self._options.verbose:
print("Server %s received request to initialize "
"scenario tree worker with name %s."
% (self.WORKERNAME, worker_name))
assert self._scenario_instance_factory is not None
assert self._full_scenario_tree is not None
if worker_name in self._worker_map:
raise RuntimeError(
"Server %s Cannot initialize worker with name '%s' "
"because a worker already exists with that name."
% (self.WORKERNAME, worker_name))
worker_type = self._registered_workers[data.worker_type]
options = worker_type.register_options()
for name, val in iteritems(data.options):
options.get(name).set_value(val)
#
# Depending on the Pyro serializer, the namedtuple
# may be been converted to a tuple
#
if not isinstance(data.worker_init, WorkerInit):
assert type(data.worker_init) is tuple
data.worker_init = WorkerInit(type_=data.worker_init[0],
names=data.worker_init[1],
data=data.worker_init[2])
# replace enum string representation with the actual enum
# object now that we've unserialized the Pyro data
worker_init = WorkerInit(type_=getattr(WorkerInitType,
data.worker_init.type_),
names=data.worker_init.names,
data=data.worker_init.data)
self._worker_map[worker_name] = worker_type(options)
self._worker_map[worker_name].initialize(
self.WORKERNAME,
self._full_scenario_tree,
worker_name,
worker_init,
self._modules_imported)
result = True
elif data.action == "ScenarioTreeServerPyro_release":
if self._options.verbose:
print("Server %s releasing worker: %s"
% (self.WORKERNAME, data.worker_name))
self.remove_worker(data.worker_name)
result = True
elif data.action == "ScenarioTreeServerPyro_reset":
if self._options.verbose:
print("Server %s received reset request"
% (self.WORKERNAME))
self.reset()
result = True
elif data.action == "ScenarioTreeServerPyro_shutdown":
if self._options.verbose:
print("Server %s received shutdown request"
% (self.WORKERNAME))
self.reset()
self._worker_shutdown = True
result = True
else:
raise ValueError("Server %s: Invalid command: %s"
% (self.WORKERNAME, data.action))
return result
class ScenarioTreeServerPyro(TaskWorker, PySPConfiguredObject):
# Maps name to a registered worker class to instantiate
_registered_workers = {}
_declared_options = \
PySPConfigBlock("Options declared for the "
"ScenarioTreeServerPyro class")
#
# scenario instance construction
#
safe_declare_common_option(_declared_options, "model_location")
safe_declare_common_option(_declared_options, "scenario_tree_location")
#
# scenario tree generation
#
safe_declare_common_option(_declared_options, "scenario_tree_random_seed")
safe_declare_common_option(_declared_options,
"scenario_tree_downsample_fraction")
#
# various
#
safe_declare_common_option(_declared_options, "verbose")
@classmethod
def get_registered_worker_type(cls, name):
if name in cls._registered_workers:
return cls._registered_workers[name]
raise KeyError("No worker type has been registered under the name "
"'%s' for ScenarioTreeServerPyro" % (name))
def __init__(self, *args, **kwds):
# add for purposes of diagnostic output.
kwds["name"] = ("ScenarioTreeServerPyro_%d@%s" %
(os.getpid(), socket.gethostname()))
kwds["caller_name"] = kwds["name"]
self._modules_imported = kwds.pop('modules_imported', {})
TaskWorker.__init__(self, **kwds)
# This classes options get updated during the "setup" phase
options = self.register_options()
PySPConfiguredObject.__init__(self, options)
self.type = self.WORKERNAME
self.block = True
self.timeout = None
self._worker_map = {}
#
# These will be used by all subsequent workers created
# by this server. Their creation can eat up a nontrivial
# amount of initialization time when a large number of
# workers are created on this server, so we only create
# them once.
#
self._scenario_instance_factory = None
self._full_scenario_tree = None
def reset(self):
if self._scenario_instance_factory is not None:
self._scenario_instance_factory.close()
self._scenario_instance_factory = None
self._full_scenario_tree = None
for worker_name in list(self._worker_map):
self.remove_worker(worker_name)
def remove_worker(self, name):
self._worker_map[name].close()
del self._worker_map[name]
def process(self, data):
self._worker_task_return_queue = self._current_task_client
try:
return self._process(data)
except:
logger.error("Scenario tree server %s caught an exception of type "
"%s while processing a task. Going idle." %
(self.WORKERNAME, sys.exc_info()[0].__name__))
traceback.print_exception(*sys.exc_info())
self._worker_error = True
return TaskProcessingError(traceback.format_exc())
def _process(self, data):
data = pyutilib.misc.Bunch(**data)
result = None
if not data.action.startswith('ScenarioTreeServerPyro_'):
#with PauseGC() as pgc:
result = getattr(self._worker_map[data.worker_name], data.action)\
(*data.args, **data.kwds)
elif data.action == 'ScenarioTreeServerPyro_setup':
options = self.register_options()
for name, val in iteritems(data.options):
options.get(name).set_value(val)
self.set_options(options)
self._options.verbose = self._options.verbose | self._verbose
assert self._scenario_instance_factory is None
assert self._full_scenario_tree is None
if self._options.verbose:
print("Server %s received setup request." % (self.WORKERNAME))
print("Options:")
self.display_options()
# Make sure these are not archives
assert os.path.exists(self._options.model_location)
assert (self._options.scenario_tree_location is None) or \
os.path.exists(self._options.scenario_tree_location)
self._scenario_instance_factory = \
ScenarioTreeInstanceFactory(
self._options.model_location,
scenario_tree_location=self._options.scenario_tree_location,
verbose=self._options.verbose)
#
# Try prevent unnecessarily re-imported the model module
# if other callbacks are in the same location
#
self._modules_imported[
self._scenario_instance_factory._model_location] = \
self._scenario_instance_factory._model_module
self._modules_imported[
self._scenario_instance_factory._model_filename] = \
self._scenario_instance_factory._model_module
self._full_scenario_tree = \
self._scenario_instance_factory.generate_scenario_tree(
downsample_fraction=self._options.scenario_tree_downsample_fraction,
random_seed=self._options.scenario_tree_random_seed)
if self._full_scenario_tree is None:
raise RuntimeError("Unable to launch scenario tree worker - "
"scenario tree construction failed.")
result = True
elif data.action == "ScenarioTreeServerPyro_initialize":
worker_name = data.worker_name
if self._options.verbose:
print("Server %s received request to initialize "
"scenario tree worker with name %s." %
(self.WORKERNAME, worker_name))
assert self._scenario_instance_factory is not None
assert self._full_scenario_tree is not None
if worker_name in self._worker_map:
raise RuntimeError(
"Server %s Cannot initialize worker with name '%s' "
"because a worker already exists with that name." %
(self.WORKERNAME, worker_name))
worker_type = self._registered_workers[data.worker_type]
options = worker_type.register_options()
for name, val in iteritems(data.options):
options.get(name).set_value(val)
#
# Depending on the Pyro serializer, the namedtuple
# may be been converted to a tuple
#
if not isinstance(data.worker_init, WorkerInit):
assert type(data.worker_init) is tuple
data.worker_init = WorkerInit(type_=data.worker_init[0],
names=data.worker_init[1],
data=data.worker_init[2])
# replace enum string representation with the actual enum
# object now that we've unserialized the Pyro data
worker_init = WorkerInit(type_=getattr(WorkerInitType,
data.worker_init.type_),
names=data.worker_init.names,
data=data.worker_init.data)
self._worker_map[worker_name] = worker_type(options)
self._worker_map[worker_name].initialize(self.WORKERNAME,
self._full_scenario_tree,
worker_name, worker_init,
self._modules_imported)
result = True
elif data.action == "ScenarioTreeServerPyro_release":
if self._options.verbose:
print("Server %s releasing worker: %s" %
(self.WORKERNAME, data.worker_name))
self.remove_worker(data.worker_name)
result = True
elif data.action == "ScenarioTreeServerPyro_reset":
if self._options.verbose:
print("Server %s received reset request" % (self.WORKERNAME))
self.reset()
result = True
elif data.action == "ScenarioTreeServerPyro_shutdown":
if self._options.verbose:
print("Server %s received shutdown request" %
(self.WORKERNAME))
self.reset()
self._worker_shutdown = True
result = True
else:
raise ValueError("Server %s: Invalid command: %s" %
(self.WORKERNAME, data.action))
return result
class ScenarioTreeServerPyro(TaskWorker):
# Maps name to a registered worker class to instantiate
_registered_workers = {}
@classmethod
def get_registered_worker_type(cls, name):
if name in cls._registered_workers:
return cls._registered_workers[name]
raise KeyError("No worker type has been registered under the name "
"'%s' for ScenarioTreeServerPyro" % (name))
def __init__(self, *args, **kwds):
mpi = kwds.pop('mpi', None)
# add for purposes of diagnostic output.
kwds["name"] = ("ScenarioTreeServerPyro_%d@%s"
% (os.getpid(), socket.gethostname()))
if mpi is not None:
assert len(mpi) == 2
kwds["name"] += "_MPIRank_"+str(mpi[1].rank)
kwds["caller_name"] = kwds["name"]
self._modules_imported = kwds.pop('modules_imported', {})
TaskWorker.__init__(self, **kwds)
assert hasattr(self, "_bulk_task_collection")
self._bulk_task_collection = True
self._contiguous_task_processing = False
self.type = self.WORKERNAME
self.block = True
self.timeout = None
self._worker_map = {}
self._init_verbose = self._verbose
# A reference to the mpi4py.MPI namespace
self.MPI = None
# The communicator and group associated with all processors
self.mpi_comm_world = None
self.mpi_group_world = None
# The communicator associated with the workers assigned
# to the current current client
self.mpi_comm_workers = None
if mpi is not None:
assert len(mpi) == 2
self.MPI = mpi[0]
self.mpi_comm_world = mpi[1]
self.mpi_group_world = self.mpi_comm_world.Get_group()
#
# These will be used by all subsequent workers created
# by this server. Their creation can eat up a nontrivial
# amount of initialization time when a large number of
# workers are created on this server, so we only create
# them once.
#
self._scenario_instance_factory = None
self._full_scenario_tree = None
def reset(self):
if self._scenario_instance_factory is not None:
self._scenario_instance_factory.close()
self._scenario_instance_factory = None
self._full_scenario_tree = None
for worker_name in list(self._worker_map):
self.remove_worker(worker_name)
if self.mpi_comm_workers is not None:
self.mpi_comm_workers.Free()
self.mpi_comm_workers = None
self._verbose = self._init_verbose
def remove_worker(self, name):
self._worker_map[name].close()
del self._worker_map[name]
def process(self, data):
self._worker_task_return_queue = self._current_task_client
try:
# The only reason we are go through this much
# effort to deal with the serpent serializer
# is because it is the default in Pyro4.
if using_pyro4 and \
(Pyro4.config.SERIALIZER == 'serpent'):
if six.PY3:
assert type(data) is dict
assert data['encoding'] == 'base64'
data = base64.b64decode(data['data'])
else:
assert type(data) is unicode
data = str(data)
return pickle.dumps(self._process(pickle.loads(data)))
except:
logger.error(
"Scenario tree server %s caught an exception of type "
"%s while processing a task. Going idle."
% (self.WORKERNAME, sys.exc_info()[0].__name__))
traceback.print_exception(*sys.exc_info())
self._worker_error = True
return pickle.dumps(TaskProcessingError(traceback.format_exc()))
def _process(self, data):
data = pyutilib.misc.Bunch(**data)
result = None
if not data.action.startswith('ScenarioTreeServerPyro_'):
#with PauseGC() as pgc:
result = getattr(self._worker_map[data.worker_name], data.action)\
(*data.args, **data.kwds)
elif data.action == 'ScenarioTreeServerPyro_setup':
model_input = data.options.pop('model', None)
if model_input is None:
model_input = data.options.pop('model_callback')
assert dill_available
model_input = dill.loads(model_input)
scenario_tree_input = data.options.pop('scenario_tree')
data_input = data.options.pop('data')
mpi_group = data.options.pop("mpi_group",None)
verbose = data.options.pop("verbose", False)
assert len(data.options) == 0
self._verbose |= verbose
assert self._scenario_instance_factory is None
assert self._full_scenario_tree is None
if self._verbose:
print("Server %s received setup request."
% (self.WORKERNAME))
# Make sure these are not archives
assert (not isinstance(model_input, six.string_types)) or \
os.path.exists(model_input)
assert isinstance(scenario_tree_input, ScenarioTree)
self._scenario_instance_factory = \
ScenarioTreeInstanceFactory(
model_input,
scenario_tree_input,
data=data_input)
#
# Try to prevent unnecessarily re-importing the model module
# if other callbacks are in the same location. Doing so might
# have serious consequences.
#
if self._scenario_instance_factory._model_module is not None:
self._modules_imported[self._scenario_instance_factory.\
_model_filename] = \
self._scenario_instance_factory._model_module
assert self._scenario_instance_factory._scenario_tree_module is None
self._full_scenario_tree = \
self._scenario_instance_factory.generate_scenario_tree()
assert self.mpi_comm_workers is None
if self.mpi_comm_world is not None:
assert self.mpi_group_world is not None
assert mpi_group is not None
mpi_group = self.mpi_group_world.Incl(mpi_group)
self.mpi_comm_workers = \
self.mpi_comm_world.Create_group(mpi_group)
else:
assert mpi_group is None
if self._full_scenario_tree is None:
raise RuntimeError("Unable to launch scenario tree worker - "
"scenario tree construction failed.")
result = True
elif data.action == "ScenarioTreeServerPyro_initialize":
worker_name = data.worker_name
if self._verbose:
print("Server %s received request to initialize "
"scenario tree worker with name %s."
% (self.WORKERNAME, worker_name))
assert self._scenario_instance_factory is not None
assert self._full_scenario_tree is not None
if worker_name in self._worker_map:
raise RuntimeError(
"Server %s Cannot initialize worker with name '%s' "
"because a worker already exists with that name."
% (self.WORKERNAME, worker_name))
worker_type = self._registered_workers[data.worker_type]
self._worker_map[worker_name] = worker_type(
self,
worker_name,
*data.init_args,
**data.init_kwds)
result = True
elif data.action == "ScenarioTreeServerPyro_release":
if self._verbose:
print("Server %s releasing worker: %s"
% (self.WORKERNAME, data.worker_name))
self.remove_worker(data.worker_name)
result = True
elif data.action == "ScenarioTreeServerPyro_reset":
if self._verbose:
print("Server %s received reset request"
% (self.WORKERNAME))
self.reset()
result = True
elif data.action == "ScenarioTreeServerPyro_shutdown":
if self._verbose:
print("Server %s received shutdown request"
% (self.WORKERNAME))
self.reset()
self._worker_shutdown = True
result = True
else:
raise ValueError("Server %s: Invalid command: %s"
% (self.WORKERNAME, data.action))
return result
def __init__(self, fsfile,
fsfct = None,
tree_model = None,
phopts = None):
"""Initialize a StochSolver object.
"""
if fsfct is None:
# Changed in October 2018: None implies AbstractModel
args_list = _optiondict_2_list(phopts)
parser = phinit.construct_ph_options_parser("")
options = parser.parse_args(args_list)
scenario_instance_factory = \
ScenarioTreeInstanceFactory(fsfile, tree_model)
try:
self.scenario_tree = \
phinit.GenerateScenarioTreeForPH(options,
scenario_instance_factory)
except:
print ("ERROR in StochSolver called from",inspect.stack()[1][3])
raise RuntimeError("fsfct is None, so assuming",
"AbstractModel but could not find all ingredients.")
else: # concrete model
if callable(fsfct):
scen_function = fsfct
else: # better be a string
fsfile = fsfile.replace('.py','') # import does not like .py
# __import__ only gives the top level module
# probably need to be dealing with modules installed via setup.py
m = __import__(fsfile)
for n in fsfile.split(".")[1:]:
m = getattr(m, n)
scen_function = getattr(m, fsfct)
if tree_model is None:
treecbname = "pysp_scenario_tree_model_callback"
tree_maker = getattr(m, treecbname)
tree = tree_maker()
if isinstance(tree, Pyo.ConcreteModel):
tree_model = tree
else:
raise RuntimeError("The tree returned by",treecbname,
"must be a ConcreteModel")
scenario_instance_factory = ScenarioTreeInstanceFactory(scen_function, tree_model)
else:
# DLW March 21: still not correct
scenario_instance_factory = \
ScenarioTreeInstanceFactory(scen_function, tree_model)
kwargs = _kwfromphopts(phopts)
self.scenario_tree = \
scenario_instance_factory.generate_scenario_tree(**kwargs) #verbose = True)
instances = scenario_instance_factory. \
construct_instances_for_scenario_tree(self.scenario_tree)
self.scenario_tree.linkInInstances(instances)