def __init__(self, fsfile,
fsfct = None,
tree_model = None,
phopts = None):
"""Initialize a StochSolver object.
"""
if fsfct is None:
# Changed in October 2018: None implies AbstractModel
args_list = _optiondict_2_list(phopts)
parser = construct_ph_options_parser("")
options = parser.parse_args(args_list)
scenario_instance_factory = \
ScenarioTreeInstanceFactory(fsfile, tree_model)
try:
self.scenario_tree = \
GenerateScenarioTreeForPH(options,
scenario_instance_factory)
except:
print ("ERROR in StochSolver called from",inspect.stack()[1][3])
raise RuntimeError("fsfct is None, so assuming",
"AbstractModel but could not find all ingredients.")
else: # concrete model
if callable(fsfct):
scen_function = fsfct
else: # better be a string
fsfile = fsfile.replace('.py','') # import does not like .py
# __import__ only gives the top level module
# probably need to be dealing with modules installed via setup.py
m = __import__(fsfile)
for n in fsfile.split(".")[1:]:
m = getattr(m, n)
scen_function = getattr(m, fsfct)
if tree_model is None:
treecbname = "pysp_scenario_tree_model_callback"
tree_maker = getattr(m, treecbname)
tree = tree_maker()
scenario_instance_factory = ScenarioTreeInstanceFactory(scen_function, tree_model)
else:
# DLW March 21: still not correct
scenario_instance_factory = \
ScenarioTreeInstanceFactory(scen_function, tree_model)
kwargs = _kwfromphopts(phopts)
self.scenario_tree = \
scenario_instance_factory.generate_scenario_tree(**kwargs) #verbose = True)
instances = scenario_instance_factory. \
construct_instances_for_scenario_tree(self.scenario_tree)
self.scenario_tree.linkInInstances(instances)
class ScenarioTreeServerPyro(pyu_pyro.TaskWorker):
# Maps name to a registered worker class to instantiate
_registered_workers = {}
@classmethod
def get_registered_worker_type(cls, name):
if name in cls._registered_workers:
return cls._registered_workers[name]
raise KeyError("No worker type has been registered under the name "
"'%s' for ScenarioTreeServerPyro" % (name))
def __init__(self, *args, **kwds):
mpi = kwds.pop('mpi', None)
# add for purposes of diagnostic output.
kwds["name"] = ("ScenarioTreeServerPyro_%d@%s" %
(os.getpid(), socket.gethostname()))
if mpi is not None:
assert len(mpi) == 2
kwds["name"] += "_MPIRank_" + str(mpi[1].rank)
kwds["caller_name"] = kwds["name"]
self._modules_imported = kwds.pop('modules_imported', {})
pyu_pyro.TaskWorker.__init__(self, **kwds)
assert hasattr(self, "_bulk_task_collection")
self._bulk_task_collection = True
self._contiguous_task_processing = False
self.type = self.WORKERNAME
self.block = True
self.timeout = None
self._worker_map = {}
self._init_verbose = self._verbose
# A reference to the mpi4py.MPI namespace
self.MPI = None
# The communicator and group associated with all processors
self.mpi_comm_world = None
self.mpi_group_world = None
# The communicator associated with the workers assigned
# to the current current client
self.mpi_comm_workers = None
if mpi is not None:
assert len(mpi) == 2
self.MPI = mpi[0]
self.mpi_comm_world = mpi[1]
self.mpi_group_world = self.mpi_comm_world.Get_group()
#
# These will be used by all subsequent workers created
# by this server. Their creation can eat up a nontrivial
# amount of initialization time when a large number of
# workers are created on this server, so we only create
# them once.
#
self._scenario_instance_factory = None
self._full_scenario_tree = None
def reset(self):
if self._scenario_instance_factory is not None:
self._scenario_instance_factory.close()
self._scenario_instance_factory = None
self._full_scenario_tree = None
for worker_name in list(self._worker_map):
self.remove_worker(worker_name)
if self.mpi_comm_workers is not None:
self.mpi_comm_workers.Free()
self.mpi_comm_workers = None
self._verbose = self._init_verbose
def remove_worker(self, name):
self._worker_map[name].close()
del self._worker_map[name]
def process(self, data):
self._worker_task_return_queue = self._current_task_client
try:
# The only reason we are go through this much
# effort to deal with the serpent serializer
# is because it is the default in Pyro4.
if pyu_pyro.using_pyro4 and \
(Pyro4.config.SERIALIZER == 'serpent'):
if six.PY3:
assert type(data) is dict
assert data['encoding'] == 'base64'
data = base64.b64decode(data['data'])
else:
assert type(data) is unicode
data = str(data)
return pickle.dumps(self._process(pickle.loads(data)))
except:
logger.error("Scenario tree server %s caught an exception of type "
"%s while processing a task. Going idle." %
(self.WORKERNAME, sys.exc_info()[0].__name__))
traceback.print_exception(*sys.exc_info())
self._worker_error = True
return pickle.dumps(
pyu_pyro.TaskProcessingError(traceback.format_exc()))
def _process(self, data):
data = Bunch(**data)
result = None
if not data.action.startswith('ScenarioTreeServerPyro_'):
result = getattr(self._worker_map[data.worker_name], data.action)\
(*data.args, **data.kwds)
elif data.action == 'ScenarioTreeServerPyro_setup':
model_input = data.options.pop('model', None)
if model_input is None:
model_input = data.options.pop('model_callback')
assert dill_available
model_input = dill.loads(model_input)
scenario_tree_input = data.options.pop('scenario_tree')
data_input = data.options.pop('data')
mpi_group = data.options.pop("mpi_group", None)
verbose = data.options.pop("verbose", False)
assert len(data.options) == 0
self._verbose |= verbose
assert self._scenario_instance_factory is None
assert self._full_scenario_tree is None
if self._verbose:
print("Server %s received setup request." % (self.WORKERNAME))
# Make sure these are not archives
assert (not isinstance(model_input, six.string_types)) or \
os.path.exists(model_input)
assert isinstance(scenario_tree_input, ScenarioTree)
self._scenario_instance_factory = \
ScenarioTreeInstanceFactory(
model_input,
scenario_tree_input,
data=data_input)
#
# Try to prevent unnecessarily re-importing the model module
# if other callbacks are in the same location. Doing so might
# have serious consequences.
#
if self._scenario_instance_factory._model_module is not None:
self._modules_imported[self._scenario_instance_factory.\
_model_filename] = \
self._scenario_instance_factory._model_module
assert self._scenario_instance_factory._scenario_tree_module is None
self._full_scenario_tree = \
self._scenario_instance_factory.generate_scenario_tree()
assert self.mpi_comm_workers is None
if self.mpi_comm_world is not None:
assert self.mpi_group_world is not None
assert mpi_group is not None
mpi_group = self.mpi_group_world.Incl(mpi_group)
self.mpi_comm_workers = \
self.mpi_comm_world.Create_group(mpi_group)
else:
assert mpi_group is None
if self._full_scenario_tree is None:
raise RuntimeError("Unable to launch scenario tree worker - "
"scenario tree construction failed.")
result = True
elif data.action == "ScenarioTreeServerPyro_initialize":
worker_name = data.worker_name
if self._verbose:
print("Server %s received request to initialize "
"scenario tree worker with name %s." %
(self.WORKERNAME, worker_name))
assert self._scenario_instance_factory is not None
assert self._full_scenario_tree is not None
if worker_name in self._worker_map:
raise RuntimeError(
"Server %s Cannot initialize worker with name '%s' "
"because a worker already exists with that name." %
(self.WORKERNAME, worker_name))
worker_type = self._registered_workers[data.worker_type]
self._worker_map[worker_name] = worker_type(
self, worker_name, *data.init_args, **data.init_kwds)
result = True
elif data.action == "ScenarioTreeServerPyro_release":
if self._verbose:
print("Server %s releasing worker: %s" %
(self.WORKERNAME, data.worker_name))
self.remove_worker(data.worker_name)
result = True
elif data.action == "ScenarioTreeServerPyro_reset":
if self._verbose:
print("Server %s received reset request" % (self.WORKERNAME))
self.reset()
result = True
elif data.action == "ScenarioTreeServerPyro_shutdown":
if self._verbose:
print("Server %s received shutdown request" %
(self.WORKERNAME))
self.reset()
self._worker_shutdown = True
result = True
else:
raise ValueError("Server %s: Invalid command: %s" %
(self.WORKERNAME, data.action))
return result