def test_dill(self):
p = parameter(1)
f = functional_value(lambda: p())
self.assertEqual(f(), 1)
fup = dill.loads(dill.dumps(f))
p.value = 2
self.assertEqual(f(), 2)
self.assertEqual(fup(), 1)
b = block()
b.p = p
b.f = f
self.assertEqual(b.f(), 2)
bup = dill.loads(dill.dumps(b))
fup = bup.f
b.p.value = 4
self.assertEqual(b.f(), 4)
self.assertEqual(bup.f(), 2)
bup.p.value = 4
self.assertEqual(bup.f(), 4)
def _invoke_function_impl(self,
function,
module_name=None,
invocation_type=InvocationType.Single,
function_args=(),
function_kwds=None):
start_time = time.time()
if self.get_option("verbose"):
if module_name is not None:
print("Received request to invoke function=%s "
"in module=%s" % (str(function), str(module_name)))
else:
print("Received request to invoke anonymous "
"function serialized using the dill module")
# InvocationType is transmitted as (key, data) to
# avoid issues with Pyro, so this function accepts a
# tuple and converts back to InvocationType
if type(invocation_type) is tuple:
_invocation_type_key, _invocation_type_data = invocation_type
assert isinstance(_invocation_type_key, string_types)
invocation_type = getattr(InvocationType,
_invocation_type_key)
if _invocation_type_data is not None:
invocation_type = invocation_type(_invocation_type_data)
# here we assume that if the module_name is None,
# then a function was serialized by the fill module
# before being transmitted
if module_name is None:
assert dill_available
function = dill.loads(function)
result = self._invoke_function_by_worker(
function,
module_name=module_name,
invocation_type=invocation_type,
function_args=function_args,
function_kwds=function_kwds)
end_time = time.time()
if self.get_option("output_times") or \
self.get_option("verbose"):
print("External function invocation time=%.2f seconds"
% (end_time - start_time))
return result
def _invoke_function_impl(self,
function,
module_name=None,
invocation_type=InvocationType.Single,
function_args=(),
function_kwds=None):
start_time = time.time()
if self.get_option("verbose"):
if module_name is not None:
print("Received request to invoke function=%s "
"in module=%s" % (str(function), str(module_name)))
else:
print("Received request to invoke anonymous "
"function serialized using the dill module")
# pyutilib.Enum can not be serialized depending on the
# serializer type used by Pyro, so we just transmit it
# as a (key, data) tuple in that case
if type(invocation_type) is tuple:
_invocation_type_key, _invocation_type_data = invocation_type
assert isinstance(_invocation_type_key, string_types)
invocation_type = getattr(InvocationType, _invocation_type_key)
if _invocation_type_data is not None:
assert isinstance(invocation_type, _EnumValueWithData)
invocation_type = invocation_type(_invocation_type_data)
# here we assume that if the module_name is None,
# then a function was serialized by the fill module
# before being transmitted
if module_name is None:
assert dill_available
function = dill.loads(function)
result = self._invoke_function_by_worker(
function,
module_name=module_name,
invocation_type=invocation_type,
function_args=function_args,
function_kwds=function_kwds)
end_time = time.time()
if self.get_option("output_times") or \
self.get_option("verbose"):
print("External function invocation time=%.2f seconds" %
(end_time - start_time))
return result
def _process(self, data):
data = Bunch(**data)
result = None
if not data.action.startswith('ScenarioTreeServerPyro_'):
#with PauseGC() as pgc:
result = getattr(self._worker_map[data.worker_name], data.action)\
(*data.args, **data.kwds)
elif data.action == 'ScenarioTreeServerPyro_setup':
model_input = data.options.pop('model', None)
if model_input is None:
model_input = data.options.pop('model_callback')
assert dill_available
model_input = dill.loads(model_input)
scenario_tree_input = data.options.pop('scenario_tree')
data_input = data.options.pop('data')
mpi_group = data.options.pop("mpi_group", None)
verbose = data.options.pop("verbose", False)
assert len(data.options) == 0
self._verbose |= verbose
assert self._scenario_instance_factory is None
assert self._full_scenario_tree is None
if self._verbose:
print("Server %s received setup request." % (self.WORKERNAME))
# Make sure these are not archives
assert (not isinstance(model_input, six.string_types)) or \
os.path.exists(model_input)
assert isinstance(scenario_tree_input, ScenarioTree)
self._scenario_instance_factory = \
ScenarioTreeInstanceFactory(
model_input,
scenario_tree_input,
data=data_input)
#
# Try to prevent unnecessarily re-importing the model module
# if other callbacks are in the same location. Doing so might
# have serious consequences.
#
if self._scenario_instance_factory._model_module is not None:
self._modules_imported[self._scenario_instance_factory.\
_model_filename] = \
self._scenario_instance_factory._model_module
assert self._scenario_instance_factory._scenario_tree_module is None
self._full_scenario_tree = \
self._scenario_instance_factory.generate_scenario_tree()
assert self.mpi_comm_workers is None
if self.mpi_comm_world is not None:
assert self.mpi_group_world is not None
assert mpi_group is not None
mpi_group = self.mpi_group_world.Incl(mpi_group)
self.mpi_comm_workers = \
self.mpi_comm_world.Create_group(mpi_group)
else:
assert mpi_group is None
if self._full_scenario_tree is None:
raise RuntimeError("Unable to launch scenario tree worker - "
"scenario tree construction failed.")
result = True
elif data.action == "ScenarioTreeServerPyro_initialize":
worker_name = data.worker_name
if self._verbose:
print("Server %s received request to initialize "
"scenario tree worker with name %s." %
(self.WORKERNAME, worker_name))
assert self._scenario_instance_factory is not None
assert self._full_scenario_tree is not None
if worker_name in self._worker_map:
raise RuntimeError(
"Server %s Cannot initialize worker with name '%s' "
"because a worker already exists with that name." %
(self.WORKERNAME, worker_name))
worker_type = self._registered_workers[data.worker_type]
self._worker_map[worker_name] = worker_type(
self, worker_name, *data.init_args, **data.init_kwds)
result = True
elif data.action == "ScenarioTreeServerPyro_release":
if self._verbose:
print("Server %s releasing worker: %s" %
(self.WORKERNAME, data.worker_name))
self.remove_worker(data.worker_name)
result = True
elif data.action == "ScenarioTreeServerPyro_reset":
if self._verbose:
print("Server %s received reset request" % (self.WORKERNAME))
self.reset()
result = True
elif data.action == "ScenarioTreeServerPyro_shutdown":
if self._verbose:
print("Server %s received shutdown request" %
(self.WORKERNAME))
self.reset()
self._worker_shutdown = True
result = True
else:
raise ValueError("Server %s: Invalid command: %s" %
(self.WORKERNAME, data.action))
return result
