def test_dill(self): p = parameter(1) f = functional_value(lambda: p()) self.assertEqual(f(), 1) fup = dill.loads(dill.dumps(f)) p.value = 2 self.assertEqual(f(), 2) self.assertEqual(fup(), 1) b = block() b.p = p b.f = f self.assertEqual(b.f(), 2) bup = dill.loads(dill.dumps(b)) fup = bup.f b.p.value = 4 self.assertEqual(b.f(), 4) self.assertEqual(bup.f(), 2) bup.p.value = 4 self.assertEqual(bup.f(), 4)
def _invoke_function_impl(self, function, module_name=None, invocation_type=InvocationType.Single, function_args=(), function_kwds=None): start_time = time.time() if self.get_option("verbose"): if module_name is not None: print("Received request to invoke function=%s " "in module=%s" % (str(function), str(module_name))) else: print("Received request to invoke anonymous " "function serialized using the dill module") # InvocationType is transmitted as (key, data) to # avoid issues with Pyro, so this function accepts a # tuple and converts back to InvocationType if type(invocation_type) is tuple: _invocation_type_key, _invocation_type_data = invocation_type assert isinstance(_invocation_type_key, string_types) invocation_type = getattr(InvocationType, _invocation_type_key) if _invocation_type_data is not None: invocation_type = invocation_type(_invocation_type_data) # here we assume that if the module_name is None, # then a function was serialized by the fill module # before being transmitted if module_name is None: assert dill_available function = dill.loads(function) result = self._invoke_function_by_worker( function, module_name=module_name, invocation_type=invocation_type, function_args=function_args, function_kwds=function_kwds) end_time = time.time() if self.get_option("output_times") or \ self.get_option("verbose"): print("External function invocation time=%.2f seconds" % (end_time - start_time)) return result
def _invoke_function_impl(self, function, module_name=None, invocation_type=InvocationType.Single, function_args=(), function_kwds=None): start_time = time.time() if self.get_option("verbose"): if module_name is not None: print("Received request to invoke function=%s " "in module=%s" % (str(function), str(module_name))) else: print("Received request to invoke anonymous " "function serialized using the dill module") # pyutilib.Enum can not be serialized depending on the # serializer type used by Pyro, so we just transmit it # as a (key, data) tuple in that case if type(invocation_type) is tuple: _invocation_type_key, _invocation_type_data = invocation_type assert isinstance(_invocation_type_key, string_types) invocation_type = getattr(InvocationType, _invocation_type_key) if _invocation_type_data is not None: assert isinstance(invocation_type, _EnumValueWithData) invocation_type = invocation_type(_invocation_type_data) # here we assume that if the module_name is None, # then a function was serialized by the fill module # before being transmitted if module_name is None: assert dill_available function = dill.loads(function) result = self._invoke_function_by_worker( function, module_name=module_name, invocation_type=invocation_type, function_args=function_args, function_kwds=function_kwds) end_time = time.time() if self.get_option("output_times") or \ self.get_option("verbose"): print("External function invocation time=%.2f seconds" % (end_time - start_time)) return result
def _process(self, data): data = Bunch(**data) result = None if not data.action.startswith('ScenarioTreeServerPyro_'): #with PauseGC() as pgc: result = getattr(self._worker_map[data.worker_name], data.action)\ (*data.args, **data.kwds) elif data.action == 'ScenarioTreeServerPyro_setup': model_input = data.options.pop('model', None) if model_input is None: model_input = data.options.pop('model_callback') assert dill_available model_input = dill.loads(model_input) scenario_tree_input = data.options.pop('scenario_tree') data_input = data.options.pop('data') mpi_group = data.options.pop("mpi_group", None) verbose = data.options.pop("verbose", False) assert len(data.options) == 0 self._verbose |= verbose assert self._scenario_instance_factory is None assert self._full_scenario_tree is None if self._verbose: print("Server %s received setup request." % (self.WORKERNAME)) # Make sure these are not archives assert (not isinstance(model_input, six.string_types)) or \ os.path.exists(model_input) assert isinstance(scenario_tree_input, ScenarioTree) self._scenario_instance_factory = \ ScenarioTreeInstanceFactory( model_input, scenario_tree_input, data=data_input) # # Try to prevent unnecessarily re-importing the model module # if other callbacks are in the same location. Doing so might # have serious consequences. # if self._scenario_instance_factory._model_module is not None: self._modules_imported[self._scenario_instance_factory.\ _model_filename] = \ self._scenario_instance_factory._model_module assert self._scenario_instance_factory._scenario_tree_module is None self._full_scenario_tree = \ self._scenario_instance_factory.generate_scenario_tree() assert self.mpi_comm_workers is None if self.mpi_comm_world is not None: assert self.mpi_group_world is not None assert mpi_group is not None mpi_group = self.mpi_group_world.Incl(mpi_group) self.mpi_comm_workers = \ self.mpi_comm_world.Create_group(mpi_group) else: assert mpi_group is None if self._full_scenario_tree is None: raise RuntimeError("Unable to launch scenario tree worker - " "scenario tree construction failed.") result = True elif data.action == "ScenarioTreeServerPyro_initialize": worker_name = data.worker_name if self._verbose: print("Server %s received request to initialize " "scenario tree worker with name %s." % (self.WORKERNAME, worker_name)) assert self._scenario_instance_factory is not None assert self._full_scenario_tree is not None if worker_name in self._worker_map: raise RuntimeError( "Server %s Cannot initialize worker with name '%s' " "because a worker already exists with that name." % (self.WORKERNAME, worker_name)) worker_type = self._registered_workers[data.worker_type] self._worker_map[worker_name] = worker_type( self, worker_name, *data.init_args, **data.init_kwds) result = True elif data.action == "ScenarioTreeServerPyro_release": if self._verbose: print("Server %s releasing worker: %s" % (self.WORKERNAME, data.worker_name)) self.remove_worker(data.worker_name) result = True elif data.action == "ScenarioTreeServerPyro_reset": if self._verbose: print("Server %s received reset request" % (self.WORKERNAME)) self.reset() result = True elif data.action == "ScenarioTreeServerPyro_shutdown": if self._verbose: print("Server %s received shutdown request" % (self.WORKERNAME)) self.reset() self._worker_shutdown = True result = True else: raise ValueError("Server %s: Invalid command: %s" % (self.WORKERNAME, data.action)) return result