def worker_pool_executor_factory(executors, max_fanout=100 ) -> executor_factory.ExecutorFactory: """Create an executor backed by a worker pool. Args: executors: A list of `tff.framework.Executor` instances that forward work to workers in the worker pool. These can be any type of executors, but in most scenarios, they will be instances of `tff.framework.RemoteExecutor`. max_fanout: The maximum fanout at any point in the aggregation hierarchy. If `num_clients > max_fanout`, the constructed executor stack will consist of multiple levels of aggregators. The height of the stack will be on the order of `log(num_clients) / log(max_fanout)`. Returns: An instance of `executor_factory.ExecutorFactory` encapsulating the executor construction logic specified above. """ py_typecheck.check_type(executors, list) py_typecheck.check_type(max_fanout, int) if not executors: raise ValueError('The list executors cannot be empty.') if max_fanout < 2: raise ValueError('Max fanout must be greater than 1.') executors = [_complete_stack(e) for e in executors] def _stack_fn(cardinalities): del cardinalities # Unused return _aggregate_stacks(executors, max_fanout) return executor_factory.ExecutorFactoryImpl(executor_stack_fn=_stack_fn)
def test_concrete_class_instantiates_stack_fn(self): def _stack_fn(x): del x # Unused return eager_tf_executor.EagerTFExecutor() factory = executor_factory.ExecutorFactoryImpl(_stack_fn) self.assertIsInstance(factory, executor_factory.ExecutorFactoryImpl)
def worker_pool_executor_factory(executors, max_fanout=100 ) -> executor_factory.ExecutorFactory: """Create an executor backed by a worker pool. Args: executors: A list of `tff.framework.Executor` instances that forward work to workers in the worker pool. These can be any type of executors, but in most scenarios, they will be instances of `tff.framework.RemoteExecutor`. max_fanout: The maximum fanout at any point in the aggregation hierarchy. If `num_clients > max_fanout`, the constructed executor stack will consist of multiple levels of aggregators. The height of the stack will be on the order of `log(num_clients) / log(max_fanout)`. Returns: An instance of `executor_factory.ExecutorFactory` encapsulating the executor construction logic specified above. """ py_typecheck.check_type(executors, list) if not executors: raise ValueError('The list executors cannot be empty.') executors = [_wrap_executor_in_threading_stack(e) for e in executors] flat_stack_fn = lambda _: executors unplaced_ex_factory = UnplacedExecutorFactory(use_caching=True) composing_executor_factory = ComposingExecutorFactory( max_fanout=max_fanout, unplaced_ex_factory=unplaced_ex_factory, flat_stack_fn=flat_stack_fn, ) return executor_factory.ExecutorFactoryImpl( executor_stack_fn=composing_executor_factory.create_executor)
def test_cleanup_succeeds_without_init(self): def _stack_fn(x): del x # Unused return eager_tf_executor.EagerTFExecutor() factory = executor_factory.ExecutorFactoryImpl(_stack_fn) factory.clean_up_executors()
def test_executor_service_create_one_arg_computation_value_and_call(self): ex_factory = executor_factory.ExecutorFactoryImpl( lambda _: eager_tf_executor.EagerTFExecutor()) env = TestEnv(ex_factory) @computations.tf_computation(tf.int32) def comp(x): return tf.add(x, 1) value_proto, _ = executor_service_utils.serialize_value(comp) response = env.stub.CreateValue( executor_pb2.CreateValueRequest(value=value_proto)) self.assertIsInstance(response, executor_pb2.CreateValueResponse) comp_ref = response.value_ref value_proto, _ = executor_service_utils.serialize_value(10, tf.int32) response = env.stub.CreateValue( executor_pb2.CreateValueRequest(value=value_proto)) self.assertIsInstance(response, executor_pb2.CreateValueResponse) arg_ref = response.value_ref response = env.stub.CreateCall( executor_pb2.CreateCallRequest( function_ref=comp_ref, argument_ref=arg_ref)) self.assertIsInstance(response, executor_pb2.CreateCallResponse) value_id = str(response.value_ref.id) value = env.get_value(value_id) self.assertEqual(value, 11) del env
def _create_inferred_cardinality_factory( max_fanout, stack_func, clients_per_thread) -> executor_factory.ExecutorFactory: """Creates executor function with variable cardinality.""" def _create_variable_clients_executors(cardinalities): """Constructs executor stacks from `dict` argument.""" py_typecheck.check_type(cardinalities, dict) for k, v in cardinalities.items(): py_typecheck.check_type(k, placement_literals.PlacementLiteral) if k not in [ placement_literals.CLIENTS, placement_literals.SERVER ]: raise ValueError('Unsupported placement: {}.'.format(k)) if v <= 0: raise ValueError( 'Cardinality must be at ' 'least one; you have passed {} for placement {}.'.format( v, k)) return _create_full_stack( cardinalities.get(placement_literals.CLIENTS, 0), max_fanout, stack_func, clients_per_thread) return executor_factory.ExecutorFactoryImpl( executor_stack_fn=_create_variable_clients_executors)
def test_call_constructs_executor(self): def _stack_fn(x): del x # Unused return eager_tf_executor.EagerTFExecutor() factory = executor_factory.ExecutorFactoryImpl(_stack_fn) ex = factory.create_executor({}) self.assertIsInstance(ex, executor_base.Executor)
def test_executor_service_slowly_create_tensor_value(self): class SlowExecutorValue(executor_value_base.ExecutorValue): def __init__(self, v, t): self._v = v self._t = t @property def type_signature(self): return self._t async def compute(self): return self._v class SlowExecutor(executor_base.Executor): def __init__(self): self.status = 'idle' self.busy = threading.Event() self.done = threading.Event() async def create_value(self, value, type_spec=None): self.status = 'busy' self.busy.set() self.done.wait() self.status = 'done' return SlowExecutorValue(value, type_spec) async def create_call(self, comp, arg=None): raise NotImplementedError async def create_struct(self, elements): raise NotImplementedError async def create_selection(self, source, index=None, name=None): raise NotImplementedError def close(self): pass ex = SlowExecutor() ex_factory = executor_factory.ExecutorFactoryImpl(lambda _: ex) env = TestEnv(ex_factory) self.assertEqual(ex.status, 'idle') value_proto, _ = executor_service_utils.serialize_value(10, tf.int32) response = env.stub.CreateValue( executor_pb2.CreateValueRequest(value=value_proto)) ex.busy.wait() self.assertEqual(ex.status, 'busy') ex.done.set() value = env.get_value(response.value_ref.id) self.assertEqual(ex.status, 'done') self.assertEqual(value, 10)
def test_with_executor_factory(self): context_stack = context_stack_impl.context_stack executor_factory_impl = executor_factory.ExecutorFactoryImpl(lambda _: None) self.assertNotIsInstance(context_stack.current, execution_context.ExecutionContext) default_executor.set_default_executor(executor_factory_impl) self.assertIsInstance(context_stack.current, execution_context.ExecutionContext) self.assertIs(context_stack.current._executor_factory, executor_factory_impl)
def test_cleanup_calls_close(self): ex = eager_tf_executor.EagerTFExecutor() ex.close = mock.MagicMock() def _stack_fn(x): del x # Unused return ex factory = executor_factory.ExecutorFactoryImpl(_stack_fn) factory.create_executor({}) factory.clean_up_executors() ex.close.assert_called_once()
def test_runs_tf(test_obj, executor): """Tests `executor` can run a minimal TF computation.""" py_typecheck.check_type(executor, executor_base.Executor) @computations.tf_computation def comp(): return tf.math.add(5, 5) executor_factory_impl = executor_factory.ExecutorFactoryImpl( lambda _: executor) with _execution_context(executor_factory_impl): test_obj.assertEqual(comp(), 10)
def test_executor_service_create_tensor_value(self): ex_factory = executor_factory.ExecutorFactoryImpl( lambda _: eager_tf_executor.EagerTFExecutor()) env = TestEnv(ex_factory) value_proto, _ = executor_service_utils.serialize_value( tf.constant(10.0).numpy(), tf.float32) response = env.stub.CreateValue( executor_pb2.CreateValueRequest(value=value_proto)) self.assertIsInstance(response, executor_pb2.CreateValueResponse) value_id = str(response.value_ref.id) value = env.get_value(value_id) self.assertEqual(value, 10.0) del env
def test_server_context_shuts_down_uncaught_exception( self, mock_logging_info): ex = eager_tf_executor.EagerTFExecutor() ex_factory = executor_factory.ExecutorFactoryImpl(lambda _: ex) with self.assertRaises(TypeError): with server_utils.server_context( ex_factory, 1, portpicker.pick_unused_port()) as server: time.sleep(1) raise TypeError mock_logging_info.assert_called_once_with('Shutting down server.')
def test_server_context_shuts_down_under_keyboard_interrupt( self, mock_logging_info): ex = eager_tf_executor.EagerTFExecutor() ex_factory = executor_factory.ExecutorFactoryImpl(lambda _: ex) with server_utils.server_context( ex_factory, 1, portpicker.pick_unused_port()) as server: time.sleep(1) raise KeyboardInterrupt mock_logging_info.assert_has_calls([ mock.call('Server stopped by KeyboardInterrupt.'), mock.call('Shutting down server.') ])
def _create_explicit_cardinality_factory( num_clients, max_fanout, stack_func, clients_per_thread) -> executor_factory.ExecutorFactory: """Creates executor function with fixed cardinality.""" def _return_executor(cardinalities): n_requested_clients = cardinalities.get(placement_literals.CLIENTS) if n_requested_clients is not None and n_requested_clients != num_clients: raise ValueError( 'Expected to construct an executor with {} clients, ' 'but executor is hardcoded for {}'.format( n_requested_clients, num_clients)) return _create_full_stack(num_clients, max_fanout, stack_func, clients_per_thread) return executor_factory.ExecutorFactoryImpl( executor_stack_fn=_return_executor)
def test_construction_with_multiple_cardinalities_reuses_existing_stacks( self): ex = eager_tf_executor.EagerTFExecutor() ex.close = mock.MagicMock() num_times_invoked = 0 def _stack_fn(x): del x # Unused nonlocal num_times_invoked num_times_invoked += 1 return ex factory = executor_factory.ExecutorFactoryImpl(_stack_fn) for _ in range(2): factory.create_executor({}) factory.create_executor({placement_literals.SERVER: 1}) self.assertEqual(num_times_invoked, 2)
def test_basic_functionality(self): @computations.tf_computation(computation_types.SequenceType(tf.int32)) def comp(ds): return ds.take(5).reduce(np.int32(0), lambda x, y: x + y) set_default_executor.set_default_executor( executor_factory.ExecutorFactoryImpl( lambda _: eager_tf_executor.EagerTFExecutor())) ds = tf.data.Dataset.range(1).map(lambda x: tf.constant(5)).repeat() v = comp(ds) self.assertEqual(v, 25) set_default_executor.set_default_executor() self.assertIn('ExecutionContext', str(type(context_stack_impl.context_stack.current).__name__))
def test_context(rpc_mode='REQUEST_REPLY'): port = portpicker.pick_unused_port() server_pool = logging_pool.pool(max_workers=1) server = grpc.server(server_pool) server.add_insecure_port('[::]:{}'.format(port)) target_factory = executor_stacks.local_executor_factory(num_clients=3) tracers = [] def _tracer_fn(cardinalities): tracer = executor_test_utils.TracingExecutor( target_factory.create_executor(cardinalities)) tracers.append(tracer) return tracer service = executor_service.ExecutorService( executor_factory.ExecutorFactoryImpl(_tracer_fn)) executor_pb2_grpc.add_ExecutorServicer_to_server(service, server) server.start() channel = grpc.insecure_channel('localhost:{}'.format(port)) stub = executor_pb2_grpc.ExecutorStub(channel) serialized_cards = executor_service_utils.serialize_cardinalities( {placement_literals.CLIENTS: 3}) stub.SetCardinalities( executor_pb2.SetCardinalitiesRequest(cardinalities=serialized_cards)) remote_exec = remote_executor.RemoteExecutor(channel, rpc_mode) executor = reference_resolving_executor.ReferenceResolvingExecutor( remote_exec) try: yield collections.namedtuple('_', 'executor tracers')(executor, tracers) finally: executor.close() for tracer in tracers: tracer.close() try: channel.close() except AttributeError: pass # Public gRPC channel doesn't support close() finally: server.stop(None)
def test_executor_service_create_and_select_from_tuple(self): ex_factory = executor_factory.ExecutorFactoryImpl( lambda _: eager_tf_executor.EagerTFExecutor()) env = TestEnv(ex_factory) value_proto, _ = executor_service_utils.serialize_value(10, tf.int32) response = env.stub.CreateValue( executor_pb2.CreateValueRequest(value=value_proto)) self.assertIsInstance(response, executor_pb2.CreateValueResponse) ten_ref = response.value_ref self.assertEqual(env.get_value(ten_ref.id), 10) value_proto, _ = executor_service_utils.serialize_value(20, tf.int32) response = env.stub.CreateValue( executor_pb2.CreateValueRequest(value=value_proto)) self.assertIsInstance(response, executor_pb2.CreateValueResponse) twenty_ref = response.value_ref self.assertEqual(env.get_value(twenty_ref.id), 20) response = env.stub.CreateStruct( executor_pb2.CreateStructRequest(element=[ executor_pb2.CreateStructRequest.Element( name='a', value_ref=ten_ref), executor_pb2.CreateStructRequest.Element( name='b', value_ref=twenty_ref) ])) self.assertIsInstance(response, executor_pb2.CreateStructResponse) tuple_ref = response.value_ref self.assertEqual(str(env.get_value(tuple_ref.id)), '<a=10,b=20>') for arg_name, arg_val, result_val in [('name', 'a', 10), ('name', 'b', 20), ('index', 0, 10), ('index', 1, 20)]: response = env.stub.CreateSelection( executor_pb2.CreateSelectionRequest( source_ref=tuple_ref, **{arg_name: arg_val})) self.assertIsInstance(response, executor_pb2.CreateSelectionResponse) selection_ref = response.value_ref self.assertEqual(env.get_value(selection_ref.id), result_val) del env
def test_executor_service_value_unavailable_after_dispose(self): ex_factory = executor_factory.ExecutorFactoryImpl( lambda _: eager_tf_executor.EagerTFExecutor()) env = TestEnv(ex_factory) value_proto, _ = executor_service_utils.serialize_value( tf.constant(10.0).numpy(), tf.float32) # Create the value response = env.stub.CreateValue( executor_pb2.CreateValueRequest(value=value_proto)) self.assertIsInstance(response, executor_pb2.CreateValueResponse) value_id = str(response.value_ref.id) # Check that the value appears in the _values map env.get_value_future_directly(value_id) # Dispose of the value dispose_request = executor_pb2.DisposeRequest() dispose_request.value_ref.append(response.value_ref) response = env.stub.Dispose(dispose_request) self.assertIsInstance(response, executor_pb2.DisposeResponse) # Check that the value is gone from the _values map # get_value_future_directly is used here so that we can catch the # exception rather than having it occur on the GRPC thread. with self.assertRaises(KeyError): env.get_value_future_directly(value_id)
def create_test_executor_factory(): executor = eager_tf_executor.EagerTFExecutor() return executor_factory.ExecutorFactoryImpl(lambda _: executor)
def local_executor_factory( num_clients=None, max_fanout=100, num_client_executors=32, server_tf_device=None, client_tf_devices=tuple() ) -> executor_factory.ExecutorFactory: """Constructs an executor factory to execute computations locally. Note: The `tff.federated_secure_sum()` intrinsic is not implemented by this executor. Args: num_clients: The number of clients. If specified, the executor factory function returned by `local_executor_factory` will be configured to have exactly `num_clients` clients. If unspecified (`None`), then the function returned will attempt to infer cardinalities of all placements for which it is passed values. max_fanout: The maximum fanout at any point in the aggregation hierarchy. If `num_clients > max_fanout`, the constructed executor stack will consist of multiple levels of aggregators. The height of the stack will be on the order of `log(num_clients) / log(max_fanout)`. num_client_executors: The number of distinct client executors to run concurrently; executing more clients than this number results in multiple clients having their work pinned on a single executor in a synchronous fashion. server_tf_device: A `tf.config.LogicalDevice` to place server and other computation without explicit TFF placement. client_tf_devices: List/tuple of `tf.config.LogicalDevice` to place clients for simulation. Possibly accelerators returned by `tf.config.list_logical_devices()`. Returns: An instance of `executor_factory.ExecutorFactory` encapsulating the executor construction logic specified above. Raises: ValueError: If the number of clients is specified and not one or larger. """ if server_tf_device is not None: py_typecheck.check_type(server_tf_device, tf.config.LogicalDevice) py_typecheck.check_type(client_tf_devices, (tuple, list)) py_typecheck.check_type(max_fanout, int) py_typecheck.check_type(num_client_executors, int) if num_clients is not None: py_typecheck.check_type(num_clients, int) if max_fanout < 2: raise ValueError('Max fanout must be greater than 1.') unplaced_ex_factory = UnplacedExecutorFactory( use_caching=True, server_device=server_tf_device, client_devices=client_tf_devices) federating_executor_factory = FederatingExecutorFactory( num_client_executors=num_client_executors, unplaced_ex_factory=unplaced_ex_factory, num_clients=num_clients, use_sizing=False) def _factory_fn( cardinalities: executor_factory.CardinalitiesType ) -> executor_base.Executor: return _create_full_stack( cardinalities, max_fanout, stack_func=federating_executor_factory.create_executor, unplaced_ex_factory=unplaced_ex_factory) return executor_factory.ExecutorFactoryImpl(_factory_fn)
def thread_debugging_executor_factory( num_clients=None, clients_per_thread=1, ) -> executor_factory.ExecutorFactory: r"""Constructs a simplified execution stack to execute local computations. The constructed executors support a limited set of TFF's computations. In particular, the debug executor can only resolve references at the top level of the stack, and therefore assumes that all local computation is expressed in pure TensorFlow. The debug executor makes particular guarantees about the structure of the stacks it constructs which are intended to make them maximally easy to reason about. That is, the debug executor will essentially execute exactly the computation it is passed (in particular, this implies that there are no caching layers), and uses its declared inability to execute arbitrary TFF lambdas to reduce the complexity of the constructed stack. Every debugging executor will have a similar structure, the simplest structure that can execute the full expressivity of TFF while running each client in a dedicated thread: ReferenceResolvingExecutor | FederatingExecutor / ... \ ThreadDelegatingExecutor ThreadDelegatingExecutor | | EagerTFExecutor EagerTFExecutor This structure can be useful in understanding the concurrency pattern of TFF execution, and where the TFF runtime infers data dependencies. Args: num_clients: The number of clients. If specified, the executor factory function returned by `local_executor_factory` will be configured to have exactly `num_clients` clients. If unspecified (`None`), then the function returned will attempt to infer cardinalities of all placements for which it is passed values. clients_per_thread: Integer number of clients for each of TFF's threads to run in sequence. Increasing `clients_per_thread` therefore reduces the concurrency of the TFF runtime, which can be useful if client work is very lightweight or models are very large and multiple copies cannot fit in memory. Returns: An instance of `executor_factory.ExecutorFactory` encapsulating the executor construction logic specified above. Raises: ValueError: If the number of clients is specified and not one or larger. """ py_typecheck.check_type(clients_per_thread, int) if num_clients is not None: py_typecheck.check_type(num_clients, int) unplaced_ex_factory = UnplacedExecutorFactory( use_caching=False, can_resolve_references=False, ) federating_executor_factory = FederatingExecutorFactory( clients_per_thread=clients_per_thread, unplaced_ex_factory=unplaced_ex_factory, num_clients=num_clients, use_sizing=False) return executor_factory.ExecutorFactoryImpl( federating_executor_factory.create_executor)
def local_executor_factory( num_clients=None, max_fanout=100, clients_per_thread=1, server_tf_device=None, client_tf_devices=tuple() ) -> executor_factory.ExecutorFactory: """Constructs an executor factory to execute computations locally. Note: The `tff.federated_secure_sum()` intrinsic is not implemented by this executor. Args: num_clients: The number of clients. If specified, the executor factory function returned by `local_executor_factory` will be configured to have exactly `num_clients` clients. If unspecified (`None`), then the function returned will attempt to infer cardinalities of all placements for which it is passed values. max_fanout: The maximum fanout at any point in the aggregation hierarchy. If `num_clients > max_fanout`, the constructed executor stack will consist of multiple levels of aggregators. The height of the stack will be on the order of `log(num_clients) / log(max_fanout)`. clients_per_thread: Integer number of clients for each of TFF's threads to run in sequence. Increasing `clients_per_thread` therefore reduces the concurrency of the TFF runtime, which can be useful if client work is very lightweight or models are very large and multiple copies cannot fit in memory. server_tf_device: A `tf.config.LogicalDevice` to place server and other computation without explicit TFF placement. client_tf_devices: List/tuple of `tf.config.LogicalDevice` to place clients for simulation. Possibly accelerators returned by `tf.config.list_logical_devices()`. Returns: An instance of `executor_factory.ExecutorFactory` encapsulating the executor construction logic specified above. Raises: ValueError: If the number of clients is specified and not one or larger. """ if server_tf_device is not None: py_typecheck.check_type(server_tf_device, tf.config.LogicalDevice) py_typecheck.check_type(client_tf_devices, (tuple, list)) py_typecheck.check_type(max_fanout, int) py_typecheck.check_type(clients_per_thread, int) if num_clients is not None: py_typecheck.check_type(num_clients, int) if max_fanout < 2: raise ValueError('Max fanout must be greater than 1.') unplaced_ex_factory = UnplacedExecutorFactory( use_caching=True, server_device=server_tf_device, client_devices=client_tf_devices) federating_executor_factory = FederatingExecutorFactory( clients_per_thread=clients_per_thread, unplaced_ex_factory=unplaced_ex_factory, num_clients=num_clients, use_sizing=False) flat_stack_fn = create_minimal_length_flat_stack_fn( max_fanout, federating_executor_factory) full_stack_factory = ComposingExecutorFactory( max_fanout=max_fanout, unplaced_ex_factory=unplaced_ex_factory, flat_stack_fn=flat_stack_fn, ) return executor_factory.ExecutorFactoryImpl( full_stack_factory.create_executor)
def test_runs_tf(test_obj, executor): """Tests `executor` can run a minimal TF computation.""" py_typecheck.check_type(executor, executor_base.Executor) set_default_executor.set_default_executor( executor_factory.ExecutorFactoryImpl(lambda _: executor)) test_obj.assertEqual(_dummy_tf_computation(), 10)
def create_test_executor_factory(): executor = create_test_executor(num_clients=1) return executor_factory.ExecutorFactoryImpl(lambda _: executor)
def create_test_executor_factory(): executor = eager_tf_executor.EagerTFExecutor() executor = caching_executor.CachingExecutor(executor) executor = reference_resolving_executor.ReferenceResolvingExecutor( executor) return executor_factory.ExecutorFactoryImpl(lambda _: executor)
def remote_executor_factory( channels, rpc_mode='REQUEST_REPLY', thread_pool_executor=None, dispose_batch_size=20, max_fanout=100) -> executor_factory.ExecutorFactory: """Create an executor backed by remote workers. Args: channels: A list of `grpc.Channels` hosting services which can execute TFF work. rpc_mode: A string specifying the connection mode between the local host and `channels`. thread_pool_executor: Optional concurrent.futures.Executor used to wait for the reply to a streaming RPC message. Uses the default Executor if not specified. dispose_batch_size: The batch size for requests to dispose of remote worker values. Lower values will result in more requests to the remote worker, but will result in values being cleaned up sooner and therefore may result in lower memory usage on the remote worker. max_fanout: The maximum fanout at any point in the aggregation hierarchy. If `num_clients > max_fanout`, the constructed executor stack will consist of multiple levels of aggregators. The height of the stack will be on the order of `log(num_clients) / log(max_fanout)`. Returns: An instance of `executor_factory.ExecutorFactory` encapsulating the executor construction logic specified above. """ py_typecheck.check_type(channels, list) if not channels: raise ValueError('The list of channels cannot be empty.') remote_executors = [ remote_executor.RemoteExecutor(channel, rpc_mode, thread_pool_executor, dispose_batch_size) for channel in channels ] def _configure_remote_executor(ex, cardinalities, loop): """Configures `ex` to run the appropriate number of clients.""" loop.run_until_complete(ex.set_cardinalities(cardinalities)) return def _configure_remote_workers(cardinalities): loop = asyncio.new_event_loop() try: if not cardinalities.get(placement_literals.CLIENTS): for ex in remote_executors: _configure_remote_executor(ex, cardinalities, loop) return [ _wrap_executor_in_threading_stack(e) for e in remote_executors ] remaining_clients = cardinalities[placement_literals.CLIENTS] clients_per_most_executors = remaining_clients // len( remote_executors) for ex in remote_executors[:-1]: _configure_remote_executor( ex, {placement_literals.CLIENTS: clients_per_most_executors}, loop) remaining_clients -= clients_per_most_executors _configure_remote_executor( remote_executors[-1], {placement_literals.CLIENTS: remaining_clients}, loop) finally: loop.close() return [_wrap_executor_in_threading_stack(e) for e in remote_executors] flat_stack_fn = _configure_remote_workers unplaced_ex_factory = UnplacedExecutorFactory(use_caching=True) composing_executor_factory = ComposingExecutorFactory( max_fanout=max_fanout, unplaced_ex_factory=unplaced_ex_factory, flat_stack_fn=flat_stack_fn, ) return executor_factory.ExecutorFactoryImpl( executor_stack_fn=composing_executor_factory.create_executor)