def __init__(self,
                 channel,
                 rpc_mode='REQUEST_REPLY',
                 thread_pool_executor=None):
        """Creates a remote executor.

    Args:
      channel: An instance of `grpc.Channel` to use for communication with the
        remote executor service.
      rpc_mode: Optional mode of calling the remote executor. Must be either
        'REQUEST_REPLY' or 'STREAMING' (defaults to 'REQUEST_REPLY'). This
        option will be removed after the request-reply interface is deprecated.
      thread_pool_executor: Optional concurrent.futures.Executor used to wait
        for the reply to a streaming RPC message. Uses the default Executor if
        not specified.
    """
        py_typecheck.check_type(channel, grpc.Channel)
        py_typecheck.check_type(rpc_mode, str)
        if rpc_mode not in ['REQUEST_REPLY', 'STREAMING']:
            raise ValueError('Invalid rpc_mode: {}'.format(rpc_mode))

        self._stub = executor_pb2_grpc.ExecutorStub(channel)
        self._bidi_stream = None
        if rpc_mode == 'STREAMING':
            self._bidi_stream = _BidiStream(self._stub, thread_pool_executor)
Exemple #2
0
    def __init__(self, channel):
        """Creates a remote executor.

    Args:
      channel: An instance of `grpc.Channel` to use for communication with the
        remote executor service.
    """
        py_typecheck.check_type(channel, grpc.Channel)
        self._stub = executor_pb2_grpc.ExecutorStub(channel)
 def __init__(self, executor):
     port = portpicker.pick_unused_port()
     server_pool = logging_pool.pool(max_workers=1)
     self._server = grpc.server(server_pool)
     self._server.add_insecure_port('[::]:{}'.format(port))
     self._service = executor_service.ExecutorService(executor)
     executor_pb2_grpc.add_ExecutorServicer_to_server(
         self._service, self._server)
     self._server.start()
     self._channel = grpc.insecure_channel('localhost:{}'.format(port))
     self._stub = executor_pb2_grpc.ExecutorStub(self._channel)
Exemple #4
0
 def setUp(self):
     super(ExecutorServiceTest, self).setUp()
     port = portpicker.pick_unused_port()
     server_pool = logging_pool.pool(max_workers=1)
     self._server = grpc.server(server_pool)
     self._server.add_insecure_port('[::]:{}'.format(port))
     self._service = executor_service.ExecutorService(
         eager_executor.EagerExecutor())
     executor_pb2_grpc.add_ExecutorServicer_to_server(
         self._service, self._server)
     self._server.start()
     self._channel = grpc.insecure_channel('localhost:{}'.format(port))
     self._stub = executor_pb2_grpc.ExecutorStub(self._channel)
Exemple #5
0
    def __init__(self,
                 channel,
                 rpc_mode='REQUEST_REPLY',
                 thread_pool_executor=None,
                 dispose_batch_size=20):
        """Creates a remote executor.

    Args:
      channel: An instance of `grpc.Channel` to use for communication with the
        remote executor service.
      rpc_mode: Optional mode of calling the remote executor. Must be either
        'REQUEST_REPLY' or 'STREAMING' (defaults to 'REQUEST_REPLY'). This
        option will be removed after the request-reply interface is deprecated.
      thread_pool_executor: Optional concurrent.futures.Executor used to wait
        for the reply to a streaming RPC message. Uses the default Executor if
        not specified.
      dispose_batch_size: The batch size for requests to dispose of remote
        worker values. Lower values will result in more requests to the remote
        worker, but will result in values being cleaned up sooner and therefore
        may result in lower memory usage on the remote worker.
    """

        py_typecheck.check_type(channel, grpc.Channel)
        py_typecheck.check_type(rpc_mode, str)
        py_typecheck.check_type(dispose_batch_size, int)
        if rpc_mode not in ['REQUEST_REPLY', 'STREAMING']:
            raise ValueError('Invalid rpc_mode: {}'.format(rpc_mode))

        logging.debug('Creating new ExecutorStub with RPC_MODE=%s', rpc_mode)

        self._channel_status = False

        def _channel_status_callback(
                channel_connectivity: grpc.ChannelConnectivity):
            self._channel_status = channel_connectivity

        channel.subscribe(_channel_status_callback, try_to_connect=True)

        # We need to keep a reference to the channel around to prevent the Python
        # object from being GC'ed and the callback above from no-op'ing.
        self._channel = channel
        self._stub = executor_pb2_grpc.ExecutorStub(channel)
        self._bidi_stream = None
        self._dispose_batch_size = dispose_batch_size
        self._dispose_request = executor_pb2.DisposeRequest()
        if rpc_mode == 'STREAMING':
            logging.debug('Creating Bidi stream')
            self._bidi_stream = _BidiStream(self._stub, thread_pool_executor)
Exemple #6
0
    def __init__(self,
                 channel,
                 rpc_mode=None,
                 thread_pool_executor=None,
                 dispose_batch_size=20):
        """Creates a remote executor.

    Args:
      channel: An instance of `grpc.Channel` to use for communication with the
        remote executor service.
      rpc_mode: (Deprecated) string, one of 'REQUEST_REPLY' or 'STREAMING'.
        Unused, still here for backwards compatibility.
      thread_pool_executor: Optional concurrent.futures.Executor used to wait
        for the reply to a streaming RPC message. Uses the default Executor if
        not specified.
      dispose_batch_size: The batch size for requests to dispose of remote
        worker values. Lower values will result in more requests to the remote
        worker, but will result in values being cleaned up sooner and therefore
        may result in lower memory usage on the remote worker.
    """

        py_typecheck.check_type(channel, grpc.Channel)
        py_typecheck.check_type(dispose_batch_size, int)
        if rpc_mode is not None:
            warnings.warn(
                'The rpc_mode argument is deprecated and slated for '
                'removal. Please update your callsites to avoid specifying '
                'rpc_mode.')
        del rpc_mode

        logging.debug('Creating new ExecutorStub')

        self._channel_status = False

        def _channel_status_callback(
                channel_connectivity: grpc.ChannelConnectivity):
            self._channel_status = channel_connectivity

        channel.subscribe(_channel_status_callback, try_to_connect=True)

        # We need to keep a reference to the channel around to prevent the Python
        # object from being GC'ed and the callback above from no-op'ing.
        self._channel = channel
        self._stub = executor_pb2_grpc.ExecutorStub(channel)
        self._dispose_batch_size = dispose_batch_size
        self._dispose_request = executor_pb2.DisposeRequest()
  def __init__(self,
               ex_factory: executor_factory.ExecutorFactory,
               num_clients: int = 0):
    port = portpicker.pick_unused_port()
    self._server_pool = logging_pool.pool(max_workers=1)
    self._server = grpc.server(self._server_pool)
    self._server.add_insecure_port('[::]:{}'.format(port))
    self._service = executor_service.ExecutorService(ex_factory=ex_factory)
    executor_pb2_grpc.add_ExecutorServicer_to_server(self._service,
                                                     self._server)
    self._server.start()
    self._channel = grpc.insecure_channel('localhost:{}'.format(port))
    self._stub = executor_pb2_grpc.ExecutorStub(self._channel)

    serialized_cards = executor_serialization.serialize_cardinalities(
        {placement_literals.CLIENTS: num_clients})
    self._stub.SetCardinalities(
        executor_pb2.SetCardinalitiesRequest(cardinalities=serialized_cards))
Exemple #8
0
def test_context(rpc_mode='REQUEST_REPLY'):
    port = portpicker.pick_unused_port()
    server_pool = logging_pool.pool(max_workers=1)
    server = grpc.server(server_pool)
    server.add_insecure_port('[::]:{}'.format(port))
    target_factory = executor_stacks.local_executor_factory(num_clients=3)
    tracers = []

    def _tracer_fn(cardinalities):
        tracer = executor_test_utils.TracingExecutor(
            target_factory.create_executor(cardinalities))
        tracers.append(tracer)
        return tracer

    service = executor_service.ExecutorService(
        executor_stacks.ResourceManagingExecutorFactory(_tracer_fn))
    executor_pb2_grpc.add_ExecutorServicer_to_server(service, server)
    server.start()

    channel = grpc.insecure_channel('localhost:{}'.format(port))
    stub = executor_pb2_grpc.ExecutorStub(channel)
    serialized_cards = executor_service_utils.serialize_cardinalities(
        {placement_literals.CLIENTS: 3})
    stub.SetCardinalities(
        executor_pb2.SetCardinalitiesRequest(cardinalities=serialized_cards))

    remote_exec = remote_executor.RemoteExecutor(channel, rpc_mode)
    executor = reference_resolving_executor.ReferenceResolvingExecutor(
        remote_exec)
    try:
        yield collections.namedtuple('_', 'executor tracers')(executor,
                                                              tracers)
    finally:
        executor.close()
        for tracer in tracers:
            tracer.close()
        try:
            channel.close()
        except AttributeError:
            pass  # Public gRPC channel doesn't support close()
        finally:
            server.stop(None)
Exemple #9
0
    def __init__(self,
                 channel,
                 thread_pool_executor=None,
                 dispose_batch_size=20):
        """Creates a remote executor.

    Args:
      channel: An instance of `grpc.Channel` to use for communication with the
        remote executor service.
      thread_pool_executor: Optional concurrent.futures.Executor used to wait
        for the reply to a streaming RPC message. Uses the default Executor if
        not specified.
      dispose_batch_size: The batch size for requests to dispose of remote
        worker values. Lower values will result in more requests to the remote
        worker, but will result in values being cleaned up sooner and therefore
        may result in lower memory usage on the remote worker.
    """

        py_typecheck.check_type(channel, grpc.Channel)
        py_typecheck.check_type(dispose_batch_size, int)

        logging.debug('Creating new ExecutorStub')

        self._channel_status = False

        def _channel_status_callback(
                channel_connectivity: grpc.ChannelConnectivity):
            self._channel_status = channel_connectivity

        channel.subscribe(_channel_status_callback, try_to_connect=True)

        # We need to keep a reference to the channel around to prevent the Python
        # object from being GC'ed and the callback above from no-op'ing.
        self._channel = channel
        self._stub = executor_pb2_grpc.ExecutorStub(channel)
        self._dispose_batch_size = dispose_batch_size
        self._dispose_request = executor_pb2.DisposeRequest()