def start_thrift_server(self):
        processor = Echoer.Processor(EchoHandler())
        transport = TSocket.TServerSocket(port=12346)
        protocol_factory = TCompactProtocol.TCompactProtocolFactory()

        self.thrift_server = TNonblockingServer(processor, transport, protocol_factory, protocol_factory)

        ready = threading.Condition()
        ready.value = False

        def callback():
            ready.acquire()
            ready.value = True
            ready.notify()
            ready.release()

        def serve(instance):
            try:
                instance.thrift_server.serve(ready_callback=callback)
                instance.thrift_server.close()
            except RuntimeError:
                instance.thrift_server_error = sys.exc_info()

        self.thrift_server_thread = threading.Thread(target=serve, args=[self])
        self.thrift_server_thread.start()

        # Wait for the thrift server to be ready
        ready.acquire()
        while ready.value is not True:
            ready.wait()
        ready.release()
Example #2
0
    def start_server(self):
        """
        Registers the host and root scheduler processors and
        starts the thrift server.
        """
        mux_processor = TMultiplexedProcessor()

        if self._hostHandler:
            self._register_processer("Host", mux_processor)

        if self._leafSchedulerHandler:
            self._register_processer("Scheduler", mux_processor)

        if self._rootSchedulerHandler:
            self._register_processer(
                "RootScheduler", mux_processor)

        if self._enable_ping:
            self._register_processer(
                "AgentControl", mux_processor)

        transport = TSocket.TServerSocket(self._address, self._port)

        protocol_factory = TCompactProtocol.TCompactProtocolFactory()

        self._server = TNonblockingServer(
            mux_processor, transport, protocol_factory, protocol_factory)

        self._thread = threading.Thread(target=self._server.serve)
        self._thread.daemon = True
        self._thread.start()
        self._server_started = True

        self._logger.info(
            "[ThriftServer] Thrift server started. Address: {0} Port: {1}"
            .format(self._address, self._port))
Example #3
0
class ThriftServer(object):
    """
    ThriftServer
    Register both RootScheduler and Host processors
    """
    def __init__(self, address, port,
                 hostHandler=None,
                 leafSchedulerHandler=None,
                 rootSchedulerHandler=None,
                 enable_ping=True):
        """
        :param address [str]: agent thrift server hostname
        :param port [int]: agent thrift server tcp port
        :param hostHandler [HostHandler]: HostHandler instance
        :param rootSchedulerHandler [RootSchedulerHandler]:
            RootSchedulerHandler instance
        :param enable_ping [bool]: Register AgentControl processor
        """
        self._address = address
        self._port = port
        self._hostHandler = hostHandler
        self._leafSchedulerHandler = leafSchedulerHandler
        self._rootSchedulerHandler = rootSchedulerHandler
        self._enable_ping = enable_ping

        if self._hostHandler is None and self._rootSchedulerHandler is None:
            raise Exception("Handlers not set.")

        self._server = None
        self._thread = None

        # setup logger
        self._logger = logging.getLogger(__name__)

    def _register_processer(self, service_name, mux_processor):
        """
        Registers processor
        """

        if service_name == "Host":
            _processor = Host.Processor(self._hostHandler)

        if service_name == "RootScheduler":
            _processor = RootScheduler.Processor(self._rootSchedulerHandler)

        if service_name == "AgentControl":
            _processor = AgentControl.Processor(AgentControlHandler())

        if service_name == "Scheduler":
            _processor = Scheduler.Processor(self._leafSchedulerHandler)

        mux_processor.registerProcessor(
            service_name,
            _processor,
            1,
            0)  # unbounded queue

    def start_server(self):
        """
        Registers the host and root scheduler processors and
        starts the thrift server.
        """
        mux_processor = TMultiplexedProcessor()

        if self._hostHandler:
            self._register_processer("Host", mux_processor)

        if self._leafSchedulerHandler:
            self._register_processer("Scheduler", mux_processor)

        if self._rootSchedulerHandler:
            self._register_processer(
                "RootScheduler", mux_processor)

        if self._enable_ping:
            self._register_processer(
                "AgentControl", mux_processor)

        transport = TSocket.TServerSocket(self._address, self._port)

        protocol_factory = TCompactProtocol.TCompactProtocolFactory()

        self._server = TNonblockingServer(
            mux_processor, transport, protocol_factory, protocol_factory)

        self._thread = threading.Thread(target=self._server.serve)
        self._thread.daemon = True
        self._thread.start()
        self._server_started = True

        self._logger.info(
            "[ThriftServer] Thrift server started. Address: {0} Port: {1}"
            .format(self._address, self._port))

    def stop_server(self):
        """
        Stop the thrift server
        """
        self._server.stop()

        while self._thread.isAlive():
            time.sleep(1)

        # release socket resource
        self._server.close()
class TestUnitThriftServer(unittest.TestCase):
    def start_thrift_server(self):
        processor = Echoer.Processor(EchoHandler())
        transport = TSocket.TServerSocket(port=12346)
        protocol_factory = TCompactProtocol.TCompactProtocolFactory()

        self.thrift_server = TNonblockingServer(processor, transport, protocol_factory, protocol_factory)

        ready = threading.Condition()
        ready.value = False

        def callback():
            ready.acquire()
            ready.value = True
            ready.notify()
            ready.release()

        def serve(instance):
            try:
                instance.thrift_server.serve(ready_callback=callback)
                instance.thrift_server.close()
            except RuntimeError:
                instance.thrift_server_error = sys.exc_info()

        self.thrift_server_thread = threading.Thread(target=serve, args=[self])
        self.thrift_server_thread.start()

        # Wait for the thrift server to be ready
        ready.acquire()
        while ready.value is not True:
            ready.wait()
        ready.release()

    def stop_thrift_server(self):
        if self.thrift_server is not None:
            self.thrift_server.stop()
            self.thrift_server_thread.join()

            # Reset fields _after_ serve thread has returned,
            # because this thread references these fields.
            self.thrift_server = None
            self.thrift_server_thread = None

    def create_thrift_client(self):
        s = TSocket.TSocket("127.0.0.1", 12346)
        transport = TTransport.TFramedTransport(s)
        protocol = TCompactProtocol.TCompactProtocol(transport)
        client = Echoer.Client(protocol)

        transport.open()
        self.addCleanup(transport.close)

        return client

    def test_accept_after_emfile(self):
        # visor-python has an older version of unittest
        if not hasattr(self, "addCleanup"):
            raise nose.plugins.skip.SkipTest("requires unittest 2.7+")

        # on darwin, client side of the socket is closed when the initial
        # accept() fails (with EMFILE). this causes accept_then_close() to
        # block on accept() such that we can't wake_up/stop the server thread
        if sys.platform == "darwin":
            raise nose.plugins.skip.SkipTest("hangs on darwin")

        self.start_thrift_server()
        self.addCleanup(self.stop_thrift_server)

        # Reduce soft open file descriptors limits
        limits = resource.getrlimit(resource.RLIMIT_NOFILE)
        resource.setrlimit(resource.RLIMIT_NOFILE, (32, limits[1]))
        self.addCleanup(resource.setrlimit, resource.RLIMIT_NOFILE, limits)

        # Create sockets until EMFILE
        sockets = []
        while True:
            try:
                sockets.append(socket.socket())
            except EnvironmentError as e:
                # Only care about EMFILE
                if e.errno != errno.EMFILE:
                    raise
                break

        # Release one socket to use for the Thrift client
        sockets.pop().close()

        try:
            client = self.create_thrift_client()
            client.echo("foo")
        except EnvironmentError as e:
            # Happens when write(2) fails
            assert_that(e.errno, equal_to(errno.ECONNRESET))
        except TTransportException as e:
            # Happens when read(2) returns 0
            assert_that(e.type, equal_to(TTransportException.END_OF_FILE))
        else:
            # The client must experience an error
            assert False