def start_server(self): """ Registers the host and root scheduler processors and starts the thrift server. """ mux_processor = TMultiplexedProcessor() if self._hostHandler: self._register_processer("Host", mux_processor) if self._leafSchedulerHandler: self._register_processer("Scheduler", mux_processor) if self._rootSchedulerHandler: self._register_processer("RootScheduler", mux_processor) if self._enable_ping: self._register_processer("AgentControl", mux_processor) transport = TSocket.TServerSocket(self._address, self._port) protocol_factory = TCompactProtocol.TCompactProtocolFactory() self._server = TNonblockingServer(mux_processor, transport, protocol_factory, protocol_factory) self._thread = threading.Thread(target=self._server.serve) self._thread.daemon = True self._thread.start() self._server_started = True self._logger.info( "[ThriftServer] Thrift server started. Address: {0} Port: {1}". format(self._address, self._port))
def start_thrift_server(self): processor = Echoer.Processor(EchoHandler()) transport = TSocket.TServerSocket(port=12346) protocol_factory = TCompactProtocol.TCompactProtocolFactory() self.thrift_server = TNonblockingServer(processor, transport, protocol_factory, protocol_factory) ready = threading.Condition() ready.value = False def callback(): ready.acquire() ready.value = True ready.notify() ready.release() def serve(instance): try: instance.thrift_server.serve(ready_callback=callback) instance.thrift_server.close() except RuntimeError: instance.thrift_server_error = sys.exc_info() self.thrift_server_thread = threading.Thread(target=serve, args=[self]) self.thrift_server_thread.start() # Wait for the thrift server to be ready ready.acquire() while ready.value is not True: ready.wait() ready.release()
def _initialize_thrift_service(self): """ Initialize the thrift server. """ mux_processor = TMultiplexedProcessor() for plugin in thrift_services(): self._logger.info("Load thrift services %s (num_threads: %d)", plugin.name, plugin.num_threads) handler = plugin.handler processor = plugin.service.Processor(handler) mux_processor.registerProcessor(plugin.name, processor, plugin.num_threads, plugin.max_entries) transport = TSocket.TServerSocket(port=self._config.host_port) protocol_factory = TCompactProtocol.TCompactProtocolFactory() server = TNonblockingServer(mux_processor, transport, protocol_factory, protocol_factory) self._server = server
def start_server(self): """ Registers the host and root scheduler processors and starts the thrift server. """ mux_processor = TMultiplexedProcessor() if self._hostHandler: self._register_processer("Host", mux_processor) if self._leafSchedulerHandler: self._register_processer("Scheduler", mux_processor) if self._rootSchedulerHandler: self._register_processer( "RootScheduler", mux_processor) if self._enable_ping: self._register_processer( "AgentControl", mux_processor) transport = TSocket.TServerSocket(self._address, self._port) protocol_factory = TCompactProtocol.TCompactProtocolFactory() self._server = TNonblockingServer( mux_processor, transport, protocol_factory, protocol_factory) self._thread = threading.Thread(target=self._server.serve) self._thread.daemon = True self._thread.start() self._server_started = True self._logger.info( "[ThriftServer] Thrift server started. Address: {0} Port: {1}" .format(self._address, self._port))
class TestUnitThriftServer(unittest.TestCase): def start_thrift_server(self): processor = Echoer.Processor(EchoHandler()) transport = TSocket.TServerSocket(port=12346) protocol_factory = TCompactProtocol.TCompactProtocolFactory() self.thrift_server = TNonblockingServer(processor, transport, protocol_factory, protocol_factory) ready = threading.Condition() ready.value = False def callback(): ready.acquire() ready.value = True ready.notify() ready.release() def serve(instance): try: instance.thrift_server.serve(ready_callback=callback) instance.thrift_server.close() except RuntimeError: instance.thrift_server_error = sys.exc_info() self.thrift_server_thread = threading.Thread(target=serve, args=[self]) self.thrift_server_thread.start() # Wait for the thrift server to be ready ready.acquire() while ready.value is not True: ready.wait() ready.release() def stop_thrift_server(self): if self.thrift_server is not None: self.thrift_server.stop() self.thrift_server_thread.join() # Reset fields _after_ serve thread has returned, # because this thread references these fields. self.thrift_server = None self.thrift_server_thread = None def create_thrift_client(self): s = TSocket.TSocket("127.0.0.1", 12346) transport = TTransport.TFramedTransport(s) protocol = TCompactProtocol.TCompactProtocol(transport) client = Echoer.Client(protocol) transport.open() self.addCleanup(transport.close) return client def test_accept_after_emfile(self): # visor-python has an older version of unittest if not hasattr(self, "addCleanup"): raise nose.plugins.skip.SkipTest("requires unittest 2.7+") # on darwin, client side of the socket is closed when the initial # accept() fails (with EMFILE). this causes accept_then_close() to # block on accept() such that we can't wake_up/stop the server thread if sys.platform == "darwin": raise nose.plugins.skip.SkipTest("hangs on darwin") self.start_thrift_server() self.addCleanup(self.stop_thrift_server) # Reduce soft open file descriptors limits limits = resource.getrlimit(resource.RLIMIT_NOFILE) resource.setrlimit(resource.RLIMIT_NOFILE, (32, limits[1])) self.addCleanup(resource.setrlimit, resource.RLIMIT_NOFILE, limits) # Create sockets until EMFILE sockets = [] while True: try: sockets.append(socket.socket()) except EnvironmentError as e: # Only care about EMFILE if e.errno != errno.EMFILE: raise break # Release one socket to use for the Thrift client sockets.pop().close() try: client = self.create_thrift_client() client.echo("foo") except EnvironmentError as e: # Happens when write(2) fails assert_that(e.errno, equal_to(errno.ECONNRESET)) except TTransportException as e: # Happens when read(2) returns 0 assert_that(e.type, equal_to(TTransportException.END_OF_FILE)) else: # The client must experience an error assert False
class ThriftServer(object): """ ThriftServer Register both RootScheduler and Host processors """ def __init__(self, address, port, hostHandler=None, leafSchedulerHandler=None, rootSchedulerHandler=None, enable_ping=True): """ :param address [str]: agent thrift server hostname :param port [int]: agent thrift server tcp port :param hostHandler [HostHandler]: HostHandler instance :param rootSchedulerHandler [RootSchedulerHandler]: RootSchedulerHandler instance :param enable_ping [bool]: Register AgentControl processor """ self._address = address self._port = port self._hostHandler = hostHandler self._leafSchedulerHandler = leafSchedulerHandler self._rootSchedulerHandler = rootSchedulerHandler self._enable_ping = enable_ping if self._hostHandler is None and self._rootSchedulerHandler is None: raise Exception("Handlers not set.") self._server = None self._thread = None # setup logger self._logger = logging.getLogger(__name__) def _register_processer(self, service_name, mux_processor): """ Registers processor """ if service_name == "Host": _processor = Host.Processor(self._hostHandler) if service_name == "RootScheduler": _processor = RootScheduler.Processor(self._rootSchedulerHandler) if service_name == "AgentControl": _processor = AgentControl.Processor(AgentControlHandler()) if service_name == "Scheduler": _processor = Scheduler.Processor(self._leafSchedulerHandler) mux_processor.registerProcessor( service_name, _processor, 1, 0) # unbounded queue def start_server(self): """ Registers the host and root scheduler processors and starts the thrift server. """ mux_processor = TMultiplexedProcessor() if self._hostHandler: self._register_processer("Host", mux_processor) if self._leafSchedulerHandler: self._register_processer("Scheduler", mux_processor) if self._rootSchedulerHandler: self._register_processer( "RootScheduler", mux_processor) if self._enable_ping: self._register_processer( "AgentControl", mux_processor) transport = TSocket.TServerSocket(self._address, self._port) protocol_factory = TCompactProtocol.TCompactProtocolFactory() self._server = TNonblockingServer( mux_processor, transport, protocol_factory, protocol_factory) self._thread = threading.Thread(target=self._server.serve) self._thread.daemon = True self._thread.start() self._server_started = True self._logger.info( "[ThriftServer] Thrift server started. Address: {0} Port: {1}" .format(self._address, self._port)) def stop_server(self): """ Stop the thrift server """ self._server.stop() while self._thread.isAlive(): time.sleep(1) # release socket resource self._server.close()
class ThriftServer(object): """ ThriftServer Register both RootScheduler and Host processors """ def __init__(self, address, port, hostHandler=None, leafSchedulerHandler=None, rootSchedulerHandler=None, enable_ping=True): """ :param address [str]: agent thrift server hostname :param port [int]: agent thrift server tcp port :param hostHandler [HostHandler]: HostHandler instance :param rootSchedulerHandler [RootSchedulerHandler]: RootSchedulerHandler instance :param enable_ping [bool]: Register AgentControl processor """ self._address = address self._port = port self._hostHandler = hostHandler self._leafSchedulerHandler = leafSchedulerHandler self._rootSchedulerHandler = rootSchedulerHandler self._enable_ping = enable_ping if self._hostHandler is None and self._rootSchedulerHandler is None: raise Exception("Handlers not set.") self._server = None self._thread = None # setup logger self._logger = logging.getLogger(__name__) def _register_processer(self, service_name, mux_processor): """ Registers processor """ if service_name == "Host": _processor = Host.Processor(self._hostHandler) if service_name == "RootScheduler": _processor = RootScheduler.Processor(self._rootSchedulerHandler) if service_name == "AgentControl": _processor = AgentControl.Processor(AgentControlHandler()) if service_name == "Scheduler": _processor = Scheduler.Processor(self._leafSchedulerHandler) mux_processor.registerProcessor(service_name, _processor, 1, 0) # unbounded queue def start_server(self): """ Registers the host and root scheduler processors and starts the thrift server. """ mux_processor = TMultiplexedProcessor() if self._hostHandler: self._register_processer("Host", mux_processor) if self._leafSchedulerHandler: self._register_processer("Scheduler", mux_processor) if self._rootSchedulerHandler: self._register_processer("RootScheduler", mux_processor) if self._enable_ping: self._register_processer("AgentControl", mux_processor) transport = TSocket.TServerSocket(self._address, self._port) protocol_factory = TCompactProtocol.TCompactProtocolFactory() self._server = TNonblockingServer(mux_processor, transport, protocol_factory, protocol_factory) self._thread = threading.Thread(target=self._server.serve) self._thread.daemon = True self._thread.start() self._server_started = True self._logger.info( "[ThriftServer] Thrift server started. Address: {0} Port: {1}". format(self._address, self._port)) def stop_server(self): """ Stop the thrift server """ self._server.stop() while self._thread.isAlive(): time.sleep(1) # release socket resource self._server.close()