def _initialize_thrift_service(self):
"""
Initialize agents' thrif servers.
"""
for host_handler in self._hosts_handlers:
pseudo_config = host_handler.pseudo_config
mux_processor = TMultiplexedProcessor()
processor = Host.Processor(host_handler)
mux_processor.registerProcessor(
"Host",
processor,
pseudo_config.host_service_threads,
0)
transport = TSocket.TServerSocket(
pseudo_config.hostname,
pseudo_config.host_port)
protocol_factory = TCompactProtocol.TCompactProtocolFactory()
server = TNonblockingServer(
mux_processor, transport, protocol_factory, protocol_factory)
self._servers.append(server)
def echo_server():
mp = TMultiplexedProcessor()
mp.registerProcessor("echo", Echoer.Processor(EchoHandler()))
transport = TSocket.TServerSocket(port=PORT)
tf = TTransport.TFramedTransportFactory()
pf = TCompactProtocol.TCompactProtocolFactory()
return TServer.TThreadedServer(mp, transport, tf, pf, daemon=True)
def echo_server():
mp = TMultiplexedProcessor()
mp.registerProcessor("echo", Echoer.Processor(EchoHandler()))
transport = TSocket.TServerSocket(port=PORT)
tf = TTransport.TFramedTransportFactory()
pf = TCompactProtocol.TCompactProtocolFactory()
return TServer.TThreadedServer(mp, transport, tf, pf, daemon=True)
def start_server(self):
"""
Registers the host and root scheduler processors and
starts the thrift server.
"""
mux_processor = TMultiplexedProcessor()
if self._hostHandler:
self._register_processer("Host", mux_processor)
if self._leafSchedulerHandler:
self._register_processer("Scheduler", mux_processor)
if self._rootSchedulerHandler:
self._register_processer("RootScheduler", mux_processor)
if self._enable_ping:
self._register_processer("AgentControl", mux_processor)
transport = TSocket.TServerSocket(self._address, self._port)
protocol_factory = TCompactProtocol.TCompactProtocolFactory()
self._server = TNonblockingServer(mux_processor, transport,
protocol_factory, protocol_factory)
self._thread = threading.Thread(target=self._server.serve)
self._thread.daemon = True
self._thread.start()
self._server_started = True
self._logger.info(
"[ThriftServer] Thrift server started. Address: {0} Port: {1}".
format(self._address, self._port))
def test_shutdown(self):
m_processor = TMultiplexedProcessor()
processor = MagicMock()
num_workers = 2
services = ["Fake_1", "Fake_2"]
for name in services:
m_processor.registerProcessor(name, processor, num_workers)
for name in services:
self.assertEqual(len(m_processor.services[name].workers), 2)
for worker in m_processor.services[name].workers:
self.assertTrue(worker.is_alive())
m_processor.shutdown()
self.assertTrue(m_processor.shutting_down)
for name in services:
for worker in m_processor.services[name].workers:
self.assertFalse(worker.is_alive())
def _initialize_thrift_service(self):
""" Initialize the thrift server. """
mux_processor = TMultiplexedProcessor()
for plugin in thrift_services():
self._logger.info("Load thrift services %s (num_threads: %d)",
plugin.name, plugin.num_threads)
handler = plugin.handler
processor = plugin.service.Processor(handler)
mux_processor.registerProcessor(plugin.name, processor,
plugin.num_threads,
plugin.max_entries)
transport = TSocket.TServerSocket(port=self._config.host_port)
protocol_factory = TCompactProtocol.TCompactProtocolFactory()
server = TNonblockingServer(mux_processor, transport, protocol_factory,
protocol_factory)
self._server = server
def _initialize_thrift_service(self):
""" Initialize the thrift server. """
mux_processor = TMultiplexedProcessor()
for plugin in thrift_services():
self._logger.info("Load thrift services %s (num_threads: %d)",
plugin.name, plugin.num_threads)
handler = plugin.handler
processor = plugin.service.Processor(handler)
mux_processor.registerProcessor(plugin.name, processor,
plugin.num_threads,
plugin.max_entries)
transport = TSocket.TServerSocket(port=self._config.host_port)
protocol_factory = TCompactProtocol.TCompactProtocolFactory()
server = TNonblockingServer(
mux_processor, transport, protocol_factory, protocol_factory)
self._server = server
def _initialize_thrift_service(self):
"""
Initialize agents' thrif servers.
"""
for host_handler in self._hosts_handlers:
pseudo_config = host_handler.pseudo_config
mux_processor = TMultiplexedProcessor()
processor = Host.Processor(host_handler)
mux_processor.registerProcessor("Host", processor,
pseudo_config.host_service_threads,
0)
transport = TSocket.TServerSocket(pseudo_config.hostname,
pseudo_config.host_port)
protocol_factory = TCompactProtocol.TCompactProtocolFactory()
server = TNonblockingServer(mux_processor, transport,
protocol_factory, protocol_factory)
self._servers.append(server)
def test_registerProcessor(self, worker, queue):
"""
Test to check the registration of a thrift service processor.
Validates that the right number of threads are created and that all
threads created for the same processor share the same synchrnoized
Queue.
"""
test_worker = MagicMock()
worker.return_value = test_worker
test_queue = MagicMock()
queue.return_value = test_queue
m_processor = TMultiplexedProcessor()
processor = MagicMock()
num_workers = 2
service_name = "Fake"
m_processor.registerProcessor(service_name, processor, num_workers)
queue.assert_called_once_with()
self._check_worker_calls(service_name, num_workers, worker, test_queue)
# Register a second processor and verify that we have a new queue
# created, and is passed to the number of workers specified.
num_workers = 3
queue.reset_mock()
worker.reset_mock()
test_queue_1 = MagicMock()
queue.return_value = test_queue_1
m_processor = TMultiplexedProcessor()
processor = MagicMock()
m_processor.registerProcessor(service_name, processor, num_workers)
queue.assert_called_once_with()
self._check_worker_calls(service_name, num_workers, worker,
test_queue_1)
self.assertEqual(m_processor.services[service_name].processor,
processor)
self.assertEqual(m_processor.services[service_name].name, service_name)
self.assertEqual(m_processor.services[service_name].queue,
test_queue_1)
def test_shutdown(self):
m_processor = TMultiplexedProcessor()
processor = MagicMock()
num_workers = 2
services = ["Fake_1", "Fake_2"]
for name in services:
m_processor.registerProcessor(name, processor, num_workers)
for name in services:
self.assertEqual(len(m_processor.services[name].workers), 2)
for worker in m_processor.services[name].workers:
self.assertTrue(worker.is_alive())
m_processor.shutdown()
self.assertTrue(m_processor.shutting_down)
for name in services:
for worker in m_processor.services[name].workers:
self.assertFalse(worker.is_alive())
def test_registerProcessor(self, worker, queue):
"""
Test to check the registration of a thrift service processor.
Validates that the right number of threads are created and that all
threads created for the same processor share the same synchrnoized
Queue.
"""
test_worker = MagicMock()
worker.return_value = test_worker
test_queue = MagicMock()
queue.return_value = test_queue
m_processor = TMultiplexedProcessor()
processor = MagicMock()
num_workers = 2
service_name = "Fake"
m_processor.registerProcessor(service_name, processor, num_workers)
queue.assert_called_once_with()
self._check_worker_calls(service_name, num_workers, worker, test_queue)
# Register a second processor and verify that we have a new queue
# created, and is passed to the number of workers specified.
num_workers = 3
queue.reset_mock()
worker.reset_mock()
test_queue_1 = MagicMock()
queue.return_value = test_queue_1
m_processor = TMultiplexedProcessor()
processor = MagicMock()
m_processor.registerProcessor(service_name, processor, num_workers)
queue.assert_called_once_with()
self._check_worker_calls(
service_name, num_workers, worker, test_queue_1)
self.assertEqual(m_processor.services[service_name].processor,
processor)
self.assertEqual(m_processor.services[service_name].name, service_name)
self.assertEqual(m_processor.services[service_name].queue,
test_queue_1)
def test_process_queued(self, cur_time):
"""
Test the basic process_queued functionality.
+ve cases
1. Enqueues the request in the right queue for the service.
2. Checks that the queue is not full before enqueuing the request.
-ve cases
1. Fails if the processor is being shutdown.
2. Fails if the message type is not a call or one way message.
3. Fails if there is no service type in the message specified.
4. Fails if the service is not registered.
5. Fails if the queue is full.
"""
# Fake the current time
cur_time.return_value = 5
# Start by doing a lot of magic.
iprot = MagicMock()
oprot = MagicMock()
otrans = MagicMock()
otrans.getvalue.return_value = 0
callback = MagicMock()
mock_queue = MagicMock()
mock_queue.qsize.return_value = 5
processor = MagicMock()
service_mock = ServiceProcessor("Fake", processor, mock_queue,
MagicMock(), 10)
m_processor = TMultiplexedProcessor()
m_processor.services["Fake"] = service_mock
iprot.readMessageBegin.return_value = \
("Fake:test", TMessageType.CALL, 3)
standardMessage = ("test", TMessageType.CALL, 3)
m_iprot = StoredMessageProtocol(iprot, standardMessage)
m_processor.process_queued(iprot, oprot, otrans, callback)
# Invoke a custom matcher so we compare equivalence and not identity
# for the StandardMessage
matcher = Matcher(compare_standard_msg, m_iprot)
# assert that queue size and put were called
mock_queue.assert_has_calls([call.qsize()])
mock_queue.put.assert_called_with(
[processor, matcher, oprot, otrans, callback, 5])
# Negative tests
# Test shutting down.
callback.reset_mock()
m_processor = TMultiplexedProcessor()
m_processor.shutting_down = True
m_processor.process_queued(iprot, oprot, otrans, callback)
callback.assert_called_once_with(False, '')
# Test invalid_message
callback.reset_mock()
iprot.readMessageBegin.return_value = \
("Faketest", TMessageType.CALL, 3)
m_processor = TMultiplexedProcessor()
m_processor.process_queued(iprot, oprot, otrans, callback)
callback.assert_called_once_with(False, '')
# Test unknown service
callback.reset_mock()
iprot.readMessageBegin.return_value = \
("Fake1:test", TMessageType.CALL, 3)
m_processor = TMultiplexedProcessor()
m_processor.process_queued(iprot, oprot, otrans, callback)
callback.assert_called_once_with(False, '')
# Test full queue
callback.reset_mock()
iprot.readMessageBegin.return_value = \
("Fake:test", TMessageType.CALL, 3)
service_mock = ServiceProcessor("Fake", processor, mock_queue,
MagicMock(), 10)
m_processor = TMultiplexedProcessor()
m_processor.services["Fake"] = service_mock
mock_queue.qsize.return_value = 10
m_processor.process_queued(iprot, oprot, otrans, callback)
callback.assert_called_once_with(False, '')
def test_process_queued(self, cur_time):
"""
Test the basic process_queued functionality.
+ve cases
1. Enqueues the request in the right queue for the service.
2. Checks that the queue is not full before enqueuing the request.
-ve cases
1. Fails if the processor is being shutdown.
2. Fails if the message type is not a call or one way message.
3. Fails if there is no service type in the message specified.
4. Fails if the service is not registered.
5. Fails if the queue is full.
"""
# Fake the current time
cur_time.return_value = 5
# Start by doing a lot of magic.
iprot = MagicMock()
oprot = MagicMock()
otrans = MagicMock()
otrans.getvalue.return_value = 0
callback = MagicMock()
mock_queue = MagicMock()
mock_queue.qsize.return_value = 5
processor = MagicMock()
service_mock = ServiceProcessor("Fake", processor, mock_queue,
MagicMock(), 10)
m_processor = TMultiplexedProcessor()
m_processor.services["Fake"] = service_mock
iprot.readMessageBegin.return_value = \
("Fake:test", TMessageType.CALL, 3)
standardMessage = ("test", TMessageType.CALL, 3)
m_iprot = StoredMessageProtocol(iprot, standardMessage)
m_processor.process_queued(iprot, oprot, otrans, callback)
# Invoke a custom matcher so we compare equivalence and not identity
# for the StandardMessage
matcher = Matcher(compare_standard_msg, m_iprot)
# assert that queue size and put were called
mock_queue.assert_has_calls([call.qsize()])
mock_queue.put.assert_called_with([processor, matcher, oprot, otrans,
callback, 5])
# Negative tests
# Test shutting down.
callback.reset_mock()
m_processor = TMultiplexedProcessor()
m_processor.shutting_down = True
m_processor.process_queued(iprot, oprot, otrans, callback)
callback.assert_called_once_with(False, '')
# Test invalid_message
callback.reset_mock()
iprot.readMessageBegin.return_value = \
("Faketest", TMessageType.CALL, 3)
m_processor = TMultiplexedProcessor()
m_processor.process_queued(iprot, oprot, otrans, callback)
callback.assert_called_once_with(False, '')
# Test unknown service
callback.reset_mock()
iprot.readMessageBegin.return_value = \
("Fake1:test", TMessageType.CALL, 3)
m_processor = TMultiplexedProcessor()
m_processor.process_queued(iprot, oprot, otrans, callback)
callback.assert_called_once_with(False, '')
# Test full queue
callback.reset_mock()
iprot.readMessageBegin.return_value = \
("Fake:test", TMessageType.CALL, 3)
service_mock = ServiceProcessor("Fake", processor, mock_queue,
MagicMock(), 10)
m_processor = TMultiplexedProcessor()
m_processor.services["Fake"] = service_mock
mock_queue.qsize.return_value = 10
m_processor.process_queued(iprot, oprot, otrans, callback)
callback.assert_called_once_with(False, '')