def testSplit(self):
"""Test FramedTransport and BinaryProtocolAccelerated
Tests that TBinaryProtocolAccelerated and TFramedTransport
play nicely together when a read spans a frame"""
protocol_factory = TBinaryProtocol.TBinaryProtocolAcceleratedFactory()
bigstring = "".join(chr(byte) for byte in range(ord("a"), ord("z")+1))
databuf = TTransport.TMemoryBuffer()
prot = protocol_factory.getProtocol(databuf)
prot.writeI32(42)
prot.writeString(bigstring)
prot.writeI16(24)
data = databuf.getvalue()
cutpoint = len(data) // 2
parts = [ data[:cutpoint], data[cutpoint:] ]
framed_buffer = TTransport.TMemoryBuffer()
framed_writer = TTransport.TFramedTransport(framed_buffer)
for part in parts:
framed_writer.write(part)
framed_writer.flush()
self.assertEquals(len(framed_buffer.getvalue()), len(data) + 8)
# Recreate framed_buffer so we can read from it.
framed_buffer = TTransport.TMemoryBuffer(framed_buffer.getvalue())
framed_reader = TTransport.TFramedTransport(framed_buffer)
prot = protocol_factory.getProtocol(framed_reader)
self.assertEqual(prot.readI32(), 42)
self.assertEqual(prot.readString(), bigstring)
self.assertEqual(prot.readI16(), 24)
def _test_read_write(self, u, j):
protocol_factory = TBinaryProtocol.TBinaryProtocolAcceleratedFactory()
databuf = TTransport.TMemoryBuffer()
prot = protocol_factory.getProtocol(databuf)
u.write(prot)
ndatabuf = TTransport.TMemoryBuffer(databuf.getvalue())
prot = protocol_factory.getProtocol(ndatabuf)
v = u.__class__()
v.read(prot)
self.assertEquals(v, j)
def setUp(self):
self.event = Event()
self.server_port = 21212
handler = _ServiceHandler(self.event)
prot = TBinaryProtocol.TBinaryProtocolAcceleratedFactory()
self.server = server.get_server(RndServiceApi,
handler,
self.server_port,
protocol=prot)
self.t_server = Process(target=self.server.serve)
self.t_server.daemon = True
self.t_server.start()
time.sleep(.1)
def init_thrift_server(self, socket, processor):
"""
Creates a thrift server that listens in the given socket and
uses the given processor
"""
try:
os.unlink(socket)
except OSError as oerr:
if oerr.errno != errno.ENOENT:
raise
transport = TSocket.TServerSocket(unix_socket=socket)
tfactory = TTransport.TBufferedTransportFactory()
pfactory = TBinaryProtocol.TBinaryProtocolAcceleratedFactory()
listener_server = TSimpleServer(self.name, processor, transport,
tfactory, pfactory)
return listener_server
def test_request_transport_integration():
handler = ReadOnlySchedulerHandler()
processor = ReadOnlyScheduler.Processor(handler)
pfactory = TBinaryProtocol.TBinaryProtocolAcceleratedFactory()
server = THttpServer.THttpServer(processor, ('localhost', 0), pfactory)
server_thread = Thread(target=server.serve)
server_thread.start()
_, server_port = server.httpd.socket.getsockname()
try:
transport = TRequestsTransport('http://localhost:%d' % server_port)
protocol = TBinaryProtocol.TBinaryProtocolAccelerated(transport)
client = ReadOnlyScheduler.Client(protocol)
response = client.getRoleSummary()
finally:
server.httpd.shutdown()
assert response is not None
assert response.responseCode == ResponseCode.OK
assert response.serverInfo.clusterName == 'west'
transport.close()
def test_requests_transport_session_reuse():
handler = ReadOnlySchedulerHandler()
processor = ReadOnlyScheduler.Processor(handler)
pfactory = TBinaryProtocol.TBinaryProtocolAcceleratedFactory()
server = THttpServer.THttpServer(processor, ('localhost', 0), pfactory)
server_thread = Thread(target=server.serve)
server_thread.start()
_, server_port = server.httpd.socket.getsockname()
try:
transport = TRequestsTransport('http://localhost:%d' % server_port)
protocol = TBinaryProtocol.TBinaryProtocolAccelerated(transport)
client = ReadOnlyScheduler.Client(protocol)
client.getRoleSummary()
old_session = transport._session
client.getRoleSummary()
finally:
server.httpd.shutdown()
assert old_session == transport._session
transport.close()
def open_thrift_server(processor, handler, port, server_type, num_threads=None):
'''
실제 Thrift Server 를 생성한다.
@type processor: Class (Thrift Interface)
@param processor: 자동으로 생성된 Thrift Service Class
@type handler: Class
@param handler: Thrift Interface 에 연결되어 실제 수행될 함수들이 있는 Class
@type port: int
@param port: Thrift server 를 bind 할 포트
@type server_type: str
@param server_type: in ["TThreadedServer", "TForkingServer", "TThreadPoolServer"]
@type num_threads: int
@param num_threads: TThreadPoolServer 사용시 띄울 thread 의 갯수
@rtype: TServer, handler (Object)
@return: 생성된 서버 인스턴스, 서버에 연결되어 실제 함수를 처리하는 인스턴스
'''
handler_instance = handler()
processor_ = processor.Processor(handler_instance)
transport = TSocket.TServerSocket(port)
tfactory = TTransport.TBufferedTransportFactory()
# tfactory = TTransport.TFramedTransportFactory()
pfactory = TBinaryProtocol.TBinaryProtocolAcceleratedFactory()
# 서버를 어떻게 띄울 것인가?
if server_type == "TThreadedServer": # Thread per Connection
server = TServer.TThreadedServer(processor_, transport, tfactory, pfactory)
elif server_type == "TForkingServer": # Process per Connection
server = TServer.TForkingServer(processor_, transport, tfactory, pfactory)
elif server_type == "TThreadPoolServer": # Preloaded Thread with Pool
server = TServer.TThreadPoolServer(processor_, transport, tfactory, pfactory)
server.setNumThreads(num_threads)
else:
# No default
raise Exception("Unknown Server Type")
return server, handler_instance
def run(self):
self.logger.info('Multi_p_t service starts to run.')
self.run_log_receiver_thread()
handler = SomeHandler(self.cfg, self.cache_dict)
processor = SomeServer.Processor(handler)
transport = TSocket.TServerSocket(port=self.port)
tfactory = TTransport.TBufferedTransportFactory()
pfactory = TBinaryProtocol.TBinaryProtocolAcceleratedFactory()
workerNum = self.cfg.getint('default', 'server.workers')
self.server = TProcessPoolServer.TProcessPoolServer(
processor, transport, tfactory, pfactory)
# 设置在worker进程启动时的callback，在worker进程重置logging config
self.server.setPostForkCallback(self.resetWorkerLogConfig)
self.server.setNumWorkers(workerNum)
try:
self.logger.info('Start multi service')
self.server.serve()
except Exception, why:
self.logger.fatal('Serve failed, %s', why)
self.server.stop()
sys.exit(1)
class AcceleratedBinaryTest(AbstractTest):
protocol_factory = TBinaryProtocol.TBinaryProtocolAcceleratedFactory()
import numpy as np
from thrift import TSerialization
from thrift.protocol import TBinaryProtocol
import phyre.interface.scene.ttypes as scene_if
import phyre.interface.task.ttypes as task_if
from phyre import creator
from phyre import simulator_bindings
import phyre.simulation
DEFAULT_MAX_STEPS = simulator_bindings.DEFAULT_MAX_STEPS
STEPS_FOR_SOLUTION = simulator_bindings.STEPS_FOR_SOLUTION
DEFAULT_STRIDE = simulator_bindings.FPS
OBJECT_FEATURE_SIZE = simulator_bindings.OBJECT_FEATURE_SIZE
PIXELS_IN_METER = simulator_bindings.PIXELS_IN_METER
FACTORY = TBinaryProtocol.TBinaryProtocolAcceleratedFactory()
def serialize(obj):
return TSerialization.serialize(obj, protocol_factory=FACTORY)
def deserialize(obj, pickle):
return TSerialization.deserialize(obj, pickle, protocol_factory=FACTORY)
def build_user_input(points=None, rectangulars=None, balls=None):
points, rectangulars, balls = _prepare_user_input(points, rectangulars,
balls)
user_input = scene_if.UserInput(
flattened_point_list=points.flatten().tolist(), balls=[], polygons=[])
def protocol(self, trans):
return TBinaryProtocol.TBinaryProtocolAcceleratedFactory(
fallback=False).getProtocol(trans)
class AcceleratedBinaryTest(AbstractTest, unittest.TestCase):
protocol_factory = TBinaryProtocol.TBinaryProtocolAcceleratedFactory()
print("send email message " + email + " " + message + "\n")
obj = MIMEText(message, "plain", "utf-8")
obj["From"] = sender
obj["To"] = email
obj["Subject"] = Header("慕课网邮件", "utf-8")
try:
smtpObj = smtplib.SMTP("smtp.163.com")
smtpObj.login(sender, authCode)
smtpObj.sendmail(sender, [email], obj.as_string())
except smtplib.SMTPException as e:
print("send email failed with " + e)
return False
return True
if __name__ == '__main__':
handler = MessageServiceHandler()
processor = MessageService.Processor(handler)
print("main start")
transport = TSocket.TServerSocket("127.0.0.1", "9090")
tfactory = TTransport.TFramedTransportFactory()
pfactory = TBinaryProtocol.TBinaryProtocolAcceleratedFactory()
server = TServer.TSimpleServer(processor, transport, tfactory, pfactory)
print("Python thrift server started ... ")
server.serve()
print("Python thrift server exit ... ")
def __init__(self, processor, handler, config):
self._logger = logging.getLogger(__name__)
self._section_name = utils.get_module(__name__)
self._server = None
self._service_name = config.get(self._section_name,
"service",
required=True)
self._port = config.getint(self._section_name, "port", required=True)
self._zk_connect_str = config.get(self._section_name,
"zk_connect_str",
required=True)
self._auth_user = config.get(self._section_name,
"auth_user",
required=True)
self._auth_password = config.get(self._section_name,
"auth_password",
required=True)
self._monitor = config.getboolean(self._section_name,
"monitor",
default=settings.SERVICE_MONITOR)
self._c_module_serialize = config.getboolean(
self._section_name,
"c_module_serialize",
default=settings.USE_C_MODULE_SERIALIZE)
self._server_name = config.get(self._section_name, "name")
self._owner = config.get(self._section_name, "owner")
hosts = "%s/%s" % (self._zk_connect_str,
settings.DEFAULT_ZK_NAMESPACE_ROOT)
self._server_path = "%s/%s" % (self._service_name,
settings.DEFAULT_ZK_NAMESPACE_SERVERS)
self._statistic_path = "%s/%s/%s" % (
self._service_name, settings.DEFAULT_ZK_NAMESPACE_STATISTICS,
settings.DEFAULT_ZK_NAMESPACE_SERVERS)
# create zk acl
self._acls = []
self._acls.append(
make_digest_acl(self._auth_user, self._auth_password, all=True))
self._acls.append(make_acl("world", "anyone", read=True))
# create zk_client
self._zkclient = HARpcZKClientManager.make(hosts, config, "server")
self._zkclient.add_auth(
"digest", "%s:%s" % (self._auth_user, self._auth_password))
# create zkpath
if not self._zkclient.exists(self._service_name):
self._zkclient.create(self._service_name, makepath=True)
if not self._zkclient.exists(self._server_path):
self._zkclient.create(self._server_path, acl=self._acls)
if not self._zkclient.exists(self._statistic_path):
self._zkclient.create(self._statistic_path, makepath=True)
self.transport = TSocket.TServerSocket(port=self._port)
self.tfactory = TTransport.TBufferedTransportFactory()
if self._monitor:
self._collector = StatisticsCollector(self._zkclient,
config,
is_server=True)
self._processor = self._statistics_wrapper(processor)(handler)
else:
self._processor = processor(handler)
if self._c_module_serialize:
self.pfactory = TBinaryProtocol.TBinaryProtocolAcceleratedFactory()
else:
self.pfactory = TBinaryProtocol.TBinaryProtocolFactory()
def testBinaryProtocolAcceleratedEof(self):
"""Test that TBinaryProtocolAccelerated throws a TTransportException
when it reaches the end of the stream"""
self.eofTestHelper(TBinaryProtocol.TBinaryProtocolAcceleratedFactory())
self.eofTestHelperStress(
TBinaryProtocol.TBinaryProtocolAcceleratedFactory())
class TestForwardCompatibilityBinaryAccelerated(
TestForwardCompatibilityAbstract, unittest.TestCase):
protocol_factory = TBinaryProtocol.TBinaryProtocolAcceleratedFactory()
def testBinaryProtocolAcceleratedBinaryEof(self):
"""Test that TBinaryProtocolAccelerated throws an EOFError when it reaches the end of the stream"""
self.eofTestHelper(TBinaryProtocol.TBinaryProtocolAcceleratedFactory(fallback=False))
self.eofTestHelperStress(TBinaryProtocol.TBinaryProtocolAcceleratedFactory(fallback=False))
def protocol(self, trans):
return TBinaryProtocol.TBinaryProtocolAcceleratedFactory().getProtocol(
trans)
class CirrusServer(object):
DEFAULT_THRIFT_PROTOCOL_FACTORY = TBinaryProtocol.TBinaryProtocolAcceleratedFactory(
)
DEFAULT_THRIFT_LISTEN_QUEUE_SIZE = 128
DEFAULT_WORKER_PROCESS_NUMBER = 4
DEFAULT_UNPROD_WORKER_PROCESS_NUMBER = 1
DEFAULT_HARAKIRI = 5
DEFAULT_EVENT_QUEUE_SIZE = 100
DEFAULT_THRIFT_RECV_TIMEOUT = 50000
def __init__(self,
thrift_module,
handler,
port=0,
protocol_factory=DEFAULT_THRIFT_PROTOCOL_FACTORY,
thrift_listen_queue_size=DEFAULT_THRIFT_LISTEN_QUEUE_SIZE,
worker_process_number=DEFAULT_WORKER_PROCESS_NUMBER,
harikiri=DEFAULT_HARAKIRI,
tag=None,
weight=100,
event_queue_size=DEFAULT_EVENT_QUEUE_SIZE,
thrift_recv_timeout=DEFAULT_THRIFT_RECV_TIMEOUT):
# 父级启动文件路径
boot_script_path = os.path.abspath((inspect.stack()[1])[1])
# 根目录
root_dir = os.path.dirname(boot_script_path)
service_key = CommonUtil.get_service_key(thrift_module)
handler_class = type(handler)
# 处理类装饰器
handler_decorator_class = type(handler_class.__name__,
(handler_class, HandlerDecorator), {})
handler = handler_decorator_class()
# thrift processor
processor = thrift_module.Processor(handler)
port = int(port)
if port == 0:
port = CommonUtil.get_random_port()
tag = CommonUtil.get_service_tag(tag)
weight = CommonUtil.get_service_weight(weight)
hostname = CommonUtil.get_hostname()
self._instance_config_data = InstanceConfigData(tag=tag,
weight=weight,
hostname=hostname)
transport = ThriftServerSocket(queue=thrift_listen_queue_size,
recv_timeout=thrift_recv_timeout,
port=port)
tfactory = TTransport.TFramedTransportFactory()
pfactory = protocol_factory
if not CommonUtil.is_prod():
worker_process_number = self.DEFAULT_UNPROD_WORKER_PROCESS_NUMBER
server = EpollServer(processor,
transport,
tfactory,
pfactory,
harakiri=harikiri,
event_queue_size=event_queue_size,
worker_process_number=worker_process_number,
instance_config_data=self._instance_config_data,
port=port)
self._service_key = service_key
self._port = port
self._server = server
self._tag = tag
self._handler = handler
self._worker_process = None
self._local_ip = CommonUtil.get_local_ip()
instance_path = '%s:%s' % (self._local_ip, self._port)
self._zk_publisher = ZkPublisher(self._service_key, instance_path)
def start(self):
try:
logger.info('Master process id: %d', os.getpid())
CommonUtil.set_proctitle('master')
# 启动工作进程
worker_process = Process(target=self._start)
worker_process.start()
logger.info('Worker process id: %d, port: %d', worker_process.pid,
self._port)
self._worker_process = worker_process
# kill命令不加参数终止进程
signal.signal(signal.SIGTERM, self._signal_exit)
# ctrl + C 终止进程
signal.signal(signal.SIGINT, self._signal_exit)
# 段错误,try catch难以捕获此异常
signal.signal(signal.SIGSEGV, self._signal_exit)
# zookeeper注册服务
self._register_server()
# 工作进程异常退出
worker_process.join()
logger.error('Worker process id: %d unexpected exit',
worker_process.pid)
except Exception as e:
# 主进程异常退出
logger.error('Master process id: %d error: %s', os.getpid(),
e.message)
self._stop(exit_code=1)
def _start(self):
# SIG_IGN忽略子进程状态信息,子进程会被自动回收,不会产生僵尸进程
signal.signal(signal.SIGINT, signal.SIG_IGN)
CommonUtil.set_proctitle('worker')
self._server.serve()
def _signal_exit(self, signum, frame):
logger.info('Master process id: %d receive signal %d', os.getpid(),
signum)
if signum == signal.SIGINT:
self._stop(graceful=False)
else:
self._stop(graceful=True)
def _stop(self, exit_code=0, graceful=True):
logger.info('Master process id: %d stop worker process: %d',
os.getpid(), self._worker_process.pid)
self._unregister_server()
if graceful:
time.sleep(CommonUtil.get_sec_for_server_teardown())
self._server.stop()
self._worker_process.terminate()
self._worker_process.join()
sys.exit(exit_code)
def _unregister_server(self):
self._zk_publisher.stop()
def _register_server(self):
self._zk_publisher.register(self._instance_config_data)
def protocol_factory(self):
"""Specify which protocol should be used."""
return TBinaryProtocol.TBinaryProtocolAcceleratedFactory()
