def thrift(self):
from boarpc.protocol.tgunicorn.thrift.thriftapp import ThriftApplication
from boarpc.protocol.thrift import BoarpcService
from boarpc.protocol.dispatcherHandler import DispatcherHandler
from common.utils.netUtils import NetUtils
from common.config.conf import config
import multiprocessing
handler = DispatcherHandler()
app = BoarpcService.Processor(handler)
bind = "%s:%s" % (NetUtils.getIp(), str(NetUtils.getFreePort()))
options = {
'worker_class':
'thrift_gevent',
'thrift_protocol_factory':
'thrift.protocol.TCompactProtocol:TCompactProtocolAcceleratedFactory',
'thrift_transport_factory':
'thrift.transport.TTransport:TBufferedTransportFactory',
'service_register_cls':
'boarpc.protocol.tgunicorn.zk_process:ZkRigster',
'bind':
bind,
'workers':
int(config.getConf('rpc', 'processes',
multiprocessing.cpu_count())),
'on_exit':
GunicornProtocol.on_exit,
'daemon':
config.getConf('profiles', 'active', 'dev') == 'prod',
'reuse-port':
True,
}
return ThriftApplication(app, options)
class Constants:
SERVICE_KEY = ''
APP_MODULE_NAME = config.getConf('app', 'app_name', '')
APP_REGISTER_ROOT_PATH = '/boa_rpc/' + config.getConf(
'app', 'app_name', '')
APP_REGISTER_PROVIDER_PATH = '/boa_rpc/%s/%s' % (config.getConf(
'app', 'app_name', ''), 'providers')
APP_REGISTER_CONSUMER_PATH = '/boa_rpc/%s/%s' % (config.getConf(
'app', 'app_name', ''), 'consumers')
APP_REGISTER_PROVIDER_CHILD_PATH = APP_REGISTER_PROVIDER_PATH + '/{0}'
def startWorker(self):
processes = int(
config.getConf('rpc', 'processes', multiprocessing.cpu_count()))
urlSet = set()
portSet = set()
# get all endpoints
endpointSet = self.getAllEndpoints()
host = NetUtils.getIp()
for i in range(0, processes):
# get a port
port = NetUtils.getFreePort()
if port in portSet:
flag = True
while flag:
time.time(5)
port = NetUtils.getFreePort()
if port not in portSet:
break
portSet.add(port)
url = URL(self.PROTOCOL, host, port)
url.setEndpints(endpoints=endpointSet)
p = Process(target=self.worker, args=(url, ))
urlSet.add(url)
url.setProcess(p)
p.start()
return urlSet
def on_exit(server):
from common.config.conf import config
from boarpc.url import URL
from boarpc.registry.registryService import RegistryFactory
host, port = server.app.cfg.bind[0].split(":")
registry = RegistryFactory.createRegistry()
registry.unRegister(URL(config.getConf('rpc', 'protocol', defult='thrift'), host, port))
def __init__(self, protocol, host, port):
self.protocol = protocol
self.host = host
self.port = port
self.path = '/doa_rpc/%s' % (config.getConf('app', 'app_name', ''))
self.process = None
self.endpoints = set()
class ZkRigster(object):
PROTOCOL = config.getConf('rpc', 'protocol', defult='thrift')
def __init__(self, bind):
host, port = bind.split(":")
self.url = URL(self.PROTOCOL, host, port)
def run(self):
# register to zk
logger.info("-------register to zk--------")
registry = RegistryFactory.createRegistry()
urlSet = set()
urlSet.add(self.url)
registry.urlSet = urlSet
registry.registerAll()
logger.info(self.url)
# monitor children process
while True:
time.sleep(10)
providers = registry.getAllProvides()
if self.url.getHostPort() not in providers:
# Re-register if lost in the registry
registry.register(self.url)
def register_instances(self):
p = Process(target=self.run, args=())
p.daemon = True
p.start()
class BoaClient:
# defualt protocol: thrift
PROTOCOL = config.getConf('rpc', 'protocol', defult='thrift')
def invoke(self, endpointUrl, params):
invoker = InvokerFactory.createInvoker(self.PROTOCOL)
return invoker.invoke(endpointUrl, params)
class BoaRpc:
# defualt protocol: thrift
PROTOCOL = config.getConf('rpc','protocol',defult='thrift')
def __init__(self,coroutines=1000,url=None):
self.endpoint_path = os.path.join(os.path.dirname(os.path.realpath(__file__)).replace('boarpc',''),'endpoint')
sys.path.insert(0, self.endpoint_path)
self.coroutines = coroutines
self.url = url
def run(self):
from boarpc.protocol.protocol import ProtocolFactory
protocol = ProtocolFactory.createProtocol(self.PROTOCOL,self.url)
protocol.run()
def __init__(self):
connectionRetry = KazooRetry(max_tries=-1)
timeout = 60
self.zkClient = KazooClient(hosts=config.getConf('rpc', 'zk_host'), timeout=timeout, connection_retry=connectionRetry)
self.zkClient.start(timeout=60)
self.urlSet = set()
signal.signal(signal.SIGINT, self.close)
@self.zkClient.add_listener
def stateListener(state):
if state == KazooState.CONNECTED:
if ZookeeperRegistry.isRegister:
logger.info('ZooKeeper listener reconnected')
self.zkClient.handler.spawn(self.registerAll)
ZookeeperRegistry.isRegister = True
class ZkRigster(object):
PROTOCOL = config.getConf('rpc', 'protocol', defult='thrift')
def __init__(self,bind):
host,port = bind.split(":")
self.url = URL(self.PROTOCOL, host, port)
def run(self):
# register to zk
logger.info("-------register to zk--------")
registry = RegistryFactory.createRegistry()
urlSet = set()
urlSet.add(self.url)
registry.urlSet = urlSet
registry.registerAll()
logger.info(self.url)
registry.PROVIDER_MD5 = registry.getProvideMD5()
# monitor children process
while True:
time.sleep(10)
md5 = registry.getProvideMD5()
if registry.PROVIDER_MD5 != md5:
registry.PROVIDER_MD5 = md5
registry.register(self.url)
# for url in urlSet:
# p = url.process
# logger.info('Run worker process (%s) status: %s' % (p.pid, p.is_alive()))
# if not p.is_alive():
# registry.unRegister(url)
# p.terminate()
# self.reStartWork(url)
# time.sleep(5)
# registry.register(url)
# elif flag:
# registry.register(url)
def register_instances(self):
p = Process(target=self.run, args=())
p.daemon = True
p.start()
class MultiprocessBoa:
PROTOCOL = config.getConf('rpc', 'protocol', defult='thrift')
def __init__(self):
self.endpoint_path = os.path.join(
os.path.dirname(os.path.realpath(__file__)).replace('boarpc', ''),
'endpoint')
sys.path.insert(0, self.endpoint_path)
def getAllEndpoints(self):
'''
get all endpoints
:return:
'''
for filename in glob.glob(os.path.join(self.endpoint_path, r'*.py')):
moduleName = filename[filename.rindex(os.sep) +
1:filename.rindex('.')]
if '__init__' != moduleName:
__import__(moduleName)
return Endpoint.INTERFACES
def worker(self, url):
a = BoaRpc(url=url)
a.run()
def startWorker(self):
processes = int(
config.getConf('rpc', 'processes', multiprocessing.cpu_count()))
urlSet = set()
portSet = set()
# get all endpoints
endpointSet = self.getAllEndpoints()
host = NetUtils.getIp()
for i in range(0, processes):
# get a port
port = NetUtils.getFreePort()
if port in portSet:
flag = True
while flag:
time.time(5)
port = NetUtils.getFreePort()
if port not in portSet:
break
portSet.add(port)
url = URL(self.PROTOCOL, host, port)
url.setEndpints(endpoints=endpointSet)
p = Process(target=self.worker, args=(url, ))
urlSet.add(url)
url.setProcess(p)
p.start()
return urlSet
def reStartWork(self, url):
url.setProcess(None)
p = Process(target=self.worker, args=(url, ))
url.setProcess(p)
p.start()
def run(self):
print os.getpid()
#start worker
logger.info("-------start worker--------")
urlSet = self.startWorker()
#register to zk
logger.info("-------register to zk--------")
registry = RegistryFactory.createRegistry()
registry.urlSet = urlSet
registry.registerAll()
for url in urlSet:
logger.info(url)
#monitor children process
logger.info("-------monitor children process--------")
while True:
time.sleep(10)
providers = registry.getAllProvides()
for url in urlSet:
p = url.process
logger.info('Run worker process (%s) status: %s' %
(p.pid, p.is_alive()))
if not p.is_alive():
registry.unRegister(url)
p.terminate()
self.reStartWork(url)
time.sleep(5)
registry.register(url)
elif url.getHostPort() not in providers:
#Re-register if lost in the registry
registry.register(url)
while True:
time.sleep(10)
providers = registry.getAllProvides()
for url in urlSet:
p = url.process
logger.info('Run worker process (%s) status: %s' %
(p.pid, p.is_alive()))
if not p.is_alive():
registry.unRegister(url)
p.terminate()
self.reStartWork(url)
time.sleep(5)
registry.register(url)
elif url.getHostPort() not in providers:
#Re-register if lost in the registry
registry.register(url)
import platform
from common.config.conf import config
PROTOCOL = config.getConf('rpc', 'protocol', defult='thrift')
def start():
os_name = platform.system()
if 'Windows' == os_name:
MultiprocessBoa().run()
else:
from boarpc.protocol.tgunicorn.gunicorn_factory import GunicornFactory
GunicornFactory.createGunicorn(PROTOCOL).run()