def server(request):
p1 = TProcessor(mux.ThingOneService, DispatcherOne())
p2 = TProcessor(mux.ThingTwoService, DispatcherTwo())
mux_proc = TMultiplexingProcessor()
mux_proc.register_processor(p1)
mux_proc.register_processor(p2)
_server = TThreadedServer(mux_proc,
TServerSocket(unix_socket=sock_path),
iprot_factory=TBinaryProtocolFactory(),
itrans_factory=TBufferedTransportFactory())
ps = multiprocessing.Process(target=_server.serve)
ps.start()
time.sleep(0.1)
def fin():
if ps.is_alive():
ps.terminate()
try:
os.remove(sock_path)
except IOError:
pass
request.addfinalizer(fin)
def main():
dd_proc = TProcessor(dd_thrift.DingService, DingDispatcher())
pp_proc = TProcessor(pp_thrift.PingService, PingDispatcher())
mux_proc = TMultiplexedProcessor()
mux_proc.register_processor(DD_SERVICE_NAME, dd_proc)
mux_proc.register_processor(PP_SERVICE_NAME, pp_proc)
server = TThreadedServer(mux_proc, TServerSocket(),
iprot_factory=TBinaryProtocolFactory(),
itrans_factory=TBufferedTransportFactory())
server.serve()
def make_server(service, handler, host, port,
proto_factory=TBinaryProtocolFactory()):
processor = TProcessor(service, handler)
transport = TServerSocket(host=host, port=port)
server = TThreadedServer(processor, transport,
iprot_factory=proto_factory)
return server
def make_server(service,
handler,
host="localhost",
port=9090,
unix_socket=None,
proto_factory=TBinaryProtocolFactory(),
trans_factory=TBufferedTransportFactory(),
client_timeout=3000,
certfile=None):
processor = TProcessor(service, handler)
if unix_socket:
server_socket = TServerSocket(unix_socket=unix_socket)
if certfile:
warnings.warn("SSL only works with host:port, not unix_socket.")
elif host and port:
if certfile:
server_socket = TSSLServerSocket(host=host,
port=port,
client_timeout=client_timeout,
certfile=certfile)
else:
server_socket = TServerSocket(host=host,
port=port,
client_timeout=client_timeout)
else:
raise ValueError("Either host/port or unix_socket must be provided.")
server = TThreadedServer(processor,
server_socket,
iprot_factory=proto_factory,
itrans_factory=trans_factory)
return server
def get_rpc_application():
"""Creates a Gunicorn Thrift compatible TProcessor and initializes NewRelic
"""
global __new_relic
_init_django()
if agent and not __new_relic: # pragma: no cover
try:
agent.initialize()
logging.info('Initialized New Relic application')
__new_relic = True
except Exception as exc: # pylint: disable=all
logging.warning(
'Could not wrap RPC server in New Relic config. Exc: %s', exc)
for i_app in settings.INSTALLED_APPS:
if i_app.startswith('django') or 'manifold' in i_app:
continue
try:
importlib.import_module("%s.views" % i_app)
except ImportError: # pragma: no cover
logging.info(
'No module "%s.views" found, skipping RPC calls from it...',
i_app)
_print_rpc_config()
return TProcessor(load_service(), handler)
def __init__(self,
service_name,
thrift_file=None,
root_path=None,
name=None,
module_name=None):
"""
initialize an archer application
:param name: app name
:param thrift_file: the thrift file to load by thriftpy, if not set
,archer will automatically find a .thrift file
under root path
:param module_name: module name of the thrift module to be loaded,
enables us to pickle the module objects
:param service: service name of the thrift app
:param root_path: root path for file searching, default is pwd
"""
self.root_path = root_path or os.getcwd()
self.thrift_file = thrift_file or self._find_thrift_file()
self.thrift = thriftpy.load(self.thrift_file, module_name=module_name)
self.service_name = service_name
self.service = getattr(self.thrift, service_name)
self.name = name
self.app = TProcessor(self.service, self)
self.default_error_handler = None
self.before_api_call_funcs = []
self.after_api_call_funcs = []
self.tear_down_api_funcs = []
self.error_handlers = {}
self.registered_errors = []
self.api_map = {}
self.api_meta_map = {}
self.shell_context_processors = []
self.config = copy.deepcopy(self.default_config)
self.logger = logging.getLogger(self.logger_name)
def _make_server(app, host, port, daemon=True):
processor = TProcessor(app.service, app)
server_socket = TServerSocket(host=host, port=port)
server = TThreadedServer(processor, server_socket,
iprot_factory=TBinaryProtocolFactory(),
itrans_factory=TBufferedTransportFactory(),
daemon=daemon)
return server
def create_processor():
"""Creates a Gunicorn Thrift compatible TProcessor"""
service = getattr(THRIFT_MODULE, settings.THRIFT["SERVICE"])
for app in [x for x in settings.INSTALLED_APPS if not x.startswith("django")]:
importlib.import_module("%s.views" % app)
return TProcessor(service, create_handler())
def make_server(service,
handler,
host,
port,
proto_factory=TBinaryProtocolFactory()):
processor = TProcessor(service, handler)
server = THttpServer(processor, (host, port), iprot_factory=proto_factory)
return server
def create_thrift_server(service, handler, socket_config,
server_cls=TThreadedServer):
"""
创建Thrift Server对象。
"""
return server_cls(TProcessor(service, handler), # processor
TServerSocket(**socket_config), # transport
TBufferedTransportFactory(), # transportFactory
TBinaryProtocolFactory()) # protocolFactory
def make_server(service, handler,
host="localhost", port=9090, unix_socket=None,
proto_factory=TBinaryProtocolFactory(),
trans_factory=TBufferedTransportFactory()):
processor = TProcessor(service, handler)
if unix_socket:
server_socket = TServerSocket(unix_socket=unix_socket)
elif host and port:
server_socket = TServerSocket(host=host, port=port)
else:
raise ValueError("Either host/port or unix_socket must be provided.")
server = TThreadedServer(processor, server_socket,
iprot_factory=proto_factory,
itrans_factory=trans_factory)
return server
def init_event_channel(self):
"""Create a thrift server and register it at ZLP Service to receive events."""
if self._event_channel_handler and not self._event_channel:
processor = TProcessor(self.thrift_interface.ClientEventChannel,
self._event_channel_handler)
server_socket = TServerSocket(host="0.0.0.0",
port=0,
socket_family=socket.AF_INET,
client_timeout=200000)
server_socket.client_timeout = 1000 * 60 * 10
self._event_channel = TSimpleServer(processor, server_socket)
t = threading.Thread(target=self._event_channel.serve, daemon=True)
t.start()
time.sleep(1)
connection = self._event_channel.trans.sock.getsockname()
self.ConnectClientEventChannel(connection[1])
def create_multiplexed_server(services, socket_config,
server_cls=TThreadedServer):
"""
创建多路复用的Thrift Server
:param services: 多路复用服务定义，如：[(service1, handler1, service_name1),
(service2, handler2, service_name2),...]
:param socket_config: Server的socket参数
:param server_cls: 启动的服务器类型
:return: Server对象
"""
processor = TMultiplexedProcessor()
for service, handler, service_name in services:
processor.register_processor(service_name, TProcessor(service, handler))
return server_cls(processor, # processor
TServerSocket(**socket_config), # transport
TBufferedTransportFactory(), # transportFactory
TBinaryProtocolFactory()) # protocolFactory
def make_server(service,
handler,
fd=None,
host=None,
port=None,
unix_socket=None,
address_family=socket.AF_INET,
proto_factory=None,
trans_factory=None,
client_timeout=None,
backlog=128):
processor = TProcessor(service, handler)
if unix_socket is not None:
logger.info('Setting up server bound to %s', unix_socket)
server_socket = TServerSocket(unix_socket=unix_socket,
socket_family=address_family,
client_timeout=client_timeout,
backlog=backlog)
elif fd is not None:
logger.info('Setting up server bound to socket fd %s', fd)
server_socket = TFDServerSocket(fd=fd,
socket_family=address_family,
client_timeout=client_timeout,
backlog=backlog)
elif host is not None and port is not None:
logger.info('Setting up server bound to %s:%s', host, str(port))
server_socket = TServerSocket(host=host,
port=port,
socket_family=address_family,
client_timeout=client_timeout,
backlog=backlog)
else:
raise ValueError('Insufficient params')
server = TThreadedServer(processor,
server_socket,
iprot_factory=proto_factory,
itrans_factory=trans_factory)
return server
def make_simple_server(service, handler,
host="localhost",
port=9090):
"""Return a server of type TSimple Server.
Based on thriftpy's make_server(), but return TSimpleServer instead of
TThreadedServer.
Since TSimpleServer's constructor doesn't accept kwargs, some arguments of
make_server can't be used here. By default:
client_timeout: None
protocol: TBinaryProtocolFactory
transport: TBufferedTransportFactory
"""
processor = TProcessor(service, handler)
if host and port:
server_socket = TServerSocket(
host=host, port=port, client_timeout=None)
else:
raise ValueError("Either host/port or unix_socket must be provided.")
server = TSimpleServer(processor, server_socket)
return server
from thriftpy.thrift import TProcessor from koenig import koenig_thrift from koenig.dispatcher import KoenigDispatcher app = TProcessor(koenig_thrift.KoenigService, KoenigDispatcher())
#! /usr/bin/env python
# -*- coding: utf-8 -*-
import os
import time
import thriftpy
from thriftpy.thrift import TProcessor
thrift_service = thriftpy.load(
os.path.join(os.path.dirname(__file__), "pingpong.thrift"),
"pingpong_thrift") # noqa
service = thrift_service.PingService
class PingpongServer(object):
def ping(self):
if os.environ.get('about_to_shutdown') == '1':
raise service.AboutToShutDownException
return "pong"
def win(self):
return "Yes, you win"
def sleep(self, seconds):
time.sleep(seconds)
return 'good morning'
app = TProcessor(service, PingpongServer())
#! /usr/bin/env python
# -*- coding: utf-8 -*-
import os
import thriftpy
from thriftpy.thrift import TProcessor
from . import AboutToShutDownException
pingpong_thrift = thriftpy.load(
os.path.join(os.path.dirname(__file__), "pingpong.thrift"))
PingService = pingpong_thrift.PingService
class PingpongServer(object):
def ping(self):
if os.environ.get('about_to_shutdown') == '1':
raise AboutToShutDownException
return "pong"
app = TProcessor(pingpong_thrift.PingService, PingpongServer())
def fetch_all_works(self, uid):
kol.set_user(uid)
return json.dumps(kol.fetch_all_video())
def checkout_user_agent(self):
return kol.checkout_user_agent()
if sys.platform == 'win32':
kol_thrift = thriftpy.load(os.path.join(
PRO_DIR, '.\\dolphin\\service\\douyin\\data\\kol_thrift.thrift'),
module_name='kol_thrift_thrift')
else:
kol_thrift = thriftpy.load(os.path.join(
PRO_DIR, './dolphin/service/douyin/data/kol_thrift.thrift'),
module_name='kol_thrift_thrift')
app = TProcessor(kol_thrift.KolServer, KolDispatcher())
#
if __name__ == '__main__':
server = make_server(kol_thrift.KolServer,
KolDispatcher(),
'0.0.0.0',
6000,
proto_factory=TCyBinaryProtocolFactory(),
trans_factory=TCyBufferedTransportFactory())
server.serve()
# gunicorn_thrift dolphin.service.douyin.kolserver:app -k thriftpy_sync -b 0.0.0.0:6000 -w 4 --thrift-protocol-factory thriftpy.protocol:TCyBinaryProtocolFactory --thrift-transport-factory thriftpy.transport:TCyBufferedTransportFactory --thrift-client-timeout=5
# -*- coding: utf-8 -*-
import thriftpy
from thriftpy.thrift import TProcessor
pingpong = thriftpy.load("pingpong.thrift")
class Dispatcher(object):
def ping(self):
print("ping pong!")
return 'pong'
app = TProcessor(pingpong.PingService, Dispatcher())
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Author: stdrickforce (Tengyuan Fan)
# Email: <*****@*****.**> <*****@*****.**>
from thriftpy.thrift import TProcessor
from app import (
hackthon_thrift,
Handler,
)
app = TProcessor(hackthon_thrift.Hackthon, Handler())
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import thriftpy
from thriftpy.thrift import TProcessor
from dispatcher import GoodsDispatcher
goods = thriftpy.load("def/goods.thrift", module_name="goods_thrift")
app = TProcessor(goods.GoodsService, GoodsDispatcher())
#!/usr/bin/env python
# coding: utf-8
import os
import sys
import thriftpy
from thriftpy.thrift import TProcessor
helloworld_thrift = thriftpy.load("helloworld.thrift",
module_name="helloworld_thrift")
class HelloWorldHandler:
def ping(self):
return "pong"
def say(self, msg):
ret = "Received [%s]: %s" % (os.getpid(), msg)
print(ret)
sys.stdout.flush()
return ret
service = TProcessor(helloworld_thrift.HelloWorld, HelloWorldHandler())
class Archer(object):
debug = ConfigAttribute('DEBUG')
testing = ConfigAttribute('TESTING')
logger_name = ConfigAttribute('LOGGER_NAME')
default_config = {'DEBUG': False, 'TESTING': False, 'LOGGER_NAME': None}
def __init__(self,
name,
thrift_file=None,
service_name=None,
root_path=None):
"""
initialize an archer application
:param name: app name
:param thrift_file: the thrift file to load by thriftpy, if not set
,archer will automatically find a .thrift file
under root path
:param service: service name of the thrift app, if not set, archer
will automatically find a service in the dynamically
loaded thrift
:param root_path: root path for file searching, default is pwd
"""
self.root_path = root_path or os.getcwd()
self.thrift_file = thrift_file or self._find_thrift_file()
self.thrift = thriftpy.load(self.thrift_file)
self.service = self._find_service(service_name)
self.name = name
self.app = TProcessor(self.service, self)
self.default_error_handler = None
self.before_api_call_funcs = []
self.after_api_call_funcs = []
self.tear_down_api_funcs = []
self.error_handlers = {}
self.registered_errors = []
self.api_map = {}
self.api_meta_map = {}
self.shell_context_processors = []
self.config = copy.deepcopy(self.default_config)
self.logger = logging.getLogger(self.logger_name)
def _find_service(self, service_name):
if service_name is None:
for k, v in iteritems(self.thrift.__dict__):
if isinstance(v, type) and issubclass(v, _BaseService):
self.service_name = k
return v
return getattr(self.thrift, service_name)
def _find_thrift_file(self):
def _find_in_dir(path):
for f in os.listdir(path):
f = os.path.join(path, f)
if f.endswith('.thrift'):
return f
if os.path.isdir(f):
thrift_file = _find_in_dir(f)
if thrift_file:
return thrift_file
return _find_in_dir(self.root_path)
def run(self, host='127.0.0.1', port=6000, use_reloader=True, **options):
run_simple(host,
port,
self,
extra_files=[self.thrift_file],
use_reloader=use_reloader,
**options)
def make_shell_context(self):
"""Returns the shell context for an interactive shell for this
application. This runs all the registered shell context
processors.
"""
rv = {
'app': self,
'test_client': self.test_client,
'fake_client': self.fake_client,
}
for processor in self.shell_context_processors:
rv.update(processor())
return rv
def shell_context_processor(self, f):
"""registers a shell context processor function.
A processor function is just a function that takes
no arguments and return a dict object, which contains
attributes that would be loaded in to the interactive shell
e.g.::
app = Archer(__name__)
@app.shell_context_processor
def redis_cli():
import redis
return {
'redis': redis.StrictRedis()
}
.. versionadded:: 0.1
context
"""
self.shell_context_processors.append(f)
return f
def register_default_error_handler(self, handler):
self.default_error_handler = handler
def errorhandler(self, error_type):
def decorator(f):
self.register_error_handler(error_type, f)
return f
return decorator
def register_error_handler(self, error_type, f):
"""
register an error_type on a given api function
"""
assert error_type not in (Exception, BaseException), ValueError(
"Please Register with `register_default_exc_handler`")
if error_type in self.error_handlers:
existing = self.error_handlers[error_type]
raise KeyError("Handler (%s) already registered in %s" %
(error_type, existing))
self._append_error(error_type)
self.error_handlers[error_type] = f
def _append_error(self, new_exc):
"""
append an exception_type to registered error types,
ensure that the order is based on class's ``__mro__`` hierarchy
"""
registered_errors = self.registered_errors
for i, exc in enumerate(registered_errors):
if new_exc is exc:
registered_errors[i] = new_exc
break
elif issubclass(new_exc, exc):
registered_errors.insert(i, new_exc)
break
else:
registered_errors.append(new_exc)
def before_api_call(self, f):
"""
register a function which would always be called before
an api function is called,
the function would take one argument, which is an instance
of :class:`~archer.app.ApiMeta`
"""
self.before_api_call_funcs.append(f)
return f
def after_api_call(self, f):
"""
register a function which would be called after
an api function is executed successfully
the function would take one two arguments, the first one is
an instance of :class:`ApiMeta` and the second one is an instance
of :class:`~archer.app.ApiResultMeta`, same for :meth:`tear_down_api_call`.
"""
self.after_api_call_funcs.append(f)
return f
def tear_down_api_call(self, f):
"""
register a function which would always be called after
an api is executed, no matter whether it returns normally or raise
some error
"""
self.teardown_api_funcs.append(f)
return f
def api(self, name, **meta):
"""
decorator used to register a thrift rpc api
:param name: api name
:param meta: meta attributes would add to the app's api_meta_map
:return: the original api function
"""
def on_decorate(func):
self.register_api(name, func, **meta)
return func
return on_decorate
def register_api(self, name, f, **meta):
if name in self.api_map:
raise KeyError("Api_name(%s) already registered in %s" %
(name, self.api_map[name].__module__))
self.api_map[name] = self._wrap_api(name, f)
if PY2:
self.api_map[name].__wrapped__ = f
self.api_meta_map[name] = meta
def _preprocess_api(self, api_meta):
for handler in self.before_api_call_funcs:
ret_val = handler(api_meta)
def _postprocess_api(self, api_meta, result_meta):
for handler in self.after_api_call_funcs:
handler(api_meta, result_meta)
def _tear_down_api(self, api_meta, result_meta):
for handler in self.tear_down_api_funcs:
handler(api_meta, result_meta)
def _wrap_api(self, name, f):
@functools.wraps(f)
def wrapper(*args, **kwargs):
api_meta = ApiMeta(self, name, f, args, kwargs)
before_api_call.notify(api_meta)
self._preprocess_api(api_meta)
try:
ret_val = f(*args, **kwargs)
except Exception as e:
result_meta = ApiResultMeta(error=e)
for exc_type in self.registered_errors:
handler = self.error_handlers[exc_type]
if isinstance(e, exc_type):
return handler(api_meta, result_meta)
return self.handle_uncaught_exception(api_meta, result_meta)
else:
result_meta = ApiResultMeta(ret_val)
after_api_call.notify(api_meta, result_meta)
self._postprocess_api(api_meta, result_meta)
return ret_val
finally:
tear_down_api_call.notify(api_meta, result_meta)
self._tear_down_api(api_meta, result_meta)
return wrapper
def handle_uncaught_exception(self, api_meta, result_meta):
if self.default_error_handler is not None:
return self.default_error_handler(api_meta, result_meta)
raise
def processor(self, iprot, oprot):
"""
delegate the processor method to the `app` attribute,
which is an instance of :class:`thriftpy.thrift.TProcessor`.
Make an Archer app compatible with the thrift_app definition in
`gunicorn_thrift <http://github.com/eleme/gunicorn_thrift>`_
"""
return self.app.processor(iprot, oprot)
@property
def test_client(self):
"""
:return: an :class:`~archer.test.TestClient` instance
"""
return TestClient(self)
@property
def fake_client(self):
"""
:return: an :class:`~archer.test.FakeClient` instance
"""
return FakeClient(self)
def __getattr__(self, name):
if name not in self.api_map:
raise AttributeError('''app don't have `{0}` attribute,
is it an api? if you mean an api,
this api `{0}` is not registered'''.format(name))
return self.api_map[name]
def config_from_envvar(self, variable_name, silent=False):
"""Loads a configuration from an environment variable pointing to
a configuration file. This is basically just a shortcut with nicer
error messages for this line of code::
app.config_from_pyfile(os.environ['YOURAPPLICATION_SETTINGS'])
:param variable_name: name of the environment variable
:param silent: set to ``True`` if you want silent failure for missing
files.
:return: bool. ``True`` if able to load config, ``False`` otherwise.
"""
rv = os.environ.get(variable_name)
if not rv:
if silent:
return False
raise RuntimeError('The environment variable %r is not set '
'and as such configuration could not be '
'loaded. Set this variable and make it '
'point to a configuration file' % variable_name)
return self.from_pyfile(rv, silent=silent)
def config_from_object(self, obj):
"""Updates the values from the given object. An object can be of one
of the following two types:
- a string: in this case the object with that name will be imported
- an actual object reference: that object is used directly
Objects are usually either modules or classes.
Just the uppercase variables in that object are stored in the config.
Example usage::
app.config_from_object('yourapplication.default_config')
from yourapplication import default_config
app.config_from_object(default_config)
You should not use this function to load the actual configuration but
rather configuration defaults. The actual config should be loaded
with :meth:`from_pyfile` and ideally from a location not within the
package because the package might be installed system wide.
:param obj: an import name or object
"""
if isinstance(obj, string_types):
obj = import_string(obj)
for key in dir(obj):
if key.isupper():
self.config[key] = getattr(obj, key)
def config_from_pyfile(self, filename, silent=False):
"""Updates the values in the config from a Python file. This function
behaves as if the file was imported as module with the
:meth:`from_object` function.
:param filename: the filename of the config. This can either be an
absolute filename or a filename relative to the
root path.
:param silent: set to ``True`` if you want silent failure for missing
files.
"""
filename = os.path.join(self.root_path, filename)
d = imp.new_module('config')
d.__file__ = filename
try:
with open(filename) as config_file:
exec(compile(config_file.read(), filename, 'exec'), d.__dict__)
except IOError as e:
if silent and e.errno in (errno.ENOENT, errno.EISDIR):
return False
e.strerror = 'Unable to load configuration file (%s)' % e.strerror
raise
self.from_object(d)
return True
response = task_thrift.TaskResponse(code, message)
print(response)
return response
"""任务处理"""
def add_task(self, req):
logger.info("receive task:%s", req)
print(req)
proxy = ProxyClient.get_proxy(req.url)
if proxy is None:
return self.failed(500, "内部错误")
return self.success()
def main():
'''普通运行模式 '''
server = make_server(task_thrift.TaskService, TaskDispatcher(),
'127.0.0.1', 8000)
print("serving...")
server.serve()
'''
gunicorn运行模式
'''
app = TProcessor(task_thrift.TaskService, TaskDispatcher())
if __name__ == '__main__':
main()
