def server(request): p1 = TProcessor(mux.ThingOneService, DispatcherOne()) p2 = TProcessor(mux.ThingTwoService, DispatcherTwo()) mux_proc = TMultiplexingProcessor() mux_proc.register_processor(p1) mux_proc.register_processor(p2) _server = TThreadedServer(mux_proc, TServerSocket(unix_socket=sock_path), iprot_factory=TBinaryProtocolFactory(), itrans_factory=TBufferedTransportFactory()) ps = multiprocessing.Process(target=_server.serve) ps.start() time.sleep(0.1) def fin(): if ps.is_alive(): ps.terminate() try: os.remove(sock_path) except IOError: pass request.addfinalizer(fin)
def main(): dd_proc = TProcessor(dd_thrift.DingService, DingDispatcher()) pp_proc = TProcessor(pp_thrift.PingService, PingDispatcher()) mux_proc = TMultiplexedProcessor() mux_proc.register_processor(DD_SERVICE_NAME, dd_proc) mux_proc.register_processor(PP_SERVICE_NAME, pp_proc) server = TThreadedServer(mux_proc, TServerSocket(), iprot_factory=TBinaryProtocolFactory(), itrans_factory=TBufferedTransportFactory()) server.serve()
def make_server(service, handler, host, port, proto_factory=TBinaryProtocolFactory()): processor = TProcessor(service, handler) transport = TServerSocket(host=host, port=port) server = TThreadedServer(processor, transport, iprot_factory=proto_factory) return server
def make_server(service, handler, host="localhost", port=9090, unix_socket=None, proto_factory=TBinaryProtocolFactory(), trans_factory=TBufferedTransportFactory(), client_timeout=3000, certfile=None): processor = TProcessor(service, handler) if unix_socket: server_socket = TServerSocket(unix_socket=unix_socket) if certfile: warnings.warn("SSL only works with host:port, not unix_socket.") elif host and port: if certfile: server_socket = TSSLServerSocket(host=host, port=port, client_timeout=client_timeout, certfile=certfile) else: server_socket = TServerSocket(host=host, port=port, client_timeout=client_timeout) else: raise ValueError("Either host/port or unix_socket must be provided.") server = TThreadedServer(processor, server_socket, iprot_factory=proto_factory, itrans_factory=trans_factory) return server
def get_rpc_application(): """Creates a Gunicorn Thrift compatible TProcessor and initializes NewRelic """ global __new_relic _init_django() if agent and not __new_relic: # pragma: no cover try: agent.initialize() logging.info('Initialized New Relic application') __new_relic = True except Exception as exc: # pylint: disable=all logging.warning( 'Could not wrap RPC server in New Relic config. Exc: %s', exc) for i_app in settings.INSTALLED_APPS: if i_app.startswith('django') or 'manifold' in i_app: continue try: importlib.import_module("%s.views" % i_app) except ImportError: # pragma: no cover logging.info( 'No module "%s.views" found, skipping RPC calls from it...', i_app) _print_rpc_config() return TProcessor(load_service(), handler)
def __init__(self, service_name, thrift_file=None, root_path=None, name=None, module_name=None): """ initialize an archer application :param name: app name :param thrift_file: the thrift file to load by thriftpy, if not set ,archer will automatically find a .thrift file under root path :param module_name: module name of the thrift module to be loaded, enables us to pickle the module objects :param service: service name of the thrift app :param root_path: root path for file searching, default is pwd """ self.root_path = root_path or os.getcwd() self.thrift_file = thrift_file or self._find_thrift_file() self.thrift = thriftpy.load(self.thrift_file, module_name=module_name) self.service_name = service_name self.service = getattr(self.thrift, service_name) self.name = name self.app = TProcessor(self.service, self) self.default_error_handler = None self.before_api_call_funcs = [] self.after_api_call_funcs = [] self.tear_down_api_funcs = [] self.error_handlers = {} self.registered_errors = [] self.api_map = {} self.api_meta_map = {} self.shell_context_processors = [] self.config = copy.deepcopy(self.default_config) self.logger = logging.getLogger(self.logger_name)
def _make_server(app, host, port, daemon=True): processor = TProcessor(app.service, app) server_socket = TServerSocket(host=host, port=port) server = TThreadedServer(processor, server_socket, iprot_factory=TBinaryProtocolFactory(), itrans_factory=TBufferedTransportFactory(), daemon=daemon) return server
def create_processor(): """Creates a Gunicorn Thrift compatible TProcessor""" service = getattr(THRIFT_MODULE, settings.THRIFT["SERVICE"]) for app in [x for x in settings.INSTALLED_APPS if not x.startswith("django")]: importlib.import_module("%s.views" % app) return TProcessor(service, create_handler())
def make_server(service, handler, host, port, proto_factory=TBinaryProtocolFactory()): processor = TProcessor(service, handler) server = THttpServer(processor, (host, port), iprot_factory=proto_factory) return server
def create_thrift_server(service, handler, socket_config, server_cls=TThreadedServer): """ 创建Thrift Server对象。 """ return server_cls(TProcessor(service, handler), # processor TServerSocket(**socket_config), # transport TBufferedTransportFactory(), # transportFactory TBinaryProtocolFactory()) # protocolFactory
def make_server(service, handler, host="localhost", port=9090, unix_socket=None, proto_factory=TBinaryProtocolFactory(), trans_factory=TBufferedTransportFactory()): processor = TProcessor(service, handler) if unix_socket: server_socket = TServerSocket(unix_socket=unix_socket) elif host and port: server_socket = TServerSocket(host=host, port=port) else: raise ValueError("Either host/port or unix_socket must be provided.") server = TThreadedServer(processor, server_socket, iprot_factory=proto_factory, itrans_factory=trans_factory) return server
def init_event_channel(self): """Create a thrift server and register it at ZLP Service to receive events.""" if self._event_channel_handler and not self._event_channel: processor = TProcessor(self.thrift_interface.ClientEventChannel, self._event_channel_handler) server_socket = TServerSocket(host="0.0.0.0", port=0, socket_family=socket.AF_INET, client_timeout=200000) server_socket.client_timeout = 1000 * 60 * 10 self._event_channel = TSimpleServer(processor, server_socket) t = threading.Thread(target=self._event_channel.serve, daemon=True) t.start() time.sleep(1) connection = self._event_channel.trans.sock.getsockname() self.ConnectClientEventChannel(connection[1])
def create_multiplexed_server(services, socket_config, server_cls=TThreadedServer): """ 创建多路复用的Thrift Server :param services: 多路复用服务定义,如:[(service1, handler1, service_name1), (service2, handler2, service_name2),...] :param socket_config: Server的socket参数 :param server_cls: 启动的服务器类型 :return: Server对象 """ processor = TMultiplexedProcessor() for service, handler, service_name in services: processor.register_processor(service_name, TProcessor(service, handler)) return server_cls(processor, # processor TServerSocket(**socket_config), # transport TBufferedTransportFactory(), # transportFactory TBinaryProtocolFactory()) # protocolFactory
def make_server(service, handler, fd=None, host=None, port=None, unix_socket=None, address_family=socket.AF_INET, proto_factory=None, trans_factory=None, client_timeout=None, backlog=128): processor = TProcessor(service, handler) if unix_socket is not None: logger.info('Setting up server bound to %s', unix_socket) server_socket = TServerSocket(unix_socket=unix_socket, socket_family=address_family, client_timeout=client_timeout, backlog=backlog) elif fd is not None: logger.info('Setting up server bound to socket fd %s', fd) server_socket = TFDServerSocket(fd=fd, socket_family=address_family, client_timeout=client_timeout, backlog=backlog) elif host is not None and port is not None: logger.info('Setting up server bound to %s:%s', host, str(port)) server_socket = TServerSocket(host=host, port=port, socket_family=address_family, client_timeout=client_timeout, backlog=backlog) else: raise ValueError('Insufficient params') server = TThreadedServer(processor, server_socket, iprot_factory=proto_factory, itrans_factory=trans_factory) return server
def make_simple_server(service, handler, host="localhost", port=9090): """Return a server of type TSimple Server. Based on thriftpy's make_server(), but return TSimpleServer instead of TThreadedServer. Since TSimpleServer's constructor doesn't accept kwargs, some arguments of make_server can't be used here. By default: client_timeout: None protocol: TBinaryProtocolFactory transport: TBufferedTransportFactory """ processor = TProcessor(service, handler) if host and port: server_socket = TServerSocket( host=host, port=port, client_timeout=None) else: raise ValueError("Either host/port or unix_socket must be provided.") server = TSimpleServer(processor, server_socket) return server
from thriftpy.thrift import TProcessor from koenig import koenig_thrift from koenig.dispatcher import KoenigDispatcher app = TProcessor(koenig_thrift.KoenigService, KoenigDispatcher())
#! /usr/bin/env python # -*- coding: utf-8 -*- import os import time import thriftpy from thriftpy.thrift import TProcessor thrift_service = thriftpy.load( os.path.join(os.path.dirname(__file__), "pingpong.thrift"), "pingpong_thrift") # noqa service = thrift_service.PingService class PingpongServer(object): def ping(self): if os.environ.get('about_to_shutdown') == '1': raise service.AboutToShutDownException return "pong" def win(self): return "Yes, you win" def sleep(self, seconds): time.sleep(seconds) return 'good morning' app = TProcessor(service, PingpongServer())
#! /usr/bin/env python # -*- coding: utf-8 -*- import os import thriftpy from thriftpy.thrift import TProcessor from . import AboutToShutDownException pingpong_thrift = thriftpy.load( os.path.join(os.path.dirname(__file__), "pingpong.thrift")) PingService = pingpong_thrift.PingService class PingpongServer(object): def ping(self): if os.environ.get('about_to_shutdown') == '1': raise AboutToShutDownException return "pong" app = TProcessor(pingpong_thrift.PingService, PingpongServer())
def fetch_all_works(self, uid): kol.set_user(uid) return json.dumps(kol.fetch_all_video()) def checkout_user_agent(self): return kol.checkout_user_agent() if sys.platform == 'win32': kol_thrift = thriftpy.load(os.path.join( PRO_DIR, '.\\dolphin\\service\\douyin\\data\\kol_thrift.thrift'), module_name='kol_thrift_thrift') else: kol_thrift = thriftpy.load(os.path.join( PRO_DIR, './dolphin/service/douyin/data/kol_thrift.thrift'), module_name='kol_thrift_thrift') app = TProcessor(kol_thrift.KolServer, KolDispatcher()) # if __name__ == '__main__': server = make_server(kol_thrift.KolServer, KolDispatcher(), '0.0.0.0', 6000, proto_factory=TCyBinaryProtocolFactory(), trans_factory=TCyBufferedTransportFactory()) server.serve() # gunicorn_thrift dolphin.service.douyin.kolserver:app -k thriftpy_sync -b 0.0.0.0:6000 -w 4 --thrift-protocol-factory thriftpy.protocol:TCyBinaryProtocolFactory --thrift-transport-factory thriftpy.transport:TCyBufferedTransportFactory --thrift-client-timeout=5
# -*- coding: utf-8 -*- import thriftpy from thriftpy.thrift import TProcessor pingpong = thriftpy.load("pingpong.thrift") class Dispatcher(object): def ping(self): print("ping pong!") return 'pong' app = TProcessor(pingpong.PingService, Dispatcher())
#!/usr/bin/env python # -*- coding: utf-8 -*- # Author: stdrickforce (Tengyuan Fan) # Email: <*****@*****.**> <*****@*****.**> from thriftpy.thrift import TProcessor from app import ( hackthon_thrift, Handler, ) app = TProcessor(hackthon_thrift.Hackthon, Handler())
#!/usr/bin/env python # -*- coding: utf-8 -*- import thriftpy from thriftpy.thrift import TProcessor from dispatcher import GoodsDispatcher goods = thriftpy.load("def/goods.thrift", module_name="goods_thrift") app = TProcessor(goods.GoodsService, GoodsDispatcher())
#!/usr/bin/env python # coding: utf-8 import os import sys import thriftpy from thriftpy.thrift import TProcessor helloworld_thrift = thriftpy.load("helloworld.thrift", module_name="helloworld_thrift") class HelloWorldHandler: def ping(self): return "pong" def say(self, msg): ret = "Received [%s]: %s" % (os.getpid(), msg) print(ret) sys.stdout.flush() return ret service = TProcessor(helloworld_thrift.HelloWorld, HelloWorldHandler())
class Archer(object): debug = ConfigAttribute('DEBUG') testing = ConfigAttribute('TESTING') logger_name = ConfigAttribute('LOGGER_NAME') default_config = {'DEBUG': False, 'TESTING': False, 'LOGGER_NAME': None} def __init__(self, name, thrift_file=None, service_name=None, root_path=None): """ initialize an archer application :param name: app name :param thrift_file: the thrift file to load by thriftpy, if not set ,archer will automatically find a .thrift file under root path :param service: service name of the thrift app, if not set, archer will automatically find a service in the dynamically loaded thrift :param root_path: root path for file searching, default is pwd """ self.root_path = root_path or os.getcwd() self.thrift_file = thrift_file or self._find_thrift_file() self.thrift = thriftpy.load(self.thrift_file) self.service = self._find_service(service_name) self.name = name self.app = TProcessor(self.service, self) self.default_error_handler = None self.before_api_call_funcs = [] self.after_api_call_funcs = [] self.tear_down_api_funcs = [] self.error_handlers = {} self.registered_errors = [] self.api_map = {} self.api_meta_map = {} self.shell_context_processors = [] self.config = copy.deepcopy(self.default_config) self.logger = logging.getLogger(self.logger_name) def _find_service(self, service_name): if service_name is None: for k, v in iteritems(self.thrift.__dict__): if isinstance(v, type) and issubclass(v, _BaseService): self.service_name = k return v return getattr(self.thrift, service_name) def _find_thrift_file(self): def _find_in_dir(path): for f in os.listdir(path): f = os.path.join(path, f) if f.endswith('.thrift'): return f if os.path.isdir(f): thrift_file = _find_in_dir(f) if thrift_file: return thrift_file return _find_in_dir(self.root_path) def run(self, host='127.0.0.1', port=6000, use_reloader=True, **options): run_simple(host, port, self, extra_files=[self.thrift_file], use_reloader=use_reloader, **options) def make_shell_context(self): """Returns the shell context for an interactive shell for this application. This runs all the registered shell context processors. """ rv = { 'app': self, 'test_client': self.test_client, 'fake_client': self.fake_client, } for processor in self.shell_context_processors: rv.update(processor()) return rv def shell_context_processor(self, f): """registers a shell context processor function. A processor function is just a function that takes no arguments and return a dict object, which contains attributes that would be loaded in to the interactive shell e.g.:: app = Archer(__name__) @app.shell_context_processor def redis_cli(): import redis return { 'redis': redis.StrictRedis() } .. versionadded:: 0.1 context """ self.shell_context_processors.append(f) return f def register_default_error_handler(self, handler): self.default_error_handler = handler def errorhandler(self, error_type): def decorator(f): self.register_error_handler(error_type, f) return f return decorator def register_error_handler(self, error_type, f): """ register an error_type on a given api function """ assert error_type not in (Exception, BaseException), ValueError( "Please Register with `register_default_exc_handler`") if error_type in self.error_handlers: existing = self.error_handlers[error_type] raise KeyError("Handler (%s) already registered in %s" % (error_type, existing)) self._append_error(error_type) self.error_handlers[error_type] = f def _append_error(self, new_exc): """ append an exception_type to registered error types, ensure that the order is based on class's ``__mro__`` hierarchy """ registered_errors = self.registered_errors for i, exc in enumerate(registered_errors): if new_exc is exc: registered_errors[i] = new_exc break elif issubclass(new_exc, exc): registered_errors.insert(i, new_exc) break else: registered_errors.append(new_exc) def before_api_call(self, f): """ register a function which would always be called before an api function is called, the function would take one argument, which is an instance of :class:`~archer.app.ApiMeta` """ self.before_api_call_funcs.append(f) return f def after_api_call(self, f): """ register a function which would be called after an api function is executed successfully the function would take one two arguments, the first one is an instance of :class:`ApiMeta` and the second one is an instance of :class:`~archer.app.ApiResultMeta`, same for :meth:`tear_down_api_call`. """ self.after_api_call_funcs.append(f) return f def tear_down_api_call(self, f): """ register a function which would always be called after an api is executed, no matter whether it returns normally or raise some error """ self.teardown_api_funcs.append(f) return f def api(self, name, **meta): """ decorator used to register a thrift rpc api :param name: api name :param meta: meta attributes would add to the app's api_meta_map :return: the original api function """ def on_decorate(func): self.register_api(name, func, **meta) return func return on_decorate def register_api(self, name, f, **meta): if name in self.api_map: raise KeyError("Api_name(%s) already registered in %s" % (name, self.api_map[name].__module__)) self.api_map[name] = self._wrap_api(name, f) if PY2: self.api_map[name].__wrapped__ = f self.api_meta_map[name] = meta def _preprocess_api(self, api_meta): for handler in self.before_api_call_funcs: ret_val = handler(api_meta) def _postprocess_api(self, api_meta, result_meta): for handler in self.after_api_call_funcs: handler(api_meta, result_meta) def _tear_down_api(self, api_meta, result_meta): for handler in self.tear_down_api_funcs: handler(api_meta, result_meta) def _wrap_api(self, name, f): @functools.wraps(f) def wrapper(*args, **kwargs): api_meta = ApiMeta(self, name, f, args, kwargs) before_api_call.notify(api_meta) self._preprocess_api(api_meta) try: ret_val = f(*args, **kwargs) except Exception as e: result_meta = ApiResultMeta(error=e) for exc_type in self.registered_errors: handler = self.error_handlers[exc_type] if isinstance(e, exc_type): return handler(api_meta, result_meta) return self.handle_uncaught_exception(api_meta, result_meta) else: result_meta = ApiResultMeta(ret_val) after_api_call.notify(api_meta, result_meta) self._postprocess_api(api_meta, result_meta) return ret_val finally: tear_down_api_call.notify(api_meta, result_meta) self._tear_down_api(api_meta, result_meta) return wrapper def handle_uncaught_exception(self, api_meta, result_meta): if self.default_error_handler is not None: return self.default_error_handler(api_meta, result_meta) raise def processor(self, iprot, oprot): """ delegate the processor method to the `app` attribute, which is an instance of :class:`thriftpy.thrift.TProcessor`. Make an Archer app compatible with the thrift_app definition in `gunicorn_thrift <http://github.com/eleme/gunicorn_thrift>`_ """ return self.app.processor(iprot, oprot) @property def test_client(self): """ :return: an :class:`~archer.test.TestClient` instance """ return TestClient(self) @property def fake_client(self): """ :return: an :class:`~archer.test.FakeClient` instance """ return FakeClient(self) def __getattr__(self, name): if name not in self.api_map: raise AttributeError('''app don't have `{0}` attribute, is it an api? if you mean an api, this api `{0}` is not registered'''.format(name)) return self.api_map[name] def config_from_envvar(self, variable_name, silent=False): """Loads a configuration from an environment variable pointing to a configuration file. This is basically just a shortcut with nicer error messages for this line of code:: app.config_from_pyfile(os.environ['YOURAPPLICATION_SETTINGS']) :param variable_name: name of the environment variable :param silent: set to ``True`` if you want silent failure for missing files. :return: bool. ``True`` if able to load config, ``False`` otherwise. """ rv = os.environ.get(variable_name) if not rv: if silent: return False raise RuntimeError('The environment variable %r is not set ' 'and as such configuration could not be ' 'loaded. Set this variable and make it ' 'point to a configuration file' % variable_name) return self.from_pyfile(rv, silent=silent) def config_from_object(self, obj): """Updates the values from the given object. An object can be of one of the following two types: - a string: in this case the object with that name will be imported - an actual object reference: that object is used directly Objects are usually either modules or classes. Just the uppercase variables in that object are stored in the config. Example usage:: app.config_from_object('yourapplication.default_config') from yourapplication import default_config app.config_from_object(default_config) You should not use this function to load the actual configuration but rather configuration defaults. The actual config should be loaded with :meth:`from_pyfile` and ideally from a location not within the package because the package might be installed system wide. :param obj: an import name or object """ if isinstance(obj, string_types): obj = import_string(obj) for key in dir(obj): if key.isupper(): self.config[key] = getattr(obj, key) def config_from_pyfile(self, filename, silent=False): """Updates the values in the config from a Python file. This function behaves as if the file was imported as module with the :meth:`from_object` function. :param filename: the filename of the config. This can either be an absolute filename or a filename relative to the root path. :param silent: set to ``True`` if you want silent failure for missing files. """ filename = os.path.join(self.root_path, filename) d = imp.new_module('config') d.__file__ = filename try: with open(filename) as config_file: exec(compile(config_file.read(), filename, 'exec'), d.__dict__) except IOError as e: if silent and e.errno in (errno.ENOENT, errno.EISDIR): return False e.strerror = 'Unable to load configuration file (%s)' % e.strerror raise self.from_object(d) return True
response = task_thrift.TaskResponse(code, message) print(response) return response """任务处理""" def add_task(self, req): logger.info("receive task:%s", req) print(req) proxy = ProxyClient.get_proxy(req.url) if proxy is None: return self.failed(500, "内部错误") return self.success() def main(): '''普通运行模式 ''' server = make_server(task_thrift.TaskService, TaskDispatcher(), '127.0.0.1', 8000) print("serving...") server.serve() ''' gunicorn运行模式 ''' app = TProcessor(task_thrift.TaskService, TaskDispatcher()) if __name__ == '__main__': main()