class ExhibitionistServer(IProvider, threading.Thread): """ The tornado server thread, and also provides handler registration , both HTTP RequestHandlers with tornado and other types provided by providers all **kwds not consumed by the Server contructor, will be passed on as keyword arguments to tornado's application constructor. Particular examples are "static_path" and "template_path" """ def __init__(self, port=None, address='127.0.0.1', **kwds): """ If port== None, will pick one automatically """ import tornado.web import tornado.ioloop super(ExhibitionistServer, self).__init__() self.name = "ExhibitionistServer Thread" self.daemon = True self.synq = queue.Queue() self.started_ok = False # One IOLoop per thread # this was a nightmare to debug. self.ioloop = tornado.ioloop.IOLoop() if kwds.get('static_path') : assert os.path.isdir(kwds.get('static_path')) if kwds.get('template_path') : assert os.path.isdir(kwds.get('template_path')) kwds['template_path'] = kwds.get('template_path', "You_did_not_set_the_template_path") # logger.info(kwds) self.application = None # extra kwds are passed to application as settings # self.application.settings.update(kwds) self._server = None self.tornado_app_settings=kwds self.pubsub = PubSubDispatch(self.ioloop) self.http_handlers = set() self._port_requested = port self._address_requested = address self._port_used = None self._address_used = None self.providers = set() if kwds.get("__registry"): # for testing self.registry = kwds.get("__registry") else: import exhibitionist.shared self.registry = exhibitionist.shared.registry # register the provider part of self, # with the server part of self. self.register_provider(self) @staticmethod def _discover(pred, or_or_ns): """Internal: Takes a predicate + object/namespace, and finds xs for which pred(x) :param pred: lambda x: bool :param or_or_ns: a python object, including packages/modules :rtype : list """ import inspect candidates = [] if inspect.ismodule(or_or_ns): # modules and submodules of package modules = [or_or_ns] modules += [getattr(or_or_ns, x) for x in dir(or_or_ns) if inspect.ismodule(getattr(or_or_ns, x))] for m in modules: candidates.extend([getattr(m, x) for x in dir(m)]) candidates.append(or_or_ns) return [x for x in candidates if pred(x)] ####################### # The IProvider interface def register_with_server(self, server): pass def populate_context(self, context): setattr(context, GET_VIEW_ATTR, self.get_view_url) setattr(context, GET_REG_OBJ_ATTR, self.registry.get) setattr(context, PUBSUB_ATTR, self.pubsub) pass def subscribe(self, h): if self.isAlive(): raise RuntimeError( "Registering http handlers after server start is not allowed") # exst_names=[x for x in self.http_handlers # if get_view_name(x) == get_view_name(h)] # if exst_names and exst_names[0] != h: if h in self.http_handlers: raise RuntimeError("view_name collision, " "there's already a handler called %s" % http_handler.get_view_name(h)) if h not in self.http_handlers: tmpl = "Discovered {type} handler '{name}' tagged with {route}" logger.debug(tmpl.format(type='http', name=http_handler.get_view_name(h), route=http_handler.get_route(h))) self.http_handlers.add(h) # update the application http_handlers # this allows dynamic addition of http_handlers def is_http_handler(self, o): from exhibitionist.decorators import http_handler import inspect import tornado.web return (http_handler.is_decorated_as_http(o) and (inspect.isclass(o) and issubclass(o, tornado.web.RequestHandler))) is_handler = is_http_handler ####################### # The server interface def register_provider(self, provider): import inspect if self.started_ok: raise ExhibitionistError( "can only add providers before server start") # todo, refactor in to verify_provider() if inspect.isclass(provider): raise AssertionError( "Provider must be instance, not class. did you forget to add ()?") assert isinstance(provider, IProvider), "Provider must be inherit from IProvider" assert hasattr(provider, "is_handler") assert hasattr(provider, "subscribe") self.providers.add(provider) provider.register_with_server(self) if provider != self: # attach the provider instance as an attribute setattr(self, provider.name, provider) return self # fluent def add_handler(self, ns_or_h): """ called by user with http_handlers,or modules containing them, of whatever type (http, ws). They will be auto-detected and registered with the appropriate backend machinery may throw RuntimeError if provider refuses to accept new handlers (for example, new HTTP handlers after server start) """ if self.started_ok: raise ExhibitionistError( "can only add handlers before server start") for prvdr in self.providers: handlers = self._discover(prvdr.is_handler, ns_or_h) [prvdr.subscribe(x) for x in handlers] # py3 has lazy map, side-effects. return self # fluent def _register_handlers(self): """ register http_handlers with tornado application""" from tornado.web import URLSpec,Application urlconf = [URLSpec(http_handler.get_route(h), h, name=http_handler.get_view_name(h), kwargs=http_handler.get_kwds(h)) for h in self.http_handlers] self.application = Application(urlconf, **self.tornado_app_settings) # # self.application.add_handlers("", urlconf) # re-register everything def get_view_url(self, handler_name, *args, **kwds): """Returns a full url for the given view, with given argument a wrapper around tornado's reverse_url with some bells and whistles. if the handler included the {{objid}} special marker, 'objid' must be provide to reverse_url to be inserted into the returned url, and args[0] will be interpreted as the object to be viewed. If you pass in an object it will automatically be registered and the objid substituted into the url, but you can also provide an objid for an object previously registered. if you want a weakref to be used (to prevent memory leaks) to store the object in the object registry ,pass in a kwd argument `_weakref=True` when first viewing the argument. Note that not all types are supported by weakref. :param handler_name: Handler class name, or value of view_name used in @http_handler. :param obj_or_objid: objid is the return value of a previous call to registry.register :param args: values to use in unnamed capture groups in route regexp, if any. must match the number and order of groups. :param kwds: values to use in named capture groups in route regexp, if any must match the number and and names of groups. Throws: IOError if the server thread failed to initialize Examples: @http_handler("/blah/{{objid}}") class A(): ... get_view_url('A',my_object) ----------- @http_handler("/blah/{{objid}}/(\d+)/(?P<an_arg>\w+)") class A(): ... get_view_url('A',my_object,12,an_arg="baz") ----------- @http_handler("/blah/(\d+)/(?P<an_arg>\w+)") class A(): ... get_view_url('A',12,an_arg="baz") ----------- """ import exhibitionist.shared as shared from tornado import websocket from six import string_types weak = kwds.pop("_weakref",False) if self._ensure_up() != 0: # make sure the server thread has finished starting up raise ExhibitionistError("The server thread failed to start.") registry = kwds.pop('__registry', shared.registry) # for testing handler = self.application.named_handlers[handler_name].handler_class if args and http_handler.get_auto_obj(handler): obj_or_objid=args[0] objid=None # let the user provide an objid of registered object if isinstance(obj_or_objid, string_types) and registry.get(obj_or_objid): objid = obj_or_objid if objid is None: objid = registry.register(obj_or_objid,weak=weak) if objid is not None: args = (objid,) + args[1:] path = self.application.reverse_url(handler_name, *args, **kwds) # TODO: we special-case WSH here, need to be more general? if issubclass(handler, websocket.WebSocketHandler): prot = "ws" else: prot = "http" return "{prot}://{addr}:{port}{path}".format(prot=prot, addr=self.address, port=self.port, path=path) def stop(self, block=True, waitfor=0): """Initiates a full shutdown of event loop and server. if successful, will release the server'socket an the thread will stop, Note, that unless stop() is called, the socket used will not be released until shutdown, this can act as a leak for sockets. :param waitfor: in seconds, time between stopping the server (and providers) and calling stop on the event loop. """ import datetime def stopCallback2(): self.ioloop.stop() def stopCallback1(): self.ioloop.add_timeout(datetime.timedelta(seconds=waitfor), stopCallback2) def close_cb(s): return lambda : s.stop() for p in self.providers: if p != self: self.ioloop.add_callback(close_cb(p)) self.ioloop.add_callback(close_cb(self._server)) self.ioloop.add_callback(stopCallback1) if block: self.join() return self def _create_http_server(self,port_start,port_end,ioloop): import socket server = HTTPServer(self.application, io_loop=ioloop) for portnum in range(port_start, port_end): try: server.listen(portnum, address=self._address_requested) logger.info('Server listening on port {0}'.format(portnum)) self._port_used = portnum self._address_used = self._address_requested return server, portnum except socket.error as e: # try remaining ports if port used, raise otherwise if e.errno != errno.EADDRINUSE or portnum == port_end-1: logger.error(str(e)) # pragma: no cover try: server.stop() except: pass raise def run(self): import socket try: logger.info('Starting Up') for p in self.providers: p.populate_context(http_handler.get_context()) # register all discovered http_handler with Tornado application self._register_handlers() # extra kwds are passed to application as settings # self.application.settings.update(kwds) port_start = self._port_requested or settings.SERVER_PORT_BASE if self._port_requested: port_end = self._port_requested + 1 else: port_end = min(65535, port_start + settings.MAX_N_SOCKETS) try: (self._server,portnum) = self._create_http_server(port_start, port_end, self.ioloop) except (OSError,socket.error) as e: self.synq.put(e.errno)# pragma: no cover errmsg="Couldn't listen on ports: [{0},{1})" logger.error(errmsg.format(port_start, port_end)) raise except Exception as e: logger.error(str(e)) raise # we're fine, start the loop self.synq.put(0) # signal successful startup self.ioloop.start() # blocked here, until someone stops the loop, safely. # self.ioloop.close() # despite everything, still throwing fd errors logger.info('Stopped server at {0}:{1}'.format(self.address, self.port)) logger.info('Stopped IOLoop') # not Thread Safe self._port_used = None self._server = None self.application = None except Exception as e: # capture exceptions from daemonic thread to log file import traceback as tb logger.error( "Exception in server thread:\n" + str(e) + str(tb.format_exc())) def start(self, block=True, timeout=5): """ Start the server socket, bind to a socket and start listening Note, that the socket is held until stop() is called, or the main thread exists. This can act as a leak, unless stop() is called when the server is no longer needed. returns self :param timeout: in seconds :rtype : ExhibitionistServer """ super(ExhibitionistServer, self).start() if block: err = self._ensure_up(timeout) if err != 0: msg = "Server failed to start: %s" % errno.errorcode[err] raise ExhibitionistError(msg,errno=err) return self # fluent def _ensure_up(self, timeout=1): """ we must be sure that the server successfully bound :returns : 0 if success, errno <0 if an error occurred :rtype : int :param timeout: in seconds on a addr:port, before returning a url. Waits on a signal from the server thread that it is ready. :rtype : bool """ if self.started_ok: return 0 else: try: result = errno.EIO # will be returned if error in Queue get result = self.synq.get(timeout=timeout) except queue.Empty: return errno.ETIMEDOUT else: if result == 0: self.started_ok = True return 0 else: return result def notify_object(self, obj, payload, exclude=None): """ :param obj: any python object :param payload: a JSON encodable object to be send to the client :param exclude: callback or sequence of calllbacks to be ecluded from notification :rtype exclude: ISubscriber or sequence of ISubscribers :return: """ objid = self.registry.register(obj) return self.notify_channel(channel=objid, payload=payload, exclude=exclude) def notify_channel(self, channel, payload, exclude=None): """ :param channel: the name of a channel to publish a message on, usually an objid :param payload: :param exclude: callback or sequence of calllbacks to be ecluded from notification :rtype exclude: ISubscriber or sequence of ISubscribers :return: """ from six import string_types if not isinstance(channel, string_types): raise ValueError("Channel must be a string") # put message in q to notify the pubsubdispatch Thread of message # it will then queue on a send on the ioloop self.pubsub.publish(channel=channel, payload=payload, exclude=exclude) @property def port(self): """ None if server not active, listening port otherwise""" return self._port_used @property def address(self): """ None if server not active, bound address otherwise""" return self._address_used
class Testpubsubdispatch(unittest.TestCase): def setUp(self): self.pubsubdispatch = PubSubDispatch(IOLoopMock()) def tearDown(self): pass @staticmethod def wait_for_predicate(pred, timeout, interval=None): interval = interval or timeout waited = 0 while waited <= timeout: if pred(): return True time.sleep(interval) waited += interval return False def test_message_rx_tx(self): l = [] class A(ISubscriber): def notify(self, channel, payload): l.append(payload) self.pubsubdispatch.subscribe(A(), "ch1") self.assertEqual(len(l), 0) self.pubsubdispatch.publish(channel="ch1", payload="the payload") self.wait_for_predicate(lambda: len(l), 1, 0.001) self.assertEqual(len(l), 1) self.assertEqual(l.pop(), "the payload") # and again self.pubsubdispatch.publish(channel="ch1", payload="the payload2") self.wait_for_predicate(lambda: len(l), 1, 0.001) self.assertEqual(len(l), 1) self.assertEqual(l.pop(), "the payload2") # two receivers self.assertEqual(len(l), 0) self.pubsubdispatch.subscribe(A(), "ch1") self.pubsubdispatch.publish(channel="ch1", payload="the payload3") self.wait_for_predicate(lambda: len(l) >= 2, 1, 0.001) self.assertEqual(len(l), 2) self.assertEqual(l.pop(), "the payload3") self.assertEqual(l.pop(), "the payload3") # just the registered channels get the messages for a channel self.assertEqual(len(l), 0) self.pubsubdispatch.subscribe(A(), "ch2") self.pubsubdispatch.publish(channel="ch1", payload="the payload4") self.wait_for_predicate(lambda: len(l) >= 2, 1, 0.001) self.assertEqual(len(l), 2) self.assertEqual(l.pop(), "the payload4") self.assertEqual(l.pop(), "the payload4") self.assertEqual(len(l), 0) self.pubsubdispatch.publish(channel="ch2", payload="the payload5") self.wait_for_predicate(lambda: len(l) >= 1, 1, 0.001) self.assertEqual(len(l), 1) self.assertEqual(l.pop(), "the payload5") def test_make_sure_we_dont_receive_our_own_message(self): l = [] class A(ISubscriber): def notify(self, channel, payload): l.append(payload) a = A() # do self.pubsubdispatch.subscribe(a, "ch1") self.assertEqual(len(l), 0) self.pubsubdispatch.publish(channel="ch1", payload="the payload") self.wait_for_predicate(lambda: len(l), 0.2, 0.001) self.assertEqual(len(l), 1) self.assertEqual(l.pop(), "the payload") # don't self.pubsubdispatch.publish(channel="ch1", payload="the payload", exclude=a) self.wait_for_predicate(lambda: len(l), 0.2, 0.001) self.assertEqual(len(l), 0) def test_make_sure_we_dont_receive_our_own_message_multiple_subs(self): # make sure the other subscriber does get it, no matter the subscribe order l = [] class A(ISubscriber): def notify(self, channel, payload): l.append(self) a = A() b = A() # do self.pubsubdispatch.subscribe(a, "ch1") self.pubsubdispatch.subscribe(b, "ch1") self.assertEqual(len(l), 0) self.pubsubdispatch.publish(channel="ch1", payload="the payload", exclude=a) self.wait_for_predicate(lambda: len(l), 0.2, 0.001) self.assertEqual(len(l), 1) self.assertEqual(l.pop(), b) self.pubsubdispatch.publish(channel="ch1", payload="the payload", exclude=b) self.wait_for_predicate(lambda: len(l), 0.2, 0.001) self.assertEqual(len(l), 1) self.assertEqual(l.pop(), a)