Exemple #1
0
class CoreTest(TestCase):

    def setUp(self):
        self.application = Application()
        self.pubsub = BasePubSub(self.application)

        self.project_id = 'test'
        self.namespace = 'test'
        self.channel = 'test'

    def test_get_subscription_key(self):
        subscription_key = self.pubsub.get_subscription_key(
            self.project_id, self.namespace, self.channel
        )
        self.assertTrue(isinstance(subscription_key, str))

    def test_add_subscription(self):
        self.pubsub.add_subscription(self.project_id, self.namespace, self.channel, FakeClient())

        self.assertTrue(
            self.pubsub.get_subscription_key(
                self.project_id, self.namespace, self.channel
            ) in self.pubsub.subscriptions
        )

    def test_remove_subscription(self):
        self.pubsub.remove_subscription(self.project_id, self.namespace, self.channel, FakeClient())

        self.assertTrue(
            self.pubsub.get_subscription_key(
                self.project_id, self.namespace, self.channel
            ) not in self.pubsub.subscriptions
        )
Exemple #2
0
class CoreTest(TestCase):
    def setUp(self):
        self.application = Application()
        self.pubsub = BasePubSub(self.application)

        self.project_id = 'test'
        self.namespace = 'test'
        self.channel = 'test'

    def test_get_subscription_key(self):
        subscription_key = self.pubsub.get_subscription_key(
            self.project_id, self.namespace, self.channel)
        self.assertTrue(isinstance(subscription_key, str))

    def test_add_subscription(self):
        self.pubsub.add_subscription(self.project_id, self.namespace,
                                     self.channel, FakeClient())

        self.assertTrue(
            self.pubsub.get_subscription_key(self.project_id, self.namespace,
                                             self.channel) in
            self.pubsub.subscriptions)

    def test_remove_subscription(self):
        self.pubsub.remove_subscription(self.project_id, self.namespace,
                                        self.channel, FakeClient())

        self.assertTrue(
            self.pubsub.get_subscription_key(self.project_id, self.namespace,
                                             self.channel) not in
            self.pubsub.subscriptions)
Exemple #3
0
class Application(tornado.web.Application):

    # magic fake project ID for owner API purposes.
    MAGIC_PROJECT_ID = '_'

    # magic project param name to allow owner make API operations within project
    MAGIC_PROJECT_PARAM = '_project'

    # in milliseconds, how often this application will send ping message
    PING_INTERVAL = 5000

    # in seconds
    PING_MAX_DELAY = 10

    # in milliseconds, how often application will remove stale ping information
    PING_REVIEW_INTERVAL = 10000

    PERMISSION_DENIED = 'permission denied'

    INTERNAL_SERVER_ERROR = 'internal server error'

    METHOD_NOT_FOUND = 'method not found'

    PROJECT_NOT_FOUND = 'project not found'

    NAMESPACE_NOT_FOUND = 'namespace not found'

    DUPLICATE_NAME = 'duplicate name'

    def __init__(self, *args, **kwargs):

        # create unique uid for this application
        self.uid = uuid.uuid4().hex

        # PUB/SUB manager class
        self.pubsub = BasePubSub(self)

        # initialize dict to keep administrator's connections
        self.admin_connections = {}

        # initialize dict to keep client's connections
        self.connections = {}

        # dict to keep ping from nodes
        # key - node address, value - timestamp of last ping
        self.nodes = {}

        # application structure manager (projects, namespaces etc)
        self.structure = None

        # initialize dict to keep back-off information for projects
        self.back_off = {}

        # list of coroutines that must be done before message publishing
        self.pre_publish_callbacks = []

        # list of coroutines that must be done after message publishing
        self.post_publish_callbacks = []

        # initialize tornado's application
        super(Application, self).__init__(*args, **kwargs)

    def initialize(self):
        self.init_callbacks()
        self.init_structure()
        self.init_pubsub()
        self.init_state()
        self.init_ping()

    def init_structure(self):
        """
        Initialize structure manager using settings provided
        in configuration file.
        """
        custom_settings = self.settings['config']
        structure_settings = custom_settings.get('structure', {})

        # detect and apply database storage module
        storage_module = structure_settings.get(
            'storage', 'centrifuge.structure.sqlite'
        )
        storage = utils.import_module(storage_module)

        structure = Structure(self)
        structure.set_storage(storage)
        self.structure = structure

        def run_periodic_structure_update():
            structure.update()
            periodic_structure_update = tornado.ioloop.PeriodicCallback(
                structure.update, structure_settings.get('update_interval', 30)*1000
            )
            periodic_structure_update.start()

        tornado.ioloop.IOLoop.instance().add_callback(
            partial(
                storage.init_storage,
                structure,
                structure_settings.get('settings', {}),
                run_periodic_structure_update
            )
        )

        logger.info("Storage module: {0}".format(storage_module))

    def init_state(self):
        """
        Initialize state manager (for presence/history data).
        """
        config = self.settings['config']
        state_config = config.get("state", {})
        if not state_config:
            # use base fake state
            logger.info("No State configured")
            self.state = State(self, fake=True)
        else:
            state_storage = state_config.get('storage', 'centrifuge.state.base.State')
            state_storage_class = utils.namedAny(state_storage)
            self.state = state_storage_class(self)
            tornado.ioloop.IOLoop.instance().add_callback(self.state.initialize)

    def init_pubsub(self):
        """
        Initialize and configure pub/sub manager.
        """
        self.pubsub.initialize()

    def init_callbacks(self):
        """
        Fill custom callbacks with callable objects provided in config.
        """
        config = self.settings['config']
        pre_publish_callbacks = config.get('pre_publish_callbacks', [])

        for callable_path in pre_publish_callbacks:
            callback = utils.namedAny(callable_path)
            self.pre_publish_callbacks.append(callback)

        post_publish_callbacks = config.get('post_publish_callbacks', [])
        for callable_path in post_publish_callbacks:
            callback = utils.namedAny(callable_path)
            self.post_publish_callbacks.append(callback)

    def send_ping(self, ping_message):
        self.pubsub.publish_control_message(ping_message)

    def review_ping(self):
        """
        Remove outdated information about other nodes.
        """
        now = time.time()
        outdated = []
        for node, updated_at in self.nodes.items():
            if now - updated_at > self.PING_MAX_DELAY:
                outdated.append(node)
        for node in outdated:
            try:
                del self.nodes[node]
            except KeyError:
                pass

    def init_ping(self):
        """
        Start periodic tasks for sending ping and reviewing ping.
        """
        message = {
            'app_id': self.uid,
            'method': 'ping',
            'params': {'uid': self.uid}
        }
        send_ping = partial(self.pubsub.publish_control_message, message)
        ping = tornado.ioloop.PeriodicCallback(send_ping, self.PING_INTERVAL)
        tornado.ioloop.IOLoop.instance().add_timeout(
            self.PING_INTERVAL, ping.start
        )

        review_ping = tornado.ioloop.PeriodicCallback(self.review_ping, self.PING_REVIEW_INTERVAL)
        tornado.ioloop.IOLoop.instance().add_timeout(
            self.PING_INTERVAL, review_ping.start
        )

    def send_control_message(self, message):
        """
        Send message to CONTROL channel. We use this channel to
        share commands between running instances.
        """
        self.pubsub.publish_control_message(message)

    def add_connection(self, project_id, user, uid, client):
        """
        Register new client's connection.
        """
        if project_id not in self.connections:
            self.connections[project_id] = {}
        if user and user not in self.connections:
            self.connections[project_id][user] = {}
        if user:
            self.connections[project_id][user][uid] = client

    def remove_connection(self, project_id, user, uid):
        """
        Remove client's connection
        """
        try:
            del self.connections[project_id][user][uid]
        except KeyError:
            pass

        if project_id in self.connections and user in self.connections[project_id]:
            # clean connections
            if self.connections[project_id][user]:
                return
            try:
                del self.connections[project_id][user]
            except KeyError:
                pass
            if self.connections[project_id]:
                return
            try:
                del self.connections[project_id]
            except KeyError:
                pass

    def add_admin_connection(self, uid, client):
        """
        Register administrator's connection (from web-interface).
        """
        self.admin_connections[uid] = client

    def remove_admin_connection(self, uid):
        """
        Remove administrator's connection.
        """
        try:
            del self.admin_connections[uid]
        except KeyError:
            pass

    @coroutine
    def handle_ping(self, params):
        """
        Ping message received.
        """
        self.nodes[params.get('uid')] = time.time()

    @coroutine
    def handle_unsubscribe(self, params):
        """
        Unsubscribe message received - unsubscribe client from certain channels.
        """
        project = params.get("project")
        user = params.get("user")
        namespace_name = params.get("namespace", None)
        channel = params.get("channel", None)

        if not user:
            # we don't need to block anonymous users
            raise Return((True, None))

        project_id = project['_id']

        # try to find user's connection
        user_connections = self.connections.get(project_id, {}).get(user, None)
        if not user_connections:
            raise Return((True, None))

        namespace, error = yield self.structure.get_namespace_by_name(
            project, namespace_name
        )
        if error:
            raise Return((None, error))
        if channel and not namespace:
            # namespace does not exist
            raise Return((True, None))

        namespace_name = namespace['name']

        for uid, connection in six.iteritems(user_connections):

            if not namespace_name and not channel:
                # unsubscribe from all channels
                for ns, channels in six.iteritems(connection.channels):
                    for chan in channels:
                        yield connection.handle_unsubscribe({
                            "namespace": ns,
                            "channel": chan
                        })

            elif namespace_name and not channel:
                # unsubscribe from all channels in namespace
                for cat, channels in six.iteritems(connection.channels):
                    if namespace_name != cat:
                        continue
                    for chan in channels:
                        yield connection.handle_unsubscribe({
                            "namespace": namespace_name,
                            "channel": chan
                        })
                raise Return((True, None))

            else:
                # unsubscribe from certain channel
                yield connection.handle_unsubscribe({
                    "namespace": namespace_name,
                    "channel": channel
                })

        raise Return((True, None))

    @coroutine
    def handle_update_structure(self, params):
        """
        Update structure message received - structure changed and other
        node sent us a signal about update.
        """
        result, error = yield self.structure.update()
        raise Return((result, error))

    # noinspection PyCallingNonCallable
    @coroutine
    def process_call(self, project, method, params):
        """
        Call appropriate method from this class according to specified method.
        Note, that all permission checking must be done before calling this method.
        """
        handle_func = getattr(self, "process_%s" % method, None)

        if handle_func:
            result, error = yield handle_func(project, params)
            raise Return((result, error))
        else:
            raise Return((None, self.METHOD_NOT_FOUND))

    @coroutine
    def publish_message(self, message, allowed_namespaces):
        """
        Publish event into PUB socket stream
        """
        project_id = message['project_id']
        namespace_name = message['namespace']
        channel = message['channel']

        if allowed_namespaces[namespace_name]['is_watching']:
            # send to admin channel
            self.pubsub.publish_admin_message(message)

        # send to event channel
        subscription_key = self.pubsub.get_subscription_key(
            project_id, namespace_name, channel
        )

        self.pubsub.publish(subscription_key, message)

        yield self.state.add_history_message(
            project_id, namespace_name, channel, message,
            history_size=allowed_namespaces[namespace_name]['history_size']
        )

        raise Return((True, None))

    @coroutine
    def prepare_message(self, project, allowed_namespaces, params, client_id):
        """
        Prepare message before actual publishing.
        """
        namespace_name = params.get('namespace')
        namespace, error = yield self.structure.get_namespace_by_name(
            project, namespace_name
        )
        if error:
            raise Return((None, error))
        if not namespace:
            raise Return((None, self.NAMESPACE_NOT_FOUND))
        namespace_name = namespace['name']

        namespace = allowed_namespaces.get(namespace_name, None)
        if not namespace:
            raise Return(("namespace not found in allowed namespaces", None))

        data = params.get('data', None)

        message = {
            'project_id': project['_id'],
            'namespace': namespace['name'],
            'uid': uuid.uuid4().hex,
            'client_id': client_id,
            'channel': params.get('channel'),
            'data': data
        }

        for callback in self.pre_publish_callbacks:
            message, error = yield callback(message)
            if error:
                raise Return((message, error))
            if message is None:
                raise Return(('message discarded', None))

        raise Return((message, None))

    @coroutine
    def process_publish(self, project, params, allowed_namespaces=None, client_id=None):
        """
        Publish message into appropriate channel.
        """
        if allowed_namespaces is None:
            project_namespaces, error = yield self.structure.get_project_namespaces(project)
            if error:
                raise Return((None, error))

            allowed_namespaces = dict((x['name'], x) for x in project_namespaces)

        message, error = yield self.prepare_message(
            project, allowed_namespaces, params, client_id
        )
        if error:
            raise Return((None, error))

        if isinstance(message, dict):
            # event prepared for publishing
            result, error = yield self.publish_message(
                message, allowed_namespaces
            )
            if error:
                raise Return((None, error))

            for callback in self.post_publish_callbacks:
                result, error = yield callback(message)
                if error:
                    logger.error(str(error))
        else:
            # message is error description
            raise Return((None, message))

        raise Return((True, None))

    @coroutine
    def process_history(self, project, params):
        """
        Return a list of last messages sent into channel.
        """
        project_id = project['_id']

        namespace_name = params.get('namespace')
        namespace, error = yield self.structure.get_namespace_by_name(
            project, namespace_name
        )
        if error:
            raise Return((None, error))
        if not namespace:
            raise Return((None, self.NAMESPACE_NOT_FOUND))
        namespace_name = namespace['name']

        channel = params.get("channel")
        message = {
            "namespace": namespace_name,
            "channel": channel,
            "data": []
        }
        data, error = yield self.state.get_history(project_id, namespace_name, channel)
        if data:
            message['data'] = data
        raise Return((message, error))

    @coroutine
    def process_presence(self, project, params):
        """
        Return current presence information for channel.
        """
        project_id = project['_id']

        namespace_name = params.get('namespace')
        namespace, error = yield self.structure.get_namespace_by_name(
            project, namespace_name
        )
        if error:
            raise Return((None, error))
        if not namespace:
            raise Return((None, self.NAMESPACE_NOT_FOUND))
        namespace_name = namespace['name']

        channel = params.get("channel")
        message = {
            "namespace": namespace_name,
            "channel": channel,
            "data": {}
        }
        data, error = yield self.state.get_presence(project_id, namespace_name, channel)
        if data:
            message['data'] = data
        raise Return((message, error))

    @coroutine
    def process_unsubscribe(self, project, params):
        """
        Unsubscribe user from channels.
        """
        params["project"] = project
        message = {
            'app_id': self.uid,
            'method': 'unsubscribe',
            'params': params
        }

        # handle on this node
        result, error = yield self.handle_unsubscribe(params)

        # send to other nodes
        self.send_control_message(message)

        raise Return((result, error))

    @coroutine
    def process_dump_structure(self, project, params):

        projects, error = yield self.structure.project_list()
        if error:
            raise Return((None, self.INTERNAL_SERVER_ERROR))

        namespaces, error = yield self.structure.namespace_list()
        if error:
            raise Return((None, self.INTERNAL_SERVER_ERROR))

        data = {
            "projects": projects,
            "namespaces": namespaces
        }
        raise Return((data, None))

    @coroutine
    def process_project_list(self, project, params):
        projects, error = yield self.structure.project_list()
        if error:
            raise Return((None, self.INTERNAL_SERVER_ERROR))
        raise Return((projects, None))

    @coroutine
    def process_project_get(self, project, params):
        if not project:
            raise Return((None, self.PROJECT_NOT_FOUND))
        raise Return((project, None))

    @coroutine
    def process_project_by_name(self, project, params):
        project, error = yield self.structure.get_project_by_name(
            params.get("name")
        )
        if error:
            raise Return((None, self.INTERNAL_SERVER_ERROR))
        if not project:
            raise Return((None, self.PROJECT_NOT_FOUND))
        raise Return((project, None))

    @coroutine
    def process_project_create(self, project, params, error_form=False):

        form = ProjectForm(params)

        if form.validate():
            existing_project, error = yield self.structure.get_project_by_name(
                form.name.data
            )
            if error:
                raise Return((None, self.INTERNAL_SERVER_ERROR))

            if existing_project:
                form.name.errors.append(self.DUPLICATE_NAME)
                if error_form:
                    raise Return((None, form))
                raise Return((None, form.errors))
            else:
                project, error = yield self.structure.project_create(
                    **form.data
                )
                if error:
                    raise Return((None, self.INTERNAL_SERVER_ERROR))
                raise Return((project, None))
        else:
            if error_form:
                raise Return((None, form))
            raise Return((None, form.errors))

    @coroutine
    def process_project_edit(self, project, params, error_form=False):
        """
        Edit project namespace.
        """
        if not project:
            raise Return((None, self.PROJECT_NOT_FOUND))

        if "name" not in params:
            params["name"] = project["name"]

        namespaces, error = yield self.structure.get_project_namespaces(project)
        if error:
            raise Return((None, self.INTERNAL_SERVER_ERROR))

        namespace_choices = [(x['_id'], x['name']) for x in namespaces]
        namespace_choices.insert(0, ('', ''))
        form = ProjectForm(params, namespace_choices=namespace_choices)

        if form.validate():

            if "name" in params and params["name"] != project["name"]:

                existing_project, error = yield self.structure.get_project_by_name(
                    params["name"]
                )
                if error:
                    raise Return((None, self.INTERNAL_SERVER_ERROR))
                if existing_project:
                    form.name.errors.append(self.DUPLICATE_NAME)
                    if error_form:
                        raise Return((None, form))
                    raise Return((None, form.errors))

            updated_project = project.copy()
            updated_project.update(form.data)
            project, error = yield self.structure.project_edit(
                project, **updated_project
            )
            if error:
                raise Return((None, self.INTERNAL_SERVER_ERROR))
            raise Return((project, None))
        else:
            if error_form:
                raise Return((None, form))
            raise Return((None, form.errors))

    @coroutine
    def process_project_delete(self, project, params):
        if not project:
            raise Return((None, self.PROJECT_NOT_FOUND))
        result, error = yield self.structure.project_delete(project)
        if error:
            raise Return((None, self.INTERNAL_SERVER_ERROR))
        raise Return((True, None))

    @coroutine
    def process_regenerate_secret_key(self, project, params):
        if not project:
            raise Return((None, self.PROJECT_NOT_FOUND))
        result, error = yield self.structure.regenerate_project_secret_key(project)
        if error:
            raise Return((None, self.INTERNAL_SERVER_ERROR))
        raise Return((result, None))

    @coroutine
    def process_namespace_list(self, project, params):
        """
        Return a list of all namespaces for project.
        """
        if not project:
            raise Return((None, self.PROJECT_NOT_FOUND))
        namespaces, error = yield self.structure.get_project_namespaces(project)
        if error:
            raise Return((None, self.INTERNAL_SERVER_ERROR))
        raise Return((namespaces, None))

    @coroutine
    def process_namespace_get(self, project, params):
        """
        Return a list of all namespaces for project.
        """
        namespace_id = params.get('_id')
        namespace, error = yield self.structure.get_namespace_by_id(namespace_id)
        if error:
            raise Return((None, self.INTERNAL_SERVER_ERROR))
        if not namespace:
            raise Return((None, self.NAMESPACE_NOT_FOUND))
        raise Return((namespace, None))

    @coroutine
    def process_namespace_by_name(self, project, params):
        if not project:
            raise Return((None, self.PROJECT_NOT_FOUND))

        namespace, error = yield self.structure.get_namespace_by_name(
            project, params.get("name")
        )
        if error:
            raise Return((None, self.INTERNAL_SERVER_ERROR))
        if not namespace:
            raise Return((None, self.NAMESPACE_NOT_FOUND))
        raise Return((namespace, None))

    @coroutine
    def process_namespace_create(self, project, params, error_form=False):
        """
        Create new namespace in project or update if already exists.
        """
        if not project:
            raise Return((None, self.PROJECT_NOT_FOUND))

        form = NamespaceForm(params)

        if form.validate():
            existing_namespace, error = yield self.structure.get_namespace_by_name(
                project, form.name.data
            )
            if error:
                raise Return((None, self.INTERNAL_SERVER_ERROR))

            if existing_namespace:
                form.name.errors.append(self.DUPLICATE_NAME)
                if error_form:
                    raise Return((None, form))
                raise Return((None, form.errors))
            else:
                namespace, error = yield self.structure.namespace_create(
                    project, **form.data
                )
                if error:
                    raise Return((None, self.INTERNAL_SERVER_ERROR))
                raise Return((namespace, None))
        else:
            if error_form:
                raise Return((None, form))
            raise Return((None, form.errors))

    @coroutine
    def process_namespace_edit(self, project, params, error_form=False):
        """
        Edit project namespace.
        """
        namespace, error = yield self.structure.get_namespace_by_id(
            params.pop('_id')
        )
        if error:
            raise Return((None, self.INTERNAL_SERVER_ERROR))

        if not namespace:
            raise Return((None, self.NAMESPACE_NOT_FOUND))

        if not project:
            project, error = yield self.structure.get_project_by_id(
                namespace['project_id']
            )
            if error:
                raise Return((None, self.INTERNAL_SERVER_ERROR))
            if not project:
                raise Return((None, self.PROJECT_NOT_FOUND))

        if "name" not in params:
            params["name"] = namespace["name"]

        form = NamespaceForm(params)

        if form.validate():

            if "name" in params and params["name"] != namespace["name"]:

                existing_namespace, error = yield self.structure.get_namespace_by_name(
                    project, params["name"]
                )
                if error:
                    raise Return((None, self.INTERNAL_SERVER_ERROR))
                if existing_namespace:
                    form.name.errors.append(self.DUPLICATE_NAME)
                    if error_form:
                        raise Return((None, form))
                    raise Return((None, form.errors))

            updated_namespace = namespace.copy()
            updated_namespace.update(form.data)
            namespace, error = yield self.structure.namespace_edit(
                namespace, **updated_namespace
            )
            if error:
                raise Return((None, self.INTERNAL_SERVER_ERROR))
            raise Return((namespace, None))
        else:
            if error_form:
                raise Return((None, form))
            raise Return((None, form.errors))

    @coroutine
    def process_namespace_delete(self, project, params):
        """
        Delete project namespace.
        """
        namespace_id = params["_id"]

        existing_namespace, error = yield self.structure.get_namespace_by_id(namespace_id)
        if error:
            raise Return((None, self.INTERNAL_SERVER_ERROR))
        if not existing_namespace:
            raise Return((None, self.NAMESPACE_NOT_FOUND))

        result, error = yield self.structure.namespace_delete(namespace_id)
        if error:
            raise Return((None, self.INTERNAL_SERVER_ERROR))
        raise Return((True, None))
Exemple #4
0
class Application(tornado.web.Application):

    # magic fake project ID for owner API purposes.
    MAGIC_PROJECT_ID = '_'

    # magic project param name to allow owner make API operations within project
    MAGIC_PROJECT_PARAM = '_project'

    # in milliseconds, how often this application will send ping message
    PING_INTERVAL = 5000

    # in seconds
    PING_MAX_DELAY = 10

    # in milliseconds, how often application will remove stale ping information
    PING_REVIEW_INTERVAL = 10000

    UNAUTHORIZED = 'unauthorized'

    PERMISSION_DENIED = 'permission denied'

    INTERNAL_SERVER_ERROR = 'internal server error'

    METHOD_NOT_FOUND = 'method not found'

    PROJECT_NOT_FOUND = 'project not found'

    NAMESPACE_NOT_FOUND = 'namespace not found'

    DUPLICATE_NAME = 'duplicate name'

    def __init__(self, *args, **kwargs):

        # create unique uid for this application
        self.uid = uuid.uuid4().hex

        # PUB/SUB manager class
        self.pubsub = BasePubSub(self)

        # initialize dict to keep administrator's connections
        self.admin_connections = {}

        # initialize dict to keep client's connections
        self.connections = {}

        # dict to keep ping from nodes
        # key - node address, value - timestamp of last ping
        self.nodes = {}

        # application structure manager (projects, namespaces etc)
        self.structure = None

        # application state manager
        self.state = None

        # initialize dict to keep back-off information for projects
        self.back_off = {}

        # list of coroutines that must be done before message publishing
        self.pre_publish_callbacks = []

        # list of coroutines that must be done after message publishing
        self.post_publish_callbacks = []

        # initialize tornado's application
        super(Application, self).__init__(*args, **kwargs)

    def initialize(self):
        self.init_callbacks()
        self.init_structure()
        self.init_pubsub()
        self.init_state()
        self.init_ping()

    def init_structure(self):
        """
        Initialize structure manager using settings provided
        in configuration file.
        """
        custom_settings = self.settings['config']
        structure_settings = custom_settings.get('structure', {})

        # detect and apply database storage module
        storage_module = structure_settings.get(
            'storage', 'centrifuge.structure.sqlite'
        )
        storage = utils.import_module(storage_module)

        structure = Structure(self)
        structure.set_storage(storage)
        self.structure = structure

        def run_periodic_structure_update():
            structure.update()
            periodic_structure_update = tornado.ioloop.PeriodicCallback(
                structure.update, structure_settings.get('update_interval', 30)*1000
            )
            periodic_structure_update.start()

        tornado.ioloop.IOLoop.instance().add_callback(
            partial(
                storage.init_storage,
                structure,
                structure_settings.get('settings', {}),
                run_periodic_structure_update
            )
        )

        logger.info("Storage module: {0}".format(storage_module))

    def init_state(self):
        """
        Initialize state manager (for presence/history data).
        """
        config = self.settings['config']
        state_config = config.get("state", {})
        if not state_config:
            # use base fake state
            logger.info("No State configured")
            self.state = State(self, fake=True)
        else:
            state_storage = state_config.get('storage', 'centrifuge.state.base.State')
            state_storage_class = utils.namedAny(state_storage)
            self.state = state_storage_class(self)
            tornado.ioloop.IOLoop.instance().add_callback(self.state.initialize)

    def init_pubsub(self):
        """
        Initialize and configure pub/sub manager.
        """
        self.pubsub.initialize()

    def init_callbacks(self):
        """
        Fill custom callbacks with callable objects provided in config.
        """
        config = self.settings['config']
        pre_publish_callbacks = config.get('pre_publish_callbacks', [])

        for callable_path in pre_publish_callbacks:
            callback = utils.namedAny(callable_path)
            self.pre_publish_callbacks.append(callback)

        post_publish_callbacks = config.get('post_publish_callbacks', [])
        for callable_path in post_publish_callbacks:
            callback = utils.namedAny(callable_path)
            self.post_publish_callbacks.append(callback)

    def send_ping(self, ping_message):
        self.pubsub.publish_control_message(ping_message)

    def review_ping(self):
        """
        Remove outdated information about other nodes.
        """
        now = time.time()
        outdated = []
        for node, updated_at in self.nodes.items():
            if now - updated_at > self.PING_MAX_DELAY:
                outdated.append(node)
        for node in outdated:
            try:
                del self.nodes[node]
            except KeyError:
                pass

    def init_ping(self):
        """
        Start periodic tasks for sending ping and reviewing ping.
        """
        message = {
            'app_id': self.uid,
            'method': 'ping',
            'params': {'uid': self.uid}
        }
        send_ping = partial(self.pubsub.publish_control_message, message)
        ping = tornado.ioloop.PeriodicCallback(send_ping, self.PING_INTERVAL)
        tornado.ioloop.IOLoop.instance().add_timeout(
            self.PING_INTERVAL, ping.start
        )

        review_ping = tornado.ioloop.PeriodicCallback(self.review_ping, self.PING_REVIEW_INTERVAL)
        tornado.ioloop.IOLoop.instance().add_timeout(
            self.PING_INTERVAL, review_ping.start
        )

    def send_control_message(self, message):
        """
        Send message to CONTROL channel. We use this channel to
        share commands between running instances.
        """
        self.pubsub.publish_control_message(message)

    def add_connection(self, project_id, user, uid, client):
        """
        Register new client's connection.
        """
        if project_id not in self.connections:
            self.connections[project_id] = {}
        if user and user not in self.connections:
            self.connections[project_id][user] = {}
        if user:
            self.connections[project_id][user][uid] = client

    def remove_connection(self, project_id, user, uid):
        """
        Remove client's connection
        """
        try:
            del self.connections[project_id][user][uid]
        except KeyError:
            pass

        if project_id in self.connections and user in self.connections[project_id]:
            # clean connections
            if self.connections[project_id][user]:
                return
            try:
                del self.connections[project_id][user]
            except KeyError:
                pass
            if self.connections[project_id]:
                return
            try:
                del self.connections[project_id]
            except KeyError:
                pass

    def add_admin_connection(self, uid, client):
        """
        Register administrator's connection (from web-interface).
        """
        self.admin_connections[uid] = client

    def remove_admin_connection(self, uid):
        """
        Remove administrator's connection.
        """
        try:
            del self.admin_connections[uid]
        except KeyError:
            pass

    @coroutine
    def handle_ping(self, params):
        """
        Ping message received.
        """
        self.nodes[params.get('uid')] = time.time()

    @coroutine
    def handle_unsubscribe(self, params):
        """
        Unsubscribe message received - unsubscribe client from certain channels.
        """
        project = params.get("project")
        user = params.get("user")
        namespace_name = params.get("namespace", None)
        channel = params.get("channel", None)

        if not user:
            # we don't need to block anonymous users
            raise Return((True, None))

        project_id = project['_id']

        # try to find user's connection
        user_connections = self.connections.get(project_id, {}).get(user, None)
        if not user_connections:
            raise Return((True, None))

        namespace, error = yield self.structure.get_namespace_by_name(
            project, namespace_name
        )
        if error:
            raise Return((None, error))
        if channel and not namespace:
            # namespace does not exist
            raise Return((True, None))

        namespace_name = namespace['name']

        for uid, connection in six.iteritems(user_connections):

            if not namespace_name and not channel:
                # unsubscribe from all channels
                for ns, channels in six.iteritems(connection.channels):
                    for chan in channels:
                        yield connection.handle_unsubscribe({
                            "namespace": ns,
                            "channel": chan
                        })

            elif namespace_name and not channel:
                # unsubscribe from all channels in namespace
                for cat, channels in six.iteritems(connection.channels):
                    if namespace_name != cat:
                        continue
                    for chan in channels:
                        yield connection.handle_unsubscribe({
                            "namespace": namespace_name,
                            "channel": chan
                        })
                raise Return((True, None))

            else:
                # unsubscribe from certain channel
                yield connection.handle_unsubscribe({
                    "namespace": namespace_name,
                    "channel": channel
                })

        raise Return((True, None))

    @coroutine
    def handle_update_structure(self, params):
        """
        Update structure message received - structure changed and other
        node sent us a signal about update.
        """
        result, error = yield self.structure.update()
        raise Return((result, error))

    # noinspection PyCallingNonCallable
    @coroutine
    def process_call(self, project, method, params):
        """
        Call appropriate method from this class according to specified method.
        Note, that all permission checking must be done before calling this method.
        """
        handle_func = getattr(self, "process_%s" % method, None)

        if handle_func:
            result, error = yield handle_func(project, params)
            raise Return((result, error))
        else:
            raise Return((None, self.METHOD_NOT_FOUND))

    @coroutine
    def publish_message(self, message, allowed_namespaces):
        """
        Publish event into PUB socket stream
        """
        project_id = message['project_id']
        namespace_name = message['namespace']
        channel = message['channel']

        if allowed_namespaces[namespace_name]['is_watching']:
            # send to admin channel
            self.pubsub.publish_admin_message(message)

        # send to event channel
        subscription_key = self.pubsub.get_subscription_key(
            project_id, namespace_name, channel
        )

        self.pubsub.publish(subscription_key, message)

        yield self.state.add_history_message(
            project_id, namespace_name, channel, message,
            history_size=allowed_namespaces[namespace_name]['history_size']
        )

        raise Return((True, None))

    @coroutine
    def prepare_message(self, project, allowed_namespaces, params, client):
        """
        Prepare message before actual publishing.
        """
        namespace_name = params.get('namespace')
        namespace, error = yield self.structure.get_namespace_by_name(
            project, namespace_name
        )
        if error:
            raise Return((None, error))
        if not namespace:
            raise Return((None, self.NAMESPACE_NOT_FOUND))
        namespace_name = namespace['name']

        namespace = allowed_namespaces.get(namespace_name, None)
        if not namespace:
            raise Return(("namespace not found in allowed namespaces", None))

        data = params.get('data', None)

        message = {
            'project_id': project['_id'],
            'namespace': namespace['name'],
            'uid': uuid.uuid4().hex,
            'timestamp': int(time.time()),
            'client': client,
            'channel': params.get('channel'),
            'data': data
        }

        for callback in self.pre_publish_callbacks:
            message, error = yield callback(message)
            if error:
                raise Return((message, error))
            if message is None:
                raise Return(('message discarded', None))

        raise Return((message, None))

    @coroutine
    def process_publish(self, project, params, allowed_namespaces=None, client=None):
        """
        Publish message into appropriate channel.
        """
        if allowed_namespaces is None:
            project_namespaces, error = yield self.structure.get_project_namespaces(project)
            if error:
                raise Return((None, error))

            allowed_namespaces = dict((x['name'], x) for x in project_namespaces)

        message, error = yield self.prepare_message(
            project, allowed_namespaces, params, client
        )
        if error:
            raise Return((None, error))

        if isinstance(message, dict):
            # event prepared for publishing
            result, error = yield self.publish_message(
                message, allowed_namespaces
            )
            if error:
                raise Return((None, error))

            for callback in self.post_publish_callbacks:
                result, error = yield callback(message)
                if error:
                    logger.error(str(error))
        else:
            # message is error description
            raise Return((None, message))

        raise Return((True, None))

    @coroutine
    def process_history(self, project, params):
        """
        Return a list of last messages sent into channel.
        """
        project_id = project['_id']

        namespace_name = params.get('namespace')
        namespace, error = yield self.structure.get_namespace_by_name(
            project, namespace_name
        )
        if error:
            raise Return((None, error))
        if not namespace:
            raise Return((None, self.NAMESPACE_NOT_FOUND))
        namespace_name = namespace['name']

        channel = params.get("channel")
        message = {
            "namespace": namespace_name,
            "channel": channel,
            "data": []
        }
        data, error = yield self.state.get_history(project_id, namespace_name, channel)
        if data:
            message['data'] = data
        raise Return((message, error))

    @coroutine
    def process_presence(self, project, params):
        """
        Return current presence information for channel.
        """
        project_id = project['_id']

        namespace_name = params.get('namespace')
        namespace, error = yield self.structure.get_namespace_by_name(
            project, namespace_name
        )
        if error:
            raise Return((None, error))
        if not namespace:
            raise Return((None, self.NAMESPACE_NOT_FOUND))
        namespace_name = namespace['name']

        channel = params.get("channel")
        message = {
            "namespace": namespace_name,
            "channel": channel,
            "data": {}
        }
        data, error = yield self.state.get_presence(project_id, namespace_name, channel)
        if data:
            message['data'] = data
        raise Return((message, error))

    @coroutine
    def process_unsubscribe(self, project, params):
        """
        Unsubscribe user from channels.
        """
        params["project"] = project
        message = {
            'app_id': self.uid,
            'method': 'unsubscribe',
            'params': params
        }

        # handle on this node
        result, error = yield self.handle_unsubscribe(params)

        # send to other nodes
        self.send_control_message(message)

        raise Return((result, error))

    @coroutine
    def process_dump_structure(self, project, params):

        projects, error = yield self.structure.project_list()
        if error:
            raise Return((None, self.INTERNAL_SERVER_ERROR))

        namespaces, error = yield self.structure.namespace_list()
        if error:
            raise Return((None, self.INTERNAL_SERVER_ERROR))

        data = {
            "projects": projects,
            "namespaces": namespaces
        }
        raise Return((data, None))

    @coroutine
    def process_project_list(self, project, params):
        projects, error = yield self.structure.project_list()
        if error:
            raise Return((None, self.INTERNAL_SERVER_ERROR))
        raise Return((projects, None))

    @coroutine
    def process_project_get(self, project, params):
        if not project:
            raise Return((None, self.PROJECT_NOT_FOUND))
        raise Return((project, None))

    @coroutine
    def process_project_by_name(self, project, params):
        project, error = yield self.structure.get_project_by_name(
            params.get("name")
        )
        if error:
            raise Return((None, self.INTERNAL_SERVER_ERROR))
        if not project:
            raise Return((None, self.PROJECT_NOT_FOUND))
        raise Return((project, None))

    @coroutine
    def process_project_create(self, project, params, error_form=False):

        form = ProjectForm(params)

        if form.validate():
            existing_project, error = yield self.structure.get_project_by_name(
                form.name.data
            )
            if error:
                raise Return((None, self.INTERNAL_SERVER_ERROR))

            if existing_project:
                form.name.errors.append(self.DUPLICATE_NAME)
                if error_form:
                    raise Return((None, form))
                raise Return((None, form.errors))
            else:
                project, error = yield self.structure.project_create(
                    **form.data
                )
                if error:
                    raise Return((None, self.INTERNAL_SERVER_ERROR))
                raise Return((project, None))
        else:
            if error_form:
                raise Return((None, form))
            raise Return((None, form.errors))

    @coroutine
    def process_project_edit(self, project, params, error_form=False):
        """
        Edit project namespace.
        """
        if not project:
            raise Return((None, self.PROJECT_NOT_FOUND))

        if "name" not in params:
            params["name"] = project["name"]

        namespaces, error = yield self.structure.get_project_namespaces(project)
        if error:
            raise Return((None, self.INTERNAL_SERVER_ERROR))

        namespace_choices = [(x['_id'], x['name']) for x in namespaces]
        namespace_choices.insert(0, ('', ''))
        form = ProjectForm(params, namespace_choices=namespace_choices)

        if form.validate():

            if "name" in params and params["name"] != project["name"]:

                existing_project, error = yield self.structure.get_project_by_name(
                    params["name"]
                )
                if error:
                    raise Return((None, self.INTERNAL_SERVER_ERROR))
                if existing_project:
                    form.name.errors.append(self.DUPLICATE_NAME)
                    if error_form:
                        raise Return((None, form))
                    raise Return((None, form.errors))

            updated_project = project.copy()
            updated_project.update(form.data)
            project, error = yield self.structure.project_edit(
                project, **updated_project
            )
            if error:
                raise Return((None, self.INTERNAL_SERVER_ERROR))
            raise Return((project, None))
        else:
            if error_form:
                raise Return((None, form))
            raise Return((None, form.errors))

    @coroutine
    def process_project_delete(self, project, params):
        if not project:
            raise Return((None, self.PROJECT_NOT_FOUND))
        result, error = yield self.structure.project_delete(project)
        if error:
            raise Return((None, self.INTERNAL_SERVER_ERROR))
        raise Return((True, None))

    @coroutine
    def process_regenerate_secret_key(self, project, params):
        if not project:
            raise Return((None, self.PROJECT_NOT_FOUND))
        result, error = yield self.structure.regenerate_project_secret_key(project)
        if error:
            raise Return((None, self.INTERNAL_SERVER_ERROR))
        raise Return((result, None))

    @coroutine
    def process_namespace_list(self, project, params):
        """
        Return a list of all namespaces for project.
        """
        if not project:
            raise Return((None, self.PROJECT_NOT_FOUND))
        namespaces, error = yield self.structure.get_project_namespaces(project)
        if error:
            raise Return((None, self.INTERNAL_SERVER_ERROR))
        raise Return((namespaces, None))

    @coroutine
    def process_namespace_get(self, project, params):
        """
        Return a list of all namespaces for project.
        """
        namespace_id = params.get('_id')
        namespace, error = yield self.structure.get_namespace_by_id(namespace_id)
        if error:
            raise Return((None, self.INTERNAL_SERVER_ERROR))
        if not namespace:
            raise Return((None, self.NAMESPACE_NOT_FOUND))
        raise Return((namespace, None))

    @coroutine
    def process_namespace_by_name(self, project, params):
        if not project:
            raise Return((None, self.PROJECT_NOT_FOUND))

        namespace, error = yield self.structure.get_namespace_by_name(
            project, params.get("name")
        )
        if error:
            raise Return((None, self.INTERNAL_SERVER_ERROR))
        if not namespace:
            raise Return((None, self.NAMESPACE_NOT_FOUND))
        raise Return((namespace, None))

    @coroutine
    def process_namespace_create(self, project, params, error_form=False):
        """
        Create new namespace in project or update if already exists.
        """
        if not project:
            raise Return((None, self.PROJECT_NOT_FOUND))

        form = NamespaceForm(params)

        if form.validate():
            existing_namespace, error = yield self.structure.get_namespace_by_name(
                project, form.name.data
            )
            if error:
                raise Return((None, self.INTERNAL_SERVER_ERROR))

            if existing_namespace:
                form.name.errors.append(self.DUPLICATE_NAME)
                if error_form:
                    raise Return((None, form))
                raise Return((None, form.errors))
            else:
                namespace, error = yield self.structure.namespace_create(
                    project, **form.data
                )
                if error:
                    raise Return((None, self.INTERNAL_SERVER_ERROR))
                raise Return((namespace, None))
        else:
            if error_form:
                raise Return((None, form))
            raise Return((None, form.errors))

    @coroutine
    def process_namespace_edit(self, project, params, error_form=False):
        """
        Edit project namespace.
        """
        namespace, error = yield self.structure.get_namespace_by_id(
            params.pop('_id')
        )
        if error:
            raise Return((None, self.INTERNAL_SERVER_ERROR))

        if not namespace:
            raise Return((None, self.NAMESPACE_NOT_FOUND))

        if not project:
            project, error = yield self.structure.get_project_by_id(
                namespace['project_id']
            )
            if error:
                raise Return((None, self.INTERNAL_SERVER_ERROR))
            if not project:
                raise Return((None, self.PROJECT_NOT_FOUND))

        if "name" not in params:
            params["name"] = namespace["name"]

        form = NamespaceForm(params)

        if form.validate():

            if "name" in params and params["name"] != namespace["name"]:

                existing_namespace, error = yield self.structure.get_namespace_by_name(
                    project, params["name"]
                )
                if error:
                    raise Return((None, self.INTERNAL_SERVER_ERROR))
                if existing_namespace:
                    form.name.errors.append(self.DUPLICATE_NAME)
                    if error_form:
                        raise Return((None, form))
                    raise Return((None, form.errors))

            updated_namespace = namespace.copy()
            updated_namespace.update(form.data)
            namespace, error = yield self.structure.namespace_edit(
                namespace, **updated_namespace
            )
            if error:
                raise Return((None, self.INTERNAL_SERVER_ERROR))
            raise Return((namespace, None))
        else:
            if error_form:
                raise Return((None, form))
            raise Return((None, form.errors))

    @coroutine
    def process_namespace_delete(self, project, params):
        """
        Delete project namespace.
        """
        namespace_id = params["_id"]

        existing_namespace, error = yield self.structure.get_namespace_by_id(namespace_id)
        if error:
            raise Return((None, self.INTERNAL_SERVER_ERROR))
        if not existing_namespace:
            raise Return((None, self.NAMESPACE_NOT_FOUND))

        result, error = yield self.structure.namespace_delete(namespace_id)
        if error:
            raise Return((None, self.INTERNAL_SERVER_ERROR))
        raise Return((True, None))
Exemple #5
0
class Application(tornado.web.Application):

    # magic fake project ID for owner API purposes.
    MAGIC_PROJECT_ID = "_"

    # magic project param name to allow owner make API operations within project
    MAGIC_PROJECT_PARAM = "_project"

    # in milliseconds, how often this application will send ping message
    PING_INTERVAL = 5000

    # in seconds
    PING_MAX_DELAY = 10

    # in milliseconds, how often node will send its info into admin channel
    NODE_INFO_PUBLISH_INTERVAL = 10000

    # in milliseconds, how often application will remove stale ping information
    PING_REVIEW_INTERVAL = 10000

    UNAUTHORIZED = "unauthorized"

    PERMISSION_DENIED = "permission denied"

    INTERNAL_SERVER_ERROR = "internal server error"

    METHOD_NOT_FOUND = "method not found"

    PROJECT_NOT_FOUND = "project not found"

    NAMESPACE_NOT_FOUND = "namespace not found"

    DUPLICATE_NAME = "duplicate name"

    def __init__(self, *args, **kwargs):

        # create unique uid for this application
        self.uid = uuid.uuid4().hex

        # PUB/SUB manager class
        self.pubsub = BasePubSub(self)

        # initialize dict to keep administrator's connections
        self.admin_connections = {}

        # dictionary to keep client's connections
        self.connections = {}

        # dictionary to keep ping from nodes
        self.nodes = {}

        # application structure manager (projects, namespaces etc)
        self.structure = None

        # application state manager
        self.state = None

        # initialize dict to keep back-off information for projects
        self.back_off = {}

        # list of coroutines that must be done before message publishing
        self.pre_publish_callbacks = []

        # list of coroutines that must be done after message publishing
        self.post_publish_callbacks = []

        # periodic task for sending current node information into admin channel
        self.periodic_node_info = None

        # time of last node info revision
        self.node_info_revision_time = time.time()

        # count of messages published since last node info revision
        self.messages_published = 0

        # initialize tornado's application
        super(Application, self).__init__(*args, **kwargs)

    def get_node_info(self):
        current_time = time.time()
        msg_per_sec = float(self.messages_published) / (current_time - self.node_info_revision_time)
        info = {
            "uid": self.uid,
            "address": get_host(),
            "port": str(self.settings["options"].port),
            "nodes": len(self.nodes) + 1,
            "channels": len(self.pubsub.subscriptions),
            "clients": sum(len(v) for v in six.itervalues(self.pubsub.subscriptions)),
            "unique_clients": sum(len(v) for v in six.itervalues(self.connections)),
            "messages_per_second": "%0.2f" % msg_per_sec,
        }
        self.messages_published = 0
        self.node_info_revision_time = current_time
        return info

    def initialize(self):
        self.init_callbacks()
        self.init_structure()
        self.init_pubsub()
        self.init_state()
        self.init_ping()
        self.init_periodic_tasks()

    def init_structure(self):
        """
        Initialize structure manager using settings provided
        in configuration file.
        """
        custom_settings = self.settings["config"]
        structure_settings = custom_settings.get("structure", {})

        # detect and apply database storage module
        storage_backend = structure_settings.get("storage", "centrifuge.structure.sqlite.Storage")
        storage_backend_class = utils.namedAny(storage_backend)
        logger.info("Storage module: {0}".format(storage_backend))

        self.structure = Structure(self)
        storage = storage_backend_class(self.structure, structure_settings.get("settings", {}))
        self.structure.set_storage(storage)

        def run_periodic_structure_update():
            # update structure periodically from database. This is necessary to be sure
            # that application has actual and correct structure information. Structure
            # updates also triggered in real-time by message passing through control channel,
            # but in rare cases those update messages can be lost because of some kind of
            # network errors
            logger.info("Structure storage connected")
            self.structure.update()
            periodic_structure_update = tornado.ioloop.PeriodicCallback(
                self.structure.update, structure_settings.get("update_interval", 30) * 1000
            )
            periodic_structure_update.start()

        tornado.ioloop.IOLoop.instance().add_callback(partial(storage.connect, run_periodic_structure_update))

    def init_state(self):
        """
        Initialize state manager (for presence/history data).
        """
        config = self.settings["config"]
        state_config = config.get("state", {})
        if not state_config:
            # use fake state
            logger.info("No State configured")
            self.state = State(self, fake=True)
        else:
            state_storage = state_config.get("storage", "centrifuge.state.base.State")
            state_storage_class = utils.namedAny(state_storage)
            self.state = state_storage_class(self)
            tornado.ioloop.IOLoop.instance().add_callback(self.state.initialize)

    def init_pubsub(self):
        """
        Initialize and configure pub/sub manager.
        """
        self.pubsub.initialize()

    def init_callbacks(self):
        """
        Fill custom callbacks with callable objects provided in config.
        """
        config = self.settings["config"]

        pre_publish_callbacks = config.get("pre_publish_callbacks", [])
        for callable_path in pre_publish_callbacks:
            callback = utils.namedAny(callable_path)
            self.pre_publish_callbacks.append(callback)

        post_publish_callbacks = config.get("post_publish_callbacks", [])
        for callable_path in post_publish_callbacks:
            callback = utils.namedAny(callable_path)
            self.post_publish_callbacks.append(callback)

    def init_periodic_tasks(self):
        """
        Start different periodic tasks here
        """
        self.periodic_node_info = tornado.ioloop.PeriodicCallback(
            self.publish_node_info, self.NODE_INFO_PUBLISH_INTERVAL
        )
        self.periodic_node_info.start()

    def send_ping(self, ping_message):
        self.pubsub.publish_control_message(ping_message)

    def review_ping(self):
        """
        Remove outdated information about other nodes.
        """
        now = time.time()
        outdated = []
        for node, params in self.nodes.items():
            updated_at = params["updated_at"]
            if now - updated_at > self.PING_MAX_DELAY:
                outdated.append(node)
        for node in outdated:
            try:
                del self.nodes[node]
            except KeyError:
                pass

    def init_ping(self):
        """
        Start periodic tasks for sending ping and reviewing ping.
        """
        message = {"app_id": self.uid, "method": "ping", "params": self.get_node_info()}
        send_ping = partial(self.pubsub.publish_control_message, message)
        ping = tornado.ioloop.PeriodicCallback(send_ping, self.PING_INTERVAL)
        tornado.ioloop.IOLoop.instance().add_timeout(self.PING_INTERVAL, ping.start)

        review_ping = tornado.ioloop.PeriodicCallback(self.review_ping, self.PING_REVIEW_INTERVAL)
        tornado.ioloop.IOLoop.instance().add_timeout(self.PING_INTERVAL, review_ping.start)

    def publish_node_info(self):
        """
        Publish information about current node into admin channel
        """
        self.send_admin_message({"admin": True, "type": "node", "data": self.get_node_info()})

    def send_control_message(self, message):
        """
        Send message to CONTROL channel. We use this channel to
        share commands between running instances.
        """
        self.pubsub.publish_control_message(message)

    def send_admin_message(self, message):
        """
        Send message to ADMIN channel. We use this channel to
        send events to administrative interface.
        """
        self.pubsub.publish_admin_message(message)

    def add_connection(self, project_id, user, uid, client):
        """
        Register new client's connection.
        """
        if project_id not in self.connections:
            self.connections[project_id] = {}
        if user and user not in self.connections[project_id]:
            self.connections[project_id][user] = {}
        if user:
            self.connections[project_id][user][uid] = client

    def remove_connection(self, project_id, user, uid):
        """
        Remove client's connection
        """
        try:
            del self.connections[project_id][user][uid]
        except KeyError:
            pass

        if project_id in self.connections and user in self.connections[project_id]:
            # clean connections
            if self.connections[project_id][user]:
                return
            try:
                del self.connections[project_id][user]
            except KeyError:
                pass
            if self.connections[project_id]:
                return
            try:
                del self.connections[project_id]
            except KeyError:
                pass

    def add_admin_connection(self, uid, client):
        """
        Register administrator's connection (from web-interface).
        """
        self.admin_connections[uid] = client

    def remove_admin_connection(self, uid):
        """
        Remove administrator's connection.
        """
        try:
            del self.admin_connections[uid]
        except KeyError:
            pass

    @coroutine
    def handle_ping(self, params):
        """
        Ping message received.
        """
        params["updated_at"] = time.time()
        self.nodes[params.get("uid")] = params

    @coroutine
    def handle_unsubscribe(self, params):
        """
        Unsubscribe message received - unsubscribe client from certain channels.
        """
        project = params.get("project")
        user = params.get("user")
        namespace_name = params.get("namespace", None)
        channel = params.get("channel", None)

        if not user:
            # we don't need to block anonymous users
            raise Return((True, None))

        project_id = project["_id"]

        # try to find user's connection
        user_connections = self.connections.get(project_id, {}).get(user, None)
        if not user_connections:
            raise Return((True, None))

        namespace, error = yield self.structure.get_namespace_by_name(project, namespace_name)
        if error:
            raise Return((None, error))
        if channel and not namespace:
            # namespace does not exist
            raise Return((True, None))

        namespace_name = namespace["name"]

        for uid, connection in six.iteritems(user_connections):

            if not namespace_name and not channel:
                # unsubscribe from all channels
                for ns, channels in six.iteritems(connection.channels):
                    for chan in channels:
                        yield connection.handle_unsubscribe({"namespace": ns, "channel": chan})

            elif namespace_name and not channel:
                # unsubscribe from all channels in namespace
                for cat, channels in six.iteritems(connection.channels):
                    if namespace_name != cat:
                        continue
                    for chan in channels:
                        yield connection.handle_unsubscribe({"namespace": namespace_name, "channel": chan})
                raise Return((True, None))

            else:
                # unsubscribe from certain channel
                yield connection.handle_unsubscribe({"namespace": namespace_name, "channel": channel})

        raise Return((True, None))

    @coroutine
    def handle_update_structure(self, params):
        """
        Update structure message received - structure changed and other
        node sent us a signal about update.
        """
        result, error = yield self.structure.update()
        raise Return((result, error))

    # noinspection PyCallingNonCallable
    @coroutine
    def process_call(self, project, method, params):
        """
        Call appropriate method from this class according to specified method.
        Note, that all permission checking must be done before calling this method.
        """
        handle_func = getattr(self, "process_%s" % method, None)

        if handle_func:
            result, error = yield handle_func(project, params)
            raise Return((result, error))
        else:
            raise Return((None, self.METHOD_NOT_FOUND))

    @coroutine
    def publish_message(self, message, allowed_namespaces):
        """
        Publish event into PUB socket stream
        """
        project_id = message["project_id"]
        namespace_name = message["namespace"]
        channel = message["channel"]

        if allowed_namespaces[namespace_name]["is_watching"]:
            # send to admin channel
            self.pubsub.publish_admin_message(message)

        # send to event channel
        subscription_key = self.pubsub.get_subscription_key(project_id, namespace_name, channel)

        self.pubsub.publish(subscription_key, message)

        yield self.state.add_history_message(
            project_id,
            namespace_name,
            channel,
            message,
            history_size=allowed_namespaces[namespace_name]["history_size"],
        )

        self.messages_published += 1

        raise Return((True, None))

    @coroutine
    def prepare_message(self, project, allowed_namespaces, params, client):
        """
        Prepare message before actual publishing.
        """
        namespace_name = params.get("namespace")
        namespace, error = yield self.structure.get_namespace_by_name(project, namespace_name)
        if error:
            raise Return((None, error))
        if not namespace:
            raise Return((None, self.NAMESPACE_NOT_FOUND))
        namespace_name = namespace["name"]

        namespace = allowed_namespaces.get(namespace_name, None)
        if not namespace:
            raise Return((None, "namespace not found in allowed namespaces"))

        data = params.get("data", None)

        message = {
            "project_id": project["_id"],
            "namespace": namespace["name"],
            "uid": uuid.uuid4().hex,
            "timestamp": int(time.time()),
            "client": client,
            "channel": params.get("channel"),
            "data": data,
        }

        for callback in self.pre_publish_callbacks:
            try:
                message = yield callback(message)
            except Exception as err:
                logger.exception(err)
            else:
                if message is None:
                    raise Return((None, None))

        raise Return((message, None))

    @coroutine
    def process_publish(self, project, params, allowed_namespaces=None, client=None):
        """
        Publish message into appropriate channel.
        """
        if allowed_namespaces is None:
            project_namespaces, error = yield self.structure.get_project_namespaces(project)
            if error:
                raise Return((None, error))

            allowed_namespaces = dict((x["name"], x) for x in project_namespaces)

        message, error = yield self.prepare_message(project, allowed_namespaces, params, client)
        if error:
            raise Return((None, error))

        if not message:
            # message was discarded
            raise Return((True, None))

        # publish prepared message
        result, error = yield self.publish_message(message, allowed_namespaces)
        if error:
            raise Return((None, error))

        for callback in self.post_publish_callbacks:
            try:
                yield callback(message)
            except Exception as err:
                logger.exception(err)

        raise Return((True, None))

    @coroutine
    def process_history(self, project, params):
        """
        Return a list of last messages sent into channel.
        """
        project_id = project["_id"]

        namespace_name = params.get("namespace")
        namespace, error = yield self.structure.get_namespace_by_name(project, namespace_name)
        if error:
            raise Return((None, error))
        if not namespace:
            raise Return((None, self.NAMESPACE_NOT_FOUND))
        namespace_name = namespace["name"]

        channel = params.get("channel")
        message = {"namespace": namespace_name, "channel": channel, "data": []}
        data, error = yield self.state.get_history(project_id, namespace_name, channel)
        if data:
            message["data"] = data
        raise Return((message, error))

    @coroutine
    def process_presence(self, project, params):
        """
        Return current presence information for channel.
        """
        project_id = project["_id"]

        namespace_name = params.get("namespace")
        namespace, error = yield self.structure.get_namespace_by_name(project, namespace_name)
        if error:
            raise Return((None, error))
        if not namespace:
            raise Return((None, self.NAMESPACE_NOT_FOUND))
        namespace_name = namespace["name"]

        channel = params.get("channel")
        message = {"namespace": namespace_name, "channel": channel, "data": {}}
        data, error = yield self.state.get_presence(project_id, namespace_name, channel)
        if data:
            message["data"] = data
        raise Return((message, error))

    @coroutine
    def process_unsubscribe(self, project, params):
        """
        Unsubscribe user from channels.
        """
        params["project"] = project
        message = {"app_id": self.uid, "method": "unsubscribe", "params": params}

        # handle on this node
        result, error = yield self.handle_unsubscribe(params)

        # send to other nodes
        self.send_control_message(message)

        raise Return((result, error))

    @coroutine
    def process_dump_structure(self, project, params):

        projects, error = yield self.structure.project_list()
        if error:
            raise Return((None, self.INTERNAL_SERVER_ERROR))

        namespaces, error = yield self.structure.namespace_list()
        if error:
            raise Return((None, self.INTERNAL_SERVER_ERROR))

        data = {"projects": projects, "namespaces": namespaces}
        raise Return((data, None))

    @coroutine
    def process_project_list(self, project, params):
        projects, error = yield self.structure.project_list()
        if error:
            raise Return((None, self.INTERNAL_SERVER_ERROR))
        raise Return((projects, None))

    @coroutine
    def process_project_get(self, project, params):
        if not project:
            raise Return((None, self.PROJECT_NOT_FOUND))
        raise Return((project, None))

    @coroutine
    def process_project_by_name(self, project, params):
        project, error = yield self.structure.get_project_by_name(params.get("name"))
        if error:
            raise Return((None, self.INTERNAL_SERVER_ERROR))
        if not project:
            raise Return((None, self.PROJECT_NOT_FOUND))
        raise Return((project, None))

    @coroutine
    def process_project_create(self, project, params, error_form=False):

        form = ProjectForm(params)

        if form.validate():
            existing_project, error = yield self.structure.get_project_by_name(form.name.data)
            if error:
                raise Return((None, self.INTERNAL_SERVER_ERROR))

            if existing_project:
                form.name.errors.append(self.DUPLICATE_NAME)
                if error_form:
                    raise Return((None, form))
                raise Return((None, form.errors))
            else:
                project, error = yield self.structure.project_create(**form.data)
                if error:
                    raise Return((None, self.INTERNAL_SERVER_ERROR))
                raise Return((project, None))
        else:
            if error_form:
                raise Return((None, form))
            raise Return((None, form.errors))

    @coroutine
    def process_project_edit(self, project, params, error_form=False, patch=True):
        """
        Edit project namespace.
        """
        if not project:
            raise Return((None, self.PROJECT_NOT_FOUND))

        if "name" not in params:
            params["name"] = project["name"]

        namespaces, error = yield self.structure.get_project_namespaces(project)
        if error:
            raise Return((None, self.INTERNAL_SERVER_ERROR))

        boolean_patch_data = {}
        if patch:
            boolean_patch_data = utils.get_boolean_patch_data(ProjectForm.BOOLEAN_FIELDS, params)

        namespace_choices = [(x["_id"], x["name"]) for x in namespaces]
        namespace_choices.insert(0, ("", ""))
        form = ProjectForm(params, namespace_choices=namespace_choices)

        if form.validate():

            if "name" in params and params["name"] != project["name"]:

                existing_project, error = yield self.structure.get_project_by_name(params["name"])
                if error:
                    raise Return((None, self.INTERNAL_SERVER_ERROR))
                if existing_project:
                    form.name.errors.append(self.DUPLICATE_NAME)
                    if error_form:
                        raise Return((None, form))
                    raise Return((None, form.errors))

            updated_project = project.copy()

            if patch:
                data = utils.make_patch_data(form, params)
            else:
                data = form.data.copy()

            updated_project.update(data)
            if patch:
                updated_project.update(boolean_patch_data)
            project, error = yield self.structure.project_edit(project, **updated_project)
            if error:
                raise Return((None, self.INTERNAL_SERVER_ERROR))
            raise Return((project, None))
        else:
            if error_form:
                raise Return((None, form))
            raise Return((None, form.errors))

    @coroutine
    def process_project_delete(self, project, params):
        if not project:
            raise Return((None, self.PROJECT_NOT_FOUND))
        result, error = yield self.structure.project_delete(project)
        if error:
            raise Return((None, self.INTERNAL_SERVER_ERROR))
        raise Return((True, None))

    @coroutine
    def process_regenerate_secret_key(self, project, params):
        if not project:
            raise Return((None, self.PROJECT_NOT_FOUND))
        result, error = yield self.structure.regenerate_project_secret_key(project)
        if error:
            raise Return((None, self.INTERNAL_SERVER_ERROR))
        raise Return((result, None))

    @coroutine
    def process_namespace_list(self, project, params):
        """
        Return a list of all namespaces for project.
        """
        if not project:
            raise Return((None, self.PROJECT_NOT_FOUND))
        namespaces, error = yield self.structure.get_project_namespaces(project)
        if error:
            raise Return((None, self.INTERNAL_SERVER_ERROR))
        raise Return((namespaces, None))

    @coroutine
    def process_namespace_get(self, project, params):
        """
        Return a list of all namespaces for project.
        """
        namespace_id = params.get("_id")
        namespace, error = yield self.structure.get_namespace_by_id(namespace_id)
        if error:
            raise Return((None, self.INTERNAL_SERVER_ERROR))
        if not namespace:
            raise Return((None, self.NAMESPACE_NOT_FOUND))
        raise Return((namespace, None))

    @coroutine
    def process_namespace_by_name(self, project, params):
        if not project:
            raise Return((None, self.PROJECT_NOT_FOUND))

        namespace, error = yield self.structure.get_namespace_by_name(project, params.get("name"))
        if error:
            raise Return((None, self.INTERNAL_SERVER_ERROR))
        if not namespace:
            raise Return((None, self.NAMESPACE_NOT_FOUND))
        raise Return((namespace, None))

    @coroutine
    def process_namespace_create(self, project, params, error_form=False):
        """
        Create new namespace in project or update if already exists.
        """
        if not project:
            raise Return((None, self.PROJECT_NOT_FOUND))

        form = NamespaceForm(params)

        if form.validate():
            existing_namespace, error = yield self.structure.get_namespace_by_name(project, form.name.data)
            if error:
                raise Return((None, self.INTERNAL_SERVER_ERROR))

            if existing_namespace:
                form.name.errors.append(self.DUPLICATE_NAME)
                if error_form:
                    raise Return((None, form))
                raise Return((None, form.errors))
            else:
                namespace, error = yield self.structure.namespace_create(project, **form.data)
                if error:
                    raise Return((None, self.INTERNAL_SERVER_ERROR))
                raise Return((namespace, None))
        else:
            if error_form:
                raise Return((None, form))
            raise Return((None, form.errors))

    @coroutine
    def process_namespace_edit(self, project, params, error_form=False, patch=True):
        """
        Edit project namespace.
        """
        namespace, error = yield self.structure.get_namespace_by_id(params.pop("_id"))
        if error:
            raise Return((None, self.INTERNAL_SERVER_ERROR))

        if not namespace:
            raise Return((None, self.NAMESPACE_NOT_FOUND))

        if not project:
            project, error = yield self.structure.get_project_by_id(namespace["project_id"])
            if error:
                raise Return((None, self.INTERNAL_SERVER_ERROR))
            if not project:
                raise Return((None, self.PROJECT_NOT_FOUND))

        if "name" not in params:
            params["name"] = namespace["name"]

        boolean_patch_data = {}
        if patch:
            boolean_patch_data = utils.get_boolean_patch_data(NamespaceForm.BOOLEAN_FIELDS, params)

        form = NamespaceForm(params)

        if form.validate():

            if "name" in params and params["name"] != namespace["name"]:

                existing_namespace, error = yield self.structure.get_namespace_by_name(project, params["name"])
                if error:
                    raise Return((None, self.INTERNAL_SERVER_ERROR))
                if existing_namespace:
                    form.name.errors.append(self.DUPLICATE_NAME)
                    if error_form:
                        raise Return((None, form))
                    raise Return((None, form.errors))

            updated_namespace = namespace.copy()
            if patch:
                data = utils.make_patch_data(form, params)
            else:
                data = form.data.copy()
            updated_namespace.update(data)
            if patch:
                updated_namespace.update(boolean_patch_data)
            namespace, error = yield self.structure.namespace_edit(namespace, **updated_namespace)
            if error:
                raise Return((None, self.INTERNAL_SERVER_ERROR))
            raise Return((namespace, None))
        else:
            if error_form:
                raise Return((None, form))
            raise Return((None, form.errors))

    @coroutine
    def process_namespace_delete(self, project, params):
        """
        Delete project namespace.
        """
        namespace_id = params["_id"]

        existing_namespace, error = yield self.structure.get_namespace_by_id(namespace_id)
        if error:
            raise Return((None, self.INTERNAL_SERVER_ERROR))
        if not existing_namespace:
            raise Return((None, self.NAMESPACE_NOT_FOUND))

        result, error = yield self.structure.namespace_delete(existing_namespace)
        if error:
            raise Return((None, self.INTERNAL_SERVER_ERROR))
        raise Return((True, None))