Example #1
0
    def _subscribe_impl(self, ctx):
        """ Invoked by subclasses to subscribe callers using input pub/sub config context.
        """
        with self.lock('zato.pubsub.subscribe.%s.%s' % (ctx.topic_name, ctx.endpoint_id)):

            with closing(self.odb.session()) as session:

                # Non-WebSocket clients cannot subscribe to the same topic multiple times
                if not ctx.ws_channel_id:
                    if has_subscription(session, ctx.cluster_id, ctx.topic.id, ctx.endpoint_id):
                        raise PubSubSubscriptionExists(self.cid, 'Subscription to topic `{}` already exists'.format(
                            ctx.topic.name))

                ctx.creation_time = now = utcnow_as_ms()
                ctx.sub_key = new_sub_key()

                # If we subscribe a WSX client, we need to create its accompanying SQL models
                if ctx.ws_channel_id:

                    # This object persists across multiple WSX connections
                    add_wsx_subscription(
                        session, ctx.cluster_id, ctx.is_internal, ctx.sub_key, ctx.ext_client_id, ctx.ws_channel_id)

                    # This object will be transient - dropped each time a WSX disconnects
                    self.pubsub.add_ws_client_pubsub_keys(session, ctx.sql_ws_client_id, ctx.sub_key, ctx.ws_channel_name,
                        ctx.ws_pub_client_id)

                    # Let the WebSocket connection object know that it should handle this particular sub_key
                    ctx.web_socket.pubsub_tool.add_sub_key(ctx.sub_key)

                # Create a new subscription object
                ps_sub = add_subscription(session, ctx.cluster_id, ctx)

                # Flush the session because we need the subscription's ID below in INSERT from SELECT
                session.flush()

                # Move all available messages to that subscriber's queue
                total_moved = move_messages_to_sub_queue(session, ctx.cluster_id, ctx.topic.id, ctx.endpoint_id, ps_sub.id, now)

                # Commit all changes
                session.commit()

                # Produce response
                self.response.payload.sub_key = ctx.sub_key
                self.response.payload.queue_depth = total_moved

                # Notify workers of a new subscription
                broker_input = Bunch()
                broker_input.topic_name = ctx.topic.name
                broker_input.endpoint_type = self.endpoint_type

                for name in sub_broker_attrs:
                    broker_input[name] = getattr(ps_sub, name, None)

                broker_input.action = BROKER_MSG_PUBSUB.SUBSCRIPTION_CREATE.value
                self.broker_client.publish(broker_input)
Example #2
0
    def _subscribe_impl(self, ctx):
        """ Invoked by subclasses to subscribe callers using input pub/sub config context.
        """
        with self.lock('zato.pubsub.subscribe.%s' % (ctx.topic_name)):

            with closing(self.odb.session()) as session:

                # Non-WebSocket clients cannot subscribe to the same topic multiple times
                if not ctx.ws_channel_id:
                    if has_subscription(session, ctx.cluster_id, ctx.topic.id,
                                        ctx.endpoint_id):
                        raise PubSubSubscriptionExists(
                            self.cid,
                            'Endpoint `{}` is already subscribed to topic `{}`'
                            .format(
                                self.pubsub.get_endpoint_by_id(
                                    ctx.endpoint_id).name, ctx.topic.name))

                # Is it a WebSockets client?
                has_wsx = bool(ctx.ws_channel_id)

                ctx.creation_time = now = utcnow_as_ms()
                ctx.sub_key = new_sub_key()

                # Create a new subscription object and flush the session because the subscription's ID
                # may be needed for the WSX subscription
                ps_sub = add_subscription(session, ctx.cluster_id, ctx)
                session.flush()

                # If we subscribe a WSX client, we need to create its accompanying SQL models
                if has_wsx:

                    # This object persists across multiple WSX connections
                    add_wsx_subscription(session, ctx.cluster_id,
                                         ctx.is_internal, ctx.sub_key,
                                         ctx.ext_client_id, ctx.ws_channel_id,
                                         ps_sub.id)

                    # This object will be transient - dropped each time a WSX client disconnects
                    self.pubsub.add_ws_client_pubsub_keys(
                        session, ctx.sql_ws_client_id, ctx.sub_key,
                        ctx.ws_channel_name, ctx.ws_pub_client_id)

                # Common configuration for WSX and broker messages
                sub_config = Bunch()
                sub_config.topic_name = ctx.topic.name
                sub_config.task_delivery_interval = ctx.topic.task_delivery_interval
                sub_config.endpoint_type = self.endpoint_type

                for name in sub_broker_attrs:
                    sub_config[name] = getattr(ps_sub, name, None)

                #
                # Move all available messages to that subscriber's queue. Note that we are operating under a global
                # lock for the topic, the same lock that publications work under, which means that at this point
                # there may be several cases depending on whether there are already other subscriptions
                # or messages in the topic.
                #
                # * If there are subscribers, then this method will not move any messages because the messages
                #   will have been already moved to queues of other subscribers before we are called under this lock
                #
                # * If there are no subscribers but there are messages in the topic then this subscriber will become
                #   the sole recipient of the messages (we don't have any intrinsic foreknowledge of when, if at all,
                #   other subscribers can appear)
                #
                # * If there are no subscribers and no messages in the topic then this is a no-op
                #
                move_messages_to_sub_queue(session, ctx.cluster_id,
                                           ctx.topic.id, ctx.endpoint_id,
                                           ctx.sub_key, now)

                # Subscription's ID is available only now, after the session was flushed
                sub_config.id = ps_sub.id

                # Update current server's pub/sub config
                self.pubsub.add_subscription(sub_config)

                if has_wsx:

                    # Let the WebSocket connection object know that it should handle this particular sub_key
                    ctx.web_socket.pubsub_tool.add_sub_key(ctx.sub_key)

                # Commit all changes
                session.commit()

                # Produce response
                self.response.payload.sub_key = ctx.sub_key

                if has_wsx:

                    # Let the pub/sub task know it can fetch any messages possibly enqueued for that subscriber,
                    # note that since this is a new subscription, it is certain that only GD messages may be available,
                    # never non-GD ones.
                    ctx.web_socket.pubsub_tool.enqueue_gd_messages_by_sub_key(
                        ctx.sub_key)

                    gd_depth, non_gd_depth = ctx.web_socket.pubsub_tool.get_queue_depth(
                        ctx.sub_key)
                    self.response.payload.queue_depth = gd_depth + non_gd_depth
                else:

                    # TODO:
                    # This should be read from that client's delivery task instead of SQL so as to include
                    # non-GD messages too.

                    self.response.payload.queue_depth = get_queue_depth_by_sub_key(
                        session, ctx.cluster_id, ctx.sub_key, now)

                # Notify workers of a new subscription
                sub_config.action = BROKER_MSG_PUBSUB.SUBSCRIPTION_CREATE.value
                sub_config.add_subscription = not ctx.ws_channel_id  # WSX clients already had their subscriptions created above

                self.broker_client.publish(sub_config)
Example #3
0
    def _subscribe_impl(self, ctx):
        """ Invoked by subclasses to subscribe callers using input pub/sub config context.
        """
        with self.lock('zato.pubsub.subscribe.%s' % (ctx.topic_name)):

            # Emit events about an upcoming subscription
            self.pubsub.emit_about_to_subscribe({
                'stage': 'sub.sk.1',
                'sub_key': ctx.sub_key
            })

            self.pubsub.emit_about_to_subscribe({
                'stage': 'init.ctx',
                'data': ctx
            })

            self.pubsub.emit_about_to_subscribe({
                'stage': 'sub.sk.2',
                'sub_key': ctx.sub_key
            })

            # Endpoint on whose behalf the subscription will be made
            endpoint = self.pubsub.get_endpoint_by_id(ctx.endpoint_id)

            # Event log
            self.pubsub.emit_in_subscribe_impl({
                'stage': 'endpoint',
                'data': endpoint,
            })

            self.pubsub.emit_about_to_subscribe({
                'stage': 'sub.sk.3',
                'sub_key': ctx.sub_key
            })

            with closing(self.odb.session()) as session:

                with session.no_autoflush:

                    # Non-WebSocket clients cannot subscribe to the same topic multiple times
                    if not ctx.ws_channel_id:

                        # Event log
                        self.pubsub.emit_in_subscribe_impl({
                            'stage':
                            'no_ctx_ws_channel_id',
                            'data':
                            ctx.ws_channel_id
                        })

                        self.pubsub.emit_about_to_subscribe({
                            'stage':
                            'sub.sk.4',
                            'sub_key':
                            ctx.sub_key
                        })

                        if has_subscription(session, ctx.cluster_id,
                                            ctx.topic.id, ctx.endpoint_id):

                            # Event log
                            self.pubsub.emit_in_subscribe_impl({
                                'stage': 'has_subscription',
                                'data': {
                                    'ctx.cluster_id': ctx.cluster_id,
                                    'ctx.topic_id': ctx.topic.id,
                                    'ctx.topic_id': ctx.endpoint_id,
                                }
                            })

                            raise PubSubSubscriptionExists(
                                self.cid,
                                'Endpoint `{}` is already subscribed to topic `{}`'
                                .format(endpoint.name, ctx.topic.name))

                    # Is it a WebSockets client?
                    is_wsx = bool(ctx.ws_channel_id)

                    self.pubsub.emit_about_to_subscribe({
                        'stage': 'sub.sk.5',
                        'sub_key': ctx.sub_key
                    })

                    ctx.creation_time = now = utcnow_as_ms()
                    sub_key = new_sub_key(self.endpoint_type,
                                          ctx.ext_client_id)

                    self.pubsub.emit_in_subscribe_impl({
                        'stage': 'new_sk_generated',
                        'data': {
                            'sub_key': sub_key,
                        }
                    })

                    # Event log
                    self.pubsub.emit_in_subscribe_impl({
                        'stage': 'before_add_subscription',
                        'data': {
                            'is_wsx': is_wsx,
                            'ctx.creation_time': ctx.creation_time,
                            'sub_key': sub_key,
                            'sub_sk':
                            sorted(self.pubsub.subscriptions_by_sub_key),
                        }
                    })

                    # Create a new subscription object and flush the session because the subscription's ID
                    # may be needed for the WSX subscription
                    ps_sub = add_subscription(session, ctx.cluster_id, sub_key,
                                              ctx)
                    session.flush()

                    # Event log
                    self.pubsub.emit_in_subscribe_impl({
                        'stage': 'after_add_subscription',
                        'data': {
                            'ctx.cluster_id': ctx.cluster_id,
                            'ps_sub': ps_sub.asdict(),
                            'sub_sk':
                            sorted(self.pubsub.subscriptions_by_sub_key),
                        }
                    })

                    # Common configuration for WSX and broker messages
                    sub_config = Bunch()
                    sub_config.topic_name = ctx.topic.name
                    sub_config.task_delivery_interval = ctx.topic.task_delivery_interval
                    sub_config.endpoint_name = endpoint.name
                    sub_config.endpoint_type = self.endpoint_type
                    sub_config.unsub_on_wsx_close = ctx.unsub_on_wsx_close
                    sub_config.ext_client_id = ctx.ext_client_id

                    for name in sub_broker_attrs:
                        sub_config[name] = getattr(ps_sub, name, None)

                    #
                    # At this point there may be several cases depending on whether there are already other subscriptions
                    # or messages in the topic.
                    #
                    # * If there are subscribers, then this method will not move any messages because the messages
                    #   will have been already moved to queues of other subscribers before we are called
                    #
                    # * If there are no subscribers but there are messages in the topic then this subscriber will become
                    #   the sole recipient of the messages (we don't have any intrinsic foreknowledge of when, if at all,
                    #   other subscribers can appear)
                    #
                    # * If there are no subscribers and no messages in the topic then this is a no-op
                    #

                    move_messages_to_sub_queue(session, ctx.cluster_id,
                                               ctx.topic.id, ctx.endpoint_id,
                                               ctx.sub_pattern_matched,
                                               sub_key, now)

                    # Subscription's ID is available only now, after the session was flushed
                    sub_config.id = ps_sub.id

                    # Update current server's pub/sub config
                    self.pubsub.add_subscription(sub_config)

                    if is_wsx:

                        # Event log
                        self.pubsub.emit_in_subscribe_impl({
                            'stage': 'before_wsx_sub',
                            'data': {
                                'is_wsx':
                                is_wsx,
                                'sub_sk':
                                sorted(self.pubsub.subscriptions_by_sub_key),
                            }
                        })

                        # This object persists across multiple WSX connections
                        wsx_sub = add_wsx_subscription(
                            session, ctx.cluster_id, ctx.is_internal, sub_key,
                            ctx.ext_client_id, ctx.ws_channel_id, ps_sub.id)

                        # Event log
                        self.pubsub.emit_in_subscribe_impl({
                            'stage': 'after_wsx_sub',
                            'data': {
                                'wsx_sub':
                                wsx_sub.asdict(),
                                'sub_sk':
                                sorted(self.pubsub.subscriptions_by_sub_key),
                            }
                        })

                        # This object will be transient - dropped each time a WSX client disconnects
                        self.pubsub.add_wsx_client_pubsub_keys(
                            session, ctx.sql_ws_client_id, sub_key,
                            ctx.ws_channel_name, ctx.ws_pub_client_id,
                            ctx.web_socket.get_peer_info_dict())

                        # Let the WebSocket connection object know that it should handle this particular sub_key
                        ctx.web_socket.pubsub_tool.add_sub_key(sub_key)

                    # Commit all changes
                    session.commit()

                    # Produce response
                    self.response.payload.sub_key = sub_key

                    if is_wsx:

                        # Let the pub/sub task know it can fetch any messages possibly enqueued for that subscriber,
                        # note that since this is a new subscription, it is certain that only GD messages may be available,
                        # never non-GD ones.
                        ctx.web_socket.pubsub_tool.enqueue_gd_messages_by_sub_key(
                            sub_key)

                        gd_depth, non_gd_depth = ctx.web_socket.pubsub_tool.get_queue_depth(
                            sub_key)
                        self.response.payload.queue_depth = gd_depth + non_gd_depth
                    else:

                        # TODO:
                        # This should be read from that client's delivery task instead of SQL so as to include
                        # non-GD messages too.

                        self.response.payload.queue_depth = get_queue_depth_by_sub_key(
                            session, ctx.cluster_id, sub_key, now)

                # Notify workers of a new subscription
                sub_config.action = BROKER_MSG_PUBSUB.SUBSCRIPTION_CREATE.value

                # Append information about current server which will let all workers
                # know if they should create a subscription object (if they are different) or not.
                sub_config.server_receiving_subscription_id = self.server.id
                sub_config.server_receiving_subscription_pid = self.server.pid
                sub_config.is_api_call = True

                logger_pubsub.info('Subscription created `%s`', sub_config)

                self.broker_client.publish(sub_config)