def _subscribe_impl(self, ctx):
""" Invoked by subclasses to subscribe callers using input pub/sub config context.
"""
with self.lock('zato.pubsub.subscribe.%s.%s' % (ctx.topic_name, ctx.endpoint_id)):
with closing(self.odb.session()) as session:
# Non-WebSocket clients cannot subscribe to the same topic multiple times
if not ctx.ws_channel_id:
if has_subscription(session, ctx.cluster_id, ctx.topic.id, ctx.endpoint_id):
raise PubSubSubscriptionExists(self.cid, 'Subscription to topic `{}` already exists'.format(
ctx.topic.name))
ctx.creation_time = now = utcnow_as_ms()
ctx.sub_key = new_sub_key()
# If we subscribe a WSX client, we need to create its accompanying SQL models
if ctx.ws_channel_id:
# This object persists across multiple WSX connections
add_wsx_subscription(
session, ctx.cluster_id, ctx.is_internal, ctx.sub_key, ctx.ext_client_id, ctx.ws_channel_id)
# This object will be transient - dropped each time a WSX disconnects
self.pubsub.add_ws_client_pubsub_keys(session, ctx.sql_ws_client_id, ctx.sub_key, ctx.ws_channel_name,
ctx.ws_pub_client_id)
# Let the WebSocket connection object know that it should handle this particular sub_key
ctx.web_socket.pubsub_tool.add_sub_key(ctx.sub_key)
# Create a new subscription object
ps_sub = add_subscription(session, ctx.cluster_id, ctx)
# Flush the session because we need the subscription's ID below in INSERT from SELECT
session.flush()
# Move all available messages to that subscriber's queue
total_moved = move_messages_to_sub_queue(session, ctx.cluster_id, ctx.topic.id, ctx.endpoint_id, ps_sub.id, now)
# Commit all changes
session.commit()
# Produce response
self.response.payload.sub_key = ctx.sub_key
self.response.payload.queue_depth = total_moved
# Notify workers of a new subscription
broker_input = Bunch()
broker_input.topic_name = ctx.topic.name
broker_input.endpoint_type = self.endpoint_type
for name in sub_broker_attrs:
broker_input[name] = getattr(ps_sub, name, None)
broker_input.action = BROKER_MSG_PUBSUB.SUBSCRIPTION_CREATE.value
self.broker_client.publish(broker_input)
def _subscribe_impl(self, ctx):
""" Invoked by subclasses to subscribe callers using input pub/sub config context.
"""
with self.lock('zato.pubsub.subscribe.%s' % (ctx.topic_name)):
with closing(self.odb.session()) as session:
# Non-WebSocket clients cannot subscribe to the same topic multiple times
if not ctx.ws_channel_id:
if has_subscription(session, ctx.cluster_id, ctx.topic.id,
ctx.endpoint_id):
raise PubSubSubscriptionExists(
self.cid,
'Endpoint `{}` is already subscribed to topic `{}`'
.format(
self.pubsub.get_endpoint_by_id(
ctx.endpoint_id).name, ctx.topic.name))
# Is it a WebSockets client?
has_wsx = bool(ctx.ws_channel_id)
ctx.creation_time = now = utcnow_as_ms()
ctx.sub_key = new_sub_key()
# Create a new subscription object and flush the session because the subscription's ID
# may be needed for the WSX subscription
ps_sub = add_subscription(session, ctx.cluster_id, ctx)
session.flush()
# If we subscribe a WSX client, we need to create its accompanying SQL models
if has_wsx:
# This object persists across multiple WSX connections
add_wsx_subscription(session, ctx.cluster_id,
ctx.is_internal, ctx.sub_key,
ctx.ext_client_id, ctx.ws_channel_id,
ps_sub.id)
# This object will be transient - dropped each time a WSX client disconnects
self.pubsub.add_ws_client_pubsub_keys(
session, ctx.sql_ws_client_id, ctx.sub_key,
ctx.ws_channel_name, ctx.ws_pub_client_id)
# Common configuration for WSX and broker messages
sub_config = Bunch()
sub_config.topic_name = ctx.topic.name
sub_config.task_delivery_interval = ctx.topic.task_delivery_interval
sub_config.endpoint_type = self.endpoint_type
for name in sub_broker_attrs:
sub_config[name] = getattr(ps_sub, name, None)
#
# Move all available messages to that subscriber's queue. Note that we are operating under a global
# lock for the topic, the same lock that publications work under, which means that at this point
# there may be several cases depending on whether there are already other subscriptions
# or messages in the topic.
#
# * If there are subscribers, then this method will not move any messages because the messages
# will have been already moved to queues of other subscribers before we are called under this lock
#
# * If there are no subscribers but there are messages in the topic then this subscriber will become
# the sole recipient of the messages (we don't have any intrinsic foreknowledge of when, if at all,
# other subscribers can appear)
#
# * If there are no subscribers and no messages in the topic then this is a no-op
#
move_messages_to_sub_queue(session, ctx.cluster_id,
ctx.topic.id, ctx.endpoint_id,
ctx.sub_key, now)
# Subscription's ID is available only now, after the session was flushed
sub_config.id = ps_sub.id
# Update current server's pub/sub config
self.pubsub.add_subscription(sub_config)
if has_wsx:
# Let the WebSocket connection object know that it should handle this particular sub_key
ctx.web_socket.pubsub_tool.add_sub_key(ctx.sub_key)
# Commit all changes
session.commit()
# Produce response
self.response.payload.sub_key = ctx.sub_key
if has_wsx:
# Let the pub/sub task know it can fetch any messages possibly enqueued for that subscriber,
# note that since this is a new subscription, it is certain that only GD messages may be available,
# never non-GD ones.
ctx.web_socket.pubsub_tool.enqueue_gd_messages_by_sub_key(
ctx.sub_key)
gd_depth, non_gd_depth = ctx.web_socket.pubsub_tool.get_queue_depth(
ctx.sub_key)
self.response.payload.queue_depth = gd_depth + non_gd_depth
else:
# TODO:
# This should be read from that client's delivery task instead of SQL so as to include
# non-GD messages too.
self.response.payload.queue_depth = get_queue_depth_by_sub_key(
session, ctx.cluster_id, ctx.sub_key, now)
# Notify workers of a new subscription
sub_config.action = BROKER_MSG_PUBSUB.SUBSCRIPTION_CREATE.value
sub_config.add_subscription = not ctx.ws_channel_id # WSX clients already had their subscriptions created above
self.broker_client.publish(sub_config)
def _subscribe_impl(self, ctx):
""" Invoked by subclasses to subscribe callers using input pub/sub config context.
"""
with self.lock('zato.pubsub.subscribe.%s' % (ctx.topic_name)):
# Emit events about an upcoming subscription
self.pubsub.emit_about_to_subscribe({
'stage': 'sub.sk.1',
'sub_key': ctx.sub_key
})
self.pubsub.emit_about_to_subscribe({
'stage': 'init.ctx',
'data': ctx
})
self.pubsub.emit_about_to_subscribe({
'stage': 'sub.sk.2',
'sub_key': ctx.sub_key
})
# Endpoint on whose behalf the subscription will be made
endpoint = self.pubsub.get_endpoint_by_id(ctx.endpoint_id)
# Event log
self.pubsub.emit_in_subscribe_impl({
'stage': 'endpoint',
'data': endpoint,
})
self.pubsub.emit_about_to_subscribe({
'stage': 'sub.sk.3',
'sub_key': ctx.sub_key
})
with closing(self.odb.session()) as session:
with session.no_autoflush:
# Non-WebSocket clients cannot subscribe to the same topic multiple times
if not ctx.ws_channel_id:
# Event log
self.pubsub.emit_in_subscribe_impl({
'stage':
'no_ctx_ws_channel_id',
'data':
ctx.ws_channel_id
})
self.pubsub.emit_about_to_subscribe({
'stage':
'sub.sk.4',
'sub_key':
ctx.sub_key
})
if has_subscription(session, ctx.cluster_id,
ctx.topic.id, ctx.endpoint_id):
# Event log
self.pubsub.emit_in_subscribe_impl({
'stage': 'has_subscription',
'data': {
'ctx.cluster_id': ctx.cluster_id,
'ctx.topic_id': ctx.topic.id,
'ctx.topic_id': ctx.endpoint_id,
}
})
raise PubSubSubscriptionExists(
self.cid,
'Endpoint `{}` is already subscribed to topic `{}`'
.format(endpoint.name, ctx.topic.name))
# Is it a WebSockets client?
is_wsx = bool(ctx.ws_channel_id)
self.pubsub.emit_about_to_subscribe({
'stage': 'sub.sk.5',
'sub_key': ctx.sub_key
})
ctx.creation_time = now = utcnow_as_ms()
sub_key = new_sub_key(self.endpoint_type,
ctx.ext_client_id)
self.pubsub.emit_in_subscribe_impl({
'stage': 'new_sk_generated',
'data': {
'sub_key': sub_key,
}
})
# Event log
self.pubsub.emit_in_subscribe_impl({
'stage': 'before_add_subscription',
'data': {
'is_wsx': is_wsx,
'ctx.creation_time': ctx.creation_time,
'sub_key': sub_key,
'sub_sk':
sorted(self.pubsub.subscriptions_by_sub_key),
}
})
# Create a new subscription object and flush the session because the subscription's ID
# may be needed for the WSX subscription
ps_sub = add_subscription(session, ctx.cluster_id, sub_key,
ctx)
session.flush()
# Event log
self.pubsub.emit_in_subscribe_impl({
'stage': 'after_add_subscription',
'data': {
'ctx.cluster_id': ctx.cluster_id,
'ps_sub': ps_sub.asdict(),
'sub_sk':
sorted(self.pubsub.subscriptions_by_sub_key),
}
})
# Common configuration for WSX and broker messages
sub_config = Bunch()
sub_config.topic_name = ctx.topic.name
sub_config.task_delivery_interval = ctx.topic.task_delivery_interval
sub_config.endpoint_name = endpoint.name
sub_config.endpoint_type = self.endpoint_type
sub_config.unsub_on_wsx_close = ctx.unsub_on_wsx_close
sub_config.ext_client_id = ctx.ext_client_id
for name in sub_broker_attrs:
sub_config[name] = getattr(ps_sub, name, None)
#
# At this point there may be several cases depending on whether there are already other subscriptions
# or messages in the topic.
#
# * If there are subscribers, then this method will not move any messages because the messages
# will have been already moved to queues of other subscribers before we are called
#
# * If there are no subscribers but there are messages in the topic then this subscriber will become
# the sole recipient of the messages (we don't have any intrinsic foreknowledge of when, if at all,
# other subscribers can appear)
#
# * If there are no subscribers and no messages in the topic then this is a no-op
#
move_messages_to_sub_queue(session, ctx.cluster_id,
ctx.topic.id, ctx.endpoint_id,
ctx.sub_pattern_matched,
sub_key, now)
# Subscription's ID is available only now, after the session was flushed
sub_config.id = ps_sub.id
# Update current server's pub/sub config
self.pubsub.add_subscription(sub_config)
if is_wsx:
# Event log
self.pubsub.emit_in_subscribe_impl({
'stage': 'before_wsx_sub',
'data': {
'is_wsx':
is_wsx,
'sub_sk':
sorted(self.pubsub.subscriptions_by_sub_key),
}
})
# This object persists across multiple WSX connections
wsx_sub = add_wsx_subscription(
session, ctx.cluster_id, ctx.is_internal, sub_key,
ctx.ext_client_id, ctx.ws_channel_id, ps_sub.id)
# Event log
self.pubsub.emit_in_subscribe_impl({
'stage': 'after_wsx_sub',
'data': {
'wsx_sub':
wsx_sub.asdict(),
'sub_sk':
sorted(self.pubsub.subscriptions_by_sub_key),
}
})
# This object will be transient - dropped each time a WSX client disconnects
self.pubsub.add_wsx_client_pubsub_keys(
session, ctx.sql_ws_client_id, sub_key,
ctx.ws_channel_name, ctx.ws_pub_client_id,
ctx.web_socket.get_peer_info_dict())
# Let the WebSocket connection object know that it should handle this particular sub_key
ctx.web_socket.pubsub_tool.add_sub_key(sub_key)
# Commit all changes
session.commit()
# Produce response
self.response.payload.sub_key = sub_key
if is_wsx:
# Let the pub/sub task know it can fetch any messages possibly enqueued for that subscriber,
# note that since this is a new subscription, it is certain that only GD messages may be available,
# never non-GD ones.
ctx.web_socket.pubsub_tool.enqueue_gd_messages_by_sub_key(
sub_key)
gd_depth, non_gd_depth = ctx.web_socket.pubsub_tool.get_queue_depth(
sub_key)
self.response.payload.queue_depth = gd_depth + non_gd_depth
else:
# TODO:
# This should be read from that client's delivery task instead of SQL so as to include
# non-GD messages too.
self.response.payload.queue_depth = get_queue_depth_by_sub_key(
session, ctx.cluster_id, sub_key, now)
# Notify workers of a new subscription
sub_config.action = BROKER_MSG_PUBSUB.SUBSCRIPTION_CREATE.value
# Append information about current server which will let all workers
# know if they should create a subscription object (if they are different) or not.
sub_config.server_receiving_subscription_id = self.server.id
sub_config.server_receiving_subscription_pid = self.server.pid
sub_config.is_api_call = True
logger_pubsub.info('Subscription created `%s`', sub_config)
self.broker_client.publish(sub_config)