def _subscribe_impl(self, ctx):
""" Invoked by subclasses to subscribe callers using input pub/sub config context.
"""
with self.lock('zato.pubsub.subscribe.%s.%s' % (ctx.topic_name, ctx.endpoint_id)):
with closing(self.odb.session()) as session:
# Non-WebSocket clients cannot subscribe to the same topic multiple times
if not ctx.ws_channel_id:
if has_subscription(session, ctx.cluster_id, ctx.topic.id, ctx.endpoint_id):
raise PubSubSubscriptionExists(self.cid, 'Subscription to topic `{}` already exists'.format(
ctx.topic.name))
ctx.creation_time = now = utcnow_as_ms()
ctx.sub_key = new_sub_key()
# If we subscribe a WSX client, we need to create its accompanying SQL models
if ctx.ws_channel_id:
# This object persists across multiple WSX connections
add_wsx_subscription(
session, ctx.cluster_id, ctx.is_internal, ctx.sub_key, ctx.ext_client_id, ctx.ws_channel_id)
# This object will be transient - dropped each time a WSX disconnects
self.pubsub.add_ws_client_pubsub_keys(session, ctx.sql_ws_client_id, ctx.sub_key, ctx.ws_channel_name,
ctx.ws_pub_client_id)
# Let the WebSocket connection object know that it should handle this particular sub_key
ctx.web_socket.pubsub_tool.add_sub_key(ctx.sub_key)
# Create a new subscription object
ps_sub = add_subscription(session, ctx.cluster_id, ctx)
# Flush the session because we need the subscription's ID below in INSERT from SELECT
session.flush()
# Move all available messages to that subscriber's queue
total_moved = move_messages_to_sub_queue(session, ctx.cluster_id, ctx.topic.id, ctx.endpoint_id, ps_sub.id, now)
# Commit all changes
session.commit()
# Produce response
self.response.payload.sub_key = ctx.sub_key
self.response.payload.queue_depth = total_moved
# Notify workers of a new subscription
broker_input = Bunch()
broker_input.topic_name = ctx.topic.name
broker_input.endpoint_type = self.endpoint_type
for name in sub_broker_attrs:
broker_input[name] = getattr(ps_sub, name, None)
broker_input.action = BROKER_MSG_PUBSUB.SUBSCRIPTION_CREATE.value
self.broker_client.publish(broker_input)
def _subscribe_impl(self, ctx):
""" Invoked by subclasses to subscribe callers using input pub/sub config context.
"""
with self.lock('zato.pubsub.subscribe.%s' % (ctx.topic_name)):
with closing(self.odb.session()) as session:
# Non-WebSocket clients cannot subscribe to the same topic multiple times
if not ctx.ws_channel_id:
if has_subscription(session, ctx.cluster_id, ctx.topic.id,
ctx.endpoint_id):
raise PubSubSubscriptionExists(
self.cid,
'Endpoint `{}` is already subscribed to topic `{}`'
.format(
self.pubsub.get_endpoint_by_id(
ctx.endpoint_id).name, ctx.topic.name))
# Is it a WebSockets client?
has_wsx = bool(ctx.ws_channel_id)
ctx.creation_time = now = utcnow_as_ms()
ctx.sub_key = new_sub_key()
# Create a new subscription object and flush the session because the subscription's ID
# may be needed for the WSX subscription
ps_sub = add_subscription(session, ctx.cluster_id, ctx)
session.flush()
# If we subscribe a WSX client, we need to create its accompanying SQL models
if has_wsx:
# This object persists across multiple WSX connections
add_wsx_subscription(session, ctx.cluster_id,
ctx.is_internal, ctx.sub_key,
ctx.ext_client_id, ctx.ws_channel_id,
ps_sub.id)
# This object will be transient - dropped each time a WSX client disconnects
self.pubsub.add_ws_client_pubsub_keys(
session, ctx.sql_ws_client_id, ctx.sub_key,
ctx.ws_channel_name, ctx.ws_pub_client_id)
# Common configuration for WSX and broker messages
sub_config = Bunch()
sub_config.topic_name = ctx.topic.name
sub_config.task_delivery_interval = ctx.topic.task_delivery_interval
sub_config.endpoint_type = self.endpoint_type
for name in sub_broker_attrs:
sub_config[name] = getattr(ps_sub, name, None)
#
# Move all available messages to that subscriber's queue. Note that we are operating under a global
# lock for the topic, the same lock that publications work under, which means that at this point
# there may be several cases depending on whether there are already other subscriptions
# or messages in the topic.
#
# * If there are subscribers, then this method will not move any messages because the messages
# will have been already moved to queues of other subscribers before we are called under this lock
#
# * If there are no subscribers but there are messages in the topic then this subscriber will become
# the sole recipient of the messages (we don't have any intrinsic foreknowledge of when, if at all,
# other subscribers can appear)
#
# * If there are no subscribers and no messages in the topic then this is a no-op
#
move_messages_to_sub_queue(session, ctx.cluster_id,
ctx.topic.id, ctx.endpoint_id,
ctx.sub_key, now)
# Subscription's ID is available only now, after the session was flushed
sub_config.id = ps_sub.id
# Update current server's pub/sub config
self.pubsub.add_subscription(sub_config)
if has_wsx:
# Let the WebSocket connection object know that it should handle this particular sub_key
ctx.web_socket.pubsub_tool.add_sub_key(ctx.sub_key)
# Commit all changes
session.commit()
# Produce response
self.response.payload.sub_key = ctx.sub_key
if has_wsx:
# Let the pub/sub task know it can fetch any messages possibly enqueued for that subscriber,
# note that since this is a new subscription, it is certain that only GD messages may be available,
# never non-GD ones.
ctx.web_socket.pubsub_tool.enqueue_gd_messages_by_sub_key(
ctx.sub_key)
gd_depth, non_gd_depth = ctx.web_socket.pubsub_tool.get_queue_depth(
ctx.sub_key)
self.response.payload.queue_depth = gd_depth + non_gd_depth
else:
# TODO:
# This should be read from that client's delivery task instead of SQL so as to include
# non-GD messages too.
self.response.payload.queue_depth = get_queue_depth_by_sub_key(
session, ctx.cluster_id, ctx.sub_key, now)
# Notify workers of a new subscription
sub_config.action = BROKER_MSG_PUBSUB.SUBSCRIPTION_CREATE.value
sub_config.add_subscription = not ctx.ws_channel_id # WSX clients already had their subscriptions created above
self.broker_client.publish(sub_config)
def _subscribe_impl(self, ctx):
""" Invoked by subclasses to subscribe callers using input pub/sub config context.
"""
with self.lock('zato.pubsub.subscribe.%s' % (ctx.topic_name)):
# Emit events about an upcoming subscription
self.pubsub.emit_about_to_subscribe({
'stage': 'sub.sk.1',
'sub_key': ctx.sub_key
})
self.pubsub.emit_about_to_subscribe({
'stage': 'init.ctx',
'data': ctx
})
self.pubsub.emit_about_to_subscribe({
'stage': 'sub.sk.2',
'sub_key': ctx.sub_key
})
# Endpoint on whose behalf the subscription will be made
endpoint = self.pubsub.get_endpoint_by_id(ctx.endpoint_id)
# Event log
self.pubsub.emit_in_subscribe_impl({
'stage': 'endpoint',
'data': endpoint,
})
self.pubsub.emit_about_to_subscribe({
'stage': 'sub.sk.3',
'sub_key': ctx.sub_key
})
with closing(self.odb.session()) as session:
with session.no_autoflush:
# Non-WebSocket clients cannot subscribe to the same topic multiple times
if not ctx.ws_channel_id:
# Event log
self.pubsub.emit_in_subscribe_impl({
'stage':
'no_ctx_ws_channel_id',
'data':
ctx.ws_channel_id
})
self.pubsub.emit_about_to_subscribe({
'stage':
'sub.sk.4',
'sub_key':
ctx.sub_key
})
if has_subscription(session, ctx.cluster_id,
ctx.topic.id, ctx.endpoint_id):
# Event log
self.pubsub.emit_in_subscribe_impl({
'stage': 'has_subscription',
'data': {
'ctx.cluster_id': ctx.cluster_id,
'ctx.topic_id': ctx.topic.id,
'ctx.topic_id': ctx.endpoint_id,
}
})
raise PubSubSubscriptionExists(
self.cid,
'Endpoint `{}` is already subscribed to topic `{}`'
.format(endpoint.name, ctx.topic.name))
# Is it a WebSockets client?
is_wsx = bool(ctx.ws_channel_id)
self.pubsub.emit_about_to_subscribe({
'stage': 'sub.sk.5',
'sub_key': ctx.sub_key
})
ctx.creation_time = now = utcnow_as_ms()
sub_key = new_sub_key(self.endpoint_type,
ctx.ext_client_id)
self.pubsub.emit_in_subscribe_impl({
'stage': 'new_sk_generated',
'data': {
'sub_key': sub_key,
}
})
# Event log
self.pubsub.emit_in_subscribe_impl({
'stage': 'before_add_subscription',
'data': {
'is_wsx': is_wsx,
'ctx.creation_time': ctx.creation_time,
'sub_key': sub_key,
'sub_sk':
sorted(self.pubsub.subscriptions_by_sub_key),
}
})
# Create a new subscription object and flush the session because the subscription's ID
# may be needed for the WSX subscription
ps_sub = add_subscription(session, ctx.cluster_id, sub_key,
ctx)
session.flush()
# Event log
self.pubsub.emit_in_subscribe_impl({
'stage': 'after_add_subscription',
'data': {
'ctx.cluster_id': ctx.cluster_id,
'ps_sub': ps_sub.asdict(),
'sub_sk':
sorted(self.pubsub.subscriptions_by_sub_key),
}
})
# Common configuration for WSX and broker messages
sub_config = Bunch()
sub_config.topic_name = ctx.topic.name
sub_config.task_delivery_interval = ctx.topic.task_delivery_interval
sub_config.endpoint_name = endpoint.name
sub_config.endpoint_type = self.endpoint_type
sub_config.unsub_on_wsx_close = ctx.unsub_on_wsx_close
sub_config.ext_client_id = ctx.ext_client_id
for name in sub_broker_attrs:
sub_config[name] = getattr(ps_sub, name, None)
#
# At this point there may be several cases depending on whether there are already other subscriptions
# or messages in the topic.
#
# * If there are subscribers, then this method will not move any messages because the messages
# will have been already moved to queues of other subscribers before we are called
#
# * If there are no subscribers but there are messages in the topic then this subscriber will become
# the sole recipient of the messages (we don't have any intrinsic foreknowledge of when, if at all,
# other subscribers can appear)
#
# * If there are no subscribers and no messages in the topic then this is a no-op
#
move_messages_to_sub_queue(session, ctx.cluster_id,
ctx.topic.id, ctx.endpoint_id,
ctx.sub_pattern_matched,
sub_key, now)
# Subscription's ID is available only now, after the session was flushed
sub_config.id = ps_sub.id
# Update current server's pub/sub config
self.pubsub.add_subscription(sub_config)
if is_wsx:
# Event log
self.pubsub.emit_in_subscribe_impl({
'stage': 'before_wsx_sub',
'data': {
'is_wsx':
is_wsx,
'sub_sk':
sorted(self.pubsub.subscriptions_by_sub_key),
}
})
# This object persists across multiple WSX connections
wsx_sub = add_wsx_subscription(
session, ctx.cluster_id, ctx.is_internal, sub_key,
ctx.ext_client_id, ctx.ws_channel_id, ps_sub.id)
# Event log
self.pubsub.emit_in_subscribe_impl({
'stage': 'after_wsx_sub',
'data': {
'wsx_sub':
wsx_sub.asdict(),
'sub_sk':
sorted(self.pubsub.subscriptions_by_sub_key),
}
})
# This object will be transient - dropped each time a WSX client disconnects
self.pubsub.add_wsx_client_pubsub_keys(
session, ctx.sql_ws_client_id, sub_key,
ctx.ws_channel_name, ctx.ws_pub_client_id,
ctx.web_socket.get_peer_info_dict())
# Let the WebSocket connection object know that it should handle this particular sub_key
ctx.web_socket.pubsub_tool.add_sub_key(sub_key)
# Commit all changes
session.commit()
# Produce response
self.response.payload.sub_key = sub_key
if is_wsx:
# Let the pub/sub task know it can fetch any messages possibly enqueued for that subscriber,
# note that since this is a new subscription, it is certain that only GD messages may be available,
# never non-GD ones.
ctx.web_socket.pubsub_tool.enqueue_gd_messages_by_sub_key(
sub_key)
gd_depth, non_gd_depth = ctx.web_socket.pubsub_tool.get_queue_depth(
sub_key)
self.response.payload.queue_depth = gd_depth + non_gd_depth
else:
# TODO:
# This should be read from that client's delivery task instead of SQL so as to include
# non-GD messages too.
self.response.payload.queue_depth = get_queue_depth_by_sub_key(
session, ctx.cluster_id, sub_key, now)
# Notify workers of a new subscription
sub_config.action = BROKER_MSG_PUBSUB.SUBSCRIPTION_CREATE.value
# Append information about current server which will let all workers
# know if they should create a subscription object (if they are different) or not.
sub_config.server_receiving_subscription_id = self.server.id
sub_config.server_receiving_subscription_pid = self.server.pid
sub_config.is_api_call = True
logger_pubsub.info('Subscription created `%s`', sub_config)
self.broker_client.publish(sub_config)
def add_pubsub_sec_endpoints(self, session, cluster):
from zato.common.api import CONNECTION, DATA_FORMAT, PUBSUB, URL_TYPE
from zato.common.json_internal import dumps
from zato.common.odb.model import HTTPBasicAuth, HTTPSOAP, PubSubEndpoint, PubSubSubscription, PubSubTopic, \
Service
from zato.common.pubsub import new_sub_key
from zato.common.util.time_ import utcnow_as_ms
sec_demo = HTTPBasicAuth(
None, 'zato.pubsub.demo.secdef', True, 'zato.pubsub.demo', 'Zato pub/sub demo', new_password(), cluster)
session.add(sec_demo)
sec_default_internal = HTTPBasicAuth(None, 'zato.pubsub.internal.secdef', True, 'zato.pubsub.internal',
'Zato pub/sub internal', new_password(), cluster)
session.add(sec_default_internal)
impl_name1 = 'zato.server.service.internal.pubsub.pubapi.TopicService'
impl_name2 = 'zato.server.service.internal.pubsub.pubapi.SubscribeService'
impl_name3 = 'zato.server.service.internal.pubsub.pubapi.MessageService'
impl_demo = 'zato.server.service.internal.helpers.JSONRawRequestLogger'
service_topic = Service(None, 'zato.pubsub.pubapi.topic-service', True, impl_name1, True, cluster)
service_sub = Service(None, 'zato.pubsub.pubapi.subscribe-service', True, impl_name2, True, cluster)
service_msg = Service(None, 'zato.pubsub.pubapi.message-service', True, impl_name3, True, cluster)
service_demo = Service(None, 'zato.pubsub.helpers.json-raw-request-logger', True, impl_demo, True, cluster)
# Opaque data that lets clients use topic contain slash characters
opaque = dumps({'match_slash':True})
chan_topic = HTTPSOAP(None, 'zato.pubsub.topic.topic_name', True, True, CONNECTION.CHANNEL,
URL_TYPE.PLAIN_HTTP, None, '/zato/pubsub/topic/{topic_name}',
None, '', None, DATA_FORMAT.JSON, security=None, service=service_topic, opaque=opaque,
cluster=cluster)
chan_sub = HTTPSOAP(None, 'zato.pubsub.subscribe.topic.topic_name', True, True, CONNECTION.CHANNEL,
URL_TYPE.PLAIN_HTTP, None, '/zato/pubsub/subscribe/topic/{topic_name}',
None, '', None, DATA_FORMAT.JSON, security=None, service=service_sub, opaque=opaque,
cluster=cluster)
chan_msg = HTTPSOAP(None, 'zato.pubsub.msg.msg_id', True, True, CONNECTION.CHANNEL,
URL_TYPE.PLAIN_HTTP, None, '/zato/pubsub/msg/{msg_id}',
None, '', None, DATA_FORMAT.JSON, security=None, service=service_msg, opaque=opaque,
cluster=cluster)
chan_demo = HTTPSOAP(None, 'pubsub.demo.sample.channel', True, True, CONNECTION.CHANNEL,
URL_TYPE.PLAIN_HTTP, None, '/zato/pubsub/zato.demo.sample',
None, '', None, DATA_FORMAT.JSON, security=sec_demo, service=service_demo, opaque=opaque,
cluster=cluster)
outconn_demo = HTTPSOAP(None, 'pubsub.demo.sample.outconn', True, True, CONNECTION.OUTGOING,
URL_TYPE.PLAIN_HTTP, 'http://localhost:11223', '/zato/pubsub/zato.demo.sample',
None, '', None, DATA_FORMAT.JSON, security=sec_demo, opaque=opaque,
cluster=cluster)
endpoint_default_internal = PubSubEndpoint()
endpoint_default_internal.name = PUBSUB.DEFAULT.INTERNAL_ENDPOINT_NAME
endpoint_default_internal.is_internal = True
endpoint_default_internal.role = PUBSUB.ROLE.PUBLISHER_SUBSCRIBER.id
endpoint_default_internal.topic_patterns = 'pub=/*\nsub=/*'
endpoint_default_internal.security = sec_default_internal
endpoint_default_internal.cluster = cluster
endpoint_default_internal.endpoint_type = PUBSUB.ENDPOINT_TYPE.INTERNAL.id
endpoint_demo = PubSubEndpoint()
endpoint_demo.name = 'zato.pubsub.demo.endpoint'
endpoint_demo.is_internal = True
endpoint_demo.role = PUBSUB.ROLE.PUBLISHER_SUBSCRIBER.id
endpoint_demo.topic_patterns = 'pub=/zato/demo/*\nsub=/zato/demo/*'
endpoint_demo.security = sec_demo
endpoint_demo.cluster = cluster
endpoint_demo.endpoint_type = PUBSUB.ENDPOINT_TYPE.REST.id
topic = PubSubTopic()
topic.name = '/zato/demo/sample'
topic.is_active = True
topic.is_api_sub_allowed = True
topic.is_internal = True
topic.max_depth = 100
topic.has_gd = False
topic.cluster = cluster
sub = PubSubSubscription()
sub.creation_time = utcnow_as_ms()
sub.topic = topic
sub.endpoint = endpoint_demo
sub.sub_key = new_sub_key(endpoint_demo.endpoint_type)
sub.has_gd = False
sub.sub_pattern_matched = 'sub=/zato/demo/*'
sub.active_status = PUBSUB.QUEUE_ACTIVE_STATUS.FULLY_ENABLED.id
sub.cluster = cluster
sub.wrap_one_msg_in_list = False
sub.delivery_err_should_block = False
sub.out_http_soap = outconn_demo
session.add(endpoint_default_internal)
session.add(endpoint_demo)
session.add(topic)
session.add(sub)
session.add(service_topic)
session.add(service_sub)
session.add(service_msg)
session.add(chan_topic)
session.add(chan_sub)
session.add(chan_msg)
session.add(chan_demo)
session.add(outconn_demo)
