def reject_publication(self, topic_name, is_gd):
""" Raises an exception to indicate that a publication was rejected.
"""
raise ServiceUnavailable(
self.cid,
'Publication rejected - would exceed {} max depth for `{}`'.format(
'GD' if is_gd else 'non-GD', topic_name))
def handle(self):
input = self.request.input
pubsub = self.server.worker_store.pubsub # type: PubSub
endpoint_id = input.endpoint_id
# Will return publication pattern matched or raise an exception that we don't catch
endpoint_id, pub_pattern_matched = self.get_pub_pattern_matched(
endpoint_id, input)
try:
topic = pubsub.get_topic_by_name(input.topic_name) # type: Topic
except KeyError:
raise NotFound(self.cid,
'No such topic `{}`'.format(input.topic_name))
# Reject the message is topic is not active
if not topic.is_active:
raise ServiceUnavailable(
self.cid, 'Topic is inactive `{}`'.format(input.topic_name))
# We always count time in milliseconds since UNIX epoch
now = utcnow_as_ms()
# Get all subscribers for that topic from local worker store
all_subscriptions_by_topic = pubsub.get_subscriptions_by_topic(
topic.name)
len_all_sub = len(all_subscriptions_by_topic)
# If we are to deliver the message(s) to only selected subscribers only,
# filter out any unwated ones first.
if input.deliver_to_sk:
has_all = False
subscriptions_by_topic = []
# Get any matching subscriptions out of the whole set
for sub in all_subscriptions_by_topic:
if sub.sub_key in input.deliver_to_sk:
subscriptions_by_topic.append(sub)
else:
# We deliver this message to all of the topic's subscribers
has_all = True
subscriptions_by_topic = all_subscriptions_by_topic
# This is only for logging purposes
_subs_found = []
# Assume that there are no missing servers for WSX clients by default
has_wsx_no_server = False
for sub in subscriptions_by_topic:
# Prepare data for logging
_subs_found.append({sub.sub_key: sub.sub_pattern_matched})
# Is there at least one WSX subscriber to this topic that is currently not connected?
# If so, later on we will need to turn all the messages into GD ones.
sk_server = self.pubsub.get_sub_key_server(sub.sub_key)
if not sk_server:
if has_logger_pubsub_debug:
logger_pubsub.debug(
'No sk_server for sub_key `%s` among `%s`',
sub.sub_key,
sorted(self.pubsub.sub_key_servers.keys()))
has_wsx_no_server = True # We have found at least one WSX subscriber that has no server = it is not connected
logger_pubsub.info(
'Subscriptions for topic `%s` `%s` (a:%d, %d/%d, cid:%s)',
topic.name, _subs_found, has_all, len(subscriptions_by_topic),
len_all_sub, self.cid)
# If input.data is a list, it means that it is a list of messages, each of which has its own
# metadata. Otherwise, it's a string to publish and other input parameters describe it.
data_list = input.data_list if input.data_list else None
# Input messages may contain a mix of GD and non-GD messages, and we need to extract them separately.
msg_id_list, gd_msg_list, non_gd_msg_list = self._get_messages_from_data(
topic, data_list, input, now, pub_pattern_matched, endpoint_id,
subscriptions_by_topic, has_wsx_no_server,
input.get('reply_to_sk', None))
# Create a wrapper object for all the input data and metadata
ctx = PubCtx(self.server.cluster_id, pubsub, topic, endpoint_id,
pubsub.get_endpoint_by_id(endpoint_id).name,
subscriptions_by_topic, msg_id_list, gd_msg_list,
non_gd_msg_list, pub_pattern_matched,
input.get('ext_client_id'), False, now)
# We have all the input data, publish the message(s) now
self._publish(ctx)
def handle(self):
input = self.request.input
pubsub = self.server.worker_store.pubsub # type: PubSub
endpoint_id = input.endpoint_id
# Will return publication pattern matched or raise an exception that we don't catch
endpoint_id, pattern_matched = self.get_pattern_matched(
endpoint_id, input)
try:
topic = pubsub.get_topic_by_name(input.topic_name) # type: Topic
except KeyError:
raise NotFound(self.cid,
'No such topic `{}`'.format(input.topic_name))
# We always count time in milliseconds since UNIX epoch
now = utcnow_as_ms()
# If input.data is a list, it means that it is a list of messages, each of which has its own
# metadata. Otherwise, it's a string to publish and other input parameters describe it.
data_list = input.data_list if input.data_list else None
# Input messages may contain a mix of GD and non-GD messages, and we need to extract them separately.
msg_id_list, gd_msg_list, non_gd_msg_list = self._get_messages_from_data(
topic, data_list, input, now, pattern_matched, endpoint_id)
len_gd_msg_list = len(gd_msg_list)
has_gd_msg_list = bool(len_gd_msg_list)
# Get all subscribers for that topic from local worker store
subscriptions_by_topic = pubsub.get_subscriptions_by_topic(
input.topic_name)
# Just so it is not overlooked, log information that no subscribers are found for this topic
if not subscriptions_by_topic:
self.logger.warn('No subscribers found for topic `%s`',
input.topic_name)
# Local aliases
cluster_id = self.server.cluster_id
has_pubsub_audit_log = self.server.has_pubsub_audit_log
# This is initially unknown and will be set only for GD messages
current_depth = 'n/a'
# We don't always have GD messages on input so there is no point in running an SQL transaction otherwise.
if has_gd_msg_list:
# Operate under a global lock for that topic to rule out any interference from other publishers
with self.lock('zato.pubsub.publish.%s' % input.topic_name):
with closing(self.odb.session()) as session:
# Abort if max depth is already reached but check first if we should check the depth in this iteration.
topic.incr_gd_depth_check()
if topic.needs_gd_depth_check():
# Get current depth of this topic
current_depth = get_topic_depth(
session, cluster_id, topic.id)
if current_depth + len_gd_msg_list > topic.max_depth_gd:
raise ServiceUnavailable(
self.cid,
'Publication rejected - would have exceeded max depth for `{}`'
.format(topic.name))
else:
# This only updates the local variable
current_depth = current_depth + len_gd_msg_list
# This updates data in SQL
incr_topic_depth(session, cluster_id, topic.id, now,
len_gd_msg_list)
# Publish messages - INSERT rows, each representing an individual message
insert_topic_messages(session, self.cid, gd_msg_list)
# Move messages to each subscriber's queue
if subscriptions_by_topic:
insert_queue_messages(session, cluster_id,
subscriptions_by_topic,
gd_msg_list, topic.id, now)
# Run an SQL commit for all queries above
session.commit()
# Update metadata in background
spawn(self._update_pub_metadata, cluster_id, topic.id,
endpoint_id, now, gd_msg_list, pattern_matched)
# Either commit succeeded or there were no GD messages on input but in both cases we can now,
# optionally, store data in pub/sub audit log.
if has_pubsub_audit_log:
msg = 'PUB. CID:`%s`, topic:`%s`, from:`%s`, ext_client_id:`%s`, pattern:`%s`, new_depth:`%s`' \
', GD data:`%s`, non-GD data:`%s`'
logger_audit.info(msg, self.cid, topic.name,
self.pubsub.endpoints[endpoint_id].name,
input.get('ext_client_id') or 'n/a',
pattern_matched, current_depth, gd_msg_list,
non_gd_msg_list)
# Also in background, notify pub/sub task runners that there are new messages for them
if subscriptions_by_topic:
# Do not notify anything if there are no messages available - this is possible because,
# for instance, we had a list of messages on input but a hook service filtered them out.
if non_gd_msg_list or has_gd_msg_list:
self._notify_pubsub_tasks(topic.id, topic.name,
subscriptions_by_topic,
non_gd_msg_list, has_gd_msg_list)
# Return either a single msg_id if there was only one message published or a list of message IDs,
# one for each message published.
len_msg_list = len_gd_msg_list + len(non_gd_msg_list)
if len_msg_list == 1:
self.response.payload.msg_id = msg_id_list[0]
else:
self.response.payload.msg_id_list = msg_id_list