def confirm_pubsub_msg_delivered(self, sub_key, pub_msg_id):
""" Sets in SQL delivery status of a given message to True.
"""
with closing(self.server.odb.session()) as session:
_confirm_pubsub_msg_delivered(session, self.server.cluster_id,
sub_key, pub_msg_id, utcnow_as_ms())
session.commit()
def handle(self):
sub_key = self.request.input.sub_key
msg_id_list = self.request.input.msg_id_list
if msg_id_list:
with closing(self.odb.session()) as session:
# Call SQL UPDATE ..
acknowledge_delivery(
session, self.server.cluster_id, sub_key, msg_id_list, utcnow_as_ms())
# .. and confirm the transaction
session.commit()
def get_sql_messages_by_sub_key(self, sub_key, last_sql_run, session=None):
""" Returns from SQL all messages queued up for a given sub_key.
"""
if not session:
session = self.server.odb.session()
needs_close = True
else:
needs_close = False
try:
return _get_sql_messages_by_sub_key(session,
self.server.cluster_id,
sub_key, last_sql_run,
utcnow_as_ms())
finally:
if needs_close:
session.close()
def handle(self):
input = self.request.input
input.require_any('sub_key', 'sub_key_list')
# Support both on input but always pass on a list further on
sub_key_list = [input.sub_key] if input.sub_key else input.sub_key_list
# Response to return
response = {}
with closing(self.odb.session()) as session:
for item in sub_key_list:
response[item] = get_queue_depth_by_sub_key(session, self.server.cluster_id, item, utcnow_as_ms())
self.response.payload.queue_depth = response
def handle(self, _batch_size=PUBSUB.DEFAULT.GET_BATCH_SIZE):
input = self.request.input
batch_size = input.batch_size or _batch_size
with closing(self.odb.session()) as session:
msg_list = get_messages(session, self.server.cluster_id, input.sub_key, batch_size, utcnow_as_ms())
for elem in msg_list:
ext_pub_time = datetime_from_ms(elem.ext_pub_time) if elem.ext_pub_time else None
self.response.payload.append({
'msg_id': elem.msg_id,
'correl_id': elem.correl_id,
'in_reply_to': elem.in_reply_to,
'priority': elem.priority,
'size': elem.size,
'data_format': elem.data_format,
'mime_type': elem.mime_type,
'data': elem.data,
'expiration': elem.expiration,
'expiration_time': datetime_from_ms(elem.expiration_time),
'ext_client_id': elem.ext_client_id,
'ext_pub_time': ext_pub_time,
'topic_name': elem.topic_name,
'recv_time': datetime_from_ms(elem.recv_time),
'delivery_count': elem.delivery_count,
})
# We need to commit the session because the underlying query issued SELECT FOR UPDATE
session.commit()
def _cleanup(self, session):
number = delete_expired(session, self.server.cluster_id, utcnow_as_ms())
return number, 'expired'
def handle(self):
input = self.request.input
pubsub = self.server.worker_store.pubsub # type: PubSub
endpoint_id = input.endpoint_id
# Will return publication pattern matched or raise an exception that we don't catch
endpoint_id, pattern_matched = self.get_pattern_matched(
endpoint_id, input)
try:
topic = pubsub.get_topic_by_name(input.topic_name) # type: Topic
except KeyError:
raise NotFound(self.cid,
'No such topic `{}`'.format(input.topic_name))
# We always count time in milliseconds since UNIX epoch
now = utcnow_as_ms()
# If input.data is a list, it means that it is a list of messages, each of which has its own
# metadata. Otherwise, it's a string to publish and other input parameters describe it.
data_list = input.data_list if input.data_list else None
# Input messages may contain a mix of GD and non-GD messages, and we need to extract them separately.
msg_id_list, gd_msg_list, non_gd_msg_list = self._get_messages_from_data(
topic, data_list, input, now, pattern_matched, endpoint_id)
len_gd_msg_list = len(gd_msg_list)
has_gd_msg_list = bool(len_gd_msg_list)
# Get all subscribers for that topic from local worker store
subscriptions_by_topic = pubsub.get_subscriptions_by_topic(
input.topic_name)
# Just so it is not overlooked, log information that no subscribers are found for this topic
if not subscriptions_by_topic:
self.logger.warn('No subscribers found for topic `%s`',
input.topic_name)
# Local aliases
cluster_id = self.server.cluster_id
has_pubsub_audit_log = self.server.has_pubsub_audit_log
# This is initially unknown and will be set only for GD messages
current_depth = 'n/a'
# We don't always have GD messages on input so there is no point in running an SQL transaction otherwise.
if has_gd_msg_list:
# Operate under a global lock for that topic to rule out any interference from other publishers
with self.lock('zato.pubsub.publish.%s' % input.topic_name):
with closing(self.odb.session()) as session:
# Abort if max depth is already reached but check first if we should check the depth in this iteration.
topic.incr_gd_depth_check()
if topic.needs_gd_depth_check():
# Get current depth of this topic
current_depth = get_topic_depth(
session, cluster_id, topic.id)
if current_depth + len_gd_msg_list > topic.max_depth_gd:
raise ServiceUnavailable(
self.cid,
'Publication rejected - would have exceeded max depth for `{}`'
.format(topic.name))
else:
# This only updates the local variable
current_depth = current_depth + len_gd_msg_list
# This updates data in SQL
incr_topic_depth(session, cluster_id, topic.id, now,
len_gd_msg_list)
# Publish messages - INSERT rows, each representing an individual message
insert_topic_messages(session, self.cid, gd_msg_list)
# Move messages to each subscriber's queue
if subscriptions_by_topic:
insert_queue_messages(session, cluster_id,
subscriptions_by_topic,
gd_msg_list, topic.id, now)
# Run an SQL commit for all queries above
session.commit()
# Update metadata in background
spawn(self._update_pub_metadata, cluster_id, topic.id,
endpoint_id, now, gd_msg_list, pattern_matched)
# Either commit succeeded or there were no GD messages on input but in both cases we can now,
# optionally, store data in pub/sub audit log.
if has_pubsub_audit_log:
msg = 'PUB. CID:`%s`, topic:`%s`, from:`%s`, ext_client_id:`%s`, pattern:`%s`, new_depth:`%s`' \
', GD data:`%s`, non-GD data:`%s`'
logger_audit.info(msg, self.cid, topic.name,
self.pubsub.endpoints[endpoint_id].name,
input.get('ext_client_id') or 'n/a',
pattern_matched, current_depth, gd_msg_list,
non_gd_msg_list)
# Also in background, notify pub/sub task runners that there are new messages for them
if subscriptions_by_topic:
# Do not notify anything if there are no messages available - this is possible because,
# for instance, we had a list of messages on input but a hook service filtered them out.
if non_gd_msg_list or has_gd_msg_list:
self._notify_pubsub_tasks(topic.id, topic.name,
subscriptions_by_topic,
non_gd_msg_list, has_gd_msg_list)
# Return either a single msg_id if there was only one message published or a list of message IDs,
# one for each message published.
len_msg_list = len_gd_msg_list + len(non_gd_msg_list)
if len_msg_list == 1:
self.response.payload.msg_id = msg_id_list[0]
else:
self.response.payload.msg_id_list = msg_id_list
def _subscribe_impl(self, ctx):
""" Invoked by subclasses to subscribe callers using input pub/sub config context.
"""
with self.lock('zato.pubsub.subscribe.%s.%s' %
(ctx.topic_name, ctx.endpoint_id)):
with closing(self.odb.session()) as session:
# Non-WebSocket clients cannot subscribe to the same topic multiple times
if not ctx.ws_channel_id:
if has_subscription(session, ctx.cluster_id, ctx.topic.id,
ctx.endpoint_id):
raise PubSubSubscriptionExists(
self.cid,
'Subscription to topic `{}` already exists'.format(
ctx.topic.name))
ctx.creation_time = now = utcnow_as_ms()
ctx.sub_key = new_sub_key()
# If we subscribe a WSX client, we need to create its accompanying SQL models
if ctx.ws_channel_id:
# This object persists across multiple WSX connections
add_wsx_subscription(session, ctx.cluster_id,
ctx.is_internal, ctx.sub_key,
ctx.ext_client_id, ctx.ws_channel_id)
# This object will be transient - dropped each time a WSX disconnects
self.pubsub.add_ws_client_pubsub_keys(
session, ctx.sql_ws_client_id, ctx.sub_key,
ctx.ws_channel_name, ctx.ws_pub_client_id)
# Let the WebSocket connection object know that it should handle this particular sub_key
ctx.web_socket.pubsub_tool.add_sub_key(ctx.sub_key)
# Create a new subscription object
ps_sub = add_subscription(session, ctx.cluster_id, ctx)
# Flush the session because we need the subscription's ID below in INSERT from SELECT
session.flush()
# Move all available messages to that subscriber's queue
total_moved = move_messages_to_sub_queue(
session, ctx.cluster_id, ctx.topic.id, ctx.endpoint_id,
ps_sub.id, now)
# Commit all changes
session.commit()
# Produce response
self.response.payload.sub_key = ctx.sub_key
self.response.payload.queue_depth = total_moved
# Notify workers of a new subscription
broker_input = Bunch()
broker_input.topic_name = ctx.topic.name
broker_input.endpoint_type = self.endpoint_type
for name in sub_broker_attrs:
broker_input[name] = getattr(ps_sub, name, None)
broker_input.action = BROKER_MSG_PUBSUB.SUBSCRIPTION_CREATE.value
self.broker_client.publish(broker_input)