def confirm_pubsub_msg_delivered(self, sub_key, pub_msg_id): """ Sets in SQL delivery status of a given message to True. """ with closing(self.server.odb.session()) as session: _confirm_pubsub_msg_delivered(session, self.server.cluster_id, sub_key, pub_msg_id, utcnow_as_ms()) session.commit()
def handle(self): sub_key = self.request.input.sub_key msg_id_list = self.request.input.msg_id_list if msg_id_list: with closing(self.odb.session()) as session: # Call SQL UPDATE .. acknowledge_delivery( session, self.server.cluster_id, sub_key, msg_id_list, utcnow_as_ms()) # .. and confirm the transaction session.commit()
def get_sql_messages_by_sub_key(self, sub_key, last_sql_run, session=None): """ Returns from SQL all messages queued up for a given sub_key. """ if not session: session = self.server.odb.session() needs_close = True else: needs_close = False try: return _get_sql_messages_by_sub_key(session, self.server.cluster_id, sub_key, last_sql_run, utcnow_as_ms()) finally: if needs_close: session.close()
def handle(self): input = self.request.input input.require_any('sub_key', 'sub_key_list') # Support both on input but always pass on a list further on sub_key_list = [input.sub_key] if input.sub_key else input.sub_key_list # Response to return response = {} with closing(self.odb.session()) as session: for item in sub_key_list: response[item] = get_queue_depth_by_sub_key(session, self.server.cluster_id, item, utcnow_as_ms()) self.response.payload.queue_depth = response
def handle(self, _batch_size=PUBSUB.DEFAULT.GET_BATCH_SIZE): input = self.request.input batch_size = input.batch_size or _batch_size with closing(self.odb.session()) as session: msg_list = get_messages(session, self.server.cluster_id, input.sub_key, batch_size, utcnow_as_ms()) for elem in msg_list: ext_pub_time = datetime_from_ms(elem.ext_pub_time) if elem.ext_pub_time else None self.response.payload.append({ 'msg_id': elem.msg_id, 'correl_id': elem.correl_id, 'in_reply_to': elem.in_reply_to, 'priority': elem.priority, 'size': elem.size, 'data_format': elem.data_format, 'mime_type': elem.mime_type, 'data': elem.data, 'expiration': elem.expiration, 'expiration_time': datetime_from_ms(elem.expiration_time), 'ext_client_id': elem.ext_client_id, 'ext_pub_time': ext_pub_time, 'topic_name': elem.topic_name, 'recv_time': datetime_from_ms(elem.recv_time), 'delivery_count': elem.delivery_count, }) # We need to commit the session because the underlying query issued SELECT FOR UPDATE session.commit()
def _cleanup(self, session): number = delete_expired(session, self.server.cluster_id, utcnow_as_ms()) return number, 'expired'
def handle(self): input = self.request.input pubsub = self.server.worker_store.pubsub # type: PubSub endpoint_id = input.endpoint_id # Will return publication pattern matched or raise an exception that we don't catch endpoint_id, pattern_matched = self.get_pattern_matched( endpoint_id, input) try: topic = pubsub.get_topic_by_name(input.topic_name) # type: Topic except KeyError: raise NotFound(self.cid, 'No such topic `{}`'.format(input.topic_name)) # We always count time in milliseconds since UNIX epoch now = utcnow_as_ms() # If input.data is a list, it means that it is a list of messages, each of which has its own # metadata. Otherwise, it's a string to publish and other input parameters describe it. data_list = input.data_list if input.data_list else None # Input messages may contain a mix of GD and non-GD messages, and we need to extract them separately. msg_id_list, gd_msg_list, non_gd_msg_list = self._get_messages_from_data( topic, data_list, input, now, pattern_matched, endpoint_id) len_gd_msg_list = len(gd_msg_list) has_gd_msg_list = bool(len_gd_msg_list) # Get all subscribers for that topic from local worker store subscriptions_by_topic = pubsub.get_subscriptions_by_topic( input.topic_name) # Just so it is not overlooked, log information that no subscribers are found for this topic if not subscriptions_by_topic: self.logger.warn('No subscribers found for topic `%s`', input.topic_name) # Local aliases cluster_id = self.server.cluster_id has_pubsub_audit_log = self.server.has_pubsub_audit_log # This is initially unknown and will be set only for GD messages current_depth = 'n/a' # We don't always have GD messages on input so there is no point in running an SQL transaction otherwise. if has_gd_msg_list: # Operate under a global lock for that topic to rule out any interference from other publishers with self.lock('zato.pubsub.publish.%s' % input.topic_name): with closing(self.odb.session()) as session: # Abort if max depth is already reached but check first if we should check the depth in this iteration. topic.incr_gd_depth_check() if topic.needs_gd_depth_check(): # Get current depth of this topic current_depth = get_topic_depth( session, cluster_id, topic.id) if current_depth + len_gd_msg_list > topic.max_depth_gd: raise ServiceUnavailable( self.cid, 'Publication rejected - would have exceeded max depth for `{}`' .format(topic.name)) else: # This only updates the local variable current_depth = current_depth + len_gd_msg_list # This updates data in SQL incr_topic_depth(session, cluster_id, topic.id, now, len_gd_msg_list) # Publish messages - INSERT rows, each representing an individual message insert_topic_messages(session, self.cid, gd_msg_list) # Move messages to each subscriber's queue if subscriptions_by_topic: insert_queue_messages(session, cluster_id, subscriptions_by_topic, gd_msg_list, topic.id, now) # Run an SQL commit for all queries above session.commit() # Update metadata in background spawn(self._update_pub_metadata, cluster_id, topic.id, endpoint_id, now, gd_msg_list, pattern_matched) # Either commit succeeded or there were no GD messages on input but in both cases we can now, # optionally, store data in pub/sub audit log. if has_pubsub_audit_log: msg = 'PUB. CID:`%s`, topic:`%s`, from:`%s`, ext_client_id:`%s`, pattern:`%s`, new_depth:`%s`' \ ', GD data:`%s`, non-GD data:`%s`' logger_audit.info(msg, self.cid, topic.name, self.pubsub.endpoints[endpoint_id].name, input.get('ext_client_id') or 'n/a', pattern_matched, current_depth, gd_msg_list, non_gd_msg_list) # Also in background, notify pub/sub task runners that there are new messages for them if subscriptions_by_topic: # Do not notify anything if there are no messages available - this is possible because, # for instance, we had a list of messages on input but a hook service filtered them out. if non_gd_msg_list or has_gd_msg_list: self._notify_pubsub_tasks(topic.id, topic.name, subscriptions_by_topic, non_gd_msg_list, has_gd_msg_list) # Return either a single msg_id if there was only one message published or a list of message IDs, # one for each message published. len_msg_list = len_gd_msg_list + len(non_gd_msg_list) if len_msg_list == 1: self.response.payload.msg_id = msg_id_list[0] else: self.response.payload.msg_id_list = msg_id_list
def _subscribe_impl(self, ctx): """ Invoked by subclasses to subscribe callers using input pub/sub config context. """ with self.lock('zato.pubsub.subscribe.%s.%s' % (ctx.topic_name, ctx.endpoint_id)): with closing(self.odb.session()) as session: # Non-WebSocket clients cannot subscribe to the same topic multiple times if not ctx.ws_channel_id: if has_subscription(session, ctx.cluster_id, ctx.topic.id, ctx.endpoint_id): raise PubSubSubscriptionExists( self.cid, 'Subscription to topic `{}` already exists'.format( ctx.topic.name)) ctx.creation_time = now = utcnow_as_ms() ctx.sub_key = new_sub_key() # If we subscribe a WSX client, we need to create its accompanying SQL models if ctx.ws_channel_id: # This object persists across multiple WSX connections add_wsx_subscription(session, ctx.cluster_id, ctx.is_internal, ctx.sub_key, ctx.ext_client_id, ctx.ws_channel_id) # This object will be transient - dropped each time a WSX disconnects self.pubsub.add_ws_client_pubsub_keys( session, ctx.sql_ws_client_id, ctx.sub_key, ctx.ws_channel_name, ctx.ws_pub_client_id) # Let the WebSocket connection object know that it should handle this particular sub_key ctx.web_socket.pubsub_tool.add_sub_key(ctx.sub_key) # Create a new subscription object ps_sub = add_subscription(session, ctx.cluster_id, ctx) # Flush the session because we need the subscription's ID below in INSERT from SELECT session.flush() # Move all available messages to that subscriber's queue total_moved = move_messages_to_sub_queue( session, ctx.cluster_id, ctx.topic.id, ctx.endpoint_id, ps_sub.id, now) # Commit all changes session.commit() # Produce response self.response.payload.sub_key = ctx.sub_key self.response.payload.queue_depth = total_moved # Notify workers of a new subscription broker_input = Bunch() broker_input.topic_name = ctx.topic.name broker_input.endpoint_type = self.endpoint_type for name in sub_broker_attrs: broker_input[name] = getattr(ps_sub, name, None) broker_input.action = BROKER_MSG_PUBSUB.SUBSCRIPTION_CREATE.value self.broker_client.publish(broker_input)