def handle(self): # Get basic pub/sub subscription context ctx = self._get_sub_ctx() # Confirm correctness of input data, including whether the caller can subscribe # to this topic and if the topic exists at all. ctx.sub_pattern_matched = self._get_sub_pattern_matched( ctx.topic_name, ctx.ws_channel_id, ctx.sql_ws_client_id, ctx.security_id, ctx.endpoint_id) try: topic = ctx.pubsub.get_topic_by_name(ctx.topic_name) except KeyError: raise NotFound(self.cid, 'No such topic `{}`'.format(ctx.topic_name)) else: ctx.topic = topic # Inherit GD from topic if it is not set explicitly ctx.has_gd = ctx.has_gd if isinstance(ctx.has_gd, bool) else topic.has_gd # Ok, we can actually subscribe the caller now self._handle_subscription(ctx)
def handle(self): input = self.request.input pubsub = self.server.worker_store.pubsub # type: PubSub endpoint_id = input.endpoint_id # Will return publication pattern matched or raise an exception that we don't catch endpoint_id, pattern_matched = self.get_pattern_matched( endpoint_id, input) try: topic = pubsub.get_topic_by_name(input.topic_name) # type: Topic except KeyError: raise NotFound(self.cid, 'No such topic `{}`'.format(input.topic_name)) # We always count time in milliseconds since UNIX epoch now = utcnow_as_ms() # Get all subscribers for that topic from local worker store subscriptions_by_topic = pubsub.get_subscriptions_by_topic(topic.name) # If input.data is a list, it means that it is a list of messages, each of which has its own # metadata. Otherwise, it's a string to publish and other input parameters describe it. data_list = input.data_list if input.data_list else None # Input messages may contain a mix of GD and non-GD messages, and we need to extract them separately. msg_id_list, gd_msg_list, non_gd_msg_list = self._get_messages_from_data( topic, data_list, input, now, pattern_matched, endpoint_id, bool(subscriptions_by_topic)) # Create a wrapper object for all the input data and metadata ctx = PubCtx(self.server.cluster_id, pubsub, topic, endpoint_id, pubsub.get_endpoint_by_id(endpoint_id).name, subscriptions_by_topic, msg_id_list, gd_msg_list, non_gd_msg_list, pattern_matched, input.get('ext_client_id'), False, now) # We have all the input data, publish the message(s) now self._publish(ctx)
def handle(self): with closing(self.odb.session()) as session: ps_msg = session.query(PubSubMessage).\ filter(PubSubMessage.cluster_id==self.request.input.cluster_id).\ filter(PubSubMessage.pub_msg_id==self.request.input.msg_id).\ first() if not ps_msg: raise NotFound(self.cid, 'Message not found `{}`'.format(self.request.input.msg_id)) ps_topic = session.query(PubSubTopic).\ filter(PubSubTopic.cluster_id==self.request.input.cluster_id).\ filter(PubSubTopic.id==ps_msg.topic_id).\ one() # Delete the message and decrement its topic's current depth .. session.delete(ps_msg) ps_topic.current_depth = ps_topic.current_depth - 1 # .. but do it under a global lock because other transactions may want to update the topic in parallel. with self.lock('zato.pubsub.publish.%s' % ps_topic.name): session.commit()
def handle_POST(self): """ Stores new cache entries or updates existing ones, including setting their expiry time. """ input = self.request.input key = input['key'] cache = self._get_cache(input) # This is an update of value and, possibly, an entry's expiry if input.get('value'): prev_value = cache.set(input['key'], input.value, input.get('expiry') or 0.0) if input.get('return_prev'): self.response.payload.prev_value = prev_value # We only update the expiry time else: if not input.get('expiry'): raise BadRequest( self.cid, 'At least one of `value` or `expiry` is needed on input') else: found_key = cache.expire(input['key'], input.expiry) if not found_key: raise NotFound(self.cid, 'No such key `{}`'.format(key))
def handle(self): input = self.request.input pubsub = self.server.worker_store.pubsub # type: PubSub endpoint_id = input.endpoint_id # Will return publication pattern matched or raise an exception that we don't catch endpoint_id, pub_pattern_matched = self.get_pub_pattern_matched( endpoint_id, input) try: topic = pubsub.get_topic_by_name(input.topic_name) # type: Topic except KeyError: raise NotFound(self.cid, 'No such topic `{}`'.format(input.topic_name)) # Reject the message is topic is not active if not topic.is_active: raise ServiceUnavailable( self.cid, 'Topic is inactive `{}`'.format(input.topic_name)) # We always count time in milliseconds since UNIX epoch now = utcnow_as_ms() # Get all subscribers for that topic from local worker store all_subscriptions_by_topic = pubsub.get_subscriptions_by_topic( topic.name) len_all_sub = len(all_subscriptions_by_topic) # If we are to deliver the message(s) to only selected subscribers only, # filter out any unwated ones first. if input.deliver_to_sk: has_all = False subscriptions_by_topic = [] # Get any matching subscriptions out of the whole set for sub in all_subscriptions_by_topic: if sub.sub_key in input.deliver_to_sk: subscriptions_by_topic.append(sub) else: # We deliver this message to all of the topic's subscribers has_all = True subscriptions_by_topic = all_subscriptions_by_topic # This is only for logging purposes _subs_found = [] # Assume that there are no missing servers for WSX clients by default has_wsx_no_server = False for sub in subscriptions_by_topic: # Prepare data for logging _subs_found.append({sub.sub_key: sub.sub_pattern_matched}) # Is there at least one WSX subscriber to this topic that is currently not connected? # If so, later on we will need to turn all the messages into GD ones. sk_server = self.pubsub.get_sub_key_server(sub.sub_key) if not sk_server: if has_logger_pubsub_debug: logger_pubsub.debug( 'No sk_server for sub_key `%s` among `%s`', sub.sub_key, sorted(self.pubsub.sub_key_servers.keys())) has_wsx_no_server = True # We have found at least one WSX subscriber that has no server = it is not connected logger_pubsub.info( 'Subscriptions for topic `%s` `%s` (a:%d, %d/%d, cid:%s)', topic.name, _subs_found, has_all, len(subscriptions_by_topic), len_all_sub, self.cid) # If input.data is a list, it means that it is a list of messages, each of which has its own # metadata. Otherwise, it's a string to publish and other input parameters describe it. data_list = input.data_list if input.data_list else None # Input messages may contain a mix of GD and non-GD messages, and we need to extract them separately. msg_id_list, gd_msg_list, non_gd_msg_list = self._get_messages_from_data( topic, data_list, input, now, pub_pattern_matched, endpoint_id, subscriptions_by_topic, has_wsx_no_server, input.get('reply_to_sk', None)) # Create a wrapper object for all the input data and metadata ctx = PubCtx(self.server.cluster_id, pubsub, topic, endpoint_id, pubsub.get_endpoint_by_id(endpoint_id).name, subscriptions_by_topic, msg_id_list, gd_msg_list, non_gd_msg_list, pub_pattern_matched, input.get('ext_client_id'), False, now) # We have all the input data, publish the message(s) now self._publish(ctx)
def handle(self): input = self.request.input pubsub = self.server.worker_store.pubsub # type: PubSub endpoint_id = input.endpoint_id # Will return publication pattern matched or raise an exception that we don't catch endpoint_id, pattern_matched = self.get_pattern_matched( endpoint_id, input) try: topic = pubsub.get_topic_by_name(input.topic_name) # type: Topic except KeyError: raise NotFound(self.cid, 'No such topic `{}`'.format(input.topic_name)) # We always count time in milliseconds since UNIX epoch now = utcnow_as_ms() # If input.data is a list, it means that it is a list of messages, each of which has its own # metadata. Otherwise, it's a string to publish and other input parameters describe it. data_list = input.data_list if input.data_list else None # Input messages may contain a mix of GD and non-GD messages, and we need to extract them separately. msg_id_list, gd_msg_list, non_gd_msg_list = self._get_messages_from_data( topic, data_list, input, now, pattern_matched, endpoint_id) len_gd_msg_list = len(gd_msg_list) has_gd_msg_list = bool(len_gd_msg_list) # Get all subscribers for that topic from local worker store subscriptions_by_topic = pubsub.get_subscriptions_by_topic( input.topic_name) # Just so it is not overlooked, log information that no subscribers are found for this topic if not subscriptions_by_topic: self.logger.warn('No subscribers found for topic `%s`', input.topic_name) # Local aliases cluster_id = self.server.cluster_id has_pubsub_audit_log = self.server.has_pubsub_audit_log # This is initially unknown and will be set only for GD messages current_depth = 'n/a' # We don't always have GD messages on input so there is no point in running an SQL transaction otherwise. if has_gd_msg_list: # Operate under a global lock for that topic to rule out any interference from other publishers with self.lock('zato.pubsub.publish.%s' % input.topic_name): with closing(self.odb.session()) as session: # Abort if max depth is already reached but check first if we should check the depth in this iteration. topic.incr_gd_depth_check() if topic.needs_gd_depth_check(): # Get current depth of this topic current_depth = get_topic_depth( session, cluster_id, topic.id) if current_depth + len_gd_msg_list > topic.max_depth_gd: raise ServiceUnavailable( self.cid, 'Publication rejected - would have exceeded max depth for `{}`' .format(topic.name)) else: # This only updates the local variable current_depth = current_depth + len_gd_msg_list # This updates data in SQL incr_topic_depth(session, cluster_id, topic.id, now, len_gd_msg_list) # Publish messages - INSERT rows, each representing an individual message insert_topic_messages(session, self.cid, gd_msg_list) # Move messages to each subscriber's queue if subscriptions_by_topic: insert_queue_messages(session, cluster_id, subscriptions_by_topic, gd_msg_list, topic.id, now) # Run an SQL commit for all queries above session.commit() # Update metadata in background spawn(self._update_pub_metadata, cluster_id, topic.id, endpoint_id, now, gd_msg_list, pattern_matched) # Either commit succeeded or there were no GD messages on input but in both cases we can now, # optionally, store data in pub/sub audit log. if has_pubsub_audit_log: msg = 'PUB. CID:`%s`, topic:`%s`, from:`%s`, ext_client_id:`%s`, pattern:`%s`, new_depth:`%s`' \ ', GD data:`%s`, non-GD data:`%s`' logger_audit.info(msg, self.cid, topic.name, self.pubsub.endpoints[endpoint_id].name, input.get('ext_client_id') or 'n/a', pattern_matched, current_depth, gd_msg_list, non_gd_msg_list) # Also in background, notify pub/sub task runners that there are new messages for them if subscriptions_by_topic: # Do not notify anything if there are no messages available - this is possible because, # for instance, we had a list of messages on input but a hook service filtered them out. if non_gd_msg_list or has_gd_msg_list: self._notify_pubsub_tasks(topic.id, topic.name, subscriptions_by_topic, non_gd_msg_list, has_gd_msg_list) # Return either a single msg_id if there was only one message published or a list of message IDs, # one for each message published. len_msg_list = len_gd_msg_list + len(non_gd_msg_list) if len_msg_list == 1: self.response.payload.msg_id = msg_id_list[0] else: self.response.payload.msg_id_list = msg_id_list