def _impl(): while self.keep_running: try: msg = self.conn.receive(self.queue_name, 100) if self.has_debug: self.logger.debug('Message received `%s`' % str(msg).decode('utf-8')) if msg == _connection_closing: self.logger.info( 'Received request to quit, closing channel for queue `%s` (%s)', self.queue_name, self.conn.get_connection_info()) self.keep_running = False return if msg: start_new_thread(_invoke_callback, (_MessageCtx( msg, self.id, self.queue_name, self.service_name, self.data_format), )) except NoMessageAvailableException as e: if self.has_debug: self.logger.debug( 'Consumer for queue `%s` did not receive a message. `%s`' % (self.queue_name, self._get_destination_info(self.queue_name))) except self.pymqi.MQMIError as e: if e.reason == self.pymqi.CMQC.MQRC_UNKNOWN_OBJECT_NAME: self.logger.warn('No such queue `%s` found for %s', self.queue_name, self.conn.get_connection_info()) else: self.logger.warn( '%s in run, reason_code:`%s`, comp_code:`%s`' % (e.__class__.__name__, e.reason, e.comp)) # In case of any low-level PyMQI error, sleep for some time # because there is nothing we can do at this time about it. self.logger.info('Sleeping for %ss', sleep_on_error) sleep(sleep_on_error) except WebSphereMQException as e: # If current connection is broken we may try to re-estalish it. sleep(sleep_on_error) if e.completion_code == _cc_failed and e.reason_code == _rc_conn_broken: self.logger.warn( 'Caught MQRC_CONNECTION_BROKEN in receive, will try to reconnect connection to %s ', self.conn.get_connection_info()) self.conn.reconnect() self.conn.ping() else: raise except Exception as e: self.logger.error('Exception in the main loop %r %s %s', e.args, type(e), format_exc()) sleep(sleep_on_error)
def run(self): logger.debug('Starting broker client, host:`%s`, port:`%s`, name:`%s`, topics:`%s`', self.kvdb.config.host, self.kvdb.config.port, self.name, sorted(self.topic_callbacks)) self.pub_client = _ClientThread(self.kvdb.copy(), 'pub', self.name) self.sub_client = _ClientThread(self.kvdb.copy(), 'sub', self.name, self.topic_callbacks, self.on_message) start_new_thread(self.pub_client.run, ()) start_new_thread(self.sub_client.run, ()) for client in(self.pub_client, self.sub_client): while client.keep_running == ZATO_NONE: time.sleep(0.01) self.ready = True
def BrokerClient(kvdb, client_type, topic_callbacks, _initial_lua_programs): # Imported here so it's guaranteed to be monkey-patched using gevent.monkey.patch_all by whoever called us from zato.common.py23_ import start_new_thread class _ClientThread(object): def __init__(self, kvdb, pubsub, name, topic_callbacks=None, on_message=None): self.kvdb = kvdb.copy() self.kvdb.init() self.pubsub = pubsub self.topic_callbacks = topic_callbacks self.on_message = on_message self.client = None self.keep_running = ZATO_NONE self.connect_sleep_time = 1 def set_up_pub_sub_client(self): try: self.kvdb = self.kvdb.copy() self.kvdb.init() self.kvdb.conn.ping() self.client = self.kvdb.pubsub() self.client.subscribe(self.topic_callbacks.keys()) except Exception: logger.warn('Redis connection error, will retry after %ss.\n%s', self.connect_sleep_time, format_exc()) sleep(self.connect_sleep_time) def run(self): # We're in a new thread and we can initialize the KVDB connection now. self.kvdb.init() if self.pubsub == 'sub': self.set_up_pub_sub_client() self.keep_running = True try: while self.keep_running: try: for msg in self.client.listen(): try: msg = Bunch(msg) msg.channel = msg.channel if isinstance(msg.data, bytes): msg.data = msg.data self.on_message(msg) except Exception: logger.warn('Could not handle broker message `%s`, e:`%s`', msg, format_exc()) except redis.ConnectionError as e: if e.message not in EXPECTED_CONNECTION_ERRORS: # Hm, there's no error code, only the message logger.warn('Caught Redis exception `%s`', e.message) self.set_up_pub_sub_client() except KeyboardInterrupt: self.keep_running = False else: self.client = self.kvdb self.keep_running = True def publish(self, topic, msg): if has_debug: logger.debug('Publishing `%r` (%s) to `%s` (%s)', msg, type(msg), topic, self.client) return self.client.publish(topic, msg) def close(self): self.keep_running = False self.client.close() class _BrokerClient(object): """ Zato broker client. Starts two background threads, one for publishing and one for receiving of the messages. There may be 3 types of messages sent out: 1) to the singleton server 2) to all the servers/connectors/all connectors of a certain type (e.g. only AMQP ones) 3) to one of the parallel servers 1) and 2) are straightforward, a message is being published on a topic, off which it is read by broker client(s). 3) needs more work - the actual message is added to Redis and what is really being published is a Redis key it's been stored under. The first client to read it will be the one to handle it. Yup, it means the messages are sent across to all of the clients and the winning one is the one that picked up the Redis message; it's not that bad as it may seem, there will be at most as many clients as there are servers in the cluster and truth to be told, Zero MQ < 3.x also would do client-side PUB/SUB filtering and it did scale nicely. """ def __init__(self, kvdb, client_type, topic_callbacks, initial_lua_programs): self.kvdb = kvdb self.decrypt_func = kvdb.decrypt_func self.name = '{}-{}'.format(client_type, new_cid()) self.topic_callbacks = topic_callbacks self.lua_container = LuaContainer(self.kvdb.conn, initial_lua_programs) self.ready = False def run(self): logger.debug('Starting broker client, host:`%s`, port:`%s`, name:`%s`, topics:`%s`', self.kvdb.config.host, self.kvdb.config.port, self.name, sorted(self.topic_callbacks)) self.pub_client = _ClientThread(self.kvdb.copy(), 'pub', self.name) self.sub_client = _ClientThread(self.kvdb.copy(), 'sub', self.name, self.topic_callbacks, self.on_message) start_new_thread(self.pub_client.run, ()) start_new_thread(self.sub_client.run, ()) for client in(self.pub_client, self.sub_client): while client.keep_running == ZATO_NONE: time.sleep(0.01) self.ready = True def publish(self, msg, msg_type=MESSAGE_TYPE.TO_PARALLEL_ALL, *ignored_args, **ignored_kwargs): msg['msg_type'] = msg_type topic = TOPICS[msg_type] msg = dumps(msg) self.pub_client.publish(topic, msg) def invoke_async(self, msg, msg_type=MESSAGE_TYPE.TO_PARALLEL_ANY, expiration=BROKER.DEFAULT_EXPIRATION): msg['msg_type'] = msg_type try: msg = dumps(msg) except Exception: error_msg = 'JSON serialization failed for msg:`%r`, e:`%s`' logger.error(error_msg, msg, format_exc()) raise else: topic = TOPICS[msg_type] key = broker_msg = 'zato:broker{}:{}'.format(KEYS[msg_type], new_cid()) self.kvdb.conn.set(key, str(msg)) self.kvdb.conn.expire(key, expiration) # In seconds self.pub_client.publish(topic, broker_msg) def on_message(self, msg): if has_debug: logger.warn('Got broker message `%s`', msg) if msg.type == 'message': # Replace payload with stuff read off the KVDB in case this is where the actual message happens to reside. if msg.channel in NEEDS_TMP_KEY: tmp_key = '{}.tmp'.format(msg.data) if self.lua_container.run_lua('zato.rename_if_exists', [msg.data, tmp_key]) == CODE_NO_SUCH_FROM_KEY: payload = None else: payload = self.kvdb.conn.get(tmp_key) self.kvdb.conn.delete(tmp_key) # Note that it would've expired anyway if not payload: logger.info('No KVDB payload for key `%s` (already expired?)', tmp_key) else: if isinstance(payload, bytes): payload = payload payload = loads(payload) else: if isinstance(msg.data, bytes): msg.data = msg.data payload = loads(msg.data) if payload: payload = Bunch(payload) if has_debug: logger.debug('Got broker message payload `%s`', payload) callback = self.topic_callbacks[msg.channel] spawn_greenlet(callback, payload) else: if has_debug: logger.debug('No payload in msg: `%s`', msg) def close(self): for client in(self.pub_client, self.sub_client): client.keep_running = False client.kvdb.close() client = _BrokerClient(kvdb, client_type, topic_callbacks, _initial_lua_programs) start_new_thread(client.run, ()) return client
def _impl(): while self.keep_running: try: msg = self.conn.receive(self.queue_name, 100) if self.has_debug: self.logger.debug('Message received `%s`' % str(msg).decode('utf-8')) if msg == _connection_closing: self.logger.info('Received request to quit, closing channel for queue `%s` (%s)', self.queue_name, self.conn.get_connection_info()) self.keep_running = False return if msg: start_new_thread(_invoke_callback, ( _MessageCtx(msg, self.id, self.queue_name, self.service_name, self.data_format),)) except NoMessageAvailableException as e: if self.has_debug: self.logger.debug('Consumer for queue `%s` did not receive a message. `%s`' % ( self.queue_name, self._get_destination_info(self.queue_name))) except self.pymqi.MQMIError as e: if e.reason == self.pymqi.CMQC.MQRC_UNKNOWN_OBJECT_NAME: self.logger.warn('No such queue `%s` found for %s', self.queue_name, self.conn.get_connection_info()) else: self.logger.warn('%s in run, reason_code:`%s`, comp_code:`%s`' % ( e.__class__.__name__, e.reason, e.comp)) # In case of any low-level PyMQI error, sleep for some time # because there is nothing we can do at this time about it. self.logger.info('Sleeping for %ss', sleep_on_error) sleep(sleep_on_error) except WebSphereMQException as e: sleep(sleep_on_error) conn_info = self.conn.get_connection_info() # Try to reconnect if the reason code points to one that is of a transient nature while self.keep_running and e.completion_code == _cc_failed and e.reason_code in _rc_reconnect_list: try: self.logger.warn('Reconnecting channel `%s` due to MQRC `%s` and MQCC `%s`', conn_info, e.reason_code, e.completion_code) self.conn.reconnect() self.conn.ping() break except WebSphereMQException as exc: e = exc sleep(sleep_on_error) except Exception as e: self.logger.error('Stopping channel `%s` due to `%s`', conn_info, format_exc()) raise else: self.logger.error( 'Stopped channel `%s` due to MQRC `%s` and MQCC `%s`', conn_info, e.reason_code, e.completion_code) raise except Exception as e: self.logger.error('Exception in the main loop %r %s %s', e.args, type(e), format_exc()) sleep(sleep_on_error)