def __init__(self, kvdb, client_type, topic_callbacks, initial_lua_programs): self.kvdb = kvdb self.decrypt_func = kvdb.decrypt_func self.name = '{}-{}'.format(client_type, new_cid()) self.topic_callbacks = topic_callbacks self.lua_container = LuaContainer(self.kvdb.conn, initial_lua_programs) self.ready = False
class _BrokerClient(object): """ Zato broker client. Starts two background threads, one for publishing and one for receiving of the messages. There may be 3 types of messages sent out: 1) to the singleton server 2) to all the servers/connectors/all connectors of a certain type (e.g. only AMQP ones) 3) to one of the parallel servers 1) and 2) are straightforward, a message is being published on a topic, off which it is read by broker client(s). 3) needs more work - the actual message is added to Redis and what is really being published is a Redis key it's been stored under. The first client to read it will be the one to handle it. Yup, it means the messages are sent across to all of the clients and the winning one is the one that picked up the Redis message; it's not that bad as it may seem, there will be at most as many clients as there are servers in the cluster and truth to be told, Zero MQ < 3.x also would do client-side PUB/SUB filtering and it did scale nicely. """ def __init__(self, kvdb, client_type, topic_callbacks, initial_lua_programs): self.kvdb = kvdb self.decrypt_func = kvdb.decrypt_func self.name = '{}-{}'.format(client_type, new_cid()) self.topic_callbacks = topic_callbacks self.lua_container = LuaContainer(self.kvdb.conn, initial_lua_programs) self.ready = False def run(self): logger.debug('Starting broker client, host:`%s`, port:`%s`, name:`%s`, topics:`%s`', self.kvdb.config.host, self.kvdb.config.port, self.name, sorted(self.topic_callbacks)) self.pub_client = _ClientThread(self.kvdb.copy(), 'pub', self.name) self.sub_client = _ClientThread(self.kvdb.copy(), 'sub', self.name, self.topic_callbacks, self.on_message) start_new_thread(self.pub_client.run, ()) start_new_thread(self.sub_client.run, ()) for client in(self.pub_client, self.sub_client): while client.keep_running == ZATO_NONE: time.sleep(0.01) self.ready = True def publish(self, msg, msg_type=MESSAGE_TYPE.TO_PARALLEL_ALL, *ignored_args, **ignored_kwargs): msg['msg_type'] = msg_type topic = TOPICS[msg_type] msg = dumps(msg) self.pub_client.publish(topic, msg) def invoke_async(self, msg, msg_type=MESSAGE_TYPE.TO_PARALLEL_ANY, expiration=BROKER.DEFAULT_EXPIRATION): msg['msg_type'] = msg_type try: msg = dumps(msg) except Exception: error_msg = 'JSON serialization failed for msg:`%r`, e:`%s`' logger.error(error_msg, msg, format_exc()) raise else: topic = TOPICS[msg_type] key = broker_msg = 'zato:broker{}:{}'.format(KEYS[msg_type], new_cid()) self.kvdb.conn.set(key, str(msg)) self.kvdb.conn.expire(key, expiration) # In seconds self.pub_client.publish(topic, broker_msg) def on_message(self, msg): if has_debug: logger.warn('Got broker message `%s`', msg) if msg.type == 'message': # Replace payload with stuff read off the KVDB in case this is where the actual message happens to reside. if msg.channel in NEEDS_TMP_KEY: tmp_key = '{}.tmp'.format(msg.data) if self.lua_container.run_lua('zato.rename_if_exists', [msg.data, tmp_key]) == CODE_NO_SUCH_FROM_KEY: payload = None else: payload = self.kvdb.conn.get(tmp_key) self.kvdb.conn.delete(tmp_key) # Note that it would've expired anyway if not payload: logger.info('No KVDB payload for key `%s` (already expired?)', tmp_key) else: if isinstance(payload, bytes): payload = payload payload = loads(payload) else: if isinstance(msg.data, bytes): msg.data = msg.data payload = loads(msg.data) if payload: payload = Bunch(payload) if has_debug: logger.debug('Got broker message payload `%s`', payload) callback = self.topic_callbacks[msg.channel] spawn_greenlet(callback, payload) else: if has_debug: logger.debug('No payload in msg: `%s`', msg) def close(self): for client in(self.pub_client, self.sub_client): client.keep_running = False client.kvdb.close()