def test_22_info(self, r): """ Older Redis versions contained 'allocation_stats' in INFO that was the cause of a number of bugs when parsing. """ info = "allocation_stats:6=1,7=1,8=7141,9=180,10=92,11=116,12=5330," \ "13=123,14=3091,15=11048,16=225842,17=1784,18=814,19=12020," \ "20=2530,21=645,22=15113,23=8695,24=142860,25=318,26=3303," \ "27=20561,28=54042,29=37390,30=1884,31=18071,32=31367,33=160," \ "34=169,35=201,36=10155,37=1045,38=15078,39=22985,40=12523," \ "41=15588,42=265,43=1287,44=142,45=382,46=945,47=426,48=171," \ "49=56,50=516,51=43,52=41,53=46,54=54,55=75,56=647,57=332," \ "58=32,59=39,60=48,61=35,62=62,63=32,64=221,65=26,66=30," \ "67=36,68=41,69=44,70=26,71=144,72=169,73=24,74=37,75=25," \ "76=42,77=21,78=126,79=374,80=27,81=40,82=43,83=47,84=46," \ "85=114,86=34,87=37,88=7240,89=34,90=38,91=18,92=99,93=20," \ "94=18,95=17,96=15,97=22,98=18,99=69,100=17,101=22,102=15," \ "103=29,104=39,105=30,106=70,107=22,108=21,109=26,110=52," \ "111=45,112=33,113=67,114=41,115=44,116=48,117=53,118=54," \ "119=51,120=75,121=44,122=57,123=44,124=66,125=56,126=52," \ "127=81,128=108,129=70,130=50,131=51,132=53,133=45,134=62," \ "135=12,136=13,137=7,138=15,139=21,140=11,141=20,142=6,143=7," \ "144=11,145=6,146=16,147=19,148=1112,149=1,151=83,154=1," \ "155=1,156=1,157=1,160=1,161=1,162=2,166=1,169=1,170=1,171=2," \ "172=1,174=1,176=2,177=9,178=34,179=73,180=30,181=1,185=3," \ "187=1,188=1,189=1,192=1,196=1,198=1,200=1,201=1,204=1,205=1," \ "207=1,208=1,209=1,214=2,215=31,216=78,217=28,218=5,219=2," \ "220=1,222=1,225=1,227=1,234=1,242=1,250=1,252=1,253=1," \ ">=256=203" parsed = parse_info(info) assert 'allocation_stats' in parsed assert '6' in parsed['allocation_stats'] assert '>=256' in parsed['allocation_stats']
def monitor_commandstats(self): if self.connection is None: self.connection = self.connection_pool.get_connection( 'info commandstats', None) self.connection.send_command("info commandstats") info = parse_info(self.connection.read_response()) return info
def log_cluster_replication_info(self, logger): # Gather replication information from Redis replication_info = parse_info( self.redis.execute_command("INFO REPLICATION")) # We only care about information from Redis master, which contains offsets from both master and replica if replication_info["role"] != "master": return master_repl_offset = replication_info["master_repl_offset"] if master_repl_offset == 0: return master_replid = replication_info["master_replid"] connected_replicas = replication_info["connected_slaves"] # If there are more than one replicas, log the most up-to-date max_replica_offset = 0 for n in range(connected_replicas): max_replica_offset = max(max_replica_offset, replication_info["slave%d" % n]["offset"]) offset_difference = int(master_repl_offset - max_replica_offset) now = time.time() # Offset difference between master and replica doesn't indicate how far behind the slave is, in wall clock time # # we calculate how many seconds the replica is falling behind, by comparing # a. offset difference between master and replica # and # b. master offset's change rate (per second) master_offset_change_per_sec = int( (master_repl_offset - self.last_master_offset) / (now - self.last_replication_sample_timestamp)) replica_lag_secs = int(offset_difference / master_offset_change_per_sec) time_format = "%Y-%m-%d %H:%M:%SZ" logger.emit_value( "redis", "replication", extra_fields={ "host": self.display_string, "ts": time.strftime(time_format, time.gmtime(now)), "masterId": master_replid, "masterOffset": master_repl_offset, "connectedReplicas": connected_replicas, "maxReplicaOffset": max_replica_offset, "offsetDiff": offset_difference, "masterOffsetDiffPerSec": master_offset_change_per_sec, "replicaLagSecs": replica_lag_secs, }, ) self.last_master_offset = master_repl_offset self.last_replication_sample_timestamp = now
def _configure(self): """ given the servers we know about, find the current master once we have the master, find all the slaves """ logger.debug("Running configure") to_check = Queue() for x in self._hosts: to_check.put(x) while not to_check.empty(): x = to_check.get() try: conn = self.connection_class(host=x.host, port=x.port, **self.connection_kwargs) conn.send_command("INFO") info = parse_info(conn.read_response()) if info['role'] == 'slave': self._slave_pool.add(conn) elif info['role'] == 'master': self._current_master = x logger.debug("Current master {}:{}".format(x.host, x.port)) self._master_pool.add(conn) slaves = filter(lambda x: x[0:5] == 'slave', info.keys()) slaves = [info[y] for y in slaves] slaves = [y.split(',') for y in slaves] slaves = filter(lambda x: x[2] == 'online', slaves) slaves = [Server(x[0], int(x[1])) for x in slaves] for y in slaves: if y not in self._hosts: self._hosts.add(y) to_check.put(y) # add the slaves except: # remove from list to_remove = [] logger.debug("Configure complete, host list: {}".format(self._hosts))
def _configure(self): """Given the servers we know about, find the current master once we have the master, find all the slaves. """ logging.debug("Running configure") self._current_master = None # (host, port) self._master_pool = self.queue_class(self.max_connections) self._slave_pool = set() to_check = Queue() for x in self._hosts: to_check.put(x) while not to_check.empty(): x = to_check.get() try: conn = self.connection_class( host=x.host, port=x.port, **self.connection_kwargs) conn.send_command("INFO") info = parse_info(conn.read_response()) if info['role'] == 'slave': self._slave_pool.add(conn) elif info['role'] == 'master': self._current_master = x logging.debug("Current master {0}:{1}".format( x.host, x.port)) self._master_pool.put_nowait(conn) slaves = filter(lambda x: x[0:5] == 'slave', info.keys()) slaves = [info[y] for y in slaves] if info['redis_version'] >= '2.8': slaves = filter(lambda x: x['state'] == 'online', slaves) slaves = [Server(s['ip'], int(s['port'])) for s in slaves] else: slaves = [y.split(',') for y in slaves] slaves = filter(lambda x: x[2] == 'online', slaves) slaves = [Server(s[0], int(s[1])) for s in slaves] for y in slaves: if y not in self._hosts: self._hosts.add(y) to_check.put(y) except ConnectionError: logging.exception("Can't connect to " "{host}:{port}/{db}, remove from " "hosts".format( host=x.host, port=x.port, db=self.connection_kwargs.get('db'))) self._hosts.remove(x) # Fill master connection pool with ``None`` while True: try: self._master_pool.put_nowait(None) except Full: break logging.debug("Configure complete, host list: {0}".format(self._hosts))
def _configure(self): """Given the servers we know about, find the current master once we have the master, find all the slaves. """ self._connections = [] logging.debug("Running configure") self._current_master = None # (host, port) self._master_pool = self.queue_class(self.max_connections) self._slave_pool = set() to_check = Queue() for x in self._hosts: to_check.put(x) while not to_check.empty(): x = to_check.get() try: conn = self.connection_class(host=x.host, port=x.port, **self.connection_kwargs) conn.send_command("INFO") info = parse_info(conn.read_response()) if info['role'] == 'slave': self._slave_pool.add(conn) elif info['role'] == 'master': self._current_master = x logging.debug("Current master {0}:{1}".format( x.host, x.port)) self._master_pool.put_nowait(conn) slaves = filter(lambda x: x[0:5] == 'slave', info.keys()) slaves = [info[y] for y in slaves] if info['redis_version'] >= '2.8': slaves = filter(lambda x: x['state'] == 'online', slaves) slaves = [ Server(s['ip'], int(s['port'])) for s in slaves ] else: slaves = [y.split(',') for y in slaves] slaves = filter(lambda x: x[2] == 'online', slaves) slaves = [Server(s[0], int(s[1])) for s in slaves] for y in slaves: if y not in self._hosts: self._hosts.add(y) to_check.put(y) except ConnectionError: logging.exception("Can't connect to " "{host}:{port}/{db}, remove from " "hosts".format( host=x.host, port=x.port, db=self.connection_kwargs.get('db'))) self._hosts.discard(x) # Fill master connection pool with ``None`` while True: try: self._master_pool.put_nowait(None) except Full: break logging.debug("Configure complete, host list: {0}".format(self._hosts))