def test_22_info(self, r):
"""
Older Redis versions contained 'allocation_stats' in INFO that
was the cause of a number of bugs when parsing.
"""
info = "allocation_stats:6=1,7=1,8=7141,9=180,10=92,11=116,12=5330," \
"13=123,14=3091,15=11048,16=225842,17=1784,18=814,19=12020," \
"20=2530,21=645,22=15113,23=8695,24=142860,25=318,26=3303," \
"27=20561,28=54042,29=37390,30=1884,31=18071,32=31367,33=160," \
"34=169,35=201,36=10155,37=1045,38=15078,39=22985,40=12523," \
"41=15588,42=265,43=1287,44=142,45=382,46=945,47=426,48=171," \
"49=56,50=516,51=43,52=41,53=46,54=54,55=75,56=647,57=332," \
"58=32,59=39,60=48,61=35,62=62,63=32,64=221,65=26,66=30," \
"67=36,68=41,69=44,70=26,71=144,72=169,73=24,74=37,75=25," \
"76=42,77=21,78=126,79=374,80=27,81=40,82=43,83=47,84=46," \
"85=114,86=34,87=37,88=7240,89=34,90=38,91=18,92=99,93=20," \
"94=18,95=17,96=15,97=22,98=18,99=69,100=17,101=22,102=15," \
"103=29,104=39,105=30,106=70,107=22,108=21,109=26,110=52," \
"111=45,112=33,113=67,114=41,115=44,116=48,117=53,118=54," \
"119=51,120=75,121=44,122=57,123=44,124=66,125=56,126=52," \
"127=81,128=108,129=70,130=50,131=51,132=53,133=45,134=62," \
"135=12,136=13,137=7,138=15,139=21,140=11,141=20,142=6,143=7," \
"144=11,145=6,146=16,147=19,148=1112,149=1,151=83,154=1," \
"155=1,156=1,157=1,160=1,161=1,162=2,166=1,169=1,170=1,171=2," \
"172=1,174=1,176=2,177=9,178=34,179=73,180=30,181=1,185=3," \
"187=1,188=1,189=1,192=1,196=1,198=1,200=1,201=1,204=1,205=1," \
"207=1,208=1,209=1,214=2,215=31,216=78,217=28,218=5,219=2," \
"220=1,222=1,225=1,227=1,234=1,242=1,250=1,252=1,253=1," \
">=256=203"
parsed = parse_info(info)
assert 'allocation_stats' in parsed
assert '6' in parsed['allocation_stats']
assert '>=256' in parsed['allocation_stats']
def monitor_commandstats(self):
if self.connection is None:
self.connection = self.connection_pool.get_connection(
'info commandstats', None)
self.connection.send_command("info commandstats")
info = parse_info(self.connection.read_response())
return info
def test_22_info(self, r):
"""
Older Redis versions contained 'allocation_stats' in INFO that
was the cause of a number of bugs when parsing.
"""
info = "allocation_stats:6=1,7=1,8=7141,9=180,10=92,11=116,12=5330," \
"13=123,14=3091,15=11048,16=225842,17=1784,18=814,19=12020," \
"20=2530,21=645,22=15113,23=8695,24=142860,25=318,26=3303," \
"27=20561,28=54042,29=37390,30=1884,31=18071,32=31367,33=160," \
"34=169,35=201,36=10155,37=1045,38=15078,39=22985,40=12523," \
"41=15588,42=265,43=1287,44=142,45=382,46=945,47=426,48=171," \
"49=56,50=516,51=43,52=41,53=46,54=54,55=75,56=647,57=332," \
"58=32,59=39,60=48,61=35,62=62,63=32,64=221,65=26,66=30," \
"67=36,68=41,69=44,70=26,71=144,72=169,73=24,74=37,75=25," \
"76=42,77=21,78=126,79=374,80=27,81=40,82=43,83=47,84=46," \
"85=114,86=34,87=37,88=7240,89=34,90=38,91=18,92=99,93=20," \
"94=18,95=17,96=15,97=22,98=18,99=69,100=17,101=22,102=15," \
"103=29,104=39,105=30,106=70,107=22,108=21,109=26,110=52," \
"111=45,112=33,113=67,114=41,115=44,116=48,117=53,118=54," \
"119=51,120=75,121=44,122=57,123=44,124=66,125=56,126=52," \
"127=81,128=108,129=70,130=50,131=51,132=53,133=45,134=62," \
"135=12,136=13,137=7,138=15,139=21,140=11,141=20,142=6,143=7," \
"144=11,145=6,146=16,147=19,148=1112,149=1,151=83,154=1," \
"155=1,156=1,157=1,160=1,161=1,162=2,166=1,169=1,170=1,171=2," \
"172=1,174=1,176=2,177=9,178=34,179=73,180=30,181=1,185=3," \
"187=1,188=1,189=1,192=1,196=1,198=1,200=1,201=1,204=1,205=1," \
"207=1,208=1,209=1,214=2,215=31,216=78,217=28,218=5,219=2," \
"220=1,222=1,225=1,227=1,234=1,242=1,250=1,252=1,253=1," \
">=256=203"
parsed = parse_info(info)
assert 'allocation_stats' in parsed
assert '6' in parsed['allocation_stats']
assert '>=256' in parsed['allocation_stats']
def log_cluster_replication_info(self, logger):
# Gather replication information from Redis
replication_info = parse_info(
self.redis.execute_command("INFO REPLICATION"))
# We only care about information from Redis master, which contains offsets from both master and replica
if replication_info["role"] != "master":
return
master_repl_offset = replication_info["master_repl_offset"]
if master_repl_offset == 0:
return
master_replid = replication_info["master_replid"]
connected_replicas = replication_info["connected_slaves"]
# If there are more than one replicas, log the most up-to-date
max_replica_offset = 0
for n in range(connected_replicas):
max_replica_offset = max(max_replica_offset,
replication_info["slave%d" % n]["offset"])
offset_difference = int(master_repl_offset - max_replica_offset)
now = time.time()
# Offset difference between master and replica doesn't indicate how far behind the slave is, in wall clock time
#
# we calculate how many seconds the replica is falling behind, by comparing
# a. offset difference between master and replica
# and
# b. master offset's change rate (per second)
master_offset_change_per_sec = int(
(master_repl_offset - self.last_master_offset) /
(now - self.last_replication_sample_timestamp))
replica_lag_secs = int(offset_difference /
master_offset_change_per_sec)
time_format = "%Y-%m-%d %H:%M:%SZ"
logger.emit_value(
"redis",
"replication",
extra_fields={
"host": self.display_string,
"ts": time.strftime(time_format, time.gmtime(now)),
"masterId": master_replid,
"masterOffset": master_repl_offset,
"connectedReplicas": connected_replicas,
"maxReplicaOffset": max_replica_offset,
"offsetDiff": offset_difference,
"masterOffsetDiffPerSec": master_offset_change_per_sec,
"replicaLagSecs": replica_lag_secs,
},
)
self.last_master_offset = master_repl_offset
self.last_replication_sample_timestamp = now
def _configure(self):
"""
given the servers we know about, find the current master
once we have the master, find all the slaves
"""
logger.debug("Running configure")
to_check = Queue()
for x in self._hosts:
to_check.put(x)
while not to_check.empty():
x = to_check.get()
try:
conn = self.connection_class(host=x.host,
port=x.port,
**self.connection_kwargs)
conn.send_command("INFO")
info = parse_info(conn.read_response())
if info['role'] == 'slave':
self._slave_pool.add(conn)
elif info['role'] == 'master':
self._current_master = x
logger.debug("Current master {}:{}".format(x.host, x.port))
self._master_pool.add(conn)
slaves = filter(lambda x: x[0:5] == 'slave', info.keys())
slaves = [info[y] for y in slaves]
slaves = [y.split(',') for y in slaves]
slaves = filter(lambda x: x[2] == 'online', slaves)
slaves = [Server(x[0], int(x[1])) for x in slaves]
for y in slaves:
if y not in self._hosts:
self._hosts.add(y)
to_check.put(y)
# add the slaves
except:
# remove from list
to_remove = []
logger.debug("Configure complete, host list: {}".format(self._hosts))
def _configure(self):
"""
given the servers we know about, find the current master
once we have the master, find all the slaves
"""
logger.debug("Running configure")
to_check = Queue()
for x in self._hosts:
to_check.put(x)
while not to_check.empty():
x = to_check.get()
try:
conn = self.connection_class(host=x.host, port=x.port, **self.connection_kwargs)
conn.send_command("INFO")
info = parse_info(conn.read_response())
if info['role'] == 'slave':
self._slave_pool.add(conn)
elif info['role'] == 'master':
self._current_master = x
logger.debug("Current master {}:{}".format(x.host, x.port))
self._master_pool.add(conn)
slaves = filter(lambda x: x[0:5] == 'slave', info.keys())
slaves = [info[y] for y in slaves]
slaves = [y.split(',') for y in slaves]
slaves = filter(lambda x: x[2] == 'online', slaves)
slaves = [Server(x[0], int(x[1])) for x in slaves]
for y in slaves:
if y not in self._hosts:
self._hosts.add(y)
to_check.put(y)
# add the slaves
except:
# remove from list
to_remove = []
logger.debug("Configure complete, host list: {}".format(self._hosts))
def _configure(self):
"""Given the servers we know about, find the current master
once we have the master, find all the slaves.
"""
logging.debug("Running configure")
self._current_master = None # (host, port)
self._master_pool = self.queue_class(self.max_connections)
self._slave_pool = set()
to_check = Queue()
for x in self._hosts:
to_check.put(x)
while not to_check.empty():
x = to_check.get()
try:
conn = self.connection_class(
host=x.host, port=x.port, **self.connection_kwargs)
conn.send_command("INFO")
info = parse_info(conn.read_response())
if info['role'] == 'slave':
self._slave_pool.add(conn)
elif info['role'] == 'master':
self._current_master = x
logging.debug("Current master {0}:{1}".format(
x.host, x.port))
self._master_pool.put_nowait(conn)
slaves = filter(lambda x: x[0:5] == 'slave', info.keys())
slaves = [info[y] for y in slaves]
if info['redis_version'] >= '2.8':
slaves = filter(lambda x: x['state'] == 'online',
slaves)
slaves = [Server(s['ip'], int(s['port']))
for s in slaves]
else:
slaves = [y.split(',') for y in slaves]
slaves = filter(lambda x: x[2] == 'online', slaves)
slaves = [Server(s[0], int(s[1])) for s in slaves]
for y in slaves:
if y not in self._hosts:
self._hosts.add(y)
to_check.put(y)
except ConnectionError:
logging.exception("Can't connect to "
"{host}:{port}/{db}, remove from "
"hosts".format(
host=x.host, port=x.port,
db=self.connection_kwargs.get('db')))
self._hosts.remove(x)
# Fill master connection pool with ``None``
while True:
try:
self._master_pool.put_nowait(None)
except Full:
break
logging.debug("Configure complete, host list: {0}".format(self._hosts))
def _configure(self):
"""Given the servers we know about, find the current master
once we have the master, find all the slaves.
"""
self._connections = []
logging.debug("Running configure")
self._current_master = None # (host, port)
self._master_pool = self.queue_class(self.max_connections)
self._slave_pool = set()
to_check = Queue()
for x in self._hosts:
to_check.put(x)
while not to_check.empty():
x = to_check.get()
try:
conn = self.connection_class(host=x.host,
port=x.port,
**self.connection_kwargs)
conn.send_command("INFO")
info = parse_info(conn.read_response())
if info['role'] == 'slave':
self._slave_pool.add(conn)
elif info['role'] == 'master':
self._current_master = x
logging.debug("Current master {0}:{1}".format(
x.host, x.port))
self._master_pool.put_nowait(conn)
slaves = filter(lambda x: x[0:5] == 'slave', info.keys())
slaves = [info[y] for y in slaves]
if info['redis_version'] >= '2.8':
slaves = filter(lambda x: x['state'] == 'online',
slaves)
slaves = [
Server(s['ip'], int(s['port'])) for s in slaves
]
else:
slaves = [y.split(',') for y in slaves]
slaves = filter(lambda x: x[2] == 'online', slaves)
slaves = [Server(s[0], int(s[1])) for s in slaves]
for y in slaves:
if y not in self._hosts:
self._hosts.add(y)
to_check.put(y)
except ConnectionError:
logging.exception("Can't connect to "
"{host}:{port}/{db}, remove from "
"hosts".format(
host=x.host,
port=x.port,
db=self.connection_kwargs.get('db')))
self._hosts.discard(x)
# Fill master connection pool with ``None``
while True:
try:
self._master_pool.put_nowait(None)
except Full:
break
logging.debug("Configure complete, host list: {0}".format(self._hosts))
