def master_slave_master_off():
master = RedisServer(address="127.0.0.1", port=1234)
slave = RedisServer(unix_socket="/tmp/redis.socket", master=master)
filter = Logs(redis_server=slave)
filter.configure(FLOGS_CONF_TEMPLATE)
master.stop()
filter.start()
try:
filter.log(b'It s dangerous out there. Take this sword.')
except Exception:
logging.error("master_slave_master_off: Could not connect to logs filter")
return False
if not filter.check_run():
logging.error("master_slave_master_off: filter seems to have crashed when master Redis got offline")
return False
sleep(1)
number = slave.get_number_of_connections()
if number != 2:
logging.error("master_slave_master_off: filter is not connected to slave")
return False
return True
def master_replica_discovery_rate_limiting():
master = RedisServer(address="127.0.0.1", port=1234)
replica = RedisServer(unix_socket=REDIS_SOCKET_PATH, master=master)
filter = Filter(filter_name='test', nb_threads=5)
filter.configure(FTEST_CONF_TEMPLATE)
filter.start()
try:
# success
filter.send_single(REDIS_LIST_TRIGGER)
except Exception as e:
logging.error(
"master_replica_discovery_rate_limiting: Could not connect to test filter: {}"
.format(function_name, e))
return False
# master shuts down
master.stop()
#brutal
def thread_brute_time(filter, time_end):
end = time() + time_end
while time() < end:
# accelerate throughput by avoiding constant calls to time()
for i in range(0, 9):
try:
filter.send_single(REDIS_LIST_TRIGGER)
except:
return False
return True
thread_list = []
for num in range(0, 5):
thread_list.append(
threading.Thread(target=thread_brute_time, args=(filter, 9)))
# ought to crash if command fails
initial_connections_num = replica.connect().info(
)['total_connections_received']
for thread in thread_list:
thread.start()
for thread in thread_list:
thread.join()
# new generated connections generated, minus the one generated by the call to get it
new_connections = replica.connect().info(
)['total_connections_received'] - initial_connections_num - 1
if new_connections > 10:
logging.error(
"master_replica_discovery_rate_limiting: Wrong number of new connection attempts, "
+ "was supposed to have 10 new at most, but got ".format(
new_connections))
return False
return True
def master_replica_failover(function_name, healthcheck):
master = RedisServer(address="127.0.0.1", port=1234)
replica = RedisServer(unix_socket=REDIS_SOCKET_PATH, master=master)
filter = Filter(filter_name='test')
filter.configure(FTEST_CONF_TEMPLATE)
filter.valgrind_start()
try:
# success
filter.send_single(REDIS_LIST_TRIGGER)
except Exception as e:
logging.error("{}: Could not connect to test filter: {}".format(
function_name, e))
return False
# master shuts down
master.stop()
#replica becomes master
with replica.connect() as replica_connection:
replica_connection.slaveof()
sleep(1)
if healthcheck:
sleep(8)
try:
# success
return_code = filter.send_single(REDIS_LIST_TRIGGER)
except Exception as e:
logging.error("{}: Could not connect to test filter: {}".format(
function_name, e))
return False
if return_code != 0:
logging.error(
"{}: Filter didn't return correct code, " +
"waited for 0 but got {}".format(function_name, return_code))
return False
with replica.connect() as new_master_connection:
num_entries = new_master_connection.llen(REDIS_LIST_NAME)
if num_entries != 2:
logging.error("{}: Wrong number of entries in {}, " +
"expected 2 but got {}".format(
function_name, REDIS_LIST_NAME, num_entries))
return False
return True
def master_replica_master_temp_fail():
master = RedisServer(address="127.0.0.1", port=1234)
replica = RedisServer(address="127.0.0.1",
unix_socket=REDIS_SOCKET_PATH,
master=master)
filter = Filter(filter_name='test')
filter.configure(FTEST_CONF_TEMPLATE)
filter.valgrind_start()
try:
# success
filter.send_single(REDIS_LIST_TRIGGER)
except Exception as e:
logging.error(
"master_replica_master_temp_fail: Could not connect to test filter: {}"
.format(e))
return False
master.stop()
sleep(1)
try:
# failure
filter.send_single(REDIS_LIST_TRIGGER)
except Exception as e:
logging.error(
"master_replica_master_temp_fail: Could not connect to test filter: {}"
.format(e))
return False
sleep(1)
if not filter.check_run():
logging.error(
"master_replica_master_temp_fail: filter seems to have crashed when master Redis got offline"
)
return False
master.start()
# rate limiting will prevent to reconnect immediately after failure, so there should have a wait
# (rate limiting is not tested here)
sleep(8)
try:
# success
filter.send_single(REDIS_LIST_TRIGGER)
except Exception as e:
logging.error(
"master_replica_master_temp_fail: Could not connect to test filter: {}"
.format(e))
return False
with master.connect() as master_connection:
num_list_entries = master_connection.llen(REDIS_LIST_NAME)
if num_list_entries != 1:
logging.error(
"master_replica_master_temp_fail: wrong number of entries in the redis list {}: "
+
"expected 1 but got {}".format(REDIS_LIST_NAME, num_list_entries))
return False
return True