def setUp(self):
self.redis_monitor = RedisMonitor("localsettings.py")
self.redis_monitor.settings = self.redis_monitor.wrapper.load(
"localsettings.py")
self.redis_monitor.logger = MagicMock()
self.redis_monitor.settings['KAFKA_TOPIC_PREFIX'] = "demo_test"
self.redis_monitor.settings['STATS_TOTAL'] = False
self.redis_monitor.settings['STATS_PLUGINS'] = False
self.redis_monitor.settings['PLUGINS'] = {
'plugins.info_monitor.InfoMonitor': None,
'plugins.stop_monitor.StopMonitor': None,
'plugins.expire_monitor.ExpireMonitor': None,
'tests.tests_online.CustomMonitor': 100,
}
self.redis_monitor.redis_conn = redis.Redis(
host=self.redis_monitor.settings['REDIS_HOST'],
port=self.redis_monitor.settings['REDIS_PORT'])
self.redis_monitor._load_plugins()
self.redis_monitor.stats_dict = {}
self.kafka_conn = KafkaClient(
self.redis_monitor.settings['KAFKA_HOSTS'])
self.kafka_conn.ensure_topic_exists("demo_test.outbound_firehose")
self.consumer = SimpleConsumer(self.kafka_conn, "demo-id",
"demo_test.outbound_firehose")
def setUp(self):
self.redis_monitor = RedisMonitor("localsettings.py")
self.redis_monitor.settings = self.redis_monitor.wrapper.load(
"localsettings.py")
self.redis_monitor.logger = MagicMock()
self.redis_monitor.settings['KAFKA_TOPIC_PREFIX'] = "demo_test"
self.redis_monitor.settings['STATS_TOTAL'] = False
self.redis_monitor.settings['STATS_PLUGINS'] = False
self.redis_monitor.settings['PLUGINS'] = {
'plugins.info_monitor.InfoMonitor': None,
'plugins.stop_monitor.StopMonitor': None,
'plugins.expire_monitor.ExpireMonitor': None,
'tests.online.CustomMonitor': 100,
}
self.redis_monitor.redis_conn = redis.Redis(
host=self.redis_monitor.settings['REDIS_HOST'],
port=self.redis_monitor.settings['REDIS_PORT'],
db=self.redis_monitor.settings['REDIS_DB'])
self.redis_monitor._load_plugins()
self.redis_monitor.stats_dict = {}
self.consumer = KafkaConsumer(
"demo_test.outbound_firehose",
bootstrap_servers=self.redis_monitor.settings['KAFKA_HOSTS'],
group_id="demo-id",
auto_commit_interval_ms=10,
consumer_timeout_ms=5000,
auto_offset_reset='earliest')
sleep(1)
def setUp(self):
print "### setUp ###"
self.monitor = RedisMonitor(
zk=None,
redis_hosts="localhost:6379,localhost:6389,localhost:6399",
sleep_time=30,
zk_path="/test/redis")
self.monitor.zk = MockZooKeeper("zk")
self.monitor.redis_class = MockRedis
self.monitor.zk_properties = MockZooKeeper("zk_properties")
self.monitor.queue = Queue.Queue()
def setUp(self):
print "### setUp ###"
self.monitor = RedisMonitor(zk=None, redis_hosts="localhost:6379,localhost:6389,localhost:6399", sleep_time=30, zk_path="/test/redis")
self.monitor.zk = MockZooKeeper("zk")
self.monitor.redis_class = MockRedis
self.monitor.zk_properties = MockZooKeeper("zk_properties")
self.monitor.queue = Queue.Queue()
def setUp(self):
self.redis_monitor = RedisMonitor("localsettings.py")
self.redis_monitor.settings = self.redis_monitor.wrapper.load("localsettings.py")
self.redis_monitor.logger = MagicMock()
self.redis_monitor.settings['KAFKA_TOPIC_PREFIX'] = "demo_test"
self.redis_monitor.settings['STATS_TOTAL'] = False
self.redis_monitor.settings['STATS_PLUGINS'] = False
self.redis_monitor.settings['PLUGINS'] = {
'plugins.info_monitor.InfoMonitor': None,
'plugins.stop_monitor.StopMonitor': None,
'plugins.expire_monitor.ExpireMonitor': None,
'tests.tests_online.CustomMonitor': 100,
}
self.redis_monitor.redis_conn = redis.Redis(
host=self.redis_monitor.settings['REDIS_HOST'],
port=self.redis_monitor.settings['REDIS_PORT'])
self.redis_monitor._load_plugins()
self.redis_monitor.stats_dict = {}
self.kafka_conn = KafkaClient(self.redis_monitor.settings[
'KAFKA_HOSTS'])
self.kafka_conn.ensure_topic_exists("demo_test.outbound_firehose")
self.consumer = SimpleConsumer(
self.kafka_conn,
"demo-id",
"demo_test.outbound_firehose"
)
def setUp(self):
self.redis_monitor = RedisMonitor("localsettings.py")
self.redis_monitor.settings = self.redis_monitor.wrapper.load("localsettings.py")
self.redis_monitor.logger = MagicMock()
self.redis_monitor.settings['KAFKA_TOPIC_PREFIX'] = "demo_test"
self.redis_monitor.settings['STATS_TOTAL'] = False
self.redis_monitor.settings['STATS_PLUGINS'] = False
self.redis_monitor.settings['PLUGINS'] = {
'plugins.info_monitor.InfoMonitor': None,
'plugins.stop_monitor.StopMonitor': None,
'plugins.expire_monitor.ExpireMonitor': None,
'tests.online.CustomMonitor': 100,
}
self.redis_monitor.redis_conn = redis.Redis(
host=self.redis_monitor.settings['REDIS_HOST'],
port=self.redis_monitor.settings['REDIS_PORT'],
db=self.redis_monitor.settings['REDIS_DB'])
self.redis_monitor._load_plugins()
self.redis_monitor.stats_dict = {}
self.consumer = KafkaConsumer(
"demo_test.outbound_firehose",
bootstrap_servers=self.redis_monitor.settings['KAFKA_HOSTS'],
group_id="demo-id",
auto_commit_interval_ms=10,
consumer_timeout_ms=5000,
auto_offset_reset='earliest'
)
sleep(1)
def setUp(self):
self.redis_monitor = RedisMonitor("settings.py", True)
self.redis_monitor.settings = self.redis_monitor.wrapper.load(
"settings.py")
self.redis_monitor.logger = MagicMock()
class TestRedisMonitor(TestCase):
def setUp(self):
self.redis_monitor = RedisMonitor("settings.py", True)
self.redis_monitor.settings = self.redis_monitor.wrapper.load(
"settings.py")
self.redis_monitor.logger = MagicMock()
def test_load_plugins(self):
# test loading default plugins
assert_keys = [100, 200, 300, 400, 500]
self.redis_monitor._load_plugins()
self.assertEqual(list(self.redis_monitor.plugins_dict.keys()),
assert_keys)
# test removing a plugin from settings
assert_keys = [100, 300, 400, 500]
self.redis_monitor.settings['PLUGINS'] \
['plugins.stop_monitor.StopMonitor'] = None
self.redis_monitor._load_plugins()
self.assertEqual(list(self.redis_monitor.plugins_dict.keys()),
assert_keys)
self.redis_monitor.settings['PLUGINS'] \
['plugins.stop_monitor.StopMonitor'] = 200
# fail if the class is not found
self.redis_monitor.settings['PLUGINS'] \
['plugins.crazy_class.CrazyHandler'] = 400,
self.assertRaises(ImportError, self.redis_monitor._load_plugins)
del self.redis_monitor.settings['PLUGINS'] \
['plugins.crazy_class.CrazyHandler']
self.redis_monitor.settings['PLUGINS'] = {}
def test_active_plugins(self):
# test that exceptions are caught within each plugin
# assuming now all plugins are loaded
self.redis_monitor._load_plugins()
self.redis_monitor.stats_dict = {}
# BaseExceptions are never raised normally
list(self.redis_monitor.plugins_dict.items()
)[0][1]['instance'].handle = MagicMock(
side_effect=BaseException("info"))
list(self.redis_monitor.plugins_dict.items()
)[1][1]['instance'].handle = MagicMock(
side_effect=BaseException("stop"))
list(self.redis_monitor.plugins_dict.items()
)[2][1]['instance'].handle = MagicMock(
side_effect=BaseException("expire"))
self.redis_monitor.redis_conn = MagicMock()
self.redis_monitor.redis_conn.scan_iter = MagicMock()
self.redis_monitor._create_lock_object = MagicMock()
# lets just assume the regex worked
self.redis_monitor.redis_conn.scan_iter.return_value = ['somekey1']
# info
try:
plugin = list(self.redis_monitor.plugins_dict.items())[0][1]
self.redis_monitor._process_plugin(plugin)
self.fail("Info not called")
except BaseException as e:
self.assertEqual("info", str(e))
# action
try:
plugin = list(self.redis_monitor.plugins_dict.items())[1][1]
self.redis_monitor._process_plugin(plugin)
self.fail("Stop not called")
except BaseException as e:
self.assertEqual("stop", str(e))
# expire
try:
plugin = list(self.redis_monitor.plugins_dict.items())[2][1]
self.redis_monitor._process_plugin(plugin)
self.fail("Expire not called")
except BaseException as e:
self.assertEqual("expire", str(e))
# test that an exception within a handle method is caught
self.redis_monitor._process_failures = MagicMock()
self.redis_monitor._increment_fail_stat = MagicMock()
try:
list(self.redis_monitor.plugins_dict.items()
)[0][1]['instance'].handle = MagicMock(
side_effect=Exception("normal"))
plugin = list(self.redis_monitor.plugins_dict.items())[0][1]
self.redis_monitor._process_plugin(plugin)
except Exception as e:
self.fail("Normal Exception not handled")
self.assertTrue(self.redis_monitor._increment_fail_stat.called)
self.assertTrue(self.redis_monitor._process_failures.called)
def test_locking_actions(self):
# tests for _process_plugins with locking
self.redis_monitor._load_plugins()
self.redis_monitor.redis_conn = MagicMock()
self.redis_monitor.redis_conn = MagicMock()
self.redis_monitor.redis_conn.scan_iter = MagicMock()
self.redis_monitor.redis_conn.get = MagicMock(return_value=5)
lock = MagicMock()
lock.acquire
lock.acquire = MagicMock(return_value=False)
lock.release = MagicMock()
lock._held = False
self.redis_monitor._create_lock_object = MagicMock(return_value=lock)
self.redis_monitor._increment_fail_stat = MagicMock()
self.redis_monitor._process_failures = MagicMock()
self.redis_monitor._process_key_val = MagicMock()
# lets just assume the regex worked
self.redis_monitor.redis_conn.scan_iter.return_value = ['somekey1']
plugin = {'instance': 'test', 'regex': 'abc123'}
# test didnt acquire lock
self.redis_monitor._process_key_val.assert_not_called()
self.redis_monitor._process_plugin(plugin)
self.redis_monitor._process_key_val.assert_not_called()
lock.release.assert_not_called()
# test got lock
lock.acquire = MagicMock(return_value=True)
lock._held = True
self.redis_monitor._process_plugin(plugin)
self.redis_monitor._process_key_val.assert_called_once_with(
'test', 'somekey1', 5)
# test lock released
lock.release.assert_called_once_with()
# test lock not held not released
lock._held = False
lock.release.reset_mock()
self.redis_monitor._process_plugin(plugin)
lock.release.assert_not_called()
def test_load_stats_plugins(self):
# lets assume we are loading the default plugins
self.redis_monitor._load_plugins()
self.redis_monitor.redis_conn = MagicMock()
# test no rolling stats
self.redis_monitor.stats_dict = {}
self.redis_monitor.settings['STATS_TIMES'] = []
self.redis_monitor._setup_stats_plugins()
defaults = [
'ExpireMonitor', 'StopMonitor', 'InfoMonitor', 'StatsMonitor',
'ZookeeperMonitor'
]
self.assertEqual(
sorted(self.redis_monitor.stats_dict['plugins'].keys()),
sorted(defaults))
for key in self.redis_monitor.plugins_dict:
plugin_name = self.redis_monitor.plugins_dict[key][
'instance'].__class__.__name__
self.assertEqual(
list(self.redis_monitor.stats_dict['plugins']
[plugin_name].keys()), ['lifetime'])
# test good/bad rolling stats
self.redis_monitor.stats_dict = {}
self.redis_monitor.settings['STATS_TIMES'] = [
'SECONDS_15_MINUTE',
'SECONDS_1_HOUR',
'SECONDS_DUMB',
]
good = [
'lifetime', # for totals, not DUMB
'900',
'3600',
]
self.redis_monitor._setup_stats_plugins()
self.assertEqual(
sorted(self.redis_monitor.stats_dict['plugins'].keys()),
sorted(defaults))
for key in self.redis_monitor.plugins_dict:
plugin_name = self.redis_monitor.plugins_dict[key][
'instance'].__class__.__name__
self.assertEqual(
sorted([
str(x) for x in self.redis_monitor.stats_dict['plugins']
[plugin_name].keys()
]), sorted(good))
for plugin_key in self.redis_monitor.stats_dict['plugins']:
k1 = 'stats:redis-monitor:{p}'.format(p=plugin_key)
for time_key in self.redis_monitor.stats_dict['plugins'][
plugin_key]:
if time_key == 0:
self.assertEqual(
self.redis_monitor.stats_dict['plugins'][plugin_key]
[0].key, '{k}:lifetime'.format(k=k1))
else:
self.assertEqual(
self.redis_monitor.stats_dict['plugins'][plugin_key]
[time_key].key, '{k}:{t}'.format(k=k1, t=time_key))
def test_main_loop(self):
self.redis_monitor._load_plugins()
self.redis_monitor._process_plugin = MagicMock(
side_effect=Exception("normal"))
try:
self.redis_monitor._main_loop()
self.fail("_process_plugin not called")
except BaseException as e:
self.assertEqual("normal", str(e))
def test_precondition(self):
self.redis_monitor.stats_dict = {}
instance = MagicMock()
instance.check_precondition = MagicMock(return_value=False)
instance.handle = MagicMock(side_effect=Exception("handler"))
key = 'stuff'
value = 'blah'
# this should not raise an exception
self.redis_monitor._process_key_val(instance, key, value)
# this should
instance.check_precondition = MagicMock(return_value=True)
try:
self.redis_monitor._process_key_val(instance, key, value)
self.fail('handler not called')
except BaseException as e:
self.assertEqual('handler', str(e))
def test_get_fail_key(self):
key = 'test'
result = 'lock:test:failures'
self.assertEqual(self.redis_monitor._get_fail_key(key), result)
def test_process_failures(self):
self.redis_monitor.settings = {
'RETRY_FAILURES': True,
'RETRY_FAILURES_MAX': 3
}
self.redis_monitor._get_fail_key = MagicMock(return_value='the_key')
self.redis_monitor.redis_conn = MagicMock()
self.redis_monitor.redis_conn.set = MagicMock()
# test not set
self.redis_monitor.redis_conn.get = MagicMock(return_value=None)
self.redis_monitor._process_failures('key1')
self.redis_monitor.redis_conn.set.assert_called_once_with('the_key', 1)
# test set
self.redis_monitor.redis_conn.set.reset_mock()
self.redis_monitor.redis_conn.get = MagicMock(return_value=2)
self.redis_monitor._process_failures('key1')
self.redis_monitor.redis_conn.set.assert_called_once_with('the_key', 3)
# test exceeded
self.redis_monitor.redis_conn.delete = MagicMock()
self.redis_monitor.redis_conn.set.reset_mock()
self.redis_monitor.redis_conn.get = MagicMock(return_value=3)
self.redis_monitor._process_failures('key1')
calls = [call('the_key'), call('key1')]
self.redis_monitor.redis_conn.delete.assert_has_calls(calls)
@mock.patch('socket.gethostname', return_value='host')
@mock.patch('time.time', return_value=5)
def test_report_self(self, h, t):
self.redis_monitor.my_uuid = '1234'
self.redis_monitor.redis_conn = MagicMock()
self.redis_monitor.redis_conn.set = MagicMock()
self.redis_monitor.redis_conn.expire = MagicMock()
self.redis_monitor._report_self()
self.redis_monitor.redis_conn.set.assert_called_once_with(
'stats:redis-monitor:self:host:1234', 5)
self.redis_monitor.redis_conn.expire.assert_called_once_with(
'stats:redis-monitor:self:host:1234', 120)
class TestRedisMonitor(unittest.TestCase):
def setUp(self):
print "### setUp ###"
self.monitor = RedisMonitor(zk=None, redis_hosts="localhost:6379,localhost:6389,localhost:6399", sleep_time=30, zk_path="/test/redis")
self.monitor.zk = MockZooKeeper("zk")
self.monitor.redis_class = MockRedis
self.monitor.zk_properties = MockZooKeeper("zk_properties")
self.monitor.queue = Queue.Queue()
def test_all_ok(self):
print "test_all_ok"
master = self.monitor.cluster.add_node(HOST, PORT, ROLE_MASTER, REDIS_STATUS_OK)
slave1 = self.monitor.cluster.add_node(HOST, PORT+1, ROLE_SLAVE, REDIS_STATUS_OK)
slave2 = self.monitor.cluster.add_node(HOST, PORT+2, ROLE_SLAVE, REDIS_STATUS_OK)
self.send_OK_Message(HOST, PORT)
self.monitor._parse_message_from_queue()
self.send_OK_Message(HOST, PORT+1)
self.monitor._parse_message_from_queue()
self.send_OK_Message(HOST, PORT+2)
self.monitor._parse_message_from_queue()
self.assertTrue(master.is_master())
self.assertTrue(master.is_alive())
self.assertTrue(slave1.is_slave())
self.assertTrue(slave1.is_alive())
self.assertTrue(slave2.is_slave())
self.assertTrue(slave2.is_alive())
def test_one_slave_KO(self):
print "test_one_slave_KO"
master = self.monitor.cluster.add_node(HOST, PORT, ROLE_MASTER, REDIS_STATUS_OK)
slave1 = self.monitor.cluster.add_node(HOST, PORT+1, ROLE_SLAVE, REDIS_STATUS_OK)
slave2 = self.monitor.cluster.add_node(HOST, PORT+2, ROLE_SLAVE, REDIS_STATUS_OK)
self.send_OK_Message(HOST, PORT+1)
self.monitor._parse_message_from_queue()
self.send_OK_Message(HOST, PORT)
self.monitor._parse_message_from_queue()
self.send_KO_Message(HOST, PORT+2)
self.monitor._parse_message_from_queue()
self.assertTrue(master.is_master())
self.assertTrue(master.is_alive())
self.assertTrue(slave1.is_slave())
self.assertTrue(slave1.is_alive())
self.assertTrue(slave2.is_slave())
self.assertFalse(slave2.is_alive())
def test_master_KO(self):
print "test_master_KO"
master = self.monitor.cluster.add_node(HOST, PORT, ROLE_MASTER, REDIS_STATUS_OK)
slave1 = self.monitor.cluster.add_node(HOST, PORT+1, ROLE_SLAVE, REDIS_STATUS_OK)
slave2 = self.monitor.cluster.add_node(HOST, PORT+2, ROLE_SLAVE, REDIS_STATUS_OK)
self.send_OK_Message(HOST, PORT+1)
self.monitor._parse_message_from_queue()
self.send_KO_Message(HOST, PORT)
self.monitor._parse_message_from_queue()
self.send_OK_Message(HOST, PORT+2)
self.monitor._parse_message_from_queue()
self.assertTrue(master.is_slave())
self.assertFalse(master.is_alive())
self.assertTrue(slave1.is_master())
self.assertTrue(slave1.is_alive())
self.assertTrue(slave2.is_slave())
self.assertTrue(slave2.is_alive())
def test_all_going_KO_master(self):
print "test_all_going_KO_master"
master = self.monitor.cluster.add_node(HOST, PORT, ROLE_MASTER, REDIS_STATUS_OK)
slave1 = self.monitor.cluster.add_node(HOST, PORT+1, ROLE_SLAVE, REDIS_STATUS_OK)
slave2 = self.monitor.cluster.add_node(HOST, PORT+2, ROLE_SLAVE, REDIS_STATUS_OK)
# kill slave1
self.send_KO_Message(HOST, PORT+1)
self.monitor._parse_message_from_queue()
self.assertTrue(slave1.is_slave())
self.assertFalse(slave1.is_alive())
# kill master
self.send_KO_Message(HOST, PORT)
self.monitor._parse_message_from_queue()
self.assertTrue(master.is_slave())
self.assertFalse(master.is_alive())
# kill slave2
self.send_KO_Message(HOST, PORT+2)
self.monitor._parse_message_from_queue()
self.assertTrue(slave2.is_master())
self.assertFalse(slave2.is_alive())
# slave2 is resurrected as master
self.send_OK_Message(HOST, PORT+2)
self.monitor._parse_message_from_queue()
self.assertTrue(slave2.is_master())
self.assertTrue(slave2.is_alive())
# slave1 is resurrected as slave
self.send_OK_Message(HOST, PORT+1)
self.monitor._parse_message_from_queue()
self.assertTrue(slave1.is_slave())
self.assertTrue(slave1.is_alive())
def test_all_going_KO_slave(self):
print "test_all_going_KO_slave"
master = self.monitor.cluster.add_node(HOST, PORT, ROLE_MASTER, REDIS_STATUS_OK)
slave1 = self.monitor.cluster.add_node(HOST, PORT+1, ROLE_SLAVE, REDIS_STATUS_OK)
slave2 = self.monitor.cluster.add_node(HOST, PORT+2, ROLE_SLAVE, REDIS_STATUS_OK)
# kill slave1
self.send_KO_Message(HOST, PORT+1)
self.monitor._parse_message_from_queue()
self.assertTrue(slave1.is_slave())
self.assertFalse(slave1.is_alive())
# kill master
self.send_KO_Message(HOST, PORT)
self.monitor._parse_message_from_queue()
self.assertTrue(master.is_slave())
self.assertFalse(master.is_alive())
# kill slave2
self.send_KO_Message(HOST, PORT+2)
self.monitor._parse_message_from_queue()
self.assertTrue(slave2.is_master())
self.assertFalse(slave2.is_alive())
# slave1 is resurrected
self.send_OK_Message(HOST, PORT+1)
self.monitor._parse_message_from_queue()
self.assertTrue(slave1.is_master())
self.assertTrue(slave1.is_alive())
# slave2 is resurrected
self.send_OK_Message(HOST, PORT+2)
self.monitor._parse_message_from_queue()
self.assertTrue(slave2.is_alive())
self.assertTrue(slave2.is_slave())
def test_all_resurrecting(self):
print "test_all_resurrecting"
master = self.monitor.cluster.add_node(HOST, PORT, ROLE_MASTER, REDIS_STATUS_KO)
slave1 = self.monitor.cluster.add_node(HOST, PORT+1, ROLE_SLAVE, REDIS_STATUS_KO)
slave2 = self.monitor.cluster.add_node(HOST, PORT+2, ROLE_SLAVE, REDIS_STATUS_KO)
# kill slave1
self.send_KO_Message(HOST, PORT+1)
self.send_KO_Message(HOST, PORT)
self.send_KO_Message(HOST, PORT+2)
self.monitor._parse_message_from_queue()
self.assertTrue(slave1.is_slave())
self.assertFalse(slave1.is_alive())
self.monitor._parse_message_from_queue()
self.assertTrue(master.is_master())
self.assertFalse(master.is_alive())
self.monitor._parse_message_from_queue()
self.assertTrue(slave2.is_slave())
self.assertFalse(slave2.is_alive())
def send_OK_Message(self, redis_host, redis_port):
self.send_message(redis_host, redis_port, REDIS_STATUS_OK)
def send_KO_Message(self, redis_host, redis_port):
self.send_message(redis_host, redis_port, REDIS_STATUS_KO)
def send_message(self, redis_host, redis_port, redis_status):
message = self.create_message(redis_host, redis_port, redis_status)
self.monitor.queue.put(message)
def create_message(self, redis_host, redis_port, redis_status):
worker_name = '%s:%s' % (redis_host, redis_port)
text = "%s,%s" % (worker_name, redis_status)
return text
def tearDown(self):
print "### tearDown ###"
class TestRedisMonitor(unittest.TestCase):
def setUp(self):
print "### setUp ###"
self.monitor = RedisMonitor(
zk=None,
redis_hosts="localhost:6379,localhost:6389,localhost:6399",
sleep_time=30,
zk_path="/test/redis")
self.monitor.zk = MockZooKeeper("zk")
self.monitor.redis_class = MockRedis
self.monitor.zk_properties = MockZooKeeper("zk_properties")
self.monitor.queue = Queue.Queue()
def test_all_ok(self):
print "test_all_ok"
master = self.monitor.cluster.add_node(HOST, PORT, ROLE_MASTER,
REDIS_STATUS_OK)
slave1 = self.monitor.cluster.add_node(HOST, PORT + 1, ROLE_SLAVE,
REDIS_STATUS_OK)
slave2 = self.monitor.cluster.add_node(HOST, PORT + 2, ROLE_SLAVE,
REDIS_STATUS_OK)
self.send_OK_Message(HOST, PORT)
self.monitor._parse_message_from_queue()
self.send_OK_Message(HOST, PORT + 1)
self.monitor._parse_message_from_queue()
self.send_OK_Message(HOST, PORT + 2)
self.monitor._parse_message_from_queue()
self.assertTrue(master.is_master())
self.assertTrue(master.is_alive())
self.assertTrue(slave1.is_slave())
self.assertTrue(slave1.is_alive())
self.assertTrue(slave2.is_slave())
self.assertTrue(slave2.is_alive())
def test_one_slave_KO(self):
print "test_one_slave_KO"
master = self.monitor.cluster.add_node(HOST, PORT, ROLE_MASTER,
REDIS_STATUS_OK)
slave1 = self.monitor.cluster.add_node(HOST, PORT + 1, ROLE_SLAVE,
REDIS_STATUS_OK)
slave2 = self.monitor.cluster.add_node(HOST, PORT + 2, ROLE_SLAVE,
REDIS_STATUS_OK)
self.send_OK_Message(HOST, PORT + 1)
self.monitor._parse_message_from_queue()
self.send_OK_Message(HOST, PORT)
self.monitor._parse_message_from_queue()
self.send_KO_Message(HOST, PORT + 2)
self.monitor._parse_message_from_queue()
self.assertTrue(master.is_master())
self.assertTrue(master.is_alive())
self.assertTrue(slave1.is_slave())
self.assertTrue(slave1.is_alive())
self.assertTrue(slave2.is_slave())
self.assertFalse(slave2.is_alive())
def test_master_KO(self):
print "test_master_KO"
master = self.monitor.cluster.add_node(HOST, PORT, ROLE_MASTER,
REDIS_STATUS_OK)
slave1 = self.monitor.cluster.add_node(HOST, PORT + 1, ROLE_SLAVE,
REDIS_STATUS_OK)
slave2 = self.monitor.cluster.add_node(HOST, PORT + 2, ROLE_SLAVE,
REDIS_STATUS_OK)
self.send_OK_Message(HOST, PORT + 1)
self.monitor._parse_message_from_queue()
self.send_KO_Message(HOST, PORT)
self.monitor._parse_message_from_queue()
self.send_OK_Message(HOST, PORT + 2)
self.monitor._parse_message_from_queue()
self.assertTrue(master.is_slave())
self.assertFalse(master.is_alive())
self.assertTrue(slave1.is_master())
self.assertTrue(slave1.is_alive())
self.assertTrue(slave2.is_slave())
self.assertTrue(slave2.is_alive())
def test_all_going_KO_master(self):
print "test_all_going_KO_master"
master = self.monitor.cluster.add_node(HOST, PORT, ROLE_MASTER,
REDIS_STATUS_OK)
slave1 = self.monitor.cluster.add_node(HOST, PORT + 1, ROLE_SLAVE,
REDIS_STATUS_OK)
slave2 = self.monitor.cluster.add_node(HOST, PORT + 2, ROLE_SLAVE,
REDIS_STATUS_OK)
# kill slave1
self.send_KO_Message(HOST, PORT + 1)
self.monitor._parse_message_from_queue()
self.assertTrue(slave1.is_slave())
self.assertFalse(slave1.is_alive())
# kill master
self.send_KO_Message(HOST, PORT)
self.monitor._parse_message_from_queue()
self.assertTrue(master.is_slave())
self.assertFalse(master.is_alive())
# kill slave2
self.send_KO_Message(HOST, PORT + 2)
self.monitor._parse_message_from_queue()
self.assertTrue(slave2.is_master())
self.assertFalse(slave2.is_alive())
# slave2 is resurrected as master
self.send_OK_Message(HOST, PORT + 2)
self.monitor._parse_message_from_queue()
self.assertTrue(slave2.is_master())
self.assertTrue(slave2.is_alive())
# slave1 is resurrected as slave
self.send_OK_Message(HOST, PORT + 1)
self.monitor._parse_message_from_queue()
self.assertTrue(slave1.is_slave())
self.assertTrue(slave1.is_alive())
def test_all_going_KO_slave(self):
print "test_all_going_KO_slave"
master = self.monitor.cluster.add_node(HOST, PORT, ROLE_MASTER,
REDIS_STATUS_OK)
slave1 = self.monitor.cluster.add_node(HOST, PORT + 1, ROLE_SLAVE,
REDIS_STATUS_OK)
slave2 = self.monitor.cluster.add_node(HOST, PORT + 2, ROLE_SLAVE,
REDIS_STATUS_OK)
# kill slave1
self.send_KO_Message(HOST, PORT + 1)
self.monitor._parse_message_from_queue()
self.assertTrue(slave1.is_slave())
self.assertFalse(slave1.is_alive())
# kill master
self.send_KO_Message(HOST, PORT)
self.monitor._parse_message_from_queue()
self.assertTrue(master.is_slave())
self.assertFalse(master.is_alive())
# kill slave2
self.send_KO_Message(HOST, PORT + 2)
self.monitor._parse_message_from_queue()
self.assertTrue(slave2.is_master())
self.assertFalse(slave2.is_alive())
# slave1 is resurrected
self.send_OK_Message(HOST, PORT + 1)
self.monitor._parse_message_from_queue()
self.assertTrue(slave1.is_master())
self.assertTrue(slave1.is_alive())
# slave2 is resurrected
self.send_OK_Message(HOST, PORT + 2)
self.monitor._parse_message_from_queue()
self.assertTrue(slave2.is_alive())
self.assertTrue(slave2.is_slave())
def test_all_resurrecting(self):
print "test_all_resurrecting"
master = self.monitor.cluster.add_node(HOST, PORT, ROLE_MASTER,
REDIS_STATUS_KO)
slave1 = self.monitor.cluster.add_node(HOST, PORT + 1, ROLE_SLAVE,
REDIS_STATUS_KO)
slave2 = self.monitor.cluster.add_node(HOST, PORT + 2, ROLE_SLAVE,
REDIS_STATUS_KO)
# kill slave1
self.send_KO_Message(HOST, PORT + 1)
self.send_KO_Message(HOST, PORT)
self.send_KO_Message(HOST, PORT + 2)
self.monitor._parse_message_from_queue()
self.assertTrue(slave1.is_slave())
self.assertFalse(slave1.is_alive())
self.monitor._parse_message_from_queue()
self.assertTrue(master.is_master())
self.assertFalse(master.is_alive())
self.monitor._parse_message_from_queue()
self.assertTrue(slave2.is_slave())
self.assertFalse(slave2.is_alive())
def send_OK_Message(self, redis_host, redis_port):
self.send_message(redis_host, redis_port, REDIS_STATUS_OK)
def send_KO_Message(self, redis_host, redis_port):
self.send_message(redis_host, redis_port, REDIS_STATUS_KO)
def send_message(self, redis_host, redis_port, redis_status):
message = self.create_message(redis_host, redis_port, redis_status)
self.monitor.queue.put(message)
def create_message(self, redis_host, redis_port, redis_status):
worker_name = '%s:%s' % (redis_host, redis_port)
text = "%s,%s" % (worker_name, redis_status)
return text
def tearDown(self):
print "### tearDown ###"
class TestRedisMonitor(TestCase):
maxDiff = None
queue_key = "link:istresearch.com:queue"
def setUp(self):
self.redis_monitor = RedisMonitor("localsettings.py")
self.redis_monitor.settings = self.redis_monitor.wrapper.load(
"localsettings.py")
self.redis_monitor.logger = MagicMock()
self.redis_monitor.settings['KAFKA_TOPIC_PREFIX'] = "demo_test"
self.redis_monitor.settings['STATS_TOTAL'] = False
self.redis_monitor.settings['STATS_PLUGINS'] = False
self.redis_monitor.settings['PLUGINS'] = {
'plugins.info_monitor.InfoMonitor': None,
'plugins.stop_monitor.StopMonitor': None,
'plugins.expire_monitor.ExpireMonitor': None,
'tests.tests_online.CustomMonitor': 100,
}
self.redis_monitor.redis_conn = redis.Redis(
host=self.redis_monitor.settings['REDIS_HOST'],
port=self.redis_monitor.settings['REDIS_PORT'])
self.redis_monitor._load_plugins()
self.redis_monitor.stats_dict = {}
self.kafka_conn = KafkaClient(
self.redis_monitor.settings['KAFKA_HOSTS'])
self.kafka_conn.ensure_topic_exists("demo_test.outbound_firehose")
self.consumer = SimpleConsumer(self.kafka_conn, "demo-id",
"demo_test.outbound_firehose")
def test_process_item(self):
# we only want to go to the end now, not after this test is ran
self.consumer.seek(0, 2)
# set the info flag
key = "info-test:blah"
value = "ABC123"
self.redis_monitor.redis_conn.set(key, value)
# process the request
plugin = self.redis_monitor.plugins_dict.items()[0][1]
self.redis_monitor._process_plugin(plugin)
# ensure the key is gone
self.assertEquals(self.redis_monitor.redis_conn.get(key), None)
def test_sent_to_kafka(self):
success = {u'info-test': "ABC123", u"appid": u"someapp"}
# ensure it was sent out to kafka
message_count = 0
for message in self.consumer.get_messages():
if message is None:
break
else:
the_dict = json.loads(message.message.value)
self.assertEquals(success, the_dict)
message_count += 1
self.assertEquals(message_count, 1)
class TestRedisMonitor(TestCase):
def setUp(self):
self.redis_monitor = RedisMonitor("settings.py", True)
self.redis_monitor.settings = self.redis_monitor.wrapper.load("settings.py")
self.redis_monitor.logger = MagicMock()
def test_load_plugins(self):
# test loading default plugins
assert_keys = [100, 200, 300, 400, 500]
self.redis_monitor._load_plugins()
self.assertEqual(list(self.redis_monitor.plugins_dict.keys()), assert_keys)
# test removing a plugin from settings
assert_keys = [100, 300, 400, 500]
self.redis_monitor.settings['PLUGINS'] \
['plugins.stop_monitor.StopMonitor'] = None
self.redis_monitor._load_plugins()
self.assertEqual(list(self.redis_monitor.plugins_dict.keys()), assert_keys)
self.redis_monitor.settings['PLUGINS'] \
['plugins.stop_monitor.StopMonitor'] = 200
# fail if the class is not found
self.redis_monitor.settings['PLUGINS'] \
['plugins.crazy_class.CrazyHandler'] = 400,
self.assertRaises(ImportError, self.redis_monitor._load_plugins)
del self.redis_monitor.settings['PLUGINS'] \
['plugins.crazy_class.CrazyHandler']
self.redis_monitor.settings['PLUGINS'] = {}
def test_active_plugins(self):
# test that exceptions are caught within each plugin
# assuming now all plugins are loaded
self.redis_monitor._load_plugins()
self.redis_monitor.stats_dict = {}
# BaseExceptions are never raised normally
list(self.redis_monitor.plugins_dict.items())[0][1]['instance'].handle = MagicMock(side_effect=BaseException("info"))
list(self.redis_monitor.plugins_dict.items())[1][1]['instance'].handle = MagicMock(side_effect=BaseException("stop"))
list(self.redis_monitor.plugins_dict.items())[2][1]['instance'].handle = MagicMock(side_effect=BaseException("expire"))
self.redis_monitor.redis_conn = MagicMock()
self.redis_monitor.redis_conn.scan_iter = MagicMock()
self.redis_monitor._create_lock_object = MagicMock()
# lets just assume the regex worked
self.redis_monitor.redis_conn.scan_iter.return_value = ['somekey1']
# info
try:
plugin = list(self.redis_monitor.plugins_dict.items())[0][1]
self.redis_monitor._process_plugin(plugin)
self.fail("Info not called")
except BaseException as e:
self.assertEquals("info", str(e))
# action
try:
plugin = list(self.redis_monitor.plugins_dict.items())[1][1]
self.redis_monitor._process_plugin(plugin)
self.fail("Stop not called")
except BaseException as e:
self.assertEquals("stop", str(e))
# expire
try:
plugin = list(self.redis_monitor.plugins_dict.items())[2][1]
self.redis_monitor._process_plugin(plugin)
self.fail("Expire not called")
except BaseException as e:
self.assertEquals("expire", str(e))
# test that an exception within a handle method is caught
self.redis_monitor._process_failures = MagicMock()
self.redis_monitor._increment_fail_stat = MagicMock()
try:
list(self.redis_monitor.plugins_dict.items())[0][1]['instance'].handle = MagicMock(side_effect=Exception("normal"))
plugin = list(self.redis_monitor.plugins_dict.items())[0][1]
self.redis_monitor._process_plugin(plugin)
except Exception as e:
self.fail("Normal Exception not handled")
self.assertTrue(self.redis_monitor._increment_fail_stat.called)
self.assertTrue(self.redis_monitor._process_failures.called)
def test_locking_actions(self):
# tests for _process_plugins with locking
self.redis_monitor._load_plugins()
self.redis_monitor.redis_conn = MagicMock()
self.redis_monitor.redis_conn = MagicMock()
self.redis_monitor.redis_conn.scan_iter = MagicMock()
self.redis_monitor.redis_conn.get = MagicMock(return_value=5)
lock = MagicMock()
lock.acquire
lock.acquire = MagicMock(return_value=False)
lock.release = MagicMock()
lock._held = False
self.redis_monitor._create_lock_object = MagicMock(return_value=lock)
self.redis_monitor._increment_fail_stat = MagicMock()
self.redis_monitor._process_failures = MagicMock()
self.redis_monitor._process_key_val = MagicMock()
# lets just assume the regex worked
self.redis_monitor.redis_conn.scan_iter.return_value = ['somekey1']
plugin = {'instance': 'test',
'regex': 'abc123'}
# test didnt acquire lock
self.redis_monitor._process_key_val.assert_not_called()
self.redis_monitor._process_plugin(plugin)
self.redis_monitor._process_key_val.assert_not_called()
lock.release.assert_not_called()
# test got lock
lock.acquire = MagicMock(return_value=True)
lock._held = True
self.redis_monitor._process_plugin(plugin)
self.redis_monitor._process_key_val.assert_called_once_with('test', 'somekey1', 5)
# test lock released
lock.release.assert_called_once_with()
# test lock not held not released
lock._held = False
lock.release.reset_mock()
self.redis_monitor._process_plugin(plugin)
lock.release.assert_not_called()
def test_load_stats_plugins(self):
# lets assume we are loading the default plugins
self.redis_monitor._load_plugins()
self.redis_monitor.redis_conn = MagicMock()
# test no rolling stats
self.redis_monitor.stats_dict = {}
self.redis_monitor.settings['STATS_TIMES'] = []
self.redis_monitor._setup_stats_plugins()
defaults = [
'ExpireMonitor',
'StopMonitor',
'InfoMonitor',
'StatsMonitor',
'ZookeeperMonitor'
]
self.assertEquals(
sorted(self.redis_monitor.stats_dict['plugins'].keys()),
sorted(defaults))
for key in self.redis_monitor.plugins_dict:
plugin_name = self.redis_monitor.plugins_dict[key]['instance'].__class__.__name__
self.assertEquals(
list(self.redis_monitor.stats_dict['plugins'][plugin_name].keys()),
['lifetime'])
# test good/bad rolling stats
self.redis_monitor.stats_dict = {}
self.redis_monitor.settings['STATS_TIMES'] = [
'SECONDS_15_MINUTE',
'SECONDS_1_HOUR',
'SECONDS_DUMB',
]
good = [
'lifetime', # for totals, not DUMB
'900',
'3600',
]
self.redis_monitor._setup_stats_plugins()
self.assertEquals(
sorted(self.redis_monitor.stats_dict['plugins'].keys()),
sorted(defaults))
for key in self.redis_monitor.plugins_dict:
plugin_name = self.redis_monitor.plugins_dict[key]['instance'].__class__.__name__
self.assertEquals(
sorted([str(x) for x in self.redis_monitor.stats_dict['plugins'][plugin_name].keys()]),
sorted(good))
for plugin_key in self.redis_monitor.stats_dict['plugins']:
k1 = 'stats:redis-monitor:{p}'.format(p=plugin_key)
for time_key in self.redis_monitor.stats_dict['plugins'][plugin_key]:
if time_key == 0:
self.assertEquals(
self.redis_monitor.stats_dict['plugins'][plugin_key][0].key,
'{k}:lifetime'.format(k=k1)
)
else:
self.assertEquals(
self.redis_monitor.stats_dict['plugins'][plugin_key][time_key].key,
'{k}:{t}'.format(k=k1, t=time_key)
)
def test_main_loop(self):
self.redis_monitor._load_plugins()
self.redis_monitor._process_plugin = MagicMock(side_effect=Exception(
"normal"))
try:
self.redis_monitor._main_loop()
self.fail("_process_plugin not called")
except BaseException as e:
self.assertEquals("normal", str(e))
def test_precondition(self):
self.redis_monitor.stats_dict = {}
instance = MagicMock()
instance.check_precondition = MagicMock(return_value=False)
instance.handle = MagicMock(side_effect=Exception("handler"))
key = 'stuff'
value = 'blah'
# this should not raise an exception
self.redis_monitor._process_key_val(instance, key, value)
# this should
instance.check_precondition = MagicMock(return_value=True)
try:
self.redis_monitor._process_key_val(instance, key, value)
self.fail('handler not called')
except BaseException as e:
self.assertEquals('handler', str(e))
def test_get_fail_key(self):
key = 'test'
result = 'lock:test:failures'
self.assertEquals(self.redis_monitor._get_fail_key(key), result)
def test_process_failures(self):
self.redis_monitor.settings = {'RETRY_FAILURES':True,
'RETRY_FAILURES_MAX': 3}
self.redis_monitor._get_fail_key = MagicMock(return_value='the_key')
self.redis_monitor.redis_conn = MagicMock()
self.redis_monitor.redis_conn.set = MagicMock()
# test not set
self.redis_monitor.redis_conn.get = MagicMock(return_value=None)
self.redis_monitor._process_failures('key1')
self.redis_monitor.redis_conn.set.assert_called_once_with('the_key', 1)
# test set
self.redis_monitor.redis_conn.set.reset_mock()
self.redis_monitor.redis_conn.get = MagicMock(return_value=2)
self.redis_monitor._process_failures('key1')
self.redis_monitor.redis_conn.set.assert_called_once_with('the_key', 3)
# test exceeded
self.redis_monitor.redis_conn.delete = MagicMock()
self.redis_monitor.redis_conn.set.reset_mock()
self.redis_monitor.redis_conn.get = MagicMock(return_value=3)
self.redis_monitor._process_failures('key1')
calls = [call('the_key'), call('key1')]
self.redis_monitor.redis_conn.delete.assert_has_calls(calls)
@mock.patch('socket.gethostname', return_value='host')
@mock.patch('time.time', return_value=5)
def test_report_self(self, h, t):
self.redis_monitor.my_uuid = '1234'
self.redis_monitor.redis_conn = MagicMock()
self.redis_monitor.redis_conn.set = MagicMock()
self.redis_monitor.redis_conn.expire = MagicMock()
self.redis_monitor._report_self()
self.redis_monitor.redis_conn.set.assert_called_once_with('stats:redis-monitor:self:host:1234', 5)
self.redis_monitor.redis_conn.expire.assert_called_once_with('stats:redis-monitor:self:host:1234', 120)
class TestRedisMonitor(TestCase):
maxDiff = None
queue_key = "link:istresearch.com:queue"
consumer = None
def setUp(self):
self.redis_monitor = RedisMonitor("localsettings.py")
self.redis_monitor.settings = self.redis_monitor.wrapper.load(
"localsettings.py")
self.redis_monitor.logger = MagicMock()
self.redis_monitor.settings['KAFKA_TOPIC_PREFIX'] = "demo_test"
self.redis_monitor.settings['STATS_TOTAL'] = False
self.redis_monitor.settings['STATS_PLUGINS'] = False
self.redis_monitor.settings['PLUGINS'] = {
'plugins.info_monitor.InfoMonitor': None,
'plugins.stop_monitor.StopMonitor': None,
'plugins.expire_monitor.ExpireMonitor': None,
'tests.online.CustomMonitor': 100,
}
self.redis_monitor.redis_conn = redis.Redis(
host=self.redis_monitor.settings['REDIS_HOST'],
port=self.redis_monitor.settings['REDIS_PORT'],
db=self.redis_monitor.settings['REDIS_DB'])
self.redis_monitor._load_plugins()
self.redis_monitor.stats_dict = {}
self.consumer = KafkaConsumer(
"demo_test.outbound_firehose",
bootstrap_servers=self.redis_monitor.settings['KAFKA_HOSTS'],
group_id="demo-id",
auto_commit_interval_ms=10,
consumer_timeout_ms=5000,
auto_offset_reset='earliest')
sleep(1)
def test_process_item(self):
# set the info flag
key = "info-test:blah"
value = "ABC1234"
self.redis_monitor.redis_conn.set(key, value)
# process the request
plugin = list(self.redis_monitor.plugins_dict.items())[0][1]
self.redis_monitor._process_plugin(plugin)
# ensure the key is gone
self.assertEquals(self.redis_monitor.redis_conn.get(key), None)
self.redis_monitor.close()
sleep(10)
# now test the message was sent to kafka
success = {u'info-test': "ABC1234", u"appid": u"someapp"}
message_count = 0
m = next(self.consumer)
if m is None:
pass
else:
the_dict = json.loads(m.value)
self.assertEquals(success, the_dict)
message_count += 1
self.assertEquals(message_count, 1)
def tearDown(self):
# if for some reason the tests fail, we end up falling behind on
# the consumer
for m in self.consumer:
pass
self.consumer.close()
class TestRedisMonitor(TestCase):
def setUp(self):
self.redis_monitor = RedisMonitor("settings.py", True)
self.redis_monitor.settings = self.redis_monitor.wrapper.load("settings.py")
self.redis_monitor.logger = MagicMock()
def test_load_plugins(self):
# test loading default plugins
assert_keys = [100, 200, 300, 400]
self.redis_monitor._load_plugins()
self.assertEqual(self.redis_monitor.plugins_dict.keys(), assert_keys)
# test removing a plugin from settings
assert_keys = [100, 300, 400]
self.redis_monitor.settings['PLUGINS'] \
['plugins.stop_monitor.StopMonitor'] = None
self.redis_monitor._load_plugins()
self.assertEqual(self.redis_monitor.plugins_dict.keys(), assert_keys)
self.redis_monitor.settings['PLUGINS'] \
['plugins.stop_monitor.StopMonitor'] = 200
# fail if the class is not found
self.redis_monitor.settings['PLUGINS'] \
['plugins.crazy_class.CrazyHandler'] = 400,
self.assertRaises(ImportError, self.redis_monitor._load_plugins)
del self.redis_monitor.settings['PLUGINS'] \
['plugins.crazy_class.CrazyHandler']
self.redis_monitor.settings['PLUGINS'] = {}
def test_active_plugins(self):
# test that exceptions are caught within each plugin
# assuming now all plugins are loaded
self.redis_monitor._load_plugins()
self.redis_monitor.stats_dict = {}
# BaseExceptions are never raised normally
self.redis_monitor.plugins_dict.items()[0][1]['instance'].handle = MagicMock(side_effect=BaseException("info"))
self.redis_monitor.plugins_dict.items()[1][1]['instance'].handle = MagicMock(side_effect=BaseException("stop"))
self.redis_monitor.plugins_dict.items()[2][1]['instance'].handle = MagicMock(side_effect=BaseException("expire"))
self.redis_monitor.redis_conn = MagicMock()
self.redis_monitor.redis_conn.scan_iter = MagicMock()
# lets just assume the regex worked
self.redis_monitor.redis_conn.scan_iter.return_value = ['somekey1']
# info
try:
plugin = self.redis_monitor.plugins_dict.items()[0][1]
self.redis_monitor._process_plugin(plugin)
self.fail("Info not called")
except BaseException as e:
self.assertEquals("info", e.message)
# action
try:
plugin = self.redis_monitor.plugins_dict.items()[1][1]
self.redis_monitor._process_plugin(plugin)
self.fail("Stop not called")
except BaseException as e:
self.assertEquals("stop", e.message)
# expire
try:
plugin = self.redis_monitor.plugins_dict.items()[2][1]
self.redis_monitor._process_plugin(plugin)
self.fail("Expire not called")
except BaseException as e:
self.assertEquals("expire", e.message)
# test that an exception within a handle method is caught
try:
self.redis_monitor.plugins_dict.items()[0][1]['instance'].handle = MagicMock(side_effect=Exception("normal"))
plugin = self.redis_monitor.plugins_dict.items()[0][1]
self.redis_monitor._process_plugin(plugin)
except Exception as e:
self.fail("Normal Exception not handled")
def test_load_stats_plugins(self):
# lets assume we are loading the default plugins
self.redis_monitor._load_plugins()
self.redis_monitor.redis_conn = MagicMock()
# test no rolling stats
self.redis_monitor.stats_dict = {}
self.redis_monitor.settings['STATS_TIMES'] = []
self.redis_monitor._setup_stats_plugins()
defaults = [
'ExpireMonitor',
'StopMonitor',
'InfoMonitor',
'StatsMonitor'
]
self.assertEquals(
sorted(self.redis_monitor.stats_dict['plugins'].keys()),
sorted(defaults))
for key in self.redis_monitor.plugins_dict:
plugin_name = self.redis_monitor.plugins_dict[key]['instance'].__class__.__name__
self.assertEquals(
self.redis_monitor.stats_dict['plugins'][plugin_name].keys(),
['lifetime'])
# test good/bad rolling stats
self.redis_monitor.stats_dict = {}
self.redis_monitor.settings['STATS_TIMES'] = [
'SECONDS_15_MINUTE',
'SECONDS_1_HOUR',
'SECONDS_DUMB',
]
good = [
'lifetime', # for totals, not DUMB
900,
3600,
]
self.redis_monitor._setup_stats_plugins()
self.assertEquals(
sorted(self.redis_monitor.stats_dict['plugins'].keys()),
sorted(defaults))
for key in self.redis_monitor.plugins_dict:
plugin_name = self.redis_monitor.plugins_dict[key]['instance'].__class__.__name__
self.assertEquals(
sorted(self.redis_monitor.stats_dict['plugins'][plugin_name].keys()),
sorted(good))
for plugin_key in self.redis_monitor.stats_dict['plugins']:
k1 = 'stats:redis-monitor:{p}'.format(p=plugin_key)
for time_key in self.redis_monitor.stats_dict['plugins'][plugin_key]:
if time_key == 0:
self.assertEquals(
self.redis_monitor.stats_dict['plugins'][plugin_key][0].key,
'{k}:lifetime'.format(k=k1)
)
else:
self.assertEquals(
self.redis_monitor.stats_dict['plugins'][plugin_key][time_key].key,
'{k}:{t}'.format(k=k1, t=time_key)
)
def test_main_loop(self):
self.redis_monitor._load_plugins()
self.redis_monitor._process_plugin = MagicMock(side_effect=Exception(
"normal"))
try:
self.redis_monitor._main_loop()
self.fail("_process_plugin not called")
except BaseException as e:
self.assertEquals("normal", e.message)
def test_precondition(self):
self.redis_monitor.stats_dict = {}
instance = MagicMock()
instance.check_precondition = MagicMock(return_value=False)
instance.handle = MagicMock(side_effect=Exception("handler"))
key = 'stuff'
value = 'blah'
# this should not raise an exception
self.redis_monitor._process_key_val(instance, key, value)
# this should
instance.check_precondition = MagicMock(return_value=True)
try:
self.redis_monitor._process_key_val(instance, key, value)
self.fail('handler not called')
except BaseException as e:
self.assertEquals('handler', e.message)
class TestRedisMonitor(TestCase):
maxDiff = None
queue_key = "link:istresearch.com:queue"
consumer = None
def setUp(self):
self.redis_monitor = RedisMonitor("localsettings.py")
self.redis_monitor.settings = self.redis_monitor.wrapper.load("localsettings.py")
self.redis_monitor.logger = MagicMock()
self.redis_monitor.settings['KAFKA_TOPIC_PREFIX'] = "demo_test"
self.redis_monitor.settings['STATS_TOTAL'] = False
self.redis_monitor.settings['STATS_PLUGINS'] = False
self.redis_monitor.settings['PLUGINS'] = {
'plugins.info_monitor.InfoMonitor': None,
'plugins.stop_monitor.StopMonitor': None,
'plugins.expire_monitor.ExpireMonitor': None,
'tests.online.CustomMonitor': 100,
}
self.redis_monitor.redis_conn = redis.Redis(
host=self.redis_monitor.settings['REDIS_HOST'],
port=self.redis_monitor.settings['REDIS_PORT'],
db=self.redis_monitor.settings['REDIS_DB'])
self.redis_monitor._load_plugins()
self.redis_monitor.stats_dict = {}
self.consumer = KafkaConsumer(
"demo_test.outbound_firehose",
bootstrap_servers=self.redis_monitor.settings['KAFKA_HOSTS'],
group_id="demo-id",
auto_commit_interval_ms=10,
consumer_timeout_ms=5000,
auto_offset_reset='earliest'
)
sleep(1)
def test_process_item(self):
# set the info flag
key = "info-test:blah"
value = "ABC1234"
self.redis_monitor.redis_conn.set(key, value)
# process the request
plugin = list(self.redis_monitor.plugins_dict.items())[0][1]
self.redis_monitor._process_plugin(plugin)
# ensure the key is gone
self.assertEquals(self.redis_monitor.redis_conn.get(key), None)
self.redis_monitor.close()
sleep(10)
# now test the message was sent to kafka
success = {
u'info-test': "ABC1234",
u"appid": u"someapp"
}
message_count = 0
m = next(self.consumer)
if m is None:
pass
else:
the_dict = json.loads(m.value)
self.assertEquals(success, the_dict)
message_count += 1
self.assertEquals(message_count, 1)
def tearDown(self):
# if for some reason the tests fail, we end up falling behind on
# the consumer
for m in self.consumer:
pass
self.consumer.close()
class TestRedisMonitor(TestCase):
def setUp(self):
self.redis_monitor = RedisMonitor("settings.py", True)
self.redis_monitor.settings = self.redis_monitor.wrapper.load("settings.py")
self.redis_monitor.logger = MagicMock()
def test_load_plugins(self):
# test loading default plugins
assert_keys = [100, 200, 300, 400]
self.redis_monitor._load_plugins()
self.assertEqual(self.redis_monitor.plugins_dict.keys(), assert_keys)
# test removing a plugin from settings
assert_keys = [100, 300, 400]
self.redis_monitor.settings["PLUGINS"]["plugins.stop_monitor.StopMonitor"] = None
self.redis_monitor._load_plugins()
self.assertEqual(self.redis_monitor.plugins_dict.keys(), assert_keys)
self.redis_monitor.settings["PLUGINS"]["plugins.stop_monitor.StopMonitor"] = 200
# fail if the class is not found
self.redis_monitor.settings["PLUGINS"]["plugins.crazy_class.CrazyHandler"] = (400,)
self.assertRaises(ImportError, self.redis_monitor._load_plugins)
del self.redis_monitor.settings["PLUGINS"]["plugins.crazy_class.CrazyHandler"]
self.redis_monitor.settings["PLUGINS"] = {}
def test_active_plugins(self):
# test that exceptions are caught within each plugin
# assuming now all plugins are loaded
self.redis_monitor._load_plugins()
self.redis_monitor.stats_dict = {}
# BaseExceptions are never raised normally
self.redis_monitor.plugins_dict.items()[0][1]["instance"].handle = MagicMock(side_effect=BaseException("info"))
self.redis_monitor.plugins_dict.items()[1][1]["instance"].handle = MagicMock(side_effect=BaseException("stop"))
self.redis_monitor.plugins_dict.items()[2][1]["instance"].handle = MagicMock(
side_effect=BaseException("expire")
)
self.redis_monitor.redis_conn = MagicMock()
self.redis_monitor.redis_conn.scan_iter = MagicMock()
# lets just assume the regex worked
self.redis_monitor.redis_conn.scan_iter.return_value = ["somekey1"]
# info
try:
plugin = self.redis_monitor.plugins_dict.items()[0][1]
self.redis_monitor._process_plugin(plugin)
self.fail("Info not called")
except BaseException as e:
self.assertEquals("info", e.message)
# action
try:
plugin = self.redis_monitor.plugins_dict.items()[1][1]
self.redis_monitor._process_plugin(plugin)
self.fail("Stop not called")
except BaseException as e:
self.assertEquals("stop", e.message)
# expire
try:
plugin = self.redis_monitor.plugins_dict.items()[2][1]
self.redis_monitor._process_plugin(plugin)
self.fail("Expire not called")
except BaseException as e:
self.assertEquals("expire", e.message)
# test that an exception within a handle method is caught
try:
self.redis_monitor.plugins_dict.items()[0][1]["instance"].handle = MagicMock(
side_effect=Exception("normal")
)
plugin = self.redis_monitor.plugins_dict.items()[0][1]
self.redis_monitor._process_plugin(plugin)
except Exception as e:
self.fail("Normal Exception not handled")
def test_load_stats_plugins(self):
# lets assume we are loading the default plugins
self.redis_monitor._load_plugins()
self.redis_monitor.redis_conn = MagicMock()
# test no rolling stats
self.redis_monitor.stats_dict = {}
self.redis_monitor.settings["STATS_TIMES"] = []
self.redis_monitor._setup_stats_plugins()
defaults = ["ExpireMonitor", "StopMonitor", "InfoMonitor", "StatsMonitor"]
self.assertEquals(sorted(self.redis_monitor.stats_dict["plugins"].keys()), sorted(defaults))
for key in self.redis_monitor.plugins_dict:
plugin_name = self.redis_monitor.plugins_dict[key]["instance"].__class__.__name__
self.assertEquals(self.redis_monitor.stats_dict["plugins"][plugin_name].keys(), ["lifetime"])
# test good/bad rolling stats
self.redis_monitor.stats_dict = {}
self.redis_monitor.settings["STATS_TIMES"] = ["SECONDS_15_MINUTE", "SECONDS_1_HOUR", "SECONDS_DUMB"]
good = ["lifetime", 900, 3600] # for totals, not DUMB
self.redis_monitor._setup_stats_plugins()
self.assertEquals(sorted(self.redis_monitor.stats_dict["plugins"].keys()), sorted(defaults))
for key in self.redis_monitor.plugins_dict:
plugin_name = self.redis_monitor.plugins_dict[key]["instance"].__class__.__name__
self.assertEquals(sorted(self.redis_monitor.stats_dict["plugins"][plugin_name].keys()), sorted(good))
for plugin_key in self.redis_monitor.stats_dict["plugins"]:
k1 = "stats:redis-monitor:{p}".format(p=plugin_key)
for time_key in self.redis_monitor.stats_dict["plugins"][plugin_key]:
if time_key == 0:
self.assertEquals(
self.redis_monitor.stats_dict["plugins"][plugin_key][0].key, "{k}:lifetime".format(k=k1)
)
else:
self.assertEquals(
self.redis_monitor.stats_dict["plugins"][plugin_key][time_key].key,
"{k}:{t}".format(k=k1, t=time_key),
)
def test_main_loop(self):
self.redis_monitor._load_plugins()
self.redis_monitor._process_plugin = MagicMock(side_effect=Exception("normal"))
try:
self.redis_monitor._main_loop()
self.fail("_process_plugin not called")
except BaseException as e:
self.assertEquals("normal", e.message)
def test_precondition(self):
self.redis_monitor.stats_dict = {}
instance = MagicMock()
instance.check_precondition = MagicMock(return_value=False)
instance.handle = MagicMock(side_effect=Exception("handler"))
key = "stuff"
value = "blah"
# this should not raise an exception
self.redis_monitor._process_key_val(instance, key, value)
# this should
instance.check_precondition = MagicMock(return_value=True)
try:
self.redis_monitor._process_key_val(instance, key, value)
self.fail("handler not called")
except BaseException as e:
self.assertEquals("handler", e.message)
def setUp(self):
self.redis_monitor = RedisMonitor("settings.py", True)
self.redis_monitor.settings = self.redis_monitor.wrapper.load("settings.py")
self.redis_monitor.logger = MagicMock()
class TestRedisMonitor(TestCase):
maxDiff = None
queue_key = "link:istresearch.com:queue"
def setUp(self):
self.redis_monitor = RedisMonitor("localsettings.py")
self.redis_monitor.settings = self.redis_monitor.wrapper.load("localsettings.py")
self.redis_monitor.logger = MagicMock()
self.redis_monitor.settings['KAFKA_TOPIC_PREFIX'] = "demo_test"
self.redis_monitor.settings['STATS_TOTAL'] = False
self.redis_monitor.settings['STATS_PLUGINS'] = False
self.redis_monitor.settings['PLUGINS'] = {
'plugins.info_monitor.InfoMonitor': None,
'plugins.stop_monitor.StopMonitor': None,
'plugins.expire_monitor.ExpireMonitor': None,
'tests.tests_online.CustomMonitor': 100,
}
self.redis_monitor.redis_conn = redis.Redis(
host=self.redis_monitor.settings['REDIS_HOST'],
port=self.redis_monitor.settings['REDIS_PORT'])
self.redis_monitor._load_plugins()
self.redis_monitor.stats_dict = {}
self.kafka_conn = KafkaClient(self.redis_monitor.settings[
'KAFKA_HOSTS'])
self.kafka_conn.ensure_topic_exists("demo_test.outbound_firehose")
self.consumer = SimpleConsumer(
self.kafka_conn,
"demo-id",
"demo_test.outbound_firehose"
)
def test_process_item(self):
# we only want to go to the end now, not after this test is ran
self.consumer.seek(0, 2)
# set the info flag
key = "info-test:blah"
value = "ABC123"
self.redis_monitor.redis_conn.set(key, value)
# process the request
plugin = self.redis_monitor.plugins_dict.items()[0][1]
self.redis_monitor._process_plugin(plugin)
# ensure the key is gone
self.assertEquals(self.redis_monitor.redis_conn.get(key), None)
def test_sent_to_kafka(self):
success = {
u'info-test': "ABC123",
u"appid": u"someapp"
}
# ensure it was sent out to kafka
message_count = 0
for message in self.consumer.get_messages():
if message is None:
break
else:
the_dict = json.loads(message.message.value)
self.assertEquals(success, the_dict)
message_count += 1
self.assertEquals(message_count, 1)