def setUp(self): self.redis_monitor = RedisMonitor("localsettings.py") self.redis_monitor.settings = self.redis_monitor.wrapper.load( "localsettings.py") self.redis_monitor.logger = MagicMock() self.redis_monitor.settings['KAFKA_TOPIC_PREFIX'] = "demo_test" self.redis_monitor.settings['STATS_TOTAL'] = False self.redis_monitor.settings['STATS_PLUGINS'] = False self.redis_monitor.settings['PLUGINS'] = { 'plugins.info_monitor.InfoMonitor': None, 'plugins.stop_monitor.StopMonitor': None, 'plugins.expire_monitor.ExpireMonitor': None, 'tests.tests_online.CustomMonitor': 100, } self.redis_monitor.redis_conn = redis.Redis( host=self.redis_monitor.settings['REDIS_HOST'], port=self.redis_monitor.settings['REDIS_PORT']) self.redis_monitor._load_plugins() self.redis_monitor.stats_dict = {} self.kafka_conn = KafkaClient( self.redis_monitor.settings['KAFKA_HOSTS']) self.kafka_conn.ensure_topic_exists("demo_test.outbound_firehose") self.consumer = SimpleConsumer(self.kafka_conn, "demo-id", "demo_test.outbound_firehose")
def setUp(self): self.redis_monitor = RedisMonitor("localsettings.py") self.redis_monitor.settings = self.redis_monitor.wrapper.load( "localsettings.py") self.redis_monitor.logger = MagicMock() self.redis_monitor.settings['KAFKA_TOPIC_PREFIX'] = "demo_test" self.redis_monitor.settings['STATS_TOTAL'] = False self.redis_monitor.settings['STATS_PLUGINS'] = False self.redis_monitor.settings['PLUGINS'] = { 'plugins.info_monitor.InfoMonitor': None, 'plugins.stop_monitor.StopMonitor': None, 'plugins.expire_monitor.ExpireMonitor': None, 'tests.online.CustomMonitor': 100, } self.redis_monitor.redis_conn = redis.Redis( host=self.redis_monitor.settings['REDIS_HOST'], port=self.redis_monitor.settings['REDIS_PORT'], db=self.redis_monitor.settings['REDIS_DB']) self.redis_monitor._load_plugins() self.redis_monitor.stats_dict = {} self.consumer = KafkaConsumer( "demo_test.outbound_firehose", bootstrap_servers=self.redis_monitor.settings['KAFKA_HOSTS'], group_id="demo-id", auto_commit_interval_ms=10, consumer_timeout_ms=5000, auto_offset_reset='earliest') sleep(1)
def setUp(self): print "### setUp ###" self.monitor = RedisMonitor( zk=None, redis_hosts="localhost:6379,localhost:6389,localhost:6399", sleep_time=30, zk_path="/test/redis") self.monitor.zk = MockZooKeeper("zk") self.monitor.redis_class = MockRedis self.monitor.zk_properties = MockZooKeeper("zk_properties") self.monitor.queue = Queue.Queue()
def setUp(self): print "### setUp ###" self.monitor = RedisMonitor(zk=None, redis_hosts="localhost:6379,localhost:6389,localhost:6399", sleep_time=30, zk_path="/test/redis") self.monitor.zk = MockZooKeeper("zk") self.monitor.redis_class = MockRedis self.monitor.zk_properties = MockZooKeeper("zk_properties") self.monitor.queue = Queue.Queue()
def setUp(self): self.redis_monitor = RedisMonitor("localsettings.py") self.redis_monitor.settings = self.redis_monitor.wrapper.load("localsettings.py") self.redis_monitor.logger = MagicMock() self.redis_monitor.settings['KAFKA_TOPIC_PREFIX'] = "demo_test" self.redis_monitor.settings['STATS_TOTAL'] = False self.redis_monitor.settings['STATS_PLUGINS'] = False self.redis_monitor.settings['PLUGINS'] = { 'plugins.info_monitor.InfoMonitor': None, 'plugins.stop_monitor.StopMonitor': None, 'plugins.expire_monitor.ExpireMonitor': None, 'tests.tests_online.CustomMonitor': 100, } self.redis_monitor.redis_conn = redis.Redis( host=self.redis_monitor.settings['REDIS_HOST'], port=self.redis_monitor.settings['REDIS_PORT']) self.redis_monitor._load_plugins() self.redis_monitor.stats_dict = {} self.kafka_conn = KafkaClient(self.redis_monitor.settings[ 'KAFKA_HOSTS']) self.kafka_conn.ensure_topic_exists("demo_test.outbound_firehose") self.consumer = SimpleConsumer( self.kafka_conn, "demo-id", "demo_test.outbound_firehose" )
def setUp(self): self.redis_monitor = RedisMonitor("localsettings.py") self.redis_monitor.settings = self.redis_monitor.wrapper.load("localsettings.py") self.redis_monitor.logger = MagicMock() self.redis_monitor.settings['KAFKA_TOPIC_PREFIX'] = "demo_test" self.redis_monitor.settings['STATS_TOTAL'] = False self.redis_monitor.settings['STATS_PLUGINS'] = False self.redis_monitor.settings['PLUGINS'] = { 'plugins.info_monitor.InfoMonitor': None, 'plugins.stop_monitor.StopMonitor': None, 'plugins.expire_monitor.ExpireMonitor': None, 'tests.online.CustomMonitor': 100, } self.redis_monitor.redis_conn = redis.Redis( host=self.redis_monitor.settings['REDIS_HOST'], port=self.redis_monitor.settings['REDIS_PORT'], db=self.redis_monitor.settings['REDIS_DB']) self.redis_monitor._load_plugins() self.redis_monitor.stats_dict = {} self.consumer = KafkaConsumer( "demo_test.outbound_firehose", bootstrap_servers=self.redis_monitor.settings['KAFKA_HOSTS'], group_id="demo-id", auto_commit_interval_ms=10, consumer_timeout_ms=5000, auto_offset_reset='earliest' ) sleep(1)
def setUp(self): self.redis_monitor = RedisMonitor("settings.py", True) self.redis_monitor.settings = self.redis_monitor.wrapper.load( "settings.py") self.redis_monitor.logger = MagicMock()
class TestRedisMonitor(TestCase): def setUp(self): self.redis_monitor = RedisMonitor("settings.py", True) self.redis_monitor.settings = self.redis_monitor.wrapper.load( "settings.py") self.redis_monitor.logger = MagicMock() def test_load_plugins(self): # test loading default plugins assert_keys = [100, 200, 300, 400, 500] self.redis_monitor._load_plugins() self.assertEqual(list(self.redis_monitor.plugins_dict.keys()), assert_keys) # test removing a plugin from settings assert_keys = [100, 300, 400, 500] self.redis_monitor.settings['PLUGINS'] \ ['plugins.stop_monitor.StopMonitor'] = None self.redis_monitor._load_plugins() self.assertEqual(list(self.redis_monitor.plugins_dict.keys()), assert_keys) self.redis_monitor.settings['PLUGINS'] \ ['plugins.stop_monitor.StopMonitor'] = 200 # fail if the class is not found self.redis_monitor.settings['PLUGINS'] \ ['plugins.crazy_class.CrazyHandler'] = 400, self.assertRaises(ImportError, self.redis_monitor._load_plugins) del self.redis_monitor.settings['PLUGINS'] \ ['plugins.crazy_class.CrazyHandler'] self.redis_monitor.settings['PLUGINS'] = {} def test_active_plugins(self): # test that exceptions are caught within each plugin # assuming now all plugins are loaded self.redis_monitor._load_plugins() self.redis_monitor.stats_dict = {} # BaseExceptions are never raised normally list(self.redis_monitor.plugins_dict.items() )[0][1]['instance'].handle = MagicMock( side_effect=BaseException("info")) list(self.redis_monitor.plugins_dict.items() )[1][1]['instance'].handle = MagicMock( side_effect=BaseException("stop")) list(self.redis_monitor.plugins_dict.items() )[2][1]['instance'].handle = MagicMock( side_effect=BaseException("expire")) self.redis_monitor.redis_conn = MagicMock() self.redis_monitor.redis_conn.scan_iter = MagicMock() self.redis_monitor._create_lock_object = MagicMock() # lets just assume the regex worked self.redis_monitor.redis_conn.scan_iter.return_value = ['somekey1'] # info try: plugin = list(self.redis_monitor.plugins_dict.items())[0][1] self.redis_monitor._process_plugin(plugin) self.fail("Info not called") except BaseException as e: self.assertEqual("info", str(e)) # action try: plugin = list(self.redis_monitor.plugins_dict.items())[1][1] self.redis_monitor._process_plugin(plugin) self.fail("Stop not called") except BaseException as e: self.assertEqual("stop", str(e)) # expire try: plugin = list(self.redis_monitor.plugins_dict.items())[2][1] self.redis_monitor._process_plugin(plugin) self.fail("Expire not called") except BaseException as e: self.assertEqual("expire", str(e)) # test that an exception within a handle method is caught self.redis_monitor._process_failures = MagicMock() self.redis_monitor._increment_fail_stat = MagicMock() try: list(self.redis_monitor.plugins_dict.items() )[0][1]['instance'].handle = MagicMock( side_effect=Exception("normal")) plugin = list(self.redis_monitor.plugins_dict.items())[0][1] self.redis_monitor._process_plugin(plugin) except Exception as e: self.fail("Normal Exception not handled") self.assertTrue(self.redis_monitor._increment_fail_stat.called) self.assertTrue(self.redis_monitor._process_failures.called) def test_locking_actions(self): # tests for _process_plugins with locking self.redis_monitor._load_plugins() self.redis_monitor.redis_conn = MagicMock() self.redis_monitor.redis_conn = MagicMock() self.redis_monitor.redis_conn.scan_iter = MagicMock() self.redis_monitor.redis_conn.get = MagicMock(return_value=5) lock = MagicMock() lock.acquire lock.acquire = MagicMock(return_value=False) lock.release = MagicMock() lock._held = False self.redis_monitor._create_lock_object = MagicMock(return_value=lock) self.redis_monitor._increment_fail_stat = MagicMock() self.redis_monitor._process_failures = MagicMock() self.redis_monitor._process_key_val = MagicMock() # lets just assume the regex worked self.redis_monitor.redis_conn.scan_iter.return_value = ['somekey1'] plugin = {'instance': 'test', 'regex': 'abc123'} # test didnt acquire lock self.redis_monitor._process_key_val.assert_not_called() self.redis_monitor._process_plugin(plugin) self.redis_monitor._process_key_val.assert_not_called() lock.release.assert_not_called() # test got lock lock.acquire = MagicMock(return_value=True) lock._held = True self.redis_monitor._process_plugin(plugin) self.redis_monitor._process_key_val.assert_called_once_with( 'test', 'somekey1', 5) # test lock released lock.release.assert_called_once_with() # test lock not held not released lock._held = False lock.release.reset_mock() self.redis_monitor._process_plugin(plugin) lock.release.assert_not_called() def test_load_stats_plugins(self): # lets assume we are loading the default plugins self.redis_monitor._load_plugins() self.redis_monitor.redis_conn = MagicMock() # test no rolling stats self.redis_monitor.stats_dict = {} self.redis_monitor.settings['STATS_TIMES'] = [] self.redis_monitor._setup_stats_plugins() defaults = [ 'ExpireMonitor', 'StopMonitor', 'InfoMonitor', 'StatsMonitor', 'ZookeeperMonitor' ] self.assertEqual( sorted(self.redis_monitor.stats_dict['plugins'].keys()), sorted(defaults)) for key in self.redis_monitor.plugins_dict: plugin_name = self.redis_monitor.plugins_dict[key][ 'instance'].__class__.__name__ self.assertEqual( list(self.redis_monitor.stats_dict['plugins'] [plugin_name].keys()), ['lifetime']) # test good/bad rolling stats self.redis_monitor.stats_dict = {} self.redis_monitor.settings['STATS_TIMES'] = [ 'SECONDS_15_MINUTE', 'SECONDS_1_HOUR', 'SECONDS_DUMB', ] good = [ 'lifetime', # for totals, not DUMB '900', '3600', ] self.redis_monitor._setup_stats_plugins() self.assertEqual( sorted(self.redis_monitor.stats_dict['plugins'].keys()), sorted(defaults)) for key in self.redis_monitor.plugins_dict: plugin_name = self.redis_monitor.plugins_dict[key][ 'instance'].__class__.__name__ self.assertEqual( sorted([ str(x) for x in self.redis_monitor.stats_dict['plugins'] [plugin_name].keys() ]), sorted(good)) for plugin_key in self.redis_monitor.stats_dict['plugins']: k1 = 'stats:redis-monitor:{p}'.format(p=plugin_key) for time_key in self.redis_monitor.stats_dict['plugins'][ plugin_key]: if time_key == 0: self.assertEqual( self.redis_monitor.stats_dict['plugins'][plugin_key] [0].key, '{k}:lifetime'.format(k=k1)) else: self.assertEqual( self.redis_monitor.stats_dict['plugins'][plugin_key] [time_key].key, '{k}:{t}'.format(k=k1, t=time_key)) def test_main_loop(self): self.redis_monitor._load_plugins() self.redis_monitor._process_plugin = MagicMock( side_effect=Exception("normal")) try: self.redis_monitor._main_loop() self.fail("_process_plugin not called") except BaseException as e: self.assertEqual("normal", str(e)) def test_precondition(self): self.redis_monitor.stats_dict = {} instance = MagicMock() instance.check_precondition = MagicMock(return_value=False) instance.handle = MagicMock(side_effect=Exception("handler")) key = 'stuff' value = 'blah' # this should not raise an exception self.redis_monitor._process_key_val(instance, key, value) # this should instance.check_precondition = MagicMock(return_value=True) try: self.redis_monitor._process_key_val(instance, key, value) self.fail('handler not called') except BaseException as e: self.assertEqual('handler', str(e)) def test_get_fail_key(self): key = 'test' result = 'lock:test:failures' self.assertEqual(self.redis_monitor._get_fail_key(key), result) def test_process_failures(self): self.redis_monitor.settings = { 'RETRY_FAILURES': True, 'RETRY_FAILURES_MAX': 3 } self.redis_monitor._get_fail_key = MagicMock(return_value='the_key') self.redis_monitor.redis_conn = MagicMock() self.redis_monitor.redis_conn.set = MagicMock() # test not set self.redis_monitor.redis_conn.get = MagicMock(return_value=None) self.redis_monitor._process_failures('key1') self.redis_monitor.redis_conn.set.assert_called_once_with('the_key', 1) # test set self.redis_monitor.redis_conn.set.reset_mock() self.redis_monitor.redis_conn.get = MagicMock(return_value=2) self.redis_monitor._process_failures('key1') self.redis_monitor.redis_conn.set.assert_called_once_with('the_key', 3) # test exceeded self.redis_monitor.redis_conn.delete = MagicMock() self.redis_monitor.redis_conn.set.reset_mock() self.redis_monitor.redis_conn.get = MagicMock(return_value=3) self.redis_monitor._process_failures('key1') calls = [call('the_key'), call('key1')] self.redis_monitor.redis_conn.delete.assert_has_calls(calls) @mock.patch('socket.gethostname', return_value='host') @mock.patch('time.time', return_value=5) def test_report_self(self, h, t): self.redis_monitor.my_uuid = '1234' self.redis_monitor.redis_conn = MagicMock() self.redis_monitor.redis_conn.set = MagicMock() self.redis_monitor.redis_conn.expire = MagicMock() self.redis_monitor._report_self() self.redis_monitor.redis_conn.set.assert_called_once_with( 'stats:redis-monitor:self:host:1234', 5) self.redis_monitor.redis_conn.expire.assert_called_once_with( 'stats:redis-monitor:self:host:1234', 120)
class TestRedisMonitor(unittest.TestCase): def setUp(self): print "### setUp ###" self.monitor = RedisMonitor(zk=None, redis_hosts="localhost:6379,localhost:6389,localhost:6399", sleep_time=30, zk_path="/test/redis") self.monitor.zk = MockZooKeeper("zk") self.monitor.redis_class = MockRedis self.monitor.zk_properties = MockZooKeeper("zk_properties") self.monitor.queue = Queue.Queue() def test_all_ok(self): print "test_all_ok" master = self.monitor.cluster.add_node(HOST, PORT, ROLE_MASTER, REDIS_STATUS_OK) slave1 = self.monitor.cluster.add_node(HOST, PORT+1, ROLE_SLAVE, REDIS_STATUS_OK) slave2 = self.monitor.cluster.add_node(HOST, PORT+2, ROLE_SLAVE, REDIS_STATUS_OK) self.send_OK_Message(HOST, PORT) self.monitor._parse_message_from_queue() self.send_OK_Message(HOST, PORT+1) self.monitor._parse_message_from_queue() self.send_OK_Message(HOST, PORT+2) self.monitor._parse_message_from_queue() self.assertTrue(master.is_master()) self.assertTrue(master.is_alive()) self.assertTrue(slave1.is_slave()) self.assertTrue(slave1.is_alive()) self.assertTrue(slave2.is_slave()) self.assertTrue(slave2.is_alive()) def test_one_slave_KO(self): print "test_one_slave_KO" master = self.monitor.cluster.add_node(HOST, PORT, ROLE_MASTER, REDIS_STATUS_OK) slave1 = self.monitor.cluster.add_node(HOST, PORT+1, ROLE_SLAVE, REDIS_STATUS_OK) slave2 = self.monitor.cluster.add_node(HOST, PORT+2, ROLE_SLAVE, REDIS_STATUS_OK) self.send_OK_Message(HOST, PORT+1) self.monitor._parse_message_from_queue() self.send_OK_Message(HOST, PORT) self.monitor._parse_message_from_queue() self.send_KO_Message(HOST, PORT+2) self.monitor._parse_message_from_queue() self.assertTrue(master.is_master()) self.assertTrue(master.is_alive()) self.assertTrue(slave1.is_slave()) self.assertTrue(slave1.is_alive()) self.assertTrue(slave2.is_slave()) self.assertFalse(slave2.is_alive()) def test_master_KO(self): print "test_master_KO" master = self.monitor.cluster.add_node(HOST, PORT, ROLE_MASTER, REDIS_STATUS_OK) slave1 = self.monitor.cluster.add_node(HOST, PORT+1, ROLE_SLAVE, REDIS_STATUS_OK) slave2 = self.monitor.cluster.add_node(HOST, PORT+2, ROLE_SLAVE, REDIS_STATUS_OK) self.send_OK_Message(HOST, PORT+1) self.monitor._parse_message_from_queue() self.send_KO_Message(HOST, PORT) self.monitor._parse_message_from_queue() self.send_OK_Message(HOST, PORT+2) self.monitor._parse_message_from_queue() self.assertTrue(master.is_slave()) self.assertFalse(master.is_alive()) self.assertTrue(slave1.is_master()) self.assertTrue(slave1.is_alive()) self.assertTrue(slave2.is_slave()) self.assertTrue(slave2.is_alive()) def test_all_going_KO_master(self): print "test_all_going_KO_master" master = self.monitor.cluster.add_node(HOST, PORT, ROLE_MASTER, REDIS_STATUS_OK) slave1 = self.monitor.cluster.add_node(HOST, PORT+1, ROLE_SLAVE, REDIS_STATUS_OK) slave2 = self.monitor.cluster.add_node(HOST, PORT+2, ROLE_SLAVE, REDIS_STATUS_OK) # kill slave1 self.send_KO_Message(HOST, PORT+1) self.monitor._parse_message_from_queue() self.assertTrue(slave1.is_slave()) self.assertFalse(slave1.is_alive()) # kill master self.send_KO_Message(HOST, PORT) self.monitor._parse_message_from_queue() self.assertTrue(master.is_slave()) self.assertFalse(master.is_alive()) # kill slave2 self.send_KO_Message(HOST, PORT+2) self.monitor._parse_message_from_queue() self.assertTrue(slave2.is_master()) self.assertFalse(slave2.is_alive()) # slave2 is resurrected as master self.send_OK_Message(HOST, PORT+2) self.monitor._parse_message_from_queue() self.assertTrue(slave2.is_master()) self.assertTrue(slave2.is_alive()) # slave1 is resurrected as slave self.send_OK_Message(HOST, PORT+1) self.monitor._parse_message_from_queue() self.assertTrue(slave1.is_slave()) self.assertTrue(slave1.is_alive()) def test_all_going_KO_slave(self): print "test_all_going_KO_slave" master = self.monitor.cluster.add_node(HOST, PORT, ROLE_MASTER, REDIS_STATUS_OK) slave1 = self.monitor.cluster.add_node(HOST, PORT+1, ROLE_SLAVE, REDIS_STATUS_OK) slave2 = self.monitor.cluster.add_node(HOST, PORT+2, ROLE_SLAVE, REDIS_STATUS_OK) # kill slave1 self.send_KO_Message(HOST, PORT+1) self.monitor._parse_message_from_queue() self.assertTrue(slave1.is_slave()) self.assertFalse(slave1.is_alive()) # kill master self.send_KO_Message(HOST, PORT) self.monitor._parse_message_from_queue() self.assertTrue(master.is_slave()) self.assertFalse(master.is_alive()) # kill slave2 self.send_KO_Message(HOST, PORT+2) self.monitor._parse_message_from_queue() self.assertTrue(slave2.is_master()) self.assertFalse(slave2.is_alive()) # slave1 is resurrected self.send_OK_Message(HOST, PORT+1) self.monitor._parse_message_from_queue() self.assertTrue(slave1.is_master()) self.assertTrue(slave1.is_alive()) # slave2 is resurrected self.send_OK_Message(HOST, PORT+2) self.monitor._parse_message_from_queue() self.assertTrue(slave2.is_alive()) self.assertTrue(slave2.is_slave()) def test_all_resurrecting(self): print "test_all_resurrecting" master = self.monitor.cluster.add_node(HOST, PORT, ROLE_MASTER, REDIS_STATUS_KO) slave1 = self.monitor.cluster.add_node(HOST, PORT+1, ROLE_SLAVE, REDIS_STATUS_KO) slave2 = self.monitor.cluster.add_node(HOST, PORT+2, ROLE_SLAVE, REDIS_STATUS_KO) # kill slave1 self.send_KO_Message(HOST, PORT+1) self.send_KO_Message(HOST, PORT) self.send_KO_Message(HOST, PORT+2) self.monitor._parse_message_from_queue() self.assertTrue(slave1.is_slave()) self.assertFalse(slave1.is_alive()) self.monitor._parse_message_from_queue() self.assertTrue(master.is_master()) self.assertFalse(master.is_alive()) self.monitor._parse_message_from_queue() self.assertTrue(slave2.is_slave()) self.assertFalse(slave2.is_alive()) def send_OK_Message(self, redis_host, redis_port): self.send_message(redis_host, redis_port, REDIS_STATUS_OK) def send_KO_Message(self, redis_host, redis_port): self.send_message(redis_host, redis_port, REDIS_STATUS_KO) def send_message(self, redis_host, redis_port, redis_status): message = self.create_message(redis_host, redis_port, redis_status) self.monitor.queue.put(message) def create_message(self, redis_host, redis_port, redis_status): worker_name = '%s:%s' % (redis_host, redis_port) text = "%s,%s" % (worker_name, redis_status) return text def tearDown(self): print "### tearDown ###"
class TestRedisMonitor(unittest.TestCase): def setUp(self): print "### setUp ###" self.monitor = RedisMonitor( zk=None, redis_hosts="localhost:6379,localhost:6389,localhost:6399", sleep_time=30, zk_path="/test/redis") self.monitor.zk = MockZooKeeper("zk") self.monitor.redis_class = MockRedis self.monitor.zk_properties = MockZooKeeper("zk_properties") self.monitor.queue = Queue.Queue() def test_all_ok(self): print "test_all_ok" master = self.monitor.cluster.add_node(HOST, PORT, ROLE_MASTER, REDIS_STATUS_OK) slave1 = self.monitor.cluster.add_node(HOST, PORT + 1, ROLE_SLAVE, REDIS_STATUS_OK) slave2 = self.monitor.cluster.add_node(HOST, PORT + 2, ROLE_SLAVE, REDIS_STATUS_OK) self.send_OK_Message(HOST, PORT) self.monitor._parse_message_from_queue() self.send_OK_Message(HOST, PORT + 1) self.monitor._parse_message_from_queue() self.send_OK_Message(HOST, PORT + 2) self.monitor._parse_message_from_queue() self.assertTrue(master.is_master()) self.assertTrue(master.is_alive()) self.assertTrue(slave1.is_slave()) self.assertTrue(slave1.is_alive()) self.assertTrue(slave2.is_slave()) self.assertTrue(slave2.is_alive()) def test_one_slave_KO(self): print "test_one_slave_KO" master = self.monitor.cluster.add_node(HOST, PORT, ROLE_MASTER, REDIS_STATUS_OK) slave1 = self.monitor.cluster.add_node(HOST, PORT + 1, ROLE_SLAVE, REDIS_STATUS_OK) slave2 = self.monitor.cluster.add_node(HOST, PORT + 2, ROLE_SLAVE, REDIS_STATUS_OK) self.send_OK_Message(HOST, PORT + 1) self.monitor._parse_message_from_queue() self.send_OK_Message(HOST, PORT) self.monitor._parse_message_from_queue() self.send_KO_Message(HOST, PORT + 2) self.monitor._parse_message_from_queue() self.assertTrue(master.is_master()) self.assertTrue(master.is_alive()) self.assertTrue(slave1.is_slave()) self.assertTrue(slave1.is_alive()) self.assertTrue(slave2.is_slave()) self.assertFalse(slave2.is_alive()) def test_master_KO(self): print "test_master_KO" master = self.monitor.cluster.add_node(HOST, PORT, ROLE_MASTER, REDIS_STATUS_OK) slave1 = self.monitor.cluster.add_node(HOST, PORT + 1, ROLE_SLAVE, REDIS_STATUS_OK) slave2 = self.monitor.cluster.add_node(HOST, PORT + 2, ROLE_SLAVE, REDIS_STATUS_OK) self.send_OK_Message(HOST, PORT + 1) self.monitor._parse_message_from_queue() self.send_KO_Message(HOST, PORT) self.monitor._parse_message_from_queue() self.send_OK_Message(HOST, PORT + 2) self.monitor._parse_message_from_queue() self.assertTrue(master.is_slave()) self.assertFalse(master.is_alive()) self.assertTrue(slave1.is_master()) self.assertTrue(slave1.is_alive()) self.assertTrue(slave2.is_slave()) self.assertTrue(slave2.is_alive()) def test_all_going_KO_master(self): print "test_all_going_KO_master" master = self.monitor.cluster.add_node(HOST, PORT, ROLE_MASTER, REDIS_STATUS_OK) slave1 = self.monitor.cluster.add_node(HOST, PORT + 1, ROLE_SLAVE, REDIS_STATUS_OK) slave2 = self.monitor.cluster.add_node(HOST, PORT + 2, ROLE_SLAVE, REDIS_STATUS_OK) # kill slave1 self.send_KO_Message(HOST, PORT + 1) self.monitor._parse_message_from_queue() self.assertTrue(slave1.is_slave()) self.assertFalse(slave1.is_alive()) # kill master self.send_KO_Message(HOST, PORT) self.monitor._parse_message_from_queue() self.assertTrue(master.is_slave()) self.assertFalse(master.is_alive()) # kill slave2 self.send_KO_Message(HOST, PORT + 2) self.monitor._parse_message_from_queue() self.assertTrue(slave2.is_master()) self.assertFalse(slave2.is_alive()) # slave2 is resurrected as master self.send_OK_Message(HOST, PORT + 2) self.monitor._parse_message_from_queue() self.assertTrue(slave2.is_master()) self.assertTrue(slave2.is_alive()) # slave1 is resurrected as slave self.send_OK_Message(HOST, PORT + 1) self.monitor._parse_message_from_queue() self.assertTrue(slave1.is_slave()) self.assertTrue(slave1.is_alive()) def test_all_going_KO_slave(self): print "test_all_going_KO_slave" master = self.monitor.cluster.add_node(HOST, PORT, ROLE_MASTER, REDIS_STATUS_OK) slave1 = self.monitor.cluster.add_node(HOST, PORT + 1, ROLE_SLAVE, REDIS_STATUS_OK) slave2 = self.monitor.cluster.add_node(HOST, PORT + 2, ROLE_SLAVE, REDIS_STATUS_OK) # kill slave1 self.send_KO_Message(HOST, PORT + 1) self.monitor._parse_message_from_queue() self.assertTrue(slave1.is_slave()) self.assertFalse(slave1.is_alive()) # kill master self.send_KO_Message(HOST, PORT) self.monitor._parse_message_from_queue() self.assertTrue(master.is_slave()) self.assertFalse(master.is_alive()) # kill slave2 self.send_KO_Message(HOST, PORT + 2) self.monitor._parse_message_from_queue() self.assertTrue(slave2.is_master()) self.assertFalse(slave2.is_alive()) # slave1 is resurrected self.send_OK_Message(HOST, PORT + 1) self.monitor._parse_message_from_queue() self.assertTrue(slave1.is_master()) self.assertTrue(slave1.is_alive()) # slave2 is resurrected self.send_OK_Message(HOST, PORT + 2) self.monitor._parse_message_from_queue() self.assertTrue(slave2.is_alive()) self.assertTrue(slave2.is_slave()) def test_all_resurrecting(self): print "test_all_resurrecting" master = self.monitor.cluster.add_node(HOST, PORT, ROLE_MASTER, REDIS_STATUS_KO) slave1 = self.monitor.cluster.add_node(HOST, PORT + 1, ROLE_SLAVE, REDIS_STATUS_KO) slave2 = self.monitor.cluster.add_node(HOST, PORT + 2, ROLE_SLAVE, REDIS_STATUS_KO) # kill slave1 self.send_KO_Message(HOST, PORT + 1) self.send_KO_Message(HOST, PORT) self.send_KO_Message(HOST, PORT + 2) self.monitor._parse_message_from_queue() self.assertTrue(slave1.is_slave()) self.assertFalse(slave1.is_alive()) self.monitor._parse_message_from_queue() self.assertTrue(master.is_master()) self.assertFalse(master.is_alive()) self.monitor._parse_message_from_queue() self.assertTrue(slave2.is_slave()) self.assertFalse(slave2.is_alive()) def send_OK_Message(self, redis_host, redis_port): self.send_message(redis_host, redis_port, REDIS_STATUS_OK) def send_KO_Message(self, redis_host, redis_port): self.send_message(redis_host, redis_port, REDIS_STATUS_KO) def send_message(self, redis_host, redis_port, redis_status): message = self.create_message(redis_host, redis_port, redis_status) self.monitor.queue.put(message) def create_message(self, redis_host, redis_port, redis_status): worker_name = '%s:%s' % (redis_host, redis_port) text = "%s,%s" % (worker_name, redis_status) return text def tearDown(self): print "### tearDown ###"
class TestRedisMonitor(TestCase): maxDiff = None queue_key = "link:istresearch.com:queue" def setUp(self): self.redis_monitor = RedisMonitor("localsettings.py") self.redis_monitor.settings = self.redis_monitor.wrapper.load( "localsettings.py") self.redis_monitor.logger = MagicMock() self.redis_monitor.settings['KAFKA_TOPIC_PREFIX'] = "demo_test" self.redis_monitor.settings['STATS_TOTAL'] = False self.redis_monitor.settings['STATS_PLUGINS'] = False self.redis_monitor.settings['PLUGINS'] = { 'plugins.info_monitor.InfoMonitor': None, 'plugins.stop_monitor.StopMonitor': None, 'plugins.expire_monitor.ExpireMonitor': None, 'tests.tests_online.CustomMonitor': 100, } self.redis_monitor.redis_conn = redis.Redis( host=self.redis_monitor.settings['REDIS_HOST'], port=self.redis_monitor.settings['REDIS_PORT']) self.redis_monitor._load_plugins() self.redis_monitor.stats_dict = {} self.kafka_conn = KafkaClient( self.redis_monitor.settings['KAFKA_HOSTS']) self.kafka_conn.ensure_topic_exists("demo_test.outbound_firehose") self.consumer = SimpleConsumer(self.kafka_conn, "demo-id", "demo_test.outbound_firehose") def test_process_item(self): # we only want to go to the end now, not after this test is ran self.consumer.seek(0, 2) # set the info flag key = "info-test:blah" value = "ABC123" self.redis_monitor.redis_conn.set(key, value) # process the request plugin = self.redis_monitor.plugins_dict.items()[0][1] self.redis_monitor._process_plugin(plugin) # ensure the key is gone self.assertEquals(self.redis_monitor.redis_conn.get(key), None) def test_sent_to_kafka(self): success = {u'info-test': "ABC123", u"appid": u"someapp"} # ensure it was sent out to kafka message_count = 0 for message in self.consumer.get_messages(): if message is None: break else: the_dict = json.loads(message.message.value) self.assertEquals(success, the_dict) message_count += 1 self.assertEquals(message_count, 1)
class TestRedisMonitor(TestCase): def setUp(self): self.redis_monitor = RedisMonitor("settings.py", True) self.redis_monitor.settings = self.redis_monitor.wrapper.load("settings.py") self.redis_monitor.logger = MagicMock() def test_load_plugins(self): # test loading default plugins assert_keys = [100, 200, 300, 400, 500] self.redis_monitor._load_plugins() self.assertEqual(list(self.redis_monitor.plugins_dict.keys()), assert_keys) # test removing a plugin from settings assert_keys = [100, 300, 400, 500] self.redis_monitor.settings['PLUGINS'] \ ['plugins.stop_monitor.StopMonitor'] = None self.redis_monitor._load_plugins() self.assertEqual(list(self.redis_monitor.plugins_dict.keys()), assert_keys) self.redis_monitor.settings['PLUGINS'] \ ['plugins.stop_monitor.StopMonitor'] = 200 # fail if the class is not found self.redis_monitor.settings['PLUGINS'] \ ['plugins.crazy_class.CrazyHandler'] = 400, self.assertRaises(ImportError, self.redis_monitor._load_plugins) del self.redis_monitor.settings['PLUGINS'] \ ['plugins.crazy_class.CrazyHandler'] self.redis_monitor.settings['PLUGINS'] = {} def test_active_plugins(self): # test that exceptions are caught within each plugin # assuming now all plugins are loaded self.redis_monitor._load_plugins() self.redis_monitor.stats_dict = {} # BaseExceptions are never raised normally list(self.redis_monitor.plugins_dict.items())[0][1]['instance'].handle = MagicMock(side_effect=BaseException("info")) list(self.redis_monitor.plugins_dict.items())[1][1]['instance'].handle = MagicMock(side_effect=BaseException("stop")) list(self.redis_monitor.plugins_dict.items())[2][1]['instance'].handle = MagicMock(side_effect=BaseException("expire")) self.redis_monitor.redis_conn = MagicMock() self.redis_monitor.redis_conn.scan_iter = MagicMock() self.redis_monitor._create_lock_object = MagicMock() # lets just assume the regex worked self.redis_monitor.redis_conn.scan_iter.return_value = ['somekey1'] # info try: plugin = list(self.redis_monitor.plugins_dict.items())[0][1] self.redis_monitor._process_plugin(plugin) self.fail("Info not called") except BaseException as e: self.assertEquals("info", str(e)) # action try: plugin = list(self.redis_monitor.plugins_dict.items())[1][1] self.redis_monitor._process_plugin(plugin) self.fail("Stop not called") except BaseException as e: self.assertEquals("stop", str(e)) # expire try: plugin = list(self.redis_monitor.plugins_dict.items())[2][1] self.redis_monitor._process_plugin(plugin) self.fail("Expire not called") except BaseException as e: self.assertEquals("expire", str(e)) # test that an exception within a handle method is caught self.redis_monitor._process_failures = MagicMock() self.redis_monitor._increment_fail_stat = MagicMock() try: list(self.redis_monitor.plugins_dict.items())[0][1]['instance'].handle = MagicMock(side_effect=Exception("normal")) plugin = list(self.redis_monitor.plugins_dict.items())[0][1] self.redis_monitor._process_plugin(plugin) except Exception as e: self.fail("Normal Exception not handled") self.assertTrue(self.redis_monitor._increment_fail_stat.called) self.assertTrue(self.redis_monitor._process_failures.called) def test_locking_actions(self): # tests for _process_plugins with locking self.redis_monitor._load_plugins() self.redis_monitor.redis_conn = MagicMock() self.redis_monitor.redis_conn = MagicMock() self.redis_monitor.redis_conn.scan_iter = MagicMock() self.redis_monitor.redis_conn.get = MagicMock(return_value=5) lock = MagicMock() lock.acquire lock.acquire = MagicMock(return_value=False) lock.release = MagicMock() lock._held = False self.redis_monitor._create_lock_object = MagicMock(return_value=lock) self.redis_monitor._increment_fail_stat = MagicMock() self.redis_monitor._process_failures = MagicMock() self.redis_monitor._process_key_val = MagicMock() # lets just assume the regex worked self.redis_monitor.redis_conn.scan_iter.return_value = ['somekey1'] plugin = {'instance': 'test', 'regex': 'abc123'} # test didnt acquire lock self.redis_monitor._process_key_val.assert_not_called() self.redis_monitor._process_plugin(plugin) self.redis_monitor._process_key_val.assert_not_called() lock.release.assert_not_called() # test got lock lock.acquire = MagicMock(return_value=True) lock._held = True self.redis_monitor._process_plugin(plugin) self.redis_monitor._process_key_val.assert_called_once_with('test', 'somekey1', 5) # test lock released lock.release.assert_called_once_with() # test lock not held not released lock._held = False lock.release.reset_mock() self.redis_monitor._process_plugin(plugin) lock.release.assert_not_called() def test_load_stats_plugins(self): # lets assume we are loading the default plugins self.redis_monitor._load_plugins() self.redis_monitor.redis_conn = MagicMock() # test no rolling stats self.redis_monitor.stats_dict = {} self.redis_monitor.settings['STATS_TIMES'] = [] self.redis_monitor._setup_stats_plugins() defaults = [ 'ExpireMonitor', 'StopMonitor', 'InfoMonitor', 'StatsMonitor', 'ZookeeperMonitor' ] self.assertEquals( sorted(self.redis_monitor.stats_dict['plugins'].keys()), sorted(defaults)) for key in self.redis_monitor.plugins_dict: plugin_name = self.redis_monitor.plugins_dict[key]['instance'].__class__.__name__ self.assertEquals( list(self.redis_monitor.stats_dict['plugins'][plugin_name].keys()), ['lifetime']) # test good/bad rolling stats self.redis_monitor.stats_dict = {} self.redis_monitor.settings['STATS_TIMES'] = [ 'SECONDS_15_MINUTE', 'SECONDS_1_HOUR', 'SECONDS_DUMB', ] good = [ 'lifetime', # for totals, not DUMB '900', '3600', ] self.redis_monitor._setup_stats_plugins() self.assertEquals( sorted(self.redis_monitor.stats_dict['plugins'].keys()), sorted(defaults)) for key in self.redis_monitor.plugins_dict: plugin_name = self.redis_monitor.plugins_dict[key]['instance'].__class__.__name__ self.assertEquals( sorted([str(x) for x in self.redis_monitor.stats_dict['plugins'][plugin_name].keys()]), sorted(good)) for plugin_key in self.redis_monitor.stats_dict['plugins']: k1 = 'stats:redis-monitor:{p}'.format(p=plugin_key) for time_key in self.redis_monitor.stats_dict['plugins'][plugin_key]: if time_key == 0: self.assertEquals( self.redis_monitor.stats_dict['plugins'][plugin_key][0].key, '{k}:lifetime'.format(k=k1) ) else: self.assertEquals( self.redis_monitor.stats_dict['plugins'][plugin_key][time_key].key, '{k}:{t}'.format(k=k1, t=time_key) ) def test_main_loop(self): self.redis_monitor._load_plugins() self.redis_monitor._process_plugin = MagicMock(side_effect=Exception( "normal")) try: self.redis_monitor._main_loop() self.fail("_process_plugin not called") except BaseException as e: self.assertEquals("normal", str(e)) def test_precondition(self): self.redis_monitor.stats_dict = {} instance = MagicMock() instance.check_precondition = MagicMock(return_value=False) instance.handle = MagicMock(side_effect=Exception("handler")) key = 'stuff' value = 'blah' # this should not raise an exception self.redis_monitor._process_key_val(instance, key, value) # this should instance.check_precondition = MagicMock(return_value=True) try: self.redis_monitor._process_key_val(instance, key, value) self.fail('handler not called') except BaseException as e: self.assertEquals('handler', str(e)) def test_get_fail_key(self): key = 'test' result = 'lock:test:failures' self.assertEquals(self.redis_monitor._get_fail_key(key), result) def test_process_failures(self): self.redis_monitor.settings = {'RETRY_FAILURES':True, 'RETRY_FAILURES_MAX': 3} self.redis_monitor._get_fail_key = MagicMock(return_value='the_key') self.redis_monitor.redis_conn = MagicMock() self.redis_monitor.redis_conn.set = MagicMock() # test not set self.redis_monitor.redis_conn.get = MagicMock(return_value=None) self.redis_monitor._process_failures('key1') self.redis_monitor.redis_conn.set.assert_called_once_with('the_key', 1) # test set self.redis_monitor.redis_conn.set.reset_mock() self.redis_monitor.redis_conn.get = MagicMock(return_value=2) self.redis_monitor._process_failures('key1') self.redis_monitor.redis_conn.set.assert_called_once_with('the_key', 3) # test exceeded self.redis_monitor.redis_conn.delete = MagicMock() self.redis_monitor.redis_conn.set.reset_mock() self.redis_monitor.redis_conn.get = MagicMock(return_value=3) self.redis_monitor._process_failures('key1') calls = [call('the_key'), call('key1')] self.redis_monitor.redis_conn.delete.assert_has_calls(calls) @mock.patch('socket.gethostname', return_value='host') @mock.patch('time.time', return_value=5) def test_report_self(self, h, t): self.redis_monitor.my_uuid = '1234' self.redis_monitor.redis_conn = MagicMock() self.redis_monitor.redis_conn.set = MagicMock() self.redis_monitor.redis_conn.expire = MagicMock() self.redis_monitor._report_self() self.redis_monitor.redis_conn.set.assert_called_once_with('stats:redis-monitor:self:host:1234', 5) self.redis_monitor.redis_conn.expire.assert_called_once_with('stats:redis-monitor:self:host:1234', 120)
class TestRedisMonitor(TestCase): maxDiff = None queue_key = "link:istresearch.com:queue" consumer = None def setUp(self): self.redis_monitor = RedisMonitor("localsettings.py") self.redis_monitor.settings = self.redis_monitor.wrapper.load( "localsettings.py") self.redis_monitor.logger = MagicMock() self.redis_monitor.settings['KAFKA_TOPIC_PREFIX'] = "demo_test" self.redis_monitor.settings['STATS_TOTAL'] = False self.redis_monitor.settings['STATS_PLUGINS'] = False self.redis_monitor.settings['PLUGINS'] = { 'plugins.info_monitor.InfoMonitor': None, 'plugins.stop_monitor.StopMonitor': None, 'plugins.expire_monitor.ExpireMonitor': None, 'tests.online.CustomMonitor': 100, } self.redis_monitor.redis_conn = redis.Redis( host=self.redis_monitor.settings['REDIS_HOST'], port=self.redis_monitor.settings['REDIS_PORT'], db=self.redis_monitor.settings['REDIS_DB']) self.redis_monitor._load_plugins() self.redis_monitor.stats_dict = {} self.consumer = KafkaConsumer( "demo_test.outbound_firehose", bootstrap_servers=self.redis_monitor.settings['KAFKA_HOSTS'], group_id="demo-id", auto_commit_interval_ms=10, consumer_timeout_ms=5000, auto_offset_reset='earliest') sleep(1) def test_process_item(self): # set the info flag key = "info-test:blah" value = "ABC1234" self.redis_monitor.redis_conn.set(key, value) # process the request plugin = list(self.redis_monitor.plugins_dict.items())[0][1] self.redis_monitor._process_plugin(plugin) # ensure the key is gone self.assertEquals(self.redis_monitor.redis_conn.get(key), None) self.redis_monitor.close() sleep(10) # now test the message was sent to kafka success = {u'info-test': "ABC1234", u"appid": u"someapp"} message_count = 0 m = next(self.consumer) if m is None: pass else: the_dict = json.loads(m.value) self.assertEquals(success, the_dict) message_count += 1 self.assertEquals(message_count, 1) def tearDown(self): # if for some reason the tests fail, we end up falling behind on # the consumer for m in self.consumer: pass self.consumer.close()
class TestRedisMonitor(TestCase): def setUp(self): self.redis_monitor = RedisMonitor("settings.py", True) self.redis_monitor.settings = self.redis_monitor.wrapper.load("settings.py") self.redis_monitor.logger = MagicMock() def test_load_plugins(self): # test loading default plugins assert_keys = [100, 200, 300, 400] self.redis_monitor._load_plugins() self.assertEqual(self.redis_monitor.plugins_dict.keys(), assert_keys) # test removing a plugin from settings assert_keys = [100, 300, 400] self.redis_monitor.settings['PLUGINS'] \ ['plugins.stop_monitor.StopMonitor'] = None self.redis_monitor._load_plugins() self.assertEqual(self.redis_monitor.plugins_dict.keys(), assert_keys) self.redis_monitor.settings['PLUGINS'] \ ['plugins.stop_monitor.StopMonitor'] = 200 # fail if the class is not found self.redis_monitor.settings['PLUGINS'] \ ['plugins.crazy_class.CrazyHandler'] = 400, self.assertRaises(ImportError, self.redis_monitor._load_plugins) del self.redis_monitor.settings['PLUGINS'] \ ['plugins.crazy_class.CrazyHandler'] self.redis_monitor.settings['PLUGINS'] = {} def test_active_plugins(self): # test that exceptions are caught within each plugin # assuming now all plugins are loaded self.redis_monitor._load_plugins() self.redis_monitor.stats_dict = {} # BaseExceptions are never raised normally self.redis_monitor.plugins_dict.items()[0][1]['instance'].handle = MagicMock(side_effect=BaseException("info")) self.redis_monitor.plugins_dict.items()[1][1]['instance'].handle = MagicMock(side_effect=BaseException("stop")) self.redis_monitor.plugins_dict.items()[2][1]['instance'].handle = MagicMock(side_effect=BaseException("expire")) self.redis_monitor.redis_conn = MagicMock() self.redis_monitor.redis_conn.scan_iter = MagicMock() # lets just assume the regex worked self.redis_monitor.redis_conn.scan_iter.return_value = ['somekey1'] # info try: plugin = self.redis_monitor.plugins_dict.items()[0][1] self.redis_monitor._process_plugin(plugin) self.fail("Info not called") except BaseException as e: self.assertEquals("info", e.message) # action try: plugin = self.redis_monitor.plugins_dict.items()[1][1] self.redis_monitor._process_plugin(plugin) self.fail("Stop not called") except BaseException as e: self.assertEquals("stop", e.message) # expire try: plugin = self.redis_monitor.plugins_dict.items()[2][1] self.redis_monitor._process_plugin(plugin) self.fail("Expire not called") except BaseException as e: self.assertEquals("expire", e.message) # test that an exception within a handle method is caught try: self.redis_monitor.plugins_dict.items()[0][1]['instance'].handle = MagicMock(side_effect=Exception("normal")) plugin = self.redis_monitor.plugins_dict.items()[0][1] self.redis_monitor._process_plugin(plugin) except Exception as e: self.fail("Normal Exception not handled") def test_load_stats_plugins(self): # lets assume we are loading the default plugins self.redis_monitor._load_plugins() self.redis_monitor.redis_conn = MagicMock() # test no rolling stats self.redis_monitor.stats_dict = {} self.redis_monitor.settings['STATS_TIMES'] = [] self.redis_monitor._setup_stats_plugins() defaults = [ 'ExpireMonitor', 'StopMonitor', 'InfoMonitor', 'StatsMonitor' ] self.assertEquals( sorted(self.redis_monitor.stats_dict['plugins'].keys()), sorted(defaults)) for key in self.redis_monitor.plugins_dict: plugin_name = self.redis_monitor.plugins_dict[key]['instance'].__class__.__name__ self.assertEquals( self.redis_monitor.stats_dict['plugins'][plugin_name].keys(), ['lifetime']) # test good/bad rolling stats self.redis_monitor.stats_dict = {} self.redis_monitor.settings['STATS_TIMES'] = [ 'SECONDS_15_MINUTE', 'SECONDS_1_HOUR', 'SECONDS_DUMB', ] good = [ 'lifetime', # for totals, not DUMB 900, 3600, ] self.redis_monitor._setup_stats_plugins() self.assertEquals( sorted(self.redis_monitor.stats_dict['plugins'].keys()), sorted(defaults)) for key in self.redis_monitor.plugins_dict: plugin_name = self.redis_monitor.plugins_dict[key]['instance'].__class__.__name__ self.assertEquals( sorted(self.redis_monitor.stats_dict['plugins'][plugin_name].keys()), sorted(good)) for plugin_key in self.redis_monitor.stats_dict['plugins']: k1 = 'stats:redis-monitor:{p}'.format(p=plugin_key) for time_key in self.redis_monitor.stats_dict['plugins'][plugin_key]: if time_key == 0: self.assertEquals( self.redis_monitor.stats_dict['plugins'][plugin_key][0].key, '{k}:lifetime'.format(k=k1) ) else: self.assertEquals( self.redis_monitor.stats_dict['plugins'][plugin_key][time_key].key, '{k}:{t}'.format(k=k1, t=time_key) ) def test_main_loop(self): self.redis_monitor._load_plugins() self.redis_monitor._process_plugin = MagicMock(side_effect=Exception( "normal")) try: self.redis_monitor._main_loop() self.fail("_process_plugin not called") except BaseException as e: self.assertEquals("normal", e.message) def test_precondition(self): self.redis_monitor.stats_dict = {} instance = MagicMock() instance.check_precondition = MagicMock(return_value=False) instance.handle = MagicMock(side_effect=Exception("handler")) key = 'stuff' value = 'blah' # this should not raise an exception self.redis_monitor._process_key_val(instance, key, value) # this should instance.check_precondition = MagicMock(return_value=True) try: self.redis_monitor._process_key_val(instance, key, value) self.fail('handler not called') except BaseException as e: self.assertEquals('handler', e.message)
class TestRedisMonitor(TestCase): maxDiff = None queue_key = "link:istresearch.com:queue" consumer = None def setUp(self): self.redis_monitor = RedisMonitor("localsettings.py") self.redis_monitor.settings = self.redis_monitor.wrapper.load("localsettings.py") self.redis_monitor.logger = MagicMock() self.redis_monitor.settings['KAFKA_TOPIC_PREFIX'] = "demo_test" self.redis_monitor.settings['STATS_TOTAL'] = False self.redis_monitor.settings['STATS_PLUGINS'] = False self.redis_monitor.settings['PLUGINS'] = { 'plugins.info_monitor.InfoMonitor': None, 'plugins.stop_monitor.StopMonitor': None, 'plugins.expire_monitor.ExpireMonitor': None, 'tests.online.CustomMonitor': 100, } self.redis_monitor.redis_conn = redis.Redis( host=self.redis_monitor.settings['REDIS_HOST'], port=self.redis_monitor.settings['REDIS_PORT'], db=self.redis_monitor.settings['REDIS_DB']) self.redis_monitor._load_plugins() self.redis_monitor.stats_dict = {} self.consumer = KafkaConsumer( "demo_test.outbound_firehose", bootstrap_servers=self.redis_monitor.settings['KAFKA_HOSTS'], group_id="demo-id", auto_commit_interval_ms=10, consumer_timeout_ms=5000, auto_offset_reset='earliest' ) sleep(1) def test_process_item(self): # set the info flag key = "info-test:blah" value = "ABC1234" self.redis_monitor.redis_conn.set(key, value) # process the request plugin = list(self.redis_monitor.plugins_dict.items())[0][1] self.redis_monitor._process_plugin(plugin) # ensure the key is gone self.assertEquals(self.redis_monitor.redis_conn.get(key), None) self.redis_monitor.close() sleep(10) # now test the message was sent to kafka success = { u'info-test': "ABC1234", u"appid": u"someapp" } message_count = 0 m = next(self.consumer) if m is None: pass else: the_dict = json.loads(m.value) self.assertEquals(success, the_dict) message_count += 1 self.assertEquals(message_count, 1) def tearDown(self): # if for some reason the tests fail, we end up falling behind on # the consumer for m in self.consumer: pass self.consumer.close()
class TestRedisMonitor(TestCase): def setUp(self): self.redis_monitor = RedisMonitor("settings.py", True) self.redis_monitor.settings = self.redis_monitor.wrapper.load("settings.py") self.redis_monitor.logger = MagicMock() def test_load_plugins(self): # test loading default plugins assert_keys = [100, 200, 300, 400] self.redis_monitor._load_plugins() self.assertEqual(self.redis_monitor.plugins_dict.keys(), assert_keys) # test removing a plugin from settings assert_keys = [100, 300, 400] self.redis_monitor.settings["PLUGINS"]["plugins.stop_monitor.StopMonitor"] = None self.redis_monitor._load_plugins() self.assertEqual(self.redis_monitor.plugins_dict.keys(), assert_keys) self.redis_monitor.settings["PLUGINS"]["plugins.stop_monitor.StopMonitor"] = 200 # fail if the class is not found self.redis_monitor.settings["PLUGINS"]["plugins.crazy_class.CrazyHandler"] = (400,) self.assertRaises(ImportError, self.redis_monitor._load_plugins) del self.redis_monitor.settings["PLUGINS"]["plugins.crazy_class.CrazyHandler"] self.redis_monitor.settings["PLUGINS"] = {} def test_active_plugins(self): # test that exceptions are caught within each plugin # assuming now all plugins are loaded self.redis_monitor._load_plugins() self.redis_monitor.stats_dict = {} # BaseExceptions are never raised normally self.redis_monitor.plugins_dict.items()[0][1]["instance"].handle = MagicMock(side_effect=BaseException("info")) self.redis_monitor.plugins_dict.items()[1][1]["instance"].handle = MagicMock(side_effect=BaseException("stop")) self.redis_monitor.plugins_dict.items()[2][1]["instance"].handle = MagicMock( side_effect=BaseException("expire") ) self.redis_monitor.redis_conn = MagicMock() self.redis_monitor.redis_conn.scan_iter = MagicMock() # lets just assume the regex worked self.redis_monitor.redis_conn.scan_iter.return_value = ["somekey1"] # info try: plugin = self.redis_monitor.plugins_dict.items()[0][1] self.redis_monitor._process_plugin(plugin) self.fail("Info not called") except BaseException as e: self.assertEquals("info", e.message) # action try: plugin = self.redis_monitor.plugins_dict.items()[1][1] self.redis_monitor._process_plugin(plugin) self.fail("Stop not called") except BaseException as e: self.assertEquals("stop", e.message) # expire try: plugin = self.redis_monitor.plugins_dict.items()[2][1] self.redis_monitor._process_plugin(plugin) self.fail("Expire not called") except BaseException as e: self.assertEquals("expire", e.message) # test that an exception within a handle method is caught try: self.redis_monitor.plugins_dict.items()[0][1]["instance"].handle = MagicMock( side_effect=Exception("normal") ) plugin = self.redis_monitor.plugins_dict.items()[0][1] self.redis_monitor._process_plugin(plugin) except Exception as e: self.fail("Normal Exception not handled") def test_load_stats_plugins(self): # lets assume we are loading the default plugins self.redis_monitor._load_plugins() self.redis_monitor.redis_conn = MagicMock() # test no rolling stats self.redis_monitor.stats_dict = {} self.redis_monitor.settings["STATS_TIMES"] = [] self.redis_monitor._setup_stats_plugins() defaults = ["ExpireMonitor", "StopMonitor", "InfoMonitor", "StatsMonitor"] self.assertEquals(sorted(self.redis_monitor.stats_dict["plugins"].keys()), sorted(defaults)) for key in self.redis_monitor.plugins_dict: plugin_name = self.redis_monitor.plugins_dict[key]["instance"].__class__.__name__ self.assertEquals(self.redis_monitor.stats_dict["plugins"][plugin_name].keys(), ["lifetime"]) # test good/bad rolling stats self.redis_monitor.stats_dict = {} self.redis_monitor.settings["STATS_TIMES"] = ["SECONDS_15_MINUTE", "SECONDS_1_HOUR", "SECONDS_DUMB"] good = ["lifetime", 900, 3600] # for totals, not DUMB self.redis_monitor._setup_stats_plugins() self.assertEquals(sorted(self.redis_monitor.stats_dict["plugins"].keys()), sorted(defaults)) for key in self.redis_monitor.plugins_dict: plugin_name = self.redis_monitor.plugins_dict[key]["instance"].__class__.__name__ self.assertEquals(sorted(self.redis_monitor.stats_dict["plugins"][plugin_name].keys()), sorted(good)) for plugin_key in self.redis_monitor.stats_dict["plugins"]: k1 = "stats:redis-monitor:{p}".format(p=plugin_key) for time_key in self.redis_monitor.stats_dict["plugins"][plugin_key]: if time_key == 0: self.assertEquals( self.redis_monitor.stats_dict["plugins"][plugin_key][0].key, "{k}:lifetime".format(k=k1) ) else: self.assertEquals( self.redis_monitor.stats_dict["plugins"][plugin_key][time_key].key, "{k}:{t}".format(k=k1, t=time_key), ) def test_main_loop(self): self.redis_monitor._load_plugins() self.redis_monitor._process_plugin = MagicMock(side_effect=Exception("normal")) try: self.redis_monitor._main_loop() self.fail("_process_plugin not called") except BaseException as e: self.assertEquals("normal", e.message) def test_precondition(self): self.redis_monitor.stats_dict = {} instance = MagicMock() instance.check_precondition = MagicMock(return_value=False) instance.handle = MagicMock(side_effect=Exception("handler")) key = "stuff" value = "blah" # this should not raise an exception self.redis_monitor._process_key_val(instance, key, value) # this should instance.check_precondition = MagicMock(return_value=True) try: self.redis_monitor._process_key_val(instance, key, value) self.fail("handler not called") except BaseException as e: self.assertEquals("handler", e.message)
def setUp(self): self.redis_monitor = RedisMonitor("settings.py", True) self.redis_monitor.settings = self.redis_monitor.wrapper.load("settings.py") self.redis_monitor.logger = MagicMock()
class TestRedisMonitor(TestCase): maxDiff = None queue_key = "link:istresearch.com:queue" def setUp(self): self.redis_monitor = RedisMonitor("localsettings.py") self.redis_monitor.settings = self.redis_monitor.wrapper.load("localsettings.py") self.redis_monitor.logger = MagicMock() self.redis_monitor.settings['KAFKA_TOPIC_PREFIX'] = "demo_test" self.redis_monitor.settings['STATS_TOTAL'] = False self.redis_monitor.settings['STATS_PLUGINS'] = False self.redis_monitor.settings['PLUGINS'] = { 'plugins.info_monitor.InfoMonitor': None, 'plugins.stop_monitor.StopMonitor': None, 'plugins.expire_monitor.ExpireMonitor': None, 'tests.tests_online.CustomMonitor': 100, } self.redis_monitor.redis_conn = redis.Redis( host=self.redis_monitor.settings['REDIS_HOST'], port=self.redis_monitor.settings['REDIS_PORT']) self.redis_monitor._load_plugins() self.redis_monitor.stats_dict = {} self.kafka_conn = KafkaClient(self.redis_monitor.settings[ 'KAFKA_HOSTS']) self.kafka_conn.ensure_topic_exists("demo_test.outbound_firehose") self.consumer = SimpleConsumer( self.kafka_conn, "demo-id", "demo_test.outbound_firehose" ) def test_process_item(self): # we only want to go to the end now, not after this test is ran self.consumer.seek(0, 2) # set the info flag key = "info-test:blah" value = "ABC123" self.redis_monitor.redis_conn.set(key, value) # process the request plugin = self.redis_monitor.plugins_dict.items()[0][1] self.redis_monitor._process_plugin(plugin) # ensure the key is gone self.assertEquals(self.redis_monitor.redis_conn.get(key), None) def test_sent_to_kafka(self): success = { u'info-test': "ABC123", u"appid": u"someapp" } # ensure it was sent out to kafka message_count = 0 for message in self.consumer.get_messages(): if message is None: break else: the_dict = json.loads(message.message.value) self.assertEquals(success, the_dict) message_count += 1 self.assertEquals(message_count, 1)