Exemplo n.º 1
0
    def setUp(self):
        self.redis_monitor = RedisMonitor("localsettings.py")
        self.redis_monitor.settings = self.redis_monitor.wrapper.load(
            "localsettings.py")
        self.redis_monitor.logger = MagicMock()
        self.redis_monitor.settings['KAFKA_TOPIC_PREFIX'] = "demo_test"
        self.redis_monitor.settings['STATS_TOTAL'] = False
        self.redis_monitor.settings['STATS_PLUGINS'] = False
        self.redis_monitor.settings['PLUGINS'] = {
            'plugins.info_monitor.InfoMonitor': None,
            'plugins.stop_monitor.StopMonitor': None,
            'plugins.expire_monitor.ExpireMonitor': None,
            'tests.tests_online.CustomMonitor': 100,
        }
        self.redis_monitor.redis_conn = redis.Redis(
            host=self.redis_monitor.settings['REDIS_HOST'],
            port=self.redis_monitor.settings['REDIS_PORT'])

        self.redis_monitor._load_plugins()
        self.redis_monitor.stats_dict = {}

        self.kafka_conn = KafkaClient(
            self.redis_monitor.settings['KAFKA_HOSTS'])
        self.kafka_conn.ensure_topic_exists("demo_test.outbound_firehose")

        self.consumer = SimpleConsumer(self.kafka_conn, "demo-id",
                                       "demo_test.outbound_firehose")
Exemplo n.º 2
0
    def setUp(self):
        self.redis_monitor = RedisMonitor("localsettings.py")
        self.redis_monitor.settings = self.redis_monitor.wrapper.load(
            "localsettings.py")
        self.redis_monitor.logger = MagicMock()
        self.redis_monitor.settings['KAFKA_TOPIC_PREFIX'] = "demo_test"
        self.redis_monitor.settings['STATS_TOTAL'] = False
        self.redis_monitor.settings['STATS_PLUGINS'] = False
        self.redis_monitor.settings['PLUGINS'] = {
            'plugins.info_monitor.InfoMonitor': None,
            'plugins.stop_monitor.StopMonitor': None,
            'plugins.expire_monitor.ExpireMonitor': None,
            'tests.online.CustomMonitor': 100,
        }
        self.redis_monitor.redis_conn = redis.Redis(
            host=self.redis_monitor.settings['REDIS_HOST'],
            port=self.redis_monitor.settings['REDIS_PORT'],
            db=self.redis_monitor.settings['REDIS_DB'])

        self.redis_monitor._load_plugins()
        self.redis_monitor.stats_dict = {}

        self.consumer = KafkaConsumer(
            "demo_test.outbound_firehose",
            bootstrap_servers=self.redis_monitor.settings['KAFKA_HOSTS'],
            group_id="demo-id",
            auto_commit_interval_ms=10,
            consumer_timeout_ms=5000,
            auto_offset_reset='earliest')
        sleep(1)
Exemplo n.º 3
0
 def setUp(self):
     print "### setUp ###"
     self.monitor = RedisMonitor(
         zk=None,
         redis_hosts="localhost:6379,localhost:6389,localhost:6399",
         sleep_time=30,
         zk_path="/test/redis")
     self.monitor.zk = MockZooKeeper("zk")
     self.monitor.redis_class = MockRedis
     self.monitor.zk_properties = MockZooKeeper("zk_properties")
     self.monitor.queue = Queue.Queue()
 def setUp(self):
     print "### setUp ###"
     self.monitor = RedisMonitor(zk=None, redis_hosts="localhost:6379,localhost:6389,localhost:6399", sleep_time=30, zk_path="/test/redis")
     self.monitor.zk = MockZooKeeper("zk")
     self.monitor.redis_class = MockRedis
     self.monitor.zk_properties = MockZooKeeper("zk_properties")
     self.monitor.queue = Queue.Queue()
Exemplo n.º 5
0
    def setUp(self):
        self.redis_monitor = RedisMonitor("localsettings.py")
        self.redis_monitor.settings = self.redis_monitor.wrapper.load("localsettings.py")
        self.redis_monitor.logger = MagicMock()
        self.redis_monitor.settings['KAFKA_TOPIC_PREFIX'] = "demo_test"
        self.redis_monitor.settings['STATS_TOTAL'] = False
        self.redis_monitor.settings['STATS_PLUGINS'] = False
        self.redis_monitor.settings['PLUGINS'] = {
            'plugins.info_monitor.InfoMonitor': None,
            'plugins.stop_monitor.StopMonitor': None,
            'plugins.expire_monitor.ExpireMonitor': None,
            'tests.tests_online.CustomMonitor': 100,
        }
        self.redis_monitor.redis_conn = redis.Redis(
            host=self.redis_monitor.settings['REDIS_HOST'],
            port=self.redis_monitor.settings['REDIS_PORT'])

        self.redis_monitor._load_plugins()
        self.redis_monitor.stats_dict = {}

        self.kafka_conn = KafkaClient(self.redis_monitor.settings[
                                      'KAFKA_HOSTS'])
        self.kafka_conn.ensure_topic_exists("demo_test.outbound_firehose")

        self.consumer = SimpleConsumer(
            self.kafka_conn,
            "demo-id",
            "demo_test.outbound_firehose"
        )
Exemplo n.º 6
0
    def setUp(self):
        self.redis_monitor = RedisMonitor("localsettings.py")
        self.redis_monitor.settings = self.redis_monitor.wrapper.load("localsettings.py")
        self.redis_monitor.logger = MagicMock()
        self.redis_monitor.settings['KAFKA_TOPIC_PREFIX'] = "demo_test"
        self.redis_monitor.settings['STATS_TOTAL'] = False
        self.redis_monitor.settings['STATS_PLUGINS'] = False
        self.redis_monitor.settings['PLUGINS'] = {
            'plugins.info_monitor.InfoMonitor': None,
            'plugins.stop_monitor.StopMonitor': None,
            'plugins.expire_monitor.ExpireMonitor': None,
            'tests.online.CustomMonitor': 100,
        }
        self.redis_monitor.redis_conn = redis.Redis(
            host=self.redis_monitor.settings['REDIS_HOST'],
            port=self.redis_monitor.settings['REDIS_PORT'],
            db=self.redis_monitor.settings['REDIS_DB'])

        self.redis_monitor._load_plugins()
        self.redis_monitor.stats_dict = {}

        self.consumer = KafkaConsumer(
            "demo_test.outbound_firehose",
            bootstrap_servers=self.redis_monitor.settings['KAFKA_HOSTS'],
            group_id="demo-id",
            auto_commit_interval_ms=10,
            consumer_timeout_ms=5000,
            auto_offset_reset='earliest'
        )
        sleep(1)
 def setUp(self):
     self.redis_monitor = RedisMonitor("settings.py", True)
     self.redis_monitor.settings = self.redis_monitor.wrapper.load(
         "settings.py")
     self.redis_monitor.logger = MagicMock()
class TestRedisMonitor(TestCase):
    def setUp(self):
        self.redis_monitor = RedisMonitor("settings.py", True)
        self.redis_monitor.settings = self.redis_monitor.wrapper.load(
            "settings.py")
        self.redis_monitor.logger = MagicMock()

    def test_load_plugins(self):
        # test loading default plugins
        assert_keys = [100, 200, 300, 400, 500]
        self.redis_monitor._load_plugins()
        self.assertEqual(list(self.redis_monitor.plugins_dict.keys()),
                         assert_keys)

        # test removing a plugin from settings
        assert_keys = [100, 300, 400, 500]
        self.redis_monitor.settings['PLUGINS'] \
            ['plugins.stop_monitor.StopMonitor'] = None
        self.redis_monitor._load_plugins()
        self.assertEqual(list(self.redis_monitor.plugins_dict.keys()),
                         assert_keys)
        self.redis_monitor.settings['PLUGINS'] \
            ['plugins.stop_monitor.StopMonitor'] = 200

        # fail if the class is not found
        self.redis_monitor.settings['PLUGINS'] \
            ['plugins.crazy_class.CrazyHandler'] = 400,

        self.assertRaises(ImportError, self.redis_monitor._load_plugins)
        del self.redis_monitor.settings['PLUGINS'] \
            ['plugins.crazy_class.CrazyHandler']
        self.redis_monitor.settings['PLUGINS'] = {}

    def test_active_plugins(self):
        # test that exceptions are caught within each plugin
        # assuming now all plugins are loaded
        self.redis_monitor._load_plugins()
        self.redis_monitor.stats_dict = {}

        # BaseExceptions are never raised normally
        list(self.redis_monitor.plugins_dict.items()
             )[0][1]['instance'].handle = MagicMock(
                 side_effect=BaseException("info"))
        list(self.redis_monitor.plugins_dict.items()
             )[1][1]['instance'].handle = MagicMock(
                 side_effect=BaseException("stop"))
        list(self.redis_monitor.plugins_dict.items()
             )[2][1]['instance'].handle = MagicMock(
                 side_effect=BaseException("expire"))
        self.redis_monitor.redis_conn = MagicMock()
        self.redis_monitor.redis_conn.scan_iter = MagicMock()
        self.redis_monitor._create_lock_object = MagicMock()

        # lets just assume the regex worked
        self.redis_monitor.redis_conn.scan_iter.return_value = ['somekey1']

        # info
        try:
            plugin = list(self.redis_monitor.plugins_dict.items())[0][1]
            self.redis_monitor._process_plugin(plugin)
            self.fail("Info not called")
        except BaseException as e:
            self.assertEqual("info", str(e))

        # action
        try:
            plugin = list(self.redis_monitor.plugins_dict.items())[1][1]
            self.redis_monitor._process_plugin(plugin)
            self.fail("Stop not called")
        except BaseException as e:
            self.assertEqual("stop", str(e))

        # expire
        try:
            plugin = list(self.redis_monitor.plugins_dict.items())[2][1]
            self.redis_monitor._process_plugin(plugin)
            self.fail("Expire not called")
        except BaseException as e:
            self.assertEqual("expire", str(e))

        # test that an exception within a handle method is caught
        self.redis_monitor._process_failures = MagicMock()
        self.redis_monitor._increment_fail_stat = MagicMock()
        try:
            list(self.redis_monitor.plugins_dict.items()
                 )[0][1]['instance'].handle = MagicMock(
                     side_effect=Exception("normal"))
            plugin = list(self.redis_monitor.plugins_dict.items())[0][1]
            self.redis_monitor._process_plugin(plugin)

        except Exception as e:
            self.fail("Normal Exception not handled")

        self.assertTrue(self.redis_monitor._increment_fail_stat.called)
        self.assertTrue(self.redis_monitor._process_failures.called)

    def test_locking_actions(self):
        # tests for _process_plugins with locking
        self.redis_monitor._load_plugins()
        self.redis_monitor.redis_conn = MagicMock()
        self.redis_monitor.redis_conn = MagicMock()
        self.redis_monitor.redis_conn.scan_iter = MagicMock()
        self.redis_monitor.redis_conn.get = MagicMock(return_value=5)

        lock = MagicMock()
        lock.acquire
        lock.acquire = MagicMock(return_value=False)
        lock.release = MagicMock()
        lock._held = False
        self.redis_monitor._create_lock_object = MagicMock(return_value=lock)

        self.redis_monitor._increment_fail_stat = MagicMock()
        self.redis_monitor._process_failures = MagicMock()
        self.redis_monitor._process_key_val = MagicMock()
        # lets just assume the regex worked
        self.redis_monitor.redis_conn.scan_iter.return_value = ['somekey1']

        plugin = {'instance': 'test', 'regex': 'abc123'}

        # test didnt acquire lock
        self.redis_monitor._process_key_val.assert_not_called()
        self.redis_monitor._process_plugin(plugin)
        self.redis_monitor._process_key_val.assert_not_called()
        lock.release.assert_not_called()

        # test got lock
        lock.acquire = MagicMock(return_value=True)
        lock._held = True
        self.redis_monitor._process_plugin(plugin)
        self.redis_monitor._process_key_val.assert_called_once_with(
            'test', 'somekey1', 5)
        # test lock released
        lock.release.assert_called_once_with()

        # test lock not held not released
        lock._held = False
        lock.release.reset_mock()
        self.redis_monitor._process_plugin(plugin)
        lock.release.assert_not_called()

    def test_load_stats_plugins(self):
        # lets assume we are loading the default plugins
        self.redis_monitor._load_plugins()
        self.redis_monitor.redis_conn = MagicMock()

        # test no rolling stats
        self.redis_monitor.stats_dict = {}
        self.redis_monitor.settings['STATS_TIMES'] = []
        self.redis_monitor._setup_stats_plugins()
        defaults = [
            'ExpireMonitor', 'StopMonitor', 'InfoMonitor', 'StatsMonitor',
            'ZookeeperMonitor'
        ]

        self.assertEqual(
            sorted(self.redis_monitor.stats_dict['plugins'].keys()),
            sorted(defaults))

        for key in self.redis_monitor.plugins_dict:
            plugin_name = self.redis_monitor.plugins_dict[key][
                'instance'].__class__.__name__
            self.assertEqual(
                list(self.redis_monitor.stats_dict['plugins']
                     [plugin_name].keys()), ['lifetime'])

        # test good/bad rolling stats
        self.redis_monitor.stats_dict = {}
        self.redis_monitor.settings['STATS_TIMES'] = [
            'SECONDS_15_MINUTE',
            'SECONDS_1_HOUR',
            'SECONDS_DUMB',
        ]
        good = [
            'lifetime',  # for totals, not DUMB
            '900',
            '3600',
        ]

        self.redis_monitor._setup_stats_plugins()

        self.assertEqual(
            sorted(self.redis_monitor.stats_dict['plugins'].keys()),
            sorted(defaults))

        for key in self.redis_monitor.plugins_dict:
            plugin_name = self.redis_monitor.plugins_dict[key][
                'instance'].__class__.__name__
            self.assertEqual(
                sorted([
                    str(x) for x in self.redis_monitor.stats_dict['plugins']
                    [plugin_name].keys()
                ]), sorted(good))

        for plugin_key in self.redis_monitor.stats_dict['plugins']:
            k1 = 'stats:redis-monitor:{p}'.format(p=plugin_key)
            for time_key in self.redis_monitor.stats_dict['plugins'][
                    plugin_key]:
                if time_key == 0:
                    self.assertEqual(
                        self.redis_monitor.stats_dict['plugins'][plugin_key]
                        [0].key, '{k}:lifetime'.format(k=k1))
                else:
                    self.assertEqual(
                        self.redis_monitor.stats_dict['plugins'][plugin_key]
                        [time_key].key, '{k}:{t}'.format(k=k1, t=time_key))

    def test_main_loop(self):
        self.redis_monitor._load_plugins()
        self.redis_monitor._process_plugin = MagicMock(
            side_effect=Exception("normal"))

        try:
            self.redis_monitor._main_loop()
            self.fail("_process_plugin not called")
        except BaseException as e:
            self.assertEqual("normal", str(e))

    def test_precondition(self):
        self.redis_monitor.stats_dict = {}
        instance = MagicMock()
        instance.check_precondition = MagicMock(return_value=False)
        instance.handle = MagicMock(side_effect=Exception("handler"))
        key = 'stuff'
        value = 'blah'

        # this should not raise an exception
        self.redis_monitor._process_key_val(instance, key, value)

        # this should
        instance.check_precondition = MagicMock(return_value=True)
        try:
            self.redis_monitor._process_key_val(instance, key, value)
            self.fail('handler not called')
        except BaseException as e:
            self.assertEqual('handler', str(e))

    def test_get_fail_key(self):
        key = 'test'
        result = 'lock:test:failures'
        self.assertEqual(self.redis_monitor._get_fail_key(key), result)

    def test_process_failures(self):
        self.redis_monitor.settings = {
            'RETRY_FAILURES': True,
            'RETRY_FAILURES_MAX': 3
        }
        self.redis_monitor._get_fail_key = MagicMock(return_value='the_key')
        self.redis_monitor.redis_conn = MagicMock()
        self.redis_monitor.redis_conn.set = MagicMock()

        # test not set
        self.redis_monitor.redis_conn.get = MagicMock(return_value=None)
        self.redis_monitor._process_failures('key1')
        self.redis_monitor.redis_conn.set.assert_called_once_with('the_key', 1)

        # test set
        self.redis_monitor.redis_conn.set.reset_mock()
        self.redis_monitor.redis_conn.get = MagicMock(return_value=2)
        self.redis_monitor._process_failures('key1')
        self.redis_monitor.redis_conn.set.assert_called_once_with('the_key', 3)

        # test exceeded
        self.redis_monitor.redis_conn.delete = MagicMock()
        self.redis_monitor.redis_conn.set.reset_mock()
        self.redis_monitor.redis_conn.get = MagicMock(return_value=3)
        self.redis_monitor._process_failures('key1')
        calls = [call('the_key'), call('key1')]
        self.redis_monitor.redis_conn.delete.assert_has_calls(calls)

    @mock.patch('socket.gethostname', return_value='host')
    @mock.patch('time.time', return_value=5)
    def test_report_self(self, h, t):
        self.redis_monitor.my_uuid = '1234'
        self.redis_monitor.redis_conn = MagicMock()
        self.redis_monitor.redis_conn.set = MagicMock()
        self.redis_monitor.redis_conn.expire = MagicMock()

        self.redis_monitor._report_self()
        self.redis_monitor.redis_conn.set.assert_called_once_with(
            'stats:redis-monitor:self:host:1234', 5)
        self.redis_monitor.redis_conn.expire.assert_called_once_with(
            'stats:redis-monitor:self:host:1234', 120)
class TestRedisMonitor(unittest.TestCase):

    def setUp(self):
        print "### setUp ###"
        self.monitor = RedisMonitor(zk=None, redis_hosts="localhost:6379,localhost:6389,localhost:6399", sleep_time=30, zk_path="/test/redis")
        self.monitor.zk = MockZooKeeper("zk")
        self.monitor.redis_class = MockRedis
        self.monitor.zk_properties = MockZooKeeper("zk_properties")
        self.monitor.queue = Queue.Queue()

    def test_all_ok(self):
        print "test_all_ok"
        master = self.monitor.cluster.add_node(HOST, PORT, ROLE_MASTER, REDIS_STATUS_OK)
        slave1 = self.monitor.cluster.add_node(HOST, PORT+1, ROLE_SLAVE, REDIS_STATUS_OK)
        slave2 = self.monitor.cluster.add_node(HOST, PORT+2, ROLE_SLAVE, REDIS_STATUS_OK)
        
        self.send_OK_Message(HOST, PORT)
        self.monitor._parse_message_from_queue()
        
        self.send_OK_Message(HOST, PORT+1)
        self.monitor._parse_message_from_queue()
        
        self.send_OK_Message(HOST, PORT+2)
        self.monitor._parse_message_from_queue()

        self.assertTrue(master.is_master())
        self.assertTrue(master.is_alive())
        
        self.assertTrue(slave1.is_slave())
        self.assertTrue(slave1.is_alive())
        
        self.assertTrue(slave2.is_slave())
        self.assertTrue(slave2.is_alive())

    def test_one_slave_KO(self):
        print "test_one_slave_KO"
        master = self.monitor.cluster.add_node(HOST, PORT, ROLE_MASTER, REDIS_STATUS_OK)
        slave1 = self.monitor.cluster.add_node(HOST, PORT+1, ROLE_SLAVE, REDIS_STATUS_OK)
        slave2 = self.monitor.cluster.add_node(HOST, PORT+2, ROLE_SLAVE, REDIS_STATUS_OK)
        
        self.send_OK_Message(HOST, PORT+1)
        self.monitor._parse_message_from_queue()
        
        self.send_OK_Message(HOST, PORT)
        self.monitor._parse_message_from_queue()
        
        self.send_KO_Message(HOST, PORT+2)
        self.monitor._parse_message_from_queue()

        self.assertTrue(master.is_master())
        self.assertTrue(master.is_alive())

        self.assertTrue(slave1.is_slave())
        self.assertTrue(slave1.is_alive())

        self.assertTrue(slave2.is_slave())
        self.assertFalse(slave2.is_alive())

    def test_master_KO(self):
        print "test_master_KO"
        master = self.monitor.cluster.add_node(HOST, PORT, ROLE_MASTER, REDIS_STATUS_OK)
        slave1 = self.monitor.cluster.add_node(HOST, PORT+1, ROLE_SLAVE, REDIS_STATUS_OK)
        slave2 = self.monitor.cluster.add_node(HOST, PORT+2, ROLE_SLAVE, REDIS_STATUS_OK)
        
        self.send_OK_Message(HOST, PORT+1)
        self.monitor._parse_message_from_queue()
        
        self.send_KO_Message(HOST, PORT)
        self.monitor._parse_message_from_queue()
        
        self.send_OK_Message(HOST, PORT+2)
        self.monitor._parse_message_from_queue()

        self.assertTrue(master.is_slave())
        self.assertFalse(master.is_alive())

        self.assertTrue(slave1.is_master())
        self.assertTrue(slave1.is_alive())

        self.assertTrue(slave2.is_slave())
        self.assertTrue(slave2.is_alive())

    def test_all_going_KO_master(self):
        print "test_all_going_KO_master"
        master = self.monitor.cluster.add_node(HOST, PORT, ROLE_MASTER, REDIS_STATUS_OK)
        slave1 = self.monitor.cluster.add_node(HOST, PORT+1, ROLE_SLAVE, REDIS_STATUS_OK)
        slave2 = self.monitor.cluster.add_node(HOST, PORT+2, ROLE_SLAVE, REDIS_STATUS_OK)
        
        # kill slave1
        self.send_KO_Message(HOST, PORT+1)
        self.monitor._parse_message_from_queue()

        self.assertTrue(slave1.is_slave())
        self.assertFalse(slave1.is_alive())
        
        # kill master
        self.send_KO_Message(HOST, PORT)
        self.monitor._parse_message_from_queue()

        self.assertTrue(master.is_slave())
        self.assertFalse(master.is_alive())
        
        # kill slave2
        self.send_KO_Message(HOST, PORT+2)
        self.monitor._parse_message_from_queue()

        self.assertTrue(slave2.is_master())
        self.assertFalse(slave2.is_alive())

        # slave2 is resurrected as master
        self.send_OK_Message(HOST, PORT+2)
        self.monitor._parse_message_from_queue()

        self.assertTrue(slave2.is_master())
        self.assertTrue(slave2.is_alive())

        # slave1 is resurrected as slave
        self.send_OK_Message(HOST, PORT+1)
        self.monitor._parse_message_from_queue()

        self.assertTrue(slave1.is_slave())
        self.assertTrue(slave1.is_alive())

    def test_all_going_KO_slave(self):
        print "test_all_going_KO_slave"
        master = self.monitor.cluster.add_node(HOST, PORT, ROLE_MASTER, REDIS_STATUS_OK)
        slave1 = self.monitor.cluster.add_node(HOST, PORT+1, ROLE_SLAVE, REDIS_STATUS_OK)
        slave2 = self.monitor.cluster.add_node(HOST, PORT+2, ROLE_SLAVE, REDIS_STATUS_OK)
        
        # kill slave1
        self.send_KO_Message(HOST, PORT+1)
        self.monitor._parse_message_from_queue()

        self.assertTrue(slave1.is_slave())
        self.assertFalse(slave1.is_alive())
        
        # kill master
        self.send_KO_Message(HOST, PORT)
        self.monitor._parse_message_from_queue()

        self.assertTrue(master.is_slave())
        self.assertFalse(master.is_alive())
        
        # kill slave2
        self.send_KO_Message(HOST, PORT+2)
        self.monitor._parse_message_from_queue()

        self.assertTrue(slave2.is_master())
        self.assertFalse(slave2.is_alive())

        # slave1 is resurrected
        self.send_OK_Message(HOST, PORT+1)
        self.monitor._parse_message_from_queue()

        self.assertTrue(slave1.is_master())
        self.assertTrue(slave1.is_alive())

        # slave2 is resurrected
        self.send_OK_Message(HOST, PORT+2)
        self.monitor._parse_message_from_queue()

        self.assertTrue(slave2.is_alive())
        self.assertTrue(slave2.is_slave())

    def test_all_resurrecting(self):
        print "test_all_resurrecting"
        master = self.monitor.cluster.add_node(HOST, PORT, ROLE_MASTER, REDIS_STATUS_KO)
        slave1 = self.monitor.cluster.add_node(HOST, PORT+1, ROLE_SLAVE, REDIS_STATUS_KO)
        slave2 = self.monitor.cluster.add_node(HOST, PORT+2, ROLE_SLAVE, REDIS_STATUS_KO)
        # kill slave1
        self.send_KO_Message(HOST, PORT+1)
        self.send_KO_Message(HOST, PORT)
        self.send_KO_Message(HOST, PORT+2)

        self.monitor._parse_message_from_queue()
        self.assertTrue(slave1.is_slave())
        self.assertFalse(slave1.is_alive())
        
        self.monitor._parse_message_from_queue()
        self.assertTrue(master.is_master())
        self.assertFalse(master.is_alive())
        
        self.monitor._parse_message_from_queue()
        self.assertTrue(slave2.is_slave())
        self.assertFalse(slave2.is_alive())

    def send_OK_Message(self, redis_host, redis_port):
        self.send_message(redis_host, redis_port, REDIS_STATUS_OK)

    def send_KO_Message(self, redis_host, redis_port):
        self.send_message(redis_host, redis_port, REDIS_STATUS_KO)

    def send_message(self, redis_host, redis_port, redis_status):
        message = self.create_message(redis_host, redis_port, redis_status)
        self.monitor.queue.put(message)

    def create_message(self, redis_host, redis_port, redis_status):
        worker_name = '%s:%s' % (redis_host, redis_port)
        text = "%s,%s" % (worker_name, redis_status)
        return text

    def tearDown(self):
        print "### tearDown ###"
Exemplo n.º 10
0
class TestRedisMonitor(unittest.TestCase):
    def setUp(self):
        print "### setUp ###"
        self.monitor = RedisMonitor(
            zk=None,
            redis_hosts="localhost:6379,localhost:6389,localhost:6399",
            sleep_time=30,
            zk_path="/test/redis")
        self.monitor.zk = MockZooKeeper("zk")
        self.monitor.redis_class = MockRedis
        self.monitor.zk_properties = MockZooKeeper("zk_properties")
        self.monitor.queue = Queue.Queue()

    def test_all_ok(self):
        print "test_all_ok"
        master = self.monitor.cluster.add_node(HOST, PORT, ROLE_MASTER,
                                               REDIS_STATUS_OK)
        slave1 = self.monitor.cluster.add_node(HOST, PORT + 1, ROLE_SLAVE,
                                               REDIS_STATUS_OK)
        slave2 = self.monitor.cluster.add_node(HOST, PORT + 2, ROLE_SLAVE,
                                               REDIS_STATUS_OK)

        self.send_OK_Message(HOST, PORT)
        self.monitor._parse_message_from_queue()

        self.send_OK_Message(HOST, PORT + 1)
        self.monitor._parse_message_from_queue()

        self.send_OK_Message(HOST, PORT + 2)
        self.monitor._parse_message_from_queue()

        self.assertTrue(master.is_master())
        self.assertTrue(master.is_alive())

        self.assertTrue(slave1.is_slave())
        self.assertTrue(slave1.is_alive())

        self.assertTrue(slave2.is_slave())
        self.assertTrue(slave2.is_alive())

    def test_one_slave_KO(self):
        print "test_one_slave_KO"
        master = self.monitor.cluster.add_node(HOST, PORT, ROLE_MASTER,
                                               REDIS_STATUS_OK)
        slave1 = self.monitor.cluster.add_node(HOST, PORT + 1, ROLE_SLAVE,
                                               REDIS_STATUS_OK)
        slave2 = self.monitor.cluster.add_node(HOST, PORT + 2, ROLE_SLAVE,
                                               REDIS_STATUS_OK)

        self.send_OK_Message(HOST, PORT + 1)
        self.monitor._parse_message_from_queue()

        self.send_OK_Message(HOST, PORT)
        self.monitor._parse_message_from_queue()

        self.send_KO_Message(HOST, PORT + 2)
        self.monitor._parse_message_from_queue()

        self.assertTrue(master.is_master())
        self.assertTrue(master.is_alive())

        self.assertTrue(slave1.is_slave())
        self.assertTrue(slave1.is_alive())

        self.assertTrue(slave2.is_slave())
        self.assertFalse(slave2.is_alive())

    def test_master_KO(self):
        print "test_master_KO"
        master = self.monitor.cluster.add_node(HOST, PORT, ROLE_MASTER,
                                               REDIS_STATUS_OK)
        slave1 = self.monitor.cluster.add_node(HOST, PORT + 1, ROLE_SLAVE,
                                               REDIS_STATUS_OK)
        slave2 = self.monitor.cluster.add_node(HOST, PORT + 2, ROLE_SLAVE,
                                               REDIS_STATUS_OK)

        self.send_OK_Message(HOST, PORT + 1)
        self.monitor._parse_message_from_queue()

        self.send_KO_Message(HOST, PORT)
        self.monitor._parse_message_from_queue()

        self.send_OK_Message(HOST, PORT + 2)
        self.monitor._parse_message_from_queue()

        self.assertTrue(master.is_slave())
        self.assertFalse(master.is_alive())

        self.assertTrue(slave1.is_master())
        self.assertTrue(slave1.is_alive())

        self.assertTrue(slave2.is_slave())
        self.assertTrue(slave2.is_alive())

    def test_all_going_KO_master(self):
        print "test_all_going_KO_master"
        master = self.monitor.cluster.add_node(HOST, PORT, ROLE_MASTER,
                                               REDIS_STATUS_OK)
        slave1 = self.monitor.cluster.add_node(HOST, PORT + 1, ROLE_SLAVE,
                                               REDIS_STATUS_OK)
        slave2 = self.monitor.cluster.add_node(HOST, PORT + 2, ROLE_SLAVE,
                                               REDIS_STATUS_OK)

        # kill slave1
        self.send_KO_Message(HOST, PORT + 1)
        self.monitor._parse_message_from_queue()

        self.assertTrue(slave1.is_slave())
        self.assertFalse(slave1.is_alive())

        # kill master
        self.send_KO_Message(HOST, PORT)
        self.monitor._parse_message_from_queue()

        self.assertTrue(master.is_slave())
        self.assertFalse(master.is_alive())

        # kill slave2
        self.send_KO_Message(HOST, PORT + 2)
        self.monitor._parse_message_from_queue()

        self.assertTrue(slave2.is_master())
        self.assertFalse(slave2.is_alive())

        # slave2 is resurrected as master
        self.send_OK_Message(HOST, PORT + 2)
        self.monitor._parse_message_from_queue()

        self.assertTrue(slave2.is_master())
        self.assertTrue(slave2.is_alive())

        # slave1 is resurrected as slave
        self.send_OK_Message(HOST, PORT + 1)
        self.monitor._parse_message_from_queue()

        self.assertTrue(slave1.is_slave())
        self.assertTrue(slave1.is_alive())

    def test_all_going_KO_slave(self):
        print "test_all_going_KO_slave"
        master = self.monitor.cluster.add_node(HOST, PORT, ROLE_MASTER,
                                               REDIS_STATUS_OK)
        slave1 = self.monitor.cluster.add_node(HOST, PORT + 1, ROLE_SLAVE,
                                               REDIS_STATUS_OK)
        slave2 = self.monitor.cluster.add_node(HOST, PORT + 2, ROLE_SLAVE,
                                               REDIS_STATUS_OK)

        # kill slave1
        self.send_KO_Message(HOST, PORT + 1)
        self.monitor._parse_message_from_queue()

        self.assertTrue(slave1.is_slave())
        self.assertFalse(slave1.is_alive())

        # kill master
        self.send_KO_Message(HOST, PORT)
        self.monitor._parse_message_from_queue()

        self.assertTrue(master.is_slave())
        self.assertFalse(master.is_alive())

        # kill slave2
        self.send_KO_Message(HOST, PORT + 2)
        self.monitor._parse_message_from_queue()

        self.assertTrue(slave2.is_master())
        self.assertFalse(slave2.is_alive())

        # slave1 is resurrected
        self.send_OK_Message(HOST, PORT + 1)
        self.monitor._parse_message_from_queue()

        self.assertTrue(slave1.is_master())
        self.assertTrue(slave1.is_alive())

        # slave2 is resurrected
        self.send_OK_Message(HOST, PORT + 2)
        self.monitor._parse_message_from_queue()

        self.assertTrue(slave2.is_alive())
        self.assertTrue(slave2.is_slave())

    def test_all_resurrecting(self):
        print "test_all_resurrecting"
        master = self.monitor.cluster.add_node(HOST, PORT, ROLE_MASTER,
                                               REDIS_STATUS_KO)
        slave1 = self.monitor.cluster.add_node(HOST, PORT + 1, ROLE_SLAVE,
                                               REDIS_STATUS_KO)
        slave2 = self.monitor.cluster.add_node(HOST, PORT + 2, ROLE_SLAVE,
                                               REDIS_STATUS_KO)
        # kill slave1
        self.send_KO_Message(HOST, PORT + 1)
        self.send_KO_Message(HOST, PORT)
        self.send_KO_Message(HOST, PORT + 2)

        self.monitor._parse_message_from_queue()
        self.assertTrue(slave1.is_slave())
        self.assertFalse(slave1.is_alive())

        self.monitor._parse_message_from_queue()
        self.assertTrue(master.is_master())
        self.assertFalse(master.is_alive())

        self.monitor._parse_message_from_queue()
        self.assertTrue(slave2.is_slave())
        self.assertFalse(slave2.is_alive())

    def send_OK_Message(self, redis_host, redis_port):
        self.send_message(redis_host, redis_port, REDIS_STATUS_OK)

    def send_KO_Message(self, redis_host, redis_port):
        self.send_message(redis_host, redis_port, REDIS_STATUS_KO)

    def send_message(self, redis_host, redis_port, redis_status):
        message = self.create_message(redis_host, redis_port, redis_status)
        self.monitor.queue.put(message)

    def create_message(self, redis_host, redis_port, redis_status):
        worker_name = '%s:%s' % (redis_host, redis_port)
        text = "%s,%s" % (worker_name, redis_status)
        return text

    def tearDown(self):
        print "### tearDown ###"
Exemplo n.º 11
0
class TestRedisMonitor(TestCase):

    maxDiff = None
    queue_key = "link:istresearch.com:queue"

    def setUp(self):
        self.redis_monitor = RedisMonitor("localsettings.py")
        self.redis_monitor.settings = self.redis_monitor.wrapper.load(
            "localsettings.py")
        self.redis_monitor.logger = MagicMock()
        self.redis_monitor.settings['KAFKA_TOPIC_PREFIX'] = "demo_test"
        self.redis_monitor.settings['STATS_TOTAL'] = False
        self.redis_monitor.settings['STATS_PLUGINS'] = False
        self.redis_monitor.settings['PLUGINS'] = {
            'plugins.info_monitor.InfoMonitor': None,
            'plugins.stop_monitor.StopMonitor': None,
            'plugins.expire_monitor.ExpireMonitor': None,
            'tests.tests_online.CustomMonitor': 100,
        }
        self.redis_monitor.redis_conn = redis.Redis(
            host=self.redis_monitor.settings['REDIS_HOST'],
            port=self.redis_monitor.settings['REDIS_PORT'])

        self.redis_monitor._load_plugins()
        self.redis_monitor.stats_dict = {}

        self.kafka_conn = KafkaClient(
            self.redis_monitor.settings['KAFKA_HOSTS'])
        self.kafka_conn.ensure_topic_exists("demo_test.outbound_firehose")

        self.consumer = SimpleConsumer(self.kafka_conn, "demo-id",
                                       "demo_test.outbound_firehose")

    def test_process_item(self):
        # we only want to go to the end now, not after this test is ran
        self.consumer.seek(0, 2)

        # set the info flag
        key = "info-test:blah"
        value = "ABC123"
        self.redis_monitor.redis_conn.set(key, value)

        # process the request
        plugin = self.redis_monitor.plugins_dict.items()[0][1]
        self.redis_monitor._process_plugin(plugin)

        # ensure the key is gone
        self.assertEquals(self.redis_monitor.redis_conn.get(key), None)

    def test_sent_to_kafka(self):
        success = {u'info-test': "ABC123", u"appid": u"someapp"}

        # ensure it was sent out to kafka
        message_count = 0
        for message in self.consumer.get_messages():
            if message is None:
                break
            else:
                the_dict = json.loads(message.message.value)
                self.assertEquals(success, the_dict)
                message_count += 1

        self.assertEquals(message_count, 1)
Exemplo n.º 12
0
class TestRedisMonitor(TestCase):

    def setUp(self):
        self.redis_monitor = RedisMonitor("settings.py", True)
        self.redis_monitor.settings = self.redis_monitor.wrapper.load("settings.py")
        self.redis_monitor.logger = MagicMock()

    def test_load_plugins(self):
        # test loading default plugins
        assert_keys = [100, 200, 300, 400, 500]
        self.redis_monitor._load_plugins()
        self.assertEqual(list(self.redis_monitor.plugins_dict.keys()), assert_keys)

        # test removing a plugin from settings
        assert_keys = [100, 300, 400, 500]
        self.redis_monitor.settings['PLUGINS'] \
            ['plugins.stop_monitor.StopMonitor'] = None
        self.redis_monitor._load_plugins()
        self.assertEqual(list(self.redis_monitor.plugins_dict.keys()), assert_keys)
        self.redis_monitor.settings['PLUGINS'] \
            ['plugins.stop_monitor.StopMonitor'] = 200

        # fail if the class is not found
        self.redis_monitor.settings['PLUGINS'] \
            ['plugins.crazy_class.CrazyHandler'] = 400,

        self.assertRaises(ImportError, self.redis_monitor._load_plugins)
        del self.redis_monitor.settings['PLUGINS'] \
            ['plugins.crazy_class.CrazyHandler']
        self.redis_monitor.settings['PLUGINS'] = {}

    def test_active_plugins(self):
        # test that exceptions are caught within each plugin
        # assuming now all plugins are loaded
        self.redis_monitor._load_plugins()
        self.redis_monitor.stats_dict = {}

        # BaseExceptions are never raised normally
        list(self.redis_monitor.plugins_dict.items())[0][1]['instance'].handle = MagicMock(side_effect=BaseException("info"))
        list(self.redis_monitor.plugins_dict.items())[1][1]['instance'].handle = MagicMock(side_effect=BaseException("stop"))
        list(self.redis_monitor.plugins_dict.items())[2][1]['instance'].handle = MagicMock(side_effect=BaseException("expire"))
        self.redis_monitor.redis_conn = MagicMock()
        self.redis_monitor.redis_conn.scan_iter = MagicMock()
        self.redis_monitor._create_lock_object = MagicMock()

        # lets just assume the regex worked
        self.redis_monitor.redis_conn.scan_iter.return_value = ['somekey1']

        # info
        try:
            plugin = list(self.redis_monitor.plugins_dict.items())[0][1]
            self.redis_monitor._process_plugin(plugin)
            self.fail("Info not called")
        except BaseException as e:
            self.assertEquals("info", str(e))

        # action
        try:
            plugin = list(self.redis_monitor.plugins_dict.items())[1][1]
            self.redis_monitor._process_plugin(plugin)
            self.fail("Stop not called")
        except BaseException as e:
            self.assertEquals("stop", str(e))

        # expire
        try:
            plugin = list(self.redis_monitor.plugins_dict.items())[2][1]
            self.redis_monitor._process_plugin(plugin)
            self.fail("Expire not called")
        except BaseException as e:
            self.assertEquals("expire", str(e))

        # test that an exception within a handle method is caught
        self.redis_monitor._process_failures = MagicMock()
        self.redis_monitor._increment_fail_stat = MagicMock()
        try:
            list(self.redis_monitor.plugins_dict.items())[0][1]['instance'].handle = MagicMock(side_effect=Exception("normal"))
            plugin = list(self.redis_monitor.plugins_dict.items())[0][1]
            self.redis_monitor._process_plugin(plugin)

        except Exception as e:
            self.fail("Normal Exception not handled")

        self.assertTrue(self.redis_monitor._increment_fail_stat.called)
        self.assertTrue(self.redis_monitor._process_failures.called)

    def test_locking_actions(self):
        # tests for _process_plugins with locking
        self.redis_monitor._load_plugins()
        self.redis_monitor.redis_conn = MagicMock()
        self.redis_monitor.redis_conn = MagicMock()
        self.redis_monitor.redis_conn.scan_iter = MagicMock()
        self.redis_monitor.redis_conn.get = MagicMock(return_value=5)

        lock = MagicMock()
        lock.acquire
        lock.acquire = MagicMock(return_value=False)
        lock.release = MagicMock()
        lock._held = False
        self.redis_monitor._create_lock_object = MagicMock(return_value=lock)

        self.redis_monitor._increment_fail_stat = MagicMock()
        self.redis_monitor._process_failures = MagicMock()
        self.redis_monitor._process_key_val = MagicMock()
        # lets just assume the regex worked
        self.redis_monitor.redis_conn.scan_iter.return_value = ['somekey1']

        plugin = {'instance': 'test',
                  'regex': 'abc123'}

        # test didnt acquire lock
        self.redis_monitor._process_key_val.assert_not_called()
        self.redis_monitor._process_plugin(plugin)
        self.redis_monitor._process_key_val.assert_not_called()
        lock.release.assert_not_called()

        # test got lock
        lock.acquire = MagicMock(return_value=True)
        lock._held = True
        self.redis_monitor._process_plugin(plugin)
        self.redis_monitor._process_key_val.assert_called_once_with('test', 'somekey1', 5)
        # test lock released
        lock.release.assert_called_once_with()

        # test lock not held not released
        lock._held = False
        lock.release.reset_mock()
        self.redis_monitor._process_plugin(plugin)
        lock.release.assert_not_called()


    def test_load_stats_plugins(self):
        # lets assume we are loading the default plugins
        self.redis_monitor._load_plugins()
        self.redis_monitor.redis_conn = MagicMock()

        # test no rolling stats
        self.redis_monitor.stats_dict = {}
        self.redis_monitor.settings['STATS_TIMES'] = []
        self.redis_monitor._setup_stats_plugins()
        defaults = [
            'ExpireMonitor',
            'StopMonitor',
            'InfoMonitor',
            'StatsMonitor',
            'ZookeeperMonitor'
        ]

        self.assertEquals(
            sorted(self.redis_monitor.stats_dict['plugins'].keys()),
            sorted(defaults))

        for key in self.redis_monitor.plugins_dict:
            plugin_name = self.redis_monitor.plugins_dict[key]['instance'].__class__.__name__
            self.assertEquals(
                list(self.redis_monitor.stats_dict['plugins'][plugin_name].keys()),
                ['lifetime'])

        # test good/bad rolling stats
        self.redis_monitor.stats_dict = {}
        self.redis_monitor.settings['STATS_TIMES'] = [
            'SECONDS_15_MINUTE',
            'SECONDS_1_HOUR',
            'SECONDS_DUMB',
        ]
        good = [
            'lifetime', # for totals, not DUMB
            '900',
            '3600',
        ]

        self.redis_monitor._setup_stats_plugins()

        self.assertEquals(
            sorted(self.redis_monitor.stats_dict['plugins'].keys()),
            sorted(defaults))

        for key in self.redis_monitor.plugins_dict:
            plugin_name = self.redis_monitor.plugins_dict[key]['instance'].__class__.__name__
            self.assertEquals(
                sorted([str(x) for x in self.redis_monitor.stats_dict['plugins'][plugin_name].keys()]),
                sorted(good))

        for plugin_key in self.redis_monitor.stats_dict['plugins']:
            k1 = 'stats:redis-monitor:{p}'.format(p=plugin_key)
            for time_key in self.redis_monitor.stats_dict['plugins'][plugin_key]:
                if time_key == 0:
                    self.assertEquals(
                        self.redis_monitor.stats_dict['plugins'][plugin_key][0].key,
                        '{k}:lifetime'.format(k=k1)
                        )
                else:
                    self.assertEquals(
                        self.redis_monitor.stats_dict['plugins'][plugin_key][time_key].key,
                        '{k}:{t}'.format(k=k1, t=time_key)
                        )

    def test_main_loop(self):
        self.redis_monitor._load_plugins()
        self.redis_monitor._process_plugin = MagicMock(side_effect=Exception(
                                                       "normal"))

        try:
            self.redis_monitor._main_loop()
            self.fail("_process_plugin not called")
        except BaseException as e:
            self.assertEquals("normal", str(e))

    def test_precondition(self):
        self.redis_monitor.stats_dict = {}
        instance = MagicMock()
        instance.check_precondition = MagicMock(return_value=False)
        instance.handle = MagicMock(side_effect=Exception("handler"))
        key = 'stuff'
        value = 'blah'

        # this should not raise an exception
        self.redis_monitor._process_key_val(instance, key, value)

        # this should
        instance.check_precondition = MagicMock(return_value=True)
        try:
            self.redis_monitor._process_key_val(instance, key, value)
            self.fail('handler not called')
        except BaseException as e:
            self.assertEquals('handler', str(e))

    def test_get_fail_key(self):
        key = 'test'
        result = 'lock:test:failures'
        self.assertEquals(self.redis_monitor._get_fail_key(key), result)

    def test_process_failures(self):
        self.redis_monitor.settings = {'RETRY_FAILURES':True,
                                       'RETRY_FAILURES_MAX': 3}
        self.redis_monitor._get_fail_key = MagicMock(return_value='the_key')
        self.redis_monitor.redis_conn = MagicMock()
        self.redis_monitor.redis_conn.set = MagicMock()

        # test not set
        self.redis_monitor.redis_conn.get = MagicMock(return_value=None)
        self.redis_monitor._process_failures('key1')
        self.redis_monitor.redis_conn.set.assert_called_once_with('the_key', 1)

        # test set
        self.redis_monitor.redis_conn.set.reset_mock()
        self.redis_monitor.redis_conn.get = MagicMock(return_value=2)
        self.redis_monitor._process_failures('key1')
        self.redis_monitor.redis_conn.set.assert_called_once_with('the_key', 3)

        # test exceeded
        self.redis_monitor.redis_conn.delete = MagicMock()
        self.redis_monitor.redis_conn.set.reset_mock()
        self.redis_monitor.redis_conn.get = MagicMock(return_value=3)
        self.redis_monitor._process_failures('key1')
        calls = [call('the_key'), call('key1')]
        self.redis_monitor.redis_conn.delete.assert_has_calls(calls)

    @mock.patch('socket.gethostname', return_value='host')
    @mock.patch('time.time', return_value=5)
    def test_report_self(self, h, t):
        self.redis_monitor.my_uuid = '1234'
        self.redis_monitor.redis_conn = MagicMock()
        self.redis_monitor.redis_conn.set = MagicMock()
        self.redis_monitor.redis_conn.expire = MagicMock()

        self.redis_monitor._report_self()
        self.redis_monitor.redis_conn.set.assert_called_once_with('stats:redis-monitor:self:host:1234', 5)
        self.redis_monitor.redis_conn.expire.assert_called_once_with('stats:redis-monitor:self:host:1234', 120)
Exemplo n.º 13
0
class TestRedisMonitor(TestCase):

    maxDiff = None
    queue_key = "link:istresearch.com:queue"
    consumer = None

    def setUp(self):
        self.redis_monitor = RedisMonitor("localsettings.py")
        self.redis_monitor.settings = self.redis_monitor.wrapper.load(
            "localsettings.py")
        self.redis_monitor.logger = MagicMock()
        self.redis_monitor.settings['KAFKA_TOPIC_PREFIX'] = "demo_test"
        self.redis_monitor.settings['STATS_TOTAL'] = False
        self.redis_monitor.settings['STATS_PLUGINS'] = False
        self.redis_monitor.settings['PLUGINS'] = {
            'plugins.info_monitor.InfoMonitor': None,
            'plugins.stop_monitor.StopMonitor': None,
            'plugins.expire_monitor.ExpireMonitor': None,
            'tests.online.CustomMonitor': 100,
        }
        self.redis_monitor.redis_conn = redis.Redis(
            host=self.redis_monitor.settings['REDIS_HOST'],
            port=self.redis_monitor.settings['REDIS_PORT'],
            db=self.redis_monitor.settings['REDIS_DB'])

        self.redis_monitor._load_plugins()
        self.redis_monitor.stats_dict = {}

        self.consumer = KafkaConsumer(
            "demo_test.outbound_firehose",
            bootstrap_servers=self.redis_monitor.settings['KAFKA_HOSTS'],
            group_id="demo-id",
            auto_commit_interval_ms=10,
            consumer_timeout_ms=5000,
            auto_offset_reset='earliest')
        sleep(1)

    def test_process_item(self):
        # set the info flag
        key = "info-test:blah"
        value = "ABC1234"
        self.redis_monitor.redis_conn.set(key, value)

        # process the request
        plugin = list(self.redis_monitor.plugins_dict.items())[0][1]
        self.redis_monitor._process_plugin(plugin)

        # ensure the key is gone
        self.assertEquals(self.redis_monitor.redis_conn.get(key), None)
        self.redis_monitor.close()
        sleep(10)
        # now test the message was sent to kafka
        success = {u'info-test': "ABC1234", u"appid": u"someapp"}

        message_count = 0
        m = next(self.consumer)

        if m is None:
            pass
        else:
            the_dict = json.loads(m.value)
            self.assertEquals(success, the_dict)
            message_count += 1

        self.assertEquals(message_count, 1)

    def tearDown(self):
        # if for some reason the tests fail, we end up falling behind on
        # the consumer
        for m in self.consumer:
            pass
        self.consumer.close()
Exemplo n.º 14
0
class TestRedisMonitor(TestCase):

    def setUp(self):
        self.redis_monitor = RedisMonitor("settings.py", True)
        self.redis_monitor.settings = self.redis_monitor.wrapper.load("settings.py")
        self.redis_monitor.logger = MagicMock()

    def test_load_plugins(self):
        # test loading default plugins
        assert_keys = [100, 200, 300, 400]
        self.redis_monitor._load_plugins()
        self.assertEqual(self.redis_monitor.plugins_dict.keys(), assert_keys)

        # test removing a plugin from settings
        assert_keys = [100, 300, 400]
        self.redis_monitor.settings['PLUGINS'] \
            ['plugins.stop_monitor.StopMonitor'] = None
        self.redis_monitor._load_plugins()
        self.assertEqual(self.redis_monitor.plugins_dict.keys(), assert_keys)
        self.redis_monitor.settings['PLUGINS'] \
            ['plugins.stop_monitor.StopMonitor'] = 200

        # fail if the class is not found
        self.redis_monitor.settings['PLUGINS'] \
            ['plugins.crazy_class.CrazyHandler'] = 400,

        self.assertRaises(ImportError, self.redis_monitor._load_plugins)
        del self.redis_monitor.settings['PLUGINS'] \
            ['plugins.crazy_class.CrazyHandler']
        self.redis_monitor.settings['PLUGINS'] = {}

    def test_active_plugins(self):
        # test that exceptions are caught within each plugin
        # assuming now all plugins are loaded
        self.redis_monitor._load_plugins()
        self.redis_monitor.stats_dict = {}

        # BaseExceptions are never raised normally
        self.redis_monitor.plugins_dict.items()[0][1]['instance'].handle = MagicMock(side_effect=BaseException("info"))
        self.redis_monitor.plugins_dict.items()[1][1]['instance'].handle = MagicMock(side_effect=BaseException("stop"))
        self.redis_monitor.plugins_dict.items()[2][1]['instance'].handle = MagicMock(side_effect=BaseException("expire"))
        self.redis_monitor.redis_conn = MagicMock()
        self.redis_monitor.redis_conn.scan_iter = MagicMock()
        # lets just assume the regex worked
        self.redis_monitor.redis_conn.scan_iter.return_value = ['somekey1']

        # info
        try:
            plugin = self.redis_monitor.plugins_dict.items()[0][1]
            self.redis_monitor._process_plugin(plugin)
            self.fail("Info not called")
        except BaseException as e:
            self.assertEquals("info", e.message)

        # action
        try:
            plugin = self.redis_monitor.plugins_dict.items()[1][1]
            self.redis_monitor._process_plugin(plugin)
            self.fail("Stop not called")
        except BaseException as e:
            self.assertEquals("stop", e.message)

        # expire
        try:
            plugin = self.redis_monitor.plugins_dict.items()[2][1]
            self.redis_monitor._process_plugin(plugin)
            self.fail("Expire not called")
        except BaseException as e:
            self.assertEquals("expire", e.message)

        # test that an exception within a handle method is caught
        try:
            self.redis_monitor.plugins_dict.items()[0][1]['instance'].handle = MagicMock(side_effect=Exception("normal"))
            plugin = self.redis_monitor.plugins_dict.items()[0][1]
            self.redis_monitor._process_plugin(plugin)
        except Exception as e:
            self.fail("Normal Exception not handled")

    def test_load_stats_plugins(self):
        # lets assume we are loading the default plugins
        self.redis_monitor._load_plugins()
        self.redis_monitor.redis_conn = MagicMock()

        # test no rolling stats
        self.redis_monitor.stats_dict = {}
        self.redis_monitor.settings['STATS_TIMES'] = []
        self.redis_monitor._setup_stats_plugins()
        defaults = [
            'ExpireMonitor',
            'StopMonitor',
            'InfoMonitor',
            'StatsMonitor'
        ]

        self.assertEquals(
            sorted(self.redis_monitor.stats_dict['plugins'].keys()),
            sorted(defaults))

        for key in self.redis_monitor.plugins_dict:
            plugin_name = self.redis_monitor.plugins_dict[key]['instance'].__class__.__name__
            self.assertEquals(
                self.redis_monitor.stats_dict['plugins'][plugin_name].keys(),
                ['lifetime'])

        # test good/bad rolling stats
        self.redis_monitor.stats_dict = {}
        self.redis_monitor.settings['STATS_TIMES'] = [
            'SECONDS_15_MINUTE',
            'SECONDS_1_HOUR',
            'SECONDS_DUMB',
        ]
        good = [
            'lifetime', # for totals, not DUMB
            900,
            3600,
        ]

        self.redis_monitor._setup_stats_plugins()

        self.assertEquals(
            sorted(self.redis_monitor.stats_dict['plugins'].keys()),
            sorted(defaults))

        for key in self.redis_monitor.plugins_dict:
            plugin_name = self.redis_monitor.plugins_dict[key]['instance'].__class__.__name__
            self.assertEquals(
                sorted(self.redis_monitor.stats_dict['plugins'][plugin_name].keys()),
                sorted(good))

        for plugin_key in self.redis_monitor.stats_dict['plugins']:
            k1 = 'stats:redis-monitor:{p}'.format(p=plugin_key)
            for time_key in self.redis_monitor.stats_dict['plugins'][plugin_key]:
                if time_key == 0:
                    self.assertEquals(
                        self.redis_monitor.stats_dict['plugins'][plugin_key][0].key,
                        '{k}:lifetime'.format(k=k1)
                        )
                else:
                    self.assertEquals(
                        self.redis_monitor.stats_dict['plugins'][plugin_key][time_key].key,
                        '{k}:{t}'.format(k=k1, t=time_key)
                        )

    def test_main_loop(self):
        self.redis_monitor._load_plugins()
        self.redis_monitor._process_plugin = MagicMock(side_effect=Exception(
                                                       "normal"))

        try:
            self.redis_monitor._main_loop()
            self.fail("_process_plugin not called")
        except BaseException as e:
            self.assertEquals("normal", e.message)

    def test_precondition(self):
        self.redis_monitor.stats_dict = {}
        instance = MagicMock()
        instance.check_precondition = MagicMock(return_value=False)
        instance.handle = MagicMock(side_effect=Exception("handler"))
        key = 'stuff'
        value = 'blah'

        # this should not raise an exception
        self.redis_monitor._process_key_val(instance, key, value)

        # this should
        instance.check_precondition = MagicMock(return_value=True)
        try:
            self.redis_monitor._process_key_val(instance, key, value)
            self.fail('handler not called')
        except BaseException as e:
            self.assertEquals('handler', e.message)
Exemplo n.º 15
0
class TestRedisMonitor(TestCase):

    maxDiff = None
    queue_key = "link:istresearch.com:queue"
    consumer = None

    def setUp(self):
        self.redis_monitor = RedisMonitor("localsettings.py")
        self.redis_monitor.settings = self.redis_monitor.wrapper.load("localsettings.py")
        self.redis_monitor.logger = MagicMock()
        self.redis_monitor.settings['KAFKA_TOPIC_PREFIX'] = "demo_test"
        self.redis_monitor.settings['STATS_TOTAL'] = False
        self.redis_monitor.settings['STATS_PLUGINS'] = False
        self.redis_monitor.settings['PLUGINS'] = {
            'plugins.info_monitor.InfoMonitor': None,
            'plugins.stop_monitor.StopMonitor': None,
            'plugins.expire_monitor.ExpireMonitor': None,
            'tests.online.CustomMonitor': 100,
        }
        self.redis_monitor.redis_conn = redis.Redis(
            host=self.redis_monitor.settings['REDIS_HOST'],
            port=self.redis_monitor.settings['REDIS_PORT'],
            db=self.redis_monitor.settings['REDIS_DB'])

        self.redis_monitor._load_plugins()
        self.redis_monitor.stats_dict = {}

        self.consumer = KafkaConsumer(
            "demo_test.outbound_firehose",
            bootstrap_servers=self.redis_monitor.settings['KAFKA_HOSTS'],
            group_id="demo-id",
            auto_commit_interval_ms=10,
            consumer_timeout_ms=5000,
            auto_offset_reset='earliest'
        )
        sleep(1)

    def test_process_item(self):
        # set the info flag
        key = "info-test:blah"
        value = "ABC1234"
        self.redis_monitor.redis_conn.set(key, value)

        # process the request
        plugin = list(self.redis_monitor.plugins_dict.items())[0][1]
        self.redis_monitor._process_plugin(plugin)

        # ensure the key is gone
        self.assertEquals(self.redis_monitor.redis_conn.get(key), None)
        self.redis_monitor.close()
        sleep(10)
        # now test the message was sent to kafka
        success = {
            u'info-test': "ABC1234",
            u"appid": u"someapp"
        }

        message_count = 0
        m = next(self.consumer)

        if m is None:
            pass
        else:
            the_dict = json.loads(m.value)
            self.assertEquals(success, the_dict)
            message_count += 1

        self.assertEquals(message_count, 1)

    def tearDown(self):
        # if for some reason the tests fail, we end up falling behind on
        # the consumer
        for m in self.consumer:
            pass
        self.consumer.close()
Exemplo n.º 16
0
class TestRedisMonitor(TestCase):
    def setUp(self):
        self.redis_monitor = RedisMonitor("settings.py", True)
        self.redis_monitor.settings = self.redis_monitor.wrapper.load("settings.py")
        self.redis_monitor.logger = MagicMock()

    def test_load_plugins(self):
        # test loading default plugins
        assert_keys = [100, 200, 300, 400]
        self.redis_monitor._load_plugins()
        self.assertEqual(self.redis_monitor.plugins_dict.keys(), assert_keys)

        # test removing a plugin from settings
        assert_keys = [100, 300, 400]
        self.redis_monitor.settings["PLUGINS"]["plugins.stop_monitor.StopMonitor"] = None
        self.redis_monitor._load_plugins()
        self.assertEqual(self.redis_monitor.plugins_dict.keys(), assert_keys)
        self.redis_monitor.settings["PLUGINS"]["plugins.stop_monitor.StopMonitor"] = 200

        # fail if the class is not found
        self.redis_monitor.settings["PLUGINS"]["plugins.crazy_class.CrazyHandler"] = (400,)

        self.assertRaises(ImportError, self.redis_monitor._load_plugins)
        del self.redis_monitor.settings["PLUGINS"]["plugins.crazy_class.CrazyHandler"]
        self.redis_monitor.settings["PLUGINS"] = {}

    def test_active_plugins(self):
        # test that exceptions are caught within each plugin
        # assuming now all plugins are loaded
        self.redis_monitor._load_plugins()
        self.redis_monitor.stats_dict = {}

        # BaseExceptions are never raised normally
        self.redis_monitor.plugins_dict.items()[0][1]["instance"].handle = MagicMock(side_effect=BaseException("info"))
        self.redis_monitor.plugins_dict.items()[1][1]["instance"].handle = MagicMock(side_effect=BaseException("stop"))
        self.redis_monitor.plugins_dict.items()[2][1]["instance"].handle = MagicMock(
            side_effect=BaseException("expire")
        )
        self.redis_monitor.redis_conn = MagicMock()
        self.redis_monitor.redis_conn.scan_iter = MagicMock()
        # lets just assume the regex worked
        self.redis_monitor.redis_conn.scan_iter.return_value = ["somekey1"]

        # info
        try:
            plugin = self.redis_monitor.plugins_dict.items()[0][1]
            self.redis_monitor._process_plugin(plugin)
            self.fail("Info not called")
        except BaseException as e:
            self.assertEquals("info", e.message)

        # action
        try:
            plugin = self.redis_monitor.plugins_dict.items()[1][1]
            self.redis_monitor._process_plugin(plugin)
            self.fail("Stop not called")
        except BaseException as e:
            self.assertEquals("stop", e.message)

        # expire
        try:
            plugin = self.redis_monitor.plugins_dict.items()[2][1]
            self.redis_monitor._process_plugin(plugin)
            self.fail("Expire not called")
        except BaseException as e:
            self.assertEquals("expire", e.message)

        # test that an exception within a handle method is caught
        try:
            self.redis_monitor.plugins_dict.items()[0][1]["instance"].handle = MagicMock(
                side_effect=Exception("normal")
            )
            plugin = self.redis_monitor.plugins_dict.items()[0][1]
            self.redis_monitor._process_plugin(plugin)
        except Exception as e:
            self.fail("Normal Exception not handled")

    def test_load_stats_plugins(self):
        # lets assume we are loading the default plugins
        self.redis_monitor._load_plugins()
        self.redis_monitor.redis_conn = MagicMock()

        # test no rolling stats
        self.redis_monitor.stats_dict = {}
        self.redis_monitor.settings["STATS_TIMES"] = []
        self.redis_monitor._setup_stats_plugins()
        defaults = ["ExpireMonitor", "StopMonitor", "InfoMonitor", "StatsMonitor"]

        self.assertEquals(sorted(self.redis_monitor.stats_dict["plugins"].keys()), sorted(defaults))

        for key in self.redis_monitor.plugins_dict:
            plugin_name = self.redis_monitor.plugins_dict[key]["instance"].__class__.__name__
            self.assertEquals(self.redis_monitor.stats_dict["plugins"][plugin_name].keys(), ["lifetime"])

        # test good/bad rolling stats
        self.redis_monitor.stats_dict = {}
        self.redis_monitor.settings["STATS_TIMES"] = ["SECONDS_15_MINUTE", "SECONDS_1_HOUR", "SECONDS_DUMB"]
        good = ["lifetime", 900, 3600]  # for totals, not DUMB

        self.redis_monitor._setup_stats_plugins()

        self.assertEquals(sorted(self.redis_monitor.stats_dict["plugins"].keys()), sorted(defaults))

        for key in self.redis_monitor.plugins_dict:
            plugin_name = self.redis_monitor.plugins_dict[key]["instance"].__class__.__name__
            self.assertEquals(sorted(self.redis_monitor.stats_dict["plugins"][plugin_name].keys()), sorted(good))

        for plugin_key in self.redis_monitor.stats_dict["plugins"]:
            k1 = "stats:redis-monitor:{p}".format(p=plugin_key)
            for time_key in self.redis_monitor.stats_dict["plugins"][plugin_key]:
                if time_key == 0:
                    self.assertEquals(
                        self.redis_monitor.stats_dict["plugins"][plugin_key][0].key, "{k}:lifetime".format(k=k1)
                    )
                else:
                    self.assertEquals(
                        self.redis_monitor.stats_dict["plugins"][plugin_key][time_key].key,
                        "{k}:{t}".format(k=k1, t=time_key),
                    )

    def test_main_loop(self):
        self.redis_monitor._load_plugins()
        self.redis_monitor._process_plugin = MagicMock(side_effect=Exception("normal"))

        try:
            self.redis_monitor._main_loop()
            self.fail("_process_plugin not called")
        except BaseException as e:
            self.assertEquals("normal", e.message)

    def test_precondition(self):
        self.redis_monitor.stats_dict = {}
        instance = MagicMock()
        instance.check_precondition = MagicMock(return_value=False)
        instance.handle = MagicMock(side_effect=Exception("handler"))
        key = "stuff"
        value = "blah"

        # this should not raise an exception
        self.redis_monitor._process_key_val(instance, key, value)

        # this should
        instance.check_precondition = MagicMock(return_value=True)
        try:
            self.redis_monitor._process_key_val(instance, key, value)
            self.fail("handler not called")
        except BaseException as e:
            self.assertEquals("handler", e.message)
Exemplo n.º 17
0
 def setUp(self):
     self.redis_monitor = RedisMonitor("settings.py", True)
     self.redis_monitor.settings = self.redis_monitor.wrapper.load("settings.py")
     self.redis_monitor.logger = MagicMock()
Exemplo n.º 18
0
class TestRedisMonitor(TestCase):

    maxDiff = None
    queue_key = "link:istresearch.com:queue"

    def setUp(self):
        self.redis_monitor = RedisMonitor("localsettings.py")
        self.redis_monitor.settings = self.redis_monitor.wrapper.load("localsettings.py")
        self.redis_monitor.logger = MagicMock()
        self.redis_monitor.settings['KAFKA_TOPIC_PREFIX'] = "demo_test"
        self.redis_monitor.settings['STATS_TOTAL'] = False
        self.redis_monitor.settings['STATS_PLUGINS'] = False
        self.redis_monitor.settings['PLUGINS'] = {
            'plugins.info_monitor.InfoMonitor': None,
            'plugins.stop_monitor.StopMonitor': None,
            'plugins.expire_monitor.ExpireMonitor': None,
            'tests.tests_online.CustomMonitor': 100,
        }
        self.redis_monitor.redis_conn = redis.Redis(
            host=self.redis_monitor.settings['REDIS_HOST'],
            port=self.redis_monitor.settings['REDIS_PORT'])

        self.redis_monitor._load_plugins()
        self.redis_monitor.stats_dict = {}

        self.kafka_conn = KafkaClient(self.redis_monitor.settings[
                                      'KAFKA_HOSTS'])
        self.kafka_conn.ensure_topic_exists("demo_test.outbound_firehose")

        self.consumer = SimpleConsumer(
            self.kafka_conn,
            "demo-id",
            "demo_test.outbound_firehose"
        )

    def test_process_item(self):
        # we only want to go to the end now, not after this test is ran
        self.consumer.seek(0, 2)

        # set the info flag
        key = "info-test:blah"
        value = "ABC123"
        self.redis_monitor.redis_conn.set(key, value)

        # process the request
        plugin = self.redis_monitor.plugins_dict.items()[0][1]
        self.redis_monitor._process_plugin(plugin)

        # ensure the key is gone
        self.assertEquals(self.redis_monitor.redis_conn.get(key), None)

    def test_sent_to_kafka(self):
        success = {
            u'info-test': "ABC123",
            u"appid": u"someapp"
        }

        # ensure it was sent out to kafka
        message_count = 0
        for message in self.consumer.get_messages():
            if message is None:
                break
            else:
                the_dict = json.loads(message.message.value)
                self.assertEquals(success, the_dict)
                message_count += 1

        self.assertEquals(message_count, 1)