def disconnect_from_rabbit(rabbit: RabbitMQApi, attempts: int = 3) -> None: tries = 0 while tries < attempts: try: rabbit.disconnect() return except Exception as e: tries += 1 print("Could not disconnect to rabbit. Attempts so " "far: {}".format(tries)) print(e) if tries >= attempts: raise e
class TestDataTransformersManager(unittest.TestCase): def setUp(self) -> None: self.dummy_logger = logging.getLogger('Dummy') self.dummy_logger.disabled = True self.connection_check_time_interval = timedelta(seconds=0) self.rabbit_ip = env.RABBIT_IP self.rabbitmq = RabbitMQApi( self.dummy_logger, self.rabbit_ip, connection_check_time_interval=self.connection_check_time_interval) self.manager_name = 'test_data_transformers_manager' self.test_queue_name = 'Test Queue' self.test_data_str = 'test data' self.test_timestamp = datetime(2012, 1, 1).timestamp() self.test_heartbeat = { 'component_name': 'Test Component', 'is_alive': True, 'timestamp': self.test_timestamp, } self.dummy_process1 = Process(target=infinite_fn, args=()) self.dummy_process1.daemon = True self.dummy_process2 = Process(target=infinite_fn, args=()) self.dummy_process2.daemon = True self.dummy_process3 = Process(target=infinite_fn, args=()) self.dummy_process3.daemon = True self.transformer_process_dict_example = { SYSTEM_DATA_TRANSFORMER_NAME: self.dummy_process1, GITHUB_DATA_TRANSFORMER_NAME: self.dummy_process2, } self.test_manager = DataTransformersManager(self.dummy_logger, self.manager_name, self.rabbitmq) self.test_exception = PANICException('test_exception', 1) def tearDown(self) -> None: # Delete any queues and exchanges which are common across many tests try: self.test_manager.rabbitmq.connect() # Declare them before just in case there are tests which do not # use these queues and exchanges self.test_manager.rabbitmq.queue_declare( queue=self.test_queue_name, durable=True, exclusive=False, auto_delete=False, passive=False) self.test_manager.rabbitmq.queue_declare(DT_MAN_INPUT_QUEUE, False, True, False, False) self.test_manager.rabbitmq.exchange_declare( HEALTH_CHECK_EXCHANGE, 'topic', False, True, False, False) self.test_manager.rabbitmq.queue_purge(self.test_queue_name) self.test_manager.rabbitmq.queue_purge(DT_MAN_INPUT_QUEUE) self.test_manager.rabbitmq.queue_delete(self.test_queue_name) self.test_manager.rabbitmq.queue_delete(DT_MAN_INPUT_QUEUE) self.test_manager.rabbitmq.exchange_delete(HEALTH_CHECK_EXCHANGE) self.test_manager.rabbitmq.disconnect() except Exception as e: print("Deletion of queues and exchanges failed: {}".format(e)) self.dummy_logger = None self.rabbitmq = None self.dummy_process1 = None self.dummy_process2 = None self.dummy_process3 = None self.test_manager = None self.test_exception = None self.transformer_process_dict_example = None def test_str_returns_name(self) -> None: self.assertEqual(self.manager_name, str(self.test_manager)) def test_name_returns_name(self) -> None: self.assertEqual(self.manager_name, self.test_manager.name) def test_transformer_process_dict_returns_transformer_process_dict( self) -> None: self.test_manager._transformer_process_dict = \ self.transformer_process_dict_example self.assertEqual(self.transformer_process_dict_example, self.test_manager.transformer_process_dict) def test_initialise_rabbitmq_initializes_everything_as_expected( self) -> None: try: # To make sure that there is no connection/channel already # established self.assertIsNone(self.rabbitmq.connection) self.assertIsNone(self.rabbitmq.channel) # To make sure that the exchange and queue have not already been # declared self.rabbitmq.connect() self.test_manager.rabbitmq.queue_delete(DT_MAN_INPUT_QUEUE) self.test_manager.rabbitmq.exchange_delete(HEALTH_CHECK_EXCHANGE) self.rabbitmq.disconnect() self.test_manager._initialise_rabbitmq() # Perform checks that the connection has been opened, marked as open # and that the delivery confirmation variable is set. self.assertTrue(self.test_manager.rabbitmq.is_connected) self.assertTrue(self.test_manager.rabbitmq.connection.is_open) self.assertTrue( self.test_manager.rabbitmq.channel._delivery_confirmation) # Check whether the exchange and queue have been creating by # sending messages with the same routing keys as for the queue. We # will also check if the size of the queue is 0 to confirm that # basic_consume was called (it will store the msg in the component # memory immediately). If one of the exchange or queue is not # created, then either an exception will be thrown or the queue size # would be 1. Note when deleting the exchange in the beginning we # also released every binding, hence there is no other queue binded # with the same routing key to any exchange at this point. self.test_manager.rabbitmq.basic_publish_confirm( exchange=HEALTH_CHECK_EXCHANGE, routing_key=DT_MAN_INPUT_ROUTING_KEY, body=self.test_data_str, is_body_dict=False, properties=pika.BasicProperties(delivery_mode=2), mandatory=True) # Re-declare queue to get the number of messages res = self.test_manager.rabbitmq.queue_declare( DT_MAN_INPUT_QUEUE, False, True, False, False) self.assertEqual(0, res.method.message_count) except Exception as e: self.fail("Test failed: {}".format(e)) @mock.patch.object(RabbitMQApi, "start_consuming") def test_listen_for_data_calls_start_consuming( self, mock_start_consuming) -> None: mock_start_consuming.return_value = None self.test_manager._listen_for_data() mock_start_consuming.assert_called_once() def test_send_heartbeat_sends_a_heartbeat_correctly(self) -> None: # This test creates a queue which receives messages with the same # routing key as the ones sent by send_heartbeat, and checks that the # heartbeat is received try: self.test_manager._initialise_rabbitmq() # Delete the queue before to avoid messages in the queue on error. self.test_manager.rabbitmq.queue_delete(self.test_queue_name) res = self.test_manager.rabbitmq.queue_declare( queue=self.test_queue_name, durable=True, exclusive=False, auto_delete=False, passive=False) self.assertEqual(0, res.method.message_count) self.test_manager.rabbitmq.queue_bind( queue=self.test_queue_name, exchange=HEALTH_CHECK_EXCHANGE, routing_key='heartbeat.manager') self.test_manager._send_heartbeat(self.test_heartbeat) # By re-declaring the queue again we can get the number of messages # in the queue. res = self.test_manager.rabbitmq.queue_declare( queue=self.test_queue_name, durable=True, exclusive=False, auto_delete=False, passive=True) self.assertEqual(1, res.method.message_count) # Check that the message received is actually the HB _, _, body = self.test_manager.rabbitmq.basic_get( self.test_queue_name) self.assertEqual(self.test_heartbeat, json.loads(body)) except Exception as e: self.fail("Test failed: {}".format(e)) @mock.patch.object(multiprocessing.Process, "start") def test_start_transformers_processes_creates_correct_sys_proc_first_time( self, mock_start) -> None: mock_start.return_value = None self.test_manager._start_transformers_processes() system_data_trans_proc = self.test_manager.transformer_process_dict[ SYSTEM_DATA_TRANSFORMER_NAME] self.assertTrue(system_data_trans_proc.daemon) self.assertEqual(0, len(system_data_trans_proc._args)) self.assertEqual(start_system_data_transformer, system_data_trans_proc._target) @mock.patch.object(multiprocessing.Process, "start") def test_start_transformers_processes_creates_correct_sys_proc_if_not_alive( self, mock_start) -> None: mock_start.return_value = None # Make the state non-empty to mock the case when creation is not being # done for the first time self.test_manager._transformer_process_dict = \ self.transformer_process_dict_example self.test_manager._start_transformers_processes() system_data_trans_proc = self.test_manager.transformer_process_dict[ SYSTEM_DATA_TRANSFORMER_NAME] self.assertTrue(system_data_trans_proc.daemon) self.assertEqual(0, len(system_data_trans_proc._args)) self.assertEqual(start_system_data_transformer, system_data_trans_proc._target) @mock.patch.object(multiprocessing.Process, "start") @mock.patch.object(multiprocessing, "Process") def test_start_transformers_processes_stores_sys_proc_correct_first_time( self, mock_init, mock_start) -> None: mock_start.return_value = None mock_init.return_value = self.dummy_process3 self.test_manager._start_transformers_processes() expected_trans_proc_dict_without_gh = { SYSTEM_DATA_TRANSFORMER_NAME: self.dummy_process3, } actual_trans_proc_dict_without_gh = \ self.test_manager._transformer_process_dict del actual_trans_proc_dict_without_gh[GITHUB_DATA_TRANSFORMER_NAME] self.assertEqual(expected_trans_proc_dict_without_gh, actual_trans_proc_dict_without_gh) @mock.patch.object(multiprocessing.Process, "start") @mock.patch.object(multiprocessing, "Process") def test_start_transformers_processes_stores_sys_proc_correct_if_not_alive( self, mock_init, mock_start) -> None: mock_start.return_value = None mock_init.return_value = self.dummy_process3 # Make the state non-empty to mock the case when creation is not being # done for the first time self.test_manager._transformer_process_dict = \ self.transformer_process_dict_example self.test_manager._start_transformers_processes() expected_trans_proc_dict_without_gh = { SYSTEM_DATA_TRANSFORMER_NAME: self.dummy_process3, } actual_trans_proc_dict_without_gh = \ self.test_manager._transformer_process_dict del actual_trans_proc_dict_without_gh[GITHUB_DATA_TRANSFORMER_NAME] self.assertEqual(expected_trans_proc_dict_without_gh, actual_trans_proc_dict_without_gh) @mock.patch("src.data_transformers.starters.create_logger") def test_start_transformers_processes_starts_system_process_if_first_time( self, mock_create_logger) -> None: mock_create_logger.return_value = self.dummy_logger self.test_manager._start_transformers_processes() # We need to sleep to give some time for the data transformer to be # initialized, otherwise the process would not terminate time.sleep(1) system_data_trans_process = self.test_manager.transformer_process_dict[ SYSTEM_DATA_TRANSFORMER_NAME] github_data_trans_process = self.test_manager.transformer_process_dict[ GITHUB_DATA_TRANSFORMER_NAME] self.assertTrue(system_data_trans_process.is_alive()) system_data_trans_process.terminate() system_data_trans_process.join() github_data_trans_process.terminate() github_data_trans_process.join() @mock.patch("src.data_transformers.starters.create_logger") def test_start_transformers_processes_starts_system_process_if_not_alive( self, mock_create_logger) -> None: mock_create_logger.return_value = self.dummy_logger # Make the state non-empty to mock the case when creation is not being # done for the first time self.test_manager._transformer_process_dict = \ self.transformer_process_dict_example self.test_manager._start_transformers_processes() # We need to sleep to give some time for the data transformer to be # initialized, otherwise the process would not terminate time.sleep(1) system_data_trans_process = self.test_manager.transformer_process_dict[ SYSTEM_DATA_TRANSFORMER_NAME] github_data_trans_process = self.test_manager.transformer_process_dict[ GITHUB_DATA_TRANSFORMER_NAME] self.assertTrue(system_data_trans_process.is_alive()) system_data_trans_process.terminate() system_data_trans_process.join() github_data_trans_process.terminate() github_data_trans_process.join() @mock.patch.object(multiprocessing.Process, "start") def test_start_transformers_processes_creates_correct_git_proc_first_time( self, mock_start) -> None: mock_start.return_value = None self.test_manager._start_transformers_processes() github_data_trans_proc = self.test_manager.transformer_process_dict[ GITHUB_DATA_TRANSFORMER_NAME] self.assertTrue(github_data_trans_proc.daemon) self.assertEqual(0, len(github_data_trans_proc._args)) self.assertEqual(start_github_data_transformer, github_data_trans_proc._target) @mock.patch.object(multiprocessing.Process, "start") def test_start_transformers_processes_creates_correct_git_proc_if_not_alive( self, mock_start) -> None: mock_start.return_value = None # Make the state non-empty to mock the case when creation is not being # done for the first time self.test_manager._transformer_process_dict = \ self.transformer_process_dict_example self.test_manager._start_transformers_processes() github_data_trans_proc = self.test_manager.transformer_process_dict[ GITHUB_DATA_TRANSFORMER_NAME] self.assertTrue(github_data_trans_proc.daemon) self.assertEqual(0, len(github_data_trans_proc._args)) self.assertEqual(start_github_data_transformer, github_data_trans_proc._target) @mock.patch.object(multiprocessing.Process, "start") @mock.patch.object(multiprocessing, "Process") def test_start_transformers_processes_stores_github_proc_correct_first_time( self, mock_init, mock_start) -> None: mock_start.return_value = None mock_init.return_value = self.dummy_process3 self.test_manager._start_transformers_processes() expected_trans_proc_dict_without_sys = { GITHUB_DATA_TRANSFORMER_NAME: self.dummy_process3, } actual_trans_proc_dict_without_sys = \ self.test_manager._transformer_process_dict del actual_trans_proc_dict_without_sys[SYSTEM_DATA_TRANSFORMER_NAME] self.assertEqual(expected_trans_proc_dict_without_sys, actual_trans_proc_dict_without_sys) @mock.patch.object(multiprocessing.Process, "start") @mock.patch.object(multiprocessing, "Process") def test_start_transformers_processes_stores_git_proc_correct_if_not_alive( self, mock_init, mock_start) -> None: mock_start.return_value = None mock_init.return_value = self.dummy_process3 # Make the state non-empty to mock the case when creation is not being # done for the first time self.test_manager._transformer_process_dict = \ self.transformer_process_dict_example self.test_manager._start_transformers_processes() expected_trans_proc_dict_without_sys = { GITHUB_DATA_TRANSFORMER_NAME: self.dummy_process3, } actual_trans_proc_dict_without_sys = \ self.test_manager._transformer_process_dict del actual_trans_proc_dict_without_sys[SYSTEM_DATA_TRANSFORMER_NAME] self.assertEqual(expected_trans_proc_dict_without_sys, actual_trans_proc_dict_without_sys) @mock.patch("src.data_transformers.starters.create_logger") def test_start_transformers_processes_starts_github_process_if_first_time( self, mock_create_logger) -> None: mock_create_logger.return_value = self.dummy_logger self.test_manager._start_transformers_processes() # We need to sleep to give some time for the data transformer to be # initialized, otherwise the process would not terminate time.sleep(1) system_data_trans_process = self.test_manager.transformer_process_dict[ SYSTEM_DATA_TRANSFORMER_NAME] github_data_trans_process = self.test_manager.transformer_process_dict[ GITHUB_DATA_TRANSFORMER_NAME] self.assertTrue(github_data_trans_process.is_alive()) system_data_trans_process.terminate() system_data_trans_process.join() github_data_trans_process.terminate() github_data_trans_process.join() @mock.patch("src.data_transformers.starters.create_logger") def test_start_transformers_processes_starts_github_process_if_not_alive( self, mock_create_logger) -> None: mock_create_logger.return_value = self.dummy_logger # Make the state non-empty to mock the case when creation is not being # done for the first time self.test_manager._transformer_process_dict = \ self.transformer_process_dict_example self.test_manager._start_transformers_processes() # We need to sleep to give some time for the data transformer to be # initialized, otherwise the process would not terminate time.sleep(1) system_data_trans_process = self.test_manager.transformer_process_dict[ SYSTEM_DATA_TRANSFORMER_NAME] github_data_trans_process = self.test_manager.transformer_process_dict[ GITHUB_DATA_TRANSFORMER_NAME] self.assertTrue(github_data_trans_process.is_alive()) system_data_trans_process.terminate() system_data_trans_process.join() github_data_trans_process.terminate() github_data_trans_process.join() @mock.patch.object(multiprocessing.Process, "is_alive") @mock.patch.object(multiprocessing.Process, "start") def test_start_transformers_processes_does_not_start_any_process_if_alive( self, mock_start, mock_is_alive) -> None: mock_start.return_value = None mock_is_alive.return_value = True # Make the state non-empty to mock the case when creation is not being # done for the first time self.test_manager._transformer_process_dict = \ self.transformer_process_dict_example self.test_manager._start_transformers_processes() mock_start.assert_not_called() @freeze_time("2012-01-01") @mock.patch.object(DataTransformersManager, "_send_heartbeat") @mock.patch.object(multiprocessing.Process, "is_alive") def test_process_ping_sends_a_valid_hb_if_all_processes_are_alive( self, mock_is_alive, mock_send_hb) -> None: # We will perform this test by checking that send_hb is called with the # correct heartbeat. The actual sending was already tested above. Note # we wil mock is_alive by setting it to always return true to avoid # creating processes mock_is_alive.return_value = True mock_send_hb.return_value = None try: # Some of the variables below are needed as parameters for the # process_ping function self.test_manager._initialise_rabbitmq() blocking_channel = self.test_manager.rabbitmq.channel method = pika.spec.Basic.Deliver( routing_key=DT_MAN_INPUT_ROUTING_KEY) body = 'ping' properties = pika.spec.BasicProperties() # Make the state non-empty to mock the fact that the processes were # already created self.test_manager._transformer_process_dict = \ self.transformer_process_dict_example self.test_manager._process_ping(blocking_channel, method, properties, body) expected_hb = { 'component_name': self.test_manager.name, 'running_processes': [SYSTEM_DATA_TRANSFORMER_NAME, GITHUB_DATA_TRANSFORMER_NAME], 'dead_processes': [], 'timestamp': datetime.now().timestamp() } mock_send_hb.assert_called_once_with(expected_hb) except Exception as e: self.fail("Test failed: {}".format(e)) @freeze_time("2012-01-01") @mock.patch.object(DataTransformersManager, "_send_heartbeat") @mock.patch.object(DataTransformersManager, "_start_transformers_processes") @mock.patch.object(multiprocessing.Process, "is_alive") @mock.patch.object(multiprocessing.Process, "join") def test_process_ping_sends_a_valid_hb_if_some_processes_alive_some_dead( self, mock_join, mock_is_alive, mock_start_trans, mock_send_hb) -> None: # We will perform this test by checking that send_hb is called with the # correct heartbeat as the actual sending was already tested above. Note # we wil mock is_alive by setting it to first return true and then # return false (note we only have two processes). By this we can avoid # creating processes. mock_is_alive.side_effect = [True, False] mock_send_hb.return_value = None mock_join.return_value = None mock_start_trans.return_value = None try: # Some of the variables below are needed as parameters for the # process_ping function self.test_manager._initialise_rabbitmq() blocking_channel = self.test_manager.rabbitmq.channel method = pika.spec.Basic.Deliver( routing_key=DT_MAN_INPUT_ROUTING_KEY) body = 'ping' properties = pika.spec.BasicProperties() # Make the state non-empty to mock the fact that the processes were # already created self.test_manager._transformer_process_dict = \ self.transformer_process_dict_example self.test_manager._process_ping(blocking_channel, method, properties, body) expected_hb = { 'component_name': self.test_manager.name, 'running_processes': [SYSTEM_DATA_TRANSFORMER_NAME], 'dead_processes': [GITHUB_DATA_TRANSFORMER_NAME], 'timestamp': datetime.now().timestamp() } mock_send_hb.assert_called_once_with(expected_hb) except Exception as e: self.fail("Test failed: {}".format(e)) @freeze_time("2012-01-01") @mock.patch.object(DataTransformersManager, "_send_heartbeat") @mock.patch.object(DataTransformersManager, "_start_transformers_processes") @mock.patch.object(multiprocessing.Process, "is_alive") @mock.patch.object(multiprocessing.Process, "join") def test_process_ping_sends_a_valid_hb_if_all_processes_dead( self, mock_join, mock_is_alive, mock_start_trans, mock_send_hb) -> None: # We will perform this test by checking that send_hb is called with the # correct heartbeat as the actual sending was already tested above. Note # we wil mock is_alive by setting it to always return False. By this we # can avoid creating processes. mock_is_alive.return_value = False mock_send_hb.return_value = None mock_join.return_value = None mock_start_trans.return_value = None try: # Some of the variables below are needed as parameters for the # process_ping function self.test_manager._initialise_rabbitmq() blocking_channel = self.test_manager.rabbitmq.channel method = pika.spec.Basic.Deliver( routing_key=DT_MAN_INPUT_ROUTING_KEY) body = 'ping' properties = pika.spec.BasicProperties() # Make the state non-empty to mock the fact that the processes were # already created self.test_manager._transformer_process_dict = \ self.transformer_process_dict_example self.test_manager._process_ping(blocking_channel, method, properties, body) expected_hb = { 'component_name': self.test_manager.name, 'running_processes': [], 'dead_processes': [SYSTEM_DATA_TRANSFORMER_NAME, GITHUB_DATA_TRANSFORMER_NAME], 'timestamp': datetime.now().timestamp() } mock_send_hb.assert_called_once_with(expected_hb) except Exception as e: self.fail("Test failed: {}".format(e)) @freeze_time("2012-01-01") @mock.patch.object(DataTransformersManager, "_send_heartbeat") @mock.patch.object(DataTransformersManager, "_start_transformers_processes") @mock.patch.object(multiprocessing.Process, "is_alive") @mock.patch.object(multiprocessing.Process, "join") def test_process_ping_restarts_dead_processes_all_dead( self, mock_join, mock_is_alive, mock_start_trans, mock_send_hb) -> None: # We will perform this test by checking that # _start_transformers_processes is called, as the actual restarting # logic was already tested above. Note we wil mock is_alive by setting # it to always return False. By this we can avoid creating processes. mock_is_alive.return_value = False mock_send_hb.return_value = None mock_join.return_value = None mock_start_trans.return_value = None try: # Some of the variables below are needed as parameters for the # process_ping function self.test_manager._initialise_rabbitmq() blocking_channel = self.test_manager.rabbitmq.channel method = pika.spec.Basic.Deliver( routing_key=DT_MAN_INPUT_ROUTING_KEY) body = 'ping' properties = pika.spec.BasicProperties() # Make the state non-empty to mock the fact that the processes were # already created self.test_manager._transformer_process_dict = \ self.transformer_process_dict_example self.test_manager._process_ping(blocking_channel, method, properties, body) mock_start_trans.assert_called_once() except Exception as e: self.fail("Test failed: {}".format(e)) @freeze_time("2012-01-01") @mock.patch.object(DataTransformersManager, "_send_heartbeat") @mock.patch.object(DataTransformersManager, "_start_transformers_processes") @mock.patch.object(multiprocessing.Process, "is_alive") @mock.patch.object(multiprocessing.Process, "join") def test_process_ping_restarts_dead_processes_some_alive_some_dead( self, mock_join, mock_is_alive, mock_start_trans, mock_send_hb) -> None: # We will perform this test by checking that # _start_transformers_processes is called, as the actual restarting # logic was already tested above. Note we wil mock is_alive by setting # it to first return False and then True. By this we can avoid creating # processes. mock_is_alive.side_effect = [False, True] mock_send_hb.return_value = None mock_join.return_value = None mock_start_trans.return_value = None try: # Some of the variables below are needed as parameters for the # process_ping function self.test_manager._initialise_rabbitmq() blocking_channel = self.test_manager.rabbitmq.channel method = pika.spec.Basic.Deliver( routing_key=DT_MAN_INPUT_ROUTING_KEY) body = 'ping' properties = pika.spec.BasicProperties() # Make the state non-empty to mock the fact that the processes were # already created self.test_manager._transformer_process_dict = \ self.transformer_process_dict_example self.test_manager._process_ping(blocking_channel, method, properties, body) mock_start_trans.assert_called_once() except Exception as e: self.fail("Test failed: {}".format(e)) @freeze_time("2012-01-01") @mock.patch.object(DataTransformersManager, "_send_heartbeat") @mock.patch.object(DataTransformersManager, "_start_transformers_processes") @mock.patch.object(multiprocessing.Process, "is_alive") @mock.patch.object(multiprocessing.Process, "join") def test_process_ping_does_not_restart_dead_processes_if_all_alive( self, mock_join, mock_is_alive, mock_start_trans, mock_send_hb) -> None: # We will perform this test by checking that # _start_transformers_processes is called, as the actual restarting # logic was already tested above. Note we wil mock is_alive by setting # it to always return True. By this we can avoid creating processes. mock_is_alive.return_value = True mock_send_hb.return_value = None mock_join.return_value = None mock_start_trans.return_value = None try: # Some of the variables below are needed as parameters for the # process_ping function self.test_manager._initialise_rabbitmq() blocking_channel = self.test_manager.rabbitmq.channel method = pika.spec.Basic.Deliver( routing_key=DT_MAN_INPUT_ROUTING_KEY) body = 'ping' properties = pika.spec.BasicProperties() # Make the state non-empty to mock the fact that the processes were # already created self.test_manager._transformer_process_dict = \ self.transformer_process_dict_example self.test_manager._process_ping(blocking_channel, method, properties, body) mock_start_trans.assert_not_called() except Exception as e: self.fail("Test failed: {}".format(e)) @freeze_time("2012-01-01") @mock.patch.object(DataTransformersManager, "_send_heartbeat") @mock.patch.object(multiprocessing.Process, "is_alive") @mock.patch.object(multiprocessing.Process, "start") @mock.patch.object(multiprocessing, 'Process') def test_process_ping_does_not_send_hb_if_processing_fails( self, mock_process, mock_start, mock_is_alive, mock_send_hb) -> None: # We will perform this test by checking that _send_heartbeat is not # called. Note we will generate an exception from is_alive mock_is_alive.side_effect = self.test_exception mock_send_hb.return_value = None mock_start.return_value = None mock_process.side_effect = self.dummy_process1 try: # Some of the variables below are needed as parameters for the # process_ping function self.test_manager._initialise_rabbitmq() blocking_channel = self.test_manager.rabbitmq.channel method = pika.spec.Basic.Deliver( routing_key=DT_MAN_INPUT_ROUTING_KEY) body = 'ping' properties = pika.spec.BasicProperties() # Make the state non-empty to mock the fact that the processes were # already created self.test_manager._transformer_process_dict = \ self.transformer_process_dict_example self.test_manager._process_ping(blocking_channel, method, properties, body) mock_send_hb.assert_not_called() except Exception as e: self.fail("Test failed: {}".format(e)) @mock.patch.object(DataTransformersManager, "_send_heartbeat") def test_proc_ping_send_hb_does_not_raise_msg_not_del_exce_if_hb_not_routed( self, mock_send_hb) -> None: # This test would fail if a msg not del excep is raised, as it is not # caught in the test. mock_send_hb.side_effect = MessageWasNotDeliveredException('test') try: # Some of the variables below are needed as parameters for the # process_ping function self.test_manager._initialise_rabbitmq() blocking_channel = self.test_manager.rabbitmq.channel method = pika.spec.Basic.Deliver( routing_key=DT_MAN_INPUT_ROUTING_KEY) body = 'ping' properties = pika.spec.BasicProperties() self.test_manager._process_ping(blocking_channel, method, properties, body) except Exception as e: self.fail("Test failed: {}".format(e)) @mock.patch.object(DataTransformersManager, "_send_heartbeat") def test_process_ping_send_hb_raises_amqp_connection_err_on_connection_err( self, mock_send_hb) -> None: mock_send_hb.side_effect = pika.exceptions.AMQPConnectionError('test') try: # Some of the variables below are needed as parameters for the # process_ping function self.test_manager._initialise_rabbitmq() blocking_channel = self.test_manager.rabbitmq.channel method = pika.spec.Basic.Deliver( routing_key=DT_MAN_INPUT_ROUTING_KEY) body = 'ping' properties = pika.spec.BasicProperties() self.assertRaises(pika.exceptions.AMQPConnectionError, self.test_manager._process_ping, blocking_channel, method, properties, body) except Exception as e: self.fail("Test failed: {}".format(e)) @mock.patch.object(DataTransformersManager, "_send_heartbeat") def test_process_ping_send_hb_raises_amqp_channel_err_on_channel_err( self, mock_send_hb) -> None: mock_send_hb.side_effect = pika.exceptions.AMQPChannelError('test') try: # Some of the variables below are needed as parameters for the # process_ping function self.test_manager._initialise_rabbitmq() blocking_channel = self.test_manager.rabbitmq.channel method = pika.spec.Basic.Deliver( routing_key=DT_MAN_INPUT_ROUTING_KEY) body = 'ping' properties = pika.spec.BasicProperties() self.assertRaises(pika.exceptions.AMQPChannelError, self.test_manager._process_ping, blocking_channel, method, properties, body) except Exception as e: self.fail("Test failed: {}".format(e)) @mock.patch.object(DataTransformersManager, "_send_heartbeat") def test_process_ping_send_hb_raises_exception_on_unexpected_exception( self, mock_send_hb) -> None: mock_send_hb.side_effect = self.test_exception try: # Some of the variables below are needed as parameters for the # process_ping function self.test_manager._initialise_rabbitmq() blocking_channel = self.test_manager.rabbitmq.channel method = pika.spec.Basic.Deliver( routing_key=DT_MAN_INPUT_ROUTING_KEY) body = 'ping' properties = pika.spec.BasicProperties() self.assertRaises(PANICException, self.test_manager._process_ping, blocking_channel, method, properties, body) except Exception as e: self.fail("Test failed: {}".format(e))
def _initialise_and_declare_config_queues() -> None: # TODO: This can be refactored by storing the queue configurations in # : constant.py so that it is easier to maintain. dummy_logger = logging.getLogger('Dummy') while True: try: rabbitmq = RabbitMQApi(dummy_logger, env.RABBIT_IP) log_and_print( "Connecting with RabbitMQ to create and bind " "configuration queues.", dummy_logger) ret = rabbitmq.connect() if ret == -1: log_and_print( "RabbitMQ is temporarily unavailable. Re-trying in {} " "seconds.".format(RE_INITIALISE_SLEEPING_PERIOD), dummy_logger) time.sleep(RE_INITIALISE_SLEEPING_PERIOD) continue # Config exchange declaration log_and_print("Creating {} exchange.".format(CONFIG_EXCHANGE), dummy_logger) rabbitmq.exchange_declare(CONFIG_EXCHANGE, 'topic', False, True, False, False) # Alert router queues log_and_print( "Creating queue '{}'".format(ALERT_ROUTER_CONFIGS_QUEUE_NAME), dummy_logger) rabbitmq.queue_declare(ALERT_ROUTER_CONFIGS_QUEUE_NAME, False, True, False, False) log_and_print( "Binding queue '{}' to '{}' exchange with routing " "key {}.".format(ALERT_ROUTER_CONFIGS_QUEUE_NAME, CONFIG_EXCHANGE, 'channels.*'), dummy_logger) rabbitmq.queue_bind(ALERT_ROUTER_CONFIGS_QUEUE_NAME, CONFIG_EXCHANGE, 'channels.*') # System Alerters Manager queues log_and_print( "Creating queue '{}'".format( SYSTEM_ALERTERS_MANAGER_CONFIGS_QUEUE_NAME), dummy_logger) rabbitmq.queue_declare(SYSTEM_ALERTERS_MANAGER_CONFIGS_QUEUE_NAME, False, True, False, False) log_and_print( "Binding queue '{}' to '{}' exchange with routing " "key {}.".format(SYSTEM_ALERTERS_MANAGER_CONFIGS_QUEUE_NAME, CONFIG_EXCHANGE, 'chains.*.*.alerts_config'), dummy_logger) rabbitmq.queue_bind(SYSTEM_ALERTERS_MANAGER_CONFIGS_QUEUE_NAME, CONFIG_EXCHANGE, 'chains.*.*.alerts_config') log_and_print( "Binding queue '{}' to '{}' exchange with routing " "key {}.".format(SYSTEM_ALERTERS_MANAGER_CONFIGS_QUEUE_NAME, CONFIG_EXCHANGE, 'general.alerts_config'), dummy_logger) rabbitmq.queue_bind(SYSTEM_ALERTERS_MANAGER_CONFIGS_QUEUE_NAME, CONFIG_EXCHANGE, 'general.alerts_config') # Channels manager queues log_and_print( "Creating queue '{}'".format( CHANNELS_MANAGER_CONFIGS_QUEUE_NAME), dummy_logger) rabbitmq.queue_declare(CHANNELS_MANAGER_CONFIGS_QUEUE_NAME, False, True, False, False) log_and_print( "Binding queue '{}' to '{}' exchange with routing " "key {}.".format(CHANNELS_MANAGER_CONFIGS_QUEUE_NAME, CONFIG_EXCHANGE, 'channels.*'), dummy_logger) rabbitmq.queue_bind(CHANNELS_MANAGER_CONFIGS_QUEUE_NAME, CONFIG_EXCHANGE, 'channels.*') # GitHub Monitors Manager queues log_and_print( "Creating queue '{}'".format( GITHUB_MONITORS_MANAGER_CONFIGS_QUEUE_NAME), dummy_logger) rabbitmq.queue_declare(GITHUB_MONITORS_MANAGER_CONFIGS_QUEUE_NAME, False, True, False, False) log_and_print( "Binding queue '{}' to '{}' exchange with routing " "key {}.".format(GITHUB_MONITORS_MANAGER_CONFIGS_QUEUE_NAME, CONFIG_EXCHANGE, 'chains.*.*.repos_config'), dummy_logger) rabbitmq.queue_bind(GITHUB_MONITORS_MANAGER_CONFIGS_QUEUE_NAME, CONFIG_EXCHANGE, 'chains.*.*.repos_config') log_and_print( "Binding queue '{}' to '{}' exchange with routing " "key {}.".format(GITHUB_MONITORS_MANAGER_CONFIGS_QUEUE_NAME, CONFIG_EXCHANGE, 'general.repos_config'), dummy_logger) rabbitmq.queue_bind(GITHUB_MONITORS_MANAGER_CONFIGS_QUEUE_NAME, CONFIG_EXCHANGE, 'general.repos_config') # System Monitors Manager queues log_and_print( "Creating queue '{}'".format( SYSTEM_MONITORS_MANAGER_CONFIGS_QUEUE_NAME), dummy_logger) rabbitmq.queue_declare(SYSTEM_MONITORS_MANAGER_CONFIGS_QUEUE_NAME, False, True, False, False) log_and_print( "Binding queue '{}' to '{}' exchange with routing " "key {}.".format(SYSTEM_MONITORS_MANAGER_CONFIGS_QUEUE_NAME, CONFIG_EXCHANGE, 'chains.*.*.nodes_config'), dummy_logger) rabbitmq.queue_bind(SYSTEM_MONITORS_MANAGER_CONFIGS_QUEUE_NAME, CONFIG_EXCHANGE, 'chains.*.*.nodes_config') log_and_print( "Binding queue '{}' to '{}' exchange with routing " "key {}.".format(SYSTEM_MONITORS_MANAGER_CONFIGS_QUEUE_NAME, CONFIG_EXCHANGE, 'general.systems_config'), dummy_logger) rabbitmq.queue_bind(SYSTEM_MONITORS_MANAGER_CONFIGS_QUEUE_NAME, CONFIG_EXCHANGE, 'general.systems_config') # Config Store queues log_and_print( "Creating queue '{}'".format(STORE_CONFIGS_QUEUE_NAME), dummy_logger) rabbitmq.queue_declare(STORE_CONFIGS_QUEUE_NAME, False, True, False, False) log_and_print( "Binding queue '{}' to '{}' exchange with routing " "key {}.".format(STORE_CONFIGS_QUEUE_NAME, CONFIG_EXCHANGE, '#'), dummy_logger) rabbitmq.queue_bind(STORE_CONFIGS_QUEUE_NAME, CONFIG_EXCHANGE, '#') ret = rabbitmq.disconnect() if ret == -1: log_and_print( "RabbitMQ is temporarily unavailable. Re-trying in {} " "seconds.".format(RE_INITIALISE_SLEEPING_PERIOD), dummy_logger) time.sleep(RE_INITIALISE_SLEEPING_PERIOD) continue log_and_print( "Configuration queues initialisation procedure has " "completed successfully. Disconnecting with " "RabbitMQ.", dummy_logger) break except pika.exceptions.AMQPChannelError as e: log_and_print( "Channel error while initialising the configuration " "queues: {}. Re-trying in {} " "seconds.".format(repr(e), RE_INITIALISE_SLEEPING_PERIOD), dummy_logger) time.sleep(RE_INITIALISE_SLEEPING_PERIOD) except pika.exceptions.AMQPConnectionError as e: log_and_print( "RabbitMQ connection error while initialising the " "configuration queues: {}. Re-trying in {} " "seconds.".format(repr(e), RE_INITIALISE_SLEEPING_PERIOD), dummy_logger) time.sleep(RE_INITIALISE_SLEEPING_PERIOD) except Exception as e: log_and_print( "Unexpected exception while initialising the " "configuration queues: {}. Re-trying in {} " "seconds.".format(repr(e), RE_INITIALISE_SLEEPING_PERIOD), dummy_logger) time.sleep(RE_INITIALISE_SLEEPING_PERIOD)
class TestGitHubMonitor(unittest.TestCase): def setUp(self) -> None: self.dummy_logger = logging.getLogger('Dummy') self.dummy_logger.disabled = True self.connection_check_time_interval = timedelta(seconds=0) self.rabbit_ip = env.RABBIT_IP self.rabbitmq = RabbitMQApi( self.dummy_logger, self.rabbit_ip, connection_check_time_interval=self.connection_check_time_interval) self.monitor_name = 'test_monitor' self.monitoring_period = 10 self.repo_id = 'test_repo_id' self.parent_id = 'test_parent_id' self.repo_name = 'test_repo' self.monitor_repo = True self.releases_page = 'test_url' self.routing_key = 'test_routing_key' self.test_data_str = 'test data' self.test_data_dict = { 'test_key_1': 'test_val_1', 'test_key_2': 'test_val_2', } self.test_heartbeat = { 'component_name': 'Test Component', 'is_alive': True, 'timestamp': datetime(2012, 1, 1).timestamp(), } self.test_queue_name = 'Test Queue' # In the real retrieved data there are more fields, but these are the # only ones that interest us so far. self.retrieved_metrics_example = [ { 'name': 'First Release 😮', 'tag_name': 'v1.0.0' }, { 'name': 'Release Candidate 1', 'tag_name': 'v0.1.0' }, ] self.processed_data_example = { '0': { 'release_name': 'First Release 😮', 'tag_name': 'v1.0.0' }, '1': { 'release_name': 'Release Candidate 1', 'tag_name': 'v0.1.0' }, } self.test_exception = PANICException('test_exception', 1) self.repo_config = RepoConfig(self.repo_id, self.parent_id, self.repo_name, self.monitor_repo, self.releases_page) self.test_monitor = GitHubMonitor(self.monitor_name, self.repo_config, self.dummy_logger, self.monitoring_period, self.rabbitmq) def tearDown(self) -> None: # Delete any queues and exchanges which are common across many tests try: self.test_monitor.rabbitmq.connect() # Declare them before just in case there are tests which do not # use these queues and exchanges self.test_monitor.rabbitmq.queue_declare( queue=self.test_queue_name, durable=True, exclusive=False, auto_delete=False, passive=False) self.test_monitor.rabbitmq.exchange_declare( HEALTH_CHECK_EXCHANGE, 'topic', False, True, False, False) self.test_monitor.rabbitmq.exchange_declare( RAW_DATA_EXCHANGE, 'direct', False, True, False, False) self.test_monitor.rabbitmq.queue_purge(self.test_queue_name) self.test_monitor.rabbitmq.queue_delete(self.test_queue_name) self.test_monitor.rabbitmq.exchange_delete(RAW_DATA_EXCHANGE) self.test_monitor.rabbitmq.exchange_delete(HEALTH_CHECK_EXCHANGE) self.test_monitor.rabbitmq.disconnect() except Exception as e: print("Deletion of queues and exchanges failed: {}".format(e)) self.dummy_logger = None self.rabbitmq = None self.test_exception = None self.repo_config = None self.test_monitor = None def test_str_returns_monitor_name(self) -> None: self.assertEqual(self.monitor_name, str(self.test_monitor)) def test_get_monitor_period_returns_monitor_period(self) -> None: self.assertEqual(self.monitoring_period, self.test_monitor.monitor_period) def test_get_monitor_name_returns_monitor_name(self) -> None: self.assertEqual(self.monitor_name, self.test_monitor.monitor_name) def test_repo_config_returns_repo_config(self) -> None: self.assertEqual(self.repo_config, self.test_monitor.repo_config) def test_initialise_rabbitmq_initialises_everything_as_expected( self) -> None: try: # To make sure that there is no connection/channel already # established self.assertIsNone(self.rabbitmq.connection) self.assertIsNone(self.rabbitmq.channel) # To make sure that the exchanges have not already been declared self.rabbitmq.connect() self.rabbitmq.exchange_delete(RAW_DATA_EXCHANGE) self.rabbitmq.exchange_delete(HEALTH_CHECK_EXCHANGE) self.rabbitmq.disconnect() self.test_monitor._initialise_rabbitmq() # Perform checks that the connection has been opened, marked as open # and that the delivery confirmation variable is set. self.assertTrue(self.test_monitor.rabbitmq.is_connected) self.assertTrue(self.test_monitor.rabbitmq.connection.is_open) self.assertTrue( self.test_monitor.rabbitmq.channel._delivery_confirmation) # Check whether the exchange has been creating by sending messages # to it. If this fails an exception is raised, hence the test fails. self.test_monitor.rabbitmq.basic_publish_confirm( exchange=RAW_DATA_EXCHANGE, routing_key=self.routing_key, body=self.test_data_str, is_body_dict=False, properties=pika.BasicProperties(delivery_mode=2), mandatory=False) self.test_monitor.rabbitmq.basic_publish_confirm( exchange=HEALTH_CHECK_EXCHANGE, routing_key=self.routing_key, body=self.test_data_str, is_body_dict=False, properties=pika.BasicProperties(delivery_mode=2), mandatory=False) except Exception as e: self.fail("Test failed: {}".format(e)) @mock.patch.object(GitHubMonitor, "_process_retrieved_data") @mock.patch.object(GitHubMonitor, "_process_error") def test_process_data_calls_process_error_on_retrieval_error( self, mock_process_error, mock_process_retrieved_data) -> None: # Do not test the processing of data for now mock_process_error.return_value = self.test_data_dict self.test_monitor._process_data(self.test_data_dict, True, self.test_exception) # Test passes if _process_error is called once and # process_retrieved_data is not called self.assertEqual(1, mock_process_error.call_count) self.assertEqual(0, mock_process_retrieved_data.call_count) @mock.patch.object(GitHubMonitor, "_process_retrieved_data") @mock.patch.object(GitHubMonitor, "_process_error") def test_process_data_calls_process_retrieved_data_on_retrieval_success( self, mock_process_error, mock_process_retrieved_data) -> None: # Do not test the processing of data for now mock_process_retrieved_data.return_value = self.test_data_dict self.test_monitor._process_data(self.test_data_dict, False, None) # Test passes if _process_error is called once and # process_retrieved_data is not called self.assertEqual(0, mock_process_error.call_count) self.assertEqual(1, mock_process_retrieved_data.call_count) def test_send_heartbeat_sends_a_heartbeat_correctly(self) -> None: # This test creates a queue which receives messages with the same # routing key as the ones sent by send_heartbeat, and checks that the # heartbeat is received try: self.test_monitor._initialise_rabbitmq() # Delete the queue before to avoid messages in the queue on error. self.test_monitor.rabbitmq.queue_delete(self.test_queue_name) res = self.test_monitor.rabbitmq.queue_declare( queue=self.test_queue_name, durable=True, exclusive=False, auto_delete=False, passive=False) self.assertEqual(0, res.method.message_count) self.test_monitor.rabbitmq.queue_bind( queue=self.test_queue_name, exchange=HEALTH_CHECK_EXCHANGE, routing_key='heartbeat.worker') self.test_monitor._send_heartbeat(self.test_heartbeat) # By re-declaring the queue again we can get the number of messages # in the queue. res = self.test_monitor.rabbitmq.queue_declare( queue=self.test_queue_name, durable=True, exclusive=False, auto_delete=False, passive=True) self.assertEqual(1, res.method.message_count) # Check that the message received is actually the HB _, _, body = self.test_monitor.rabbitmq.basic_get( self.test_queue_name) self.assertEqual(self.test_heartbeat, json.loads(body)) except Exception as e: self.fail("Test failed: {}".format(e)) def test_display_data_returns_the_correct_string(self) -> None: expected_output = json.dumps( self.processed_data_example, ensure_ascii=False).encode('utf8').decode() actual_output = self.test_monitor._display_data( self.processed_data_example) self.assertEqual(expected_output, actual_output) @freeze_time("2012-01-01") def test_process_error_returns_expected_data(self) -> None: expected_output = { 'error': { 'meta_data': { 'monitor_name': self.test_monitor.monitor_name, 'repo_name': self.test_monitor.repo_config.repo_name, 'repo_id': self.test_monitor.repo_config.repo_id, 'repo_parent_id': self.test_monitor.repo_config.parent_id, 'time': datetime(2012, 1, 1).timestamp() }, 'message': self.test_exception.message, 'code': self.test_exception.code, } } actual_output = self.test_monitor._process_error(self.test_exception) self.assertEqual(actual_output, expected_output) @freeze_time("2012-01-01") def test_process_retrieved_data_returns_expected_data(self) -> None: expected_output = { 'result': { 'meta_data': { 'monitor_name': self.test_monitor.monitor_name, 'repo_name': self.test_monitor.repo_config.repo_name, 'repo_id': self.test_monitor.repo_config.repo_id, 'repo_parent_id': self.test_monitor.repo_config.parent_id, 'time': datetime(2012, 1, 1).timestamp() }, 'data': self.processed_data_example, } } actual_output = self.test_monitor._process_retrieved_data( self.retrieved_metrics_example) self.assertEqual(expected_output, actual_output) def test_send_data_sends_data_correctly(self) -> None: # This test creates a queue which receives messages with the same # routing key as the ones sent by send_data, and checks that the # data is received try: self.test_monitor._initialise_rabbitmq() # Delete the queue before to avoid messages in the queue on error. self.test_monitor.rabbitmq.queue_delete(self.test_queue_name) res = self.test_monitor.rabbitmq.queue_declare( queue=self.test_queue_name, durable=True, exclusive=False, auto_delete=False, passive=False) self.assertEqual(0, res.method.message_count) self.test_monitor.rabbitmq.queue_bind(queue=self.test_queue_name, exchange=RAW_DATA_EXCHANGE, routing_key='github') self.test_monitor._send_data(self.processed_data_example) # By re-declaring the queue again we can get the number of messages # in the queue. res = self.test_monitor.rabbitmq.queue_declare( queue=self.test_queue_name, durable=True, exclusive=False, auto_delete=False, passive=True) self.assertEqual(1, res.method.message_count) # Check that the message received is actually the processed data _, _, body = self.test_monitor.rabbitmq.basic_get( self.test_queue_name) self.assertEqual(self.processed_data_example, json.loads(body)) except Exception as e: self.fail("Test failed: {}".format(e)) @freeze_time("2012-01-01") @mock.patch.object(GitHubMonitor, "_get_data") def test_monitor_sends_data_and_hb_if_data_retrieve_and_processing_success( self, mock_get_data) -> None: expected_output_data = { 'result': { 'meta_data': { 'monitor_name': self.test_monitor.monitor_name, 'repo_name': self.test_monitor.repo_config.repo_name, 'repo_id': self.test_monitor.repo_config.repo_id, 'repo_parent_id': self.test_monitor.repo_config.parent_id, 'time': datetime(2012, 1, 1).timestamp() }, 'data': self.processed_data_example, } } expected_output_hb = { 'component_name': self.test_monitor.monitor_name, 'is_alive': True, 'timestamp': datetime(2012, 1, 1).timestamp() } try: mock_get_data.return_value = self.retrieved_metrics_example self.test_monitor._initialise_rabbitmq() # Delete the queue before to avoid messages in the queue on error. self.test_monitor.rabbitmq.queue_delete(self.test_queue_name) res = self.test_monitor.rabbitmq.queue_declare( queue=self.test_queue_name, durable=True, exclusive=False, auto_delete=False, passive=False) self.assertEqual(0, res.method.message_count) self.test_monitor.rabbitmq.queue_bind(queue=self.test_queue_name, exchange=RAW_DATA_EXCHANGE, routing_key='github') self.test_monitor.rabbitmq.queue_bind( queue=self.test_queue_name, exchange=HEALTH_CHECK_EXCHANGE, routing_key='heartbeat.worker') self.test_monitor._monitor() # By re-declaring the queue again we can get the number of messages # in the queue. res = self.test_monitor.rabbitmq.queue_declare( queue=self.test_queue_name, durable=True, exclusive=False, auto_delete=False, passive=True) # There must be 2 messages in the queue, the heartbeat and the # processed data self.assertEqual(2, res.method.message_count) # Check that the message received is actually the processed data _, _, body = self.test_monitor.rabbitmq.basic_get( self.test_queue_name) self.assertEqual(expected_output_data, json.loads(body)) # Check that the message received is actually the HB _, _, body = self.test_monitor.rabbitmq.basic_get( self.test_queue_name) self.assertEqual(expected_output_hb, json.loads(body)) except Exception as e: self.fail("Test failed: {}".format(e)) @mock.patch.object(GitHubMonitor, "_process_data") @mock.patch.object(GitHubMonitor, "_get_data") def test_monitor_sends_no_data_and_hb_if_data_ret_success_and_proc_fails( self, mock_get_data, mock_process_data) -> None: mock_process_data.side_effect = self.test_exception mock_get_data.return_value = self.retrieved_metrics_example try: self.test_monitor._initialise_rabbitmq() # Delete the queue before to avoid messages in the queue on error. self.test_monitor.rabbitmq.queue_delete(self.test_queue_name) res = self.test_monitor.rabbitmq.queue_declare( queue=self.test_queue_name, durable=True, exclusive=False, auto_delete=False, passive=False) self.assertEqual(0, res.method.message_count) self.test_monitor.rabbitmq.queue_bind(queue=self.test_queue_name, exchange=RAW_DATA_EXCHANGE, routing_key='github') self.test_monitor.rabbitmq.queue_bind( queue=self.test_queue_name, exchange=HEALTH_CHECK_EXCHANGE, routing_key='heartbeat.worker') self.test_monitor._monitor() # By re-declaring the queue again we can get the number of messages # in the queue. res = self.test_monitor.rabbitmq.queue_declare( queue=self.test_queue_name, durable=True, exclusive=False, auto_delete=False, passive=True) # There must be 0 messages in the queue. self.assertEqual(0, res.method.message_count) except Exception as e: self.fail("Test failed: {}".format(e)) @mock.patch.object(GitHubMonitor, "_get_data") def test_monitor_sends_no_data_and_no_hb_on_get_data_unexpected_exception( self, mock_get_data) -> None: mock_get_data.side_effect = self.test_exception try: self.test_monitor._initialise_rabbitmq() # Delete the queue before to avoid messages in the queue on error. self.test_monitor.rabbitmq.queue_delete(self.test_queue_name) res = self.test_monitor.rabbitmq.queue_declare( queue=self.test_queue_name, durable=True, exclusive=False, auto_delete=False, passive=False) self.assertEqual(0, res.method.message_count) self.test_monitor.rabbitmq.queue_bind(queue=self.test_queue_name, exchange=RAW_DATA_EXCHANGE, routing_key='github') self.test_monitor.rabbitmq.queue_bind( queue=self.test_queue_name, exchange=HEALTH_CHECK_EXCHANGE, routing_key='heartbeat.worker') self.assertRaises(PANICException, self.test_monitor._monitor) # By re-declaring the queue again we can get the number of messages # in the queue. res = self.test_monitor.rabbitmq.queue_declare( queue=self.test_queue_name, durable=True, exclusive=False, auto_delete=False, passive=True) # There must be 0 messages in the queue. self.assertEqual(0, res.method.message_count) except Exception as e: self.fail("Test failed: {}".format(e)) @freeze_time("2012-01-01") @mock.patch.object(GitHubMonitor, "_get_data") def test_monitor_sends_gh_api_call_exception_data_and_hb_on_api_call_err( self, mock_get_data) -> None: api_call_err_return = { "message": "Not Found", "documentation_url": "https://docs.github.com/rest/reference/repos#list-releases" } mock_get_data.return_value = api_call_err_return data_ret_exception = GitHubAPICallException( api_call_err_return['message']) expected_output_data = { 'error': { 'meta_data': { 'monitor_name': self.test_monitor.monitor_name, 'repo_name': self.test_monitor.repo_config.repo_name, 'repo_id': self.test_monitor.repo_config.repo_id, 'repo_parent_id': self.test_monitor.repo_config.parent_id, 'time': datetime(2012, 1, 1).timestamp() }, 'message': data_ret_exception.message, 'code': data_ret_exception.code, } } expected_output_hb = { 'component_name': self.test_monitor.monitor_name, 'is_alive': True, 'timestamp': datetime(2012, 1, 1).timestamp() } try: self.test_monitor._initialise_rabbitmq() # Delete the queue before to avoid messages in the queue on error. self.test_monitor.rabbitmq.queue_delete(self.test_queue_name) res = self.test_monitor.rabbitmq.queue_declare( queue=self.test_queue_name, durable=True, exclusive=False, auto_delete=False, passive=False) self.assertEqual(0, res.method.message_count) self.test_monitor.rabbitmq.queue_bind(queue=self.test_queue_name, exchange=RAW_DATA_EXCHANGE, routing_key='github') self.test_monitor.rabbitmq.queue_bind( queue=self.test_queue_name, exchange=HEALTH_CHECK_EXCHANGE, routing_key='heartbeat.worker') self.test_monitor._monitor() # By re-declaring the queue again we can get the number of messages # in the queue. res = self.test_monitor.rabbitmq.queue_declare( queue=self.test_queue_name, durable=True, exclusive=False, auto_delete=False, passive=True) # There must be 2 messages in the queue, the heartbeat and the # exception details. self.assertEqual(2, res.method.message_count) # Check that the message received is actually the processed data _, _, body = self.test_monitor.rabbitmq.basic_get( self.test_queue_name) self.assertEqual(expected_output_data, json.loads(body)) # Check that the message received is actually the HB _, _, body = self.test_monitor.rabbitmq.basic_get( self.test_queue_name) self.assertEqual(expected_output_hb, json.loads(body)) except Exception as e: self.fail("Test failed: {}".format(e)) @freeze_time("2012-01-01") @mock.patch.object(GitHubMonitor, "_get_data") def test_monitor_sends_exception_data_and_hb_on_expected_exceptions( self, mock_get_data) -> None: json_decode_error = json.JSONDecodeError(msg='test error', doc='test', pos=2) errors_exceptions_dict = { ReqConnectionError('test'): CannotAccessGitHubPageException(self.repo_config.releases_page), ReadTimeout('test'): CannotAccessGitHubPageException(self.repo_config.releases_page), IncompleteRead('test'): DataReadingException(self.monitor_name, self.repo_config.releases_page), ChunkedEncodingError('test'): DataReadingException(self.monitor_name, self.repo_config.releases_page), ProtocolError('test'): DataReadingException(self.monitor_name, self.repo_config.releases_page), json_decode_error: JSONDecodeException(json_decode_error) } try: self.test_monitor._initialise_rabbitmq() for error, data_ret_exception in errors_exceptions_dict.items(): mock_get_data.side_effect = error expected_output_data = { 'error': { 'meta_data': { 'monitor_name': self.test_monitor.monitor_name, 'repo_name': self.test_monitor.repo_config.repo_name, 'repo_id': self.test_monitor.repo_config.repo_id, 'repo_parent_id': self.test_monitor.repo_config.parent_id, 'time': datetime(2012, 1, 1).timestamp() }, 'message': data_ret_exception.message, 'code': data_ret_exception.code, } } expected_output_hb = { 'component_name': self.test_monitor.monitor_name, 'is_alive': True, 'timestamp': datetime(2012, 1, 1).timestamp() } # Delete the queue before to avoid messages in the queue on # error. self.test_monitor.rabbitmq.queue_delete(self.test_queue_name) res = self.test_monitor.rabbitmq.queue_declare( queue=self.test_queue_name, durable=True, exclusive=False, auto_delete=False, passive=False) self.assertEqual(0, res.method.message_count) self.test_monitor.rabbitmq.queue_bind( queue=self.test_queue_name, exchange=RAW_DATA_EXCHANGE, routing_key='github') self.test_monitor.rabbitmq.queue_bind( queue=self.test_queue_name, exchange=HEALTH_CHECK_EXCHANGE, routing_key='heartbeat.worker') self.test_monitor._monitor() # By re-declaring the queue again we can get the number of # messages in the queue. res = self.test_monitor.rabbitmq.queue_declare( queue=self.test_queue_name, durable=True, exclusive=False, auto_delete=False, passive=True) # There must be 2 messages in the queue, the heartbeat and the # processed data self.assertEqual(2, res.method.message_count) # Check that the message received is actually the processed data _, _, body = self.test_monitor.rabbitmq.basic_get( self.test_queue_name) self.assertEqual(expected_output_data, json.loads(body)) # Check that the message received is actually the HB _, _, body = self.test_monitor.rabbitmq.basic_get( self.test_queue_name) self.assertEqual(expected_output_hb, json.loads(body)) except Exception as e: self.fail("Test failed: {}".format(e)) @mock.patch.object(GitHubMonitor, "_get_data") def test_monitor_raises_msg_not_delivered_exception_if_data_not_routed( self, mock_get_data) -> None: mock_get_data.return_value = self.retrieved_metrics_example try: self.test_monitor._initialise_rabbitmq() self.assertRaises(MessageWasNotDeliveredException, self.test_monitor._monitor) except Exception as e: self.fail("Test failed: {}".format(e)) @freeze_time("2012-01-01") @mock.patch.object(GitHubMonitor, "_get_data") def test_monitor_raises_msg_not_del_except_if_hb_not_routed_and_sends_data( self, mock_get_data) -> None: mock_get_data.return_value = self.retrieved_metrics_example expected_output_data = { 'result': { 'meta_data': { 'monitor_name': self.test_monitor.monitor_name, 'repo_name': self.test_monitor.repo_config.repo_name, 'repo_id': self.test_monitor.repo_config.repo_id, 'repo_parent_id': self.test_monitor.repo_config.parent_id, 'time': datetime(2012, 1, 1).timestamp() }, 'data': self.processed_data_example, } } try: self.test_monitor._initialise_rabbitmq() self.test_monitor.rabbitmq.queue_delete(self.test_queue_name) res = self.test_monitor.rabbitmq.queue_declare( queue=self.test_queue_name, durable=True, exclusive=False, auto_delete=False, passive=False) self.assertEqual(0, res.method.message_count) self.test_monitor.rabbitmq.queue_bind(queue=self.test_queue_name, exchange=RAW_DATA_EXCHANGE, routing_key='github') self.assertRaises(MessageWasNotDeliveredException, self.test_monitor._monitor) # By re-declaring the queue again we can get the number of # messages in the queue. res = self.test_monitor.rabbitmq.queue_declare( queue=self.test_queue_name, durable=True, exclusive=False, auto_delete=False, passive=True) # There must be 1 message in the queue, the processed data self.assertEqual(1, res.method.message_count) # Check that the message received is actually the processed data _, _, body = self.test_monitor.rabbitmq.basic_get( self.test_queue_name) self.assertEqual(expected_output_data, json.loads(body)) except Exception as e: self.fail("Test failed: {}".format(e)) @mock.patch.object(GitHubMonitor, "_send_data") @mock.patch.object(GitHubMonitor, "_get_data") def test_monitor_send_data_raises_amqp_channel_error_on_channel_error( self, mock_get_data, mock_send_data) -> None: mock_get_data.return_value = self.retrieved_metrics_example mock_send_data.side_effect = pika.exceptions.AMQPChannelError('test') try: self.test_monitor._initialise_rabbitmq() self.assertRaises(pika.exceptions.AMQPChannelError, self.test_monitor._monitor) except Exception as e: self.fail("Test failed: {}".format(e)) @freeze_time("2012-01-01") @mock.patch.object(GitHubMonitor, "_send_heartbeat") @mock.patch.object(GitHubMonitor, "_get_data") def test_monitor_send_hb_raises_amqp_chan_err_on_chan_err_and_sends_data( self, mock_get_data, mock_send_heartbeat) -> None: mock_get_data.return_value = self.retrieved_metrics_example mock_send_heartbeat.side_effect = \ pika.exceptions.AMQPChannelError('test') expected_output_data = { 'result': { 'meta_data': { 'monitor_name': self.test_monitor.monitor_name, 'repo_name': self.test_monitor.repo_config.repo_name, 'repo_id': self.test_monitor.repo_config.repo_id, 'repo_parent_id': self.test_monitor.repo_config.parent_id, 'time': datetime(2012, 1, 1).timestamp() }, 'data': self.processed_data_example, } } try: self.test_monitor._initialise_rabbitmq() self.test_monitor.rabbitmq.queue_delete(self.test_queue_name) res = self.test_monitor.rabbitmq.queue_declare( queue=self.test_queue_name, durable=True, exclusive=False, auto_delete=False, passive=False) self.assertEqual(0, res.method.message_count) self.test_monitor.rabbitmq.queue_bind(queue=self.test_queue_name, exchange=RAW_DATA_EXCHANGE, routing_key='github') self.test_monitor.rabbitmq.queue_bind( queue=self.test_queue_name, exchange=HEALTH_CHECK_EXCHANGE, routing_key='heartbeat.worker') self.assertRaises(pika.exceptions.AMQPChannelError, self.test_monitor._monitor) # By re-declaring the queue again we can get the number of # messages in the queue. res = self.test_monitor.rabbitmq.queue_declare( queue=self.test_queue_name, durable=True, exclusive=False, auto_delete=False, passive=True) # There must be 1 message in the queue, the processed data self.assertEqual(1, res.method.message_count) # Check that the message received is actually the processed data _, _, body = self.test_monitor.rabbitmq.basic_get( self.test_queue_name) self.assertEqual(expected_output_data, json.loads(body)) except Exception as e: self.fail("Test failed: {}".format(e)) @mock.patch.object(GitHubMonitor, "_send_data") @mock.patch.object(GitHubMonitor, "_get_data") def test_monitor_send_data_raises_amqp_conn_error_on_conn_error( self, mock_get_data, mock_send_data) -> None: mock_get_data.return_value = self.retrieved_metrics_example mock_send_data.side_effect = pika.exceptions.AMQPConnectionError( 'test') try: self.test_monitor._initialise_rabbitmq() self.assertRaises(pika.exceptions.AMQPConnectionError, self.test_monitor._monitor) except Exception as e: self.fail("Test failed: {}".format(e)) @freeze_time("2012-01-01") @mock.patch.object(GitHubMonitor, "_send_heartbeat") @mock.patch.object(GitHubMonitor, "_get_data") def test_monitor_send_hb_raises_amqp_conn_err_on_conn_err_and_sends_data( self, mock_get_data, mock_send_heartbeat) -> None: mock_get_data.return_value = self.retrieved_metrics_example mock_send_heartbeat.side_effect = \ pika.exceptions.AMQPConnectionError('test') expected_output_data = { 'result': { 'meta_data': { 'monitor_name': self.test_monitor.monitor_name, 'repo_name': self.test_monitor.repo_config.repo_name, 'repo_id': self.test_monitor.repo_config.repo_id, 'repo_parent_id': self.test_monitor.repo_config.parent_id, 'time': datetime(2012, 1, 1).timestamp() }, 'data': self.processed_data_example, } } try: self.test_monitor._initialise_rabbitmq() self.test_monitor.rabbitmq.queue_delete(self.test_queue_name) res = self.test_monitor.rabbitmq.queue_declare( queue=self.test_queue_name, durable=True, exclusive=False, auto_delete=False, passive=False) self.assertEqual(0, res.method.message_count) self.test_monitor.rabbitmq.queue_bind(queue=self.test_queue_name, exchange=RAW_DATA_EXCHANGE, routing_key='github') self.test_monitor.rabbitmq.queue_bind( queue=self.test_queue_name, exchange=HEALTH_CHECK_EXCHANGE, routing_key='heartbeat.worker') self.assertRaises(pika.exceptions.AMQPConnectionError, self.test_monitor._monitor) # By re-declaring the queue again we can get the number of # messages in the queue. res = self.test_monitor.rabbitmq.queue_declare( queue=self.test_queue_name, durable=True, exclusive=False, auto_delete=False, passive=True) # There must be 1 message in the queue, the processed data self.assertEqual(1, res.method.message_count) # Check that the message received is actually the processed data _, _, body = self.test_monitor.rabbitmq.basic_get( self.test_queue_name) self.assertEqual(expected_output_data, json.loads(body)) except Exception as e: self.fail("Test failed: {}".format(e)) @mock.patch.object(GitHubMonitor, "_send_data") @mock.patch.object(GitHubMonitor, "_get_data") def test_monitor_does_not_send_hb_and_data_if_send_data_fails( self, mock_get_data, mock_send_data) -> None: mock_get_data.return_value = self.retrieved_metrics_example exception_types_dict = \ { Exception('test'): Exception, pika.exceptions.AMQPConnectionError('test'): pika.exceptions.AMQPConnectionError, pika.exceptions.AMQPChannelError('test'): pika.exceptions.AMQPChannelError, MessageWasNotDeliveredException('test'): MessageWasNotDeliveredException } try: self.test_monitor._initialise_rabbitmq() for exception, exception_type in exception_types_dict.items(): mock_send_data.side_effect = exception self.test_monitor.rabbitmq.queue_delete(self.test_queue_name) res = self.test_monitor.rabbitmq.queue_declare( queue=self.test_queue_name, durable=True, exclusive=False, auto_delete=False, passive=False) self.assertEqual(0, res.method.message_count) self.test_monitor.rabbitmq.queue_bind( queue=self.test_queue_name, exchange=HEALTH_CHECK_EXCHANGE, routing_key='heartbeat.worker') self.test_monitor.rabbitmq.queue_bind( queue=self.test_queue_name, exchange=RAW_DATA_EXCHANGE, routing_key='github') self.assertRaises(exception_type, self.test_monitor._monitor) # By re-declaring the queue again we can get the number of # messages in the queue. res = self.test_monitor.rabbitmq.queue_declare( queue=self.test_queue_name, durable=True, exclusive=False, auto_delete=False, passive=True) # There must be no messages in the queue. self.assertEqual(0, res.method.message_count) except Exception as e: self.fail("Test failed: {}".format(e))
class TestConfigStore(unittest.TestCase): def setUp(self) -> None: self.dummy_logger = logging.getLogger('Dummy') self.dummy_logger.disabled = True self.connection_check_time_interval = timedelta(seconds=0) self.rabbit_ip = env.RABBIT_IP self.rabbitmq = RabbitMQApi( self.dummy_logger, self.rabbit_ip, connection_check_time_interval=self.connection_check_time_interval) self.test_rabbit_manager = RabbitMQApi( self.dummy_logger, self.rabbit_ip, connection_check_time_interval=self.connection_check_time_interval) self.redis_db = env.REDIS_DB self.redis_host = env.REDIS_IP self.redis_port = env.REDIS_PORT self.redis_namespace = env.UNIQUE_ALERTER_IDENTIFIER self.redis = RedisApi(self.dummy_logger, self.redis_db, self.redis_host, self.redis_port, '', self.redis_namespace, self.connection_check_time_interval) self.mongo_ip = env.DB_IP self.mongo_db = env.DB_NAME self.mongo_port = env.DB_PORT self.test_store_name = 'store name' self.test_store = ConfigStore(self.test_store_name, self.dummy_logger, self.rabbitmq) self.routing_key = 'heartbeat.worker' self.test_queue_name = 'test queue' connect_to_rabbit(self.rabbitmq) self.rabbitmq.exchange_declare(HEALTH_CHECK_EXCHANGE, 'topic', False, True, False, False) self.rabbitmq.exchange_declare(CONFIG_EXCHANGE, 'topic', False, True, False, False) self.rabbitmq.queue_declare(STORE_CONFIGS_QUEUE_NAME, False, True, False, False) self.rabbitmq.queue_bind(STORE_CONFIGS_QUEUE_NAME, CONFIG_EXCHANGE, STORE_CONFIGS_ROUTING_KEY_CHAINS) connect_to_rabbit(self.test_rabbit_manager) self.test_rabbit_manager.queue_declare(self.test_queue_name, False, True, False, False) self.test_rabbit_manager.queue_bind(self.test_queue_name, HEALTH_CHECK_EXCHANGE, self.routing_key) self.test_parent_id = 'parent_id' self.test_config_type = 'config_type' self.test_data_str = 'test data' self.test_exception = PANICException('test_exception', 1) self.last_monitored = datetime(2012, 1, 1).timestamp() self.routing_key_1 = 'chains.cosmos.cosmos.nodes_config' self.routing_key_2 = 'chains.cosmos.cosmos.alerts_config' self.routing_key_3 = 'chains.cosmos.cosmos.repos_config' self.routing_key_4 = 'general.repos_config' self.routing_key_5 = 'general.alerts_config' self.routing_key_6 = 'general.systems_config' self.routing_key_7 = 'channels.email_config' self.routing_key_8 = 'channels.pagerduty_config' self.routing_key_9 = 'channels.opsgenie_config' self.routing_key_10 = 'channels.telegram_config' self.routing_key_11 = 'channels.twilio_config' self.nodes_config_1 = { "node_3e0a5189-f474-4120-a0a4-d5ab817c0504": { "id": "node_3e0a5189-f474-4120-a0a4-d5ab817c0504", "parent_id": "chain_name_7f4bc842-21b1-4bcb-8ab9-d86e08149548", "name": "cosmos_sentry_1(46.166.146.165:9100)", "monitor_tendermint": "false", "monitor_rpc": "false", "monitor_prometheus": "false", "exporter_url": "http://46.166.146.165:9100/metrics", "monitor_system": "true", "is_validator": "false", "monitor_node": "true", "is_archive_node": "true", "use_as_data_source": "true" }, "node_f8ebf267-9b53-4aa1-9c45-e84a9cba5fbc": { "id": "node_f8ebf267-9b53-4aa1-9c45-e84a9cba5fbc", "parent_id": "chain_name_7f4bc842-21b1-4bcb-8ab9-d86e08149548", "name": "cosmos_sentry_2(172.16.151.10:9100)", "monitor_tendermint": "false", "monitor_rpc": "false", "monitor_prometheus": "false", "exporter_url": "http://172.16.151.10:9100/metrics", "monitor_system": "true", "is_validator": "false", "monitor_node": "true", "is_archive_node": "true", "use_as_data_source": "true" } } self.repos_config_1 = { "repo_4ea76d87-d291-4b68-88af-da2bd1e16e2e": { "id": "repo_4ea76d87-d291-4b68-88af-da2bd1e16e2e", "parent_id": "chain_name_7f4bc842-21b1-4bcb-8ab9-d86e08149548", "repo_name": "tendermint/tendermint/", "monitor_repo": "true" }, "repo_83713022-4155-420b-ada1-73a863f58282": { "id": "repo_83713022-4155-420b-ada1-73a863f58282", "parent_id": "chain_name_7f4bc842-21b1-4bcb-8ab9-d86e08149548", "repo_name": "SimplyVC/panic_cosmos/", "monitor_repo": "true" } } self.alerts_config_1 = { "1": { "name": "open_file_descriptors", "enabled": "true", "parent_id": "GLOBAL", "critical_threshold": "95", "critical_repeat": "300", "critical_enabled": "true", "warning_threshold": "85", "warning_enabled": "true" }, "2": { "name": "system_cpu_usage", "enabled": "true", "parent_id": "GLOBAL", "critical_threshold": "95", "critical_repeat": "300", "critical_enabled": "true", "warning_threshold": "85", "warning_enabled": "true" }, "3": { "name": "system_storage_usage", "enabled": "true", "parent_id": "GLOBAL", "critical_threshold": "95", "critical_repeat": "300", "critical_enabled": "true", "warning_threshold": "85", "warning_enabled": "true" }, "4": { "name": "system_ram_usage", "enabled": "true", "parent_id": "GLOBAL", "critical_threshold": "95", "critical_repeat": "300", "critical_enabled": "true", "warning_threshold": "85", "warning_enabled": "true" }, "5": { "name": "system_is_down", "enabled": "true", "parent_id": "GLOBAL", "critical_threshold": "200", "critical_repeat": "300", "critical_enabled": "true", "warning_threshold": "0", "warning_enabled": "true" } } self.systems_config_1 = { "system_1d026af1-6cab-403d-8256-c8faa462930a": { "id": "system_1d026af1-6cab-403d-8256-c8faa462930a", "parent_id": "GLOBAL", "name": "matic_full_node_nl(172.26.10.137:9100)", "exporter_url": "http://172.26.10.137:9100/metrics", "monitor_system": "true" }, "system_a51b3a33-cb3f-4f53-a657-8a5a0efe0822": { "id": "system_a51b3a33-cb3f-4f53-a657-8a5a0efe0822", "parent_id": "GLOBAL", "name": "matic_full_node_mt(172.16.152.137:9100)", "exporter_url": "http://172.16.152.137:9100/metrics", "monitor_system": "true" } } self.telegram_config_1 = { "telegram_8431a28e-a2ce-4e9b-839c-299b62e3d5b9": { "id": "telegram_8431a28e-a2ce-4e9b-839c-299b62e3d5b9", "channel_name": "telegram_chat_1", "bot_token": "1277777773:AAF-78AENtsYXxxdqTL3Ip987N7gmIKJaBE", "chat_id": "-759538717", "info": "true", "warning": "true", "critical": "true", "error": "true", "alerts": "false", "commands": "false", "parent_ids": "chain_name_7f4bc842-21b1-4bcb-8ab9-d86e08149548,chain_name_94aafe04-8287-463a-8416-0401852b3ca2,GLOBAL", "parent_names": "cosmos,kusama,GLOBAL" } } self.twilio_config_1 = { "twilio_a7016a6b-9394-4584-abe3-5a5c434b6b7c": { "id": "twilio_a7016a6b-9394-4584-abe3-5a5c434b6b7c", "channel_name": "twilio_caller_main", "account_sid": "ACb77777284e97e49eb2260aada0220e12", "auth_token": "d19f777777a0b8e274470d599e5bcc5e8", "twilio_phone_no": "+19893077770", "twilio_phone_numbers_to_dial_valid": "+35697777380", "parent_ids": "chain_name_7f4bc842-21b1-4bcb-8ab9-d86e08149548,chain_name_94aafe04-8287-463a-8416-0401852b3ca2,GLOBAL", "parent_names": "cosmos,kusama,GLOBAL" } } self.pagerduty_config_1 = { "pagerduty_4092d0ed-ac45-462b-b62a-89cffd4833cc": { "id": "pagerduty_4092d0ed-ac45-462b-b62a-89cffd4833cc", "channel_name": "pager_duty_1", "api_token": "meVp_vyQybcX7dA3o1fS", "integration_key": "4a520ce3577777ad89a3518096f3a5189", "info": "true", "warning": "true", "critical": "true", "error": "true", "parent_ids": "chain_name_7f4bc842-21b1-4bcb-8ab9-d86e08149548,chain_name_94aafe04-8287-463a-8416-0401852b3ca2,GLOBAL", "parent_names": "cosmos,kusama,GLOBAL" } } self.opsgenie_config_1 = { "opsgenie_9550bee1-5880-41f6-bdcf-a289472d7c35": { "id": "opsgenie_9550bee1-5880-41f6-bdcf-a289472d7c35", "channel_name": "ops_genie_main", "api_token": "77777777-0708-4b7e-a46f-496c85fa0b06", "eu": "true", "info": "true", "warning": "true", "critical": "true", "error": "true", "parent_ids": "chain_name_7f4bc842-21b1-4bcb-8ab9-d86e08149548,chain_name_94aafe04-8287-463a-8416-0401852b3ca2,GLOBAL", "parent_names": "cosmos,kusama,GLOBAL" } } self.email_config_1 = { "email_01b23d79-10f5-4815-a11f-034f53974b23": { "id": "email_01b23d79-10f5-4815-a11f-034f53974b23", "channel_name": "main_email_channel", "port": "25", "smtp": "exchange.olive.com", "email_from": "*****@*****.**", "emails_to": "*****@*****.**", "info": "true", "warning": "true", "critical": "true", "error": "true", "parent_ids": "chain_name_7f4bc842-21b1-4bcb-8ab9-d86e08149548,chain_name_94aafe04-8287-463a-8416-0401852b3ca2,GLOBAL", "parent_names": "cosmos,kusama,GLOBAL" } } self.config_data_unexpected = {"unexpected": {}} def tearDown(self) -> None: connect_to_rabbit(self.rabbitmq) delete_queue_if_exists(self.rabbitmq, STORE_CONFIGS_QUEUE_NAME) delete_exchange_if_exists(self.rabbitmq, CONFIG_EXCHANGE) delete_exchange_if_exists(self.rabbitmq, HEALTH_CHECK_EXCHANGE) disconnect_from_rabbit(self.rabbitmq) connect_to_rabbit(self.test_rabbit_manager) delete_queue_if_exists(self.test_rabbit_manager, self.test_queue_name) disconnect_from_rabbit(self.test_rabbit_manager) self.redis.delete_all_unsafe() self.redis = None self.dummy_logger = None self.connection_check_time_interval = None self.rabbitmq = None self.test_rabbit_manager = None def test__str__returns_name_correctly(self) -> None: self.assertEqual(self.test_store_name, str(self.test_store)) def test_name_property_returns_name_correctly(self) -> None: self.assertEqual(self.test_store_name, self.test_store.name) def test_mongo_ip_property_returns_mongo_ip_correctly(self) -> None: self.assertEqual(self.mongo_ip, self.test_store.mongo_ip) def test_mongo_db_property_returns_mongo_db_correctly(self) -> None: self.assertEqual(self.mongo_db, self.test_store.mongo_db) def test_mongo_port_property_returns_mongo_port_correctly(self) -> None: self.assertEqual(self.mongo_port, self.test_store.mongo_port) def test_redis_property_returns_redis_correctly(self) -> None: self.assertEqual(type(self.redis), type(self.test_store.redis)) def test_mongo_property_returns_none_when_mongo_not_init(self) -> None: self.assertEqual(None, self.test_store.mongo) def test_initialise_rabbitmq_initialises_everything_as_expected( self) -> None: try: # To make sure that the exchanges have not already been declared self.rabbitmq.connect() self.rabbitmq.exchange_delete(HEALTH_CHECK_EXCHANGE) self.rabbitmq.exchange_delete(CONFIG_EXCHANGE) self.rabbitmq.disconnect() self.test_store._initialise_rabbitmq() # Perform checks that the connection has been opened, marked as open # and that the delivery confirmation variable is set. self.assertTrue(self.test_store.rabbitmq.is_connected) self.assertTrue(self.test_store.rabbitmq.connection.is_open) self.assertTrue( self.test_store.rabbitmq.channel._delivery_confirmation) # Check whether the producing exchanges have been created by # using passive=True. If this check fails an exception is raised # automatically. self.test_store.rabbitmq.exchange_declare(CONFIG_EXCHANGE, passive=True) self.test_store.rabbitmq.exchange_declare(HEALTH_CHECK_EXCHANGE, passive=True) # Check whether the exchange has been creating by sending messages # to it. If this fails an exception is raised, hence the test fails. self.test_store.rabbitmq.basic_publish_confirm( exchange=HEALTH_CHECK_EXCHANGE, routing_key=self.routing_key, body=self.test_data_str, is_body_dict=False, properties=pika.BasicProperties(delivery_mode=2), mandatory=False) # Check whether the exchange has been creating by sending messages # to it. If this fails an exception is raised, hence the test fails. self.test_store.rabbitmq.basic_publish_confirm( exchange=CONFIG_EXCHANGE, routing_key=STORE_CONFIGS_ROUTING_KEY_CHAINS, body=self.test_data_str, is_body_dict=False, properties=pika.BasicProperties(delivery_mode=2), mandatory=False) # Re-declare queue to get the number of messages res = self.test_store.rabbitmq.queue_declare( STORE_CONFIGS_QUEUE_NAME, False, True, False, False) self.assertEqual(1, res.method.message_count) except Exception as e: self.fail("Test failed: {}".format(e)) @mock.patch("src.data_store.stores.store.RedisApi.hset", autospec=True) def test_process_redis_store_does_nothing_on_error_key(self, mock_hset) -> None: self.test_store._process_redis_store(self.test_parent_id, self.config_data_unexpected) mock_hset.assert_not_called() @parameterized.expand([ ("self.nodes_config_1", "self.routing_key_1"), ("self.alerts_config_1", "self.routing_key_2"), ("self.repos_config_1", "self.routing_key_3"), ("self.repos_config_1", "self.routing_key_4"), ("self.alerts_config_1", "self.routing_key_5"), ("self.systems_config_1", "self.routing_key_6"), ("self.email_config_1", "self.routing_key_7"), ("self.pagerduty_config_1", "self.routing_key_8"), ("self.opsgenie_config_1", "self.routing_key_9"), ("self.telegram_config_1", "self.routing_key_10"), ("self.twilio_config_1", "self.routing_key_11"), ]) @mock.patch("src.data_store.stores.store.RabbitMQApi.basic_ack", autospec=True) @mock.patch("src.data_store.stores.store.Store._send_heartbeat", autospec=True) def test_process_data_saves_in_redis(self, mock_config_data, mock_routing_key, mock_send_hb, mock_ack) -> None: self.rabbitmq.connect() mock_ack.return_value = None try: data = eval(mock_config_data) routing_key = eval(mock_routing_key) self.test_store._initialise_rabbitmq() blocking_channel = self.test_store.rabbitmq.channel method_chains = pika.spec.Basic.Deliver( routing_key=eval(mock_routing_key)) properties = pika.spec.BasicProperties() self.test_store._process_data(blocking_channel, method_chains, properties, json.dumps(data).encode()) mock_ack.assert_called_once() mock_send_hb.assert_called_once() self.assertEqual( data, json.loads( self.redis.get( Keys.get_config(routing_key)).decode("utf-8"))) except Exception as e: self.fail("Test failed: {}".format(e)) @freeze_time("2012-01-01") @mock.patch("src.data_store.stores.store.RabbitMQApi.basic_ack", autospec=True) @mock.patch( "src.data_store.stores.config.ConfigStore._process_redis_store", autospec=True) def test_process_data_sends_heartbeat_correctly(self, mock_process_redis_store, mock_basic_ack) -> None: mock_basic_ack.return_value = None try: self.test_rabbit_manager.connect() self.test_store._initialise_rabbitmq() self.test_rabbit_manager.queue_delete(self.test_queue_name) res = self.test_rabbit_manager.queue_declare( queue=self.test_queue_name, durable=True, exclusive=False, auto_delete=False, passive=False) self.assertEqual(0, res.method.message_count) self.test_rabbit_manager.queue_bind(queue=self.test_queue_name, exchange=HEALTH_CHECK_EXCHANGE, routing_key=self.routing_key) blocking_channel = self.test_store.rabbitmq.channel method_chains = pika.spec.Basic.Deliver( routing_key=self.routing_key_1) properties = pika.spec.BasicProperties() self.test_store._process_data( blocking_channel, method_chains, properties, json.dumps(self.nodes_config_1).encode()) res = self.test_rabbit_manager.queue_declare( queue=self.test_queue_name, durable=True, exclusive=False, auto_delete=False, passive=True) self.assertEqual(1, res.method.message_count) heartbeat_test = { 'component_name': self.test_store_name, 'is_alive': True, 'timestamp': datetime(2012, 1, 1).timestamp() } _, _, body = self.test_rabbit_manager.basic_get( self.test_queue_name) self.assertEqual(heartbeat_test, json.loads(body)) mock_process_redis_store.assert_called_once() except Exception as e: self.fail("Test failed: {}".format(e)) @mock.patch("src.data_store.stores.store.RabbitMQApi.basic_ack", autospec=True) def test_process_data_doesnt_send_heartbeat_on_processing_error( self, mock_basic_ack) -> None: mock_basic_ack.return_value = None try: self.test_rabbit_manager.connect() self.test_store._initialise_rabbitmq() self.test_rabbit_manager.queue_delete(self.test_queue_name) res = self.test_rabbit_manager.queue_declare( queue=self.test_queue_name, durable=True, exclusive=False, auto_delete=False, passive=False) self.assertEqual(0, res.method.message_count) self.test_rabbit_manager.queue_bind(queue=self.test_queue_name, exchange=HEALTH_CHECK_EXCHANGE, routing_key=self.routing_key) blocking_channel = self.test_store.rabbitmq.channel method_chains = pika.spec.Basic.Deliver(routing_key=None) properties = pika.spec.BasicProperties() self.test_store._process_data( blocking_channel, method_chains, properties, json.dumps(self.nodes_config_1).encode()) res = self.test_rabbit_manager.queue_declare( queue=self.test_queue_name, durable=True, exclusive=False, auto_delete=False, passive=True) self.assertEqual(0, res.method.message_count) except Exception as e: self.fail("Test failed: {}".format(e)) @parameterized.expand([ ("self.nodes_config_1", "self.routing_key_1"), ("self.alerts_config_1", "self.routing_key_2"), ("self.repos_config_1", "self.routing_key_3"), ("self.repos_config_1", "self.routing_key_4"), ("self.alerts_config_1", "self.routing_key_5"), ("self.systems_config_1", "self.routing_key_6"), ("self.email_config_1", "self.routing_key_7"), ("self.pagerduty_config_1", "self.routing_key_8"), ("self.opsgenie_config_1", "self.routing_key_9"), ("self.telegram_config_1", "self.routing_key_10"), ("self.twilio_config_1", "self.routing_key_11"), ]) @mock.patch("src.data_store.stores.store.RabbitMQApi.basic_ack", autospec=True) @mock.patch("src.data_store.stores.store.Store._send_heartbeat", autospec=True) def test_process_data_saves_in_redis_then_removes_it_on_empty_config( self, mock_config_data, mock_routing_key, mock_send_hb, mock_ack) -> None: self.rabbitmq.connect() mock_ack.return_value = None try: data = eval(mock_config_data) routing_key = eval(mock_routing_key) self.test_store._initialise_rabbitmq() blocking_channel = self.test_store.rabbitmq.channel method_chains = pika.spec.Basic.Deliver(routing_key=routing_key) properties = pika.spec.BasicProperties() self.test_store._process_data(blocking_channel, method_chains, properties, json.dumps(data).encode()) mock_ack.assert_called_once() mock_send_hb.assert_called_once() self.assertEqual( data, json.loads( self.redis.get( Keys.get_config(routing_key)).decode("utf-8"))) self.test_store._process_data(blocking_channel, method_chains, properties, json.dumps({}).encode()) self.assertEqual(None, self.redis.get(Keys.get_config(routing_key))) except Exception as e: self.fail("Test failed: {}".format(e))
class TestStoreManager(unittest.TestCase): def setUp(self) -> None: self.dummy_logger = logging.getLogger('Dummy') self.dummy_logger.disabled = True self.connection_check_time_interval = timedelta(seconds=0) self.rabbitmq = RabbitMQApi( self.dummy_logger, env.RABBIT_IP, connection_check_time_interval=self.connection_check_time_interval) self.test_rabbit_manager = RabbitMQApi( self.dummy_logger, env.RABBIT_IP, connection_check_time_interval=self.connection_check_time_interval) self.manager_name = 'test_store_manager' self.routing_key = 'heartbeat.manager' self.test_queue_name = 'test queue' self.test_store_manager = StoreManager(self.dummy_logger, self.manager_name, self.rabbitmq) # Adding dummy process self.dummy_process = Process(target=infinite_fn, args=()) self.dummy_process.daemon = True connect_to_rabbit(self.rabbitmq) connect_to_rabbit(self.test_rabbit_manager) self.rabbitmq.exchange_declare(HEALTH_CHECK_EXCHANGE, 'topic', False, True, False, False) self.rabbitmq.queue_declare(DATA_STORE_MAN_INPUT_QUEUE, False, True, False, False) self.test_rabbit_manager.queue_declare(self.test_queue_name, False, True, False, False) self.rabbitmq.queue_bind(DATA_STORE_MAN_INPUT_QUEUE, HEALTH_CHECK_EXCHANGE, DATA_STORE_MAN_INPUT_ROUTING_KEY) self.test_data_str = 'test data' self.test_heartbeat = { 'component_name': self.manager_name, 'is_alive': True, 'timestamp': datetime(2012, 1, 1).timestamp(), } self.test_exception = PANICException('test_exception', 1) def tearDown(self) -> None: connect_to_rabbit(self.rabbitmq) delete_queue_if_exists(self.rabbitmq, DATA_STORE_MAN_INPUT_QUEUE) delete_exchange_if_exists(self.rabbitmq, HEALTH_CHECK_EXCHANGE) disconnect_from_rabbit(self.rabbitmq) connect_to_rabbit(self.test_rabbit_manager) delete_queue_if_exists(self.test_rabbit_manager, self.test_queue_name) disconnect_from_rabbit(self.test_rabbit_manager) self.dummy_logger = None self.dummy_process = None self.connection_check_time_interval = None self.rabbitmq = None self.test_rabbit_manager = None def test__str__returns_name_correctly(self) -> None: self.assertEqual(self.manager_name, str(self.test_store_manager)) def test_name_property_returns_name_correctly(self) -> None: self.assertEqual(self.manager_name, self.test_store_manager.name) def test_logger_property_returns_logger_correctly(self) -> None: self.assertEqual(self.dummy_logger, self.test_store_manager.logger) def test_rabbitmq_property_returns_rabbitmq_correctly(self) -> None: self.assertEqual(self.rabbitmq, self.test_store_manager.rabbitmq) def test_initialise_rabbitmq_initialises_everything_as_expected( self) -> None: try: # To make sure that the exchanges have not already been declared self.rabbitmq.connect() self.rabbitmq.exchange_delete(HEALTH_CHECK_EXCHANGE) self.rabbitmq.disconnect() self.test_store_manager._initialise_rabbitmq() # Perform checks that the connection has been opened, marked as open # and that the delivery confirmation variable is set. self.assertTrue(self.test_store_manager.rabbitmq.is_connected) self.assertTrue( self.test_store_manager.rabbitmq.connection.is_open) self.assertTrue(self.test_store_manager.rabbitmq.channel. _delivery_confirmation) # Check whether the producing exchanges have been created by # using passive=True. If this check fails an exception is raised # automatically. self.test_store_manager.rabbitmq.exchange_declare( HEALTH_CHECK_EXCHANGE, passive=True) self.test_rabbit_manager.queue_declare(self.test_queue_name, False, True, False, False) self.test_rabbit_manager.queue_bind(self.test_queue_name, HEALTH_CHECK_EXCHANGE, self.routing_key) # Check whether the exchange has been creating by sending messages # to it. If this fails an exception is raised, hence the test fails. self.test_store_manager.rabbitmq.basic_publish_confirm( exchange=HEALTH_CHECK_EXCHANGE, routing_key=self.routing_key, body=self.test_data_str, is_body_dict=False, properties=pika.BasicProperties(delivery_mode=2), mandatory=False) res = self.test_rabbit_manager.queue_declare( self.test_queue_name, False, True, False, False) self.assertEqual(1, res.method.message_count) except Exception as e: self.fail("Test failed: {}".format(e)) def test_send_heartbeat_sends_a_heartbeat_correctly(self) -> None: # This test creates a queue which receives messages with the same # routing key as the ones sent by send_heartbeat, and checks that the # heartbeat is received try: self.test_store_manager._initialise_rabbitmq() self.test_rabbit_manager.connect() res = self.test_rabbit_manager.queue_declare( queue=self.test_queue_name, durable=True, exclusive=False, auto_delete=False, passive=False) self.assertEqual(0, res.method.message_count) self.test_store_manager.rabbitmq.queue_bind( queue=self.test_queue_name, exchange=HEALTH_CHECK_EXCHANGE, routing_key=self.routing_key) self.test_store_manager._send_heartbeat(self.test_heartbeat) # By re-declaring the queue again we can get the number of messages # in the queue. res = self.test_rabbit_manager.queue_declare( queue=self.test_queue_name, durable=True, exclusive=False, auto_delete=False, passive=True) self.assertEqual(1, res.method.message_count) # Check that the message received is actually the HB _, _, body = self.test_rabbit_manager.basic_get( self.test_queue_name) self.assertEqual(self.test_heartbeat, json.loads(body)) except Exception as e: self.fail("Test failed: {}".format(e)) @mock.patch.object(multiprocessing.Process, "start") def test_start_stores_processes_starts_system_store_correctly( self, mock_start) -> None: mock_start.return_value = None self.test_store_manager._start_stores_processes() new_entry_process = self.test_store_manager._store_process_dict[ SYSTEM_STORE_NAME] self.assertTrue(new_entry_process.daemon) self.assertEqual(0, len(new_entry_process._args)) self.assertEqual(start_system_store, new_entry_process._target) @mock.patch.object(multiprocessing.Process, "start") def test_start_stores_processes_starts_github_store_correctly( self, mock_start) -> None: mock_start.return_value = None self.test_store_manager._start_stores_processes() new_entry_process = self.test_store_manager._store_process_dict[ GITHUB_STORE_NAME] self.assertTrue(new_entry_process.daemon) self.assertEqual(0, len(new_entry_process._args)) self.assertEqual(start_github_store, new_entry_process._target) @mock.patch.object(multiprocessing.Process, "start") def test_start_stores_processes_starts_alert_store_correctly( self, mock_start) -> None: mock_start.return_value = None self.test_store_manager._start_stores_processes() new_entry_process = self.test_store_manager._store_process_dict[ ALERT_STORE_NAME] self.assertTrue(new_entry_process.daemon) self.assertEqual(0, len(new_entry_process._args)) self.assertEqual(start_alert_store, new_entry_process._target) @mock.patch.object(multiprocessing.Process, "start") def test_start_stores_processes_starts_config_store_correctly( self, mock_start) -> None: mock_start.return_value = None self.test_store_manager._start_stores_processes() new_entry_process = self.test_store_manager._store_process_dict[ CONFIG_STORE_NAME] self.assertTrue(new_entry_process.daemon) self.assertEqual(0, len(new_entry_process._args)) self.assertEqual(start_config_store, new_entry_process._target) @mock.patch("src.data_store.starters.create_logger") def test_start_stores_processes_starts_the_processes_correctly( self, mock_create_logger) -> None: mock_create_logger.return_value = self.dummy_logger self.test_store_manager._start_stores_processes() # We need to sleep to give some time for the stores to be initialised, # otherwise the process would not terminate time.sleep(1) new_system_process = self.test_store_manager._store_process_dict[ SYSTEM_STORE_NAME] self.assertTrue(new_system_process.is_alive()) new_system_process.terminate() new_system_process.join() new_github_process = self.test_store_manager._store_process_dict[ GITHUB_STORE_NAME] self.assertTrue(new_github_process.is_alive()) new_github_process.terminate() new_github_process.join() new_alert_process = self.test_store_manager._store_process_dict[ ALERT_STORE_NAME] self.assertTrue(new_alert_process.is_alive()) new_alert_process.terminate() new_alert_process.join() new_config_process = self.test_store_manager._store_process_dict[ CONFIG_STORE_NAME] self.assertTrue(new_config_process.is_alive()) new_config_process.terminate() new_config_process.join() @freeze_time("2012-01-01") @mock.patch("src.data_store.starters.create_logger") @mock.patch.object(RabbitMQApi, "basic_ack") def test_process_ping_sends_a_valid_hb_if_process_is_alive( self, mock_ack, mock_create_logger) -> None: # This test creates a queue which receives messages with the same # routing key as the ones sent by send_heartbeat, and checks that the # received heartbeat is valid. mock_create_logger.return_value = self.dummy_logger mock_ack.return_value = None try: self.test_store_manager._initialise_rabbitmq() self.test_store_manager._start_stores_processes() # Give time for the processes to start time.sleep(1) self.test_rabbit_manager.queue_declare(queue=self.test_queue_name, durable=True, exclusive=False, auto_delete=False, passive=False) # Delete the queue before to avoid messages in the queue on error. self.test_rabbit_manager.queue_delete(self.test_queue_name) # initialise blocking_channel = self.test_store_manager.rabbitmq.channel properties = pika.spec.BasicProperties() method_hb = pika.spec.Basic.Deliver(routing_key=self.routing_key) body = 'ping' res = self.test_rabbit_manager.queue_declare( queue=self.test_queue_name, durable=True, exclusive=False, auto_delete=False, passive=False) self.assertEqual(0, res.method.message_count) self.test_rabbit_manager.queue_bind(queue=self.test_queue_name, exchange=HEALTH_CHECK_EXCHANGE, routing_key=self.routing_key) self.test_store_manager._process_ping(blocking_channel, method_hb, properties, body) time.sleep(1) # By re-declaring the queue again we can get the number of messages # in the queue. res = self.test_rabbit_manager.queue_declare( queue=self.test_queue_name, durable=True, exclusive=False, auto_delete=False, passive=True) self.assertEqual(1, res.method.message_count) expected_output = { "component_name": self.manager_name, "dead_processes": [], "running_processes": [ SYSTEM_STORE_NAME, GITHUB_STORE_NAME, ALERT_STORE_NAME, CONFIG_STORE_NAME ], "timestamp": datetime(2012, 1, 1).timestamp() } # Check that the message received is a valid HB _, _, body = self.test_rabbit_manager.basic_get( self.test_queue_name) self.assertEqual(expected_output, json.loads(body)) # Clean before test finishes self.test_store_manager._store_process_dict[ SYSTEM_STORE_NAME].terminate() self.test_store_manager._store_process_dict[ GITHUB_STORE_NAME].terminate() self.test_store_manager._store_process_dict[ ALERT_STORE_NAME].terminate() self.test_store_manager._store_process_dict[ CONFIG_STORE_NAME].terminate() self.test_store_manager._store_process_dict[ SYSTEM_STORE_NAME].join() self.test_store_manager._store_process_dict[ GITHUB_STORE_NAME].join() self.test_store_manager._store_process_dict[ALERT_STORE_NAME].join( ) self.test_store_manager._store_process_dict[ CONFIG_STORE_NAME].join() except Exception as e: self.fail("Test failed: {}".format(e)) @freeze_time("2012-01-01") @mock.patch("src.data_store.starters.create_logger") @mock.patch.object(RabbitMQApi, "basic_ack") def test_process_ping_sends_a_valid_hb_if_all_processes_are_dead( self, mock_ack, mock_create_logger) -> None: # This test creates a queue which receives messages with the same # routing key as the ones sent by send_heartbeat, and checks that the # received heartbeat is valid. mock_create_logger.return_value = self.dummy_logger mock_ack.return_value = None try: self.test_store_manager._initialise_rabbitmq() self.test_store_manager._start_stores_processes() # Give time for the processes to start time.sleep(1) self.test_store_manager._store_process_dict[ SYSTEM_STORE_NAME].terminate() self.test_store_manager._store_process_dict[ GITHUB_STORE_NAME].terminate() self.test_store_manager._store_process_dict[ ALERT_STORE_NAME].terminate() self.test_store_manager._store_process_dict[ CONFIG_STORE_NAME].terminate() self.test_store_manager._store_process_dict[ SYSTEM_STORE_NAME].join() self.test_store_manager._store_process_dict[ GITHUB_STORE_NAME].join() self.test_store_manager._store_process_dict[ALERT_STORE_NAME].join( ) self.test_store_manager._store_process_dict[ CONFIG_STORE_NAME].join() # Time for processes to terminate time.sleep(1) # Delete the queue before to avoid messages in the queue on error. self.test_rabbit_manager.queue_delete(self.test_queue_name) # initialise blocking_channel = self.test_store_manager.rabbitmq.channel properties = pika.spec.BasicProperties() method_hb = pika.spec.Basic.Deliver(routing_key=self.routing_key) body = 'ping' res = self.test_rabbit_manager.queue_declare( queue=self.test_queue_name, durable=True, exclusive=False, auto_delete=False, passive=False) self.assertEqual(0, res.method.message_count) self.test_rabbit_manager.queue_bind(queue=self.test_queue_name, exchange=HEALTH_CHECK_EXCHANGE, routing_key=self.routing_key) self.test_store_manager._process_ping(blocking_channel, method_hb, properties, body) time.sleep(1) # By re-declaring the queue again we can get the number of messages # in the queue. res = self.test_rabbit_manager.queue_declare( queue=self.test_queue_name, durable=True, exclusive=False, auto_delete=False, passive=True) self.assertEqual(1, res.method.message_count) expected_output = { "component_name": self.manager_name, "dead_processes": [ SYSTEM_STORE_NAME, GITHUB_STORE_NAME, ALERT_STORE_NAME, CONFIG_STORE_NAME ], "running_processes": [], "timestamp": datetime(2012, 1, 1).timestamp() } # Check that the message received is a valid HB _, _, body = self.test_rabbit_manager.basic_get( self.test_queue_name) self.assertEqual(expected_output, json.loads(body)) # Clean before test finishes self.test_store_manager._store_process_dict[ SYSTEM_STORE_NAME].terminate() self.test_store_manager._store_process_dict[ GITHUB_STORE_NAME].terminate() self.test_store_manager._store_process_dict[ ALERT_STORE_NAME].terminate() self.test_store_manager._store_process_dict[ CONFIG_STORE_NAME].terminate() self.test_store_manager._store_process_dict[ SYSTEM_STORE_NAME].join() self.test_store_manager._store_process_dict[ GITHUB_STORE_NAME].join() self.test_store_manager._store_process_dict[ALERT_STORE_NAME].join( ) self.test_store_manager._store_process_dict[ CONFIG_STORE_NAME].join() self.rabbitmq.disconnect() except Exception as e: self.fail("Test failed: {}".format(e)) @freeze_time("2012-01-01") @mock.patch.object(RabbitMQApi, "basic_ack") @mock.patch("src.data_store.starters.create_logger") @mock.patch.object(StoreManager, "_send_heartbeat") def test_process_ping_restarts_dead_processes(self, send_hb_mock, mock_create_logger, mock_ack) -> None: send_hb_mock.return_value = None mock_create_logger.return_value = self.dummy_logger mock_ack.return_value = None try: self.test_store_manager._initialise_rabbitmq() self.test_store_manager._start_stores_processes() # Give time for the processes to start time.sleep(1) self.test_store_manager._store_process_dict[ SYSTEM_STORE_NAME].terminate() self.test_store_manager._store_process_dict[ GITHUB_STORE_NAME].terminate() self.test_store_manager._store_process_dict[ ALERT_STORE_NAME].terminate() self.test_store_manager._store_process_dict[ CONFIG_STORE_NAME].terminate() self.test_store_manager._store_process_dict[ SYSTEM_STORE_NAME].join() self.test_store_manager._store_process_dict[ GITHUB_STORE_NAME].join() self.test_store_manager._store_process_dict[ALERT_STORE_NAME].join( ) self.test_store_manager._store_process_dict[ CONFIG_STORE_NAME].join() # Give time for the processes to terminate time.sleep(1) # Check that that the processes have terminated self.assertFalse(self.test_store_manager. _store_process_dict[SYSTEM_STORE_NAME].is_alive()) self.assertFalse(self.test_store_manager. _store_process_dict[GITHUB_STORE_NAME].is_alive()) self.assertFalse(self.test_store_manager. _store_process_dict[ALERT_STORE_NAME].is_alive()) self.assertFalse(self.test_store_manager. _store_process_dict[CONFIG_STORE_NAME].is_alive()) # initialise blocking_channel = self.test_store_manager.rabbitmq.channel properties = pika.spec.BasicProperties() method_hb = pika.spec.Basic.Deliver(routing_key=self.routing_key) body = 'ping' self.test_store_manager._process_ping(blocking_channel, method_hb, properties, body) # Give time for the processes to start time.sleep(1) self.assertTrue(self.test_store_manager. _store_process_dict[SYSTEM_STORE_NAME].is_alive()) self.assertTrue(self.test_store_manager. _store_process_dict[GITHUB_STORE_NAME].is_alive()) self.assertTrue(self.test_store_manager. _store_process_dict[ALERT_STORE_NAME].is_alive()) self.assertTrue(self.test_store_manager. _store_process_dict[CONFIG_STORE_NAME].is_alive()) # Clean before test finishes self.test_store_manager._store_process_dict[ SYSTEM_STORE_NAME].terminate() self.test_store_manager._store_process_dict[ GITHUB_STORE_NAME].terminate() self.test_store_manager._store_process_dict[ ALERT_STORE_NAME].terminate() self.test_store_manager._store_process_dict[ CONFIG_STORE_NAME].terminate() self.test_store_manager._store_process_dict[ SYSTEM_STORE_NAME].join() self.test_store_manager._store_process_dict[ GITHUB_STORE_NAME].join() self.test_store_manager._store_process_dict[ALERT_STORE_NAME].join( ) self.test_store_manager._store_process_dict[ CONFIG_STORE_NAME].join() except Exception as e: self.fail("Test failed: {}".format(e)) @mock.patch.object(multiprocessing.Process, "is_alive") @mock.patch.object(multiprocessing.Process, "start") @mock.patch.object(multiprocessing, 'Process') def test_process_ping_does_not_send_hb_if_processing_fails( self, mock_process, mock_start, is_alive_mock) -> None: # This test creates a queue which receives messages with the same # routing key as the ones sent by send_heartbeat. In this test we will # check that no heartbeat is sent when mocking a raised exception. is_alive_mock.side_effect = self.test_exception mock_start.return_value = None mock_process.side_effect = self.dummy_process try: self.test_store_manager._initialise_rabbitmq() self.test_store_manager._start_stores_processes() time.sleep(1) # Delete the queue before to avoid messages in the queue on error. self.test_rabbit_manager.queue_delete(self.test_queue_name) # initialise blocking_channel = self.test_store_manager.rabbitmq.channel method = pika.spec.Basic.Deliver(routing_key=self.routing_key) properties = pika.spec.BasicProperties() body = 'ping' res = self.test_rabbit_manager.queue_declare( queue=self.test_queue_name, durable=True, exclusive=False, auto_delete=False, passive=False) self.assertEqual(0, res.method.message_count) self.test_rabbit_manager.queue_bind(queue=self.test_queue_name, exchange=HEALTH_CHECK_EXCHANGE, routing_key=self.routing_key) self.test_store_manager._process_ping(blocking_channel, method, properties, body) time.sleep(1) # By re-declaring the queue again we can get the number of messages # in the queue. res = self.test_rabbit_manager.queue_declare( queue=self.test_queue_name, durable=True, exclusive=False, auto_delete=False, passive=True) self.assertEqual(0, res.method.message_count) except Exception as e: self.fail("Test failed: {}".format(e)) def test_proc_ping_send_hb_does_not_raise_msg_not_del_exce_if_hb_not_routed( self) -> None: try: self.test_store_manager._initialise_rabbitmq() self.test_store_manager._start_stores_processes() time.sleep(1) # initialise blocking_channel = self.test_store_manager.rabbitmq.channel method = pika.spec.Basic.Deliver(routing_key='heartbeat.manager') properties = pika.spec.BasicProperties() body = 'ping' self.test_store_manager._process_ping(blocking_channel, method, properties, body) except Exception as e: self.fail("Test failed: {}".format(e)) @parameterized.expand([("pika.exceptions.AMQPChannelError('test')", "pika.exceptions.AMQPChannelError"), ("self.test_exception", "PANICException"), ("pika.exceptions.AMQPConnectionError", "pika.exceptions.AMQPConnectionError")]) @mock.patch.object(StoreManager, "_send_heartbeat") def test_process_ping_send_hb_raises_exceptions(self, param_input, param_expected, hb_mock) -> None: hb_mock.side_effect = eval(param_input) try: self.test_store_manager._initialise_rabbitmq() # initialise blocking_channel = self.test_store_manager.rabbitmq.channel method = pika.spec.Basic.Deliver(routing_key=self.routing_key) properties = pika.spec.BasicProperties() body = 'ping' self.assertRaises(eval(param_expected), self.test_store_manager._process_ping, blocking_channel, method, properties, body) except Exception as e: self.fail("Test failed: {}".format(e))
class TestGithubStore(unittest.TestCase): def setUp(self) -> None: self.dummy_logger = logging.getLogger('Dummy') self.dummy_logger.disabled = True self.connection_check_time_interval = timedelta(seconds=0) self.rabbit_ip = env.RABBIT_IP self.rabbitmq = RabbitMQApi( self.dummy_logger, self.rabbit_ip, connection_check_time_interval=self.connection_check_time_interval) self.test_rabbit_manager = RabbitMQApi( self.dummy_logger, self.rabbit_ip, connection_check_time_interval=self.connection_check_time_interval) self.redis_db = env.REDIS_DB self.redis_host = env.REDIS_IP self.redis_port = env.REDIS_PORT self.redis_namespace = env.UNIQUE_ALERTER_IDENTIFIER self.redis = RedisApi(self.dummy_logger, self.redis_db, self.redis_host, self.redis_port, '', self.redis_namespace, self.connection_check_time_interval) self.mongo_ip = env.DB_IP self.mongo_db = env.DB_NAME self.mongo_port = env.DB_PORT self.test_store_name = 'store name' self.test_store = GithubStore(self.test_store_name, self.dummy_logger, self.rabbitmq) self.routing_key = 'heartbeat.worker' self.test_queue_name = 'test queue' connect_to_rabbit(self.rabbitmq) self.rabbitmq.exchange_declare(HEALTH_CHECK_EXCHANGE, 'topic', False, True, False, False) self.rabbitmq.exchange_declare(STORE_EXCHANGE, 'direct', False, True, False, False) self.rabbitmq.queue_declare(GITHUB_STORE_INPUT_QUEUE, False, True, False, False) self.rabbitmq.queue_bind(GITHUB_STORE_INPUT_QUEUE, STORE_EXCHANGE, GITHUB_STORE_INPUT_ROUTING_KEY) connect_to_rabbit(self.test_rabbit_manager) self.test_rabbit_manager.queue_declare(self.test_queue_name, False, True, False, False) self.test_rabbit_manager.queue_bind(self.test_queue_name, HEALTH_CHECK_EXCHANGE, self.routing_key) self.test_data_str = 'test data' self.test_exception = PANICException('test_exception', 1) self.repo_name = 'simplyvc/panic/' self.repo_id = 'test_repo_id' self.parent_id = 'test_parent_id' self.repo_name_2 = 'simplyvc/panic_oasis/' self.repo_id_2 = 'test_repo_id_2' self.parent_id_2 = 'test_parent_id_2' self.last_monitored = datetime(2012, 1, 1).timestamp() self.github_data_1 = { "result": { "meta_data": { "repo_name": self.repo_name, "repo_id": self.repo_id, "repo_parent_id": self.parent_id, "last_monitored": self.last_monitored }, "data": { "no_of_releases": { "current": 5, "previous": 4, } } } } self.github_data_2 = { "result": { "meta_data": { "repo_name": self.repo_name, "repo_id": self.repo_id, "repo_parent_id": self.parent_id, "last_monitored": self.last_monitored }, "data": { "no_of_releases": { "current": 5, "previous": 5, } } } } self.github_data_3 = { "result": { "meta_data": { "repo_name": self.repo_name_2, "repo_id": self.repo_id_2, "repo_parent_id": self.parent_id_2, "last_monitored": self.last_monitored }, "data": { "no_of_releases": { "current": 8, "previous": 1, } } } } self.github_data_error = { "error": { "meta_data": { "repo_name": self.repo_name, "repo_id": self.repo_id, "repo_parent_id": self.parent_id, "time": self.last_monitored }, "code": "5006", "message": "error message" } } self.github_data_key_error = { "result": { "data": { "repo_name": self.repo_name_2, "repo_id": self.repo_id_2, "repo_parent_id": self.parent_id_2, "last_monitored": self.last_monitored }, "wrong_data": { "no_of_releases": { "current": 8, "previous": 1, } } } } self.github_data_unexpected = {"unexpected": {}} def tearDown(self) -> None: connect_to_rabbit(self.rabbitmq) delete_queue_if_exists(self.rabbitmq, GITHUB_STORE_INPUT_QUEUE) delete_exchange_if_exists(self.rabbitmq, STORE_EXCHANGE) delete_exchange_if_exists(self.rabbitmq, HEALTH_CHECK_EXCHANGE) disconnect_from_rabbit(self.rabbitmq) connect_to_rabbit(self.test_rabbit_manager) delete_queue_if_exists(self.test_rabbit_manager, self.test_queue_name) disconnect_from_rabbit(self.test_rabbit_manager) self.redis.delete_all_unsafe() self.redis = None self.dummy_logger = None self.connection_check_time_interval = None self.rabbitmq = None self.test_rabbit_manager = None def test__str__returns_name_correctly(self) -> None: self.assertEqual(self.test_store_name, str(self.test_store)) def test_name_property_returns_name_correctly(self) -> None: self.assertEqual(self.test_store_name, self.test_store.name) def test_mongo_ip_property_returns_mongo_ip_correctly(self) -> None: self.assertEqual(self.mongo_ip, self.test_store.mongo_ip) def test_mongo_db_property_returns_mongo_db_correctly(self) -> None: self.assertEqual(self.mongo_db, self.test_store.mongo_db) def test_mongo_port_property_returns_mongo_port_correctly(self) -> None: self.assertEqual(self.mongo_port, self.test_store.mongo_port) def test_redis_property_returns_redis_correctly(self) -> None: self.assertEqual(type(self.redis), type(self.test_store.redis)) def test_mongo_property_returns_none_when_mongo_not_init(self) -> None: self.assertEqual(None, self.test_store.mongo) def test_initialise_rabbitmq_initialises_everything_as_expected( self) -> None: try: # To make sure that the exchanges have not already been declared self.rabbitmq.connect() self.rabbitmq.exchange_delete(HEALTH_CHECK_EXCHANGE) self.rabbitmq.exchange_delete(STORE_EXCHANGE) self.rabbitmq.disconnect() self.test_store._initialise_rabbitmq() # Perform checks that the connection has been opened, marked as open # and that the delivery confirmation variable is set. self.assertTrue(self.test_store.rabbitmq.is_connected) self.assertTrue(self.test_store.rabbitmq.connection.is_open) self.assertTrue( self.test_store.rabbitmq.channel._delivery_confirmation) # Check whether the producing exchanges have been created by # using passive=True. If this check fails an exception is raised # automatically. self.test_store.rabbitmq.exchange_declare(STORE_EXCHANGE, passive=True) self.test_store.rabbitmq.exchange_declare(HEALTH_CHECK_EXCHANGE, passive=True) # Check whether the exchange has been creating by sending messages # to it. If this fails an exception is raised, hence the test fails. self.test_store.rabbitmq.basic_publish_confirm( exchange=HEALTH_CHECK_EXCHANGE, routing_key=self.routing_key, body=self.test_data_str, is_body_dict=False, properties=pika.BasicProperties(delivery_mode=2), mandatory=False) # Check whether the exchange has been creating by sending messages # to it. If this fails an exception is raised, hence the test fails. self.test_store.rabbitmq.basic_publish_confirm( exchange=STORE_EXCHANGE, routing_key=GITHUB_STORE_INPUT_ROUTING_KEY, body=self.test_data_str, is_body_dict=False, properties=pika.BasicProperties(delivery_mode=2), mandatory=False) # Re-declare queue to get the number of messages res = self.test_store.rabbitmq.queue_declare( GITHUB_STORE_INPUT_QUEUE, False, True, False, False) self.assertEqual(1, res.method.message_count) except Exception as e: self.fail("Test failed: {}".format(e)) @parameterized.expand([ ("self.github_data_1", ), ("self.github_data_2", ), ("self.github_data_3", ), ]) @mock.patch.object(RedisApi, "hset_multiple") def test_process_redis_store_redis_is_called_correctly( self, mock_github_data, mock_hset_multiple) -> None: data = eval(mock_github_data) self.test_store._process_redis_store(data) meta_data = data['result']['meta_data'] repo_id = meta_data['repo_id'] parent_id = meta_data['repo_parent_id'] metrics = data['result']['data'] call_1 = call( Keys.get_hash_parent(parent_id), { Keys.get_github_no_of_releases(repo_id): str(metrics['no_of_releases']), Keys.get_github_last_monitored(repo_id): str(meta_data['last_monitored']), }) mock_hset_multiple.assert_has_calls([call_1]) @mock.patch("src.data_store.stores.store.RedisApi.hset_multiple", autospec=True) def test_process_redis_store_does_nothing_on_error_key( self, mock_hset_multiple) -> None: self.test_store._process_redis_store(self.github_data_error) mock_hset_multiple.assert_not_called() def test_process_redis_store_raises_exception_on_unexpected_key( self) -> None: self.assertRaises(ReceivedUnexpectedDataException, self.test_store._process_redis_store, self.github_data_unexpected) @parameterized.expand([ ("self.github_data_1", ), ("self.github_data_2", ), ("self.github_data_3", ), ]) def test_process_redis_store_redis_stores_correctly( self, mock_github_data) -> None: data = eval(mock_github_data) self.test_store._process_redis_store(data) meta_data = data['result']['meta_data'] repo_id = meta_data['repo_id'] parent_id = meta_data['repo_parent_id'] metrics = data['result']['data'] self.assertEqual( str(metrics['no_of_releases']), self.redis.hget( Keys.get_hash_parent(parent_id), Keys.get_github_no_of_releases(repo_id)).decode("utf-8")) self.assertEqual( str(meta_data['last_monitored']), self.redis.hget( Keys.get_hash_parent(parent_id), Keys.get_github_last_monitored(repo_id)).decode("utf-8")) @parameterized.expand([ ("self.github_data_1", ), ("self.github_data_2", ), ("self.github_data_3", ), ]) @mock.patch("src.data_store.stores.store.RabbitMQApi.basic_ack", autospec=True) @mock.patch("src.data_store.stores.store.Store._send_heartbeat", autospec=True) def test_process_data_saves_in_redis(self, mock_github_data, mock_send_hb, mock_ack) -> None: self.rabbitmq.connect() mock_ack.return_value = None try: self.test_store._initialise_rabbitmq() data = eval(mock_github_data) blocking_channel = self.test_store.rabbitmq.channel method_chains = pika.spec.Basic.Deliver( routing_key=GITHUB_STORE_INPUT_ROUTING_KEY) properties = pika.spec.BasicProperties() self.test_store._process_data(blocking_channel, method_chains, properties, json.dumps(data).encode()) mock_ack.assert_called_once() mock_send_hb.assert_called_once() meta_data = data['result']['meta_data'] repo_id = meta_data['repo_id'] parent_id = meta_data['repo_parent_id'] metrics = data['result']['data'] self.assertEqual( str(metrics['no_of_releases']), self.redis.hget( Keys.get_hash_parent(parent_id), Keys.get_github_no_of_releases(repo_id)).decode("utf-8")) self.assertEqual( str(meta_data['last_monitored']), self.redis.hget( Keys.get_hash_parent(parent_id), Keys.get_github_last_monitored(repo_id)).decode("utf-8")) except Exception as e: self.fail("Test failed: {}".format(e)) @parameterized.expand([ ("KeyError", "self.github_data_key_error "), ("ReceivedUnexpectedDataException", "self.github_data_unexpected"), ]) @mock.patch("src.data_store.stores.store.RabbitMQApi.basic_ack", autospec=True) @mock.patch("src.data_store.stores.store.Store._send_heartbeat", autospec=True) def test_process_data_with_bad_data_does_raises_exceptions( self, mock_error, mock_bad_data, mock_send_hb, mock_ack) -> None: self.rabbitmq.connect() mock_ack.return_value = None try: self.test_store._initialise_rabbitmq() blocking_channel = self.test_store.rabbitmq.channel method_chains = pika.spec.Basic.Deliver( routing_key=GITHUB_STORE_INPUT_ROUTING_KEY) properties = pika.spec.BasicProperties() self.test_store._process_data( blocking_channel, method_chains, properties, json.dumps(self.github_data_unexpected).encode()) self.assertRaises(eval(mock_error), self.test_store._process_redis_store, eval(mock_bad_data)) mock_ack.assert_called_once() mock_send_hb.assert_not_called() except Exception as e: self.fail("Test failed: {}".format(e)) @freeze_time("2012-01-01") @mock.patch("src.data_store.stores.store.RabbitMQApi.basic_ack", autospec=True) @mock.patch( "src.data_store.stores.github.GithubStore._process_redis_store", autospec=True) def test_process_data_sends_heartbeat_correctly(self, mock_process_redis_store, mock_basic_ack) -> None: mock_basic_ack.return_value = None try: self.test_rabbit_manager.connect() self.test_store._initialise_rabbitmq() self.test_rabbit_manager.queue_delete(self.test_queue_name) res = self.test_rabbit_manager.queue_declare( queue=self.test_queue_name, durable=True, exclusive=False, auto_delete=False, passive=False) self.assertEqual(0, res.method.message_count) self.test_rabbit_manager.queue_bind(queue=self.test_queue_name, exchange=HEALTH_CHECK_EXCHANGE, routing_key=self.routing_key) blocking_channel = self.test_store.rabbitmq.channel method_chains = pika.spec.Basic.Deliver( routing_key=GITHUB_STORE_INPUT_ROUTING_KEY) properties = pika.spec.BasicProperties() self.test_store._process_data( blocking_channel, method_chains, properties, json.dumps(self.github_data_1).encode()) res = self.test_rabbit_manager.queue_declare( queue=self.test_queue_name, durable=True, exclusive=False, auto_delete=False, passive=True) self.assertEqual(1, res.method.message_count) heartbeat_test = { 'component_name': self.test_store_name, 'is_alive': True, 'timestamp': datetime(2012, 1, 1).timestamp() } _, _, body = self.test_rabbit_manager.basic_get( self.test_queue_name) self.assertEqual(heartbeat_test, json.loads(body)) mock_process_redis_store.assert_called_once() except Exception as e: self.fail("Test failed: {}".format(e)) @mock.patch("src.data_store.stores.store.RabbitMQApi.basic_ack", autospec=True) def test_process_data_doesnt_send_heartbeat_on_processing_error( self, mock_basic_ack) -> None: mock_basic_ack.return_value = None try: self.test_rabbit_manager.connect() self.test_store._initialise_rabbitmq() self.test_rabbit_manager.queue_delete(self.test_queue_name) res = self.test_rabbit_manager.queue_declare( queue=self.test_queue_name, durable=True, exclusive=False, auto_delete=False, passive=False) self.assertEqual(0, res.method.message_count) self.test_rabbit_manager.queue_bind(queue=self.test_queue_name, exchange=HEALTH_CHECK_EXCHANGE, routing_key=self.routing_key) blocking_channel = self.test_store.rabbitmq.channel method_chains = pika.spec.Basic.Deliver( routing_key=GITHUB_STORE_INPUT_ROUTING_KEY) properties = pika.spec.BasicProperties() self.test_store._process_data( blocking_channel, method_chains, properties, json.dumps(self.github_data_unexpected).encode()) res = self.test_rabbit_manager.queue_declare( queue=self.test_queue_name, durable=True, exclusive=False, auto_delete=False, passive=True) self.assertEqual(0, res.method.message_count) except Exception as e: self.fail("Test failed: {}".format(e))
class TestSystemMonitor(unittest.TestCase): def setUp(self) -> None: self.dummy_logger = logging.getLogger('Dummy') self.dummy_logger.disabled = True self.connection_check_time_interval = timedelta(seconds=0) self.rabbit_ip = env.RABBIT_IP self.rabbitmq = RabbitMQApi( self.dummy_logger, self.rabbit_ip, connection_check_time_interval=self.connection_check_time_interval) self.monitor_name = 'test_monitor' self.monitoring_period = 10 self.system_id = 'test_system_id' self.parent_id = 'test_parent_id' self.system_name = 'test_system' self.monitor_system = True self.node_exporter_url = 'test_url' self.routing_key = 'test_routing_key' self.test_data_str = 'test data' self.test_data_dict = { 'test_key_1': 'test_val_1', 'test_key_2': 'test_val_2', } self.test_heartbeat = { 'component_name': 'Test Component', 'is_alive': True, 'timestamp': datetime(2012, 1, 1).timestamp(), } self.test_queue_name = 'Test Queue' self.metrics_to_monitor = [ 'process_cpu_seconds_total', 'go_memstats_alloc_bytes', 'go_memstats_alloc_bytes_total', 'process_virtual_memory_bytes', 'process_max_fds', 'process_open_fds', 'node_cpu_seconds_total', 'node_filesystem_avail_bytes', 'node_filesystem_size_bytes', 'node_memory_MemTotal_bytes', 'node_memory_MemAvailable_bytes', 'node_network_transmit_bytes_total', 'node_network_receive_bytes_total', 'node_disk_io_time_seconds_total'] self.retrieved_metrics_example = { 'go_memstats_alloc_bytes': 2003024.0, 'go_memstats_alloc_bytes_total': 435777412600.0, 'node_cpu_seconds_total': { '{"cpu": "0", "mode": "idle"}': 3626110.54, '{"cpu": "0", "mode": "iowait"}': 16892.07, '{"cpu": "0", "mode": "irq"}': 0.0, '{"cpu": "0", "mode": "nice"}': 131.77, '{"cpu": "0", "mode": "softirq"}': 8165.66, '{"cpu": "0", "mode": "steal"}': 0.0, '{"cpu": "0", "mode": "system"}': 46168.15, '{"cpu": "0", "mode": "user"}': 238864.68, '{"cpu": "1", "mode": "idle"}': 3630087.24, '{"cpu": "1", "mode": "iowait"}': 17084.42, '{"cpu": "1", "mode": "irq"}': 0.0, '{"cpu": "1", "mode": "nice"}': 145.18, '{"cpu": "1", "mode": "softirq"}': 5126.93, '{"cpu": "1", "mode": "steal"}': 0.0, '{"cpu": "1", "mode": "system"}': 46121.4, '{"cpu": "1", "mode": "user"}': 239419.51}, 'node_disk_io_time_seconds_total': { '{"device": "dm-0"}': 38359.0, '{"device": "sda"}': 38288.0, '{"device": "sr0"}': 0.0}, 'node_filesystem_avail_bytes': { '{"device": "/dev/mapper/ubuntu--vg-ubuntu--lv", ' '"fstype": "ext4", "mountpoint": "/"}': 57908170752.0, '{"device": "/dev/sda2", "fstype": "ext4", ' '"mountpoint": "/boot"}': 729411584.0, '{"device": "lxcfs", "fstype": "fuse.lxcfs", ' '"mountpoint": "/var/lib/lxcfs"}': 0.0, '{"device": "tmpfs", "fstype": "tmpfs", ' '"mountpoint": "/run"}': 207900672.0, '{"device": "tmpfs", "fstype": "tmpfs", "mountpoint": ' '"/run/lock"}': 5242880.0}, 'node_filesystem_size_bytes': { '{"device": "/dev/mapper/ubuntu--vg-ubuntu--lv", "fstype": ' '"ext4", "mountpoint": "/"}': 104560844800.0, '{"device": "/dev/sda2", "fstype": "ext4", "mountpoint": ' '"/boot"}': 1023303680.0, '{"device": "lxcfs", "fstype": "fuse.lxcfs", "mountpoint": ' '"/var/lib/lxcfs"}': 0.0, '{"device": "tmpfs", "fstype": "tmpfs", "mountpoint": "/run"}': 209027072.0, '{"device": "tmpfs", "fstype": "tmpfs", "mountpoint": ' '"/run/lock"}': 5242880.0}, 'node_memory_MemAvailable_bytes': 1377767424.0, 'node_memory_MemTotal_bytes': 2090237952.0, 'node_network_receive_bytes_total': { '{"device": "ens160"}': 722358765622.0, '{"device": "lo"}': 381405.0}, 'node_network_transmit_bytes_total': { '{"device": "ens160"}': 1011571824152.0, '{"device": "lo"}': 381405.0}, 'process_cpu_seconds_total': 2786.82, 'process_max_fds': 1024.0, 'process_open_fds': 8.0, 'process_virtual_memory_bytes': 118513664.0} self.processed_data_example = { 'process_cpu_seconds_total': 2786.82, 'process_memory_usage': 0.0, 'virtual_memory_usage': 118513664.0, 'open_file_descriptors': 0.78125, 'system_cpu_usage': 7.85, 'system_ram_usage': 34.09, 'system_storage_usage': 44.37, 'network_transmit_bytes_total': 1011572205557.0, 'network_receive_bytes_total': 722359147027.0, 'disk_io_time_seconds_total': 76647.0, } self.test_exception = PANICException('test_exception', 1) self.system_config = SystemConfig(self.system_id, self.parent_id, self.system_name, self.monitor_system, self.node_exporter_url) self.test_monitor = SystemMonitor(self.monitor_name, self.system_config, self.dummy_logger, self.monitoring_period, self.rabbitmq) def tearDown(self) -> None: # Delete any queues and exchanges which are common across many tests try: self.test_monitor.rabbitmq.connect() # Declare them before just in case there are tests which do not # use these queues and exchanges self.test_monitor.rabbitmq.queue_declare( queue=self.test_queue_name, durable=True, exclusive=False, auto_delete=False, passive=False ) self.test_monitor.rabbitmq.exchange_declare( HEALTH_CHECK_EXCHANGE, 'topic', False, True, False, False) self.test_monitor.rabbitmq.exchange_declare( RAW_DATA_EXCHANGE, 'direct', False, True, False, False) self.test_monitor.rabbitmq.queue_purge(self.test_queue_name) self.test_monitor.rabbitmq.queue_delete(self.test_queue_name) self.test_monitor.rabbitmq.exchange_delete(RAW_DATA_EXCHANGE) self.test_monitor.rabbitmq.exchange_delete(HEALTH_CHECK_EXCHANGE) self.test_monitor.rabbitmq.disconnect() except Exception as e: print("Deletion of queues and exchanges failed: {}".format(e)) self.dummy_logger = None self.rabbitmq = None self.test_exception = None self.system_config = None self.test_monitor = None def test_str_returns_monitor_name(self) -> None: self.assertEqual(self.monitor_name, str(self.test_monitor)) def test_get_monitor_period_returns_monitor_period(self) -> None: self.assertEqual(self.monitoring_period, self.test_monitor.monitor_period) def test_get_monitor_name_returns_monitor_name(self) -> None: self.assertEqual(self.monitor_name, self.test_monitor.monitor_name) def test_system_config_returns_system_config(self) -> None: self.assertEqual(self.system_config, self.test_monitor.system_config) def test_metrics_to_monitor_returns_metrics_to_monitor(self) -> None: self.assertEqual(self.metrics_to_monitor, self.test_monitor.metrics_to_monitor) def test_initialise_rabbitmq_initialises_everything_as_expected( self) -> None: try: # To make sure that there is no connection/channel already # established self.assertIsNone(self.rabbitmq.connection) self.assertIsNone(self.rabbitmq.channel) # To make sure that the exchanges have not already been declared self.rabbitmq.connect() self.rabbitmq.exchange_delete(RAW_DATA_EXCHANGE) self.rabbitmq.exchange_delete(HEALTH_CHECK_EXCHANGE) self.rabbitmq.disconnect() self.test_monitor._initialise_rabbitmq() # Perform checks that the connection has been opened, marked as open # and that the delivery confirmation variable is set. self.assertTrue(self.test_monitor.rabbitmq.is_connected) self.assertTrue(self.test_monitor.rabbitmq.connection.is_open) self.assertTrue( self.test_monitor.rabbitmq.channel._delivery_confirmation) # Check whether the exchange has been creating by sending messages # to it. If this fails an exception is raised hence the test fails. self.test_monitor.rabbitmq.basic_publish_confirm( exchange=RAW_DATA_EXCHANGE, routing_key=self.routing_key, body=self.test_data_str, is_body_dict=False, properties=pika.BasicProperties(delivery_mode=2), mandatory=False) self.test_monitor.rabbitmq.basic_publish_confirm( exchange=HEALTH_CHECK_EXCHANGE, routing_key=self.routing_key, body=self.test_data_str, is_body_dict=False, properties=pika.BasicProperties(delivery_mode=2), mandatory=False) except Exception as e: self.fail("Test failed: {}".format(e)) @mock.patch.object(SystemMonitor, "_process_retrieved_data") @mock.patch.object(SystemMonitor, "_process_error") def test_process_data_calls_process_error_on_retrieval_error( self, mock_process_error, mock_process_retrieved_data) -> None: # Do not test the processing of data for now mock_process_error.return_value = self.test_data_dict self.test_monitor._process_data(self.test_data_dict, True, self.test_exception) # Test passes if _process_error is called once and # process_retrieved_data is not called self.assertEqual(1, mock_process_error.call_count) self.assertEqual(0, mock_process_retrieved_data.call_count) @mock.patch.object(SystemMonitor, "_process_retrieved_data") @mock.patch.object(SystemMonitor, "_process_error") def test_process_data_calls_process_retrieved_data_on_retrieval_success( self, mock_process_error, mock_process_retrieved_data) -> None: # Do not test the processing of data for now mock_process_retrieved_data.return_value = self.test_data_dict self.test_monitor._process_data(self.test_data_dict, False, None) # Test passes if _process_error is called once and # process_retrieved_data is not called self.assertEqual(0, mock_process_error.call_count) self.assertEqual(1, mock_process_retrieved_data.call_count) def test_send_heartbeat_sends_a_heartbeat_correctly(self) -> None: # This test creates a queue which receives messages with the same # routing key as the ones sent by send_heartbeat, and checks that the # heartbeat is received try: self.test_monitor._initialise_rabbitmq() # Delete the queue before to avoid messages in the queue on error. self.test_monitor.rabbitmq.queue_delete(self.test_queue_name) res = self.test_monitor.rabbitmq.queue_declare( queue=self.test_queue_name, durable=True, exclusive=False, auto_delete=False, passive=False ) self.assertEqual(0, res.method.message_count) self.test_monitor.rabbitmq.queue_bind( queue=self.test_queue_name, exchange=HEALTH_CHECK_EXCHANGE, routing_key='heartbeat.worker') self.test_monitor._send_heartbeat(self.test_heartbeat) # By re-declaring the queue again we can get the number of messages # in the queue. res = self.test_monitor.rabbitmq.queue_declare( queue=self.test_queue_name, durable=True, exclusive=False, auto_delete=False, passive=True ) self.assertEqual(1, res.method.message_count) # Check that the message received is actually the HB _, _, body = self.test_monitor.rabbitmq.basic_get( self.test_queue_name) self.assertEqual(self.test_heartbeat, json.loads(body)) except Exception as e: self.fail("Test failed: {}".format(e)) def test_display_data_returns_the_correct_string(self) -> None: expected_output = \ "process_cpu_seconds_total={}, process_memory_usage={}, " \ "virtual_memory_usage={}, open_file_descriptors={}, " \ "system_cpu_usage={}, system_ram_usage={}, " \ "system_storage_usage={}, network_transmit_bytes_total={}, " \ "network_receive_bytes_total={}, disk_io_time_seconds_total={}" \ "".format(self.processed_data_example['process_cpu_seconds_total'], self.processed_data_example['process_memory_usage'], self.processed_data_example['virtual_memory_usage'], self.processed_data_example['open_file_descriptors'], self.processed_data_example['system_cpu_usage'], self.processed_data_example['system_ram_usage'], self.processed_data_example['system_storage_usage'], self.processed_data_example[ 'network_transmit_bytes_total'], self.processed_data_example[ 'network_receive_bytes_total'], self.processed_data_example['disk_io_time_seconds_total']) actual_output = self.test_monitor._display_data( self.processed_data_example) self.assertEqual(expected_output, actual_output) @freeze_time("2012-01-01") def test_process_error_returns_expected_data(self) -> None: expected_output = { 'error': { 'meta_data': { 'monitor_name': self.test_monitor.monitor_name, 'system_name': self.test_monitor.system_config.system_name, 'system_id': self.test_monitor.system_config.system_id, 'system_parent_id': self.test_monitor.system_config.parent_id, 'time': datetime(2012, 1, 1).timestamp() }, 'message': self.test_exception.message, 'code': self.test_exception.code, } } actual_output = self.test_monitor._process_error(self.test_exception) self.assertEqual(actual_output, expected_output) @freeze_time("2012-01-01") def test_process_retrieved_data_returns_expected_data(self) -> None: expected_output = { 'result': { 'meta_data': { 'monitor_name': self.test_monitor.monitor_name, 'system_name': self.test_monitor.system_config.system_name, 'system_id': self.test_monitor.system_config.system_id, 'system_parent_id': self.test_monitor.system_config.parent_id, 'time': datetime(2012, 1, 1).timestamp() }, 'data': self.processed_data_example, } } actual_output = self.test_monitor._process_retrieved_data( self.retrieved_metrics_example) self.assertEqual(expected_output, actual_output) def test_send_data_sends_data_correctly(self) -> None: # This test creates a queue which receives messages with the same # routing key as the ones sent by send_data, and checks that the # data is received try: self.test_monitor._initialise_rabbitmq() # Delete the queue before to avoid messages in the queue on error. self.test_monitor.rabbitmq.queue_delete(self.test_queue_name) res = self.test_monitor.rabbitmq.queue_declare( queue=self.test_queue_name, durable=True, exclusive=False, auto_delete=False, passive=False ) self.assertEqual(0, res.method.message_count) self.test_monitor.rabbitmq.queue_bind( queue=self.test_queue_name, exchange=RAW_DATA_EXCHANGE, routing_key='system') self.test_monitor._send_data(self.processed_data_example) # By re-declaring the queue again we can get the number of messages # in the queue. res = self.test_monitor.rabbitmq.queue_declare( queue=self.test_queue_name, durable=True, exclusive=False, auto_delete=False, passive=True ) self.assertEqual(1, res.method.message_count) # Check that the message received is actually the processed data _, _, body = self.test_monitor.rabbitmq.basic_get( self.test_queue_name) self.assertEqual(self.processed_data_example, json.loads(body)) except Exception as e: self.fail("Test failed: {}".format(e)) @freeze_time("2012-01-01") @mock.patch.object(SystemMonitor, "_get_data") def test_monitor_sends_data_and_hb_if_data_retrieve_and_processing_success( self, mock_get_data) -> None: expected_output_data = { 'result': { 'meta_data': { 'monitor_name': self.test_monitor.monitor_name, 'system_name': self.test_monitor.system_config.system_name, 'system_id': self.test_monitor.system_config.system_id, 'system_parent_id': self.test_monitor.system_config.parent_id, 'time': datetime(2012, 1, 1).timestamp() }, 'data': self.processed_data_example, } } expected_output_hb = { 'component_name': self.test_monitor.monitor_name, 'is_alive': True, 'timestamp': datetime(2012, 1, 1).timestamp() } try: mock_get_data.return_value = self.retrieved_metrics_example self.test_monitor._initialise_rabbitmq() # Delete the queue before to avoid messages in the queue on error. self.test_monitor.rabbitmq.queue_delete(self.test_queue_name) res = self.test_monitor.rabbitmq.queue_declare( queue=self.test_queue_name, durable=True, exclusive=False, auto_delete=False, passive=False ) self.assertEqual(0, res.method.message_count) self.test_monitor.rabbitmq.queue_bind( queue=self.test_queue_name, exchange=RAW_DATA_EXCHANGE, routing_key='system') self.test_monitor.rabbitmq.queue_bind( queue=self.test_queue_name, exchange=HEALTH_CHECK_EXCHANGE, routing_key='heartbeat.worker') self.test_monitor._monitor() # By re-declaring the queue again we can get the number of messages # in the queue. res = self.test_monitor.rabbitmq.queue_declare( queue=self.test_queue_name, durable=True, exclusive=False, auto_delete=False, passive=True ) # There must be 2 messages in the queue, the heartbeat and the # processed data self.assertEqual(2, res.method.message_count) # Check that the message received is actually the processed data _, _, body = self.test_monitor.rabbitmq.basic_get( self.test_queue_name) self.assertEqual(expected_output_data, json.loads(body)) # Check that the message received is actually the HB _, _, body = self.test_monitor.rabbitmq.basic_get( self.test_queue_name) self.assertEqual(expected_output_hb, json.loads(body)) except Exception as e: self.fail("Test failed: {}".format(e)) @mock.patch.object(SystemMonitor, "_process_data") @mock.patch.object(SystemMonitor, "_get_data") def test_monitor_sends_no_data_and_hb_if_data_ret_success_and_proc_fails( self, mock_get_data, mock_process_data) -> None: mock_process_data.side_effect = self.test_exception mock_get_data.return_value = self.retrieved_metrics_example try: self.test_monitor._initialise_rabbitmq() # Delete the queue before to avoid messages in the queue on error. self.test_monitor.rabbitmq.queue_delete(self.test_queue_name) res = self.test_monitor.rabbitmq.queue_declare( queue=self.test_queue_name, durable=True, exclusive=False, auto_delete=False, passive=False ) self.assertEqual(0, res.method.message_count) self.test_monitor.rabbitmq.queue_bind( queue=self.test_queue_name, exchange=RAW_DATA_EXCHANGE, routing_key='system') self.test_monitor.rabbitmq.queue_bind( queue=self.test_queue_name, exchange=HEALTH_CHECK_EXCHANGE, routing_key='heartbeat.worker') self.test_monitor._monitor() # By re-declaring the queue again we can get the number of messages # in the queue. res = self.test_monitor.rabbitmq.queue_declare( queue=self.test_queue_name, durable=True, exclusive=False, auto_delete=False, passive=True ) # There must be 0 messages in the queue. self.assertEqual(0, res.method.message_count) except Exception as e: self.fail("Test failed: {}".format(e)) @mock.patch.object(SystemMonitor, "_get_data") def test_monitor_sends_no_data_and_no_hb_on_get_data_unexpected_exception( self, mock_get_data) -> None: mock_get_data.side_effect = self.test_exception try: self.test_monitor._initialise_rabbitmq() # Delete the queue before to avoid messages in the queue on error. self.test_monitor.rabbitmq.queue_delete(self.test_queue_name) res = self.test_monitor.rabbitmq.queue_declare( queue=self.test_queue_name, durable=True, exclusive=False, auto_delete=False, passive=False ) self.assertEqual(0, res.method.message_count) self.test_monitor.rabbitmq.queue_bind( queue=self.test_queue_name, exchange=RAW_DATA_EXCHANGE, routing_key='system') self.test_monitor.rabbitmq.queue_bind( queue=self.test_queue_name, exchange=HEALTH_CHECK_EXCHANGE, routing_key='heartbeat.worker') self.assertRaises(PANICException, self.test_monitor._monitor) # By re-declaring the queue again we can get the number of messages # in the queue. res = self.test_monitor.rabbitmq.queue_declare( queue=self.test_queue_name, durable=True, exclusive=False, auto_delete=False, passive=True ) # There must be 0 messages in the queue. self.assertEqual(0, res.method.message_count) except Exception as e: self.fail("Test failed: {}".format(e)) @freeze_time("2012-01-01") @mock.patch.object(SystemMonitor, "_get_data") def test_monitor_sends_exception_data_and_hb_on_expected_exceptions( self, mock_get_data) -> None: errors_exceptions_dict = { ReqConnectionError('test'): SystemIsDownException( self.test_monitor.system_config.system_name), ReadTimeout('test'): SystemIsDownException( self.test_monitor.system_config.system_name), IncompleteRead('test'): DataReadingException( self.test_monitor.monitor_name, self.test_monitor.system_config.system_name), ChunkedEncodingError('test'): DataReadingException( self.test_monitor.monitor_name, self.test_monitor.system_config.system_name), ProtocolError('test'): DataReadingException( self.test_monitor.monitor_name, self.test_monitor.system_config.system_name), InvalidURL('test'): InvalidUrlException( self.test_monitor.system_config.node_exporter_url), InvalidSchema('test'): InvalidUrlException( self.test_monitor.system_config.node_exporter_url), MissingSchema('test'): InvalidUrlException( self.test_monitor.system_config.node_exporter_url), MetricNotFoundException('test_metric', 'test_endpoint'): MetricNotFoundException('test_metric', 'test_endpoint') } try: self.test_monitor._initialise_rabbitmq() for error, data_ret_exception in errors_exceptions_dict.items(): mock_get_data.side_effect = error expected_output_data = { 'error': { 'meta_data': { 'monitor_name': self.test_monitor.monitor_name, 'system_name': self.test_monitor.system_config.system_name, 'system_id': self.test_monitor.system_config.system_id, 'system_parent_id': self.test_monitor.system_config.parent_id, 'time': datetime(2012, 1, 1).timestamp() }, 'message': data_ret_exception.message, 'code': data_ret_exception.code, } } expected_output_hb = { 'component_name': self.test_monitor.monitor_name, 'is_alive': True, 'timestamp': datetime(2012, 1, 1).timestamp() } # Delete the queue before to avoid messages in the queue on # error. self.test_monitor.rabbitmq.queue_delete(self.test_queue_name) res = self.test_monitor.rabbitmq.queue_declare( queue=self.test_queue_name, durable=True, exclusive=False, auto_delete=False, passive=False ) self.assertEqual(0, res.method.message_count) self.test_monitor.rabbitmq.queue_bind( queue=self.test_queue_name, exchange=RAW_DATA_EXCHANGE, routing_key='system') self.test_monitor.rabbitmq.queue_bind( queue=self.test_queue_name, exchange=HEALTH_CHECK_EXCHANGE, routing_key='heartbeat.worker') self.test_monitor._monitor() # By re-declaring the queue again we can get the number of # messages in the queue. res = self.test_monitor.rabbitmq.queue_declare( queue=self.test_queue_name, durable=True, exclusive=False, auto_delete=False, passive=True ) # There must be 2 messages in the queue, the heartbeat and the # processed data self.assertEqual(2, res.method.message_count) # Check that the message received is actually the processed data _, _, body = self.test_monitor.rabbitmq.basic_get( self.test_queue_name) self.assertEqual(expected_output_data, json.loads(body)) # Check that the message received is actually the HB _, _, body = self.test_monitor.rabbitmq.basic_get( self.test_queue_name) self.assertEqual(expected_output_hb, json.loads(body)) except Exception as e: self.fail("Test failed: {}".format(e)) @mock.patch.object(SystemMonitor, "_get_data") def test_monitor_raises_msg_not_delivered_exception_if_data_not_routed( self, mock_get_data) -> None: mock_get_data.return_value = self.retrieved_metrics_example try: self.test_monitor._initialise_rabbitmq() self.assertRaises(MessageWasNotDeliveredException, self.test_monitor._monitor) except Exception as e: self.fail("Test failed: {}".format(e)) @freeze_time("2012-01-01") @mock.patch.object(SystemMonitor, "_get_data") def test_monitor_raises_msg_not_del_except_if_hb_not_routed_and_sends_data( self, mock_get_data) -> None: mock_get_data.return_value = self.retrieved_metrics_example expected_output_data = { 'result': { 'meta_data': { 'monitor_name': self.test_monitor.monitor_name, 'system_name': self.test_monitor.system_config.system_name, 'system_id': self.test_monitor.system_config.system_id, 'system_parent_id': self.test_monitor.system_config.parent_id, 'time': datetime(2012, 1, 1).timestamp() }, 'data': self.processed_data_example, } } try: self.test_monitor._initialise_rabbitmq() self.test_monitor.rabbitmq.queue_delete(self.test_queue_name) res = self.test_monitor.rabbitmq.queue_declare( queue=self.test_queue_name, durable=True, exclusive=False, auto_delete=False, passive=False ) self.assertEqual(0, res.method.message_count) self.test_monitor.rabbitmq.queue_bind( queue=self.test_queue_name, exchange=RAW_DATA_EXCHANGE, routing_key='system') self.assertRaises(MessageWasNotDeliveredException, self.test_monitor._monitor) # By re-declaring the queue again we can get the number of # messages in the queue. res = self.test_monitor.rabbitmq.queue_declare( queue=self.test_queue_name, durable=True, exclusive=False, auto_delete=False, passive=True ) # There must be 1 message in the queue, the processed data self.assertEqual(1, res.method.message_count) # Check that the message received is actually the processed data _, _, body = self.test_monitor.rabbitmq.basic_get( self.test_queue_name) self.assertEqual(expected_output_data, json.loads(body)) except Exception as e: self.fail("Test failed: {}".format(e)) @mock.patch.object(SystemMonitor, "_send_data") @mock.patch.object(SystemMonitor, "_get_data") def test_monitor_send_data_raises_amqp_channel_error_on_channel_error( self, mock_get_data, mock_send_data) -> None: mock_get_data.return_value = self.retrieved_metrics_example mock_send_data.side_effect = pika.exceptions.AMQPChannelError('test') try: self.test_monitor._initialise_rabbitmq() self.assertRaises(pika.exceptions.AMQPChannelError, self.test_monitor._monitor) except Exception as e: self.fail("Test failed: {}".format(e)) @freeze_time("2012-01-01") @mock.patch.object(SystemMonitor, "_send_heartbeat") @mock.patch.object(SystemMonitor, "_get_data") def test_monitor_send_hb_raises_amqp_chan_err_on_chan_err_and_sends_data( self, mock_get_data, mock_send_heartbeat) -> None: mock_get_data.return_value = self.retrieved_metrics_example mock_send_heartbeat.side_effect = \ pika.exceptions.AMQPChannelError('test') expected_output_data = { 'result': { 'meta_data': { 'monitor_name': self.test_monitor.monitor_name, 'system_name': self.test_monitor.system_config.system_name, 'system_id': self.test_monitor.system_config.system_id, 'system_parent_id': self.test_monitor.system_config.parent_id, 'time': datetime(2012, 1, 1).timestamp() }, 'data': self.processed_data_example, } } try: self.test_monitor._initialise_rabbitmq() self.test_monitor.rabbitmq.queue_delete(self.test_queue_name) res = self.test_monitor.rabbitmq.queue_declare( queue=self.test_queue_name, durable=True, exclusive=False, auto_delete=False, passive=False ) self.assertEqual(0, res.method.message_count) self.test_monitor.rabbitmq.queue_bind( queue=self.test_queue_name, exchange=HEALTH_CHECK_EXCHANGE, routing_key='heartbeat.worker') self.test_monitor.rabbitmq.queue_bind( queue=self.test_queue_name, exchange=RAW_DATA_EXCHANGE, routing_key='system') self.assertRaises(pika.exceptions.AMQPChannelError, self.test_monitor._monitor) # By re-declaring the queue again we can get the number of # messages in the queue. res = self.test_monitor.rabbitmq.queue_declare( queue=self.test_queue_name, durable=True, exclusive=False, auto_delete=False, passive=True ) # There must be 1 message in the queue, the processed data self.assertEqual(1, res.method.message_count) # Check that the message received is actually the processed data _, _, body = self.test_monitor.rabbitmq.basic_get( self.test_queue_name) self.assertEqual(expected_output_data, json.loads(body)) except Exception as e: self.fail("Test failed: {}".format(e)) @mock.patch.object(SystemMonitor, "_send_data") @mock.patch.object(SystemMonitor, "_get_data") def test_monitor_send_data_raises_amqp_conn_error_on_conn_error( self, mock_get_data, mock_send_data) -> None: mock_get_data.return_value = self.retrieved_metrics_example mock_send_data.side_effect = pika.exceptions.AMQPConnectionError('test') try: self.test_monitor._initialise_rabbitmq() self.assertRaises(pika.exceptions.AMQPConnectionError, self.test_monitor._monitor) except Exception as e: self.fail("Test failed: {}".format(e)) @freeze_time("2012-01-01") @mock.patch.object(SystemMonitor, "_send_heartbeat") @mock.patch.object(SystemMonitor, "_get_data") def test_monitor_send_hb_raises_amqp_conn_err_on_conn_err_and_sends_data( self, mock_get_data, mock_send_heartbeat) -> None: mock_get_data.return_value = self.retrieved_metrics_example mock_send_heartbeat.side_effect = \ pika.exceptions.AMQPConnectionError('test') expected_output_data = { 'result': { 'meta_data': { 'monitor_name': self.test_monitor.monitor_name, 'system_name': self.test_monitor.system_config.system_name, 'system_id': self.test_monitor.system_config.system_id, 'system_parent_id': self.test_monitor.system_config.parent_id, 'time': datetime(2012, 1, 1).timestamp() }, 'data': self.processed_data_example, } } try: self.test_monitor._initialise_rabbitmq() self.test_monitor.rabbitmq.queue_delete(self.test_queue_name) res = self.test_monitor.rabbitmq.queue_declare( queue=self.test_queue_name, durable=True, exclusive=False, auto_delete=False, passive=False ) self.assertEqual(0, res.method.message_count) self.test_monitor.rabbitmq.queue_bind( queue=self.test_queue_name, exchange=HEALTH_CHECK_EXCHANGE, routing_key='heartbeat.worker') self.test_monitor.rabbitmq.queue_bind( queue=self.test_queue_name, exchange=RAW_DATA_EXCHANGE, routing_key='system') self.assertRaises(pika.exceptions.AMQPConnectionError, self.test_monitor._monitor) # By re-declaring the queue again we can get the number of # messages in the queue. res = self.test_monitor.rabbitmq.queue_declare( queue=self.test_queue_name, durable=True, exclusive=False, auto_delete=False, passive=True ) # There must be 1 message in the queue, the processed data self.assertEqual(1, res.method.message_count) # Check that the message received is actually the processed data _, _, body = self.test_monitor.rabbitmq.basic_get( self.test_queue_name) self.assertEqual(expected_output_data, json.loads(body)) except Exception as e: self.fail("Test failed: {}".format(e)) @mock.patch.object(SystemMonitor, "_send_data") @mock.patch.object(SystemMonitor, "_get_data") def test_monitor_does_not_send_hb_and_data_if_send_data_fails( self, mock_get_data, mock_send_data) -> None: mock_get_data.return_value = self.retrieved_metrics_example exception_types_dict = \ { Exception('test'): Exception, pika.exceptions.AMQPConnectionError('test'): pika.exceptions.AMQPConnectionError, pika.exceptions.AMQPChannelError('test'): pika.exceptions.AMQPChannelError, MessageWasNotDeliveredException('test'): MessageWasNotDeliveredException } try: self.test_monitor._initialise_rabbitmq() for exception, exception_type in exception_types_dict.items(): mock_send_data.side_effect = exception self.test_monitor.rabbitmq.queue_delete( self.test_queue_name) res = self.test_monitor.rabbitmq.queue_declare( queue=self.test_queue_name, durable=True, exclusive=False, auto_delete=False, passive=False ) self.assertEqual(0, res.method.message_count) self.test_monitor.rabbitmq.queue_bind( queue=self.test_queue_name, exchange=HEALTH_CHECK_EXCHANGE, routing_key='heartbeat.worker') self.test_monitor.rabbitmq.queue_bind( queue=self.test_queue_name, exchange=RAW_DATA_EXCHANGE, routing_key='system') self.assertRaises(exception_type, self.test_monitor._monitor) # By re-declaring the queue again we can get the number of # messages in the queue. res = self.test_monitor.rabbitmq.queue_declare( queue=self.test_queue_name, durable=True, exclusive=False, auto_delete=False, passive=True ) # There must be no messages in the queue. self.assertEqual(0, res.method.message_count) except Exception as e: self.fail("Test failed: {}".format(e))
class TestAlertStore(unittest.TestCase): def setUp(self) -> None: self.dummy_logger = logging.getLogger('Dummy') self.dummy_logger.disabled = True self.connection_check_time_interval = timedelta(seconds=0) self.rabbit_ip = env.RABBIT_IP self.rabbitmq = RabbitMQApi( self.dummy_logger, self.rabbit_ip, connection_check_time_interval=self.connection_check_time_interval) self.test_rabbit_manager = RabbitMQApi( self.dummy_logger, self.rabbit_ip, connection_check_time_interval=self.connection_check_time_interval) self.mongo_ip = env.DB_IP self.mongo_db = env.DB_NAME self.mongo_port = env.DB_PORT self.mongo = MongoApi(logger=self.dummy_logger.getChild( MongoApi.__name__), db_name=self.mongo_db, host=self.mongo_ip, port=self.mongo_port) self.redis_db = env.REDIS_DB self.redis_host = env.REDIS_IP self.redis_port = env.REDIS_PORT self.redis_namespace = env.UNIQUE_ALERTER_IDENTIFIER self.redis = RedisApi(self.dummy_logger, self.redis_db, self.redis_host, self.redis_port, '', self.redis_namespace, self.connection_check_time_interval) self.test_store_name = 'store name' self.test_store = AlertStore(self.test_store_name, self.dummy_logger, self.rabbitmq) self.routing_key = 'heartbeat.worker' self.test_queue_name = 'test queue' connect_to_rabbit(self.rabbitmq) self.rabbitmq.exchange_declare(HEALTH_CHECK_EXCHANGE, 'topic', False, True, False, False) self.rabbitmq.exchange_declare(STORE_EXCHANGE, 'direct', False, True, False, False) self.rabbitmq.queue_declare(ALERT_STORE_INPUT_QUEUE, False, True, False, False) self.rabbitmq.queue_bind(ALERT_STORE_INPUT_QUEUE, STORE_EXCHANGE, ALERT_STORE_INPUT_ROUTING_KEY) connect_to_rabbit(self.test_rabbit_manager) self.test_rabbit_manager.queue_declare(self.test_queue_name, False, True, False, False) self.test_rabbit_manager.queue_bind(self.test_queue_name, HEALTH_CHECK_EXCHANGE, self.routing_key) self.test_data_str = 'test data' self.test_exception = PANICException('test_exception', 1) self.parent_id = 'test_parent_id' self.alert_id = 'test_alert_id' self.origin_id = 'test_origin_id' self.alert_name = 'test_alert' self.metric = 'system_is_down' self.severity = 'warning' self.message = 'alert message' self.value = 'alert_code_1' self.alert_id_2 = 'test_alert_id_2' self.origin_id_2 = 'test_origin_id_2' self.alert_name_2 = 'test_alert_2' self.severity_2 = 'critical' self.message_2 = 'alert message 2' self.value_2 = 'alert_code_2' self.alert_id_3 = 'test_alert_id_3' self.origin_id_3 = 'test_origin_id_3' self.alert_name_3 = 'test_alert_3' self.severity_3 = 'info' self.message_3 = 'alert message 3' self.value_3 = 'alert_code_3' self.last_monitored = datetime(2012, 1, 1).timestamp() self.none = None self.alert_data_1 = { 'parent_id': self.parent_id, 'origin_id': self.origin_id, 'alert_code': { 'name': self.alert_name, 'value': self.value, }, 'severity': self.severity, 'metric': self.metric, 'message': self.message, 'timestamp': self.last_monitored, } self.alert_data_2 = { 'parent_id': self.parent_id, 'origin_id': self.origin_id_2, 'alert_code': { 'name': self.alert_name_2, 'value': self.value_2, }, 'severity': self.severity_2, 'metric': self.metric, 'message': self.message_2, 'timestamp': self.last_monitored, } self.alert_data_3 = { 'parent_id': self.parent_id, 'origin_id': self.origin_id_3, 'alert_code': { 'name': self.alert_name_3, 'value': self.value_3, }, 'severity': self.severity_3, 'metric': self.metric, 'message': self.message_3, 'timestamp': self.last_monitored, } self.alert_data_key_error = {"result": {"data": {}, "data2": {}}} self.alert_data_unexpected = {"unexpected": {}} def tearDown(self) -> None: connect_to_rabbit(self.rabbitmq) delete_queue_if_exists(self.rabbitmq, ALERT_STORE_INPUT_QUEUE) delete_exchange_if_exists(self.rabbitmq, STORE_EXCHANGE) delete_exchange_if_exists(self.rabbitmq, HEALTH_CHECK_EXCHANGE) disconnect_from_rabbit(self.rabbitmq) connect_to_rabbit(self.test_rabbit_manager) delete_queue_if_exists(self.test_rabbit_manager, self.test_queue_name) disconnect_from_rabbit(self.test_rabbit_manager) self.dummy_logger = None self.connection_check_time_interval = None self.rabbitmq = None self.test_rabbit_manager = None self.redis.delete_all_unsafe() self.redis = None self.mongo.drop_collection(self.parent_id) self.mongo = None self.test_store = None def test__str__returns_name_correctly(self) -> None: self.assertEqual(self.test_store_name, str(self.test_store)) def test_name_property_returns_name_correctly(self) -> None: self.assertEqual(self.test_store_name, self.test_store.name) def test_mongo_ip_property_returns_mongo_ip_correctly(self) -> None: self.assertEqual(self.mongo_ip, self.test_store.mongo_ip) def test_mongo_db_property_returns_mongo_db_correctly(self) -> None: self.assertEqual(self.mongo_db, self.test_store.mongo_db) def test_mongo_port_property_returns_mongo_port_correctly(self) -> None: self.assertEqual(self.mongo_port, self.test_store.mongo_port) def test_mongo_property_returns_mongo(self) -> None: self.assertEqual(type(self.mongo), type(self.test_store.mongo)) def test_redis_property_returns_redis_correctly(self) -> None: self.assertEqual(type(self.redis), type(self.test_store.redis)) def test_initialise_rabbitmq_initialises_everything_as_expected( self) -> None: try: # To make sure that the exchanges have not already been declared self.rabbitmq.connect() self.rabbitmq.queue_delete(ALERT_STORE_INPUT_QUEUE) self.test_rabbit_manager.queue_delete(self.test_queue_name) self.rabbitmq.exchange_delete(HEALTH_CHECK_EXCHANGE) self.rabbitmq.exchange_delete(STORE_EXCHANGE) self.rabbitmq.disconnect() self.test_store._initialise_rabbitmq() # Perform checks that the connection has been opened, marked as open # and that the delivery confirmation variable is set. self.assertTrue(self.test_store.rabbitmq.is_connected) self.assertTrue(self.test_store.rabbitmq.connection.is_open) self.assertTrue( self.test_store.rabbitmq.channel._delivery_confirmation) # Check whether the producing exchanges have been created by # using passive=True. If this check fails an exception is raised # automatically. self.test_store.rabbitmq.exchange_declare(STORE_EXCHANGE, passive=True) self.test_store.rabbitmq.exchange_declare(HEALTH_CHECK_EXCHANGE, passive=True) # Check whether the exchange has been creating by sending messages # to it. If this fails an exception is raised, hence the test fails. self.test_store.rabbitmq.basic_publish_confirm( exchange=HEALTH_CHECK_EXCHANGE, routing_key=self.routing_key, body=self.test_data_str, is_body_dict=False, properties=pika.BasicProperties(delivery_mode=2), mandatory=False) # Check whether the exchange has been creating by sending messages # to it. If this fails an exception is raised, hence the test fails. self.test_store.rabbitmq.basic_publish_confirm( exchange=STORE_EXCHANGE, routing_key=ALERT_STORE_INPUT_ROUTING_KEY, body=self.test_data_str, is_body_dict=False, properties=pika.BasicProperties(delivery_mode=2), mandatory=False) # Re-declare queue to get the number of messages res = self.test_store.rabbitmq.queue_declare( ALERT_STORE_INPUT_QUEUE, False, True, False, False) self.assertEqual(1, res.method.message_count) except Exception as e: self.fail("Test failed: {}".format(e)) @parameterized.expand([ ("KeyError", "self.alert_data_key_error "), ]) @mock.patch("src.data_store.stores.store.RabbitMQApi.basic_ack", autospec=True) @mock.patch("src.data_store.stores.store.Store._send_heartbeat", autospec=True) def test_process_data_with_bad_data_does_raises_exceptions( self, mock_error, mock_bad_data, mock_send_hb, mock_ack) -> None: mock_ack.return_value = None try: self.test_store._initialise_rabbitmq() blocking_channel = self.test_store.rabbitmq.channel method_chains = pika.spec.Basic.Deliver( routing_key=ALERT_STORE_INPUT_ROUTING_KEY) properties = pika.spec.BasicProperties() self.test_store._process_data( blocking_channel, method_chains, properties, json.dumps(self.alert_data_unexpected).encode()) self.assertRaises(eval(mock_error), self.test_store._process_mongo_store, eval(mock_bad_data)) mock_ack.assert_called_once() mock_send_hb.assert_not_called() except Exception as e: self.fail("Test failed: {}".format(e)) @freeze_time("2012-01-01") @mock.patch("src.data_store.stores.store.RabbitMQApi.basic_ack", autospec=True) @mock.patch("src.data_store.stores.alert.AlertStore._process_redis_store", autospec=True) @mock.patch("src.data_store.stores.alert.AlertStore._process_mongo_store", autospec=True) def test_process_data_sends_heartbeat_correctly(self, mock_process_mongo_store, mock_process_redis_store, mock_basic_ack) -> None: mock_basic_ack.return_value = None try: self.test_rabbit_manager.connect() self.test_store._initialise_rabbitmq() self.test_rabbit_manager.queue_delete(self.test_queue_name) res = self.test_rabbit_manager.queue_declare( queue=self.test_queue_name, durable=True, exclusive=False, auto_delete=False, passive=False) self.assertEqual(0, res.method.message_count) self.test_rabbit_manager.queue_bind(queue=self.test_queue_name, exchange=HEALTH_CHECK_EXCHANGE, routing_key=self.routing_key) blocking_channel = self.test_store.rabbitmq.channel method_chains = pika.spec.Basic.Deliver( routing_key=ALERT_STORE_INPUT_ROUTING_KEY) properties = pika.spec.BasicProperties() self.test_store._process_data( blocking_channel, method_chains, properties, json.dumps(self.alert_data_1).encode()) res = self.test_rabbit_manager.queue_declare( queue=self.test_queue_name, durable=True, exclusive=False, auto_delete=False, passive=True) self.assertEqual(1, res.method.message_count) heartbeat_test = { 'component_name': self.test_store_name, 'is_alive': True, 'timestamp': datetime(2012, 1, 1).timestamp() } _, _, body = self.test_rabbit_manager.basic_get( self.test_queue_name) self.assertEqual(heartbeat_test, json.loads(body)) mock_process_mongo_store.assert_called_once() mock_process_redis_store.assert_called_once() except Exception as e: self.fail("Test failed: {}".format(e)) @mock.patch("src.data_store.stores.store.RabbitMQApi.basic_ack", autospec=True) def test_process_data_doesnt_send_heartbeat_on_processing_error( self, mock_basic_ack) -> None: mock_basic_ack.return_value = None try: self.test_rabbit_manager.connect() self.test_store._initialise_rabbitmq() self.test_rabbit_manager.queue_delete(self.test_queue_name) res = self.test_rabbit_manager.queue_declare( queue=self.test_queue_name, durable=True, exclusive=False, auto_delete=False, passive=False) self.assertEqual(0, res.method.message_count) self.test_rabbit_manager.queue_bind(queue=self.test_queue_name, exchange=HEALTH_CHECK_EXCHANGE, routing_key=self.routing_key) blocking_channel = self.test_store.rabbitmq.channel method_chains = pika.spec.Basic.Deliver( routing_key=ALERT_STORE_INPUT_ROUTING_KEY) properties = pika.spec.BasicProperties() self.test_store._process_data( blocking_channel, method_chains, properties, json.dumps(self.alert_data_unexpected).encode()) res = self.test_rabbit_manager.queue_declare( queue=self.test_queue_name, durable=True, exclusive=False, auto_delete=False, passive=True) self.assertEqual(0, res.method.message_count) except Exception as e: self.fail("Test failed: {}".format(e)) @mock.patch.object(MongoApi, "update_one") def test_process_mongo_store_calls_update_one(self, mock_update_one) -> None: self.test_store._process_mongo_store(self.alert_data_1) mock_update_one.assert_called_once() @mock.patch.object(RedisApi, "hset") def test_process_redis_store_calls_hset(self, mock_hset) -> None: self.test_store._process_redis_store(self.alert_data_1) mock_hset.assert_called_once() @parameterized.expand([ ("self.alert_data_1", ), ("self.alert_data_2", ), ("self.alert_data_3", ), ]) @freeze_time("2012-01-01") @mock.patch.object(MongoApi, "update_one") def test_process_mongo_store_calls_mongo_correctly( self, mock_system_data, mock_update_one) -> None: data = eval(mock_system_data) self.test_store._process_mongo_store(data) call_1 = call(data['parent_id'], { 'doc_type': 'alert', 'n_alerts': { '$lt': 1000 } }, { '$push': { 'alerts': { 'origin': data['origin_id'], 'alert_name': data['alert_code']['name'], 'severity': data['severity'], 'metric': data['metric'], 'message': data['message'], 'timestamp': str(data['timestamp']), } }, '$min': { 'first': data['timestamp'] }, '$max': { 'last': data['timestamp'] }, '$inc': { 'n_alerts': 1 }, }) mock_update_one.assert_has_calls([call_1]) @parameterized.expand([ ("self.alert_data_1", ), ("self.alert_data_2", ), ("self.alert_data_3", ), ]) @freeze_time("2012-01-01") @mock.patch.object(RedisApi, "hset") def test_process_redis_store_calls_redis_correctly(self, mock_system_data, mock_hset) -> None: data = eval(mock_system_data) self.test_store._process_redis_store(data) metric_data = { 'severity': data['severity'], 'message': data['message'] } key = data['origin_id'] call_1 = call(Keys.get_hash_parent(data['parent_id']), eval('Keys.get_alert_{}(key)'.format(data['metric'])), json.dumps(metric_data)) mock_hset.assert_has_calls([call_1]) @parameterized.expand([ ("self.alert_data_1", ), ("self.alert_data_2", ), ("self.alert_data_3", ), ]) @freeze_time("2012-01-01") @mock.patch("src.data_store.stores.store.RabbitMQApi.basic_ack", autospec=True) @mock.patch("src.data_store.stores.alert.AlertStore._process_redis_store", autospec=True) @mock.patch("src.data_store.stores.store.Store._send_heartbeat", autospec=True) @mock.patch.object(MongoApi, "update_one") def test_process_data_calls_mongo_correctly(self, mock_system_data, mock_update_one, mock_send_hb, mock_process_redis_store, mock_ack) -> None: mock_ack.return_value = None try: self.test_store._initialise_rabbitmq() data = eval(mock_system_data) blocking_channel = self.test_store.rabbitmq.channel method_chains = pika.spec.Basic.Deliver( routing_key=ALERT_STORE_INPUT_ROUTING_KEY) properties = pika.spec.BasicProperties() self.test_store._process_data(blocking_channel, method_chains, properties, json.dumps(data).encode()) mock_ack.assert_called_once() mock_send_hb.assert_called_once() call_1 = call(data['parent_id'], { 'doc_type': 'alert', 'n_alerts': { '$lt': 1000 } }, { '$push': { 'alerts': { 'origin': data['origin_id'], 'alert_name': data['alert_code']['name'], 'severity': data['severity'], 'metric': data['metric'], 'message': data['message'], 'timestamp': str(data['timestamp']), } }, '$min': { 'first': data['timestamp'] }, '$max': { 'last': data['timestamp'] }, '$inc': { 'n_alerts': 1 }, }) mock_update_one.assert_has_calls([call_1]) mock_process_redis_store.assert_called_once() except Exception as e: self.fail("Test failed: {}".format(e)) @parameterized.expand([ ("self.alert_data_1", ), ("self.alert_data_2", ), ("self.alert_data_3", ), ]) @freeze_time("2012-01-01") @mock.patch("src.data_store.stores.store.RabbitMQApi.basic_ack", autospec=True) @mock.patch("src.data_store.stores.alert.AlertStore._process_mongo_store", autospec=True) @mock.patch("src.data_store.stores.store.Store._send_heartbeat", autospec=True) @mock.patch.object(RedisApi, "hset") def test_process_data_calls_redis_correctly(self, mock_system_data, mock_hset, mock_send_hb, mock_process_mongo_store, mock_ack) -> None: mock_ack.return_value = None try: self.test_store._initialise_rabbitmq() data = eval(mock_system_data) blocking_channel = self.test_store.rabbitmq.channel method_chains = pika.spec.Basic.Deliver( routing_key=ALERT_STORE_INPUT_ROUTING_KEY) properties = pika.spec.BasicProperties() self.test_store._process_data(blocking_channel, method_chains, properties, json.dumps(data).encode()) mock_ack.assert_called_once() mock_send_hb.assert_called_once() metric_data = { 'severity': data['severity'], 'message': data['message'] } key = data['origin_id'] call_1 = call( Keys.get_hash_parent(data['parent_id']), eval('Keys.get_alert_{}(key)'.format(data['metric'])), json.dumps(metric_data)) mock_hset.assert_has_calls([call_1]) mock_process_mongo_store.assert_called_once() except Exception as e: self.fail("Test failed: {}".format(e)) @parameterized.expand([ ("self.alert_data_1", ), ("self.alert_data_2", ), ("self.alert_data_3", ), ]) def test_process_mongo_store_mongo_stores_correctly( self, mock_system_data) -> None: data = eval(mock_system_data) self.test_store._process_mongo_store(data) documents = self.mongo.get_all(data['parent_id']) document = documents[0] expected = [ 'alert', 1, str(data['origin_id']), str(data['alert_code']['name']), str(data['severity']), str(data['metric']), str(data['message']), str(data['timestamp']) ] actual = [ document['doc_type'], document['n_alerts'], document['alerts'][0]['origin'], document['alerts'][0]['alert_name'], document['alerts'][0]['severity'], document['alerts'][0]['metric'], document['alerts'][0]['message'], document['alerts'][0]['timestamp'] ] self.assertListEqual(expected, actual) @parameterized.expand([ ("self.alert_data_1", ), ("self.alert_data_2", ), ("self.alert_data_3", ), ]) def test_process_redis_store_redis_stores_correctly( self, mock_system_data) -> None: data = eval(mock_system_data) self.test_store._process_redis_store(data) key = data['origin_id'] stored_data = self.redis.hget( Keys.get_hash_parent(data['parent_id']), eval('Keys.get_alert_{}(key)'.format(data['metric']))) expected_data = { 'severity': data['severity'], 'message': data['message'] } self.assertEqual(expected_data, json.loads(stored_data)) @parameterized.expand([ ("self.alert_data_1", ), ("self.alert_data_2", ), ("self.alert_data_3", ), ]) @mock.patch("src.data_store.stores.store.RabbitMQApi.basic_ack", autospec=True) @mock.patch("src.data_store.stores.alert.AlertStore._process_redis_store", autospec=True) @mock.patch("src.data_store.stores.store.Store._send_heartbeat", autospec=True) def test_process_data_results_stores_in_mongo_correctly( self, mock_system_data, mock_send_hb, mock_process_redis_store, mock_ack) -> None: mock_ack.return_value = None try: self.test_store._initialise_rabbitmq() data = eval(mock_system_data) blocking_channel = self.test_store.rabbitmq.channel method_chains = pika.spec.Basic.Deliver( routing_key=ALERT_STORE_INPUT_ROUTING_KEY) properties = pika.spec.BasicProperties() self.test_store._process_data(blocking_channel, method_chains, properties, json.dumps(data).encode()) mock_process_redis_store.assert_called_once() mock_ack.assert_called_once() mock_send_hb.assert_called_once() documents = self.mongo.get_all(data['parent_id']) document = documents[0] expected = [ 'alert', 1, str(data['origin_id']), str(data['alert_code']['name']), str(data['severity']), str(data['message']), str(data['timestamp']) ] actual = [ document['doc_type'], document['n_alerts'], document['alerts'][0]['origin'], document['alerts'][0]['alert_name'], document['alerts'][0]['severity'], document['alerts'][0]['message'], document['alerts'][0]['timestamp'] ] self.assertListEqual(expected, actual) except Exception as e: self.fail("Test failed: {}".format(e)) @parameterized.expand([ ("self.alert_data_1", ), ("self.alert_data_2", ), ("self.alert_data_3", ), ]) @mock.patch("src.data_store.stores.store.RabbitMQApi.basic_ack", autospec=True) @mock.patch("src.data_store.stores.alert.AlertStore._process_mongo_store", autospec=True) @mock.patch("src.data_store.stores.store.Store._send_heartbeat", autospec=True) def test_process_data_results_stores_in_redis_correctly( self, mock_system_data, mock_send_hb, mock_process_mongo_store, mock_ack) -> None: mock_ack.return_value = None try: self.test_store._initialise_rabbitmq() data = eval(mock_system_data) blocking_channel = self.test_store.rabbitmq.channel method_chains = pika.spec.Basic.Deliver( routing_key=ALERT_STORE_INPUT_ROUTING_KEY) properties = pika.spec.BasicProperties() self.test_store._process_data(blocking_channel, method_chains, properties, json.dumps(data).encode()) mock_process_mongo_store.assert_called_once() mock_ack.assert_called_once() mock_send_hb.assert_called_once() key = data['origin_id'] stored_data = self.redis.hget( Keys.get_hash_parent(data['parent_id']), eval('Keys.get_alert_{}(key)'.format(data['metric']))) expected_data = { 'severity': data['severity'], 'message': data['message'] } self.assertEqual(expected_data, json.loads(stored_data)) except Exception as e: self.fail("Test failed: {}".format(e))
class TestGithubAlertersManager(unittest.TestCase): def setUp(self) -> None: self.dummy_logger = logging.getLogger('Dummy') self.dummy_logger.disabled = True self.connection_check_time_interval = timedelta(seconds=0) self.rabbitmq = RabbitMQApi( self.dummy_logger, env.RABBIT_IP, connection_check_time_interval=self.connection_check_time_interval) self.manager_name = 'test_github_alerters_manager' self.test_queue_name = 'Test Queue' self.test_data_str = 'test data' self.test_heartbeat = { 'component_name': self.manager_name, 'is_alive': True, 'timestamp': datetime(2012, 1, 1).timestamp(), } self.github_alerter_name = GITHUB_ALERTER_NAME self.dummy_process1 = Process(target=infinite_fn, args=()) self.dummy_process1.daemon = True self.dummy_process2 = Process(target=infinite_fn, args=()) self.dummy_process2.daemon = True self.dummy_process3 = Process(target=infinite_fn, args=()) self.dummy_process3.daemon = True self.test_rabbit_manager = RabbitMQApi( self.dummy_logger, env.RABBIT_IP, connection_check_time_interval=self.connection_check_time_interval) self.test_manager = GithubAlerterManager( self.dummy_logger, self.manager_name, self.rabbitmq) self.test_exception = PANICException('test_exception', 1) def tearDown(self) -> None: # Delete any queues and exchanges which are common across many tests try: self.test_rabbit_manager.connect() self.test_manager.rabbitmq.connect() # Declare queues incase they haven't been declared already self.test_manager.rabbitmq.queue_declare( queue=self.test_queue_name, durable=True, exclusive=False, auto_delete=False, passive=False ) self.test_manager.rabbitmq.queue_declare( queue=GITHUB_MANAGER_INPUT_QUEUE, durable=True, exclusive=False, auto_delete=False, passive=False ) self.test_manager.rabbitmq.queue_purge(self.test_queue_name) self.test_manager.rabbitmq.queue_purge(GITHUB_MANAGER_INPUT_QUEUE) self.test_manager.rabbitmq.queue_delete(self.test_queue_name) self.test_manager.rabbitmq.queue_delete(GITHUB_MANAGER_INPUT_QUEUE) self.test_manager.rabbitmq.exchange_delete(HEALTH_CHECK_EXCHANGE) self.test_manager.rabbitmq.disconnect() self.test_rabbit_manager.disconnect() except Exception as e: print("Test failed: {}".format(e)) self.dummy_logger = None self.rabbitmq = None self.test_manager = None self.test_exception = None self.test_rabbit_manager = None self.dummy_process1 = None self.dummy_process2 = None self.dummy_process3 = None def test_str_returns_manager_name(self) -> None: self.assertEqual(self.manager_name, self.test_manager.__str__()) def test_name_returns_manager_name(self) -> None: self.assertEqual(self.manager_name, self.test_manager.name) @mock.patch.object(RabbitMQApi, "start_consuming") def test_listen_for_data_calls_start_consuming( self, mock_start_consuming) -> None: mock_start_consuming.return_value = None self.test_manager._listen_for_data() self.assertEqual(1, mock_start_consuming.call_count) @mock.patch.object(GithubAlerterManager, "_process_ping") def test_initialise_rabbitmq_initialises_everything_as_expected( self, mock_process_ping) -> None: mock_process_ping.return_value = None try: self.test_rabbit_manager.connect() # To make sure that there is no connection/channel already # established self.assertIsNone(self.rabbitmq.connection) self.assertIsNone(self.rabbitmq.channel) # To make sure that the exchanges and queues have not already been # declared self.test_manager.rabbitmq.connect() self.test_manager.rabbitmq.queue_delete(GITHUB_MANAGER_INPUT_QUEUE) self.test_manager.rabbitmq.exchange_delete(HEALTH_CHECK_EXCHANGE) self.test_manager._initialise_rabbitmq() # Perform checks that the connection has been opened, marked as open # and that the delivery confirmation variable is set. self.assertTrue(self.test_manager.rabbitmq.is_connected) self.assertTrue(self.test_manager.rabbitmq.connection.is_open) self.assertTrue( self.test_manager.rabbitmq.channel._delivery_confirmation) # Check whether the exchanges and queues have been creating by # sending messages with the same routing keys as for the queues. We # will also check if the size of the queues is 0 to confirm that # basic_consume was called (it will store the msg in the component # memory immediately). If one of the exchanges or queues is not # created, then either an exception will be thrown or the queue size # would be 1. Note when deleting the exchanges in the beginning we # also released every binding, hence there are no other queue binded # with the same routing key to any exchange at this point. self.test_rabbit_manager.basic_publish_confirm( exchange=HEALTH_CHECK_EXCHANGE, routing_key=GITHUB_MANAGER_INPUT_ROUTING_KEY, body=self.test_data_str, is_body_dict=False, properties=pika.BasicProperties(delivery_mode=2), mandatory=True) # Re-declare queue to get the number of messages res = self.test_rabbit_manager.queue_declare( GITHUB_MANAGER_INPUT_QUEUE, False, True, False, False) self.assertEqual(0, res.method.message_count) except Exception as e: self.fail("Test failed: {}".format(e)) def test_send_heartbeat_sends_a_heartbeat_correctly(self) -> None: # This test creates a queue which receives messages with the same # routing key as the ones sent by send_heartbeat, and checks that the # heartbeat is received try: self.test_manager._initialise_rabbitmq() self.test_rabbit_manager.connect() res = self.test_rabbit_manager.queue_declare( queue=self.test_queue_name, durable=True, exclusive=False, auto_delete=False, passive=False ) self.assertEqual(0, res.method.message_count) self.test_manager.rabbitmq.queue_bind( queue=self.test_queue_name, exchange=HEALTH_CHECK_EXCHANGE, routing_key='heartbeat.manager') self.test_manager._send_heartbeat(self.test_heartbeat) # By re-declaring the queue again we can get the number of messages # in the queue. res = self.test_rabbit_manager.queue_declare( queue=self.test_queue_name, durable=True, exclusive=False, auto_delete=False, passive=True ) self.assertEqual(1, res.method.message_count) # Check that the message received is actually the HB _, _, body = self.test_rabbit_manager.basic_get( self.test_queue_name) self.assertEqual(self.test_heartbeat, json.loads(body)) except Exception as e: self.fail("Test failed: {}".format(e)) @mock.patch.object(multiprocessing.Process, "start") def test_create_and_start_alerter_process_creates_the_correct_process( self, mock_start) -> None: mock_start.return_value = None self.test_manager._start_alerters_processes() new_entry_process = self.test_manager.alerter_process_dict[ GITHUB_ALERTER_NAME] self.assertTrue(new_entry_process.daemon) self.assertEqual(0, len(new_entry_process._args)) self.assertEqual(start_github_alerter, new_entry_process._target) @mock.patch("src.alerter.alerter_starters.create_logger") def test_start_alerters_process_starts_the_process( self, mock_create_logger) -> None: mock_create_logger.return_value = self.dummy_logger self.test_manager._start_alerters_processes() # We need to sleep to give some time for the alerter to be initialised, # otherwise the process would not terminate time.sleep(1) new_entry_process = self.test_manager.alerter_process_dict[ GITHUB_ALERTER_NAME] self.assertTrue(new_entry_process.is_alive()) new_entry_process.terminate() new_entry_process.join() @freeze_time("2012-01-01") @mock.patch.object(RabbitMQApi, "basic_ack") @mock.patch.object(multiprocessing.Process, "is_alive") @mock.patch.object(multiprocessing.Process, "start") @mock.patch.object(multiprocessing.Process, "join") @mock.patch.object(multiprocessing.Process, "terminate") def test_process_ping_sends_a_valid_hb_if_process_is_alive( self, mock_terminate, mock_join, mock_start, mock_is_alive, mock_ack) -> None: # This test creates a queue which receives messages with the same # routing key as the ones sent by send_heartbeat, and checks that the # received heartbeat is valid. mock_ack.return_value = None mock_is_alive.return_value = True mock_start.return_value = None mock_join.return_value = None mock_terminate.return_value = None try: self.test_manager._initialise_rabbitmq() self.test_manager._start_alerters_processes() # Delete the queue before to avoid messages in the queue on error. self.test_manager.rabbitmq.queue_delete(self.test_queue_name) # initialise blocking_channel = self.test_manager.rabbitmq.channel properties = pika.spec.BasicProperties() method_hb = pika.spec.Basic.Deliver(routing_key='heartbeat.manager') body = 'ping' res = self.test_manager.rabbitmq.queue_declare( queue=self.test_queue_name, durable=True, exclusive=False, auto_delete=False, passive=False ) self.assertEqual(0, res.method.message_count) self.test_manager.rabbitmq.queue_bind( queue=self.test_queue_name, exchange=HEALTH_CHECK_EXCHANGE, routing_key='heartbeat.manager') self.test_manager._process_ping(blocking_channel, method_hb, properties, body) # By re-declaring the queue again we can get the number of messages # in the queue. res = self.test_manager.rabbitmq.queue_declare( queue=self.test_queue_name, durable=True, exclusive=False, auto_delete=False, passive=True ) self.assertEqual(1, res.method.message_count) expected_output = { "component_name": self.manager_name, "dead_processes": [], "running_processes": [GITHUB_ALERTER_NAME], "timestamp": datetime(2012, 1, 1).timestamp() } # Check that the message received is a valid HB _, _, body = self.test_manager.rabbitmq.basic_get( self.test_queue_name) self.assertEqual(expected_output, json.loads(body)) except Exception as e: self.fail("Test failed: {}".format(e)) @freeze_time("2012-01-01") @mock.patch.object(RabbitMQApi, "basic_ack") @mock.patch.object(multiprocessing.Process, "is_alive") @mock.patch.object(multiprocessing.Process, "start") @mock.patch.object(multiprocessing.Process, "join") @mock.patch.object(multiprocessing.Process, "terminate") def test_process_ping_sends_a_valid_hb_if_process_is_dead( self, mock_terminate, mock_join, mock_start, mock_is_alive, mock_ack) -> None: # This test creates a queue which receives messages with the same # routing key as the ones sent by send_heartbeat, and checks that the # received heartbeat is valid. mock_ack.return_value = None mock_is_alive.return_value = False mock_start.return_value = None mock_join.return_value = None mock_terminate.return_value = None try: self.test_manager._initialise_rabbitmq() self.test_manager._start_alerters_processes() # Delete the queue before to avoid messages in the queue on error. self.test_manager.rabbitmq.queue_delete(self.test_queue_name) # initialise blocking_channel = self.test_manager.rabbitmq.channel properties = pika.spec.BasicProperties() method_hb = pika.spec.Basic.Deliver(routing_key='heartbeat.manager') body = 'ping' res = self.test_manager.rabbitmq.queue_declare( queue=self.test_queue_name, durable=True, exclusive=False, auto_delete=False, passive=False ) self.assertEqual(0, res.method.message_count) self.test_manager.rabbitmq.queue_bind( queue=self.test_queue_name, exchange=HEALTH_CHECK_EXCHANGE, routing_key='heartbeat.manager') self.test_manager._process_ping(blocking_channel, method_hb, properties, body) # By re-declaring the queue again we can get the number of messages # in the queue. res = self.test_manager.rabbitmq.queue_declare( queue=self.test_queue_name, durable=True, exclusive=False, auto_delete=False, passive=True ) self.assertEqual(1, res.method.message_count) expected_output = { "component_name": self.manager_name, "dead_processes": [GITHUB_ALERTER_NAME], "running_processes": [], "timestamp": datetime(2012, 1, 1).timestamp() } # Check that the message received is a valid HB _, _, body = self.test_manager.rabbitmq.basic_get( self.test_queue_name) self.assertEqual(expected_output, json.loads(body)) except Exception as e: self.fail("Test failed: {}".format(e)) @freeze_time("2012-01-01") @mock.patch.object(RabbitMQApi, "basic_ack") @mock.patch("src.alerter.alerter_starters.create_logger") @mock.patch.object(GithubAlerterManager, "_send_heartbeat") def test_process_ping_restarts_dead_processes( self, send_hb_mock, mock_create_logger, mock_ack) -> None: send_hb_mock.return_value = None mock_create_logger.return_value = self.dummy_logger mock_ack.return_value = None try: self.test_manager._initialise_rabbitmq() self.test_manager._start_alerters_processes() # Give time for the processes to start time.sleep(1) # Automate the case when having all processes dead self.test_manager.alerter_process_dict[ GITHUB_ALERTER_NAME].terminate() self.test_manager.alerter_process_dict[GITHUB_ALERTER_NAME].join() # Give time for the processes to terminate time.sleep(1) # Check that that the processes have terminated self.assertFalse(self.test_manager.alerter_process_dict[ GITHUB_ALERTER_NAME].is_alive()) # initialise blocking_channel = self.test_manager.rabbitmq.channel properties = pika.spec.BasicProperties() method_hb = pika.spec.Basic.Deliver(routing_key='heartbeat.manager') body = 'ping' self.test_manager._process_ping(blocking_channel, method_hb, properties, body) # Give time for the processes to start time.sleep(1) self.assertTrue(self.test_manager.alerter_process_dict[ GITHUB_ALERTER_NAME].is_alive()) # Clean before test finishes self.test_manager.alerter_process_dict[ GITHUB_ALERTER_NAME].terminate() self.test_manager.alerter_process_dict[GITHUB_ALERTER_NAME].join() except Exception as e: self.fail("Test failed: {}".format(e)) @mock.patch.object(multiprocessing.Process, "is_alive") @mock.patch.object(multiprocessing.Process, "start") @mock.patch.object(multiprocessing, 'Process') def test_process_ping_does_not_send_hb_if_processing_fails( self, mock_process, mock_start, is_alive_mock) -> None: # This test creates a queue which receives messages with the same # routing key as the ones sent by send_heartbeat. In this test we will # check that no heartbeat is sent when mocking a raised exception. is_alive_mock.side_effect = self.test_exception mock_start.return_value = None mock_process.side_effect = self.dummy_process1 try: self.test_manager._initialise_rabbitmq() self.test_manager._start_alerters_processes() self.test_manager.rabbitmq.queue_declare( queue=self.test_queue_name, durable=True, exclusive=False, auto_delete=False, passive=False ) # Delete the queue before to avoid messages in the queue on error. self.test_manager.rabbitmq.queue_delete(self.test_queue_name) # initialise blocking_channel = self.test_manager.rabbitmq.channel method = pika.spec.Basic.Deliver(routing_key='heartbeat.manager') properties = pika.spec.BasicProperties() body = 'ping' res = self.test_manager.rabbitmq.queue_declare( queue=self.test_queue_name, durable=True, exclusive=False, auto_delete=False, passive=False ) self.assertEqual(0, res.method.message_count) self.test_manager.rabbitmq.queue_bind( queue=self.test_queue_name, exchange=HEALTH_CHECK_EXCHANGE, routing_key='heartbeat.manager') self.test_manager._process_ping(blocking_channel, method, properties, body) # By re-declaring the queue again we can get the number of messages # in the queue. res = self.test_manager.rabbitmq.queue_declare( queue=self.test_queue_name, durable=True, exclusive=False, auto_delete=False, passive=True ) self.assertEqual(0, res.method.message_count) except Exception as e: self.fail("Test failed: {}".format(e)) def test_proc_ping_send_hb_does_not_raise_msg_not_del_exce_if_hb_not_routed( self) -> None: try: self.test_manager._initialise_rabbitmq() self.test_manager._start_alerters_processes() # initialise blocking_channel = self.test_manager.rabbitmq.channel method = pika.spec.Basic.Deliver(routing_key='heartbeat.manager') properties = pika.spec.BasicProperties() body = 'ping' self.test_manager._process_ping(blocking_channel, method, properties, body) except Exception as e: self.fail("Test failed: {}".format(e)) @parameterized.expand([ ("pika.exceptions.AMQPChannelError('test')", "pika.exceptions.AMQPChannelError"), ("self.test_exception", "PANICException"), ]) @mock.patch.object(GithubAlerterManager, "_send_heartbeat") def test_process_ping_send_hb_raises_exceptions( self, param_input, param_expected, hb_mock) -> None: hb_mock.side_effect = eval(param_input) try: self.test_manager._initialise_rabbitmq() # initialise blocking_channel = self.test_manager.rabbitmq.channel method = pika.spec.Basic.Deliver(routing_key='heartbeat.manager') properties = pika.spec.BasicProperties() body = 'ping' self.assertRaises(eval(param_expected), self.test_manager._process_ping, blocking_channel, method, properties, body) except Exception as e: self.fail("Test failed: {}".format(e))