class TestMongoApiWithMongoOnline(unittest.TestCase):
@classmethod
def setUpClass(cls) -> None:
# Same as in setUp(), to avoid running all tests if Mongo is offline
dummy_logger = logging.getLogger('dummy')
dummy_logger.disabled = True
db = env.DB_NAME
host = env.DB_IP
port = env.DB_PORT
user = ''
password = ''
mongo = MongoApi(logger=dummy_logger,
db_name=db,
host=host,
port=port,
username=user,
password=password)
# Ping Mongo
try:
mongo.ping_unsafe()
except PyMongoError:
raise Exception("Mongo is not online.")
def setUp(self) -> None:
self.dummy_logger = logging.getLogger('dummy')
self.dummy_logger.disabled = True
self.db = env.DB_NAME
self.host = env.DB_IP
self.port = env.DB_PORT
self.user = ''
self.password = ''
self.mongo = MongoApi(logger=self.dummy_logger,
db_name=self.db,
host=self.host,
port=self.port,
username=self.user,
password=self.password)
try:
self.mongo.ping_unsafe()
except PyMongoError:
self.fail("Mongo is not online.")
# Clear test database
self.mongo.drop_db()
self.col1 = 'collection1'
self.col2 = 'collection2'
self.val1 = {'a': 'b', 'c': 'd'}
self.val2 = {'e': 'f', 'g': 'h'}
self.val3 = {'i': 'j'}
self.val4 = {'k': 'l', 'm': {'n': ['o', 'p', True, False, 1, 2.1]}}
self.id_1 = '1'
self.id_2 = '2'
self.key_m_1 = 'key_m_1'
self.key_m_2 = 'key_m_2'
self.val_m_1 = 'm1'
self.val_m_2 = 'm2'
self.test_1 = 'test_1'
self.test_2 = 'test_2'
self.time_used = datetime(2021, 1, 28).timestamp()
self.query1 = {'doc_type': self.test_1, 'd': self.time_used}
self.query2 = {'doc_type': self.test_2, 'd': self.time_used}
self.doc_1 = {
'$push': {
self.id_1: {
self.key_m_1: self.val_m_1
}
},
'$inc': {
'n_entries': 1
}
}
self.doc_2 = {
'$push': {
self.id_2: {
self.key_m_2: self.val_m_2
}
},
'$inc': {
'n_entries': 1
}
}
self.doc_3 = {
'$push': {
self.id_1: {
self.key_m_2: self.val_m_2
}
},
'$inc': {
'n_entries': 1
}
}
self.time = timedelta(seconds=3)
self.time_with_error_margin = timedelta(seconds=4)
self.default_str = 'DEFAULT'
self.default_int = 789
self.default_bool = False
def tearDown(self) -> None:
self.dummy_logger = None
self.mongo.drop_db()
self.mongo = None
def test_insert_one_inserts_value_into_the_specified_collection(self):
# Check that col1 is empty
get_result = list(self.mongo._db[self.col1].find({}))
self.assertEqual(len(get_result), 0)
# Insert val1 into col1
self.mongo.insert_one(self.col1, self.val1)
# Check that value was added to col1
get_result = list(self.mongo._db[self.col1].find({}))
self.assertEqual(len(get_result), 1)
self.assertEqual(dict(get_result[0]), self.val1)
def test_insert_one_supports_more_complex_documents(self):
# Check that col1 is empty
get_result = list(self.mongo._db[self.col1].find({}))
self.assertEqual(len(get_result), 0)
# Insert val4 into col1
self.mongo.insert_one(self.col1, self.val4)
# Check that value was added to col1
get_result = list(self.mongo._db[self.col1].find({}))
self.assertEqual(len(get_result), 1)
self.assertEqual(dict(get_result[0]), self.val4)
def test_insert_many_inserts_all_values_into_the_specified_collection(
self):
# Check that col1 is empty
get_result = list(self.mongo._db[self.col1].find({}))
self.assertEqual(len(get_result), 0)
# Insert val1, val2, and val3 into col1
self.mongo.insert_many(self.col1, [self.val1, self.val2, self.val3])
# Check that the values was added to col1
get_result = list(self.mongo._db[self.col1].find({}))
self.assertEqual(len(get_result), 3)
self.assertEqual(dict(get_result[0]), self.val1)
self.assertEqual(dict(get_result[1]), self.val2)
self.assertEqual(dict(get_result[2]), self.val3)
def test_update_one_inserts_value_into_the_specified_collection(self):
# Check that col1 is empty
get_result = list(self.mongo._db[self.col1].find({}))
self.assertEqual(len(get_result), 0)
self.mongo.update_one(self.col1, self.query1, self.doc_1)
# Check that value was added to col1
get_result = list(self.mongo._db[self.col1].find({}))
self.assertEqual(len(get_result), 1)
self.assertEqual(get_result[0]['1'][0][self.key_m_1], self.val_m_1)
self.assertEqual(get_result[0]['n_entries'], 1)
self.assertEqual(get_result[0]['doc_type'], self.test_1)
self.assertEqual(get_result[0]['d'], self.time_used)
def test_update_one_multiple_times_inserts_values_into_the_specified_collection(
self):
# Check that col1 is empty
get_result = list(self.mongo._db[self.col1].find({}))
self.assertEqual(len(get_result), 0)
self.mongo.update_one(self.col1, self.query1, self.doc_1)
self.mongo.update_one(self.col1, self.query1, self.doc_3)
# Check that value was added to col1
get_result = list(self.mongo._db[self.col1].find({}))
self.assertEqual(len(get_result), 1)
self.assertEqual(get_result[0]['1'][0][self.key_m_1], self.val_m_1)
self.assertEqual(get_result[0]['1'][1][self.key_m_2], self.val_m_2)
self.assertEqual(get_result[0]['n_entries'], 2)
self.assertEqual(get_result[0]['doc_type'], self.test_1)
self.assertEqual(get_result[0]['d'], self.time_used)
def test_update_one_multiple_documents_inserts_values_into_the_specified_collection(
self):
# Check that col1 is empty
get_result = list(self.mongo._db[self.col1].find({}))
self.assertEqual(len(get_result), 0)
self.mongo.update_one(self.col1, self.query1, self.doc_1)
self.mongo.update_one(self.col1, self.query2, self.doc_2)
# Check that value was added to col1
get_result = list(self.mongo._db[self.col1].find({}))
self.assertEqual(len(get_result), 2)
self.assertEqual(get_result[0]['1'][0][self.key_m_1], self.val_m_1)
self.assertEqual(get_result[0]['n_entries'], 1)
self.assertEqual(get_result[0]['doc_type'], self.test_1)
self.assertEqual(get_result[0]['d'], self.time_used)
self.assertEqual(get_result[1]['2'][0][self.key_m_2], self.val_m_2)
self.assertEqual(get_result[1]['n_entries'], 1)
self.assertEqual(get_result[1]['doc_type'], self.test_2)
self.assertEqual(get_result[1]['d'], self.time_used)
def test_get_all_returns_inserted_values_in_order_of_insert(self):
# Check that col1 is empty
get_result = self.mongo.get_all(self.col1)
self.assertEqual(len(get_result), 0)
# Insert val1, val2, and val3 into col1
self.mongo._db[self.col1].insert_many(
[self.val1, self.val2, self.val3])
# Check that the values was added to col1
get_result = self.mongo.get_all(self.col1)
self.assertEqual(len(get_result), 3)
self.assertEqual(dict(get_result[0]), self.val1)
self.assertEqual(dict(get_result[1]), self.val2)
self.assertEqual(dict(get_result[2]), self.val3)
def test_drop_collection_deletes_the_specified_collection(self):
# Check that col1 and col2 are empty
get_result1 = list(self.mongo._db[self.col1].find({}))
get_result2 = list(self.mongo._db[self.col2].find({}))
self.assertEqual(len(get_result1), 0)
self.assertEqual(len(get_result2), 0)
# Insert val1, val2, and val3 into col1 and val4 into col2
self.mongo._db[self.col1].insert_many(
[self.val1, self.val2, self.val3])
self.mongo._db[self.col2].insert_one(self.val4)
# Check that col1 and col2 are not empty
get_result1 = list(self.mongo._db[self.col1].find({}))
get_result2 = list(self.mongo._db[self.col2].find({}))
self.assertEqual(len(get_result1), 3)
self.assertEqual(len(get_result2), 1)
# Delete col1
self.mongo.drop_collection(self.col1)
# Check that col1 is back to being empty but col2 is not empty
get_result1 = list(self.mongo._db[self.col1].find({}))
get_result2 = list(self.mongo._db[self.col2].find({}))
self.assertEqual(len(get_result1), 0)
self.assertEqual(len(get_result2), 1)
def test_drop_db_deletes_all_collections(self):
# Check that col1 and col2 are empty
get_result1 = list(self.mongo._db[self.col1].find({}))
get_result2 = list(self.mongo._db[self.col2].find({}))
self.assertEqual(len(get_result1), 0)
self.assertEqual(len(get_result2), 0)
# Insert val1, val2, and val3 into col1 and val4 into col2
self.mongo._db[self.col1].insert_many(
[self.val1, self.val2, self.val3])
self.mongo._db[self.col2].insert_one(self.val4)
# Check that col1 and col2 are not empty
get_result1 = list(self.mongo._db[self.col1].find({}))
get_result2 = list(self.mongo._db[self.col2].find({}))
self.assertEqual(len(get_result1), 3)
self.assertEqual(len(get_result2), 1)
# Drop db
self.mongo.drop_db()
# Check that col1 and col2 are back to being empty
get_result1 = list(self.mongo._db[self.col1].find({}))
get_result2 = list(self.mongo._db[self.col2].find({}))
self.assertEqual(len(get_result1), 0)
self.assertEqual(len(get_result2), 0)
def test_ping_returns_true(self):
self.assertTrue(self.mongo.ping_unsafe())
def test_ping_auth_throws_value_error_for_empty_password(self):
self.assertRaises(ValueError, self.mongo.ping_auth, self.user, '')
def test_ping_auth_throws_operation_failure_for_wrong_password(self):
self.assertRaises(OperationFailure, self.mongo.ping_auth, self.user,
'incorrect_password')
class TestMongoApiWithMongoOffline(unittest.TestCase):
def setUp(self) -> None:
self.dummy_logger = logging.getLogger('dummy')
self.dummy_logger.disabled = True
self.db = env.DB_NAME
self.host = 'dummyhost'
self.port = env.DB_PORT
self.user = ''
self.password = ''
self.mongo = MongoApi(self.dummy_logger,
self.db,
self.host,
self.port,
timeout_ms=1)
# timeout_ms is set to 1ms to speed up tests. It cannot be 0
self.col1 = 'collection1'
self.val1 = {'a': 'b', 'c': 'd'}
self.val2 = {'e': 'f', 'g': 'h'}
self.val3 = {'i': 'j'}
self.id_1 = '1'
self.key_m_1 = 'key_m_1'
self.val_m_1 = 'm1'
self.test_1 = 'test_1'
self.time_used = datetime(2021, 1, 28).timestamp()
self.query1 = {'doc_type': self.test_1, 'd': self.time_used}
self.doc_1 = {
'$push': {
self.id_1: {
self.key_m_1: self.val_m_1
}
},
'$inc': {
'n_entries': 1
}
}
def tearDown(self) -> None:
self.dummy_logger = None
self.mongo.drop_db()
self.mongo = None
def test_insert_one_returns_none_first_time_round(self):
default_return = self.mongo.insert_one(self.col1, self.val1)
self.assertIsNone(default_return)
def test_insert_many_returns_none_first_time_round(self):
default_return = self.mongo.insert_many(
self.col1, [self.val1, self.val2, self.val3])
self.assertIsNone(default_return)
def test_update_one_returns_none_first_time_round(self):
default_return = self.mongo.update_one(self.col1, self.query1,
self.doc_1)
self.assertIsNone(default_return)
def test_get_all_returns_none_first_time_round(self):
default_return = self.mongo.get_all(self.col1)
self.assertIsNone(default_return)
def test_drop_collection_returns_none_first_time_round(self):
default_return = self.mongo.drop_collection(self.col1)
self.assertIsNone(default_return)
def test_drop_db_returns_none_first_time_round(self):
default_return = self.mongo.drop_db()
self.assertIsNone(default_return)
def test_ping_unsafe_throws_exception_first_time_round(self):
self.assertRaises(ServerSelectionTimeoutError, self.mongo.ping_unsafe)
def test_ping_auth_throws_exception_first_time_round(self):
self.assertRaises(ServerSelectionTimeoutError, self.mongo.ping_auth,
self.user, self.password)
def test_insert_one_returns_none_if_mongo_already_down(self):
self.mongo._set_as_down()
self.assertIsNone(self.mongo.insert_one(self.col1, self.val1))
def test_update_one_returns_none_if_mongo_already_down(self):
self.mongo._set_as_down()
self.assertIsNone(
self.mongo.update_one(self.col1, self.query1, self.doc_1))
def test_insert_many_returns_none_if_mongo_already_down(self):
self.mongo._set_as_down()
documents = [self.val1, self.val2, self.val3]
self.assertIsNone(self.mongo.insert_many(self.col1, documents))
def test_get_all_returns_none_if_mongo_already_down(self):
self.mongo._set_as_down()
self.assertIsNone(self.mongo.get_all(self.col1))
def test_drop_collection_returns_none_if_mongo_already_down(self):
self.mongo._set_as_down()
self.assertIsNone(self.mongo.drop_collection(self.col1))
def test_drop_db_returns_none_if_mongo_already_down(self):
self.mongo._set_as_down()
self.assertIsNone(self.mongo.drop_db())
def test_ping_unsafe_throws_exception_if_mongo_already_down(self):
self.mongo._set_as_down()
self.assertRaises(ServerSelectionTimeoutError, self.mongo.ping_unsafe)
def test_ping_auth_throws_exception_if_mongo_already_down(self):
self.mongo._set_as_down()
self.assertRaises(ServerSelectionTimeoutError, self.mongo.ping_auth,
self.user, self.password)
class TestAlertStore(unittest.TestCase):
def setUp(self) -> None:
self.dummy_logger = logging.getLogger('Dummy')
self.dummy_logger.disabled = True
self.connection_check_time_interval = timedelta(seconds=0)
self.rabbit_ip = env.RABBIT_IP
self.rabbitmq = RabbitMQApi(
self.dummy_logger,
self.rabbit_ip,
connection_check_time_interval=self.connection_check_time_interval)
self.test_rabbit_manager = RabbitMQApi(
self.dummy_logger,
self.rabbit_ip,
connection_check_time_interval=self.connection_check_time_interval)
self.mongo_ip = env.DB_IP
self.mongo_db = env.DB_NAME
self.mongo_port = env.DB_PORT
self.mongo = MongoApi(logger=self.dummy_logger.getChild(
MongoApi.__name__),
db_name=self.mongo_db,
host=self.mongo_ip,
port=self.mongo_port)
self.redis_db = env.REDIS_DB
self.redis_host = env.REDIS_IP
self.redis_port = env.REDIS_PORT
self.redis_namespace = env.UNIQUE_ALERTER_IDENTIFIER
self.redis = RedisApi(self.dummy_logger, self.redis_db,
self.redis_host, self.redis_port, '',
self.redis_namespace,
self.connection_check_time_interval)
self.test_store_name = 'store name'
self.test_store = AlertStore(self.test_store_name, self.dummy_logger,
self.rabbitmq)
self.routing_key = 'heartbeat.worker'
self.test_queue_name = 'test queue'
connect_to_rabbit(self.rabbitmq)
self.rabbitmq.exchange_declare(HEALTH_CHECK_EXCHANGE, 'topic', False,
True, False, False)
self.rabbitmq.exchange_declare(STORE_EXCHANGE, 'direct', False, True,
False, False)
self.rabbitmq.queue_declare(ALERT_STORE_INPUT_QUEUE, False, True,
False, False)
self.rabbitmq.queue_bind(ALERT_STORE_INPUT_QUEUE, STORE_EXCHANGE,
ALERT_STORE_INPUT_ROUTING_KEY)
connect_to_rabbit(self.test_rabbit_manager)
self.test_rabbit_manager.queue_declare(self.test_queue_name, False,
True, False, False)
self.test_rabbit_manager.queue_bind(self.test_queue_name,
HEALTH_CHECK_EXCHANGE,
self.routing_key)
self.test_data_str = 'test data'
self.test_exception = PANICException('test_exception', 1)
self.parent_id = 'test_parent_id'
self.alert_id = 'test_alert_id'
self.origin_id = 'test_origin_id'
self.alert_name = 'test_alert'
self.metric = 'system_is_down'
self.severity = 'warning'
self.message = 'alert message'
self.value = 'alert_code_1'
self.alert_id_2 = 'test_alert_id_2'
self.origin_id_2 = 'test_origin_id_2'
self.alert_name_2 = 'test_alert_2'
self.severity_2 = 'critical'
self.message_2 = 'alert message 2'
self.value_2 = 'alert_code_2'
self.alert_id_3 = 'test_alert_id_3'
self.origin_id_3 = 'test_origin_id_3'
self.alert_name_3 = 'test_alert_3'
self.severity_3 = 'info'
self.message_3 = 'alert message 3'
self.value_3 = 'alert_code_3'
self.last_monitored = datetime(2012, 1, 1).timestamp()
self.none = None
self.alert_data_1 = {
'parent_id': self.parent_id,
'origin_id': self.origin_id,
'alert_code': {
'name': self.alert_name,
'value': self.value,
},
'severity': self.severity,
'metric': self.metric,
'message': self.message,
'timestamp': self.last_monitored,
}
self.alert_data_2 = {
'parent_id': self.parent_id,
'origin_id': self.origin_id_2,
'alert_code': {
'name': self.alert_name_2,
'value': self.value_2,
},
'severity': self.severity_2,
'metric': self.metric,
'message': self.message_2,
'timestamp': self.last_monitored,
}
self.alert_data_3 = {
'parent_id': self.parent_id,
'origin_id': self.origin_id_3,
'alert_code': {
'name': self.alert_name_3,
'value': self.value_3,
},
'severity': self.severity_3,
'metric': self.metric,
'message': self.message_3,
'timestamp': self.last_monitored,
}
self.alert_data_key_error = {"result": {"data": {}, "data2": {}}}
self.alert_data_unexpected = {"unexpected": {}}
def tearDown(self) -> None:
connect_to_rabbit(self.rabbitmq)
delete_queue_if_exists(self.rabbitmq, ALERT_STORE_INPUT_QUEUE)
delete_exchange_if_exists(self.rabbitmq, STORE_EXCHANGE)
delete_exchange_if_exists(self.rabbitmq, HEALTH_CHECK_EXCHANGE)
disconnect_from_rabbit(self.rabbitmq)
connect_to_rabbit(self.test_rabbit_manager)
delete_queue_if_exists(self.test_rabbit_manager, self.test_queue_name)
disconnect_from_rabbit(self.test_rabbit_manager)
self.dummy_logger = None
self.connection_check_time_interval = None
self.rabbitmq = None
self.test_rabbit_manager = None
self.redis.delete_all_unsafe()
self.redis = None
self.mongo.drop_collection(self.parent_id)
self.mongo = None
self.test_store = None
def test__str__returns_name_correctly(self) -> None:
self.assertEqual(self.test_store_name, str(self.test_store))
def test_name_property_returns_name_correctly(self) -> None:
self.assertEqual(self.test_store_name, self.test_store.name)
def test_mongo_ip_property_returns_mongo_ip_correctly(self) -> None:
self.assertEqual(self.mongo_ip, self.test_store.mongo_ip)
def test_mongo_db_property_returns_mongo_db_correctly(self) -> None:
self.assertEqual(self.mongo_db, self.test_store.mongo_db)
def test_mongo_port_property_returns_mongo_port_correctly(self) -> None:
self.assertEqual(self.mongo_port, self.test_store.mongo_port)
def test_mongo_property_returns_mongo(self) -> None:
self.assertEqual(type(self.mongo), type(self.test_store.mongo))
def test_redis_property_returns_redis_correctly(self) -> None:
self.assertEqual(type(self.redis), type(self.test_store.redis))
def test_initialise_rabbitmq_initialises_everything_as_expected(
self) -> None:
try:
# To make sure that the exchanges have not already been declared
self.rabbitmq.connect()
self.rabbitmq.queue_delete(ALERT_STORE_INPUT_QUEUE)
self.test_rabbit_manager.queue_delete(self.test_queue_name)
self.rabbitmq.exchange_delete(HEALTH_CHECK_EXCHANGE)
self.rabbitmq.exchange_delete(STORE_EXCHANGE)
self.rabbitmq.disconnect()
self.test_store._initialise_rabbitmq()
# Perform checks that the connection has been opened, marked as open
# and that the delivery confirmation variable is set.
self.assertTrue(self.test_store.rabbitmq.is_connected)
self.assertTrue(self.test_store.rabbitmq.connection.is_open)
self.assertTrue(
self.test_store.rabbitmq.channel._delivery_confirmation)
# Check whether the producing exchanges have been created by
# using passive=True. If this check fails an exception is raised
# automatically.
self.test_store.rabbitmq.exchange_declare(STORE_EXCHANGE,
passive=True)
self.test_store.rabbitmq.exchange_declare(HEALTH_CHECK_EXCHANGE,
passive=True)
# Check whether the exchange has been creating by sending messages
# to it. If this fails an exception is raised, hence the test fails.
self.test_store.rabbitmq.basic_publish_confirm(
exchange=HEALTH_CHECK_EXCHANGE,
routing_key=self.routing_key,
body=self.test_data_str,
is_body_dict=False,
properties=pika.BasicProperties(delivery_mode=2),
mandatory=False)
# Check whether the exchange has been creating by sending messages
# to it. If this fails an exception is raised, hence the test fails.
self.test_store.rabbitmq.basic_publish_confirm(
exchange=STORE_EXCHANGE,
routing_key=ALERT_STORE_INPUT_ROUTING_KEY,
body=self.test_data_str,
is_body_dict=False,
properties=pika.BasicProperties(delivery_mode=2),
mandatory=False)
# Re-declare queue to get the number of messages
res = self.test_store.rabbitmq.queue_declare(
ALERT_STORE_INPUT_QUEUE, False, True, False, False)
self.assertEqual(1, res.method.message_count)
except Exception as e:
self.fail("Test failed: {}".format(e))
@parameterized.expand([
("KeyError", "self.alert_data_key_error "),
])
@mock.patch("src.data_store.stores.store.RabbitMQApi.basic_ack",
autospec=True)
@mock.patch("src.data_store.stores.store.Store._send_heartbeat",
autospec=True)
def test_process_data_with_bad_data_does_raises_exceptions(
self, mock_error, mock_bad_data, mock_send_hb, mock_ack) -> None:
mock_ack.return_value = None
try:
self.test_store._initialise_rabbitmq()
blocking_channel = self.test_store.rabbitmq.channel
method_chains = pika.spec.Basic.Deliver(
routing_key=ALERT_STORE_INPUT_ROUTING_KEY)
properties = pika.spec.BasicProperties()
self.test_store._process_data(
blocking_channel, method_chains, properties,
json.dumps(self.alert_data_unexpected).encode())
self.assertRaises(eval(mock_error),
self.test_store._process_mongo_store,
eval(mock_bad_data))
mock_ack.assert_called_once()
mock_send_hb.assert_not_called()
except Exception as e:
self.fail("Test failed: {}".format(e))
@freeze_time("2012-01-01")
@mock.patch("src.data_store.stores.store.RabbitMQApi.basic_ack",
autospec=True)
@mock.patch("src.data_store.stores.alert.AlertStore._process_redis_store",
autospec=True)
@mock.patch("src.data_store.stores.alert.AlertStore._process_mongo_store",
autospec=True)
def test_process_data_sends_heartbeat_correctly(self,
mock_process_mongo_store,
mock_process_redis_store,
mock_basic_ack) -> None:
mock_basic_ack.return_value = None
try:
self.test_rabbit_manager.connect()
self.test_store._initialise_rabbitmq()
self.test_rabbit_manager.queue_delete(self.test_queue_name)
res = self.test_rabbit_manager.queue_declare(
queue=self.test_queue_name,
durable=True,
exclusive=False,
auto_delete=False,
passive=False)
self.assertEqual(0, res.method.message_count)
self.test_rabbit_manager.queue_bind(queue=self.test_queue_name,
exchange=HEALTH_CHECK_EXCHANGE,
routing_key=self.routing_key)
blocking_channel = self.test_store.rabbitmq.channel
method_chains = pika.spec.Basic.Deliver(
routing_key=ALERT_STORE_INPUT_ROUTING_KEY)
properties = pika.spec.BasicProperties()
self.test_store._process_data(
blocking_channel, method_chains, properties,
json.dumps(self.alert_data_1).encode())
res = self.test_rabbit_manager.queue_declare(
queue=self.test_queue_name,
durable=True,
exclusive=False,
auto_delete=False,
passive=True)
self.assertEqual(1, res.method.message_count)
heartbeat_test = {
'component_name': self.test_store_name,
'is_alive': True,
'timestamp': datetime(2012, 1, 1).timestamp()
}
_, _, body = self.test_rabbit_manager.basic_get(
self.test_queue_name)
self.assertEqual(heartbeat_test, json.loads(body))
mock_process_mongo_store.assert_called_once()
mock_process_redis_store.assert_called_once()
except Exception as e:
self.fail("Test failed: {}".format(e))
@mock.patch("src.data_store.stores.store.RabbitMQApi.basic_ack",
autospec=True)
def test_process_data_doesnt_send_heartbeat_on_processing_error(
self, mock_basic_ack) -> None:
mock_basic_ack.return_value = None
try:
self.test_rabbit_manager.connect()
self.test_store._initialise_rabbitmq()
self.test_rabbit_manager.queue_delete(self.test_queue_name)
res = self.test_rabbit_manager.queue_declare(
queue=self.test_queue_name,
durable=True,
exclusive=False,
auto_delete=False,
passive=False)
self.assertEqual(0, res.method.message_count)
self.test_rabbit_manager.queue_bind(queue=self.test_queue_name,
exchange=HEALTH_CHECK_EXCHANGE,
routing_key=self.routing_key)
blocking_channel = self.test_store.rabbitmq.channel
method_chains = pika.spec.Basic.Deliver(
routing_key=ALERT_STORE_INPUT_ROUTING_KEY)
properties = pika.spec.BasicProperties()
self.test_store._process_data(
blocking_channel, method_chains, properties,
json.dumps(self.alert_data_unexpected).encode())
res = self.test_rabbit_manager.queue_declare(
queue=self.test_queue_name,
durable=True,
exclusive=False,
auto_delete=False,
passive=True)
self.assertEqual(0, res.method.message_count)
except Exception as e:
self.fail("Test failed: {}".format(e))
@mock.patch.object(MongoApi, "update_one")
def test_process_mongo_store_calls_update_one(self,
mock_update_one) -> None:
self.test_store._process_mongo_store(self.alert_data_1)
mock_update_one.assert_called_once()
@mock.patch.object(RedisApi, "hset")
def test_process_redis_store_calls_hset(self, mock_hset) -> None:
self.test_store._process_redis_store(self.alert_data_1)
mock_hset.assert_called_once()
@parameterized.expand([
("self.alert_data_1", ),
("self.alert_data_2", ),
("self.alert_data_3", ),
])
@freeze_time("2012-01-01")
@mock.patch.object(MongoApi, "update_one")
def test_process_mongo_store_calls_mongo_correctly(
self, mock_system_data, mock_update_one) -> None:
data = eval(mock_system_data)
self.test_store._process_mongo_store(data)
call_1 = call(data['parent_id'], {
'doc_type': 'alert',
'n_alerts': {
'$lt': 1000
}
}, {
'$push': {
'alerts': {
'origin': data['origin_id'],
'alert_name': data['alert_code']['name'],
'severity': data['severity'],
'metric': data['metric'],
'message': data['message'],
'timestamp': str(data['timestamp']),
}
},
'$min': {
'first': data['timestamp']
},
'$max': {
'last': data['timestamp']
},
'$inc': {
'n_alerts': 1
},
})
mock_update_one.assert_has_calls([call_1])
@parameterized.expand([
("self.alert_data_1", ),
("self.alert_data_2", ),
("self.alert_data_3", ),
])
@freeze_time("2012-01-01")
@mock.patch.object(RedisApi, "hset")
def test_process_redis_store_calls_redis_correctly(self, mock_system_data,
mock_hset) -> None:
data = eval(mock_system_data)
self.test_store._process_redis_store(data)
metric_data = {
'severity': data['severity'],
'message': data['message']
}
key = data['origin_id']
call_1 = call(Keys.get_hash_parent(data['parent_id']),
eval('Keys.get_alert_{}(key)'.format(data['metric'])),
json.dumps(metric_data))
mock_hset.assert_has_calls([call_1])
@parameterized.expand([
("self.alert_data_1", ),
("self.alert_data_2", ),
("self.alert_data_3", ),
])
@freeze_time("2012-01-01")
@mock.patch("src.data_store.stores.store.RabbitMQApi.basic_ack",
autospec=True)
@mock.patch("src.data_store.stores.alert.AlertStore._process_redis_store",
autospec=True)
@mock.patch("src.data_store.stores.store.Store._send_heartbeat",
autospec=True)
@mock.patch.object(MongoApi, "update_one")
def test_process_data_calls_mongo_correctly(self, mock_system_data,
mock_update_one, mock_send_hb,
mock_process_redis_store,
mock_ack) -> None:
mock_ack.return_value = None
try:
self.test_store._initialise_rabbitmq()
data = eval(mock_system_data)
blocking_channel = self.test_store.rabbitmq.channel
method_chains = pika.spec.Basic.Deliver(
routing_key=ALERT_STORE_INPUT_ROUTING_KEY)
properties = pika.spec.BasicProperties()
self.test_store._process_data(blocking_channel, method_chains,
properties,
json.dumps(data).encode())
mock_ack.assert_called_once()
mock_send_hb.assert_called_once()
call_1 = call(data['parent_id'], {
'doc_type': 'alert',
'n_alerts': {
'$lt': 1000
}
}, {
'$push': {
'alerts': {
'origin': data['origin_id'],
'alert_name': data['alert_code']['name'],
'severity': data['severity'],
'metric': data['metric'],
'message': data['message'],
'timestamp': str(data['timestamp']),
}
},
'$min': {
'first': data['timestamp']
},
'$max': {
'last': data['timestamp']
},
'$inc': {
'n_alerts': 1
},
})
mock_update_one.assert_has_calls([call_1])
mock_process_redis_store.assert_called_once()
except Exception as e:
self.fail("Test failed: {}".format(e))
@parameterized.expand([
("self.alert_data_1", ),
("self.alert_data_2", ),
("self.alert_data_3", ),
])
@freeze_time("2012-01-01")
@mock.patch("src.data_store.stores.store.RabbitMQApi.basic_ack",
autospec=True)
@mock.patch("src.data_store.stores.alert.AlertStore._process_mongo_store",
autospec=True)
@mock.patch("src.data_store.stores.store.Store._send_heartbeat",
autospec=True)
@mock.patch.object(RedisApi, "hset")
def test_process_data_calls_redis_correctly(self, mock_system_data,
mock_hset, mock_send_hb,
mock_process_mongo_store,
mock_ack) -> None:
mock_ack.return_value = None
try:
self.test_store._initialise_rabbitmq()
data = eval(mock_system_data)
blocking_channel = self.test_store.rabbitmq.channel
method_chains = pika.spec.Basic.Deliver(
routing_key=ALERT_STORE_INPUT_ROUTING_KEY)
properties = pika.spec.BasicProperties()
self.test_store._process_data(blocking_channel, method_chains,
properties,
json.dumps(data).encode())
mock_ack.assert_called_once()
mock_send_hb.assert_called_once()
metric_data = {
'severity': data['severity'],
'message': data['message']
}
key = data['origin_id']
call_1 = call(
Keys.get_hash_parent(data['parent_id']),
eval('Keys.get_alert_{}(key)'.format(data['metric'])),
json.dumps(metric_data))
mock_hset.assert_has_calls([call_1])
mock_process_mongo_store.assert_called_once()
except Exception as e:
self.fail("Test failed: {}".format(e))
@parameterized.expand([
("self.alert_data_1", ),
("self.alert_data_2", ),
("self.alert_data_3", ),
])
def test_process_mongo_store_mongo_stores_correctly(
self, mock_system_data) -> None:
data = eval(mock_system_data)
self.test_store._process_mongo_store(data)
documents = self.mongo.get_all(data['parent_id'])
document = documents[0]
expected = [
'alert', 1,
str(data['origin_id']),
str(data['alert_code']['name']),
str(data['severity']),
str(data['metric']),
str(data['message']),
str(data['timestamp'])
]
actual = [
document['doc_type'], document['n_alerts'],
document['alerts'][0]['origin'],
document['alerts'][0]['alert_name'],
document['alerts'][0]['severity'], document['alerts'][0]['metric'],
document['alerts'][0]['message'],
document['alerts'][0]['timestamp']
]
self.assertListEqual(expected, actual)
@parameterized.expand([
("self.alert_data_1", ),
("self.alert_data_2", ),
("self.alert_data_3", ),
])
def test_process_redis_store_redis_stores_correctly(
self, mock_system_data) -> None:
data = eval(mock_system_data)
self.test_store._process_redis_store(data)
key = data['origin_id']
stored_data = self.redis.hget(
Keys.get_hash_parent(data['parent_id']),
eval('Keys.get_alert_{}(key)'.format(data['metric'])))
expected_data = {
'severity': data['severity'],
'message': data['message']
}
self.assertEqual(expected_data, json.loads(stored_data))
@parameterized.expand([
("self.alert_data_1", ),
("self.alert_data_2", ),
("self.alert_data_3", ),
])
@mock.patch("src.data_store.stores.store.RabbitMQApi.basic_ack",
autospec=True)
@mock.patch("src.data_store.stores.alert.AlertStore._process_redis_store",
autospec=True)
@mock.patch("src.data_store.stores.store.Store._send_heartbeat",
autospec=True)
def test_process_data_results_stores_in_mongo_correctly(
self, mock_system_data, mock_send_hb, mock_process_redis_store,
mock_ack) -> None:
mock_ack.return_value = None
try:
self.test_store._initialise_rabbitmq()
data = eval(mock_system_data)
blocking_channel = self.test_store.rabbitmq.channel
method_chains = pika.spec.Basic.Deliver(
routing_key=ALERT_STORE_INPUT_ROUTING_KEY)
properties = pika.spec.BasicProperties()
self.test_store._process_data(blocking_channel, method_chains,
properties,
json.dumps(data).encode())
mock_process_redis_store.assert_called_once()
mock_ack.assert_called_once()
mock_send_hb.assert_called_once()
documents = self.mongo.get_all(data['parent_id'])
document = documents[0]
expected = [
'alert', 1,
str(data['origin_id']),
str(data['alert_code']['name']),
str(data['severity']),
str(data['message']),
str(data['timestamp'])
]
actual = [
document['doc_type'], document['n_alerts'],
document['alerts'][0]['origin'],
document['alerts'][0]['alert_name'],
document['alerts'][0]['severity'],
document['alerts'][0]['message'],
document['alerts'][0]['timestamp']
]
self.assertListEqual(expected, actual)
except Exception as e:
self.fail("Test failed: {}".format(e))
@parameterized.expand([
("self.alert_data_1", ),
("self.alert_data_2", ),
("self.alert_data_3", ),
])
@mock.patch("src.data_store.stores.store.RabbitMQApi.basic_ack",
autospec=True)
@mock.patch("src.data_store.stores.alert.AlertStore._process_mongo_store",
autospec=True)
@mock.patch("src.data_store.stores.store.Store._send_heartbeat",
autospec=True)
def test_process_data_results_stores_in_redis_correctly(
self, mock_system_data, mock_send_hb, mock_process_mongo_store,
mock_ack) -> None:
mock_ack.return_value = None
try:
self.test_store._initialise_rabbitmq()
data = eval(mock_system_data)
blocking_channel = self.test_store.rabbitmq.channel
method_chains = pika.spec.Basic.Deliver(
routing_key=ALERT_STORE_INPUT_ROUTING_KEY)
properties = pika.spec.BasicProperties()
self.test_store._process_data(blocking_channel, method_chains,
properties,
json.dumps(data).encode())
mock_process_mongo_store.assert_called_once()
mock_ack.assert_called_once()
mock_send_hb.assert_called_once()
key = data['origin_id']
stored_data = self.redis.hget(
Keys.get_hash_parent(data['parent_id']),
eval('Keys.get_alert_{}(key)'.format(data['metric'])))
expected_data = {
'severity': data['severity'],
'message': data['message']
}
self.assertEqual(expected_data, json.loads(stored_data))
except Exception as e:
self.fail("Test failed: {}".format(e))