def test_SubscribeBadDoubleQueueCreate(self, pikaMock):
pikaMock.return_value.channel.return_value.queue_declare.side_effect=[ConnectionClosed(1, "Boom"), ConnectionClosed(1, "Boom2")]
self.broker = self._getBroker()
exchange = "testExchange"
topic1 = "topic.one"
topics = [topic1]
with self.assertRaises(ConnectionClosed):
self.broker.subscribe(self.testCallback, topics, exchange)
self.broker.disconnect()
pikaMock.return_value.channel.return_value.queue_declare.side_effect=[ChannelClosed(1, "Boom"), ChannelClosed(1, "Boom2")]
with self.assertRaises(ChannelClosed):
self.broker.subscribe(self.testCallback, topics, exchange)
self.broker.disconnect()
pikaMock.return_value.channel.return_value.queue_declare.side_effect=[ChannelError("Boom"), ChannelError("Boom2")]
with self.assertRaises(ChannelError):
self.broker.subscribe(self.testCallback, topics, exchange)
self.broker.disconnect()
pikaMock.return_value.channel.return_value.queue_declare.side_effect=[AMQPConnectionError("Boom"), AMQPConnectionError("Boom2")]
with self.assertRaises(AMQPConnectionError):
self.broker.subscribe(self.testCallback, topics, exchange)
self.broker.disconnect()
pikaMock.return_value.channel.return_value.queue_declare.side_effect=[AttributeError("Boom"), AttributeError("Boom2")]
with self.assertRaises(AttributeError):
self.broker.subscribe(self.testCallback, topics, exchange)
self.broker.disconnect()
def __init__(self, config_file_path):
"""
Initialize the object from a configuration file.
@param config_file_path: Absolute path to configuration JSON file.
@type config_file_path: str
"""
super(HalondRMQ, self).__init__(config_file_path)
self._connection = None
self._channel = None
retry_counter = 1
while not(self._connection and self._channel) and retry_counter < 6:
self.init_connection()
if not (self._connection and self._channel):
log.warning(
'RMQ Connection Failed. Retry Attempt: {} in {} secs'.
format(retry_counter, retry_counter * 2 + 60))
time.sleep(retry_counter * 2 + 60)
retry_counter += 1
if not(self._connection and self._channel):
log.warning('RMQ connection Failed. Halon communication channel '
'could not be established.')
log.error('sspl_hl_resp channel creation FAILED. '
'Retry attempts: 3')
raise AMQPConnectionError()
else:
log.info('RMQ connection is Initialized.')
def test_test_connection_fail(self, mock_connection, _, __):
"""
Test the test connection fail returns false when ampq connection error is raised
"""
mock_connection.side_effect = AMQPConnectionError()
result = self.provider.test_connection()
self.assertFalse(result)
def test_SubscribeBadSingleQueueCreate(self, pikaMock):
# ConnectionClosed
pikaMock.return_value.channel.return_value.queue_declare.side_effect=[ConnectionClosed(1, "Boom"), pikaMock.return_value.channel.return_value.queue_declare.return_value]
self.broker = self._getBroker()
exchange = "testExchange"
topic1 = "topic.one"
topics = [topic1]
self.broker.subscribe(self.testCallback, topics, exchange)
self.broker.unsubscribe()
self.broker.disconnect()
pikaMock.return_value.channel.return_value.queue_declare.side_effect=[ChannelClosed(1, "Boom"), pikaMock.return_value.channel.return_value.queue_declare.return_value]
self.broker.subscribe(self.testCallback, topics, exchange)
self.broker.unsubscribe()
self.broker.disconnect()
pikaMock.return_value.channel.return_value.queue_declare.side_effect=[ChannelError("Boom"), pikaMock.return_value.channel.return_value.queue_declare.return_value]
self.broker.subscribe(self.testCallback, topics, exchange)
self.broker.unsubscribe()
self.broker.disconnect()
pikaMock.return_value.channel.return_value.queue_declare.side_effect=[AMQPConnectionError("Boom"), pikaMock.return_value.channel.return_value.queue_declare.return_value]
self.broker.subscribe(self.testCallback, topics, exchange)
self.broker.unsubscribe()
self.broker.disconnect()
pikaMock.return_value.channel.return_value.queue_declare.side_effect=[AttributeError("Boom"), pikaMock.return_value.channel.return_value.queue_declare.return_value]
self.broker.subscribe(self.testCallback, topics, exchange)
self.broker.unsubscribe()
self.broker.disconnect()
class WhenEncounteringConnectionErrors(_BaseTestCase):
__contexts__ = (('TornadoConnection',
patch(mod + '.TornadoConnection',
side_effect=[
AMQPConnectionError(1),
AMQPConnectionError(1), TornadoConnection
])), )
def configure(self):
self.ctx.TornadoConnection._adapter_connect = MagicMock()
self.broker = Broker({'rabbit1': {}, 'rabbit2': {}})
def execute(self):
self.connection = self.broker.connect(connection_attempts=3)
def should_try_try_again(self):
self.assertIsNotNone(self.connection)
def test_rabbitmq_down_when_run_then_raise_busconnectionerror(self):
self.consumer.channel.start_consuming.side_effect = [
AMQPConnectionError()
]
self.consumer.callback = self.callback
self.consumer.queue_name = QUEUE_NAME
self.assertRaises(BusConnectionError, self.consumer.run)
class WhenRecyclingNodesWithDelay(_BaseTestCase):
__contexts__ = (
('TornadoConnection', patch(mod + '.TornadoConnection',
side_effect=[AMQPConnectionError(1),
AMQPConnectionError(1),
TornadoConnection])),
('sleep', patch(mod + '.time.sleep')),
)
def configure(self):
self.ctx.TornadoConnection._adapter_connect = MagicMock()
self.broker = Broker({'rabbit1': {}, 'rabbit2': {}})
def execute(self):
self.connection = self.broker.connect(connection_attempts=5,
cycle_delay=sentinel.delay)
def should_sleep(self):
self.ctx.sleep.assert_called_once_with(sentinel.delay)
class WhenExhaustingNodeList(_BaseTestCase):
__contexts__ = (
('TornadoConnection', patch(mod + '.TornadoConnection',
side_effect=[AMQPConnectionError(1),
AMQPConnectionError(1),
TornadoConnection])),
('_initialize_connection_attempt',
patch(sut + '._initialize_connection_attempt')),
('_increment_connection_attempts',
patch(sut + '._increment_connection_attempts')),
)
nodes = {'rabbit1': {}, 'rabbit2': {}}
def configure(self):
self.broker = Broker(self.nodes)
self.broker._nodes = PropertyMock()
get_node_iterator = lambda: iter(self.nodes.items())
self.broker._nodes.__iter__ = MagicMock(side_effect=get_node_iterator)
def execute(self):
self.connection = self.broker.connect()
def should_recycle_nodes(self):
self.assertEqual(len(self.broker._nodes.__iter__.mock_calls), 2)
def __init__(self, name, **kwargs):
self._name = name
self._channel = None
self._connecting = False
self._connecting_error = False
self._connection = None
self._exchange = None
self._queue = None
self._host = "localhost"
self._port = 5672
self._user = "******"
self._password = "******"
self._virtual_host = "/"
if "exchange" in kwargs:
self._exchange = kwargs['exchange']
else:
raise AMQPConnectionError("It is mandatory to define an exchange.")
if "queue" in kwargs:
self._queue = kwargs['queue']
else:
raise AMQPConnectionError("It is mandatory to define a queue.")
if "host" in kwargs:
self._host = kwargs['host']
if "port" in kwargs:
self._port = kwargs['port']
if "user" in kwargs:
self._user = kwargs['user']
if "password" in kwargs:
self._password = kwargs['password']
if "virtual_host" in kwargs:
self._virtual_host = kwargs['virtual_host']
class WhenExhaustingConnectionAttempts(_BaseTestCase):
__contexts__ = (('TornadoConnection',
patch(mod + '.TornadoConnection',
side_effect=AMQPConnectionError(1))), )
def setUp(self):
pass
def configure(self):
self.broker = Broker({'rabbit1': {}, 'rabbit2': {}})
def should_raise_exception(self):
with self.context() as self.ctx:
self.configure()
with self.assertRaisesRegexp(AMQPConnectionError, r'^2$'):
self.broker.connect()
def health_check_rmq(rmq_config: Dict) -> NoReturn:
"""
Проверяем, прогрузился ли RMQ при старте сборки
docker-compose считает сервис запущенным по факту запуска сервиса,
не проверяя его окончательную загрузку (depends_on в этому случае не спасает),
поэтому зависимый сервис может начать стучаться в непрогруженный RMQ
:param rmq_config:
:return:
"""
from modules.buffering.amqp import create_rmq_connection
from pika.exceptions import AMQPConnectionError
try:
create_rmq_connection(rmq_config)
except AMQPConnectionError as err:
raise AMQPConnectionError("Fail connect to RMQ (%s)" % err)
logger.debug("RMQ OK")
class WhenExhaustingConnectionAttemptsWithCallback(_BaseTestCase):
__contexts__ = (('TornadoConnection',
patch(mod + '.TornadoConnection',
side_effect=AMQPConnectionError(1))),
('BrokerConnectionError',
patch(mod + '.BrokerConnectionError',
return_value=sentinel.broker_error)))
def configure(self):
self.broker = Broker({'rabbit1': {}, 'rabbit2': {}})
self.on_failure_callback = MagicMock()
def execute(self):
self.broker.connect(on_failure_callback=self.on_failure_callback)
def should_invoke_on_failure_callback(self):
self.on_failure_callback.assert_called_once_with(sentinel.broker_error)
class AsyncoreDispatcher(asyncore.dispatcher):
"""
We extend asyncore.dispatcher here and throw in everything we need to
handle both asyncore's needs and pika's. In the async adapter structure
we expect a ioloop behavior which includes timeouts and a start and stop
function.
"""
def __init__(self, parameters):
"""
Initialize the dispatcher, socket and our defaults. We turn of nageling
in the socket to allow for faster throughput.
"""
asyncore.dispatcher.__init__(self)
# Carry the parameters for this as well
self.parameters = parameters
# Setup defaults
self.connecting = True
self.connection = None
self._timeouts = dict()
self.map = None
# Set our remaining attempts to the value or True if it's none
remaining_attempts = self.parameters.connection_attempts or True
# Loop while we have remaining attempts
while remaining_attempts:
try:
self._socket_connect()
return
except socket.error, err:
remaining_attempts -= 1
if not remaining_attempts:
break
LOGGER.warning(
"Could not connect: %s. Retrying in %i seconds "
"with %i retry(s) left", err[-1],
self.parameters.retry_delay, remaining_attempts)
self.socket.close()
time.sleep(self.parameters.retry_delay)
# Log the errors and raise the exception
LOGGER.error("Could not connect: %s", err[-1])
raise AMQPConnectionError(err[-1])
def _reap_last_connection_workflow_error(error):
"""Extract exception value from the last connection attempt
:param Exception error: error passed by the `AMQPConnectionWorkflow`
completion callback.
:returns: Exception value from the last connection attempt
:rtype: Exception
"""
if isinstance(error, AMQPConnectionWorkflowFailed):
# Extract exception value from the last connection attempt
error = error.exceptions[-1]
if isinstance(error, AMQPConnectorSocketConnectError):
error = AMQPConnectionError(error)
elif isinstance(error, AMQPConnectorPhaseErrorBase):
error = error.exception
return error
def _adapter_connect(self):
BaseConnection._adapter_connect(self)
self.socket.setblocking(1)
# Set the timeout for reading/writing on the socket
self.socket.settimeout(self.parameters.socket_timeout or SOCKET_TIMEOUT)
self._socket_timeouts = 0
self._on_connected()
self._timeouts = dict()
# When using a high availability cluster (such as HAProxy) we are always able to connect
# even though there might be no RabbitMQ backend.
socket_timeout_retries = 0
while not self.is_open and socket_timeout_retries<SOCKET_TIMEOUT_THRESHOLD:
self._flush_outbound()
self._handle_read()
timeout_retries +=1
if not self.is_open:
raise AMQPConnectionError("No connection could be opened after %s retries" % SOCKET_TIMEOUT_THRESHOLD)
return self
class WhenEncounteringConnectionError(_BaseTestCase):
__contexts__ = (
('TornadoConnection', patch(mod + '.TornadoConnection',
side_effect=[AMQPConnectionError(1),
TornadoConnection])),
('_initialize_connection_attempt',
patch(sut + '._initialize_connection_attempt')),
('_increment_connection_attempts',
patch(sut + '._increment_connection_attempts')),
)
nodes = {'rabbit1': {}, 'rabbit2': {}}
def configure(self):
self.broker = Broker(self.nodes)
self.broker._nodes = PropertyMock()
self.broker._nodes.__iter__ = MagicMock(
return_value=iter(self.nodes.items()))
def execute(self):
self.connection = self.broker.connect()
def should_iterate_over_nodes(self):
self.broker._nodes.__iter__.assert_called_once_with()
class WhenAbortingConnectionAttempt(_BaseTestCase):
__contexts__ = (
('TornadoConnection', patch(mod + '.TornadoConnection',
side_effect=AMQPConnectionError(1))),
('_initialize_connection_attempt',
patch(sut + '._initialize_connection_attempt')),
('_increment_connection_attempts',
patch(sut + '._increment_connection_attempts')),
)
def setUp(self):
pass
def configure(self):
self.broker = Broker()
self.broker._abort_on_error = True
self.broker._attempts = 1
def should_raise_exception(self):
with self.context() as self.ctx:
self.configure()
with self.assertRaisesRegexp(AMQPConnectionError, '^1$'):
self.broker.connect()
class WhenAbortingConnectionAttemptWithCallback(_BaseTestCase):
__contexts__ = (
('TornadoConnection', patch(mod + '.TornadoConnection',
side_effect=AMQPConnectionError(1))),
('BrokerConnectionError', patch(mod + '.BrokerConnectionError')),
('_initialize_connection_attempt',
patch(sut + '._initialize_connection_attempt')),
('_increment_connection_attempts',
patch(sut + '._increment_connection_attempts')),
)
def configure(self):
self.broker = Broker()
self.broker._abort_on_error = True
self.broker._attempts = 1
self.on_failure_callback = MagicMock(autospec=True)
self.ctx.BrokerConnectionError.return_value = sentinel.exception
def execute(self):
self.broker.connect(on_failure_callback=self.on_failure_callback)
def should_invoke_on_failure_callback(self):
self.on_failure_callback.assert_called_once_with(sentinel.exception)
def throw_connection_error_once(self_obj: Any, *args: Any,
**kwargs: Any) -> None:
self.counter += 1
if self.counter <= 1:
raise AMQPConnectionError("test")
actual_publish(*args, **kwargs)
reason = err[-1]
self.socket.close()
retry = ''
if remaining_attempts:
retry = "Retrying in %i seconds with %i retry(s) left" % \
(self.parameters.retry_delay, remaining_attempts)
log.warning("Could not connect: %s. %s", reason, retry)
if remaining_attempts:
time.sleep(self.parameters.retry_delay)
# Log the errors and raise the exception
log.error("Could not connect: %s", reason)
raise AMQPConnectionError(reason)
def add_timeout(self, deadline, callback):
return self.ioloop.add_timeout(deadline, callback)
def remove_timeout(self, timeout_id):
self.ioloop.remove_timeout(timeout_id)
def _erase_credentials(self):
pass
def _flush_outbound(self):
"""
Call the state manager who will figure out that we need to write.
"""
self._manage_event_state()
class TestTwilioAlertsHandler(unittest.TestCase):
def setUp(self) -> None:
self.test_handler_name = 'test_twilio_alerts_handler'
self.dummy_logger = logging.getLogger('Dummy')
self.dummy_logger.disabled = True
self.connection_check_time_interval = timedelta(seconds=0)
self.rabbit_ip = env.RABBIT_IP
self.rabbitmq = RabbitMQApi(
self.dummy_logger,
self.rabbit_ip,
connection_check_time_interval=self.connection_check_time_interval)
self.test_channel_name = 'test_twilio_channel'
self.test_channel_id = 'test_twilio_id12345'
self.test_channel_logger = self.dummy_logger.getChild('twilio_channel')
self.test_account_sid = 'test_account_sid'
self.test_auth_token = 'test_auth_token'
self.test_call_from = '+35699999999'
self.test_call_to = ['+35611111111', '+35644545454', '+35634343434']
self.test_twiml = '<Response><Reject/></Response>'
self.test_twiml_is_url = False
self.test_api = TwilioApi(self.test_account_sid, self.test_auth_token)
self.test_channel = TwilioChannel(self.test_channel_name,
self.test_channel_id,
self.test_channel_logger,
self.test_api)
self.test_max_attempts = 3
self.test_alert_validity_threshold = 300
self.test_twilio_alerts_handler = TwilioAlertsHandler(
self.test_handler_name, self.dummy_logger, self.rabbitmq,
self.test_channel, self.test_call_from, self.test_call_to,
self.test_twiml, self.test_twiml_is_url, self.test_max_attempts,
self.test_alert_validity_threshold)
self.test_data_str = "this is a test string"
self.test_rabbit_queue_name = 'Test Queue'
self.test_timestamp = 45676565.556
self.test_heartbeat = {
'component_name': 'Test Component',
'is_alive': True,
'timestamp': self.test_timestamp,
}
self.test_system_name = 'test_system'
self.test_percentage_usage = 50
self.test_panic_severity = 'WARNING'
self.test_parent_id = 'parent_1234'
self.test_system_id = 'system_id32423'
self.test_alert = OpenFileDescriptorsIncreasedAboveThresholdAlert(
self.test_system_name, self.test_percentage_usage,
self.test_panic_severity, self.test_timestamp,
self.test_panic_severity, self.test_parent_id, self.test_system_id)
def tearDown(self) -> None:
# Delete any queues and exchanges which are common across many tests
connect_to_rabbit(self.test_twilio_alerts_handler.rabbitmq)
delete_queue_if_exists(self.test_twilio_alerts_handler.rabbitmq,
self.test_rabbit_queue_name)
delete_queue_if_exists(
self.test_twilio_alerts_handler.rabbitmq,
self.test_twilio_alerts_handler._twilio_alerts_handler_queue)
delete_exchange_if_exists(self.test_twilio_alerts_handler.rabbitmq,
HEALTH_CHECK_EXCHANGE)
delete_exchange_if_exists(self.test_twilio_alerts_handler.rabbitmq,
ALERT_EXCHANGE)
disconnect_from_rabbit(self.test_twilio_alerts_handler.rabbitmq)
self.dummy_logger = None
self.test_channel_logger = None
self.rabbitmq = None
self.test_alert = None
self.test_channel = None
self.test_api = None
self.test_twilio_alerts_handler = None
def test__str__returns_handler_name(self) -> None:
self.assertEqual(self.test_handler_name,
str(self.test_twilio_alerts_handler))
def test_handler_name_returns_handler_name(self) -> None:
self.assertEqual(self.test_handler_name,
self.test_twilio_alerts_handler.handler_name)
@mock.patch.object(RabbitMQApi, "start_consuming")
def test_listen_for_data_calls_start_consuming(
self, mock_start_consuming) -> None:
mock_start_consuming.return_value = None
self.test_twilio_alerts_handler._listen_for_data()
mock_start_consuming.assert_called_once_with()
def test_twilio_channel_returns_associated_twilio_channel(self) -> None:
self.assertEqual(self.test_channel,
self.test_twilio_alerts_handler.twilio_channel)
def test_init_initialises_handler_correctly(self) -> None:
# In this test we will check that all fields that do not have a getter
# were initialised correctly, as the previous tests test the getters.
self.assertEqual(self.test_call_from,
self.test_twilio_alerts_handler._call_from)
self.assertEqual(self.test_call_to,
self.test_twilio_alerts_handler._call_to)
self.assertEqual(self.test_twiml,
self.test_twilio_alerts_handler._twiml)
self.assertEqual(self.test_twiml_is_url,
self.test_twilio_alerts_handler._twiml_is_url)
self.assertEqual(self.test_max_attempts,
self.test_twilio_alerts_handler._max_attempts)
self.assertEqual(
self.test_alert_validity_threshold,
self.test_twilio_alerts_handler._alert_validity_threshold)
self.assertEqual(
'twilio_{}_alerts_handler_queue'.format(self.test_channel_id),
self.test_twilio_alerts_handler._twilio_alerts_handler_queue)
self.assertEqual(
'channel.{}'.format(self.test_channel_id),
self.test_twilio_alerts_handler._twilio_channel_routing_key)
@mock.patch.object(RabbitMQApi, "basic_qos")
def test_initialise_rabbitmq_initialises_rabbit_correctly(
self, mock_basic_qos) -> None:
try:
# To make sure that there is no connection/channel already
# established
self.assertIsNone(self.rabbitmq.connection)
self.assertIsNone(self.rabbitmq.channel)
# To make sure that the exchanges and queues have not already been
# declared
connect_to_rabbit(self.rabbitmq)
self.test_twilio_alerts_handler.rabbitmq.queue_delete(
self.test_twilio_alerts_handler._twilio_alerts_handler_queue)
self.test_twilio_alerts_handler.rabbitmq.exchange_delete(
HEALTH_CHECK_EXCHANGE)
self.test_twilio_alerts_handler.rabbitmq.exchange_delete(
ALERT_EXCHANGE)
disconnect_from_rabbit(self.rabbitmq)
self.test_twilio_alerts_handler._initialise_rabbitmq()
# Perform checks that the connection has been opened and marked as
# open, that the delivery confirmation variable is set and basic_qos
# called successfully.
self.assertTrue(
self.test_twilio_alerts_handler.rabbitmq.is_connected)
self.assertTrue(
self.test_twilio_alerts_handler.rabbitmq.connection.is_open)
self.assertTrue(self.test_twilio_alerts_handler.rabbitmq.channel.
_delivery_confirmation)
mock_basic_qos.assert_called_once_with(prefetch_count=200)
# Check whether the producing exchanges have been created by
# using passive=True. If this check fails an exception is raised
# automatically.
self.test_twilio_alerts_handler.rabbitmq.exchange_declare(
HEALTH_CHECK_EXCHANGE, passive=True)
# Check whether the consuming exchanges and queues have been
# creating by sending messages with the same routing keys as for the
# bindings. We will also check if the size of the queues is 0 to
# confirm that basic_consume was called (it will store the msg in
# the component memory immediately). If one of the exchanges or
# queues is not created or basic_consume is not called, then either
# an exception will be thrown or the queue size would be 1
# respectively. Note when deleting the exchanges in the beginning we
# also released every binding, hence there are no other queue binded
# with the same routing key to any exchange at this point.
self.test_twilio_alerts_handler.rabbitmq.basic_publish_confirm(
exchange=ALERT_EXCHANGE,
routing_key=self.test_twilio_alerts_handler.
_twilio_channel_routing_key,
body=self.test_data_str,
is_body_dict=False,
properties=pika.BasicProperties(delivery_mode=2),
mandatory=True)
# Re-declare queue to get the number of messages
res = self.test_twilio_alerts_handler.rabbitmq.queue_declare(
self.test_twilio_alerts_handler._twilio_alerts_handler_queue,
False, True, False, False)
self.assertEqual(0, res.method.message_count)
except Exception as e:
self.fail("Test failed: {}".format(e))
def test_send_heartbeat_sends_a_heartbeat_correctly(self) -> None:
# This test creates a queue which receives messages with the same
# routing key as the ones set by send_heartbeat, and checks that the
# heartbeat is received
try:
self.test_twilio_alerts_handler._initialise_rabbitmq()
# Delete the queue before to avoid messages in the queue on error.
self.test_twilio_alerts_handler.rabbitmq.queue_delete(
self.test_rabbit_queue_name)
res = self.test_twilio_alerts_handler.rabbitmq.queue_declare(
queue=self.test_rabbit_queue_name,
durable=True,
exclusive=False,
auto_delete=False,
passive=False)
self.assertEqual(0, res.method.message_count)
self.test_twilio_alerts_handler.rabbitmq.queue_bind(
queue=self.test_rabbit_queue_name,
exchange=HEALTH_CHECK_EXCHANGE,
routing_key='heartbeat.worker')
self.test_twilio_alerts_handler._send_heartbeat(
self.test_heartbeat)
# By re-declaring the queue again we can get the number of messages
# in the queue.
res = self.test_twilio_alerts_handler.rabbitmq.queue_declare(
queue=self.test_rabbit_queue_name,
durable=True,
exclusive=False,
auto_delete=False,
passive=True)
self.assertEqual(1, res.method.message_count)
# Check that the message received is actually the HB
_, _, body = self.test_twilio_alerts_handler.rabbitmq.basic_get(
self.test_rabbit_queue_name)
self.assertEqual(self.test_heartbeat, json.loads(body))
except Exception as e:
self.fail("Test failed: {}".format(e))
@mock.patch.object(TwilioAlertsHandler, "_call_using_twilio")
@mock.patch.object(RabbitMQApi, "basic_ack")
def test_process_alert_calls_using_twilio_if_no_processing_errors(
self, mock_basic_ack, mock_call_using_twilio) -> None:
# Setting it to failed so that there is no attempt to send the heartbeat
mock_call_using_twilio.return_value = RequestStatus.FAILED
mock_basic_ack.return_value = None
try:
self.test_twilio_alerts_handler._initialise_rabbitmq()
blocking_channel = self.test_twilio_alerts_handler.rabbitmq.channel
method = pika.spec.Basic.Deliver(
routing_key=self.test_twilio_alerts_handler.
_twilio_channel_routing_key)
body = json.dumps(self.test_alert.alert_data)
properties = pika.spec.BasicProperties()
# Send alert
self.test_twilio_alerts_handler._process_alert(
blocking_channel, method, properties, body)
args, _ = mock_call_using_twilio.call_args
self.assertEqual(self.test_alert.alert_data, args[0].alert_data)
self.assertEqual(1, len(args))
except Exception as e:
self.fail("Test failed: {}".format(e))
mock_basic_ack.assert_called_once()
@mock.patch.object(TwilioAlertsHandler, "_call_using_twilio")
@mock.patch.object(RabbitMQApi, "basic_ack")
def test_process_alert_does_not_call_using_twilio_if_processing_errors(
self, mock_basic_ack, mock_call_using_twilio) -> None:
# Setting it to failed so that there is no attempt to send the heartbeat
mock_call_using_twilio.return_value = RequestStatus.FAILED
mock_basic_ack.return_value = None
try:
self.test_twilio_alerts_handler._initialise_rabbitmq()
blocking_channel = self.test_twilio_alerts_handler.rabbitmq.channel
method = pika.spec.Basic.Deliver(
routing_key=self.test_twilio_alerts_handler.
_twilio_channel_routing_key)
data_to_send = copy.deepcopy(self.test_alert.alert_data)
del data_to_send['message']
body = json.dumps(data_to_send)
properties = pika.spec.BasicProperties()
# Send alert
self.test_twilio_alerts_handler._process_alert(
blocking_channel, method, properties, body)
mock_call_using_twilio.assert_not_called()
except Exception as e:
self.fail("Test failed: {}".format(e))
mock_basic_ack.assert_called_once()
@freeze_time("2012-01-01")
@mock.patch.object(TwilioAlertsHandler, "_send_heartbeat")
@mock.patch.object(TwilioAlertsHandler, "_call_using_twilio")
@mock.patch.object(RabbitMQApi, "basic_ack")
def test_process_alert_sends_hb_if_no_processing_error_and_call_successful(
self, mock_basic_ack, mock_call_using_twilio,
mock_send_hb) -> None:
# Setting it to failed so that there is no attempt to send the heartbeat
mock_call_using_twilio.return_value = RequestStatus.SUCCESS
mock_basic_ack.return_value = None
mock_send_hb.return_value = None
try:
self.test_twilio_alerts_handler._initialise_rabbitmq()
blocking_channel = self.test_twilio_alerts_handler.rabbitmq.channel
method = pika.spec.Basic.Deliver(
routing_key=self.test_twilio_alerts_handler.
_twilio_channel_routing_key)
body = json.dumps(self.test_alert.alert_data)
properties = pika.spec.BasicProperties()
# Send alert
self.test_twilio_alerts_handler._process_alert(
blocking_channel, method, properties, body)
expected_heartbeat = {
'component_name': self.test_handler_name,
'is_alive': True,
'timestamp': datetime.now().timestamp()
}
mock_send_hb.assert_called_once_with(expected_heartbeat)
except Exception as e:
self.fail("Test failed: {}".format(e))
mock_basic_ack.assert_called_once()
@parameterized.expand([
(RequestStatus.SUCCESS, ),
(RequestStatus.FAILED, ),
])
@mock.patch.object(TwilioAlertsHandler, "_send_heartbeat")
@mock.patch.object(TwilioAlertsHandler, "_call_using_twilio")
@mock.patch.object(RabbitMQApi, "basic_ack")
def test_process_alert_does_not_send_heartbeat_if_processing_error(
self, call_request_status, mock_basic_ack, mock_call_using_twilio,
mock_send_hb) -> None:
# Setting it to failed so that there is no attempt to send the heartbeat
mock_call_using_twilio.return_value = call_request_status
mock_basic_ack.return_value = None
mock_send_hb.return_value = None
try:
self.test_twilio_alerts_handler._initialise_rabbitmq()
blocking_channel = self.test_twilio_alerts_handler.rabbitmq.channel
method = pika.spec.Basic.Deliver(
routing_key=self.test_twilio_alerts_handler.
_twilio_channel_routing_key)
data_to_send = copy.deepcopy(self.test_alert.alert_data)
del data_to_send['message']
body = json.dumps(data_to_send)
properties = pika.spec.BasicProperties()
# Send alert
self.test_twilio_alerts_handler._process_alert(
blocking_channel, method, properties, body)
mock_send_hb.assert_not_called()
except Exception as e:
self.fail("Test failed: {}".format(e))
mock_basic_ack.assert_called_once()
@mock.patch.object(TwilioAlertsHandler, "_send_heartbeat")
@mock.patch.object(TwilioAlertsHandler, "_call_using_twilio")
@mock.patch.object(RabbitMQApi, "basic_ack")
def test_process_alert_does_not_send_heartbeat_if_call_request_fails(
self, mock_basic_ack, mock_call_using_twilio,
mock_send_hb) -> None:
# Setting it to failed so that there is no attempt to send the heartbeat
mock_call_using_twilio.return_value = RequestStatus.FAILED
mock_basic_ack.return_value = None
mock_send_hb.return_value = None
try:
self.test_twilio_alerts_handler._initialise_rabbitmq()
blocking_channel = self.test_twilio_alerts_handler.rabbitmq.channel
method = pika.spec.Basic.Deliver(
routing_key=self.test_twilio_alerts_handler.
_twilio_channel_routing_key)
body = json.dumps(self.test_alert.alert_data)
properties = pika.spec.BasicProperties()
# First test with valid alert json
self.test_twilio_alerts_handler._process_alert(
blocking_channel, method, properties, body)
# Test with invalid alert json
data_to_send = copy.deepcopy(self.test_alert.alert_data)
del data_to_send['message']
body = json.dumps(data_to_send)
self.test_twilio_alerts_handler._process_alert(
blocking_channel, method, properties, body)
mock_send_hb.assert_not_called()
except Exception as e:
self.fail("Test failed: {}".format(e))
args, _ = mock_basic_ack.call_args
self.assertEqual(2, len(args))
@mock.patch.object(TwilioAlertsHandler, "_send_heartbeat")
@mock.patch.object(TwilioAlertsHandler, "_call_using_twilio")
@mock.patch.object(RabbitMQApi, "basic_ack")
def test_process_alert_does_not_raise_msg_not_delivered_exception(
self, mock_basic_ack, mock_call_using_twilio,
mock_send_hb) -> None:
mock_basic_ack.return_value = None
mock_call_using_twilio.retrurn_value = RequestStatus.SUCCESS
mock_send_hb.side_effect = MessageWasNotDeliveredException('test')
try:
self.test_twilio_alerts_handler._initialise_rabbitmq()
blocking_channel = self.test_twilio_alerts_handler.rabbitmq.channel
method = pika.spec.Basic.Deliver(
routing_key=self.test_twilio_alerts_handler.
_twilio_channel_routing_key)
body = json.dumps(self.test_alert.alert_data)
properties = pika.spec.BasicProperties()
# This would raise a MessageWasNotDeliveredException if raised,
# hence the test would fail
self.test_twilio_alerts_handler._process_alert(
blocking_channel, method, properties, body)
except Exception as e:
self.fail("Test failed: {}".format(e))
mock_basic_ack.assert_called_once()
@parameterized.expand([
(
AMQPConnectionError,
AMQPConnectionError('test'),
),
(
AMQPChannelError,
AMQPChannelError('test'),
),
(
Exception,
Exception('test'),
),
])
@mock.patch.object(TwilioAlertsHandler, "_send_heartbeat")
@mock.patch.object(TwilioAlertsHandler, "_call_using_twilio")
@mock.patch.object(RabbitMQApi, "basic_ack")
def test_process_alert_raises_error_if_raised_by_send_hb(
self, exception_class, exception_instance, mock_basic_ack,
mock_call_using_twilio, mock_send_hb) -> None:
# For this test we will check for channel, connection and unexpected
# errors.
mock_basic_ack.return_value = None
mock_call_using_twilio.return_value = RequestStatus.SUCCESS
mock_send_hb.side_effect = exception_instance
try:
self.test_twilio_alerts_handler._initialise_rabbitmq()
blocking_channel = self.test_twilio_alerts_handler.rabbitmq.channel
method = pika.spec.Basic.Deliver(
routing_key=self.test_twilio_alerts_handler.
_twilio_channel_routing_key)
body = json.dumps(self.test_alert.alert_data)
properties = pika.spec.BasicProperties()
self.assertRaises(exception_class,
self.test_twilio_alerts_handler._process_alert,
blocking_channel, method, properties, body)
except Exception as e:
self.fail("Test failed: {}".format(e))
mock_basic_ack.assert_called_once()
@parameterized.expand([
(1, ),
(20, ),
])
@freeze_time("2012-01-01")
@mock.patch.object(BlockingConnection, "sleep")
@mock.patch.object(TwilioChannel, "alert")
def test_call_using_twilio_does_not_call_if_validity_threshold_exceeded(
self, exceeding_factor, mock_alert, mock_sleep) -> None:
mock_alert.return_value = None
mock_sleep.return_value = None
test_alert = OpenFileDescriptorsIncreasedAboveThresholdAlert(
self.test_system_name, self.test_percentage_usage,
self.test_panic_severity,
datetime.now().timestamp() - self.test_alert_validity_threshold -
exceeding_factor, self.test_panic_severity, self.test_parent_id,
self.test_system_id)
ret = self.test_twilio_alerts_handler._call_using_twilio(test_alert)
self.assertEqual(RequestStatus.FAILED, ret)
mock_alert.assert_not_called()
@parameterized.expand([
(0, ),
(1, ),
(20, ),
])
@freeze_time("2012-01-01")
@mock.patch.object(BlockingConnection, "sleep")
@mock.patch.object(TwilioChannel, "alert")
def test_call_using_twilio_calls_all_if_validity_threshold_not_exceeded(
self, exceeding_factor, mock_alert, mock_sleep) -> None:
mock_alert.return_value = RequestStatus.SUCCESS
mock_sleep.return_value = None
test_alert = OpenFileDescriptorsIncreasedAboveThresholdAlert(
self.test_system_name, self.test_percentage_usage,
self.test_panic_severity,
datetime.now().timestamp() - self.test_alert_validity_threshold +
exceeding_factor, self.test_panic_severity, self.test_parent_id,
self.test_system_id)
ret = self.test_twilio_alerts_handler._call_using_twilio(test_alert)
self.assertEqual(RequestStatus.SUCCESS, ret)
expected_calls = [
call(call_from=self.test_call_from,
call_to=self.test_call_to[0],
twiml=self.test_twiml,
twiml_is_url=self.test_twiml_is_url),
call(call_from=self.test_call_from,
call_to=self.test_call_to[1],
twiml=self.test_twiml,
twiml_is_url=self.test_twiml_is_url),
call(call_from=self.test_call_from,
call_to=self.test_call_to[2],
twiml=self.test_twiml,
twiml_is_url=self.test_twiml_is_url)
]
actual_calls = mock_alert.call_args_list
self.assertEqual(expected_calls, actual_calls)
@parameterized.expand([
([
RequestStatus.FAILED, RequestStatus.FAILED, RequestStatus.FAILED,
RequestStatus.SUCCESS, RequestStatus.SUCCESS
], [0], [3, 0, 0], RequestStatus.FAILED),
([
RequestStatus.FAILED, RequestStatus.FAILED, RequestStatus.FAILED,
RequestStatus.FAILED, RequestStatus.FAILED, RequestStatus.FAILED,
RequestStatus.SUCCESS
], [0, 1], [3, 3, 0], RequestStatus.FAILED),
([
RequestStatus.FAILED, RequestStatus.FAILED, RequestStatus.FAILED,
RequestStatus.FAILED, RequestStatus.FAILED, RequestStatus.FAILED,
RequestStatus.FAILED, RequestStatus.FAILED, RequestStatus.FAILED
], [0, 1, 2], [3, 3, 3], RequestStatus.FAILED),
([
RequestStatus.SUCCESS, RequestStatus.FAILED, RequestStatus.FAILED,
RequestStatus.FAILED, RequestStatus.SUCCESS
], [1], [0, 3, 0], RequestStatus.FAILED),
([
RequestStatus.SUCCESS,
RequestStatus.FAILED,
RequestStatus.FAILED,
RequestStatus.FAILED,
RequestStatus.FAILED,
RequestStatus.FAILED,
RequestStatus.FAILED,
], [1, 2], [0, 3, 3], RequestStatus.FAILED),
([
RequestStatus.SUCCESS, RequestStatus.SUCCESS, RequestStatus.FAILED,
RequestStatus.FAILED, RequestStatus.FAILED
], [2], [0, 0, 3], RequestStatus.FAILED),
([
RequestStatus.FAILED,
RequestStatus.FAILED,
RequestStatus.FAILED,
RequestStatus.SUCCESS,
RequestStatus.FAILED,
RequestStatus.FAILED,
RequestStatus.FAILED,
], [0, 2], [3, 0, 3], RequestStatus.FAILED),
([
RequestStatus.FAILED, RequestStatus.FAILED, RequestStatus.FAILED,
RequestStatus.FAILED, RequestStatus.FAILED, RequestStatus.FAILED,
RequestStatus.FAILED, RequestStatus.FAILED, RequestStatus.FAILED
], [0, 1, 2], [3, 3, 3], RequestStatus.FAILED),
([
RequestStatus.FAILED, RequestStatus.SUCCESS, RequestStatus.SUCCESS,
RequestStatus.SUCCESS
], [0], [2, 0, 0], RequestStatus.SUCCESS),
([
RequestStatus.FAILED, RequestStatus.FAILED, RequestStatus.SUCCESS,
RequestStatus.SUCCESS, RequestStatus.SUCCESS
], [0], [3, 0, 0], RequestStatus.SUCCESS),
([
RequestStatus.SUCCESS, RequestStatus.FAILED, RequestStatus.SUCCESS,
RequestStatus.SUCCESS
], [1], [0, 2, 0], RequestStatus.SUCCESS),
([
RequestStatus.SUCCESS, RequestStatus.FAILED, RequestStatus.FAILED,
RequestStatus.SUCCESS, RequestStatus.SUCCESS
], [1], [0, 3, 0], RequestStatus.SUCCESS),
([
RequestStatus.SUCCESS, RequestStatus.SUCCESS, RequestStatus.FAILED,
RequestStatus.SUCCESS
], [2], [0, 0, 2], RequestStatus.SUCCESS),
([
RequestStatus.SUCCESS, RequestStatus.SUCCESS, RequestStatus.FAILED,
RequestStatus.FAILED, RequestStatus.SUCCESS
], [2], [0, 0, 3], RequestStatus.SUCCESS),
])
@mock.patch.object(RabbitMQApi, "connection")
@mock.patch.object(TwilioChannel, "alert")
def test_call_using_twilio_attempts_calling_max_attempts_times_for_everyone(
self, alert_request_status, failed_callee_index, amount_of_calls,
expected_ret, mock_alert, mock_connection) -> None:
# Here we will assume that the self._alert_validity_threshold is not
# exceeded
mock_alert.side_effect = alert_request_status
mock_connection.return_value.sleep.return_value = None
successful_callee_index = [
i for i in range(len(self.test_call_to))
if i not in failed_callee_index
]
expected_calls = []
test_alert = OpenFileDescriptorsIncreasedAboveThresholdAlert(
self.test_system_name, self.test_percentage_usage,
self.test_panic_severity,
datetime.now().timestamp(), self.test_panic_severity,
self.test_parent_id, self.test_system_id)
for index in range(len(self.test_call_to)):
if index in failed_callee_index:
for _ in range(amount_of_calls[index]):
expected_calls.append(
call(call_from=self.test_call_from,
call_to=self.test_call_to[index],
twiml=self.test_twiml,
twiml_is_url=self.test_twiml_is_url))
elif index in successful_callee_index:
expected_calls.append(
call(call_from=self.test_call_from,
call_to=self.test_call_to[index],
twiml=self.test_twiml,
twiml_is_url=self.test_twiml_is_url))
ret = self.test_twilio_alerts_handler._call_using_twilio(test_alert)
self.assertEqual(expected_ret, ret)
actual_calls = mock_alert.call_args_list
self.assertEqual(expected_calls, actual_calls)
class TestLogAlertsHandler(unittest.TestCase):
def setUp(self) -> None:
self.test_handler_name = 'test_log_alerts_handler'
self.dummy_logger = logging.getLogger('Dummy')
self.dummy_logger.disabled = True
self.connection_check_time_interval = timedelta(seconds=0)
self.rabbit_ip = env.RABBIT_IP
self.rabbitmq = RabbitMQApi(
self.dummy_logger,
self.rabbit_ip,
connection_check_time_interval=self.connection_check_time_interval)
self.test_channel_name = 'test_logger_channel'
self.test_channel_id = 'test_logger1234'
self.test_channel_logger = self.dummy_logger.getChild('dummy_channel')
self.alerts_logger = self.dummy_logger.getChild('alerts_logger')
self.test_channel = LogChannel(self.test_channel_name,
self.test_channel_id,
self.test_channel_logger,
self.alerts_logger)
self.test_log_alerts_handler = LogAlertsHandler(
self.test_handler_name, self.dummy_logger, self.rabbitmq,
self.test_channel)
self.test_data_str = "this is a test string"
self.test_rabbit_queue_name = 'Test Queue'
self.test_timestamp = 45676565.556
self.test_heartbeat = {
'component_name': 'Test Component',
'is_alive': True,
'timestamp': self.test_timestamp,
}
self.test_system_name = 'test_system'
self.test_percentage_usage = 50
self.test_panic_severity = 'WARNING'
self.test_parent_id = 'parent_1234'
self.test_system_id = 'system_id32423'
self.test_alert = OpenFileDescriptorsIncreasedAboveThresholdAlert(
self.test_system_name, self.test_percentage_usage,
self.test_panic_severity, self.test_timestamp,
self.test_panic_severity, self.test_parent_id, self.test_system_id)
def tearDown(self) -> None:
# Delete any queues and exchanges which are common across many tests
connect_to_rabbit(self.test_log_alerts_handler.rabbitmq)
delete_queue_if_exists(self.test_log_alerts_handler.rabbitmq,
self.test_rabbit_queue_name)
delete_queue_if_exists(
self.test_log_alerts_handler.rabbitmq,
self.test_log_alerts_handler._log_alerts_handler_queue)
delete_exchange_if_exists(self.test_log_alerts_handler.rabbitmq,
HEALTH_CHECK_EXCHANGE)
delete_exchange_if_exists(self.test_log_alerts_handler.rabbitmq,
ALERT_EXCHANGE)
disconnect_from_rabbit(self.test_log_alerts_handler.rabbitmq)
self.dummy_logger = None
self.test_channel_logger = None
self.alerts_logger = None
self.rabbitmq = None
self.test_channel = None
self.test_log_alerts_handler = None
self.test_alert = None
def test__str__returns_handler_name(self) -> None:
self.assertEqual(self.test_handler_name,
str(self.test_log_alerts_handler))
def test_handler_name_returns_handler_name(self) -> None:
self.assertEqual(self.test_handler_name,
self.test_log_alerts_handler.handler_name)
@mock.patch.object(RabbitMQApi, "start_consuming")
def test_listen_for_data_calls_start_consuming(
self, mock_start_consuming) -> None:
mock_start_consuming.return_value = None
self.test_log_alerts_handler._listen_for_data()
mock_start_consuming.assert_called_once_with()
def test_log_channel_returns_associated_log_channel(self) -> None:
self.assertEqual(self.test_channel,
self.test_log_alerts_handler.log_channel)
def test_init_initialises_handler_correctly(self) -> None:
# In this test we will check that all fields that do not have a getter
# were initialised correctly, as the previous tests test the getters.
self.assertEqual(
'log_{}_alerts_handler_queue'.format(self.test_channel_id),
self.test_log_alerts_handler._log_alerts_handler_queue)
@mock.patch.object(RabbitMQApi, "basic_qos")
def test_initialise_rabbitmq_initialises_rabbit_correctly(
self, mock_basic_qos) -> None:
try:
# To make sure that there is no connection/channel already
# established
self.assertIsNone(self.rabbitmq.connection)
self.assertIsNone(self.rabbitmq.channel)
# To make sure that the exchanges and queues have not already been
# declared
connect_to_rabbit(self.rabbitmq)
self.test_log_alerts_handler.rabbitmq.queue_delete(
self.test_log_alerts_handler._log_alerts_handler_queue)
self.test_log_alerts_handler.rabbitmq.exchange_delete(
HEALTH_CHECK_EXCHANGE)
self.test_log_alerts_handler.rabbitmq.exchange_delete(
ALERT_EXCHANGE)
disconnect_from_rabbit(self.rabbitmq)
self.test_log_alerts_handler._initialise_rabbitmq()
# Perform checks that the connection has been opened and marked as
# open, that the delivery confirmation variable is set and basic_qos
# called successfully.
self.assertTrue(self.test_log_alerts_handler.rabbitmq.is_connected)
self.assertTrue(
self.test_log_alerts_handler.rabbitmq.connection.is_open)
self.assertTrue(self.test_log_alerts_handler.rabbitmq.channel.
_delivery_confirmation)
mock_basic_qos.assert_called_once_with(prefetch_count=200)
# Check whether the producing exchanges have been created by
# using passive=True. If this check fails an exception is raised
# automatically.
self.test_log_alerts_handler.rabbitmq.exchange_declare(
HEALTH_CHECK_EXCHANGE, passive=True)
# Check whether the consuming exchanges and queues have been
# creating by sending messages with the same routing keys as for the
# bindings. We will also check if the size of the queues is 0 to
# confirm that basic_consume was called (it will store the msg in
# the component memory immediately). If one of the exchanges or
# queues is not created or basic_consume is not called, then either
# an exception will be thrown or the queue size would be 1
# respectively. Note when deleting the exchanges in the beginning we
# also released every binding, hence there are no other queue binded
# with the same routing key to any exchange at this point.
self.test_log_alerts_handler.rabbitmq.basic_publish_confirm(
exchange=ALERT_EXCHANGE,
routing_key=LOG_HANDLER_INPUT_ROUTING_KEY,
body=self.test_data_str,
is_body_dict=False,
properties=pika.BasicProperties(delivery_mode=2),
mandatory=True)
# Re-declare queue to get the number of messages
res = self.test_log_alerts_handler.rabbitmq.queue_declare(
self.test_log_alerts_handler._log_alerts_handler_queue, False,
True, False, False)
self.assertEqual(0, res.method.message_count)
except Exception as e:
self.fail("Test failed: {}".format(e))
def test_send_heartbeat_sends_a_heartbeat_correctly(self) -> None:
# This test creates a queue which receives messages with the same
# routing key as the ones set by send_heartbeat, and checks that the
# heartbeat is received
try:
self.test_log_alerts_handler._initialise_rabbitmq()
# Delete the queue before to avoid messages in the queue on error.
self.test_log_alerts_handler.rabbitmq.queue_delete(
self.test_rabbit_queue_name)
res = self.test_log_alerts_handler.rabbitmq.queue_declare(
queue=self.test_rabbit_queue_name,
durable=True,
exclusive=False,
auto_delete=False,
passive=False)
self.assertEqual(0, res.method.message_count)
self.test_log_alerts_handler.rabbitmq.queue_bind(
queue=self.test_rabbit_queue_name,
exchange=HEALTH_CHECK_EXCHANGE,
routing_key='heartbeat.worker')
self.test_log_alerts_handler._send_heartbeat(self.test_heartbeat)
# By re-declaring the queue again we can get the number of messages
# in the queue.
res = self.test_log_alerts_handler.rabbitmq.queue_declare(
queue=self.test_rabbit_queue_name,
durable=True,
exclusive=False,
auto_delete=False,
passive=True)
self.assertEqual(1, res.method.message_count)
# Check that the message received is actually the HB
_, _, body = self.test_log_alerts_handler.rabbitmq.basic_get(
self.test_rabbit_queue_name)
self.assertEqual(self.test_heartbeat, json.loads(body))
except Exception as e:
self.fail("Test failed: {}".format(e))
@mock.patch.object(LogChannel, "alert")
@mock.patch.object(RabbitMQApi, "basic_ack")
def test_process_alert_sends_alert_to_the_logs_if_no_processing_errors(
self, mock_basic_ack, mock_alert) -> None:
# Setting it to failed so that there is no attempt to send the heartbeat
mock_alert.return_value = RequestStatus.FAILED
mock_basic_ack.return_value = None
try:
self.test_log_alerts_handler._initialise_rabbitmq()
blocking_channel = self.test_log_alerts_handler.rabbitmq.channel
method = pika.spec.Basic.Deliver(
routing_key=LOG_HANDLER_INPUT_ROUTING_KEY)
body = json.dumps(self.test_alert.alert_data)
properties = pika.spec.BasicProperties()
# Send alert
self.test_log_alerts_handler._process_alert(
blocking_channel, method, properties, body)
args, _ = mock_alert.call_args
self.assertEqual(self.test_alert.alert_data, args[0].alert_data)
self.assertEqual(1, len(args))
except Exception as e:
self.fail("Test failed: {}".format(e))
mock_basic_ack.assert_called_once()
@mock.patch.object(LogChannel, "alert")
@mock.patch.object(RabbitMQApi, "basic_ack")
def test_process_alert_does_not_send_alert_to_the_logs_if_processing_errors(
self, mock_basic_ack, mock_alert) -> None:
# Setting it to failed so that there is no attempt to send the heartbeat
mock_alert.return_value = RequestStatus.FAILED
mock_basic_ack.return_value = None
try:
self.test_log_alerts_handler._initialise_rabbitmq()
blocking_channel = self.test_log_alerts_handler.rabbitmq.channel
method = pika.spec.Basic.Deliver(
routing_key=LOG_HANDLER_INPUT_ROUTING_KEY)
data_to_send = copy.deepcopy(self.test_alert.alert_data)
del data_to_send['message']
body = json.dumps(data_to_send)
properties = pika.spec.BasicProperties()
# Send alert
self.test_log_alerts_handler._process_alert(
blocking_channel, method, properties, body)
mock_alert.assert_not_called()
except Exception as e:
self.fail("Test failed: {}".format(e))
mock_basic_ack.assert_called_once()
@freeze_time("2012-01-01")
@mock.patch.object(LogAlertsHandler, "_send_heartbeat")
@mock.patch.object(LogChannel, "alert")
@mock.patch.object(RabbitMQApi, "basic_ack")
def test_process_alert_sends_hb_if_no_process_err_and_alert_sent_to_logs(
self, mock_basic_ack, mock_alert, mock_send_heartbeat) -> None:
mock_basic_ack.return_value = None
mock_alert.return_value = RequestStatus.SUCCESS
mock_send_heartbeat.return_value = None
try:
self.test_log_alerts_handler._initialise_rabbitmq()
blocking_channel = self.test_log_alerts_handler.rabbitmq.channel
method = pika.spec.Basic.Deliver(
routing_key=LOG_HANDLER_INPUT_ROUTING_KEY)
body = json.dumps(self.test_alert.alert_data)
properties = pika.spec.BasicProperties()
# Send alert
self.test_log_alerts_handler._process_alert(
blocking_channel, method, properties, body)
expected_heartbeat = {
'component_name': self.test_handler_name,
'is_alive': True,
'timestamp': datetime.now().timestamp()
}
mock_send_heartbeat.assert_called_once_with(expected_heartbeat)
except Exception as e:
self.fail("Test failed: {}".format(e))
mock_basic_ack.assert_called_once()
@parameterized.expand([
(RequestStatus.SUCCESS, ),
(RequestStatus.FAILED, ),
])
@mock.patch.object(LogAlertsHandler, "_send_heartbeat")
@mock.patch.object(LogChannel, "alert")
@mock.patch.object(RabbitMQApi, "basic_ack")
def test_process_alert_does_not_send_hb_if_processing_error(
self, alert_return, mock_basic_ack, mock_alert,
mock_send_heartbeat) -> None:
mock_basic_ack.return_value = None
mock_alert.return_value = alert_return
mock_send_heartbeat.return_value = None
try:
self.test_log_alerts_handler._initialise_rabbitmq()
blocking_channel = self.test_log_alerts_handler.rabbitmq.channel
method = pika.spec.Basic.Deliver(
routing_key=LOG_HANDLER_INPUT_ROUTING_KEY)
data_to_send = copy.deepcopy(self.test_alert.alert_data)
del data_to_send['message']
body = json.dumps(data_to_send)
properties = pika.spec.BasicProperties()
# Send alert
self.test_log_alerts_handler._process_alert(
blocking_channel, method, properties, body)
mock_send_heartbeat.assert_not_called()
except Exception as e:
self.fail("Test failed: {}".format(e))
mock_basic_ack.assert_called_once()
@mock.patch.object(LogAlertsHandler, "_send_heartbeat")
@mock.patch.object(LogChannel, "alert")
@mock.patch.object(RabbitMQApi, "basic_ack")
def test_process_alert_does_not_send_hb_if_alert_was_not_sent(
self, mock_basic_ack, mock_alert, mock_send_heartbeat) -> None:
mock_basic_ack.return_value = None
mock_alert.return_value = RequestStatus.FAILED
mock_send_heartbeat.return_value = None
try:
self.test_log_alerts_handler._initialise_rabbitmq()
blocking_channel = self.test_log_alerts_handler.rabbitmq.channel
method = pika.spec.Basic.Deliver(
routing_key=LOG_HANDLER_INPUT_ROUTING_KEY)
body = json.dumps(self.test_alert.alert_data)
properties = pika.spec.BasicProperties()
# First test with valid alert
self.test_log_alerts_handler._process_alert(
blocking_channel, method, properties, body)
# Test with an invalid alert dict
invalid_alert = copy.deepcopy(self.test_alert.alert_data)
del invalid_alert['message']
body = json.dumps(invalid_alert)
self.test_log_alerts_handler._process_alert(
blocking_channel, method, properties, body)
mock_send_heartbeat.assert_not_called()
except Exception as e:
self.fail("Test failed: {}".format(e))
args, _ = mock_basic_ack.call_args
self.assertEqual(2, len(args))
@mock.patch.object(LogAlertsHandler, "_send_heartbeat")
@mock.patch.object(LogChannel, "alert")
@mock.patch.object(RabbitMQApi, "basic_ack")
def test_process_alert_does_not_raise_msg_not_delivered_exception(
self, mock_basic_ack, mock_alert, mock_send_heartbeat) -> None:
mock_basic_ack.return_value = None
mock_alert.return_value = RequestStatus.SUCCESS
mock_send_heartbeat.side_effect = MessageWasNotDeliveredException(
'test')
try:
self.test_log_alerts_handler._initialise_rabbitmq()
blocking_channel = self.test_log_alerts_handler.rabbitmq.channel
method = pika.spec.Basic.Deliver(
routing_key=LOG_HANDLER_INPUT_ROUTING_KEY)
body = json.dumps(self.test_alert.alert_data)
properties = pika.spec.BasicProperties()
# This would raise a MessageWasNotDeliveredException if raised,
# hence the test would fail
self.test_log_alerts_handler._process_alert(
blocking_channel, method, properties, body)
except Exception as e:
self.fail("Test failed: {}".format(e))
mock_basic_ack.assert_called_once()
@parameterized.expand([
(
AMQPConnectionError,
AMQPConnectionError('test'),
),
(
AMQPChannelError,
AMQPChannelError('test'),
),
(
Exception,
Exception('test'),
),
])
@mock.patch.object(LogAlertsHandler, "_send_heartbeat")
@mock.patch.object(LogChannel, "alert")
@mock.patch.object(RabbitMQApi, "basic_ack")
def test_process_alert_raises_error_if_raised_by_send_hb(
self, exception_class, exception_instance, mock_basic_ack,
mock_alert, mock_send_heartbeat) -> None:
# For this test we will check for channel, connection and unexpected
# errors.
mock_basic_ack.return_value = None
mock_alert.return_value = RequestStatus.SUCCESS
mock_send_heartbeat.side_effect = exception_instance
try:
self.test_log_alerts_handler._initialise_rabbitmq()
blocking_channel = self.test_log_alerts_handler.rabbitmq.channel
method = pika.spec.Basic.Deliver(
routing_key=LOG_HANDLER_INPUT_ROUTING_KEY)
body = json.dumps(self.test_alert.alert_data)
properties = pika.spec.BasicProperties()
self.assertRaises(exception_class,
self.test_log_alerts_handler._process_alert,
blocking_channel, method, properties, body)
except Exception as e:
self.fail("Test failed: {}".format(e))
mock_basic_ack.assert_called_once()
class BaseConnection(Connection):
def __init__(self, parameters=None,
on_open_callback=None,
reconnection_strategy=None):
# Let the developer know we could not import SSL
if parameters.ssl and not SSL:
raise Exception("SSL specified but it is not available")
# Set our defaults
self.fd = None
self.ioloop = None
self.socket = None
# Event states (base and current)
self.base_events = READ | ERROR
self.event_state = self.base_events
# Call our parent's __init__
Connection.__init__(self, parameters, on_open_callback,
reconnection_strategy)
def _socket_connect(self):
"""Create socket and connect to it, using SSL if enabled"""
# Create our socket and set our socket options
self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
self.socket.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, 1)
# Wrap the SSL socket if we SSL turned on
ssl_text = ""
if self.parameters.ssl:
ssl_text = " with SSL"
if self.parameters.ssl_options:
self.socket = ssl.wrap_socket(self.socket,
**self.parameters.ssl_options)
else:
self.socket = ssl.wrap_socket(self.socket)
# Try and connect
log.info("Connecting to %s:%i%s", self.parameters.host,
self.parameters.port, ssl_text)
self.socket.connect((self.parameters.host, self.parameters.port))
# Set the socket to non-blocking
self.socket.setblocking(0)
def _adapter_connect(self):
"""
Base connection function to be extended as needed
"""
# Set our remaining attempts to the value or True if it's none
remaining_attempts = self.parameters.connection_attempts or True
# Loop while we have remaining attempts
while remaining_attempts:
try:
return self._socket_connect()
except socket.error, err:
remaining_attempts -= 1
if not remaining_attempts:
break
log.warning("Could not connect: %s. Retrying in %i seconds \
with %i retry(s) left",
err[-1], self.parameters.retry_delay,
remaining_attempts)
self.socket.close()
sleep(self.parameters.retry_delay)
# Log the errors and raise the exception
log.error("Could not connect: %s", err[-1])
raise AMQPConnectionError(err[-1])
class TestOpsgenieAlertsHandler(unittest.TestCase):
def setUp(self) -> None:
self.test_handler_name = 'test_opsgenie_alerts_handler'
self.dummy_logger = logging.getLogger('Dummy')
self.dummy_logger.disabled = True
self.connection_check_time_interval = timedelta(seconds=0)
self.rabbit_ip = env.RABBIT_IP
self.rabbitmq = RabbitMQApi(
self.dummy_logger,
self.rabbit_ip,
connection_check_time_interval=self.connection_check_time_interval)
self.test_api_key = 'test api key'
self.test_channel_name = 'test_opgenie_channel'
self.test_channel_id = 'test_opsgenie_id12345'
self.test_channel_logger = self.dummy_logger.getChild('dummy_channel')
self.test_api = OpsgenieApi(self.test_api_key, True)
self.test_channel = OpsgenieChannel(self.test_channel_name,
self.test_channel_id,
self.test_channel_logger,
self.test_api)
self.test_queue_size = 1000
self.test_max_attempts = 5
self.test_alert_validity_threshold = 300
self.test_opsgenie_alerts_handler = OpsgenieAlertsHandler(
self.test_handler_name, self.dummy_logger, self.rabbitmq,
self.test_channel, self.test_queue_size, self.test_max_attempts,
self.test_alert_validity_threshold)
self.test_data_str = "this is a test string"
self.test_rabbit_queue_name = 'Test Queue'
self.test_timestamp = 45676565.556
self.test_heartbeat = {
'component_name': 'Test Component',
'is_alive': True,
'timestamp': self.test_timestamp,
}
self.test_system_name = 'test_system'
self.test_percentage_usage = 50
self.test_panic_severity = 'WARNING'
self.test_parent_id = 'parent_1234'
self.test_system_id = 'system_id32423'
self.test_alert = OpenFileDescriptorsIncreasedAboveThresholdAlert(
self.test_system_name, self.test_percentage_usage,
self.test_panic_severity, self.test_timestamp,
self.test_panic_severity, self.test_parent_id, self.test_system_id)
self.test_alerts_queue = Queue(self.test_queue_size)
def tearDown(self) -> None:
# Delete any queues and exchanges which are common across many tests
connect_to_rabbit(self.test_opsgenie_alerts_handler.rabbitmq)
delete_queue_if_exists(self.test_opsgenie_alerts_handler.rabbitmq,
self.test_rabbit_queue_name)
delete_queue_if_exists(
self.test_opsgenie_alerts_handler.rabbitmq,
self.test_opsgenie_alerts_handler._opsgenie_alerts_handler_queue)
delete_exchange_if_exists(self.test_opsgenie_alerts_handler.rabbitmq,
HEALTH_CHECK_EXCHANGE)
delete_exchange_if_exists(self.test_opsgenie_alerts_handler.rabbitmq,
ALERT_EXCHANGE)
disconnect_from_rabbit(self.test_opsgenie_alerts_handler.rabbitmq)
self.dummy_logger = None
self.test_channel_logger = None
self.rabbitmq = None
self.test_alert = None
self.test_channel = None
self.test_api = None
self.test_opsgenie_alerts_handler = None
self.test_alerts_queue = None
def test__str__returns_handler_name(self) -> None:
self.assertEqual(self.test_handler_name,
str(self.test_opsgenie_alerts_handler))
def test_handler_name_returns_handler_name(self) -> None:
self.assertEqual(self.test_handler_name,
self.test_opsgenie_alerts_handler.handler_name)
@mock.patch.object(RabbitMQApi, "start_consuming")
def test_listen_for_data_calls_start_consuming(
self, mock_start_consuming) -> None:
mock_start_consuming.return_value = None
self.test_opsgenie_alerts_handler._listen_for_data()
mock_start_consuming.assert_called_once_with()
def test_opsgenie_channel_returns_associated_opsgenie_channel(
self) -> None:
self.assertEqual(self.test_channel,
self.test_opsgenie_alerts_handler.opsgenie_channel)
def test_alerts_queue_returns_the_alerts_queue(self) -> None:
self.test_opsgenie_alerts_handler._alerts_queue = self.test_alerts_queue
self.assertEqual(self.test_alerts_queue,
self.test_opsgenie_alerts_handler.alerts_queue)
def test_init_initialises_handler_correctly(self) -> None:
# In this test we will check that all fields that do not have a getter
# were initialised correctly, as the previous tests test the getters.
self.assertEqual(
self.test_queue_size,
self.test_opsgenie_alerts_handler.alerts_queue.maxsize)
self.assertEqual(self.test_max_attempts,
self.test_opsgenie_alerts_handler._max_attempts)
self.assertEqual(
self.test_alert_validity_threshold,
self.test_opsgenie_alerts_handler._alert_validity_threshold)
self.assertEqual(
'opsgenie_{}_alerts_handler_queue'.format(self.test_channel_id),
self.test_opsgenie_alerts_handler._opsgenie_alerts_handler_queue)
self.assertEqual(
'channel.{}'.format(self.test_channel_id),
self.test_opsgenie_alerts_handler._opsgenie_channel_routing_key)
@mock.patch.object(RabbitMQApi, "basic_qos")
def test_initialise_rabbitmq_initialises_rabbit_correctly(
self, mock_basic_qos) -> None:
try:
# To make sure that there is no connection/channel already
# established
self.assertIsNone(self.rabbitmq.connection)
self.assertIsNone(self.rabbitmq.channel)
# To make sure that the exchanges and queues have not already been
# declared
connect_to_rabbit(self.rabbitmq)
self.test_opsgenie_alerts_handler.rabbitmq.queue_delete(
self.test_opsgenie_alerts_handler.
_opsgenie_alerts_handler_queue)
self.test_opsgenie_alerts_handler.rabbitmq.exchange_delete(
HEALTH_CHECK_EXCHANGE)
self.test_opsgenie_alerts_handler.rabbitmq.exchange_delete(
ALERT_EXCHANGE)
disconnect_from_rabbit(self.rabbitmq)
self.test_opsgenie_alerts_handler._initialise_rabbitmq()
# Perform checks that the connection has been opened and marked as
# open, that the delivery confirmation variable is set and basic_qos
# called successfully.
self.assertTrue(
self.test_opsgenie_alerts_handler.rabbitmq.is_connected)
self.assertTrue(
self.test_opsgenie_alerts_handler.rabbitmq.connection.is_open)
self.assertTrue(self.test_opsgenie_alerts_handler.rabbitmq.channel.
_delivery_confirmation)
mock_basic_qos.assert_called_once_with(
prefetch_count=self.test_queue_size / 5)
# Check whether the producing exchanges have been created by
# using passive=True. If this check fails an exception is raised
# automatically.
self.test_opsgenie_alerts_handler.rabbitmq.exchange_declare(
HEALTH_CHECK_EXCHANGE, passive=True)
# Check whether the consuming exchanges and queues have been
# creating by sending messages with the same routing keys as for the
# bindings. We will also check if the size of the queues is 0 to
# confirm that basic_consume was called (it will store the msg in
# the component memory immediately). If one of the exchanges or
# queues is not created or basic_consume is not called, then either
# an exception will be thrown or the queue size would be 1
# respectively. Note when deleting the exchanges in the beginning we
# also released every binding, hence there are no other queue binded
# with the same routing key to any exchange at this point.
self.test_opsgenie_alerts_handler.rabbitmq.basic_publish_confirm(
exchange=ALERT_EXCHANGE,
routing_key=self.test_opsgenie_alerts_handler.
_opsgenie_channel_routing_key,
body=self.test_data_str,
is_body_dict=False,
properties=pika.BasicProperties(delivery_mode=2),
mandatory=True)
# Re-declare queue to get the number of messages
res = self.test_opsgenie_alerts_handler.rabbitmq.queue_declare(
self.test_opsgenie_alerts_handler.
_opsgenie_alerts_handler_queue, False, True, False, False)
self.assertEqual(0, res.method.message_count)
except Exception as e:
self.fail("Test failed: {}".format(e))
def test_send_heartbeat_sends_a_heartbeat_correctly(self) -> None:
# This test creates a queue which receives messages with the same
# routing key as the ones set by send_heartbeat, and checks that the
# heartbeat is received
try:
self.test_opsgenie_alerts_handler._initialise_rabbitmq()
# Delete the queue before to avoid messages in the queue on error.
self.test_opsgenie_alerts_handler.rabbitmq.queue_delete(
self.test_rabbit_queue_name)
res = self.test_opsgenie_alerts_handler.rabbitmq.queue_declare(
queue=self.test_rabbit_queue_name,
durable=True,
exclusive=False,
auto_delete=False,
passive=False)
self.assertEqual(0, res.method.message_count)
self.test_opsgenie_alerts_handler.rabbitmq.queue_bind(
queue=self.test_rabbit_queue_name,
exchange=HEALTH_CHECK_EXCHANGE,
routing_key='heartbeat.worker')
self.test_opsgenie_alerts_handler._send_heartbeat(
self.test_heartbeat)
# By re-declaring the queue again we can get the number of messages
# in the queue.
res = self.test_opsgenie_alerts_handler.rabbitmq.queue_declare(
queue=self.test_rabbit_queue_name,
durable=True,
exclusive=False,
auto_delete=False,
passive=True)
self.assertEqual(1, res.method.message_count)
# Check that the message received is actually the HB
_, _, body = self.test_opsgenie_alerts_handler.rabbitmq.basic_get(
self.test_rabbit_queue_name)
self.assertEqual(self.test_heartbeat, json.loads(body))
except Exception as e:
self.fail("Test failed: {}".format(e))
@mock.patch.object(Queue, "empty")
@mock.patch.object(OpsgenieAlertsHandler, "_send_alerts")
@mock.patch.object(OpsgenieAlertsHandler, "_place_alert_on_queue")
@mock.patch.object(RabbitMQApi, "basic_ack")
def test_process_alert_places_data_on_queue_if_no_processing_errors(
self, mock_basic_ack, mock_place_alert, mock_send_alerts,
mock_empty) -> None:
# Setting it to non empty so that there is no attempt to send the
# heartbeat
mock_empty.return_value = False
mock_place_alert.return_value = None
mock_basic_ack.return_value = None
mock_send_alerts.return_value = None
try:
self.test_opsgenie_alerts_handler._initialise_rabbitmq()
blocking_channel = \
self.test_opsgenie_alerts_handler.rabbitmq.channel
method = pika.spec.Basic.Deliver(
routing_key=self.test_opsgenie_alerts_handler.
_opsgenie_channel_routing_key)
body = json.dumps(self.test_alert.alert_data)
properties = pika.spec.BasicProperties()
# Send alert
self.test_opsgenie_alerts_handler._process_alert(
blocking_channel, method, properties, body)
args, _ = mock_place_alert.call_args
self.assertEqual(self.test_alert.alert_data, args[0].alert_data)
self.assertEqual(1, len(args))
except Exception as e:
self.fail("Test failed: {}".format(e))
mock_basic_ack.assert_called_once()
@mock.patch.object(Queue, "empty")
@mock.patch.object(OpsgenieAlertsHandler, "_send_alerts")
@mock.patch.object(OpsgenieAlertsHandler, "_place_alert_on_queue")
@mock.patch.object(RabbitMQApi, "basic_ack")
def test_process_alert_does_not_place_data_on_queue_if_processing_errors(
self, mock_basic_ack, mock_place_alert, mock_send_alerts,
mock_empty) -> None:
# Setting it to non empty so that there is no attempt to send the
# heartbeat
mock_empty.return_value = False
mock_place_alert.return_value = None
mock_basic_ack.return_value = None
mock_send_alerts.return_value = None
try:
self.test_opsgenie_alerts_handler._initialise_rabbitmq()
blocking_channel = \
self.test_opsgenie_alerts_handler.rabbitmq.channel
method = pika.spec.Basic.Deliver(
routing_key=self.test_opsgenie_alerts_handler.
_opsgenie_channel_routing_key)
data_to_send = copy.deepcopy(self.test_alert.alert_data)
del data_to_send['message']
body = json.dumps(data_to_send)
properties = pika.spec.BasicProperties()
# Send alert
self.test_opsgenie_alerts_handler._process_alert(
blocking_channel, method, properties, body)
mock_place_alert.assert_not_called()
except Exception as e:
self.fail("Test failed: {}".format(e))
mock_basic_ack.assert_called_once()
@mock.patch.object(Queue, "empty")
@mock.patch.object(OpsgenieAlertsHandler, "_send_alerts")
@mock.patch.object(OpsgenieAlertsHandler, "_place_alert_on_queue")
@mock.patch.object(RabbitMQApi, "basic_ack")
def test_process_alert_sends_data_waiting_in_queue_if_processing_errors(
self, mock_basic_ack, mock_place_alert, mock_send_alerts,
mock_empty) -> None:
# Setting it to non empty so that there is no attempt to send the
# heartbeat
mock_empty.return_value = False
mock_place_alert.return_value = None
mock_basic_ack.return_value = None
mock_send_alerts.return_value = None
try:
self.test_opsgenie_alerts_handler._initialise_rabbitmq()
blocking_channel = \
self.test_opsgenie_alerts_handler.rabbitmq.channel
method = pika.spec.Basic.Deliver(
routing_key=self.test_opsgenie_alerts_handler.
_opsgenie_channel_routing_key)
data_to_send = copy.deepcopy(self.test_alert.alert_data)
del data_to_send['message']
body = json.dumps(data_to_send)
properties = pika.spec.BasicProperties()
# Send alert
self.test_opsgenie_alerts_handler._process_alert(
blocking_channel, method, properties, body)
mock_send_alerts.assert_called_once_with()
except Exception as e:
self.fail("Test failed: {}".format(e))
mock_basic_ack.assert_called_once()
@mock.patch.object(Queue, "empty")
@mock.patch.object(OpsgenieAlertsHandler, "_send_alerts")
@mock.patch.object(OpsgenieAlertsHandler, "_place_alert_on_queue")
@mock.patch.object(RabbitMQApi, "basic_ack")
def test_process_alert_sends_data_waiting_in_queue_if_no_processing_errors(
self, mock_basic_ack, mock_place_alert, mock_send_alerts,
mock_empty) -> None:
# Setting it to non empty so that there is no attempt to send the
# heartbeat
mock_empty.return_value = False
mock_place_alert.return_value = None
mock_basic_ack.return_value = None
mock_send_alerts.return_value = None
try:
self.test_opsgenie_alerts_handler._initialise_rabbitmq()
blocking_channel = \
self.test_opsgenie_alerts_handler.rabbitmq.channel
method = pika.spec.Basic.Deliver(
routing_key=self.test_opsgenie_alerts_handler.
_opsgenie_channel_routing_key)
body = json.dumps(self.test_alert.alert_data)
properties = pika.spec.BasicProperties()
# Send alert
self.test_opsgenie_alerts_handler._process_alert(
blocking_channel, method, properties, body)
mock_send_alerts.assert_called_once_with()
except Exception as e:
self.fail("Test failed: {}".format(e))
mock_basic_ack.assert_called_once()
@parameterized.expand([
(
AMQPConnectionError,
AMQPConnectionError('test'),
),
(
AMQPChannelError,
AMQPChannelError('test'),
),
(
Exception,
Exception('test'),
),
(PANICException, PANICException('test', 4000)),
])
@mock.patch.object(Queue, "empty")
@mock.patch.object(OpsgenieAlertsHandler, "_send_alerts")
@mock.patch.object(OpsgenieAlertsHandler, "_place_alert_on_queue")
@mock.patch.object(RabbitMQApi, "basic_ack")
def test_process_alert_raises_exception_if_sending_data_from_queue_error(
self, error_class, error_instance, mock_basic_ack,
mock_place_alert, mock_send_alerts, mock_empty) -> None:
# Setting it to non empty so that there is no attempt to send the
# heartbeat
mock_empty.return_value = False
mock_place_alert.return_value = None
mock_basic_ack.return_value = None
mock_send_alerts.side_effect = error_instance
try:
self.test_opsgenie_alerts_handler._initialise_rabbitmq()
blocking_channel = \
self.test_opsgenie_alerts_handler.rabbitmq.channel
method = pika.spec.Basic.Deliver(
routing_key=self.test_opsgenie_alerts_handler.
_opsgenie_channel_routing_key)
body = json.dumps(self.test_alert.alert_data)
properties = pika.spec.BasicProperties()
self.assertRaises(error_class,
self.test_opsgenie_alerts_handler._process_alert,
blocking_channel, method, properties, body)
except Exception as e:
self.fail("Test failed: {}".format(e))
mock_basic_ack.assert_called_once()
@freeze_time("2012-01-01")
@mock.patch.object(Queue, "empty")
@mock.patch.object(OpsgenieAlertsHandler, "_send_heartbeat")
@mock.patch.object(OpsgenieAlertsHandler, "_send_alerts")
@mock.patch.object(OpsgenieAlertsHandler, "_place_alert_on_queue")
@mock.patch.object(RabbitMQApi, "basic_ack")
def test_process_alert_sends_hb_if_data_sent_and_no_processing_errors(
self, mock_basic_ack, mock_place_alert, mock_send_alerts,
mock_send_heartbeat, mock_empty) -> None:
# Setting it to non empty so that there is no attempt to send the
# heartbeat
mock_empty.return_value = True
mock_place_alert.return_value = None
mock_basic_ack.return_value = None
mock_send_alerts.return_value = None
mock_send_heartbeat.return_value = None
try:
self.test_opsgenie_alerts_handler._initialise_rabbitmq()
blocking_channel = \
self.test_opsgenie_alerts_handler.rabbitmq.channel
method = pika.spec.Basic.Deliver(
routing_key=self.test_opsgenie_alerts_handler.
_opsgenie_channel_routing_key)
body = json.dumps(self.test_alert.alert_data)
properties = pika.spec.BasicProperties()
# Send alert
self.test_opsgenie_alerts_handler._process_alert(
blocking_channel, method, properties, body)
expected_heartbeat = {
'component_name': self.test_handler_name,
'is_alive': True,
'timestamp': datetime.now().timestamp()
}
mock_send_heartbeat.assert_called_once_with(expected_heartbeat)
except Exception as e:
self.fail("Test failed: {}".format(e))
mock_basic_ack.assert_called_once()
@mock.patch.object(Queue, "empty")
@mock.patch.object(OpsgenieAlertsHandler, "_send_heartbeat")
@mock.patch.object(OpsgenieAlertsHandler, "_send_alerts")
@mock.patch.object(OpsgenieAlertsHandler, "_place_alert_on_queue")
@mock.patch.object(RabbitMQApi, "basic_ack")
def test_process_alert_does_not_send_hb_if_not_all_data_sent_from_queue(
self, mock_basic_ack, mock_place_alert, mock_send_alerts,
mock_send_heartbeat, mock_empty) -> None:
mock_empty.return_value = False
mock_place_alert.return_value = None
mock_basic_ack.return_value = None
mock_send_alerts.return_value = None
mock_send_heartbeat.return_value = None
try:
self.test_opsgenie_alerts_handler._initialise_rabbitmq()
blocking_channel = \
self.test_opsgenie_alerts_handler.rabbitmq.channel
method = pika.spec.Basic.Deliver(
routing_key=self.test_opsgenie_alerts_handler.
_opsgenie_channel_routing_key)
body = json.dumps(self.test_alert.alert_data)
properties = pika.spec.BasicProperties()
# First test with a valid alert
self.test_opsgenie_alerts_handler._process_alert(
blocking_channel, method, properties, body)
# Test with an invalid alert dict
invalid_alert = copy.deepcopy(self.test_alert.alert_data)
del invalid_alert['message']
body = json.dumps(invalid_alert)
self.test_opsgenie_alerts_handler._process_alert(
blocking_channel, method, properties, body)
mock_send_heartbeat.assert_not_called()
except Exception as e:
self.fail("Test failed: {}".format(e))
args, _ = mock_basic_ack.call_args
self.assertEqual(2, len(args))
@parameterized.expand([
(True, ),
(False, ),
])
@mock.patch.object(Queue, "empty")
@mock.patch.object(OpsgenieAlertsHandler, "_send_heartbeat")
@mock.patch.object(OpsgenieAlertsHandler, "_send_alerts")
@mock.patch.object(OpsgenieAlertsHandler, "_place_alert_on_queue")
@mock.patch.object(RabbitMQApi, "basic_ack")
def test_process_alert_does_not_send_hb_if_processing_error(
self, is_queue_empty, mock_basic_ack, mock_place_alert,
mock_send_alerts, mock_send_heartbeat, mock_empty) -> None:
mock_empty.return_value = is_queue_empty
mock_place_alert.return_value = None
mock_basic_ack.return_value = None
mock_send_alerts.return_value = None
mock_send_heartbeat.return_value = None
try:
self.test_opsgenie_alerts_handler._initialise_rabbitmq()
blocking_channel = \
self.test_opsgenie_alerts_handler.rabbitmq.channel
method = pika.spec.Basic.Deliver(
routing_key=self.test_opsgenie_alerts_handler.
_opsgenie_channel_routing_key)
invalid_alert = copy.deepcopy(self.test_alert.alert_data)
del invalid_alert['message']
body = json.dumps(invalid_alert)
properties = pika.spec.BasicProperties()
# Send alert
self.test_opsgenie_alerts_handler._process_alert(
blocking_channel, method, properties, body)
mock_send_heartbeat.assert_not_called()
except Exception as e:
self.fail("Test failed: {}".format(e))
mock_basic_ack.assert_called_once()
@parameterized.expand([
(
True,
AMQPConnectionError,
AMQPConnectionError('test'),
),
(
True,
AMQPChannelError,
AMQPChannelError('test'),
),
(
True,
Exception,
Exception('test'),
),
(True, PANICException, PANICException('test', 4000)),
(
False,
AMQPConnectionError,
AMQPConnectionError('test'),
),
(
False,
AMQPChannelError,
AMQPChannelError('test'),
),
(
False,
Exception,
Exception('test'),
),
(False, PANICException, PANICException('test', 4000)),
])
@mock.patch.object(Queue, "empty")
@mock.patch.object(OpsgenieAlertsHandler, "_send_heartbeat")
@mock.patch.object(OpsgenieAlertsHandler, "_send_alerts")
@mock.patch.object(OpsgenieAlertsHandler, "_place_alert_on_queue")
@mock.patch.object(RabbitMQApi, "basic_ack")
def test_process_alert_does_not_send_hb_if_error_raised_when_sending_data(
self, is_queue_empty, error_class, error_instance, mock_basic_ack,
mock_place_alert, mock_send_alerts, mock_send_heartbeat,
mock_empty) -> None:
mock_empty.return_value = is_queue_empty
mock_place_alert.return_value = None
mock_basic_ack.return_value = None
mock_send_alerts.side_effect = error_instance
mock_send_heartbeat.return_value = None
try:
self.test_opsgenie_alerts_handler._initialise_rabbitmq()
blocking_channel = \
self.test_opsgenie_alerts_handler.rabbitmq.channel
method = pika.spec.Basic.Deliver(
routing_key=self.test_opsgenie_alerts_handler.
_opsgenie_channel_routing_key)
body = json.dumps(self.test_alert.alert_data)
properties = pika.spec.BasicProperties()
# Send with a valid alert
self.assertRaises(error_class,
self.test_opsgenie_alerts_handler._process_alert,
blocking_channel, method, properties, body)
# Test with an invalid alert
invalid_alert = copy.deepcopy(self.test_alert.alert_data)
del invalid_alert['message']
body = json.dumps(invalid_alert)
self.assertRaises(error_class,
self.test_opsgenie_alerts_handler._process_alert,
blocking_channel, method, properties, body)
mock_send_heartbeat.assert_not_called()
except Exception as e:
self.fail("Test failed: {}".format(e))
args, _ = mock_basic_ack.call_args
self.assertEqual(2, len(args))
@mock.patch.object(Queue, "empty")
@mock.patch.object(OpsgenieAlertsHandler, "_send_heartbeat")
@mock.patch.object(OpsgenieAlertsHandler, "_send_alerts")
@mock.patch.object(OpsgenieAlertsHandler, "_place_alert_on_queue")
@mock.patch.object(RabbitMQApi, "basic_ack")
def test_process_alert_does_not_raise_msg_not_delivered_exception(
self, mock_basic_ack, mock_place_alert, mock_send_alerts,
mock_send_heartbeat, mock_empty) -> None:
mock_basic_ack.return_value = None
mock_place_alert.return_value = None
mock_send_alerts.return_value = None
mock_empty.return_value = True
mock_send_heartbeat.side_effect = MessageWasNotDeliveredException(
'test')
try:
self.test_opsgenie_alerts_handler._initialise_rabbitmq()
blocking_channel = \
self.test_opsgenie_alerts_handler.rabbitmq.channel
method = pika.spec.Basic.Deliver(
routing_key=self.test_opsgenie_alerts_handler.
_opsgenie_channel_routing_key)
body = json.dumps(self.test_alert.alert_data)
properties = pika.spec.BasicProperties()
# This would raise a MessageWasNotDeliveredException if raised,
# hence the test would fail
self.test_opsgenie_alerts_handler._process_alert(
blocking_channel, method, properties, body)
except Exception as e:
self.fail("Test failed: {}".format(e))
mock_basic_ack.assert_called_once()
@parameterized.expand([
(
AMQPConnectionError,
AMQPConnectionError('test'),
),
(
AMQPChannelError,
AMQPChannelError('test'),
),
(
Exception,
Exception('test'),
),
])
@mock.patch.object(Queue, "empty")
@mock.patch.object(OpsgenieAlertsHandler, "_send_heartbeat")
@mock.patch.object(OpsgenieAlertsHandler, "_send_alerts")
@mock.patch.object(OpsgenieAlertsHandler, "_place_alert_on_queue")
@mock.patch.object(RabbitMQApi, "basic_ack")
def test_process_alert_raises_error_if_raised_by_send_hb(
self, exception_class, exception_instance, mock_basic_ack,
mock_place_alert, mock_send_alerts, mock_send_heartbeat,
mock_empty) -> None:
# For this test we will check for channel, connection and unexpected
# errors.
mock_basic_ack.return_value = None
mock_place_alert.return_value = None
mock_send_alerts.return_value = None
mock_send_heartbeat.side_effect = exception_instance
mock_empty.return_value = True
try:
self.test_opsgenie_alerts_handler._initialise_rabbitmq()
blocking_channel = \
self.test_opsgenie_alerts_handler.rabbitmq.channel
method = pika.spec.Basic.Deliver(
routing_key=self.test_opsgenie_alerts_handler.
_opsgenie_channel_routing_key)
body = json.dumps(self.test_alert.alert_data)
properties = pika.spec.BasicProperties()
self.assertRaises(exception_class,
self.test_opsgenie_alerts_handler._process_alert,
blocking_channel, method, properties, body)
except Exception as e:
self.fail("Test failed: {}".format(e))
mock_basic_ack.assert_called_once()
def test_place_alert_on_queue_places_alert_on_queue_if_queue_not_full(
self) -> None:
# Use a smaller queue in this case for simplicity
test_queue = Queue(3)
self.test_opsgenie_alerts_handler._alerts_queue = test_queue
test_queue.put('item1')
test_queue.put('item2')
self.test_opsgenie_alerts_handler._place_alert_on_queue(
self.test_alert)
all_queue_items = list(test_queue.queue)
self.assertEqual(['item1', 'item2', self.test_alert], all_queue_items)
def test_place_alert_on_queue_removes_oldest_and_places_if_queue_full(
self) -> None:
# Use a smaller queue in this case for simplicity
test_queue = Queue(3)
self.test_opsgenie_alerts_handler._alerts_queue = test_queue
test_queue.put('item1')
test_queue.put('item2')
test_queue.put('item3')
self.test_opsgenie_alerts_handler._place_alert_on_queue(
self.test_alert)
all_queue_items = list(test_queue.queue)
self.assertEqual(['item2', 'item3', self.test_alert], all_queue_items)
@mock.patch.object(Queue, "empty")
@mock.patch.object(Queue, "get")
@mock.patch.object(logging, "debug")
@mock.patch.object(logging, "info")
@mock.patch.object(logging, "warning")
@mock.patch.object(logging, "critical")
@mock.patch.object(logging, "error")
@mock.patch.object(logging, "exception")
def test_send_alerts_does_nothing_if_queue_is_empty(
self, mock_exception, mock_error, mock_critical, mock_warning,
mock_info, mock_debug, mock_get, mock_empty) -> None:
mock_empty.return_value = True
self.test_opsgenie_alerts_handler._send_alerts()
mock_critical.assert_not_called()
mock_info.assert_not_called()
mock_warning.assert_not_called()
mock_debug.assert_not_called()
mock_get.assert_not_called()
mock_exception.assert_not_called()
mock_error.assert_not_called()
@freeze_time("2012-01-01")
@mock.patch.object(OpsgenieChannel, "alert")
def test_send_alerts_discards_old_alerts_and_sends_the_recent(
self, mock_alert) -> None:
mock_alert.return_value = RequestStatus.SUCCESS
test_alert_old1 = OpenFileDescriptorsIncreasedAboveThresholdAlert(
self.test_system_name, self.test_percentage_usage,
self.test_panic_severity,
datetime.now().timestamp() - self.test_alert_validity_threshold -
1, self.test_panic_severity, self.test_parent_id,
self.test_system_id)
test_alert_recent1 = OpenFileDescriptorsIncreasedAboveThresholdAlert(
self.test_system_name, self.test_percentage_usage,
self.test_panic_severity,
datetime.now().timestamp() - self.test_alert_validity_threshold,
self.test_panic_severity, self.test_parent_id, self.test_system_id)
test_alert_recent2 = OpenFileDescriptorsIncreasedAboveThresholdAlert(
self.test_system_name, self.test_percentage_usage,
self.test_panic_severity,
datetime.now().timestamp(), self.test_panic_severity,
self.test_parent_id, self.test_system_id)
test_alert_recent3 = OpenFileDescriptorsIncreasedAboveThresholdAlert(
self.test_system_name, self.test_percentage_usage,
self.test_panic_severity,
datetime.now().timestamp() - self.test_alert_validity_threshold +
4, self.test_panic_severity, self.test_parent_id,
self.test_system_id)
test_queue = Queue(4)
self.test_opsgenie_alerts_handler._alerts_queue = test_queue
test_queue.put(test_alert_old1)
test_queue.put(test_alert_recent1)
test_queue.put(test_alert_recent2)
test_queue.put(test_alert_recent3)
self.test_opsgenie_alerts_handler._send_alerts()
self.assertTrue(self.test_opsgenie_alerts_handler.alerts_queue.empty())
expected_calls = [
call(test_alert_recent1),
call(test_alert_recent2),
call(test_alert_recent3)
]
actual_calls = mock_alert.call_args_list
self.assertEqual(expected_calls, actual_calls)
@parameterized.expand([
(
[RequestStatus.SUCCESS],
1,
),
(
[RequestStatus.FAILED, RequestStatus.SUCCESS],
2,
),
(
[
RequestStatus.FAILED, RequestStatus.FAILED,
RequestStatus.SUCCESS
],
3,
),
(
[
RequestStatus.FAILED, RequestStatus.FAILED,
RequestStatus.FAILED, RequestStatus.SUCCESS
],
4,
),
(
[
RequestStatus.FAILED, RequestStatus.FAILED,
RequestStatus.FAILED, RequestStatus.FAILED,
RequestStatus.SUCCESS
],
5,
),
(
[
RequestStatus.FAILED, RequestStatus.FAILED,
RequestStatus.FAILED, RequestStatus.FAILED,
RequestStatus.FAILED, RequestStatus.SUCCESS
],
5,
),
])
@freeze_time("2012-01-01")
@mock.patch.object(RabbitMQApi, "connection")
@mock.patch.object(OpsgenieChannel, "alert")
def test_send_alerts_attempts_to_send_alert_for_up_to_max_attempts_times(
self, alert_request_status_list, expected_no_calls, mock_alert,
mock_connection) -> None:
mock_alert.side_effect = alert_request_status_list
mock_connection.return_value.sleep.return_value = None
test_alert = OpenFileDescriptorsIncreasedAboveThresholdAlert(
self.test_system_name, self.test_percentage_usage,
self.test_panic_severity,
datetime.now().timestamp(), self.test_panic_severity,
self.test_parent_id, self.test_system_id)
test_queue = Queue(4)
self.test_opsgenie_alerts_handler._alerts_queue = test_queue
test_queue.put(test_alert)
self.test_opsgenie_alerts_handler._send_alerts()
expected_calls = []
for _ in range(expected_no_calls):
expected_calls.append(call(test_alert))
actual_calls = mock_alert.call_args_list
self.assertEqual(expected_calls, actual_calls)
@parameterized.expand([
([RequestStatus.SUCCESS], ),
([RequestStatus.FAILED, RequestStatus.SUCCESS], ),
([RequestStatus.FAILED, RequestStatus.FAILED,
RequestStatus.SUCCESS], ),
([
RequestStatus.FAILED, RequestStatus.FAILED, RequestStatus.FAILED,
RequestStatus.SUCCESS
], ),
([
RequestStatus.FAILED, RequestStatus.FAILED, RequestStatus.FAILED,
RequestStatus.FAILED, RequestStatus.SUCCESS
], ),
])
@freeze_time("2012-01-01")
@mock.patch.object(RabbitMQApi, "connection")
@mock.patch.object(OpsgenieChannel, "alert")
def test_send_alerts_removes_alert_if_it_was_successfully_sent(
self, alert_request_status_list, mock_alert,
mock_connection) -> None:
mock_alert.side_effect = alert_request_status_list
mock_connection.return_value.sleep.return_value = None
test_alert = OpenFileDescriptorsIncreasedAboveThresholdAlert(
self.test_system_name, self.test_percentage_usage,
self.test_panic_severity,
datetime.now().timestamp(), self.test_panic_severity,
self.test_parent_id, self.test_system_id)
test_queue = Queue(4)
self.test_opsgenie_alerts_handler._alerts_queue = test_queue
test_queue.put(test_alert)
self.test_opsgenie_alerts_handler._send_alerts()
self.assertTrue(self.test_opsgenie_alerts_handler.alerts_queue.empty())
@freeze_time("2012-01-01")
@mock.patch.object(RabbitMQApi, "connection")
@mock.patch.object(OpsgenieChannel, "alert")
def test_send_alerts_stops_sending_if_an_alert_is_not_successfully_sent(
self, mock_alert, mock_connection) -> None:
mock_alert.return_value = RequestStatus.FAILED
mock_connection.return_value.sleep.return_value = None
test_alert_1 = OpenFileDescriptorsIncreasedAboveThresholdAlert(
self.test_system_name, self.test_percentage_usage,
self.test_panic_severity,
datetime.now().timestamp(), self.test_panic_severity,
self.test_parent_id, self.test_system_id)
test_alert_2 = OpenFileDescriptorsIncreasedAboveThresholdAlert(
self.test_system_name, self.test_percentage_usage,
self.test_panic_severity,
datetime.now().timestamp() + 1, self.test_panic_severity,
self.test_parent_id, self.test_system_id)
test_queue = Queue(4)
self.test_opsgenie_alerts_handler._alerts_queue = test_queue
test_queue.put(test_alert_1)
test_queue.put(test_alert_2)
self.test_opsgenie_alerts_handler._send_alerts()
self.assertFalse(
self.test_opsgenie_alerts_handler.alerts_queue.empty())
self.assertEqual(
2, self.test_opsgenie_alerts_handler.alerts_queue.qsize())
self.assertEqual(
test_alert_1,
self.test_opsgenie_alerts_handler.alerts_queue.queue[0])
self.assertEqual(
test_alert_2,
self.test_opsgenie_alerts_handler.alerts_queue.queue[1])
def pre_handle(self):
raise AMQPConnectionError()
def on_open_error(conn, err):
f.set_exception(AMQPConnectionError(err))
class TestTelegramCommandsHandler(unittest.TestCase):
def setUp(self) -> None:
self.test_handler_name = 'test_telegram_commands_handler'
self.dummy_logger = logging.getLogger('Dummy')
self.dummy_logger.disabled = True
self.connection_check_time_interval = timedelta(seconds=0)
self.rabbit_ip = env.RABBIT_IP
self.rabbitmq = RabbitMQApi(
self.dummy_logger,
self.rabbit_ip,
connection_check_time_interval=self.connection_check_time_interval)
self.test_channel_name = 'test_telegram_channel'
self.test_channel_id = 'test_telegram_id12345'
self.test_channel_logger = self.dummy_logger.getChild('dummy_channel')
self.test_bot_token = '1234567891:ABC-67ABCrfZFdddqRT5Gh837T2rtUFHgTY'
self.test_bot_chat_id = 'test_bot_chat_id'
self.test_base_url = \
"https://api.telegram.org/bot" + self.test_bot_token
self.test_api = TelegramBotApi(self.test_bot_token,
self.test_bot_chat_id)
self.test_channel = TelegramChannel(self.test_channel_name,
self.test_channel_id,
self.test_channel_logger,
self.test_api)
self.cmd_handlers_rabbit = RabbitMQApi(
logger=self.dummy_logger.getChild(RabbitMQApi.__name__),
host=self.rabbit_ip,
connection_check_time_interval=self.connection_check_time_interval)
self.redis = RedisApi(logger=self.dummy_logger.getChild(
RedisApi.__name__),
host=env.REDIS_IP,
db=env.REDIS_DB,
port=env.REDIS_PORT,
namespace=env.UNIQUE_ALERTER_IDENTIFIER)
self.mongo = MongoApi(logger=self.dummy_logger.getChild(
MongoApi.__name__),
host=env.DB_IP,
db_name=env.DB_NAME,
port=env.DB_PORT)
self.test_command_handlers_logger = self.dummy_logger.getChild(
'command_handlers')
self.test_chain_1 = 'Kusama'
self.test_chain_2 = 'Cosmos'
self.test_chain_3 = 'Test_Chain'
self.test_chain1_id = 'kusama1234'
self.test_chain2_id = 'cosmos1234'
self.test_chain3_id = 'test_chain11123'
self.test_associated_chains = {
self.test_chain1_id: self.test_chain_1,
self.test_chain2_id: self.test_chain_2,
self.test_chain3_id: self.test_chain_3
}
self.test_telegram_command_handlers = TelegramCommandHandlers(
self.test_handler_name, self.test_command_handlers_logger,
self.test_associated_chains, self.test_channel,
self.cmd_handlers_rabbit, self.redis, self.mongo)
self.test_telegram_commands_handler = TelegramCommandsHandler(
self.test_handler_name, self.dummy_logger, self.rabbitmq,
self.test_channel, self.test_telegram_command_handlers)
self.test_data_str = "this is a test string"
self.test_rabbit_queue_name = 'Test Queue'
self.test_timestamp = 45676565.556
self.test_heartbeat = {
'component_name': 'Test Component',
'timestamp': self.test_timestamp,
}
self.test_system_name = 'test_system'
self.test_percentage_usage = 50
self.test_panic_severity = 'WARNING'
self.test_parent_id = 'parent_1234'
self.test_system_id = 'system_id32423'
self.test_alert = OpenFileDescriptorsIncreasedAboveThresholdAlert(
self.test_system_name, self.test_percentage_usage,
self.test_panic_severity, self.test_timestamp,
self.test_panic_severity, self.test_parent_id, self.test_system_id)
def tearDown(self) -> None:
# Delete any queues and exchanges which are common across many tests
connect_to_rabbit(self.test_telegram_commands_handler.rabbitmq)
delete_queue_if_exists(self.test_telegram_commands_handler.rabbitmq,
self.test_rabbit_queue_name)
delete_queue_if_exists(
self.test_telegram_commands_handler.rabbitmq, self.
test_telegram_commands_handler._telegram_commands_handler_queue)
delete_exchange_if_exists(self.test_telegram_commands_handler.rabbitmq,
HEALTH_CHECK_EXCHANGE)
disconnect_from_rabbit(self.test_telegram_commands_handler.rabbitmq)
self.dummy_logger = None
self.test_channel_logger = None
self.test_command_handlers_logger = None
self.rabbitmq = None
self.cmd_handlers_rabbit = None
self.test_alert = None
self.test_channel = None
self.test_api = None
self.redis = None
self.mongo = None
self.test_telegram_commands_handler._updater.stop()
self.test_telegram_command_handlers = None
self.test_telegram_commands_handler = None
def test__str__returns_handler_name(self) -> None:
self.assertEqual(self.test_handler_name,
str(self.test_telegram_commands_handler))
def test_handler_name_returns_handler_name(self) -> None:
self.assertEqual(self.test_handler_name,
self.test_telegram_commands_handler.handler_name)
@mock.patch.object(RabbitMQApi, "start_consuming")
def test_listen_for_data_calls_start_consuming(
self, mock_start_consuming) -> None:
mock_start_consuming.return_value = None
self.test_telegram_commands_handler._listen_for_data()
mock_start_consuming.assert_called_once_with()
def test_cmd_handlers_returns_associated_command_handlers(self) -> None:
self.assertEqual(self.test_telegram_command_handlers,
self.test_telegram_commands_handler.cmd_handlers)
def test_telegram_channel_returns_associated_telegram_channel(
self) -> None:
self.assertEqual(self.test_channel,
self.test_telegram_commands_handler.telegram_channel)
def test_init_initialises_handler_correctly(self) -> None:
# In this test we will check that all fields that do not have a getter
# were initialised correctly, as the previous tests test the getters.
expected_updater = Updater(token=self.test_bot_token, use_context=True)
expected_command_specific_handlers = [
CommandHandler('start',
self.test_telegram_command_handlers.start_callback),
CommandHandler('mute',
self.test_telegram_command_handlers.mute_callback),
CommandHandler(
'unmute', self.test_telegram_command_handlers.unmute_callback),
CommandHandler(
'muteall',
self.test_telegram_command_handlers.muteall_callback),
CommandHandler(
'unmuteall',
self.test_telegram_command_handlers.unmuteall_callback),
CommandHandler(
'status', self.test_telegram_command_handlers.status_callback),
CommandHandler('ping',
self.test_telegram_command_handlers.ping_callback),
CommandHandler('help',
self.test_telegram_command_handlers.help_callback),
MessageHandler(
Filters.command,
self.test_telegram_command_handlers.unknown_callback)
]
actual_command_specific_handlers = \
self.test_telegram_commands_handler._updater.dispatcher.handlers[0]
self.assertEqual(
'telegram_{}_commands_handler_queue'.format(self.test_channel_id),
self.test_telegram_commands_handler.
_telegram_commands_handler_queue)
self.assertEqual(
expected_updater.bot.token,
self.test_telegram_commands_handler._updater.bot.token)
self.assertTrue(self.test_telegram_commands_handler._updater.
dispatcher.use_context)
for i in range(len(actual_command_specific_handlers)):
if type(actual_command_specific_handlers[i]) == CommandHandler:
self.assertEqual(
expected_command_specific_handlers[i].callback,
actual_command_specific_handlers[i].callback)
self.assertEqual(expected_command_specific_handlers[i].command,
actual_command_specific_handlers[i].command)
elif type(actual_command_specific_handlers[i]) == MessageHandler:
self.assertEqual(
expected_command_specific_handlers[i].callback,
actual_command_specific_handlers[i].callback)
def test_initialise_rabbitmq_initialises_rabbit_correctly(self) -> None:
try:
# To make sure that there is no connection/channel already
# established
self.assertIsNone(self.rabbitmq.connection)
self.assertIsNone(self.rabbitmq.channel)
# To make sure that the exchanges and queues have not already been
# declared
connect_to_rabbit(self.rabbitmq)
self.test_telegram_commands_handler.rabbitmq.queue_delete(
self.test_telegram_commands_handler.
_telegram_commands_handler_queue)
self.test_telegram_commands_handler.rabbitmq.exchange_delete(
HEALTH_CHECK_EXCHANGE)
disconnect_from_rabbit(self.rabbitmq)
self.test_telegram_commands_handler._initialise_rabbitmq()
# Perform checks that the connection has been opened and marked as
# open, that the delivery confirmation variable is set and basic_qos
# called successfully.
self.assertTrue(
self.test_telegram_commands_handler.rabbitmq.is_connected)
self.assertTrue(self.test_telegram_commands_handler.rabbitmq.
connection.is_open)
self.assertTrue(self.test_telegram_commands_handler.rabbitmq.
channel._delivery_confirmation)
# Check whether the consuming exchanges and queues have been
# creating by sending messages with the same routing keys as for the
# bindings. We will also check if the size of the queues is 0 to
# confirm that basic_consume was called (it will store the msg in
# the component memory immediately). If one of the exchanges or
# queues is not created or basic_consume is not called, then either
# an exception will be thrown or the queue size would be 1
# respectively. Note when deleting the exchanges in the beginning we
# also released every binding, hence there are no other queue binded
# with the same routing key to any exchange at this point.
self.test_telegram_commands_handler.rabbitmq.basic_publish_confirm(
exchange=HEALTH_CHECK_EXCHANGE,
routing_key=TCH_INPUT_ROUTING_KEY,
body=self.test_data_str,
is_body_dict=False,
properties=pika.BasicProperties(delivery_mode=2),
mandatory=True)
# Re-declare queue to get the number of messages
res = self.test_telegram_commands_handler.rabbitmq.queue_declare(
self.test_telegram_commands_handler.
_telegram_commands_handler_queue, False, True, False, False)
self.assertEqual(0, res.method.message_count)
except Exception as e:
self.fail("Test failed: {}".format(e))
def test_send_heartbeat_sends_a_heartbeat_correctly(self) -> None:
# This test creates a queue which receives messages with the same
# routing key as the ones set by send_heartbeat, and checks that the
# heartbeat is received
try:
self.test_telegram_commands_handler._initialise_rabbitmq()
# Delete the queue before to avoid messages in the queue on error.
self.test_telegram_commands_handler.rabbitmq.queue_delete(
self.test_rabbit_queue_name)
res = self.test_telegram_commands_handler.rabbitmq.queue_declare(
queue=self.test_rabbit_queue_name,
durable=True,
exclusive=False,
auto_delete=False,
passive=False)
self.assertEqual(0, res.method.message_count)
self.test_telegram_commands_handler.rabbitmq.queue_bind(
queue=self.test_rabbit_queue_name,
exchange=HEALTH_CHECK_EXCHANGE,
routing_key='heartbeat.worker')
self.test_telegram_commands_handler._send_heartbeat(
self.test_heartbeat)
# By re-declaring the queue again we can get the number of messages
# in the queue.
res = self.test_telegram_commands_handler.rabbitmq.queue_declare(
queue=self.test_rabbit_queue_name,
durable=True,
exclusive=False,
auto_delete=False,
passive=True)
self.assertEqual(1, res.method.message_count)
# Check that the message received is actually the HB
_, _, body = self.test_telegram_commands_handler.rabbitmq.basic_get(
self.test_rabbit_queue_name)
self.assertEqual(self.test_heartbeat, json.loads(body))
except Exception as e:
self.fail("Test failed: {}".format(e))
@parameterized.expand([
(True, ),
(False, ),
])
@mock.patch.object(Updater, "start_polling")
@mock.patch.object(Updater, "idle")
def test_start_handling_starts_polling_if_updater_not_running(
self, run_in_background, mock_idle, mock_start_polling) -> None:
mock_idle.return_value = None
mock_start_polling.return_value = None
self.test_telegram_commands_handler._updater.running = False
self.test_telegram_commands_handler._start_handling(run_in_background)
mock_start_polling.assert_called_once_with(clean=True)
@parameterized.expand([
(True, ),
(False, ),
])
@mock.patch.object(Updater, "start_polling")
@mock.patch.object(Updater, "idle")
def test_start_handling_does_not_start_polling_if_updater_is_running(
self, run_in_background, mock_idle, mock_start_polling) -> None:
mock_idle.return_value = None
mock_start_polling.return_value = None
self.test_telegram_commands_handler._updater.running = True
self.test_telegram_commands_handler._start_handling(run_in_background)
mock_start_polling.assert_not_called()
@parameterized.expand([
(True, ),
(False, ),
])
@mock.patch.object(Updater, "start_polling")
@mock.patch.object(Updater, "idle")
def test_start_handling_blocks_if_updater_not_running_in_the_background(
self, running, mock_idle, mock_start_polling) -> None:
mock_idle.return_value = None
mock_start_polling.return_value = None
self.test_telegram_commands_handler._updater.running = running
self.test_telegram_commands_handler._start_handling(False)
mock_idle.assert_called_once_with(stop_signals=[])
@parameterized.expand([
(True, ),
(False, ),
])
@mock.patch.object(Updater, "start_polling")
@mock.patch.object(Updater, "idle")
def test_start_handling_does_not_block_if_updater_running_in_the_background(
self, running, mock_idle, mock_start_polling) -> None:
mock_idle.return_value = None
mock_start_polling.return_value = None
self.test_telegram_commands_handler._updater.running = running
self.test_telegram_commands_handler._start_handling(True)
mock_idle.assert_not_called()
@mock.patch.object(Updater, "stop")
def test_stop_handling_stops_handling(self, mock_stop_handling) -> None:
mock_stop_handling.return_value = None
self.test_telegram_commands_handler._stop_handling()
mock_stop_handling.assert_called_once_with()
@freeze_time("2012-01-01")
@mock.patch.object(TelegramCommandsHandler, "_send_heartbeat")
def test_process_ping_sends_correct_heartbeat_if_updater_is_running(
self, mock_send_hb) -> None:
# We will perform this test by checking that send_hb is called with the
# correct heartbeat. The actual sending was already tested above.
mock_send_hb.return_value = None
self.test_telegram_commands_handler._updater.running = True
try:
# Some of the variables below are needed as parameters for the
# process_ping function
self.test_telegram_commands_handler._initialise_rabbitmq()
blocking_channel = \
self.test_telegram_commands_handler.rabbitmq.channel
method = pika.spec.Basic.Deliver(routing_key=TCH_INPUT_ROUTING_KEY)
body = 'ping'
properties = pika.spec.BasicProperties()
self.test_telegram_commands_handler._process_ping(
blocking_channel, method, properties, body)
expected_hb = {
'component_name': self.test_handler_name,
'is_alive': True,
'timestamp': datetime.now().timestamp()
}
mock_send_hb.assert_called_once_with(expected_hb)
except Exception as e:
self.fail("Test failed: {}".format(e))
@freeze_time("2012-01-01")
@mock.patch.object(TelegramCommandsHandler, "_start_handling")
@mock.patch.object(TelegramCommandsHandler, "_send_heartbeat")
def test_process_ping_sends_correct_heartbeat_if_updater_is_not_running(
self, mock_send_hb, mock_start_handling) -> None:
# We will perform this test by checking that send_hb is called with the
# correct heartbeat. The actual sending was already tested above.
mock_send_hb.return_value = None
mock_start_handling.return_value = None
self.test_telegram_commands_handler._updater.running = False
try:
# Some of the variables below are needed as parameters for the
# process_ping function
self.test_telegram_commands_handler._initialise_rabbitmq()
blocking_channel = \
self.test_telegram_commands_handler.rabbitmq.channel
method = pika.spec.Basic.Deliver(routing_key=TCH_INPUT_ROUTING_KEY)
body = 'ping'
properties = pika.spec.BasicProperties()
self.test_telegram_commands_handler._process_ping(
blocking_channel, method, properties, body)
expected_hb = {
'component_name': self.test_handler_name,
'is_alive': False,
'timestamp': datetime.now().timestamp()
}
mock_send_hb.assert_called_once_with(expected_hb)
except Exception as e:
self.fail("Test failed: {}".format(e))
@mock.patch.object(TelegramCommandsHandler, "_start_handling")
@mock.patch.object(TelegramCommandsHandler, "_send_heartbeat")
def test_process_ping_restarts_updater_if_it_is_dead(
self, mock_send_hb, mock_start_handling) -> None:
mock_send_hb.return_value = None
mock_start_handling.return_value = None
self.test_telegram_commands_handler._updater.running = False
try:
# Some of the variables below are needed as parameters for the
# process_ping function
self.test_telegram_commands_handler._initialise_rabbitmq()
blocking_channel = \
self.test_telegram_commands_handler.rabbitmq.channel
method = pika.spec.Basic.Deliver(routing_key=TCH_INPUT_ROUTING_KEY)
body = 'ping'
properties = pika.spec.BasicProperties()
self.test_telegram_commands_handler._process_ping(
blocking_channel, method, properties, body)
mock_start_handling.assert_called_once_with(run_in_background=True)
except Exception as e:
self.fail("Test failed: {}".format(e))
@mock.patch.object(TelegramCommandsHandler, "_start_handling")
@mock.patch.object(TelegramCommandsHandler, "_send_heartbeat")
def test_process_ping_does_not_restart_updater_if_it_is_alive(
self, mock_send_hb, mock_start_handling) -> None:
mock_send_hb.return_value = None
mock_start_handling.return_value = None
self.test_telegram_commands_handler._updater.running = True
try:
# Some of the variables below are needed as parameters for the
# process_ping function
self.test_telegram_commands_handler._initialise_rabbitmq()
blocking_channel = \
self.test_telegram_commands_handler.rabbitmq.channel
method = pika.spec.Basic.Deliver(routing_key=TCH_INPUT_ROUTING_KEY)
body = 'ping'
properties = pika.spec.BasicProperties()
self.test_telegram_commands_handler._process_ping(
blocking_channel, method, properties, body)
mock_start_handling.assert_not_called()
except Exception as e:
self.fail("Test failed: {}".format(e))
@mock.patch.object(TelegramCommandsHandler, "_start_handling")
@mock.patch.object(TelegramCommandsHandler, "_send_heartbeat")
def test_process_ping_does_not_send_heartbeat_if_error_when_computing_hb(
self, mock_send_hb, mock_start_handling) -> None:
# In this case an exception may only be raised if the updater is
# restarted.
mock_send_hb.return_value = None
mock_start_handling.side_effect = Exception('test')
self.test_telegram_commands_handler._updater.running = False
try:
# Some of the variables below are needed as parameters for the
# process_ping function
self.test_telegram_commands_handler._initialise_rabbitmq()
blocking_channel = \
self.test_telegram_commands_handler.rabbitmq.channel
method = pika.spec.Basic.Deliver(routing_key=TCH_INPUT_ROUTING_KEY)
body = 'ping'
properties = pika.spec.BasicProperties()
self.test_telegram_commands_handler._process_ping(
blocking_channel, method, properties, body)
mock_send_hb.assert_not_called()
except Exception as e:
self.fail("Test failed: {}".format(e))
@mock.patch.object(TelegramCommandsHandler, "_send_heartbeat")
def test_process_ping_does_not_raise_msg_was_not_delivered_exception(
self, mock_send_hb) -> None:
# In this case an exception may only be raised if the updater is
# restarted.
mock_send_hb.side_effect = MessageWasNotDeliveredException('test')
self.test_telegram_commands_handler._updater.running = True
try:
# Some of the variables below are needed as parameters for the
# process_ping function
self.test_telegram_commands_handler._initialise_rabbitmq()
blocking_channel = \
self.test_telegram_commands_handler.rabbitmq.channel
method = pika.spec.Basic.Deliver(routing_key=TCH_INPUT_ROUTING_KEY)
body = 'ping'
properties = pika.spec.BasicProperties()
self.test_telegram_commands_handler._process_ping(
blocking_channel, method, properties, body)
except Exception as e:
self.fail("Test failed: {}".format(e))
@parameterized.expand([
(
AMQPConnectionError,
AMQPConnectionError('test'),
),
(
AMQPChannelError,
AMQPChannelError('test'),
),
(
Exception,
Exception('test'),
),
])
@mock.patch.object(TelegramCommandsHandler, "_send_heartbeat")
def test_process_ping_raises_error_if_raised_by_send_heartbeat(
self, exception_class, exception_instance,
mock_send_heartbeat) -> None:
# For this test we will check for channel, connection and unexpected
# errors.
mock_send_heartbeat.side_effect = exception_instance
self.test_telegram_commands_handler._updater.running = True
try:
# Some of the variables below are needed as parameters for the
# process_ping function
self.test_telegram_commands_handler._initialise_rabbitmq()
blocking_channel = \
self.test_telegram_commands_handler.rabbitmq.channel
method = pika.spec.Basic.Deliver(routing_key=TCH_INPUT_ROUTING_KEY)
body = 'ping'
properties = pika.spec.BasicProperties()
self.assertRaises(
exception_class,
self.test_telegram_commands_handler._process_ping,
blocking_channel, method, properties, body)
except Exception as e:
self.fail("Test failed: {}".format(e))