def run(self):
"""Main function of benchmarking."""
log.info('Start benchmarking ...')
log.info('Spawning %d worker threads ...' % self.workers)
sqs_talk = sqs.SQSTalk()
queue_info_obj = queue_info.QueueInfo(self.config_file, sqs_talk)
worker_threads = []
for i in range(self.workers):
worker_thread = WorkerThread(queue_info_obj, self.iterations)
worker_threads.append(worker_thread)
worker_thread.setDaemon(True)
for worker_thread in worker_threads:
worker_thread.start()
for worker_thread in worker_threads:
worker_thread.join()
all_check_time = []
while not checking_sqs_time.empty():
check_time = checking_sqs_time.get()
all_check_time.append(check_time)
log.info('=== Benchmark results ===')
count = len(all_check_time)
sum_time = sum(all_check_time)
avg_time = sum_time / count
median_time = sorted(all_check_time)[count / 2]
max_time = max(all_check_time)
log.info('Average Check Time: %f' % avg_time)
log.info('Median Check time: %f' % median_time)
log.info('Max Check Time: %f' % max_time)
def test_publish_messages_to_dead_letter_queue(self):
"""Test publisher to DLQ logic."""
sqs_inst = sqs.SQSTalk()
sqs_inst._connection = mock.MagicMock()
mock_publisher = publisher.Publisher(sqs_inst)
mock_queue = mock.MagicMock()
mock_publisher._get_or_create_queue = mock.MagicMock(
return_value=mock_queue)
payload = {'args': [], 'kwargs': {}}
sqs_msg = message.KaleMessage.create_message(
task_class=test_utils.MockTask,
task_id=test_utils.MockTask._get_task_id(),
payload=payload,
queue='queue',
current_retry_num=5)
sqs_msg.id = 'test-id'
test_body = 'test-body'
sqs_msg.get_body_encoded = mock.MagicMock(return_value=test_body)
mock_messages = [sqs_msg]
with mock.patch.object(mock_queue, 'write_batch') as mock_write:
mock_publisher.publish_messages_to_dead_letter_queue(
'dlq_name', mock_messages)
expected_args = [(sqs_msg.id, test_body, 0)]
mock_write.assert_called_once_with(expected_args)
def test_publish_messages_to_dead_letter_queue(self):
"""Test publisher to DLQ logic."""
sqs_inst = sqs.SQSTalk()
mock_publisher = publisher.Publisher(sqs_inst)
mock_queue = mock.MagicMock()
mock_publisher._get_or_create_queue = mock.MagicMock(
return_value=mock_queue)
payload = {'args': [], 'kwargs': {}}
kale_msg = message.KaleMessage(
task_class=test_utils.MockTask,
task_id=test_utils.MockTask._get_task_id(),
payload=payload,
current_retry_num=5)
kale_msg.id = 'test-id'
test_body = 'test-body'
kale_msg.encode = mock.MagicMock(return_value=test_body)
mock_messages = [kale_msg]
with mock.patch.object(mock_queue, 'send_messages') as mock_write:
mock_publisher.publish_messages_to_dead_letter_queue(
'dlq_name', mock_messages)
expected_args = [{
'Id': kale_msg.id,
'MessageBody': test_body,
'DelaySeconds': 0
}]
mock_write.assert_called_once_with(Entries=expected_args)
def _build_queue_info(self):
sqs_inst = sqs.SQSTalk()
queue_config = tempfile.NamedTemporaryFile(delete=True)
queue_config.write(self.test_string.encode('utf8'))
queue_config.seek(0)
queue_info.QueueInfo._queues = None
queue_info.QueueInfo._simple_name_queues_map = None
qinfo = queue_info.QueueInfo(queue_config.name, sqs_inst,
queue_info.TaskQueue)
return qinfo
def _get_sqs_connection(self):
with mock.patch('boto.sqs') as mock_sqs:
mock_connection = mock.MagicMock()
mock_sqs.connection.SQSConnection.return_value = mock_connection
mock_sqs.connect_to_region.return_value = mock_connection
mock_connection.lookup.return_value = self.mock_queue
mock_connection.create_queue.return_value = self.mock_queue
mock_connection.get_all_queues = self.mock_all_queues
sqs_inst = sqs.SQSTalk()
sqs_inst._connection = mock_connection
return sqs_inst
def test_get_queues(self):
boto3.setup_default_session()
sqs_inst = sqs.SQSTalk()
sqs_inst._get_or_create_queue('LowPriorityTest3')
sqs_inst._get_or_create_queue('HighPriorityTest4')
queues = sqs_inst.get_all_queues()
print(queues)
self.assertEqual(len(queues), 2)
queues = sqs_inst.get_all_queues('Low')
self.assertEqual(len(queues), 1)
sqs_inst.get_all_queues('High')
self.assertEqual(len(queues), 1)
def test_create_queue(self):
boto3.setup_default_session()
sqs_inst = sqs.SQSTalk()
sqs_inst._get_or_create_queue('LowPriorityTest1')
sqs_inst._get_or_create_queue('HighPriorityTest2')
expected_low_queue = sqs_inst._sqs.Queue('https://queue.amazonaws.com/123456789012/'
'LowPriorityTest1')
expected_hi_queue = sqs_inst._sqs.Queue('https://queue.amazonaws.com/123456789012/'
'HighPriorityTest2')
self.assertEqual(len(sqs_inst._queues), 2)
self.assertEqual(expected_low_queue, sqs_inst._queues['LowPriorityTest1'])
self.assertEqual(expected_hi_queue, sqs_inst._queues['HighPriorityTest2'])
def test_publish_with_app_data(self):
"""Test publisher logic."""
sqs_inst = sqs.SQSTalk()
with mock.patch(
'kale.queue_info.QueueInfo.get_queue') as mock_get_queue:
mock_queue = mock.MagicMock()
mock_queue.visibility_timeout_sec = 10
mock_get_queue.return_value = mock_queue
mock_publisher = publisher.Publisher(sqs_inst)
mock_publisher._get_or_create_queue = mock.MagicMock()
payload = {'args': [], 'kwargs': {}, 'app_data': {}}
mock_task_class = mock.MagicMock()
mock_task_class.time_limit = 2
mock_task_class.__name__ = 'task'
with mock.patch('kale.message.KaleMessage') as mock_message:
mock_message.create_message.return_value = mock.MagicMock()
mock_publisher.publish(mock_task_class, 1, payload)
def test_publish_invalid_delay_sec(self):
"""Test publish with invalid delay_sec value."""
sqs_inst = sqs.SQSTalk()
sqs_inst._connection = mock.MagicMock()
mock_publisher = publisher.Publisher(sqs_inst)
mock_publisher._get_or_create_queue = mock.MagicMock()
payload = {'args': [], 'kwargs': {}}
mock_task_class = mock.MagicMock()
mock_task_class.time_limit = 2
delay_sec = settings.SQS_MAX_TASK_DELAY_SEC + 1
with mock.patch('kale.message.KaleMessage') as mock_message:
mock_message.create_message.return_value = mock.MagicMock()
with self.assertRaises(exceptions.InvalidTaskDelayException):
mock_publisher.publish(mock_task_class, 1, payload,
delay_sec=delay_sec)
def test_publish_bad_time_limit_greater(self):
"""Test publish with bad time limit (greater than timeout)."""
sqs_inst = sqs.SQSTalk()
with mock.patch(
'kale.queue_info.QueueInfo.get_queue') as mock_get_queue:
mock_queue = mock.MagicMock()
mock_queue.visibility_timeout_sec = 600
mock_get_queue.return_value = mock_queue
mock_publisher = publisher.Publisher(sqs_inst)
mock_publisher._get_or_create_queue = mock.MagicMock()
payload = {'args': [], 'kwargs': {}}
mock_task_class = mock.MagicMock()
mock_task_class.time_limit = 601
with mock.patch('kale.message.KaleMessage') as mock_message:
mock_message.create_message.return_value = mock.MagicMock()
with self.assertRaises(
exceptions.InvalidTimeLimitTaskException):
mock_publisher.publish(mock_task_class, 1, payload)
def _build_queue_info(self):
with mock.patch('boto.sqs') as mock_sqs:
mock_sqs_connection = mock.MagicMock()
mock_queue = mock.MagicMock()
mock_sqs_connection.lookup.return_value = mock_queue
mock_sqs_connection.create_queue.return_value = mock_queue
conn = mock_sqs.connection
conn.SQSConnection.return_value = mock_sqs_connection
mock_sqs.connect_to_region.return_value = mock_sqs_connection
sqs_inst = sqs.SQSTalk()
sqs_inst._connection = mock_sqs_connection
queue_config = tempfile.NamedTemporaryFile(delete=True)
queue_config.write(self.test_string.encode('utf8'))
queue_config.seek(0)
queue_info.QueueInfo._queues = None
queue_info.QueueInfo._simple_name_queues_map = None
qinfo = queue_info.QueueInfo(queue_config.name, sqs_inst,
queue_info.TaskQueue)
return qinfo
def test_get_improperly_configured(self):
with mock.patch('kale.sqs.settings') as mock_settings:
mock_settings.PROPERLY_CONFIGURED = False
with self.assertRaises(exceptions.ImproperlyConfiguredException):
sqs.SQSTalk()
