def run(self):
"""Main function of benchmarking."""
log.info('Start benchmarking ...')
log.info('Spawning %d worker threads ...' % self.workers)
sqs_talk = sqs.SQSTalk()
queue_info_obj = queue_info.QueueInfo(self.config_file, sqs_talk)
worker_threads = []
for i in range(self.workers):
worker_thread = WorkerThread(queue_info_obj, self.iterations)
worker_threads.append(worker_thread)
worker_thread.setDaemon(True)
for worker_thread in worker_threads:
worker_thread.start()
for worker_thread in worker_threads:
worker_thread.join()
all_check_time = []
while not checking_sqs_time.empty():
check_time = checking_sqs_time.get()
all_check_time.append(check_time)
log.info('=== Benchmark results ===')
count = len(all_check_time)
sum_time = sum(all_check_time)
avg_time = sum_time / count
median_time = sorted(all_check_time)[count / 2]
max_time = max(all_check_time)
log.info('Average Check Time: %f' % avg_time)
log.info('Median Check time: %f' % median_time)
log.info('Max Check Time: %f' % max_time)
def _build_queue_info(self):
sqs_inst = sqs.SQSTalk()
queue_config = tempfile.NamedTemporaryFile(delete=True)
queue_config.write(self.test_string.encode('utf8'))
queue_config.seek(0)
queue_info.QueueInfo._queues = None
queue_info.QueueInfo._simple_name_queues_map = None
qinfo = queue_info.QueueInfo(queue_config.name, sqs_inst,
queue_info.TaskQueue)
return qinfo
def publish(self,
task_class,
task_id,
payload,
current_retry_num=None,
current_failure_num=None,
delay_sec=None):
"""Publish the given task type to the queue with the provided payload.
:param obj task_class: class of the task that we are publishing.
:param str task_id: unique identifying string for this task.
:param dict payload: dictionary for the task payload.
:param int current_retry_num: current task retry count. If 0, this is
the first attempt to run the task.
:param int current_failure_num: current task failure count.
:param int delay_sec: time (in seconds) that a task should stay
in the queue before being released to consumers.
:raises: TaskTooChubbyException: This task is outrageously chubby.
The publisher of the task should handle this exception and
determine how to proceed.
"""
if delay_sec is not None and delay_sec > settings.SQS_MAX_TASK_DELAY_SEC:
raise exceptions.InvalidTaskDelayException(
'Invalid task delay_sec (%d > %d).' %
(delay_sec, settings.SQS_MAX_TASK_DELAY_SEC))
queue_class = utils.class_import_from_path(settings.QUEUE_CLASS)
q_info = queue_info.QueueInfo(settings.QUEUE_CONFIG, self, queue_class)
queue_obj = q_info.get_queue(task_class.queue)
if task_class.time_limit >= queue_obj.visibility_timeout_sec:
raise exceptions.InvalidTimeLimitTaskException(
'Invalid task time limit: %d >= %d from %s' %
(task_class.time_limit, queue_obj.visibility_timeout_sec,
settings.QUEUE_CONFIG))
sqs_queue = self._get_or_create_queue(queue_obj.name)
kale_msg = message.KaleMessage(task_class=task_class,
task_id=task_id,
payload=payload,
current_retry_num=current_retry_num,
current_failure_num=current_failure_num)
sqs_queue.send_message(MessageBody=kale_msg.encode(),
DelaySeconds=delay_sec or 1)
logger.debug('Published task. Task id: %s; Task name: %s' %
(task_id, '%s.%s' %
(task_class.__module__, task_class.__name__)))
def __init__(self):
"""Initialize a worker instance."""
self._consumer = consumer.Consumer()
queue_class = utils.class_import_from_path(settings.QUEUE_CLASS)
q_info = queue_info.QueueInfo(
config_file=settings.QUEUE_CONFIG,
sqs_talk=self._consumer,
queue_cls=queue_class)
queue_selector_class = utils.class_import_from_path(
settings.QUEUE_SELECTOR)
self._queue_selector = queue_selector_class(q_info)
# The worker will publish permanently failed tasks to a
# dead-letter-queue.
self._publisher = self._get_publisher()
# Track total messages processed.
self._total_messages_processed = 0
# Intialize queue variables used by each batch.
self._incomplete_messages = []
self._successful_messages = []
self._failed_messages = []
self._permanent_failures = []
self._batch_stop_time = time.time()
self._batch_queue = None
# Monitors whether the worker has been exposed to tasks and may
# have bloated in memory.
self._dirty = False
# Setup signal handling for cleanup.
for sig in SIGNALS_TO_HANDLE:
signal.signal(sig, self._cleanup_worker)
# Allow the client of this library to do any setup before
# starting the worker.
settings.ON_WORKER_STARTUP()
def _build_queue_info(self):
with mock.patch('boto.sqs') as mock_sqs:
mock_sqs_connection = mock.MagicMock()
mock_queue = mock.MagicMock()
mock_sqs_connection.lookup.return_value = mock_queue
mock_sqs_connection.create_queue.return_value = mock_queue
conn = mock_sqs.connection
conn.SQSConnection.return_value = mock_sqs_connection
mock_sqs.connect_to_region.return_value = mock_sqs_connection
sqs_inst = sqs.SQSTalk()
sqs_inst._connection = mock_sqs_connection
queue_config = tempfile.NamedTemporaryFile(delete=True)
queue_config.write(self.test_string.encode('utf8'))
queue_config.seek(0)
queue_info.QueueInfo._queues = None
queue_info.QueueInfo._simple_name_queues_map = None
qinfo = queue_info.QueueInfo(queue_config.name, sqs_inst,
queue_info.TaskQueue)
return qinfo