def run(self): """Main function of benchmarking.""" log.info('Start benchmarking ...') log.info('Spawning %d worker threads ...' % self.workers) sqs_talk = sqs.SQSTalk() queue_info_obj = queue_info.QueueInfo(self.config_file, sqs_talk) worker_threads = [] for i in range(self.workers): worker_thread = WorkerThread(queue_info_obj, self.iterations) worker_threads.append(worker_thread) worker_thread.setDaemon(True) for worker_thread in worker_threads: worker_thread.start() for worker_thread in worker_threads: worker_thread.join() all_check_time = [] while not checking_sqs_time.empty(): check_time = checking_sqs_time.get() all_check_time.append(check_time) log.info('=== Benchmark results ===') count = len(all_check_time) sum_time = sum(all_check_time) avg_time = sum_time / count median_time = sorted(all_check_time)[count / 2] max_time = max(all_check_time) log.info('Average Check Time: %f' % avg_time) log.info('Median Check time: %f' % median_time) log.info('Max Check Time: %f' % max_time)
def _build_queue_info(self): sqs_inst = sqs.SQSTalk() queue_config = tempfile.NamedTemporaryFile(delete=True) queue_config.write(self.test_string.encode('utf8')) queue_config.seek(0) queue_info.QueueInfo._queues = None queue_info.QueueInfo._simple_name_queues_map = None qinfo = queue_info.QueueInfo(queue_config.name, sqs_inst, queue_info.TaskQueue) return qinfo
def publish(self, task_class, task_id, payload, current_retry_num=None, current_failure_num=None, delay_sec=None): """Publish the given task type to the queue with the provided payload. :param obj task_class: class of the task that we are publishing. :param str task_id: unique identifying string for this task. :param dict payload: dictionary for the task payload. :param int current_retry_num: current task retry count. If 0, this is the first attempt to run the task. :param int current_failure_num: current task failure count. :param int delay_sec: time (in seconds) that a task should stay in the queue before being released to consumers. :raises: TaskTooChubbyException: This task is outrageously chubby. The publisher of the task should handle this exception and determine how to proceed. """ if delay_sec is not None and delay_sec > settings.SQS_MAX_TASK_DELAY_SEC: raise exceptions.InvalidTaskDelayException( 'Invalid task delay_sec (%d > %d).' % (delay_sec, settings.SQS_MAX_TASK_DELAY_SEC)) queue_class = utils.class_import_from_path(settings.QUEUE_CLASS) q_info = queue_info.QueueInfo(settings.QUEUE_CONFIG, self, queue_class) queue_obj = q_info.get_queue(task_class.queue) if task_class.time_limit >= queue_obj.visibility_timeout_sec: raise exceptions.InvalidTimeLimitTaskException( 'Invalid task time limit: %d >= %d from %s' % (task_class.time_limit, queue_obj.visibility_timeout_sec, settings.QUEUE_CONFIG)) sqs_queue = self._get_or_create_queue(queue_obj.name) kale_msg = message.KaleMessage(task_class=task_class, task_id=task_id, payload=payload, current_retry_num=current_retry_num, current_failure_num=current_failure_num) sqs_queue.send_message(MessageBody=kale_msg.encode(), DelaySeconds=delay_sec or 1) logger.debug('Published task. Task id: %s; Task name: %s' % (task_id, '%s.%s' % (task_class.__module__, task_class.__name__)))
def __init__(self): """Initialize a worker instance.""" self._consumer = consumer.Consumer() queue_class = utils.class_import_from_path(settings.QUEUE_CLASS) q_info = queue_info.QueueInfo( config_file=settings.QUEUE_CONFIG, sqs_talk=self._consumer, queue_cls=queue_class) queue_selector_class = utils.class_import_from_path( settings.QUEUE_SELECTOR) self._queue_selector = queue_selector_class(q_info) # The worker will publish permanently failed tasks to a # dead-letter-queue. self._publisher = self._get_publisher() # Track total messages processed. self._total_messages_processed = 0 # Intialize queue variables used by each batch. self._incomplete_messages = [] self._successful_messages = [] self._failed_messages = [] self._permanent_failures = [] self._batch_stop_time = time.time() self._batch_queue = None # Monitors whether the worker has been exposed to tasks and may # have bloated in memory. self._dirty = False # Setup signal handling for cleanup. for sig in SIGNALS_TO_HANDLE: signal.signal(sig, self._cleanup_worker) # Allow the client of this library to do any setup before # starting the worker. settings.ON_WORKER_STARTUP()
def _build_queue_info(self): with mock.patch('boto.sqs') as mock_sqs: mock_sqs_connection = mock.MagicMock() mock_queue = mock.MagicMock() mock_sqs_connection.lookup.return_value = mock_queue mock_sqs_connection.create_queue.return_value = mock_queue conn = mock_sqs.connection conn.SQSConnection.return_value = mock_sqs_connection mock_sqs.connect_to_region.return_value = mock_sqs_connection sqs_inst = sqs.SQSTalk() sqs_inst._connection = mock_sqs_connection queue_config = tempfile.NamedTemporaryFile(delete=True) queue_config.write(self.test_string.encode('utf8')) queue_config.seek(0) queue_info.QueueInfo._queues = None queue_info.QueueInfo._simple_name_queues_map = None qinfo = queue_info.QueueInfo(queue_config.name, sqs_inst, queue_info.TaskQueue) return qinfo