def test_serialize_pickle(self): worker_task1 = WorkerTask('id-1', None, 'default', dummy_function, [], {'object': TestObject()}, 5, 0, None, True) msg = worker_task1.serialize() worker_task2 = WorkerTask.deserialize(msg) self.assertEqual(worker_task2.args, worker_task1.args) self.assertEqual(worker_task2.kwargs['object'].message, worker_task1.kwargs['object'].message)
def test_serialize_pickle(self): worker_task1 = WorkerTask('id-1', None, 'default', dummy_function, [], {'object': TestObject()}, 5, 0, None, True) msg = worker_task1.serialize() worker_task2 = WorkerTask.deserialize(msg) self.assertEqual(worker_task2.args, worker_task1.args) self.assertEqual(worker_task2.kwargs['object'].message, worker_task1.kwargs['object'].message)
def delay(self, group_id, queue_name, func, args, kwargs, max_retries, use_pickle, delay, execute_inline): # type: (unicode, unicode, Any, tuple, dict, int, bool, int, bool) -> Any worker_task = WorkerTask(str(uuid.uuid4()), group_id, queue_name, func, args, kwargs, max_retries, 0, None, use_pickle) return self._enqueue_task(worker_task, delay, execute_inline, False, True)
def test_deserialize_worker_task(self): worker_task = WorkerTask.deserialize(self.dummy_msg) self.assertEqual(worker_task.id, 'id-1') self.assertEqual(worker_task.group_id, 'group-5') self.assertEqual(worker_task.queue, 'default') self.assertEqual(worker_task.func, dummy_function) self.assertEqual(worker_task.args, []) self.assertEqual(worker_task.kwargs, {}) self.assertEqual(worker_task.max_retries, 5) self.assertEqual(worker_task.retry, 0) self.assertEqual(worker_task.retry_id, 'retry-uuid')
def test_deserialize_worker_task(self): worker_task = WorkerTask.deserialize(self.dummy_msg) self.assertEqual(worker_task.id, 'id-1') self.assertEqual(worker_task.group_id, 'group-5') self.assertEqual(worker_task.queue, 'default') self.assertEqual(worker_task.func, dummy_function) self.assertEqual(worker_task.args, []) self.assertEqual(worker_task.kwargs, {}) self.assertEqual(worker_task.max_retries, 5) self.assertEqual(worker_task.retry, 0) self.assertEqual(worker_task.retry_id, 'retry-uuid')
def execute(self, msg): # type: (unicode) -> Any try: worker_task = WorkerTask.deserialize(msg) except Exception as ex: logger.exception( 'Message %s is not a valid worker task: %s', msg, ex ) raise InvalidMessageFormatException(msg, ex) try: if settings.DEAD_LETTER_MODE: # If in dead letter mode only try to run callback. Do not execute task. logger.debug( 'Task %s (%s, retry-id: %s) not executed (dead letter queue)', worker_task.abs_func_name, worker_task.id, worker_task.retry_id, ) self._remove_from_group(worker_task) else: logger.debug( 'Execute task %s (%s, retry-id: %s) with args: %s and kwargs: %s', worker_task.abs_func_name, worker_task.id, worker_task.retry_id, worker_task.args, worker_task.kwargs ) return self._execute_task(worker_task) except QueueException: raise except MaxRetriesReachedException: raise except Exception as ex: logger.exception( 'Task %s (%s, retry-id: %s) failed to execute with args: %s and kwargs: %s: %s', worker_task.abs_func_name, worker_task.id, worker_task.retry_id, worker_task.args, worker_task.kwargs, ex ) raise ExecutionFailedException(worker_task.abs_func_name, ex)
def execute(self, msg, receive_count=1): # type: (unicode, int) -> Any try: worker_task = WorkerTask.deserialize(msg) if receive_count > 1: logger.warning( '[django-eb-sqs] SQS re-queued message {} times - queue: {} func: {} retry: {} args: {}' .format(receive_count, worker_task.queue, worker_task.abs_func_name, worker_task.retry, worker_task.args)) except Exception as ex: logger.exception('Message %s is not a valid worker task: %s', msg, ex) raise InvalidMessageFormatException(msg, ex) try: if settings.DEAD_LETTER_MODE: # If in dead letter mode only try to run callback. Do not execute task. logger.debug( 'Task %s (%s, retry-id: %s) not executed (dead letter queue)', worker_task.abs_func_name, worker_task.id, worker_task.retry_id, ) self._remove_from_group(worker_task) else: logger.debug( 'Execute task %s (%s, retry-id: %s) with args: %s and kwargs: %s', worker_task.abs_func_name, worker_task.id, worker_task.retry_id, worker_task.args, worker_task.kwargs) return self._execute_task(worker_task) except QueueException: raise except MaxRetriesReachedException: raise except Exception as ex: logger.exception( 'Task %s (%s, retry-id: %s) failed to execute with args: %s and kwargs: %s: %s', worker_task.abs_func_name, worker_task.id, worker_task.retry_id, worker_task.args, worker_task.kwargs, ex) raise ExecutionFailedException(worker_task.abs_func_name, ex)
def execute(self, msg): # type: (unicode) -> Any try: worker_task = WorkerTask.deserialize(msg) except Exception as ex: logger.exception('Message %s is not a valid worker task: %s', msg, ex) raise InvalidMessageFormatException(msg, ex) try: if settings.DEAD_LETTER_MODE: # If in dead letter mode only try to run callback. Do not execute task. logger.debug( 'Task %s (%s, retry-id: %s) not executed (dead letter queue)', worker_task.abs_func_name, worker_task.id, worker_task.retry_id, ) self._remove_from_group(worker_task) else: logger.debug( 'Execute task %s (%s, retry-id: %s) with args: %s and kwargs: %s', worker_task.abs_func_name, worker_task.id, worker_task.retry_id, worker_task.args, worker_task.kwargs) return self._execute_task(worker_task) except QueueException: raise except MaxRetriesReachedException: raise except Exception as ex: logger.exception( 'Task %s (%s, retry-id: %s) failed to execute with args: %s and kwargs: %s: %s', worker_task.abs_func_name, worker_task.id, worker_task.retry_id, worker_task.args, worker_task.kwargs, ex) raise ExecutionFailedException(worker_task.abs_func_name, ex)
def test_serialize_worker_task(self): worker_task = WorkerTask('id-1', 'group-5', 'default', dummy_function, [], {}, 5, 0, 'retry-uuid', False) msg = worker_task.serialize() self.assertDictEqual(json.loads(msg), json.loads(self.dummy_msg))
def test_serialize_worker_task(self): worker_task = WorkerTask('id-1', 'group-5', 'default', dummy_function, [], {}, 5, 0, 'retry-uuid', False) msg = worker_task.serialize() self.assertEqual(msg, self.dummy_msg)