def test_serialize_pickle(self):
worker_task1 = WorkerTask('id-1', None, 'default', dummy_function, [], {'object': TestObject()}, 5, 0, None, True)
msg = worker_task1.serialize()
worker_task2 = WorkerTask.deserialize(msg)
self.assertEqual(worker_task2.args, worker_task1.args)
self.assertEqual(worker_task2.kwargs['object'].message, worker_task1.kwargs['object'].message)
def test_serialize_pickle(self):
worker_task1 = WorkerTask('id-1', None, 'default', dummy_function, [], {'object': TestObject()}, 5, 0, None, True)
msg = worker_task1.serialize()
worker_task2 = WorkerTask.deserialize(msg)
self.assertEqual(worker_task2.args, worker_task1.args)
self.assertEqual(worker_task2.kwargs['object'].message, worker_task1.kwargs['object'].message)
def delay(self, group_id, queue_name, func, args, kwargs, max_retries,
use_pickle, delay, execute_inline):
# type: (unicode, unicode, Any, tuple, dict, int, bool, int, bool) -> Any
worker_task = WorkerTask(str(uuid.uuid4()), group_id, queue_name, func,
args, kwargs, max_retries, 0, None,
use_pickle)
return self._enqueue_task(worker_task, delay, execute_inline, False,
True)
def test_deserialize_worker_task(self):
worker_task = WorkerTask.deserialize(self.dummy_msg)
self.assertEqual(worker_task.id, 'id-1')
self.assertEqual(worker_task.group_id, 'group-5')
self.assertEqual(worker_task.queue, 'default')
self.assertEqual(worker_task.func, dummy_function)
self.assertEqual(worker_task.args, [])
self.assertEqual(worker_task.kwargs, {})
self.assertEqual(worker_task.max_retries, 5)
self.assertEqual(worker_task.retry, 0)
self.assertEqual(worker_task.retry_id, 'retry-uuid')
def test_deserialize_worker_task(self):
worker_task = WorkerTask.deserialize(self.dummy_msg)
self.assertEqual(worker_task.id, 'id-1')
self.assertEqual(worker_task.group_id, 'group-5')
self.assertEqual(worker_task.queue, 'default')
self.assertEqual(worker_task.func, dummy_function)
self.assertEqual(worker_task.args, [])
self.assertEqual(worker_task.kwargs, {})
self.assertEqual(worker_task.max_retries, 5)
self.assertEqual(worker_task.retry, 0)
self.assertEqual(worker_task.retry_id, 'retry-uuid')
def execute(self, msg):
# type: (unicode) -> Any
try:
worker_task = WorkerTask.deserialize(msg)
except Exception as ex:
logger.exception(
'Message %s is not a valid worker task: %s',
msg,
ex
)
raise InvalidMessageFormatException(msg, ex)
try:
if settings.DEAD_LETTER_MODE:
# If in dead letter mode only try to run callback. Do not execute task.
logger.debug(
'Task %s (%s, retry-id: %s) not executed (dead letter queue)',
worker_task.abs_func_name,
worker_task.id,
worker_task.retry_id,
)
self._remove_from_group(worker_task)
else:
logger.debug(
'Execute task %s (%s, retry-id: %s) with args: %s and kwargs: %s',
worker_task.abs_func_name,
worker_task.id,
worker_task.retry_id,
worker_task.args,
worker_task.kwargs
)
return self._execute_task(worker_task)
except QueueException:
raise
except MaxRetriesReachedException:
raise
except Exception as ex:
logger.exception(
'Task %s (%s, retry-id: %s) failed to execute with args: %s and kwargs: %s: %s',
worker_task.abs_func_name,
worker_task.id,
worker_task.retry_id,
worker_task.args,
worker_task.kwargs,
ex
)
raise ExecutionFailedException(worker_task.abs_func_name, ex)
def execute(self, msg, receive_count=1):
# type: (unicode, int) -> Any
try:
worker_task = WorkerTask.deserialize(msg)
if receive_count > 1:
logger.warning(
'[django-eb-sqs] SQS re-queued message {} times - queue: {} func: {} retry: {} args: {}'
.format(receive_count, worker_task.queue,
worker_task.abs_func_name, worker_task.retry,
worker_task.args))
except Exception as ex:
logger.exception('Message %s is not a valid worker task: %s', msg,
ex)
raise InvalidMessageFormatException(msg, ex)
try:
if settings.DEAD_LETTER_MODE:
# If in dead letter mode only try to run callback. Do not execute task.
logger.debug(
'Task %s (%s, retry-id: %s) not executed (dead letter queue)',
worker_task.abs_func_name,
worker_task.id,
worker_task.retry_id,
)
self._remove_from_group(worker_task)
else:
logger.debug(
'Execute task %s (%s, retry-id: %s) with args: %s and kwargs: %s',
worker_task.abs_func_name, worker_task.id,
worker_task.retry_id, worker_task.args, worker_task.kwargs)
return self._execute_task(worker_task)
except QueueException:
raise
except MaxRetriesReachedException:
raise
except Exception as ex:
logger.exception(
'Task %s (%s, retry-id: %s) failed to execute with args: %s and kwargs: %s: %s',
worker_task.abs_func_name, worker_task.id,
worker_task.retry_id, worker_task.args, worker_task.kwargs, ex)
raise ExecutionFailedException(worker_task.abs_func_name, ex)
def execute(self, msg):
# type: (unicode) -> Any
try:
worker_task = WorkerTask.deserialize(msg)
except Exception as ex:
logger.exception('Message %s is not a valid worker task: %s', msg,
ex)
raise InvalidMessageFormatException(msg, ex)
try:
if settings.DEAD_LETTER_MODE:
# If in dead letter mode only try to run callback. Do not execute task.
logger.debug(
'Task %s (%s, retry-id: %s) not executed (dead letter queue)',
worker_task.abs_func_name,
worker_task.id,
worker_task.retry_id,
)
self._remove_from_group(worker_task)
else:
logger.debug(
'Execute task %s (%s, retry-id: %s) with args: %s and kwargs: %s',
worker_task.abs_func_name, worker_task.id,
worker_task.retry_id, worker_task.args, worker_task.kwargs)
return self._execute_task(worker_task)
except QueueException:
raise
except MaxRetriesReachedException:
raise
except Exception as ex:
logger.exception(
'Task %s (%s, retry-id: %s) failed to execute with args: %s and kwargs: %s: %s',
worker_task.abs_func_name, worker_task.id,
worker_task.retry_id, worker_task.args, worker_task.kwargs, ex)
raise ExecutionFailedException(worker_task.abs_func_name, ex)
def test_serialize_worker_task(self):
worker_task = WorkerTask('id-1', 'group-5', 'default', dummy_function, [], {}, 5, 0, 'retry-uuid', False)
msg = worker_task.serialize()
self.assertDictEqual(json.loads(msg), json.loads(self.dummy_msg))
def test_serialize_worker_task(self):
worker_task = WorkerTask('id-1', 'group-5', 'default', dummy_function, [], {}, 5, 0, 'retry-uuid', False)
msg = worker_task.serialize()
self.assertEqual(msg, self.dummy_msg)