def test_send_event(self): job = TaskRequest( self.mytask.name, uuid(), [1], {'f': 'x'}, app=self.app, ) job.eventer = MockEventDispatcher() job.send_event('task-frobulated') self.assertIn('task-frobulated', job.eventer.sent)
def test_execute_using_pool(self): tid = uuid() tw = TaskRequest(mytask.name, tid, [4], {"f": "x"}) class MockPool(BasePool): target = None args = None kwargs = None def __init__(self, *args, **kwargs): pass def apply_async(self, target, args=None, kwargs=None, *margs, **mkwargs): self.target = target self.args = args self.kwargs = kwargs p = MockPool() tw.execute_using_pool(p) self.assertTrue(p.target) self.assertEqual(p.args[0], mytask.name) self.assertEqual(p.args[1], tid) self.assertEqual(p.args[2], [4]) self.assertIn("f", p.args[3]) self.assertIn([4], p.args)
def test_revoked_expires_expired(self): tw = TaskRequest(mytask.name, uuid(), [1], {"f": "x"}) tw.expires = datetime.now() - timedelta(days=1) tw.revoked() self.assertIn(tw.task_id, revoked) self.assertEqual(mytask.backend.get_status(tw.task_id), states.REVOKED)
def test_execute_using_pool_does_not_execute_revoked(self): job = TaskRequest( self.mytask.name, uuid(), [1], {'f': 'x'}, app=self.app, ) revoked.add(job.id) with self.assertRaises(TaskRevokedError): job.execute_using_pool(None)
def test_revoked(self): tw = TaskRequest(mytask.name, uuid(), [1], {"f": "x"}) with assert_signal_called(task_revoked, sender=tw.task, terminated=False, expired=False, signum=None): revoked.add(tw.id) self.assertTrue(tw.revoked()) self.assertTrue(tw._already_revoked) self.assertTrue(tw.acknowledged)
def test_execute_fail(self): tid = uuid() job = TaskRequest(self.mytask_raising.name, tid, [4], app=self.app) self.assertIsInstance(job.execute(), ExceptionInfo) meta = self.mytask_raising.backend.get_task_meta(tid) self.assertEqual(meta['status'], states.FAILURE) self.assertIsInstance(meta['result'], KeyError)
def test_execute(self): tid = uuid() job = TaskRequest(self.mytask.name, tid, [4], {'f': 'x'}, app=self.app) self.assertEqual(job.execute(), 256) meta = self.mytask.backend.get_task_meta(tid) self.assertEqual(meta['result'], 256) self.assertEqual(meta['status'], states.SUCCESS)
def test_terminate__task_started(self): pool = Mock() tw = TaskRequest(mytask.name, uuid(), [1], {"f": "x"}) tw.time_start = time.time() tw.worker_pid = 313 tw.terminate(pool, signal="KILL") pool.terminate_job.assert_called_with(tw.worker_pid, "KILL")
def test_execute_using_pool(self): tid = uuid() job = TaskRequest(self.mytask.name, tid, [4], {'f': 'x'}, app=self.app) class MockPool(BasePool): target = None args = None kwargs = None def __init__(self, *args, **kwargs): pass def apply_async(self, target, args=None, kwargs=None, *margs, **mkwargs): self.target = target self.args = args self.kwargs = kwargs p = MockPool() job.execute_using_pool(p) self.assertTrue(p.target) self.assertEqual(p.args[0], self.mytask.name) self.assertEqual(p.args[1], tid) self.assertEqual(p.args[2], [4]) self.assertIn('f', p.args[3]) self.assertIn([4], p.args) job.task.accept_magic_kwargs = False job.execute_using_pool(p)
def test_on_accepted_terminates(self): tw = TaskRequest(mytask.name, uuid(), [1], {"f": "x"}) pool = Mock() tw.terminate(pool, signal="KILL") self.assertFalse(pool.terminate_job.call_count) tw.on_accepted(pid=314, time_accepted=time.time()) pool.terminate_job.assert_called_with(314, "KILL")
def test_revoked_expires_not_expired(self): tw = TaskRequest(mytask.name, uuid(), [1], {"f": "x"}, expires=datetime.utcnow() + timedelta(days=1)) tw.revoked() self.assertNotIn(tw.id, revoked) self.assertNotEqual(mytask.backend.get_status(tw.id), states.REVOKED)
def test_from_message_missing_required_fields(self): body = {} m = Message(None, body=anyjson.dumps(body), backend="foo", content_type="application/json", content_encoding="utf-8") with self.assertRaises(KeyError): TaskRequest.from_message(m, m.decode())
def test_from_message_nonexistant_task(self): body = {"task": "cu.mytask.doesnotexist", "id": uuid(), "args": [2], "kwargs": {u"æØåveéðƒeæ": "bar"}} m = Message( None, body=anyjson.dumps(body), backend="foo", content_type="application/json", content_encoding="utf-8" ) with self.assertRaises(KeyError): TaskRequest.from_message(m, m.decode())
def test_execute_success_no_kwargs(self): tid = uuid() tw = TaskRequest(mytask_no_kwargs.name, tid, [4], {}) self.assertEqual(tw.execute(), 256) meta = mytask_no_kwargs.backend.get_task_meta(tid) self.assertEqual(meta["result"], 256) self.assertEqual(meta["status"], states.SUCCESS)
def test_execute(self): tid = uuid() tw = TaskRequest(mytask.name, tid, [4], {"f": "x"}) self.assertEqual(tw.execute(), 256) meta = mytask.backend.get_task_meta(tid) self.assertEqual(meta["result"], 256) self.assertEqual(meta["status"], states.SUCCESS)
def test_execute_success_no_kwargs(self): tid = gen_unique_id() tw = TaskRequest(mytask_no_kwargs.name, tid, [4], {}) self.assertEqual(tw.execute(), 256) meta = default_backend._get_task_meta_for(tid) self.assertEqual(meta["result"], 256) self.assertEqual(meta["status"], states.SUCCESS)
def test_send_email(self): from celery import conf from celery.worker import job old_mail_admins = job.mail_admins old_enable_mails = conf.CELERY_SEND_TASK_ERROR_EMAILS mail_sent = [False] def mock_mail_admins(*args, **kwargs): mail_sent[0] = True job.mail_admins = mock_mail_admins conf.CELERY_SEND_TASK_ERROR_EMAILS = True try: tw = TaskRequest(mytask.name, gen_unique_id(), [1], {"f": "x"}) try: raise KeyError("foo") except KeyError: einfo = ExceptionInfo(sys.exc_info()) tw.on_failure(einfo) self.assertTrue(mail_sent[0]) mail_sent[0] = False conf.CELERY_SEND_TASK_ERROR_EMAILS = False tw.on_failure(einfo) self.assertFalse(mail_sent[0]) finally: job.mail_admins = old_mail_admins conf.CELERY_SEND_TASK_ERROR_EMAILS = old_enable_mails
def test_execute_fail(self): tid = uuid() tw = TaskRequest(mytask_raising.name, tid, [4], {"f": "x"}) self.assertIsInstance(tw.execute(), ExceptionInfo) meta = mytask_raising.backend.get_task_meta(tid) self.assertEqual(meta["status"], states.FAILURE) self.assertIsInstance(meta["result"], KeyError)
def test_send_email(self): from celery.loaders import current_loader loader = current_loader() old_mail_admins = loader.mail_admins old_enable_mails = mytask.send_error_emails mail_sent = [False] def mock_mail_admins(*args, **kwargs): mail_sent[0] = True loader.mail_admins = mock_mail_admins mytask.send_error_emails = True try: tw = TaskRequest(mytask.name, gen_unique_id(), [1], {"f": "x"}) try: raise KeyError("moofoobar") except: einfo = ExceptionInfo(sys.exc_info()) tw.on_failure(einfo) self.assertTrue(mail_sent[0]) mail_sent[0] = False mytask.send_error_emails = False tw.on_failure(einfo) self.assertFalse(mail_sent[0]) finally: loader.mail_admins = old_mail_admins mytask.send_error_emails = old_enable_mails
def test_on_timeout(self): class MockLogger(object): def __init__(self): self.warnings = [] self.errors = [] def warning(self, msg, *args, **kwargs): self.warnings.append(msg % args) def error(self, msg, *args, **kwargs): self.errors.append(msg % args) tw = TaskRequest(mytask.name, uuid(), [1], {"f": "x"}) tw.logger = MockLogger() tw.on_timeout(soft=True, timeout=1337) self.assertIn("Soft time limit (1337s) exceeded", tw.logger.warnings[0]) tw.on_timeout(soft=False, timeout=1337) self.assertIn("Hard time limit (1337s) exceeded", tw.logger.errors[0]) self.assertEqual(mytask.backend.get_status(tw.id), states.FAILURE) mytask.ignore_result = True try: tw = TaskRequest(mytask.name, uuid(), [1], {"f": "x"}) tw.logger = MockLogger() finally: tw.on_timeout(soft=True, timeout=1336) self.assertEqual(mytask.backend.get_status(tw.id), states.PENDING) mytask.ignore_result = False
def test_send_email(self): app = app_or_default() old_mail_admins = app.mail_admins old_enable_mails = mytask.send_error_emails mail_sent = [False] def mock_mail_admins(*args, **kwargs): mail_sent[0] = True app.mail_admins = mock_mail_admins mytask.send_error_emails = True try: tw = TaskRequest(mytask.name, gen_unique_id(), [1], {"f": "x"}) try: raise KeyError("moofoobar") except: einfo = ExceptionInfo(sys.exc_info()) tw.on_failure(einfo) self.assertTrue(mail_sent[0]) mail_sent[0] = False mytask.send_error_emails = False tw.on_failure(einfo) self.assertFalse(mail_sent[0]) finally: app.mail_admins = old_mail_admins mytask.send_error_emails = old_enable_mails
def test_on_accepted_acks_late(self): job = TaskRequest( self.mytask.name, uuid(), [1], {'f': 'x'}, app=self.app, ) self.mytask.acks_late = True job.on_accepted(pid=os.getpid(), time_accepted=time.time()) self.assertFalse(job.acknowledged)
def test_execute_success_some_kwargs(self): tid = uuid() tw = TaskRequest(mytask_some_kwargs.name, tid, [4], {}) self.assertEqual(tw.execute(), 256) meta = mytask_some_kwargs.backend.get_task_meta(tid) self.assertEqual(some_kwargs_scratchpad.get('task_id'), tid) self.assertEqual(meta['result'], 256) self.assertEqual(meta['status'], states.SUCCESS)
def test_on_success_acks_late(self): job = TaskRequest( self.mytask.name, uuid(), [1], {'f': 'x'}, app=self.app, ) job.time_start = 1 self.mytask.acks_late = True job.on_success(42) self.assertTrue(job.acknowledged)
def test_on_accepted_acks_late(self): tw = TaskRequest(mytask.name, gen_unique_id(), [1], {"f": "x"}) mytask.acks_late = True try: tw.on_accepted() self.assertFalse(tw.acknowledged) finally: mytask.acks_late = False
def test_from_message_nonexistant_task(self): body = {'task': 'cu.mytask.doesnotexist', 'id': uuid(), 'args': [2], 'kwargs': {'æØåveéðƒeæ': 'bar'}} m = Message(None, body=anyjson.dumps(body), backend='foo', content_type='application/json', content_encoding='utf-8') with self.assertRaises(KeyError): TaskRequest.from_message(m, m.decode(), app=self.app)
def test_execute_acks_late(self): mytask_raising.acks_late = True tw = TaskRequest(mytask_raising.name, uuid(), [1], {"f": "x"}) try: tw.execute() self.assertTrue(tw.acknowledged) finally: mytask_raising.acks_late = False
def test_on_accepted_acks_late(self): tw = TaskRequest(mytask.name, uuid(), [1], {"f": "x"}) mytask.acks_late = True try: tw.on_accepted(pid=os.getpid(), time_accepted=time.time()) self.assertFalse(tw.acknowledged) finally: mytask.acks_late = False
def test_compat_properties(self): tw = TaskRequest(mytask.name, uuid(), [1], {"f": "x"}) self.assertEqual(tw.task_id, tw.id) self.assertEqual(tw.task_name, tw.name) tw.task_id = "ID" self.assertEqual(tw.id, "ID") tw.task_name = "NAME" self.assertEqual(tw.name, "NAME")
def test_execute_success_some_kwargs(self): tid = uuid() tw = TaskRequest(mytask_some_kwargs.name, tid, [4], {}) self.assertEqual(tw.execute(logfile="foobaz.log"), 256) meta = mytask_some_kwargs.backend.get_task_meta(tid) self.assertEqual(some_kwargs_scratchpad.get("logfile"), "foobaz.log") self.assertEqual(meta["result"], 256) self.assertEqual(meta["status"], states.SUCCESS)
def test_dump_reserved(self): from celery.worker import state consumer = Consumer() state.reserved_requests.add(TaskRequest(mytask.name, uuid(), args=(2, 2), kwargs={})) try: panel = self.create_panel(consumer=consumer) response = panel.handle('dump_reserved', {'safe': True}) self.assertDictContainsSubset({'name': mytask.name, 'args': (2, 2), 'kwargs': {}, 'hostname': socket.gethostname()}, response[0]) state.reserved_requests.clear() self.assertFalse(panel.handle('dump_reserved')) finally: state.reserved_requests.clear()
def test_task_wrapper_mail_attrs(self): tw = TaskRequest(mytask.name, gen_unique_id(), [], {}) x = tw.success_msg % {"name": tw.task_name, "id": tw.task_id, "return_value": 10, "runtime": 0.3641} self.assertTrue(x) x = tw.error_msg % {"name": tw.task_name, "id": tw.task_id, "exc": "FOOBARBAZ", "traceback": "foobarbaz"} self.assertTrue(x) x = tw.email_subject % {"name": tw.task_name, "id": tw.task_id, "exc": "FOOBARBAZ", "hostname": "lana"} self.assertTrue(x)
def test_terminate__task_started(self): pool = Mock() signum = signal.SIGKILL tw = TaskRequest(mytask.name, uuid(), [1], {'f': 'x'}) with assert_signal_called(task_revoked, sender=tw.task, terminated=True, expired=False, signum=signum): tw.time_start = time.time() tw.worker_pid = 313 tw.terminate(pool, signal='KILL') pool.terminate_job.assert_called_with(tw.worker_pid, signum)
def test_on_accepted_terminates(self): signum = signal.SIGKILL pool = Mock() tw = TaskRequest(mytask.name, uuid(), [1], {'f': 'x'}) with assert_signal_called(task_revoked, sender=tw.task, terminated=True, expired=False, signum=signum): tw.terminate(pool, signal='KILL') self.assertFalse(pool.terminate_job.call_count) tw.on_accepted(pid=314, time_accepted=time.time()) pool.terminate_job.assert_called_with(314, signum)
def test_on_failure_WorkerLostError(self): def get_ei(): try: raise WorkerLostError("do re mi") except WorkerLostError: return ExceptionInfo() tw = TaskRequest(mytask.name, uuid(), [1], {"f": "x"}) exc_info = get_ei() tw.on_failure(exc_info) self.assertEqual(mytask.backend.get_status(tw.id), states.FAILURE) mytask.ignore_result = True try: exc_info = get_ei() tw = TaskRequest(mytask.name, uuid(), [1], {"f": "x"}) tw.on_failure(exc_info) self.assertEqual(mytask.backend.get_status(tw.id), states.PENDING) finally: mytask.ignore_result = False
def test_send_email(self): app = self.app old_mail_admins = app.mail_admins old_enable_mails = mytask.send_error_emails mail_sent = [False] def mock_mail_admins(*args, **kwargs): mail_sent[0] = True def get_ei(): try: raise KeyError('moofoobar') except: return ExceptionInfo() app.mail_admins = mock_mail_admins mytask.send_error_emails = True try: tw = TaskRequest(mytask.name, uuid(), [1], {'f': 'x'}, app=self.app) einfo = get_ei() tw.on_failure(einfo) self.assertTrue(mail_sent[0]) einfo = get_ei() mail_sent[0] = False mytask.send_error_emails = False tw.on_failure(einfo) self.assertFalse(mail_sent[0]) einfo = get_ei() mail_sent[0] = False mytask.send_error_emails = True tw.on_failure(einfo) self.assertTrue(mail_sent[0]) finally: app.mail_admins = old_mail_admins mytask.send_error_emails = old_enable_mails
def test_on_timeout(self): class MockLogger(object): def __init__(self): self.warnings = [] self.errors = [] def warning(self, msg, *args, **kwargs): self.warnings.append(msg) def error(self, msg, *args, **kwargs): self.errors.append(msg) tw = TaskRequest(mytask.name, gen_unique_id(), [1], {"f": "x"}) tw.logger = MockLogger() tw.on_timeout(soft=True) self.assertIn("Soft time limit exceeded", tw.logger.warnings[0]) tw.on_timeout(soft=False) self.assertIn("Hard time limit exceeded", tw.logger.errors[0])
def test_from_message(self): body = { "task": mytask.name, "id": gen_unique_id(), "args": [2], "kwargs": { u"æØåveéðƒeæ": "bar" } } m = BaseMessage(body=simplejson.dumps(body), backend="foo", content_type="application/json", content_encoding="utf-8") tw = TaskRequest.from_message(m, m.decode()) self.assertIsInstance(tw, TaskRequest) self.assertEqual(tw.task_name, body["task"]) self.assertEqual(tw.task_id, body["id"]) self.assertEqual(tw.args, body["args"]) self.assertEqual(tw.kwargs.keys()[0], u"æØåveéðƒeæ".encode("utf-8")) self.assertNotIsInstance(tw.kwargs.keys()[0], unicode) self.assertTrue(tw.logger)
def test_from_message(self): us = u"æØåveéðƒeæ" body = { "task": mytask.name, "id": uuid(), "args": [2], "kwargs": { us: "bar" } } m = Message(None, body=anyjson.serialize(body), backend="foo", content_type="application/json", content_encoding="utf-8") tw = TaskRequest.from_message(m, m.decode()) self.assertIsInstance(tw, TaskRequest) self.assertEqual(tw.task_name, body["task"]) self.assertEqual(tw.task_id, body["id"]) self.assertEqual(tw.args, body["args"]) us = from_utf8(us) self.assertEqual(tw.kwargs.keys()[0], us) self.assertIsInstance(tw.kwargs.keys()[0], str) self.assertTrue(tw.logger)
def test_from_message(self): us = 'æØåveéðƒeæ' body = { 'task': mytask.name, 'id': uuid(), 'args': [2], 'kwargs': { us: 'bar' } } m = Message(None, body=anyjson.dumps(body), backend='foo', content_type='application/json', content_encoding='utf-8') tw = TaskRequest.from_message(m, m.decode(), app=self.app) self.assertIsInstance(tw, Request) self.assertEqual(tw.name, body['task']) self.assertEqual(tw.id, body['id']) self.assertEqual(tw.args, body['args']) us = from_utf8(us) if sys.version_info < (2, 6): self.assertEqual(next(keys(tw.kwargs)), us) self.assertIsInstance(next(keys(tw.kwargs)), str)
def test_on_timeout(self): class MockLogger(object): def __init__(self): self.warnings = [] self.errors = [] def warning(self, msg, *args, **kwargs): self.warnings.append(msg % args) def error(self, msg, *args, **kwargs): self.errors.append(msg % args) tw = TaskRequest(mytask.name, uuid(), [1], {"f": "x"}) tw.logger = MockLogger() tw.on_timeout(soft=True, timeout=1337) self.assertIn("Soft time limit (1337s) exceeded", tw.logger.warnings[0]) tw.on_timeout(soft=False, timeout=1337) self.assertIn("Hard time limit (1337s) exceeded", tw.logger.errors[0]) self.assertEqual(mytask.backend.get_status(tw.task_id), states.FAILURE) mytask.ignore_result = True try: tw = TaskRequest(mytask.name, uuid(), [1], {"f": "x"}) tw.logger = MockLogger() finally: tw.on_timeout(soft=True, timeout=1336) self.assertEqual(mytask.backend.get_status(tw.task_id), states.PENDING) mytask.ignore_result = False
def test_execute_does_not_execute_revoked(self): tw = TaskRequest(mytask.name, gen_unique_id(), [1], {"f": "x"}) revoked.add(tw.task_id) tw.execute()
def test_execute_using_pool_does_not_execute_revoked(self): tw = TaskRequest(mytask.name, uuid(), [1], {'f': 'x'}, app=self.app) revoked.add(tw.id) with self.assertRaises(TaskRevokedError): tw.execute_using_pool(None)
def test_send_event(self): tw = TaskRequest(mytask.name, uuid(), [1], {"f": "x"}) tw.eventer = MockEventDispatcher() tw.send_event("task-frobulated") self.assertIn("task-frobulated", tw.eventer.sent)
def test_task_wrapper_repr(self): tw = TaskRequest(mytask.name, uuid(), [1], {"f": "x"}) self.assertTrue(repr(tw))
def test_task_wrapper_repr(self): tw = TaskRequest(mytask.name, uuid(), [1], {'f': 'x'}, app=self.app) self.assertTrue(repr(tw))
def test_already_revoked(self): tw = TaskRequest(mytask.name, uuid(), [1], {'f': 'x'}, app=self.app) tw._already_revoked = True self.assertTrue(tw.revoked())
def test_execute_using_pool_does_not_execute_revoked(self): tw = TaskRequest(mytask.name, uuid(), [1], {"f": "x"}) revoked.add(tw.task_id) tw.execute_using_pool(None)
def test_on_accepted_acks_early(self): tw = TaskRequest(mytask.name, uuid(), [1], {"f": "x"}) tw.on_accepted(pid=os.getpid(), time_accepted=time.time()) self.assertTrue(tw.acknowledged)
def test_executed_bit(self): tw = TaskRequest(mytask.name, gen_unique_id(), [], {}) self.assertFalse(tw.executed) tw._set_executed_bit() self.assertTrue(tw.executed) self.assertRaises(AlreadyExecutedError, tw._set_executed_bit)
def test_on_success_acks_early(self): tw = TaskRequest(mytask.name, uuid(), [1], {"f": "x"}) tw.time_start = 1 tw.on_success(42) self.assertFalse(tw.acknowledged)
def test_on_accepted_acks_early(self): tw = TaskRequest(mytask.name, gen_unique_id(), [1], {"f": "x"}) tw.on_accepted() self.assertTrue(tw.acknowledged)
def test_from_message_invalid_kwargs(self): body = dict(task=mytask.name, id=1, args=(), kwargs="foo") with self.assertRaises(InvalidTaskError): TaskRequest.from_message(None, body)
def test_already_revoked(self): tw = TaskRequest(mytask.name, uuid(), [1], {"f": "x"}) tw._already_revoked = True self.assertTrue(tw.revoked())
def test_send_event(self): tw = TaskRequest(mytask.name, uuid(), [1], {'f': 'x'}, app=self.app) tw.eventer = MockEventDispatcher() tw.send_event('task-frobulated') self.assertIn('task-frobulated', tw.eventer.sent)
def test_revoked(self): tw = TaskRequest(mytask.name, uuid(), [1], {"f": "x"}) revoked.add(tw.task_id) self.assertTrue(tw.revoked()) self.assertTrue(tw._already_revoked) self.assertTrue(tw.acknowledged)
def test_execute_does_not_execute_revoked(self): tw = TaskRequest(mytask.name, uuid(), [1], {'f': 'x'}, app=self.app) revoked.add(tw.id) tw.execute()