def test_interface__compat(self):
def with_catch_warnings(log):
ts = TaskSet(MockTask, [[(2, 2)], [(4, 4)], [(8, 8)]])
self.assertTrue(log)
self.assertIn("Using this invocation of TaskSet is deprecated",
log[0].message.args[0])
return ts
context = catch_warnings(record=True)
ts = execute_context(context, with_catch_warnings)
self.assertListEqual(ts.tasks,
[MockTask.subtask((i, i))
for i in (2, 4, 8)])
# TaskSet.task (deprecated)
def with_catch_warnings2(log):
self.assertEqual(ts.task, MockTask)
self.assertTrue(log)
self.assertIn("TaskSet.task is deprecated",
log[0].message.args[0])
execute_context(catch_warnings(record=True), with_catch_warnings2)
# TaskSet.task_name (deprecated)
def with_catch_warnings3(log):
self.assertEqual(ts.task_name, MockTask.name)
self.assertTrue(log)
self.assertIn("TaskSet.task_name is deprecated",
log[0].message.args[0])
execute_context(catch_warnings(record=True), with_catch_warnings3)
def test_missing_SQLAlchemy_raises_ImproperlyConfigured(self):
def with_SQLAlchemy_masked(_val):
from celery.backends.database import _sqlalchemy_installed
self.assertRaises(ImproperlyConfigured, _sqlalchemy_installed)
execute_context(mask_modules("sqlalchemy"), with_SQLAlchemy_masked)
def _assertLog(self, logger, logmsg, loglevel=logging.ERROR):
def with_wrap_logger(sio):
logger.log(loglevel, logmsg)
return sio.getvalue().strip()
context = wrap_logger(logger, loglevel=loglevel)
execute_context(context, with_wrap_logger)
def with_eager_tasks(_val):
def with_mock_urlopen(_val):
d = http.URL("http://example.com/mul").post_async(x=10, y=10)
self.assertEqual(d.get(), 100)
context = mock_urlopen(success_response(100))
execute_context(context, with_mock_urlopen)
def test_emergency_error_stderr(self):
def with_override_stdouts(outs):
stdout, stderr = outs
emergency_error(None, "The lazy dog crawls under the fast fox")
self.assertIn("The lazy dog crawls under the fast fox",
stderr.getvalue())
context = override_stdouts()
execute_context(context, with_override_stdouts)
def test_tyrant_None_if_tyrant_not_installed(self):
prev = sys.modules.pop("celery.backends.pyredis")
try:
def with_redis_masked(_val):
from celery.backends.pyredis import redis
self.assertIsNone(redis)
context = mask_modules("redis")
execute_context(context, with_redis_masked)
finally:
sys.modules["celery.backends.pyredis"] = prev
def test_dispatch_failure(self):
logger = logging.getLogger("celery.unittest")
def with_mock_urlopen(_val):
d = http.HttpDispatch("http://example.com/mul", "GET", {
"x": 10, "y": 10}, logger)
self.assertRaises(http.RemoteExecuteError, d.dispatch)
context = mock_urlopen(fail_response("Invalid moon alignment"))
execute_context(context, with_mock_urlopen)
def test_dispatch_non_json(self):
logger = logging.getLogger("celery.unittest")
def with_mock_urlopen(_val):
d = http.HttpDispatch("http://example.com/mul", "GET", {
"x": 10, "y": 10}, logger)
self.assertRaises(http.InvalidResponseError, d.dispatch)
context = mock_urlopen(_response("{'#{:'''"))
execute_context(context, with_mock_urlopen)
def test_dispatch_success(self):
logger = logging.getLogger("celery.unittest")
def with_mock_urlopen(_val):
d = http.HttpDispatch("http://example.com/mul", "GET", {
"x": 10, "y": 10}, logger)
self.assertEqual(d.dispatch(), 100)
context = mock_urlopen(success_response(100))
execute_context(context, with_mock_urlopen)
def test_dispatch_unknown_status(self):
logger = logging.getLogger("celery.unittest")
def with_mock_urlopen(_val):
d = http.HttpDispatch("http://example.com/mul", "GET", {
"x": 10, "y": 10}, logger)
self.assertRaises(http.UnknownStatusError, d.dispatch)
context = mock_urlopen(unknown_response())
execute_context(context, with_mock_urlopen)
def test_redis_None_if_redis_not_installed(self):
prev = sys.modules.pop("celery.backends.pyredis")
try:
def with_redis_masked(_val):
from celery.backends.pyredis import redis
self.assertIsNone(redis)
context = mask_modules("redis")
execute_context(context, with_redis_masked)
finally:
sys.modules["celery.backends.pyredis"] = prev
def test_dispatch_POST(self):
logger = logging.getLogger("celery.unittest")
def with_mock_urlopen(_val):
d = http.HttpDispatch("http://example.com/mul", "POST", {
"x": 10,
"y": 10
}, logger)
self.assertEqual(d.dispatch(), 100)
context = mock_urlopen(success_response(100))
execute_context(context, with_mock_urlopen)
def test_URL_post_async(self):
http.HttpDispatchTask.app.conf.CELERY_ALWAYS_EAGER = True
try:
def with_mock_urlopen(_val):
d = http.URL("http://example.com/mul").post_async(x=10, y=10)
self.assertEqual(d.get(), 100)
context = mock_urlopen(success_response(100))
execute_context(context, with_mock_urlopen)
finally:
http.HttpDispatchTask.app.conf.CELERY_ALWAYS_EAGER = False
def test_URL_get_async(self):
http.HttpDispatchTask.app.conf.CELERY_ALWAYS_EAGER = True
try:
def with_mock_urlopen(_val):
d = http.URL("http://example.com/mul").get_async(x=10, y=10)
self.assertEqual(d.get(), 100)
context = mock_urlopen(success_response(100))
execute_context(context, with_mock_urlopen)
finally:
http.HttpDispatchTask.app.conf.CELERY_ALWAYS_EAGER = False
def test_dispatch_unknown_status(self):
logger = logging.getLogger("celery.unittest")
def with_mock_urlopen(_val):
d = http.HttpDispatch("http://example.com/mul", "GET", {
"x": 10,
"y": 10
}, logger)
self.assertRaises(http.UnknownStatusError, d.dispatch)
context = mock_urlopen(unknown_response())
execute_context(context, with_mock_urlopen)
def test_dispatch_non_json(self):
logger = logging.getLogger("celery.unittest")
def with_mock_urlopen(_val):
d = http.HttpDispatch("http://example.com/mul", "GET", {
"x": 10,
"y": 10
}, logger)
self.assertRaises(http.InvalidResponseError, d.dispatch)
context = mock_urlopen(_response("{'#{:'''"))
execute_context(context, with_mock_urlopen)
def test_dispatch_failure(self):
logger = logging.getLogger("celery.unittest")
def with_mock_urlopen(_val):
d = http.HttpDispatch("http://example.com/mul", "GET", {
"x": 10,
"y": 10
}, logger)
self.assertRaises(http.RemoteExecuteError, d.dispatch)
context = mock_urlopen(fail_response("Invalid moon alignment"))
execute_context(context, with_mock_urlopen)
def test_redirect_stdouts(self):
logger = setup_logger(loglevel=logging.ERROR, logfile=None)
try:
def with_wrap_logger(sio):
redirect_stdouts_to_logger(logger, loglevel=logging.ERROR)
logger.error("foo")
self.assertIn("foo", sio.getvalue())
context = wrap_logger(logger)
execute_context(context, with_wrap_logger)
finally:
sys.stdout, sys.stderr = sys.__stdout__, sys.__stderr__
def test_setup_logger_no_handlers_stream(self):
l = self.get_logger()
set_handlers(l, [])
def with_override_stdouts(outs):
stdout, stderr = outs
l = self.setup_logger(logfile=stderr, loglevel=logging.INFO,
root=False)
l.info("The quick brown fox...")
self.assertIn("The quick brown fox...", stderr.getvalue())
context = override_stdouts()
execute_context(context, with_override_stdouts)
def test_setup_logger_no_handlers_stream(self):
from multiprocessing import get_logger
l = get_logger()
l.handlers = []
def with_override_stdouts(outs):
stdout, stderr = outs
l = setup_logger(logfile=stderr, loglevel=logging.INFO)
l.info("The quick brown fox...")
self.assertIn("The quick brown fox...", stderr.getvalue())
context = override_stdouts()
execute_context(context, with_override_stdouts)
def test_no_cpickle(self):
prev = sys.modules.pop("billiard.serialization")
try:
def with_cPickle_masked(_val):
from billiard.serialization import pickle
import pickle as orig_pickle
self.assertIs(pickle.dumps, orig_pickle.dumps)
context = mask_modules("cPickle")
execute_context(context, with_cPickle_masked)
finally:
sys.modules["billiard.serialization"] = prev
def test_no_cpickle(self):
prev = sys.modules.pop("celery.utils.serialization", None)
try:
def with_cPickle_masked(_val):
from celery.utils.serialization import pickle
import pickle as orig_pickle
self.assertIs(pickle.dumps, orig_pickle.dumps)
context = mask_modules("cPickle")
execute_context(context, with_cPickle_masked)
finally:
sys.modules["celery.utils.serialization"] = prev
def test_receive_message_unknown(self):
l = MyKombuConsumer(self.ready_queue, self.eta_schedule, self.logger,
send_events=False)
backend = MockBackend()
m = create_message(backend, unknown={"baz": "!!!"})
l.event_dispatcher = MockEventDispatcher()
l.pidbox_node = MockNode()
def with_catch_warnings(log):
l.receive_message(m.decode(), m)
self.assertTrue(log)
self.assertIn("unknown message", log[0].message.args[0])
context = catch_warnings(record=True)
execute_context(context, with_catch_warnings)
def test_unconfigured_settings(self):
context_executed = [False]
class _Loader(default.Loader):
def import_from_cwd(self, name):
raise ImportError(name)
def with_catch_warnings(log):
l = _Loader()
self.assertEqual(l.conf.CELERY_RESULT_BACKEND, "amqp")
context_executed[0] = True
context = catch_warnings(record=True)
execute_context(context, with_catch_warnings)
self.assertTrue(context_executed[0])
def test_run(self):
self.Worker().run()
self.Worker(discard=True).run()
worker = self.Worker()
worker.init_loader()
worker.settings.DEBUG = True
def with_catch_warnings(log):
worker.run()
self.assertIn("memory leak", log[0].message.args[0])
context = catch_warnings(record=True)
execute_context(context, with_catch_warnings)
worker.settings.DEBUG = False
def test_receive_message_unknown(self):
l = CarrotListener(self.ready_queue, self.eta_schedule, self.logger,
send_events=False)
backend = MockBackend()
m = create_message(backend, unknown={"baz": "!!!"})
l.event_dispatcher = MockEventDispatcher()
l.control_dispatch = MockControlDispatch()
def with_catch_warnings(log):
l.receive_message(m.decode(), m)
self.assertTrue(log)
self.assertTrue("unknown message" in log[0].message.args[0])
context = catch_warnings(record=True)
execute_context(context, with_catch_warnings)
def test_unconfigured_settings(self):
context_executed = [False]
class _Loader(default.Loader):
def import_from_cwd(self, name):
raise ImportError(name)
def with_catch_warnings(log):
l = _Loader()
self.assertEqual(l.conf.CELERY_RESULT_BACKEND, "amqp")
context_executed[0] = True
context = catch_warnings(record=True)
execute_context(context, with_catch_warnings)
self.assertTrue(context_executed[0])
def test_gen_unique_id_without_ctypes(self):
old_utils = sys.modules.pop("celery.utils")
def with_ctypes_masked(_val):
from celery.utils import ctypes, gen_unique_id
self.assertIsNone(ctypes)
uuid = gen_unique_id()
self.assertTrue(uuid)
self.assertIsInstance(uuid, basestring)
try:
context = mask_modules("ctypes")
execute_context(context, with_ctypes_masked)
finally:
sys.modules["celery.utils"] = old_utils
def test_receive_message_unknown(self):
l = CarrotListener(self.ready_queue,
self.eta_schedule,
self.logger,
send_events=False)
backend = MockBackend()
m = create_message(backend, unknown={"baz": "!!!"})
l.event_dispatcher = MockEventDispatcher()
l.control_dispatch = MockControlDispatch()
def with_catch_warnings(log):
l.receive_message(m.decode(), m)
self.assertTrue(log)
self.assertIn("unknown message", log[0].message.args[0])
context = catch_warnings(record=True)
execute_context(context, with_catch_warnings)
def setUp(self):
warnings.resetwarnings()
def with_catch_warnings(log):
from celery import decorators
return decorators
context = catch_warnings(record=True)
self.decorators = execute_context(context, with_catch_warnings)
def setUp(self):
warnings.resetwarnings()
def with_catch_warnings(log):
from celery import decorators
return decorators
context = catch_warnings(record=True)
self.decorators = execute_context(context, with_catch_warnings)
def test_execute_safe_catches_exception(self):
old_exec = WorkerTaskTrace.execute
def _error_exec(self, *args, **kwargs):
raise KeyError("baz")
WorkerTaskTrace.execute = _error_exec
try:
def with_catch_warnings(log):
res = execute_and_trace(mytask.name, gen_unique_id(),
[4], {})
self.assertIsInstance(res, ExceptionInfo)
self.assertTrue(log)
self.assertIn("Exception outside", log[0].message.args[0])
self.assertIn("KeyError", log[0].message.args[0])
context = catch_warnings(record=True)
execute_context(context, with_catch_warnings)
finally:
WorkerTaskTrace.execute = old_exec
def test_warns_if_running_as_privileged_user(self):
app = app_or_default()
if app.IS_WINDOWS:
raise SkipTest("Not applicable on Windows")
warnings.resetwarnings()
def geteuid():
return 0
prev, os.geteuid = os.geteuid, geteuid
try:
def with_catch_warnings(log):
worker = self.Worker()
worker.run()
self.assertTrue(log)
self.assertIn("superuser privileges is not encouraged",
log[0].message.args[0])
context = catch_warnings(record=True)
execute_context(context, with_catch_warnings)
finally:
os.geteuid = prev
def test_TaskSet_import_from_task_base(self):
warnings.resetwarnings()
def block(log):
from celery.task.base import TaskSet, subtask
TaskSet()
subtask(PingTask)
return log[0].message, log[1].message
for w in execute_context(catch_warnings(record=True), block):
self.assertIsInstance(w, DeprecationWarning)
self.assertIn("is deprecated", w.args[0])
def test_TaskSet_import_from_task_base(self):
warnings.resetwarnings()
def block(log):
from celery.task.base import TaskSet, subtask
TaskSet()
subtask(PingTask)
return log[0].message, log[1].message
for w in execute_context(catch_warnings(record=True), block):
self.assertIsInstance(w, DeprecationWarning)
self.assertIn("is deprecated", w.args[0])
def test_interface__compat(self):
def with_catch_warnings(log):
ts = TaskSet(MockTask, [[(2, 2)], [(4, 4)], [(8, 8)]])
self.assertTrue(log)
self.assertIn("Using this invocation of TaskSet is deprecated",
log[0].message.args[0])
self.assertListEqual(ts.tasks,
[MockTask.subtask((i, i)) for i in (2, 4, 8)])
return ts
context = catch_warnings(record=True)
execute_context(context, with_catch_warnings)
# TaskSet.task (deprecated)
def with_catch_warnings2(log):
ts = TaskSet(MockTask, [[(2, 2)], [(4, 4)], [(8, 8)]])
self.assertEqual(ts.task, MockTask)
self.assertTrue(log)
self.assertIn("TaskSet.task is deprecated", log[0].message.args[0])
execute_context(catch_warnings(record=True), with_catch_warnings2)
# TaskSet.task_name (deprecated)
def with_catch_warnings3(log):
ts = TaskSet(MockTask, [[(2, 2)], [(4, 4)], [(8, 8)]])
self.assertEqual(ts.task_name, MockTask.name)
self.assertTrue(log)
self.assertIn("TaskSet.task_name is deprecated",
log[0].message.args[0])
execute_context(catch_warnings(record=True), with_catch_warnings3)
def test_logging_proxy(self):
logger = setup_logger(loglevel=logging.ERROR, logfile=None)
def with_wrap_logger(sio):
p = LoggingProxy(logger)
p.close()
p.write("foo")
self.assertNotIn("foo", sio.getvalue())
p.closed = False
p.write("foo")
self.assertIn("foo", sio.getvalue())
lines = ["baz", "xuzzy"]
p.writelines(lines)
for line in lines:
self.assertIn(line, sio.getvalue())
p.flush()
p.close()
self.assertFalse(p.isatty())
self.assertIsNone(p.fileno())
context = wrap_logger(logger)
execute_context(context, with_wrap_logger)
def test_warns_if_running_as_privileged_user(self):
app = current_app
if app.IS_WINDOWS:
raise SkipTest("Not applicable on Windows")
warnings.resetwarnings()
def geteuid():
return 0
prev, os.geteuid = os.geteuid, geteuid
try:
def with_catch_warnings(log):
worker = self.Worker()
worker.run()
self.assertTrue(log)
self.assertIn("superuser privileges is not encouraged",
log[0].message.args[0])
context = catch_warnings(record=True)
execute_context(context, with_catch_warnings)
finally:
os.geteuid = prev
def test_ping(self):
warnings.resetwarnings()
def block(log):
prev = PingTask.app.conf.CELERY_ALWAYS_EAGER
PingTask.app.conf.CELERY_ALWAYS_EAGER = True
try:
return ping(), log[0].message
finally:
PingTask.app.conf.CELERY_ALWAYS_EAGER = prev
pong, warning = execute_context(catch_warnings(record=True), block)
self.assertEqual(pong, "pong")
self.assertIsInstance(warning, DeprecationWarning)
self.assertIn("ping task has been deprecated", warning.args[0])
def test_mail_admins_errors(self):
MockMail.Mailer.raise_on_send = True
opts = dict(self.message_options, **self.server_options)
def with_catch_warnings(log):
self.loader.mail_admins(fail_silently=True, **opts)
return log[0].message
warning = execute_context(catch_warnings(record=True),
with_catch_warnings)
self.assertIsInstance(warning, MockMail.SendmailWarning)
self.assertIn("KeyError", warning.args[0])
self.assertRaises(KeyError, self.loader.mail_admins,
fail_silently=False, **opts)
def test_ping(self):
warnings.resetwarnings()
def block(log):
prev = PingTask.app.conf.CELERY_ALWAYS_EAGER
PingTask.app.conf.CELERY_ALWAYS_EAGER = True
try:
return ping(), log[0].message
finally:
PingTask.app.conf.CELERY_ALWAYS_EAGER = prev
pong, warning = execute_context(catch_warnings(record=True), block)
self.assertEqual(pong, "pong")
self.assertIsInstance(warning, DeprecationWarning)
self.assertIn("ping task has been deprecated",
warning.args[0])
def test_process_cleanup_fails(self):
backend = mytask.backend
mytask.backend = Mock()
mytask.backend.process_cleanup = Mock(side_effect=KeyError())
try:
def with_wrap_logger(sio):
uuid = gen_unique_id()
ret = jail(uuid, mytask.name, [2], {})
self.assertEqual(ret, 4)
mytask.backend.mark_as_done.assert_called_with(uuid, 4)
logs = sio.getvalue().strip()
self.assertIn("Process cleanup failed", logs)
return 1234
logger = mytask.app.log.get_default_logger()
self.assertEqual(execute_context(
wrap_logger(logger), with_wrap_logger), 1234)
finally:
mytask.backend = backend
