def test_shutdown_nodes(self, slepp, gethostname, Pidfile):
gethostname.return_value = 'e.com'
self.prepare_pidfile_for_getpids(Pidfile)
self.assertIsNone(self.t.shutdown_nodes([]))
self.t.signal_node = Mock()
node_alive = self.t.node_alive = Mock()
self.t.node_alive.return_value = False
callback = Mock()
self.t.stop(['foo', 'bar', 'baz'], 'celery worker', callback=callback)
sigs = sorted(self.t.signal_node.call_args_list)
self.assertEqual(len(sigs), 2)
self.assertIn(
('*****@*****.**', 10, signal.SIGTERM),
{tup[0]
for tup in sigs},
)
self.assertIn(
('*****@*****.**', 11, signal.SIGTERM),
{tup[0]
for tup in sigs},
)
self.t.signal_node.return_value = False
callback.assert_called()
self.t.stop(['foo', 'bar', 'baz'], 'celery worker', callback=None)
def on_node_alive(pid):
if node_alive.call_count > 4:
return True
return False
self.t.signal_node.return_value = True
self.t.node_alive.side_effect = on_node_alive
self.t.stop(['foo', 'bar', 'baz'], 'celery worker', retry=True)
def test_getpids(self, gethostname, Pidfile):
gethostname.return_value = 'e.com'
self.prepare_pidfile_for_getpids(Pidfile)
callback = Mock()
p = NamespacedOptionParser(['foo', 'bar', 'baz'])
nodes = self.t.getpids(p, 'celery worker', callback=callback)
node_0, node_1 = nodes
self.assertEqual(node_0[0], '*****@*****.**')
self.assertEqual(
sorted(node_0[1]),
sorted(('celery worker', '--pidfile=foo.pid', '-n [email protected]', '')),
)
self.assertEqual(node_0[2], 10)
self.assertEqual(node_1[0], '*****@*****.**')
self.assertEqual(
sorted(node_1[1]),
sorted(('celery worker', '--pidfile=bar.pid', '-n [email protected]', '')),
)
self.assertEqual(node_1[2], 11)
callback.assert_called()
cargs, _ = callback.call_args
self.assertEqual(cargs[0], '*****@*****.**')
self.assertItemsEqual(
cargs[1],
['celery worker', '--pidfile=baz.pid', '-n [email protected]', ''],
)
self.assertIsNone(cargs[2])
self.assertIn('DOWN', self.fh.getvalue())
# without callback, should work
nodes = self.t.getpids(p, 'celery worker', callback=None)
def test_pool_restart_reload_modules(self):
consumer = Consumer(self.app)
consumer.controller = _WC(app=self.app)
consumer.controller.consumer = consumer
consumer.controller.pool.restart = Mock()
consumer.reset_rate_limits = Mock(name='reset_rate_limits()')
consumer.update_strategies = Mock(name='update_strategies()')
panel = self.create_panel(consumer=consumer)
panel.app = self.app
_import = panel.app.loader.import_from_cwd = Mock()
_reload = Mock()
self.app.conf.worker_pool_restarts = True
with patch.dict(sys.modules, {'foo': None}):
panel.handle('pool_restart', {'modules': ['foo'],
'reload': False,
'reloader': _reload})
consumer.controller.pool.restart.assert_called()
_reload.assert_not_called()
_import.assert_not_called()
_import.reset_mock()
_reload.reset_mock()
consumer.controller.pool.restart.reset_mock()
panel.handle('pool_restart', {'modules': ['foo'],
'reload': True,
'reloader': _reload})
consumer.controller.pool.restart.assert_called()
_reload.assert_called()
_import.assert_not_called()
def test_pool_restart_reload_modules(self):
consumer = Consumer(self.app)
consumer.controller = _WC(app=self.app)
consumer.controller.consumer = consumer
consumer.controller.pool.restart = Mock()
consumer.reset_rate_limits = Mock(name='reset_rate_limits()')
consumer.update_strategies = Mock(name='update_strategies()')
panel = self.create_panel(consumer=consumer)
panel.app = self.app
_import = panel.app.loader.import_from_cwd = Mock()
_reload = Mock()
self.app.conf.worker_pool_restarts = True
with patch.dict(sys.modules, {'foo': None}):
panel.handle('pool_restart', {'modules': ['foo'],
'reload': False,
'reloader': _reload})
consumer.controller.pool.restart.assert_called()
_reload.assert_not_called()
_import.assert_not_called()
_import.reset_mock()
_reload.reset_mock()
consumer.controller.pool.restart.reset_mock()
panel.handle('pool_restart', {'modules': ['foo'],
'reload': True,
'reloader': _reload})
consumer.controller.pool.restart.assert_called()
_reload.assert_called()
_import.assert_not_called()
def test_apply_timeout(self):
class Timeout(Exception):
value = None
def __init__(self, value):
self.__class__.value = value
def __enter__(self):
return self
def __exit__(self, *exc_info):
pass
timeout_callback = Mock(name='timeout_callback')
apply_target = Mock(name='apply_target')
apply_timeout(
Mock(), timeout=10, callback=Mock(name='callback'),
timeout_callback=timeout_callback,
apply_target=apply_target, Timeout=Timeout,
)
self.assertEqual(Timeout.value, 10)
apply_target.assert_called()
apply_target.side_effect = Timeout(10)
apply_timeout(
Mock(), timeout=10, callback=Mock(),
timeout_callback=timeout_callback,
apply_target=apply_target, Timeout=Timeout,
)
timeout_callback.assert_called_with(False, 10)
def test_shutdown_nodes(self, slepp, gethostname, Pidfile):
gethostname.return_value = 'e.com'
self.prepare_pidfile_for_getpids(Pidfile)
self.assertIsNone(self.t.shutdown_nodes([]))
self.t.signal_node = Mock()
node_alive = self.t.node_alive = Mock()
self.t.node_alive.return_value = False
callback = Mock()
self.t.stop(['foo', 'bar', 'baz'], 'celery worker', callback=callback)
sigs = sorted(self.t.signal_node.call_args_list)
self.assertEqual(len(sigs), 2)
self.assertIn(
('*****@*****.**', 10, signal.SIGTERM),
{tup[0] for tup in sigs},
)
self.assertIn(
('*****@*****.**', 11, signal.SIGTERM),
{tup[0] for tup in sigs},
)
self.t.signal_node.return_value = False
callback.assert_called()
self.t.stop(['foo', 'bar', 'baz'], 'celery worker', callback=None)
def on_node_alive(pid):
if node_alive.call_count > 4:
return True
return False
self.t.signal_node.return_value = True
self.t.node_alive.side_effect = on_node_alive
self.t.stop(['foo', 'bar', 'baz'], 'celery worker', retry=True)
def test_getpids(self, gethostname, Pidfile):
gethostname.return_value = 'e.com'
self.prepare_pidfile_for_getpids(Pidfile)
callback = Mock()
p = NamespacedOptionParser(['foo', 'bar', 'baz'])
nodes = self.t.getpids(p, 'celery worker', callback=callback)
node_0, node_1 = nodes
self.assertEqual(node_0[0], '*****@*****.**')
self.assertEqual(
sorted(node_0[1]),
sorted(('celery worker', '--pidfile=foo.pid',
'-n [email protected]', '')),
)
self.assertEqual(node_0[2], 10)
self.assertEqual(node_1[0], '*****@*****.**')
self.assertEqual(
sorted(node_1[1]),
sorted(('celery worker', '--pidfile=bar.pid',
'-n [email protected]', '')),
)
self.assertEqual(node_1[2], 11)
callback.assert_called()
cargs, _ = callback.call_args
self.assertEqual(cargs[0], '*****@*****.**')
self.assertItemsEqual(
cargs[1],
['celery worker', '--pidfile=baz.pid', '-n [email protected]', ''],
)
self.assertIsNone(cargs[2])
self.assertIn('DOWN', self.fh.getvalue())
# without callback, should work
nodes = self.t.getpids(p, 'celery worker', callback=None)
def test_with_success_receivers(self):
on_success = Mock()
signals.task_success.connect(on_success)
try:
self.trace(self.add, (2, 2), {})
on_success.assert_called()
finally:
signals.task_success.receivers[:] = []
def test_with_postrun_receivers(self):
on_postrun = Mock()
signals.task_postrun.connect(on_postrun)
try:
self.trace(self.add, (2, 2), {})
on_postrun.assert_called()
finally:
signals.task_postrun.receivers[:] = []
def test_callback(self):
cb = Mock()
with self.move_context(callback=cb) as (callback, pred, republish):
pred.return_value = 'foo'
body, message = self.msgpair()
callback(body, message)
republish.assert_called()
cb.assert_called()
def test_with_postrun_receivers(self):
on_postrun = Mock()
signals.task_postrun.connect(on_postrun)
try:
self.trace(self.add, (2, 2), {})
on_postrun.assert_called()
finally:
signals.task_postrun.receivers[:] = []
def test_callback(self):
cb = Mock()
with self.move_context(callback=cb) as (callback, pred, republish):
pred.return_value = 'foo'
body, message = self.msgpair()
callback(body, message)
republish.assert_called()
cb.assert_called()
def test_with_success_receivers(self):
on_success = Mock()
signals.task_success.connect(on_success)
try:
self.trace(self.add, (2, 2), {})
on_success.assert_called()
finally:
signals.task_success.receivers[:] = []
def test_start(self):
with patch('celery.contrib.migrate.eventloop') as evloop:
app = Mock()
filt = Mock(name='filter')
conn = Connection('memory://')
evloop.side_effect = StopFiltering()
app.amqp.queues = {'foo': Queue('foo'), 'bar': Queue('bar')}
consumer = app.amqp.TaskConsumer.return_value = Mock(name='consum')
consumer.queues = list(app.amqp.queues.values())
consumer.channel = conn.default_channel
consumer.__enter__ = Mock(name='consumer.__enter__')
consumer.__exit__ = Mock(name='consumer.__exit__')
consumer.callbacks = []
def register_callback(x):
consumer.callbacks.append(x)
consumer.register_callback = register_callback
start_filter(app, conn, filt, queues='foo,bar', ack_messages=True)
body = {'task': 'add', 'id': 'id'}
for callback in consumer.callbacks:
callback(body, Message(body))
consumer.callbacks[:] = []
cb = Mock(name='callback=')
start_filter(app, conn, filt, tasks='add,mul', callback=cb)
for callback in consumer.callbacks:
callback(body, Message(body))
cb.assert_called()
on_declare_queue = Mock()
start_filter(app,
conn,
filt,
tasks='add,mul',
queues='foo',
on_declare_queue=on_declare_queue)
on_declare_queue.assert_called()
start_filter(app, conn, filt, queues=['foo', 'bar'])
consumer.callbacks[:] = []
state = State()
start_filter(app,
conn,
filt,
tasks='add,mul',
callback=cb,
state=state,
limit=1)
stop_filtering_raised = False
for callback in consumer.callbacks:
try:
callback(body, Message(body))
except StopFiltering:
stop_filtering_raised = True
self.assertTrue(state.count)
self.assertTrue(stop_filtering_raised)
def test_migrate(self, name='testcelery'):
x = Connection('memory://foo')
y = Connection('memory://foo')
# use separate state
x.default_channel.queues = {}
y.default_channel.queues = {}
ex = Exchange(name, 'direct')
q = Queue(name, exchange=ex, routing_key=name)
q(x.default_channel).declare()
Producer(x).publish('foo', exchange=name, routing_key=name)
Producer(x).publish('bar', exchange=name, routing_key=name)
Producer(x).publish('baz', exchange=name, routing_key=name)
self.assertTrue(x.default_channel.queues)
self.assertFalse(y.default_channel.queues)
migrate_tasks(x, y, accept=['text/plain'], app=self.app)
yq = q(y.default_channel)
self.assertEqual(yq.get().body, ensure_bytes('foo'))
self.assertEqual(yq.get().body, ensure_bytes('bar'))
self.assertEqual(yq.get().body, ensure_bytes('baz'))
Producer(x).publish('foo', exchange=name, routing_key=name)
callback = Mock()
migrate_tasks(x, y,
callback=callback, accept=['text/plain'], app=self.app)
callback.assert_called()
migrate = Mock()
Producer(x).publish('baz', exchange=name, routing_key=name)
migrate_tasks(x, y, callback=callback,
migrate=migrate, accept=['text/plain'], app=self.app)
migrate.assert_called()
with patch('kombu.transport.virtual.Channel.queue_declare') as qd:
def effect(*args, **kwargs):
if kwargs.get('passive'):
raise ChannelError('some channel error')
return 0, 3, 0
qd.side_effect = effect
migrate_tasks(x, y, app=self.app)
x = Connection('memory://')
x.default_channel.queues = {}
y.default_channel.queues = {}
callback = Mock()
migrate_tasks(x, y,
callback=callback, accept=['text/plain'], app=self.app)
callback.assert_not_called()
def test_start(self):
with patch('celery.contrib.migrate.eventloop') as evloop:
app = Mock()
filt = Mock(name='filter')
conn = Connection('memory://')
evloop.side_effect = StopFiltering()
app.amqp.queues = {'foo': Queue('foo'), 'bar': Queue('bar')}
consumer = app.amqp.TaskConsumer.return_value = Mock(name='consum')
consumer.queues = list(app.amqp.queues.values())
consumer.channel = conn.default_channel
consumer.__enter__ = Mock(name='consumer.__enter__')
consumer.__exit__ = Mock(name='consumer.__exit__')
consumer.callbacks = []
def register_callback(x):
consumer.callbacks.append(x)
consumer.register_callback = register_callback
start_filter(app, conn, filt,
queues='foo,bar', ack_messages=True)
body = {'task': 'add', 'id': 'id'}
for callback in consumer.callbacks:
callback(body, Message(body))
consumer.callbacks[:] = []
cb = Mock(name='callback=')
start_filter(app, conn, filt, tasks='add,mul', callback=cb)
for callback in consumer.callbacks:
callback(body, Message(body))
cb.assert_called()
on_declare_queue = Mock()
start_filter(app, conn, filt, tasks='add,mul', queues='foo',
on_declare_queue=on_declare_queue)
on_declare_queue.assert_called()
start_filter(app, conn, filt, queues=['foo', 'bar'])
consumer.callbacks[:] = []
state = State()
start_filter(app, conn, filt,
tasks='add,mul', callback=cb, state=state, limit=1)
stop_filtering_raised = False
for callback in consumer.callbacks:
try:
callback(body, Message(body))
except StopFiltering:
stop_filtering_raised = True
self.assertTrue(state.count)
self.assertTrue(stop_filtering_raised)
def test_process_initializer(self, set_mp_process_title, _signals):
with mock.restore_logging():
from celery import signals
from celery._state import _tls
from celery.concurrency.prefork import (
process_initializer,
WORKER_SIGRESET,
WORKER_SIGIGNORE,
)
on_worker_process_init = Mock()
signals.worker_process_init.connect(on_worker_process_init)
def Loader(*args, **kwargs):
loader = Mock(*args, **kwargs)
loader.conf = {}
loader.override_backends = {}
return loader
with self.Celery(loader=Loader) as app:
app.conf = AttributeDict(DEFAULTS)
process_initializer(app, 'awesome.worker.com')
_signals.ignore.assert_any_call(*WORKER_SIGIGNORE)
_signals.reset.assert_any_call(*WORKER_SIGRESET)
self.assertTrue(app.loader.init_worker.call_count)
on_worker_process_init.assert_called()
self.assertIs(_tls.current_app, app)
set_mp_process_title.assert_called_with(
'celeryd',
hostname='awesome.worker.com',
)
with patch('celery.app.trace.setup_worker_optimizations') as S:
os.environ['FORKED_BY_MULTIPROCESSING'] = '1'
try:
process_initializer(app, 'luke.worker.com')
S.assert_called_with(app, 'luke.worker.com')
finally:
os.environ.pop('FORKED_BY_MULTIPROCESSING', None)
os.environ['CELERY_LOG_FILE'] = 'worker%I.log'
app.log.setup = Mock(name='log_setup')
try:
process_initializer(app, 'luke.worker.com')
finally:
os.environ.pop('CELERY_LOG_FILE', None)
def test_process_initializer(self, set_mp_process_title, _signals):
with mock.restore_logging():
from celery import signals
from celery._state import _tls
from celery.concurrency.prefork import (
process_initializer, WORKER_SIGRESET, WORKER_SIGIGNORE,
)
on_worker_process_init = Mock()
signals.worker_process_init.connect(on_worker_process_init)
def Loader(*args, **kwargs):
loader = Mock(*args, **kwargs)
loader.conf = {}
loader.override_backends = {}
return loader
with self.Celery(loader=Loader) as app:
app.conf = AttributeDict(DEFAULTS)
process_initializer(app, 'awesome.worker.com')
_signals.ignore.assert_any_call(*WORKER_SIGIGNORE)
_signals.reset.assert_any_call(*WORKER_SIGRESET)
self.assertTrue(app.loader.init_worker.call_count)
on_worker_process_init.assert_called()
self.assertIs(_tls.current_app, app)
set_mp_process_title.assert_called_with(
'celeryd', hostname='awesome.worker.com',
)
with patch('celery.app.trace.setup_worker_optimizations') as S:
os.environ['FORKED_BY_MULTIPROCESSING'] = '1'
try:
process_initializer(app, 'luke.worker.com')
S.assert_called_with(app, 'luke.worker.com')
finally:
os.environ.pop('FORKED_BY_MULTIPROCESSING', None)
os.environ['CELERY_LOG_FILE'] = 'worker%I.log'
app.log.setup = Mock(name='log_setup')
try:
process_initializer(app, 'luke.worker.com')
finally:
os.environ.pop('CELERY_LOG_FILE', None)
def test_apply_timeout(self):
class Timeout(Exception):
value = None
def __init__(self, value):
self.__class__.value = value
def __enter__(self):
return self
def __exit__(self, *exc_info):
pass
timeout_callback = Mock(name='timeout_callback')
apply_target = Mock(name='apply_target')
apply_timeout(
Mock(),
timeout=10,
callback=Mock(name='callback'),
timeout_callback=timeout_callback,
apply_target=apply_target,
Timeout=Timeout,
)
self.assertEqual(Timeout.value, 10)
apply_target.assert_called()
apply_target.side_effect = Timeout(10)
apply_timeout(
Mock(),
timeout=10,
callback=Mock(),
timeout_callback=timeout_callback,
apply_target=apply_target,
Timeout=Timeout,
)
timeout_callback.assert_called_with(False, 10)
def test_apply_target__raises_BaseException(self):
target = Mock(name='target')
callback = Mock(name='callback')
target.side_effect = BaseException()
apply_target(target, callback=callback)
callback.assert_called()
def test_reload_from_cwd_custom_reloader(self):
reload = Mock()
reload_from_cwd('foo', reload)
reload.assert_called()
def test_import_from_cwd_custom_imp(self):
imp = Mock(name='imp')
self.loader.import_from_cwd('foo', imp=imp)
imp.assert_called()
def test_import_from_cwd_custom_imp(self):
imp = Mock(name='imp')
self.loader.import_from_cwd('foo', imp=imp)
imp.assert_called()
def test_import_from_cwd_custom_imp(self):
imp = Mock(name="imp")
self.loader.import_from_cwd("foo", imp=imp)
imp.assert_called()
def test_migrate(self, name='testcelery'):
x = Connection('memory://foo')
y = Connection('memory://foo')
# use separate state
x.default_channel.queues = {}
y.default_channel.queues = {}
ex = Exchange(name, 'direct')
q = Queue(name, exchange=ex, routing_key=name)
q(x.default_channel).declare()
Producer(x).publish('foo', exchange=name, routing_key=name)
Producer(x).publish('bar', exchange=name, routing_key=name)
Producer(x).publish('baz', exchange=name, routing_key=name)
self.assertTrue(x.default_channel.queues)
self.assertFalse(y.default_channel.queues)
migrate_tasks(x, y, accept=['text/plain'], app=self.app)
yq = q(y.default_channel)
self.assertEqual(yq.get().body, ensure_bytes('foo'))
self.assertEqual(yq.get().body, ensure_bytes('bar'))
self.assertEqual(yq.get().body, ensure_bytes('baz'))
Producer(x).publish('foo', exchange=name, routing_key=name)
callback = Mock()
migrate_tasks(x,
y,
callback=callback,
accept=['text/plain'],
app=self.app)
callback.assert_called()
migrate = Mock()
Producer(x).publish('baz', exchange=name, routing_key=name)
migrate_tasks(x,
y,
callback=callback,
migrate=migrate,
accept=['text/plain'],
app=self.app)
migrate.assert_called()
with patch('kombu.transport.virtual.Channel.queue_declare') as qd:
def effect(*args, **kwargs):
if kwargs.get('passive'):
raise ChannelError('some channel error')
return 0, 3, 0
qd.side_effect = effect
migrate_tasks(x, y, app=self.app)
x = Connection('memory://')
x.default_channel.queues = {}
y.default_channel.queues = {}
callback = Mock()
migrate_tasks(x,
y,
callback=callback,
accept=['text/plain'],
app=self.app)
callback.assert_not_called()
def test_apply_target__raises_BaseException(self):
target = Mock(name="target")
callback = Mock(name="callback")
target.side_effect = BaseException()
apply_target(target, callback=callback)
callback.assert_called()
