class test_frame_handler:
@pytest.fixture(autouse=True)
def setup_conn(self):
self.conn = Mock(name='connection')
self.conn.bytes_recv = 0
self.callback = Mock(name='callback')
self.g = frame_handler(self.conn, self.callback)
def test_header(self):
buf = pack('>HH', 60, 51)
assert self.g((1, 1, buf))
self.callback.assert_called_with(1, (60, 51), buf, None)
assert self.conn.bytes_recv
def test_header_message_empty_body(self):
assert not self.g((1, 1, pack('>HH', *spec.Basic.Deliver)))
self.callback.assert_not_called()
with pytest.raises(UnexpectedFrame):
self.g((1, 1, pack('>HH', *spec.Basic.Deliver)))
m = Message()
m.properties = {}
buf = pack('>HxxQ', m.CLASS_ID, 0)
buf += m._serialize_properties()
assert self.g((2, 1, buf))
self.callback.assert_called()
msg = self.callback.call_args[0][3]
self.callback.assert_called_with(
1, msg.frame_method, msg.frame_args, msg,
)
def test_header_message_content(self):
assert not self.g((1, 1, pack('>HH', *spec.Basic.Deliver)))
self.callback.assert_not_called()
m = Message()
m.properties = {}
buf = pack('>HxxQ', m.CLASS_ID, 16)
buf += m._serialize_properties()
assert not self.g((2, 1, buf))
self.callback.assert_not_called()
assert not self.g((3, 1, b'thequick'))
self.callback.assert_not_called()
assert self.g((3, 1, b'brownfox'))
self.callback.assert_called()
msg = self.callback.call_args[0][3]
self.callback.assert_called_with(
1, msg.frame_method, msg.frame_args, msg,
)
assert msg.body == b'thequickbrownfox'
def test_heartbeat_frame(self):
assert not self.g((8, 1, ''))
self.callback.assert_not_called()
assert self.conn.bytes_recv
def test_pool_restart_reload_modules(self):
consumer = Consumer(self.app)
consumer.controller = _WC(app=self.app)
consumer.controller.consumer = consumer
consumer.controller.pool.restart = Mock()
consumer.reset_rate_limits = Mock(name="reset_rate_limits()")
consumer.update_strategies = Mock(name="update_strategies()")
panel = self.create_panel(consumer=consumer)
panel.app = self.app
_import = panel.app.loader.import_from_cwd = Mock()
_reload = Mock()
self.app.conf.worker_pool_restarts = True
with patch.dict(sys.modules, {"foo": None}):
panel.handle("pool_restart", {"modules": ["foo"], "reload": False, "reloader": _reload})
consumer.controller.pool.restart.assert_called()
_reload.assert_not_called()
_import.assert_not_called()
_import.reset_mock()
_reload.reset_mock()
consumer.controller.pool.restart.reset_mock()
panel.handle("pool_restart", {"modules": ["foo"], "reload": True, "reloader": _reload})
consumer.controller.pool.restart.assert_called()
_reload.assert_called()
_import.assert_not_called()
def test_apply_timeout(self):
self.patching.modules(*gevent_modules)
class Timeout(Exception):
value = None
def __init__(self, value):
self.__class__.value = value
def __enter__(self):
return self
def __exit__(self, *exc_info):
pass
timeout_callback = Mock(name='timeout_callback')
apply_target = Mock(name='apply_target')
apply_timeout(
Mock(), timeout=10, callback=Mock(name='callback'),
timeout_callback=timeout_callback,
apply_target=apply_target, Timeout=Timeout,
)
assert Timeout.value == 10
apply_target.assert_called()
apply_target.side_effect = Timeout(10)
apply_timeout(
Mock(), timeout=10, callback=Mock(),
timeout_callback=timeout_callback,
apply_target=apply_target, Timeout=Timeout,
)
timeout_callback.assert_called_with(False, 10)
def test_with_prerun_receivers(self):
on_prerun = Mock()
signals.task_prerun.connect(on_prerun)
try:
self.trace(self.add, (2, 2), {})
on_prerun.assert_called()
finally:
signals.task_prerun.receivers[:] = []
def test_with_success_receivers(self):
on_success = Mock()
signals.task_success.connect(on_success)
try:
self.trace(self.add, (2, 2), {})
on_success.assert_called()
finally:
signals.task_success.receivers[:] = []
def test_callback(self):
cb = Mock()
with self.move_context(callback=cb) as (callback, pred, republish):
pred.return_value = "foo"
body, message = self.msgpair()
callback(body, message)
republish.assert_called()
cb.assert_called()
def test_reraise_error(self):
m = Message('body', channel=Mock())
callback = Mock(name='callback')
try:
raise KeyError('foo')
except KeyError:
m.errors.append(sys.exc_info())
m._reraise_error(callback)
callback.assert_called()
with pytest.raises(KeyError):
m._reraise_error(None)
def test_migrate(self, app, name='testcelery'):
connection_kwargs = {
'transport_options': {'polling_interval': 0.01}
}
x = Connection('memory://foo', **connection_kwargs)
y = Connection('memory://foo', **connection_kwargs)
# use separate state
x.default_channel.queues = {}
y.default_channel.queues = {}
ex = Exchange(name, 'direct')
q = Queue(name, exchange=ex, routing_key=name)
q(x.default_channel).declare()
Producer(x).publish('foo', exchange=name, routing_key=name)
Producer(x).publish('bar', exchange=name, routing_key=name)
Producer(x).publish('baz', exchange=name, routing_key=name)
assert x.default_channel.queues
assert not y.default_channel.queues
migrate_tasks(x, y, accept=['text/plain'], app=app)
yq = q(y.default_channel)
assert yq.get().body == ensure_bytes('foo')
assert yq.get().body == ensure_bytes('bar')
assert yq.get().body == ensure_bytes('baz')
Producer(x).publish('foo', exchange=name, routing_key=name)
callback = Mock()
migrate_tasks(x, y,
callback=callback, accept=['text/plain'], app=app)
callback.assert_called()
migrate = Mock()
Producer(x).publish('baz', exchange=name, routing_key=name)
migrate_tasks(x, y, callback=callback,
migrate=migrate, accept=['text/plain'], app=app)
migrate.assert_called()
with patch('kombu.transport.virtual.Channel.queue_declare') as qd:
def effect(*args, **kwargs):
if kwargs.get('passive'):
raise ChannelError('some channel error')
return 0, 3, 0
qd.side_effect = effect
migrate_tasks(x, y, app=app)
x = Connection('memory://', **connection_kwargs)
x.default_channel.queues = {}
y.default_channel.queues = {}
callback = Mock()
migrate_tasks(x, y,
callback=callback, accept=['text/plain'], app=app)
callback.assert_not_called()
def test_autoretry(self):
myfun = Mock()
self.conn.transport.connection_errors = (KeyError,)
def on_call(*args, **kwargs):
myfun.side_effect = None
raise KeyError('foo')
myfun.side_effect = on_call
insured = self.conn.autoretry(myfun)
insured()
myfun.assert_called()
def test_migrate(self, app, name="testcelery"):
x = Connection("memory://foo")
y = Connection("memory://foo")
# use separate state
x.default_channel.queues = {}
y.default_channel.queues = {}
ex = Exchange(name, "direct")
q = Queue(name, exchange=ex, routing_key=name)
q(x.default_channel).declare()
Producer(x).publish("foo", exchange=name, routing_key=name)
Producer(x).publish("bar", exchange=name, routing_key=name)
Producer(x).publish("baz", exchange=name, routing_key=name)
assert x.default_channel.queues
assert not y.default_channel.queues
migrate_tasks(x, y, accept=["text/plain"], app=app)
yq = q(y.default_channel)
assert yq.get().body == ensure_bytes("foo")
assert yq.get().body == ensure_bytes("bar")
assert yq.get().body == ensure_bytes("baz")
Producer(x).publish("foo", exchange=name, routing_key=name)
callback = Mock()
migrate_tasks(x, y, callback=callback, accept=["text/plain"], app=app)
callback.assert_called()
migrate = Mock()
Producer(x).publish("baz", exchange=name, routing_key=name)
migrate_tasks(x, y, callback=callback, migrate=migrate, accept=["text/plain"], app=app)
migrate.assert_called()
with patch("kombu.transport.virtual.Channel.queue_declare") as qd:
def effect(*args, **kwargs):
if kwargs.get("passive"):
raise ChannelError("some channel error")
return 0, 3, 0
qd.side_effect = effect
migrate_tasks(x, y, app=app)
x = Connection("memory://")
x.default_channel.queues = {}
y.default_channel.queues = {}
callback = Mock()
migrate_tasks(x, y, callback=callback, accept=["text/plain"], app=app)
callback.assert_not_called()
def test_accept__content_allowed(self):
conn = Connection('memory://')
q = Queue('foo', exchange=self.exchange)
p = conn.Producer()
p.publish(
{'complex': object()},
declare=[q], exchange=self.exchange, serializer='pickle',
)
callback = Mock(name='callback')
with conn.Consumer(queues=[q], accept=['pickle'],
callbacks=[callback]):
conn.drain_events(timeout=1)
callback.assert_called()
body, message = callback.call_args[0]
assert body['complex']
def test_start(self):
with patch('celery.contrib.migrate.eventloop') as evloop:
app = Mock()
filt = Mock(name='filter')
conn = Connection('memory://')
evloop.side_effect = StopFiltering()
app.amqp.queues = {'foo': Queue('foo'), 'bar': Queue('bar')}
consumer = app.amqp.TaskConsumer.return_value = Mock(name='consum')
consumer.queues = list(app.amqp.queues.values())
consumer.channel = conn.default_channel
consumer.__enter__ = Mock(name='consumer.__enter__')
consumer.__exit__ = Mock(name='consumer.__exit__')
consumer.callbacks = []
def register_callback(x):
consumer.callbacks.append(x)
consumer.register_callback = register_callback
start_filter(app, conn, filt,
queues='foo,bar', ack_messages=True)
body = {'task': 'add', 'id': 'id'}
for callback in consumer.callbacks:
callback(body, Message(body))
consumer.callbacks[:] = []
cb = Mock(name='callback=')
start_filter(app, conn, filt, tasks='add,mul', callback=cb)
for callback in consumer.callbacks:
callback(body, Message(body))
cb.assert_called()
on_declare_queue = Mock()
start_filter(app, conn, filt, tasks='add,mul', queues='foo',
on_declare_queue=on_declare_queue)
on_declare_queue.assert_called()
start_filter(app, conn, filt, queues=['foo', 'bar'])
consumer.callbacks[:] = []
state = State()
start_filter(app, conn, filt,
tasks='add,mul', callback=cb, state=state, limit=1)
stop_filtering_raised = False
for callback in consumer.callbacks:
try:
callback(body, Message(body))
except StopFiltering:
stop_filtering_raised = True
assert state.count
assert stop_filtering_raised
def test_start(self):
with patch("celery.contrib.migrate.eventloop") as evloop:
app = Mock()
filt = Mock(name="filter")
conn = Connection("memory://")
evloop.side_effect = StopFiltering()
app.amqp.queues = {"foo": Queue("foo"), "bar": Queue("bar")}
consumer = app.amqp.TaskConsumer.return_value = Mock(name="consum")
consumer.queues = list(app.amqp.queues.values())
consumer.channel = conn.default_channel
consumer.__enter__ = Mock(name="consumer.__enter__")
consumer.__exit__ = Mock(name="consumer.__exit__")
consumer.callbacks = []
def register_callback(x):
consumer.callbacks.append(x)
consumer.register_callback = register_callback
start_filter(app, conn, filt, queues="foo,bar", ack_messages=True)
body = {"task": "add", "id": "id"}
for callback in consumer.callbacks:
callback(body, Message(body))
consumer.callbacks[:] = []
cb = Mock(name="callback=")
start_filter(app, conn, filt, tasks="add,mul", callback=cb)
for callback in consumer.callbacks:
callback(body, Message(body))
cb.assert_called()
on_declare_queue = Mock()
start_filter(app, conn, filt, tasks="add,mul", queues="foo", on_declare_queue=on_declare_queue)
on_declare_queue.assert_called()
start_filter(app, conn, filt, queues=["foo", "bar"])
consumer.callbacks[:] = []
state = State()
start_filter(app, conn, filt, tasks="add,mul", callback=cb, state=state, limit=1)
stop_filtering_raised = False
for callback in consumer.callbacks:
try:
callback(body, Message(body))
except StopFiltering:
stop_filtering_raised = True
assert state.count
assert stop_filtering_raised
def test_process_initializer(self, set_mp_process_title, _signals):
with mock.restore_logging():
from celery import signals
from celery._state import _tls
from celery.concurrency.prefork import (
process_initializer, WORKER_SIGRESET, WORKER_SIGIGNORE,
)
on_worker_process_init = Mock()
signals.worker_process_init.connect(on_worker_process_init)
def Loader(*args, **kwargs):
loader = Mock(*args, **kwargs)
loader.conf = {}
loader.override_backends = {}
return loader
with self.Celery(loader=Loader) as app:
app.conf = AttributeDict(DEFAULTS)
process_initializer(app, 'awesome.worker.com')
_signals.ignore.assert_any_call(*WORKER_SIGIGNORE)
_signals.reset.assert_any_call(*WORKER_SIGRESET)
assert app.loader.init_worker.call_count
on_worker_process_init.assert_called()
assert _tls.current_app is app
set_mp_process_title.assert_called_with(
'celeryd', hostname='awesome.worker.com',
)
with patch('celery.app.trace.setup_worker_optimizations') as S:
os.environ['FORKED_BY_MULTIPROCESSING'] = '1'
try:
process_initializer(app, 'luke.worker.com')
S.assert_called_with(app, 'luke.worker.com')
finally:
os.environ.pop('FORKED_BY_MULTIPROCESSING', None)
os.environ['CELERY_LOG_FILE'] = 'worker%I.log'
app.log.setup = Mock(name='log_setup')
try:
process_initializer(app, 'luke.worker.com')
finally:
os.environ.pop('CELERY_LOG_FILE', None)
def test_custom_request_gets_instantiated(self):
_MyRequest = Mock(name='MyRequest')
class MyRequest(Request):
def __init__(self, *args, **kwargs):
Request.__init__(self, *args, **kwargs)
_MyRequest()
class MyTask(Task):
Request = MyRequest
@self.app.task(base=MyTask)
def failed():
raise AssertionError
sig = failed.s()
with self._context(sig) as C:
task_message_handler = default_strategy(
failed,
self.app,
C.consumer
)
task_message_handler(C.message, None, None, None, None)
_MyRequest.assert_called()
def test_pool_restart_reload_modules(self):
consumer = Consumer(self.app)
consumer.controller = _WC(app=self.app)
consumer.controller.consumer = consumer
consumer.controller.pool.restart = Mock()
consumer.reset_rate_limits = Mock(name='reset_rate_limits()')
consumer.update_strategies = Mock(name='update_strategies()')
panel = self.create_panel(consumer=consumer)
panel.app = self.app
_import = panel.app.loader.import_from_cwd = Mock()
_reload = Mock()
self.app.conf.worker_pool_restarts = True
with patch.dict(sys.modules, {'foo': None}):
panel.handle('pool_restart', {
'modules': ['foo'],
'reload': False,
'reloader': _reload,
})
consumer.controller.pool.restart.assert_called()
_reload.assert_not_called()
_import.assert_not_called()
_import.reset_mock()
_reload.reset_mock()
consumer.controller.pool.restart.reset_mock()
panel.handle('pool_restart', {
'modules': ['foo'],
'reload': True,
'reloader': _reload,
})
consumer.controller.pool.restart.assert_called()
_reload.assert_called()
_import.assert_not_called()
def test_ensure_connection_on_error(self):
c = Connection('amqp://A;amqp://B')
with patch('kombu.connection.retry_over_time') as rot:
c.ensure_connection()
rot.assert_called()
args = rot.call_args[0]
cb = args[4]
intervals = iter([1, 2, 3, 4, 5])
assert cb(KeyError(), intervals, 0) == 0
assert cb(KeyError(), intervals, 1) == 1
assert cb(KeyError(), intervals, 2) == 0
assert cb(KeyError(), intervals, 3) == 2
assert cb(KeyError(), intervals, 4) == 0
assert cb(KeyError(), intervals, 5) == 3
assert cb(KeyError(), intervals, 6) == 0
assert cb(KeyError(), intervals, 7) == 4
errback = Mock()
c.ensure_connection(errback=errback)
args = rot.call_args[0]
cb = args[4]
assert cb(KeyError(), intervals, 0) == 0
errback.assert_called()
def test_reload_from_cwd_custom_reloader():
reload = Mock()
reload_from_cwd('foo', reload)
reload.assert_called()
def test_apply_target__raises_BaseException(self):
target = Mock(name='target')
callback = Mock(name='callback')
target.side_effect = BaseException()
apply_target(target, callback=callback)
callback.assert_called()
def test_import_from_cwd_custom_imp(self):
imp = Mock(name='imp')
self.loader.import_from_cwd('foo', imp=imp)
imp.assert_called()