def test_worker_term_hard_handler_only_stop_MainProcess(self):
process = current_process()
name, process.name = process.name, 'OtherProcess'
try:
with patch('celery.apps.worker.active_thread_count') as c:
c.return_value = 3
worker = self._Worker()
handlers = self.psig(
cd.install_worker_term_hard_handler, worker)
try:
handlers['SIGQUIT']('SIGQUIT', object())
assert state.should_terminate
finally:
state.should_terminate = None
with patch('celery.apps.worker.active_thread_count') as c:
c.return_value = 1
worker = self._Worker()
handlers = self.psig(
cd.install_worker_term_hard_handler, worker)
try:
with pytest.raises(WorkerTerminate):
handlers['SIGQUIT']('SIGQUIT', object())
finally:
state.should_terminate = None
finally:
process.name = name
def test_arguments(self):
with patch('kombu.common.uuid',
return_value='test') as uuid_mock:
q = Broadcast(name='test_Broadcast')
uuid_mock.assert_called_with()
assert q.name == 'bcast.test'
assert q.alias == 'test_Broadcast'
assert q.auto_delete
assert q.exchange.name == 'test_Broadcast'
assert q.exchange.type == 'fanout'
q = Broadcast('test_Broadcast', 'explicit_queue_name')
assert q.name == 'explicit_queue_name'
assert q.exchange.name == 'test_Broadcast'
q2 = q(Mock())
assert q2.name == q.name
with patch('kombu.common.uuid',
return_value='test') as uuid_mock:
q = Broadcast('test_Broadcast',
'explicit_queue_name',
unique=True)
uuid_mock.assert_called_with()
assert q.name == 'explicit_queue_name.test'
q2 = q(Mock())
assert q2.name.split('.')[0] == q.name.split('.')[0]
def test_load(self):
with patch('pkg_resources.iter_entry_points') as iterep:
with patch('celery.utils.imports.symbol_by_name') as symbyname:
ep = Mock()
ep.name = 'ep'
ep.module_name = 'foo'
ep.attrs = ['bar', 'baz']
iterep.return_value = [ep]
cls = symbyname.return_value = Mock()
register = Mock()
e = Extensions('unit', register)
e.load()
symbyname.assert_called_with('foo:bar')
register.assert_called_with(cls, name='ep')
with patch('celery.utils.imports.symbol_by_name') as symbyname:
symbyname.side_effect = SyntaxError()
with patch('warnings.warn') as warn:
e.load()
warn.assert_called()
with patch('celery.utils.imports.symbol_by_name') as symbyname:
symbyname.side_effect = KeyError('foo')
with pytest.raises(KeyError):
e.load()
def test_iterate_respects_subpolling_interval(self):
r1 = self.app.AsyncResult(uuid())
r2 = self.app.AsyncResult(uuid())
backend = r1.backend = r2.backend = Mock()
backend.subpolling_interval = 10
ready = r1.ready = r2.ready = Mock()
def se(*args, **kwargs):
ready.side_effect = KeyError()
return False
ready.return_value = False
ready.side_effect = se
x = self.app.ResultSet([r1, r2])
with self.dummy_copy():
with patch('celery.result.time') as _time:
with pytest.warns(CPendingDeprecationWarning):
with pytest.raises(KeyError):
list(x.iterate())
_time.sleep.assert_called_with(10)
backend.subpolling_interval = 0
with patch('celery.result.time') as _time:
with pytest.warns(CPendingDeprecationWarning):
with pytest.raises(KeyError):
ready.return_value = False
ready.side_effect = se
list(x.iterate())
_time.sleep.assert_not_called()
def test_fire_timers_raises(self):
eback = Mock()
eback.side_effect = KeyError('foo')
self.hub.timer = Mock()
self.hub.scheduler = iter([(0, eback)])
with pytest.raises(KeyError):
self.hub.fire_timers(propagate=(KeyError,))
eback.side_effect = ValueError('foo')
self.hub.scheduler = iter([(0, eback)])
with patch('kombu.asynchronous.hub.logger') as logger:
with pytest.raises(StopIteration):
self.hub.fire_timers()
logger.error.assert_called()
eback.side_effect = MemoryError('foo')
self.hub.scheduler = iter([(0, eback)])
with pytest.raises(MemoryError):
self.hub.fire_timers()
eback.side_effect = OSError()
eback.side_effect.errno = errno.ENOMEM
self.hub.scheduler = iter([(0, eback)])
with pytest.raises(OSError):
self.hub.fire_timers()
eback.side_effect = OSError()
eback.side_effect.errno = errno.ENOENT
self.hub.scheduler = iter([(0, eback)])
with patch('kombu.asynchronous.hub.logger') as logger:
with pytest.raises(StopIteration):
self.hub.fire_timers()
logger.error.assert_called()
def move_context(self, **kwargs):
with patch("celery.contrib.migrate.start_filter") as start:
with patch("celery.contrib.migrate.republish") as republish:
pred = Mock(name="predicate")
move(pred, app=self.app, connection=self.app.connection(), **kwargs)
start.assert_called()
callback = start.call_args[0][2]
yield callback, pred, republish
def test_connect_getaddrinfo_raises_gaierror_once_recovers(self):
with patch('socket.socket', return_value=MockSocket()), \
patch('socket.getaddrinfo',
side_effect=[
socket.gaierror,
[(socket.AF_INET6, 1, socket.IPPROTO_TCP,
'', ('::1', 5672))]
]):
self.t.connect()
def mock_sax_parse(self, parser):
with patch('kombu.async.aws.connection.sax_parse') as sax_parse:
with patch('kombu.async.aws.connection.XmlHandler') as xh:
def effect(body, h):
return parser(xh.call_args[0][0], body, h)
sax_parse.side_effect = effect
yield (sax_parse, xh)
sax_parse.assert_called()
def test_find_related_module(self):
with patch('importlib.import_module') as imp:
with patch('imp.find_module') as find:
imp.return_value = Mock()
imp.return_value.__path__ = 'foo'
base.find_related_module(base, 'tasks')
find.side_effect = ImportError()
base.find_related_module(base, 'tasks')
def test_connect_survives_not_implemented_set_cloexec(self):
with patch('socket.socket', return_value=MockSocket()), \
patch('socket.getaddrinfo',
return_value=[(socket.AF_INET, 1, socket.IPPROTO_TCP,
'', ('127.0.0.1', 5672))]):
with patch('amqp.transport.set_cloexec',
side_effect=NotImplementedError) as cloexec_mock:
self.t.connect()
assert cloexec_mock.called
def test_main__multi(self):
with patch('celery.__main__.maybe_patch_concurrency') as mpc:
with patch('celery.bin.celery.main') as main:
prev, sys.argv = sys.argv, ['foo', 'multi']
try:
__main__.main()
mpc.assert_not_called()
main.assert_called_with()
finally:
sys.argv = prev
def test_compat_sign():
with patch('itsdangerous.Signer') as Signer:
with patch('thorn.utils.hmac.get_digest') as get_digest:
digest = Mock(name='digest')
key = Mock(name='key')
message = Mock(name='message')
ret = hmac.compat_sign(digest, key, message)
get_digest.assert_called_with(digest)
Signer.assert_called_with(key, digest_method=get_digest())
Signer().get_signature.assert_called_with(message)
assert ret is Signer().get_signature()
def test_sign(hashlib, digest="sha1", key="KEY", msg="MSG"):
with patch('hmac.new') as hmac_new:
with patch('base64.b64encode') as b64encode:
ret = hmac.sign(digest, key, msg)
hmac_new.assert_called_with(
bytes_if_py3(key), bytes_if_py3(msg),
digestmod=hmac.get_digest(digest),
)
hmac_new().digest.assert_called_with()
b64encode.assert_called_with(hmac_new().digest())
assert ret is b64encode()
def assert_context(self, default, full, get_returns=None, set_effect=None):
with termsigs(default, full):
with patch('signal.getsignal') as GET:
with patch('signal.signal') as SET:
GET.return_value = get_returns
SET.side_effect = set_effect
reset_signals()
GET.assert_has_calls([
call(signo(sig)) for sig in default
])
yield GET, SET
def test_process_shutdown_on_worker_shutdown(self):
from celery.concurrency.prefork import process_destructor
from celery.concurrency.asynpool import Worker
with patch('celery.signals.worker_process_shutdown') as ws:
with patch('os._exit') as _exit:
worker = Worker(None, None, on_exit=process_destructor)
worker._do_exit(22, 3.1415926)
ws.send.assert_called_with(
sender=None, pid=22, exitcode=3.1415926,
)
_exit.assert_called_with(3.1415926)
def test_connect_socket_initialization_fails(self):
with patch('socket.socket', side_effect=socket.error), \
patch('socket.getaddrinfo',
return_value=[
(socket.AF_INET, 1, socket.IPPROTO_TCP,
'', ('127.0.0.1', 5672)),
(socket.AF_INET, 1, socket.IPPROTO_TCP,
'', ('127.0.0.2', 5672))
]) as getaddrinfo:
with pytest.raises(socket.error):
self.t.connect()
def test_enable_insecure_serializers(self):
with patch('kombu.serialization.registry') as registry:
enable_insecure_serializers()
registry.assert_has_calls([
call.enable('pickle'), call.enable('yaml'),
call.enable('msgpack'),
])
registry.enable.side_effect = KeyError()
enable_insecure_serializers()
with patch('kombu.serialization.registry') as registry:
enable_insecure_serializers(['msgpack'])
registry.assert_has_calls([call.enable('msgpack')])
def test_connect_multiple_addr_entries_succeed(self):
with patch('socket.socket', return_value=MockSocket()) as sock_mock, \
patch('socket.getaddrinfo',
return_value=[
(socket.AF_INET, 1, socket.IPPROTO_TCP,
'', ('127.0.0.1', 5672)),
(socket.AF_INET, 1, socket.IPPROTO_TCP,
'', ('127.0.0.2', 5672))
]):
self.t.sock = Mock()
self.t.close()
with patch.object(sock_mock.return_value, 'connect',
side_effect=(socket.error, None)):
self.t.connect()
def test_connect_short_curcuit_on_INET_succeed(self):
with patch('socket.socket', return_value=MockSocket()), \
patch('socket.getaddrinfo',
side_effect=[
[(socket.AF_INET, 1, socket.IPPROTO_TCP,
'', ('127.0.0.1', 5672))],
[(socket.AF_INET6, 1, socket.IPPROTO_TCP,
'', ('::1', 5672))]
]) as getaddrinfo:
self.t.sock = Mock()
self.t.close()
self.t.connect()
getaddrinfo.assert_called_with(
'localhost', 5672, socket.AF_INET, ANY, ANY)
def test_run(self):
with patch('celery.result.AsyncResult.get') as get:
out = WhateverIO()
r = result(app=self.app, stdout=out)
get.return_value = 'Jerry'
r.run('id')
assert 'Jerry' in out.getvalue()
get.return_value = 'Elaine'
r.run('id', task=self.add.name)
assert 'Elaine' in out.getvalue()
with patch('celery.result.AsyncResult.traceback') as tb:
r.run('id', task=self.add.name, traceback=True)
assert str(tb) in out.getvalue()
def test_run(self):
with patch("celery.result.AsyncResult.get") as get:
out = WhateverIO()
r = result(app=self.app, stdout=out)
get.return_value = "Jerry"
r.run("id")
assert "Jerry" in out.getvalue()
get.return_value = "Elaine"
r.run("id", task=self.add.name)
assert "Elaine" in out.getvalue()
with patch("celery.result.AsyncResult.traceback") as tb:
r.run("id", task=self.add.name, traceback=True)
assert str(tb) in out.getvalue()
def test_on_node_lost(self):
c = self.Consumer()
c.app.connection_for_read = _amqp_connection()
g = Gossip(c)
with patch('celery.worker.consumer.gossip.info') as info:
g.on_node_lost(c)
info.assert_called_with('missed heartbeat from %s', '*****@*****.**')
def test_worker_term_hard_handler_when_single_threaded(self):
with patch('celery.apps.worker.active_thread_count') as c:
c.return_value = 1
worker = self._Worker()
handlers = self.psig(cd.install_worker_term_hard_handler, worker)
with pytest.raises(WorkerTerminate):
handlers['SIGQUIT']('SIGQUIT', object())
def test_pool_restart_import_modules(self, _debug):
consumer = Consumer(self.app)
consumer.controller = _WC(app=self.app)
consumer.controller.consumer = consumer
consumer.controller.pool.restart = Mock()
consumer.reset_rate_limits = Mock(name='reset_rate_limits()')
consumer.update_strategies = Mock(name='update_strategies()')
panel = self.create_panel(consumer=consumer)
panel.app = self.app
assert panel.state.consumer.controller.consumer is consumer
_import = consumer.controller.app.loader.import_from_cwd = Mock()
_reload = Mock()
self.app.conf.worker_pool_restarts = True
with patch('sys.modules'):
panel.handle('pool_restart', {
'modules': ['foo', 'bar'],
'reloader': _reload,
})
consumer.controller.pool.restart.assert_called()
consumer.reset_rate_limits.assert_called_with()
consumer.update_strategies.assert_called_with()
_reload.assert_not_called()
_import.assert_has_calls([call('bar'), call('foo')], any_order=True)
assert _import.call_count == 2
def test_on_worker_init(self):
with self.fixup_context(self.app) as (f, _, _):
with patch('celery.fixups.django.DjangoWorkerFixup') as DWF:
f.on_worker_init()
DWF.assert_called_with(f.app)
DWF.return_value.install.assert_called_with()
assert f._worker_fixup is DWF.return_value
def test_autodiscovery(self):
with patch('celery.loaders.base.autodiscover_tasks') as auto:
auto.return_value = [Mock()]
auto.return_value[0].__name__ = 'moo'
self.loader.autodiscover_tasks(['A', 'B'])
assert 'moo' in self.loader.task_modules
self.loader.task_modules.discard('moo')
def test_detect_environment_eventlet(self):
with patch('eventlet.patcher.is_monkey_patched', create=True) as m:
assert sys.modules['eventlet']
m.return_value = True
env = compat._detect_environment()
m.assert_called_with(socket)
assert env == 'eventlet'
def test_as_logger(self):
with patch('amqp.utils.NullHandler') as _NullHandler:
m = Mock(name='logger')
m.handlers = None
x = get_logger(m)
assert x is m
x.addHandler.assert_called_with(_NullHandler())
def test_restore(self):
message = Mock(name='message')
with patch('kombu.transport.redis.loads') as loads:
loads.return_value = 'M', 'EX', 'RK'
client = self.channel._create_client = Mock(name='client')
client = client()
client.pipeline = ContextMock()
restore = self.channel._do_restore_message = Mock(
name='_do_restore_message',
)
pipe = client.pipeline.return_value
pipe_hget = Mock(name='pipe.hget')
pipe.hget.return_value = pipe_hget
pipe_hget_hdel = Mock(name='pipe.hget.hdel')
pipe_hget.hdel.return_value = pipe_hget_hdel
result = Mock(name='result')
pipe_hget_hdel.execute.return_value = None, None
self.channel._restore(message)
client.pipeline.assert_called_with()
unacked_key = self.channel.unacked_key
loads.assert_not_called()
tag = message.delivery_tag
pipe.hget.assert_called_with(unacked_key, tag)
pipe_hget.hdel.assert_called_with(unacked_key, tag)
pipe_hget_hdel.execute.assert_called_with()
pipe_hget_hdel.execute.return_value = result, None
self.channel._restore(message)
loads.assert_called_with(result)
restore.assert_called_with('M', 'EX', 'RK', client, False)
def test_do_restore_message(self):
client = Mock(name='client')
pl1 = {'body': 'BODY'}
spl1 = dumps(pl1)
lookup = self.channel._lookup = Mock(name='_lookup')
lookup.return_value = {'george', 'elaine'}
self.channel._do_restore_message(
pl1, 'ex', 'rkey', client,
)
client.rpush.assert_has_calls([
call('george', spl1), call('elaine', spl1),
], any_order=True)
client = Mock(name='client')
pl2 = {'body': 'BODY2', 'headers': {'x-funny': 1}}
headers_after = dict(pl2['headers'], redelivered=True)
spl2 = dumps(dict(pl2, headers=headers_after))
self.channel._do_restore_message(
pl2, 'ex', 'rkey', client,
)
client.rpush.assert_any_call('george', spl2)
client.rpush.assert_any_call('elaine', spl2)
client.rpush.side_effect = KeyError()
with patch('kombu.transport.redis.crit') as crit:
self.channel._do_restore_message(
pl2, 'ex', 'rkey', client,
)
crit.assert_called()