def test_on_start_custom_logging(self):
with mock.stdouts():
self.app.log.redirect_stdouts = Mock()
worker = self.Worker(app=self.app, redirect_stoutds=True)
worker._custom_logging = True
worker.on_start()
self.app.log.redirect_stdouts.assert_not_called()
def test_platform_tweaks_macOS(self):
class macOSWorker(Worker):
proxy_workaround_installed = False
def macOS_proxy_detection_workaround(self):
self.proxy_workaround_installed = True
with mock.stdouts():
worker = macOSWorker(app=self.app, redirect_stdouts=False)
def install_HUP_nosupport(controller):
controller.hup_not_supported_installed = True
class Controller(object):
pass
prev = cd.install_HUP_not_supported_handler
cd.install_HUP_not_supported_handler = install_HUP_nosupport
try:
worker.app.IS_macOS = True
controller = Controller()
worker.install_platform_tweaks(controller)
assert controller.hup_not_supported_installed
assert worker.proxy_workaround_installed
finally:
cd.install_HUP_not_supported_handler = prev
def test_init_queues(self):
with mock.stdouts():
app = self.app
c = app.conf
app.amqp.queues = app.amqp.Queues({
'celery': {
'exchange': 'celery',
'routing_key': 'celery',
},
'video': {
'exchange': 'video',
'routing_key': 'video',
},
})
worker = self.Worker(app=self.app)
worker.setup_queues(['video'])
assert 'video' in app.amqp.queues
assert 'video' in app.amqp.queues.consume_from
assert 'celery' in app.amqp.queues
assert 'celery' not in app.amqp.queues.consume_from
c.task_create_missing_queues = False
del(app.amqp.queues)
with pytest.raises(ImproperlyConfigured):
self.Worker(app=self.app).setup_queues(['image'])
del(app.amqp.queues)
c.task_create_missing_queues = True
worker = self.Worker(app=self.app)
worker.setup_queues(['image'])
assert 'image' in app.amqp.queues.consume_from
assert app.amqp.queues['image'] == Queue(
'image', Exchange('image'),
routing_key='image',
)
def test_using_pidfile(self, create_pidlock):
b = MockBeat(app=self.app, pidfile='pidfilelockfilepid',
socket_timeout=None, redirect_stdouts=False)
b.install_sync_handler = Mock(name='beat.install_sync_handler')
with mock.stdouts():
b.start_scheduler()
create_pidlock.assert_called()
def test_parser(self):
x = detached_celeryd(self.app)
p = x.create_parser('celeryd_detach')
options, leftovers = p.parse_known_args([
'--logfile=foo', '--fake', '--enable',
'a', 'b', '-c1', '-d', '2',
])
assert options.logfile == 'foo'
assert leftovers, ['--enable', '-c1', '-d' == '2']
options, leftovers = p.parse_known_args([
'--fake', '--enable',
'--pidfile=/var/pid/foo.pid',
'a', 'b', '-c1', '-d', '2',
])
assert options.pidfile == '/var/pid/foo.pid'
with mock.stdouts():
with pytest.raises(SystemExit):
p.parse_args(['--logfile'])
p._option_string_actions['--logfile'].nargs = 2
with pytest.raises(SystemExit):
p.parse_args(['--logfile=a'])
with pytest.raises(SystemExit):
p.parse_args(['--fake=abc'])
assert p._option_string_actions['--logfile'].nargs == 2
p.parse_args(['--logfile', 'a', 'b'])
def test_cpu_count(self):
with mock.stdouts():
with patch('celery.worker.worker.cpu_count') as cpu_count:
cpu_count.side_effect = NotImplementedError()
w = self.app.Worker(concurrency=None)
assert w.concurrency == 2
w = self.app.Worker(concurrency=5)
assert w.concurrency == 5
def test_regression_worker_startup_info(self):
self.app.conf.result_backend = (
'cache+memcached://127.0.0.1:11211;127.0.0.2:11211;127.0.0.3/'
)
worker = self.app.Worker()
with mock.stdouts():
worker.on_start()
assert worker.startup_info()
def test_with_bogus_args(self, app):
with mock.stdouts() as (_, stderr):
cmd = MockCommand(app=app)
cmd.supports_args = False
with pytest.raises(SystemExit):
cmd.execute_from_commandline(argv=['--bogus'])
assert stderr.getvalue()
assert 'Unrecognized' in stderr.getvalue()
def test_remove_if_stale_broken_pid(self):
with mock.stdouts():
p = Pidfile('/var/pid')
p.read_pid = Mock()
p.read_pid.side_effect = ValueError()
p.remove = Mock()
assert p.remove_if_stale()
p.remove.assert_called_with()
def test_regression_worker_startup_info(self):
self.app.conf.result_backend = (
'mongodb://*****:*****@host0.com:43437,host1.com:43437'
'/work4us?replicaSet=rs&ssl=true'
)
worker = self.app.Worker()
with mock.stdouts():
worker.on_start()
assert worker.startup_info()
def test_on_consumer_ready(self):
worker_ready_sent = [False]
@signals.worker_ready.connect
def on_worker_ready(**kwargs):
worker_ready_sent[0] = True
with mock.stdouts():
self.Worker(app=self.app).on_consumer_ready(object())
assert worker_ready_sent[0]
def test_setup_logger_no_handlers_stream(self):
l = self.get_logger()
l.handlers = []
with mock.stdouts() as outs:
stdout, stderr = outs
l = self.setup_logger(logfile=sys.stderr,
loglevel=logging.INFO, root=False)
l.info('The quick brown fox...')
assert 'The quick brown fox...' in stderr.getvalue()
def test_create_pidlock(self, Pidfile):
p = Pidfile.return_value = Mock()
p.is_locked.return_value = True
p.remove_if_stale.return_value = False
with mock.stdouts() as (_, err):
with pytest.raises(SystemExit):
create_pidlock('/var/pid')
assert 'already exists' in err.getvalue()
p.remove_if_stale.return_value = True
ret = create_pidlock('/var/pid')
assert ret is p
def test_remove_if_stale_process_dead(self, kill):
with mock.stdouts():
p = Pidfile('/var/pid')
p.read_pid = Mock()
p.read_pid.return_value = 1816
p.remove = Mock()
exc = OSError()
exc.errno = errno.ESRCH
kill.side_effect = exc
assert p.remove_if_stale()
kill.assert_called_with(1816, 0)
p.remove.assert_called_with()
def test_crash(self):
class T(bgThread):
def body(self):
raise KeyError()
with patch('os._exit') as _exit:
with mock.stdouts():
_exit.side_effect = ValueError()
t = T()
with pytest.raises(ValueError):
t.run()
_exit.assert_called_with(1)
def test_warns_if_running_as_privileged_user(self, _exit, patching):
getuid = patching('os.getuid')
with mock.stdouts() as (_, stderr):
getuid.return_value = 0
self.app.conf.accept_content = ['pickle']
worker = self.Worker(app=self.app)
worker.on_start()
_exit.assert_called_with(1)
patching.setattr('celery.platforms.C_FORCE_ROOT', True)
worker = self.Worker(app=self.app)
worker.on_start()
assert 'a very bad idea' in stderr.getvalue()
patching.setattr('celery.platforms.C_FORCE_ROOT', False)
self.app.conf.accept_content = ['json']
worker = self.Worker(app=self.app)
worker.on_start()
assert 'superuser' in stderr.getvalue()
def test_general_platform_tweaks(self):
restart_worker_handler_installed = [False]
def install_worker_restart_handler(worker):
restart_worker_handler_installed[0] = True
class Controller(object):
pass
with mock.stdouts():
prev = cd.install_worker_restart_handler
cd.install_worker_restart_handler = install_worker_restart_handler
try:
worker = self.Worker(app=self.app)
worker.app.IS_macOS = False
worker.install_platform_tweaks(Controller())
assert restart_worker_handler_installed[0]
finally:
cd.install_worker_restart_handler = prev
def test_startup_info(self):
with mock.stdouts():
worker = self.Worker(app=self.app)
worker.on_start()
assert worker.startup_info()
worker.loglevel = logging.DEBUG
assert worker.startup_info()
worker.loglevel = logging.INFO
assert worker.startup_info()
worker.autoscale = 13, 10
assert worker.startup_info()
prev_loader = self.app.loader
worker = self.Worker(
app=self.app,
queues='foo,bar,baz,xuzzy,do,re,mi',
)
with patch('celery.apps.worker.qualname') as qualname:
qualname.return_value = 'acme.backed_beans.Loader'
assert worker.startup_info()
with patch('celery.apps.worker.qualname') as qualname:
qualname.return_value = 'celery.loaders.Loader'
assert worker.startup_info()
from celery.loaders.app import AppLoader
self.app.loader = AppLoader(app=self.app)
assert worker.startup_info()
self.app.loader = prev_loader
worker.task_events = True
assert worker.startup_info()
# test when there are too few output lines
# to draft the ascii art onto
prev, cd.ARTLINES = cd.ARTLINES, ['the quick brown fox']
try:
assert worker.startup_info()
finally:
cd.ARTLINES = prev
def test_run_worker(self, patching):
handlers = {}
class Signals(platforms.Signals):
def __setitem__(self, sig, handler):
handlers[sig] = handler
patching.setattr('celery.platforms.signals', Signals())
with mock.stdouts():
w = self.Worker(app=self.app)
w._isatty = False
w.on_start()
for sig in 'SIGINT', 'SIGHUP', 'SIGTERM':
assert sig in handlers
handlers.clear()
w = self.Worker(app=self.app)
w._isatty = True
w.on_start()
for sig in 'SIGINT', 'SIGTERM':
assert sig in handlers
assert 'SIGHUP' not in handlers
def test_logging_proxy(self):
logger = self.setup_logger(loglevel=logging.ERROR, logfile=None,
root=False)
with mock.wrap_logger(logger) as sio:
p = LoggingProxy(logger, loglevel=logging.ERROR)
p.close()
p.write('foo')
assert 'foo' not in sio.getvalue()
p.closed = False
p.write('foo')
assert 'foo' in sio.getvalue()
lines = ['baz', 'xuzzy']
p.writelines(lines)
for line in lines:
assert line in sio.getvalue()
p.flush()
p.close()
assert not p.isatty()
with mock.stdouts() as (stdout, stderr):
with in_sighandler():
p.write('foo')
assert stderr.getvalue()
def test_loglevel_string(self):
with mock.stdouts():
worker = self.Worker(app=self.app, loglevel='INFO')
assert worker.loglevel == logging.INFO
def test_invalid_loglevel_gives_error(self):
with mock.stdouts():
x = worker(app=self.app)
with pytest.raises(SystemExit):
x.run(loglevel='GRIM_REAPER')
def test_windows_B_option(self):
with mock.stdouts():
self.app.IS_WINDOWS = True
with pytest.raises(SystemExit):
worker(app=self.app).run(beat=True)
def test_queues_string(self):
with mock.stdouts():
w = self.app.Worker()
w.setup_queues('foo,bar,baz')
assert 'foo' in self.app.amqp.queues
def test_run(self):
with mock.stdouts():
self.Worker(app=self.app).on_start()
self.Worker(app=self.app, purge=True).on_start()
worker = self.Worker(app=self.app)
worker.on_start()
def test_purge_messages(self):
with mock.stdouts():
self.Worker(app=self.app).purge_messages()
def test_startup_info_pool_is_str(self):
with mock.stdouts():
worker = self.Worker(app=self.app, redirect_stdouts=False)
worker.pool_cls = 'foo'
worker.startup_info()
def test_autoscale_argument(self):
with mock.stdouts():
worker1 = self.Worker(app=self.app, autoscale='10,3')
assert worker1.autoscale == [10, 3]
worker2 = self.Worker(app=self.app, autoscale='10')
assert worker2.autoscale == [10, 0]
def test_redirect_stdouts(self):
with mock.stdouts():
self.Worker(app=self.app, redirect_stdouts=False)
with pytest.raises(AttributeError):
sys.stdout.logger
def test_unknown_loglevel(self):
with mock.stdouts():
with pytest.raises(SystemExit):
worker(app=self.app).run(loglevel='ALIEN')
worker1 = self.Worker(app=self.app, loglevel=0xFFFF)
assert worker1.loglevel == 0xFFFF