def run(self, hostname=None, pool_cls=None, app=None, uid=None, gid=None,
loglevel=None, logfile=None, pidfile=None, statedb=None,
**kwargs):
maybe_drop_privileges(uid=uid, gid=gid)
# Pools like eventlet/gevent needs to patch libs as early
# as possible.
pool_cls = (concurrency.get_implementation(pool_cls) or
self.app.conf.worker_pool)
if self.app.IS_WINDOWS and kwargs.get('beat'):
self.die('-B option does not work on Windows. '
'Please run celery beat as a separate service.')
hostname = self.host_format(default_nodename(hostname))
if loglevel:
try:
loglevel = mlevel(loglevel)
except KeyError: # pragma: no cover
self.die('Unknown level {0!r}. Please use one of {1}.'.format(
loglevel, '|'.join(
l for l in LOG_LEVELS if isinstance(l, string_t))))
worker = self.app.Worker(
hostname=hostname, pool_cls=pool_cls, loglevel=loglevel,
logfile=logfile, # node format handled by celery.app.log.setup
pidfile=self.node_format(pidfile, hostname),
statedb=self.node_format(statedb, hostname), **kwargs
)
worker.start()
return worker.exitcode
def beat(ctx,
detach=False,
logfile=None,
pidfile=None,
uid=None,
gid=None,
umask=None,
workdir=None,
**kwargs):
"""Start the beat periodic task scheduler."""
app = ctx.obj.app
if ctx.args:
try:
app.config_from_cmdline(ctx.args)
except (KeyError, ValueError) as e:
# TODO: Improve the error messages
raise click.UsageError(
"Unable to parse extra configuration"
" from command line.\n"
f"Reason: {e}",
ctx=ctx)
if not detach:
maybe_drop_privileges(uid=uid, gid=gid)
beat = partial(app.Beat, logfile=logfile, pidfile=pidfile, **kwargs)
if detach:
with detached(logfile, pidfile, uid, gid, umask, workdir):
return beat().run()
else:
return beat().run()
def run(self, hostname=None, pool_cls=None, app=None, uid=None, gid=None,
loglevel=None, logfile=None, pidfile=None, state_db=None,
**kwargs):
maybe_drop_privileges(uid=uid, gid=gid)
# Pools like eventlet/gevent needs to patch libs as early
# as possible.
pool_cls = (concurrency.get_implementation(pool_cls) or
self.app.conf.worker_pool)
if self.app.IS_WINDOWS and kwargs.get('beat'):
self.die('-B option does not work on Windows. '
'Please run celery beat as a separate service.')
hostname = self.host_format(default_nodename(hostname))
if loglevel:
try:
loglevel = mlevel(loglevel)
except KeyError: # pragma: no cover
self.die('Unknown level {0!r}. Please use one of {1}.'.format(
loglevel, '|'.join(
l for l in LOG_LEVELS if isinstance(l, string_t))))
worker = self.app.Worker(
hostname=hostname, pool_cls=pool_cls, loglevel=loglevel,
logfile=logfile, # node format handled by celery.app.log.setup
pidfile=self.node_format(pidfile, hostname),
state_db=self.node_format(state_db, hostname), **kwargs
)
worker.start()
return worker.exitcode
def test_with_guid(self, initgroups, setuid, setgid, parse_gid, parse_uid):
parse_uid.return_value = 5001
parse_gid.return_value = 50001
maybe_drop_privileges(uid="user", gid="group")
parse_uid.assert_called_with("user")
parse_gid.assert_called_with("group")
setgid.assert_called_with(50001)
initgroups.assert_called_with(5001, 50001)
setuid.assert_called_with(5001)
def test_with_guid(self, initgroups, setuid, setgid, parse_gid,
parse_uid):
parse_uid.return_value = 5001
parse_gid.return_value = 50001
maybe_drop_privileges(uid="user", gid="group")
parse_uid.assert_called_with("user")
parse_gid.assert_called_with("group")
setgid.assert_called_with(50001)
initgroups.assert_called_with(5001, 50001)
setuid.assert_called_with(5001)
def test_with_uid(self, initgroups, setuid, setgid, getpwuid, parse_uid):
class pw_struct(object):
pw_gid = 50001
getpwuid.return_value = pw_struct()
parse_uid.return_value = 5001
maybe_drop_privileges(uid="user")
parse_uid.assert_called_with("user")
getpwuid.assert_called_with(5001)
setgid.assert_called_with(50001)
initgroups.assert_called_with(5001, 50001)
setuid.assert_called_with(5001)
def test_with_uid(self, initgroups, setuid, setgid, getpwuid,
parse_uid):
class pw_struct(object):
pw_gid = 50001
getpwuid.return_value = pw_struct()
parse_uid.return_value = 5001
maybe_drop_privileges(uid="user")
parse_uid.assert_called_with("user")
getpwuid.assert_called_with(5001)
setgid.assert_called_with(50001)
initgroups.assert_called_with(5001, 50001)
setuid.assert_called_with(5001)
def run(self, detach=False, logfile=None, pidfile=None, uid=None,
gid=None, umask=None, workdir=None, **kwargs):
if not detach:
maybe_drop_privileges(uid=uid, gid=gid)
kwargs.pop('app', None)
beat = partial(self.app.Beat,
logfile=logfile, pidfile=pidfile, **kwargs)
if detach:
with detached(logfile, pidfile, uid, gid, umask, workdir):
return beat().run()
else:
return beat().run()
def run(self, detach=False, logfile=None, pidfile=None, uid=None,
gid=None, umask=None, workdir=None, **kwargs):
if not detach:
maybe_drop_privileges(uid=uid, gid=gid)
kwargs.pop('app', None)
beat = partial(self.app.Beat,
logfile=logfile, pidfile=pidfile, **kwargs)
if detach:
with detached(logfile, pidfile, uid, gid, umask, workdir):
return beat().run()
else:
return beat().run()
def worker(ctx, hostname=None, pool_cls=None, app=None, uid=None, gid=None,
loglevel=None, logfile=None, pidfile=None, statedb=None,
**kwargs):
"""Start worker instance.
Examples
--------
$ celery --app=proj worker -l INFO
$ celery -A proj worker -l INFO -Q hipri,lopri
$ celery -A proj worker --concurrency=4
$ celery -A proj worker --concurrency=1000 -P eventlet
$ celery worker --autoscale=10,0
"""
app = ctx.obj.app
if ctx.args:
try:
app.config_from_cmdline(ctx.args, namespace='worker')
except (KeyError, ValueError) as e:
# TODO: Improve the error messages
raise click.UsageError(
"Unable to parse extra configuration from command line.\n"
f"Reason: {e}", ctx=ctx)
if kwargs.get('detach', False):
argv = ['-m', 'celery'] + sys.argv[1:]
if '--detach' in argv:
argv.remove('--detach')
if '-D' in argv:
argv.remove('-D')
return detach(sys.executable,
argv,
logfile=logfile,
pidfile=pidfile,
uid=uid, gid=gid,
umask=kwargs.get('umask', None),
workdir=kwargs.get('workdir', None),
app=app,
executable=kwargs.get('executable', None),
hostname=hostname)
maybe_drop_privileges(uid=uid, gid=gid)
worker = app.Worker(
hostname=hostname, pool_cls=pool_cls, loglevel=loglevel,
logfile=logfile, # node format handled by celery.app.log.setup
pidfile=node_format(pidfile, hostname),
statedb=node_format(statedb, hostname),
no_color=ctx.obj.no_color,
**kwargs)
worker.start()
return worker.exitcode
def test_with_uid(
self, initgroups, setuid, setgid, getpwuid, parse_gid, parse_uid, getuid, geteuid, getgid, getegid
):
geteuid.return_value = 10
getuid.return_value = 10
class pw_struct(object):
pw_gid = 50001
def raise_on_second_call(*args, **kwargs):
setuid.side_effect = OSError()
setuid.side_effect.errno = errno.EPERM
setuid.side_effect = raise_on_second_call
getpwuid.return_value = pw_struct()
parse_uid.return_value = 5001
parse_gid.return_value = 5001
maybe_drop_privileges(uid="user")
parse_uid.assert_called_with("user")
getpwuid.assert_called_with(5001)
setgid.assert_called_with(50001)
initgroups.assert_called_with(5001, 50001)
setuid.assert_has_calls([call(5001), call(0)])
setuid.side_effect = raise_on_second_call
def to_root_on_second_call(mock, first):
return_value = [first]
def on_first_call(*args, **kwargs):
ret, return_value[0] = return_value[0], 0
return ret
mock.side_effect = on_first_call
to_root_on_second_call(geteuid, 10)
to_root_on_second_call(getuid, 10)
with self.assertRaises(AssertionError):
maybe_drop_privileges(uid="user")
getuid.return_value = getuid.side_effect = None
geteuid.return_value = geteuid.side_effect = None
getegid.return_value = 0
getgid.return_value = 0
setuid.side_effect = raise_on_second_call
with self.assertRaises(AssertionError):
maybe_drop_privileges(gid="group")
getuid.reset_mock()
geteuid.reset_mock()
setuid.reset_mock()
getuid.side_effect = geteuid.side_effect = None
def raise_on_second_call(*args, **kwargs):
setuid.side_effect = OSError()
setuid.side_effect.errno = errno.ENOENT
setuid.side_effect = raise_on_second_call
with self.assertRaises(OSError):
maybe_drop_privileges(uid="user")
def test_with_uid(self, initgroups, setuid, setgid, getpwuid,
parse_gid, parse_uid, getuid, geteuid, getgid,
getegid):
geteuid.return_value = 10
getuid.return_value = 10
class pw_struct(object):
pw_gid = 50001
def raise_on_second_call(*args, **kwargs):
setuid.side_effect = OSError()
setuid.side_effect.errno = errno.EPERM
setuid.side_effect = raise_on_second_call
getpwuid.return_value = pw_struct()
parse_uid.return_value = 5001
parse_gid.return_value = 5001
maybe_drop_privileges(uid='user')
parse_uid.assert_called_with('user')
getpwuid.assert_called_with(5001)
setgid.assert_called_with(50001)
initgroups.assert_called_with(5001, 50001)
setuid.assert_has_calls([call(5001), call(0)])
setuid.side_effect = raise_on_second_call
def to_root_on_second_call(mock, first):
return_value = [first]
def on_first_call(*args, **kwargs):
ret, return_value[0] = return_value[0], 0
return ret
mock.side_effect = on_first_call
to_root_on_second_call(geteuid, 10)
to_root_on_second_call(getuid, 10)
with self.assertRaises(AssertionError):
maybe_drop_privileges(uid='user')
getuid.return_value = getuid.side_effect = None
geteuid.return_value = geteuid.side_effect = None
getegid.return_value = 0
getgid.return_value = 0
setuid.side_effect = raise_on_second_call
with self.assertRaises(AssertionError):
maybe_drop_privileges(gid='group')
getuid.reset_mock()
geteuid.reset_mock()
setuid.reset_mock()
getuid.side_effect = geteuid.side_effect = None
def raise_on_second_call(*args, **kwargs):
setuid.side_effect = OSError()
setuid.side_effect.errno = errno.ENOENT
setuid.side_effect = raise_on_second_call
with self.assertRaises(OSError):
maybe_drop_privileges(uid='user')
def test_with_uid(self, initgroups, setuid, setgid, getpwuid, parse_uid):
class pw_struct(object):
pw_gid = 50001
def raise_on_second_call(*args, **kwargs):
setuid.side_effect = OSError()
setuid.side_effect.errno = errno.EPERM
setuid.side_effect = raise_on_second_call
getpwuid.return_value = pw_struct()
parse_uid.return_value = 5001
maybe_drop_privileges(uid="user")
parse_uid.assert_called_with("user")
getpwuid.assert_called_with(5001)
setgid.assert_called_with(50001)
initgroups.assert_called_with(5001, 50001)
setuid.assert_has_calls([call(5001), call(0)])
def test_with_uid(self, initgroups, setuid, setgid, getpwuid,
parse_uid):
class pw_struct(object):
pw_gid = 50001
def raise_on_second_call(*args, **kwargs):
setuid.side_effect = OSError()
setuid.side_effect.errno = errno.EPERM
setuid.side_effect = raise_on_second_call
getpwuid.return_value = pw_struct()
parse_uid.return_value = 5001
maybe_drop_privileges(uid='user')
parse_uid.assert_called_with('user')
getpwuid.assert_called_with(5001)
setgid.assert_called_with(50001)
initgroups.assert_called_with(5001, 50001)
setuid.assert_has_calls([call(5001), call(0)])
def run(self, hostname=None, pool_cls=None, loglevel=None,
app=None, uid=None, gid=None, **kwargs):
maybe_drop_privileges(uid=uid, gid=gid)
# Pools like eventlet/gevent needs to patch libs as early
# as possible.
pool_cls = (concurrency.get_implementation(pool_cls) or
self.app.conf.CELERYD_POOL)
if self.app.IS_WINDOWS and kwargs.get('beat'):
self.die('-B option does not work on Windows. '
'Please run celery beat as a separate service.')
hostname = self.simple_format(hostname)
if loglevel:
try:
loglevel = mlevel(loglevel)
except KeyError: # pragma: no cover
self.die('Unknown level {0!r}. Please use one of {1}.'.format(
loglevel, '|'.join(
l for l in LOG_LEVELS if isinstance(l, string_t))))
return self.app.Worker(
hostname=hostname, pool_cls=pool_cls, loglevel=loglevel, **kwargs
).start()
def test_with_guid(self, initgroups, setuid, setgid, parse_gid, parse_uid):
def raise_on_second_call(*args, **kwargs):
setuid.side_effect = OSError()
setuid.side_effect.errno = errno.EPERM
setuid.side_effect = raise_on_second_call
parse_uid.return_value = 5001
parse_gid.return_value = 50001
maybe_drop_privileges(uid="user", gid="group")
parse_uid.assert_called_with("user")
parse_gid.assert_called_with("group")
setgid.assert_called_with(50001)
initgroups.assert_called_with(5001, 50001)
setuid.assert_has_calls([call(5001), call(0)])
setuid.side_effect = None
with self.assertRaises(RuntimeError):
maybe_drop_privileges(uid="user", gid="group")
setuid.side_effect = OSError()
setuid.side_effect.errno = errno.EINVAL
with self.assertRaises(OSError):
maybe_drop_privileges(uid="user", gid="group")
def test_with_guid(self, initgroups, setuid, setgid, parse_gid, parse_uid):
def raise_on_second_call(*args, **kwargs):
setuid.side_effect = OSError()
setuid.side_effect.errno = errno.EPERM
setuid.side_effect = raise_on_second_call
parse_uid.return_value = 5001
parse_gid.return_value = 50001
maybe_drop_privileges(uid='user', gid='group')
parse_uid.assert_called_with('user')
parse_gid.assert_called_with('group')
setgid.assert_called_with(50001)
initgroups.assert_called_with(5001, 50001)
setuid.assert_has_calls([call(5001), call(0)])
setuid.side_effect = None
with pytest.raises(SecurityError):
maybe_drop_privileges(uid='user', gid='group')
setuid.side_effect = OSError()
setuid.side_effect.errno = errno.EINVAL
with pytest.raises(OSError):
maybe_drop_privileges(uid='user', gid='group')
def test_with_guid(self, initgroups, setuid, setgid,
parse_gid, parse_uid):
def raise_on_second_call(*args, **kwargs):
setuid.side_effect = OSError()
setuid.side_effect.errno = errno.EPERM
setuid.side_effect = raise_on_second_call
parse_uid.return_value = 5001
parse_gid.return_value = 50001
maybe_drop_privileges(uid='user', gid='group')
parse_uid.assert_called_with('user')
parse_gid.assert_called_with('group')
setgid.assert_called_with(50001)
initgroups.assert_called_with(5001, 50001)
setuid.assert_has_calls([call(5001), call(0)])
setuid.side_effect = None
with pytest.raises(SecurityError):
maybe_drop_privileges(uid='user', gid='group')
setuid.side_effect = OSError()
setuid.side_effect.errno = errno.EINVAL
with pytest.raises(OSError):
maybe_drop_privileges(uid='user', gid='group')
def test_only_gid(self, parse_gid, setgid, setuid):
parse_gid.return_value = 50001
maybe_drop_privileges(gid='group')
parse_gid.assert_called_with('group')
setgid.assert_called_with(50001)
setuid.assert_not_called()
def test_on_windows(self):
prev, sys.platform = sys.platform, 'win32'
try:
maybe_drop_privileges()
finally:
sys.platform = prev
def test_only_gid(self, parse_gid, setgid, setuid):
parse_gid.return_value = 50001
maybe_drop_privileges(gid='group')
parse_gid.assert_called_with('group')
setgid.assert_called_with(50001)
self.assertFalse(setuid.called)
def test_on_windows(self):
prev, sys.platform = sys.platform, 'win32'
try:
maybe_drop_privileges()
finally:
sys.platform = prev
def worker(ctx, hostname=None, pool_cls=None, app=None, uid=None, gid=None,
loglevel=None, logfile=None, pidfile=None, statedb=None,
**kwargs):
"""Start worker instance.
Examples
--------
$ celery --app=proj worker -l INFO
$ celery -A proj worker -l INFO -Q hipri,lopri
$ celery -A proj worker --concurrency=4
$ celery -A proj worker --concurrency=1000 -P eventlet
$ celery worker --autoscale=10,0
"""
app = ctx.obj.app
if ctx.args:
try:
app.config_from_cmdline(ctx.args, namespace='worker')
except (KeyError, ValueError) as e:
# TODO: Improve the error messages
raise click.UsageError(
"Unable to parse extra configuration from command line.\n"
f"Reason: {e}", ctx=ctx)
if kwargs.get('detach', False):
params = ctx.params.copy()
params.pop('detach')
params.pop('logfile')
params.pop('pidfile')
params.pop('uid')
params.pop('gid')
umask = params.pop('umask')
workdir = ctx.obj.workdir
params.pop('hostname')
executable = params.pop('executable')
argv = ['-m', 'celery', 'worker']
for arg, value in params.items():
if isinstance(value, bool) and value:
argv.append(f'--{arg}')
else:
if value is not None:
argv.append(f'--{arg}')
argv.append(str(value))
return detach(sys.executable,
argv,
logfile=logfile,
pidfile=pidfile,
uid=uid, gid=gid,
umask=umask,
workdir=workdir,
app=app,
executable=executable,
hostname=hostname)
return
maybe_drop_privileges(uid=uid, gid=gid)
worker = app.Worker(
hostname=hostname, pool_cls=pool_cls, loglevel=loglevel,
logfile=logfile, # node format handled by celery.app.log.setup
pidfile=node_format(pidfile, hostname),
statedb=node_format(statedb, hostname),
no_color=ctx.obj.no_color,
**kwargs)
worker.start()
return worker.exitcode
def test_only_gid(self, parse_gid, setgid, setuid):
parse_gid.return_value = 50001
maybe_drop_privileges(gid="group")
parse_gid.assert_called_with("group")
setgid.assert_called_with(50001)
self.assertFalse(setuid.called)
