def run(self, hostname=None, pool_cls=None, app=None, uid=None, gid=None, loglevel=None, logfile=None, pidfile=None, statedb=None, **kwargs): maybe_drop_privileges(uid=uid, gid=gid) # Pools like eventlet/gevent needs to patch libs as early # as possible. pool_cls = (concurrency.get_implementation(pool_cls) or self.app.conf.worker_pool) if self.app.IS_WINDOWS and kwargs.get('beat'): self.die('-B option does not work on Windows. ' 'Please run celery beat as a separate service.') hostname = self.host_format(default_nodename(hostname)) if loglevel: try: loglevel = mlevel(loglevel) except KeyError: # pragma: no cover self.die('Unknown level {0!r}. Please use one of {1}.'.format( loglevel, '|'.join( l for l in LOG_LEVELS if isinstance(l, string_t)))) worker = self.app.Worker( hostname=hostname, pool_cls=pool_cls, loglevel=loglevel, logfile=logfile, # node format handled by celery.app.log.setup pidfile=self.node_format(pidfile, hostname), statedb=self.node_format(statedb, hostname), **kwargs ) worker.start() return worker.exitcode
def beat(ctx, detach=False, logfile=None, pidfile=None, uid=None, gid=None, umask=None, workdir=None, **kwargs): """Start the beat periodic task scheduler.""" app = ctx.obj.app if ctx.args: try: app.config_from_cmdline(ctx.args) except (KeyError, ValueError) as e: # TODO: Improve the error messages raise click.UsageError( "Unable to parse extra configuration" " from command line.\n" f"Reason: {e}", ctx=ctx) if not detach: maybe_drop_privileges(uid=uid, gid=gid) beat = partial(app.Beat, logfile=logfile, pidfile=pidfile, **kwargs) if detach: with detached(logfile, pidfile, uid, gid, umask, workdir): return beat().run() else: return beat().run()
def run(self, hostname=None, pool_cls=None, app=None, uid=None, gid=None, loglevel=None, logfile=None, pidfile=None, state_db=None, **kwargs): maybe_drop_privileges(uid=uid, gid=gid) # Pools like eventlet/gevent needs to patch libs as early # as possible. pool_cls = (concurrency.get_implementation(pool_cls) or self.app.conf.worker_pool) if self.app.IS_WINDOWS and kwargs.get('beat'): self.die('-B option does not work on Windows. ' 'Please run celery beat as a separate service.') hostname = self.host_format(default_nodename(hostname)) if loglevel: try: loglevel = mlevel(loglevel) except KeyError: # pragma: no cover self.die('Unknown level {0!r}. Please use one of {1}.'.format( loglevel, '|'.join( l for l in LOG_LEVELS if isinstance(l, string_t)))) worker = self.app.Worker( hostname=hostname, pool_cls=pool_cls, loglevel=loglevel, logfile=logfile, # node format handled by celery.app.log.setup pidfile=self.node_format(pidfile, hostname), state_db=self.node_format(state_db, hostname), **kwargs ) worker.start() return worker.exitcode
def test_with_guid(self, initgroups, setuid, setgid, parse_gid, parse_uid): parse_uid.return_value = 5001 parse_gid.return_value = 50001 maybe_drop_privileges(uid="user", gid="group") parse_uid.assert_called_with("user") parse_gid.assert_called_with("group") setgid.assert_called_with(50001) initgroups.assert_called_with(5001, 50001) setuid.assert_called_with(5001)
def test_with_uid(self, initgroups, setuid, setgid, getpwuid, parse_uid): class pw_struct(object): pw_gid = 50001 getpwuid.return_value = pw_struct() parse_uid.return_value = 5001 maybe_drop_privileges(uid="user") parse_uid.assert_called_with("user") getpwuid.assert_called_with(5001) setgid.assert_called_with(50001) initgroups.assert_called_with(5001, 50001) setuid.assert_called_with(5001)
def run(self, detach=False, logfile=None, pidfile=None, uid=None, gid=None, umask=None, workdir=None, **kwargs): if not detach: maybe_drop_privileges(uid=uid, gid=gid) kwargs.pop('app', None) beat = partial(self.app.Beat, logfile=logfile, pidfile=pidfile, **kwargs) if detach: with detached(logfile, pidfile, uid, gid, umask, workdir): return beat().run() else: return beat().run()
def worker(ctx, hostname=None, pool_cls=None, app=None, uid=None, gid=None, loglevel=None, logfile=None, pidfile=None, statedb=None, **kwargs): """Start worker instance. Examples -------- $ celery --app=proj worker -l INFO $ celery -A proj worker -l INFO -Q hipri,lopri $ celery -A proj worker --concurrency=4 $ celery -A proj worker --concurrency=1000 -P eventlet $ celery worker --autoscale=10,0 """ app = ctx.obj.app if ctx.args: try: app.config_from_cmdline(ctx.args, namespace='worker') except (KeyError, ValueError) as e: # TODO: Improve the error messages raise click.UsageError( "Unable to parse extra configuration from command line.\n" f"Reason: {e}", ctx=ctx) if kwargs.get('detach', False): argv = ['-m', 'celery'] + sys.argv[1:] if '--detach' in argv: argv.remove('--detach') if '-D' in argv: argv.remove('-D') return detach(sys.executable, argv, logfile=logfile, pidfile=pidfile, uid=uid, gid=gid, umask=kwargs.get('umask', None), workdir=kwargs.get('workdir', None), app=app, executable=kwargs.get('executable', None), hostname=hostname) maybe_drop_privileges(uid=uid, gid=gid) worker = app.Worker( hostname=hostname, pool_cls=pool_cls, loglevel=loglevel, logfile=logfile, # node format handled by celery.app.log.setup pidfile=node_format(pidfile, hostname), statedb=node_format(statedb, hostname), no_color=ctx.obj.no_color, **kwargs) worker.start() return worker.exitcode
def test_with_uid( self, initgroups, setuid, setgid, getpwuid, parse_gid, parse_uid, getuid, geteuid, getgid, getegid ): geteuid.return_value = 10 getuid.return_value = 10 class pw_struct(object): pw_gid = 50001 def raise_on_second_call(*args, **kwargs): setuid.side_effect = OSError() setuid.side_effect.errno = errno.EPERM setuid.side_effect = raise_on_second_call getpwuid.return_value = pw_struct() parse_uid.return_value = 5001 parse_gid.return_value = 5001 maybe_drop_privileges(uid="user") parse_uid.assert_called_with("user") getpwuid.assert_called_with(5001) setgid.assert_called_with(50001) initgroups.assert_called_with(5001, 50001) setuid.assert_has_calls([call(5001), call(0)]) setuid.side_effect = raise_on_second_call def to_root_on_second_call(mock, first): return_value = [first] def on_first_call(*args, **kwargs): ret, return_value[0] = return_value[0], 0 return ret mock.side_effect = on_first_call to_root_on_second_call(geteuid, 10) to_root_on_second_call(getuid, 10) with self.assertRaises(AssertionError): maybe_drop_privileges(uid="user") getuid.return_value = getuid.side_effect = None geteuid.return_value = geteuid.side_effect = None getegid.return_value = 0 getgid.return_value = 0 setuid.side_effect = raise_on_second_call with self.assertRaises(AssertionError): maybe_drop_privileges(gid="group") getuid.reset_mock() geteuid.reset_mock() setuid.reset_mock() getuid.side_effect = geteuid.side_effect = None def raise_on_second_call(*args, **kwargs): setuid.side_effect = OSError() setuid.side_effect.errno = errno.ENOENT setuid.side_effect = raise_on_second_call with self.assertRaises(OSError): maybe_drop_privileges(uid="user")
def test_with_uid(self, initgroups, setuid, setgid, getpwuid, parse_gid, parse_uid, getuid, geteuid, getgid, getegid): geteuid.return_value = 10 getuid.return_value = 10 class pw_struct(object): pw_gid = 50001 def raise_on_second_call(*args, **kwargs): setuid.side_effect = OSError() setuid.side_effect.errno = errno.EPERM setuid.side_effect = raise_on_second_call getpwuid.return_value = pw_struct() parse_uid.return_value = 5001 parse_gid.return_value = 5001 maybe_drop_privileges(uid='user') parse_uid.assert_called_with('user') getpwuid.assert_called_with(5001) setgid.assert_called_with(50001) initgroups.assert_called_with(5001, 50001) setuid.assert_has_calls([call(5001), call(0)]) setuid.side_effect = raise_on_second_call def to_root_on_second_call(mock, first): return_value = [first] def on_first_call(*args, **kwargs): ret, return_value[0] = return_value[0], 0 return ret mock.side_effect = on_first_call to_root_on_second_call(geteuid, 10) to_root_on_second_call(getuid, 10) with self.assertRaises(AssertionError): maybe_drop_privileges(uid='user') getuid.return_value = getuid.side_effect = None geteuid.return_value = geteuid.side_effect = None getegid.return_value = 0 getgid.return_value = 0 setuid.side_effect = raise_on_second_call with self.assertRaises(AssertionError): maybe_drop_privileges(gid='group') getuid.reset_mock() geteuid.reset_mock() setuid.reset_mock() getuid.side_effect = geteuid.side_effect = None def raise_on_second_call(*args, **kwargs): setuid.side_effect = OSError() setuid.side_effect.errno = errno.ENOENT setuid.side_effect = raise_on_second_call with self.assertRaises(OSError): maybe_drop_privileges(uid='user')
def test_with_uid(self, initgroups, setuid, setgid, getpwuid, parse_uid): class pw_struct(object): pw_gid = 50001 def raise_on_second_call(*args, **kwargs): setuid.side_effect = OSError() setuid.side_effect.errno = errno.EPERM setuid.side_effect = raise_on_second_call getpwuid.return_value = pw_struct() parse_uid.return_value = 5001 maybe_drop_privileges(uid="user") parse_uid.assert_called_with("user") getpwuid.assert_called_with(5001) setgid.assert_called_with(50001) initgroups.assert_called_with(5001, 50001) setuid.assert_has_calls([call(5001), call(0)])
def test_with_uid(self, initgroups, setuid, setgid, getpwuid, parse_uid): class pw_struct(object): pw_gid = 50001 def raise_on_second_call(*args, **kwargs): setuid.side_effect = OSError() setuid.side_effect.errno = errno.EPERM setuid.side_effect = raise_on_second_call getpwuid.return_value = pw_struct() parse_uid.return_value = 5001 maybe_drop_privileges(uid='user') parse_uid.assert_called_with('user') getpwuid.assert_called_with(5001) setgid.assert_called_with(50001) initgroups.assert_called_with(5001, 50001) setuid.assert_has_calls([call(5001), call(0)])
def run(self, hostname=None, pool_cls=None, loglevel=None, app=None, uid=None, gid=None, **kwargs): maybe_drop_privileges(uid=uid, gid=gid) # Pools like eventlet/gevent needs to patch libs as early # as possible. pool_cls = (concurrency.get_implementation(pool_cls) or self.app.conf.CELERYD_POOL) if self.app.IS_WINDOWS and kwargs.get('beat'): self.die('-B option does not work on Windows. ' 'Please run celery beat as a separate service.') hostname = self.simple_format(hostname) if loglevel: try: loglevel = mlevel(loglevel) except KeyError: # pragma: no cover self.die('Unknown level {0!r}. Please use one of {1}.'.format( loglevel, '|'.join( l for l in LOG_LEVELS if isinstance(l, string_t)))) return self.app.Worker( hostname=hostname, pool_cls=pool_cls, loglevel=loglevel, **kwargs ).start()
def test_with_guid(self, initgroups, setuid, setgid, parse_gid, parse_uid): def raise_on_second_call(*args, **kwargs): setuid.side_effect = OSError() setuid.side_effect.errno = errno.EPERM setuid.side_effect = raise_on_second_call parse_uid.return_value = 5001 parse_gid.return_value = 50001 maybe_drop_privileges(uid="user", gid="group") parse_uid.assert_called_with("user") parse_gid.assert_called_with("group") setgid.assert_called_with(50001) initgroups.assert_called_with(5001, 50001) setuid.assert_has_calls([call(5001), call(0)]) setuid.side_effect = None with self.assertRaises(RuntimeError): maybe_drop_privileges(uid="user", gid="group") setuid.side_effect = OSError() setuid.side_effect.errno = errno.EINVAL with self.assertRaises(OSError): maybe_drop_privileges(uid="user", gid="group")
def test_with_guid(self, initgroups, setuid, setgid, parse_gid, parse_uid): def raise_on_second_call(*args, **kwargs): setuid.side_effect = OSError() setuid.side_effect.errno = errno.EPERM setuid.side_effect = raise_on_second_call parse_uid.return_value = 5001 parse_gid.return_value = 50001 maybe_drop_privileges(uid='user', gid='group') parse_uid.assert_called_with('user') parse_gid.assert_called_with('group') setgid.assert_called_with(50001) initgroups.assert_called_with(5001, 50001) setuid.assert_has_calls([call(5001), call(0)]) setuid.side_effect = None with pytest.raises(SecurityError): maybe_drop_privileges(uid='user', gid='group') setuid.side_effect = OSError() setuid.side_effect.errno = errno.EINVAL with pytest.raises(OSError): maybe_drop_privileges(uid='user', gid='group')
def test_only_gid(self, parse_gid, setgid, setuid): parse_gid.return_value = 50001 maybe_drop_privileges(gid='group') parse_gid.assert_called_with('group') setgid.assert_called_with(50001) setuid.assert_not_called()
def test_on_windows(self): prev, sys.platform = sys.platform, 'win32' try: maybe_drop_privileges() finally: sys.platform = prev
def test_only_gid(self, parse_gid, setgid, setuid): parse_gid.return_value = 50001 maybe_drop_privileges(gid='group') parse_gid.assert_called_with('group') setgid.assert_called_with(50001) self.assertFalse(setuid.called)
def worker(ctx, hostname=None, pool_cls=None, app=None, uid=None, gid=None, loglevel=None, logfile=None, pidfile=None, statedb=None, **kwargs): """Start worker instance. Examples -------- $ celery --app=proj worker -l INFO $ celery -A proj worker -l INFO -Q hipri,lopri $ celery -A proj worker --concurrency=4 $ celery -A proj worker --concurrency=1000 -P eventlet $ celery worker --autoscale=10,0 """ app = ctx.obj.app if ctx.args: try: app.config_from_cmdline(ctx.args, namespace='worker') except (KeyError, ValueError) as e: # TODO: Improve the error messages raise click.UsageError( "Unable to parse extra configuration from command line.\n" f"Reason: {e}", ctx=ctx) if kwargs.get('detach', False): params = ctx.params.copy() params.pop('detach') params.pop('logfile') params.pop('pidfile') params.pop('uid') params.pop('gid') umask = params.pop('umask') workdir = ctx.obj.workdir params.pop('hostname') executable = params.pop('executable') argv = ['-m', 'celery', 'worker'] for arg, value in params.items(): if isinstance(value, bool) and value: argv.append(f'--{arg}') else: if value is not None: argv.append(f'--{arg}') argv.append(str(value)) return detach(sys.executable, argv, logfile=logfile, pidfile=pidfile, uid=uid, gid=gid, umask=umask, workdir=workdir, app=app, executable=executable, hostname=hostname) return maybe_drop_privileges(uid=uid, gid=gid) worker = app.Worker( hostname=hostname, pool_cls=pool_cls, loglevel=loglevel, logfile=logfile, # node format handled by celery.app.log.setup pidfile=node_format(pidfile, hostname), statedb=node_format(statedb, hostname), no_color=ctx.obj.no_color, **kwargs) worker.start() return worker.exitcode
def test_only_gid(self, parse_gid, setgid, setuid): parse_gid.return_value = 50001 maybe_drop_privileges(gid="group") parse_gid.assert_called_with("group") setgid.assert_called_with(50001) self.assertFalse(setuid.called)