def test_without_resource(self):
prev, platforms.resource = platforms.resource, None
try:
with pytest.raises(RuntimeError):
detached()
finally:
platforms.resource = prev
def test_without_resource(self):
prev, platforms.resource = platforms.resource, None
try:
with self.assertRaises(RuntimeError):
detached()
finally:
platforms.resource = prev
def detach(path,
argv,
logfile=None,
pidfile=None,
uid=None,
gid=None,
umask=None,
working_directory=None,
fake=False,
app=None):
fake = 1 if C_FAKEFORK else fake
with detached(logfile,
pidfile,
uid,
gid,
umask,
working_directory,
fake,
after_forkers=False):
try:
os.execv(path, [path] + argv)
except Exception:
if app is None:
from celery import current_app
app = current_app
app.log.setup_logging_subsystem('ERROR', logfile)
logger.critical("Can't exec %r",
' '.join([path] + argv),
exc_info=True)
return EX_FAILURE
def detach(
path,
argv,
logfile=None,
pidfile=None,
uid=None,
gid=None,
umask=None,
working_directory=None,
fake=False,
app=None,
executable=None,
hostname=None,
):
hostname = default_nodename(hostname)
logfile = node_format(logfile, hostname)
pidfile = node_format(pidfile, hostname)
fake = 1 if C_FAKEFORK else fake
with detached(logfile, pidfile, uid, gid, umask, working_directory, fake, after_forkers=False):
try:
if executable is not None:
path = executable
os.execv(path, [path] + argv)
except Exception:
if app is None:
from celery import current_app
app = current_app
app.log.setup_logging_subsystem("ERROR", logfile, hostname=hostname)
logger.critical("Can't exec %r", " ".join([path] + argv), exc_info=True)
return EX_FAILURE
def run_evcam(self,
camera,
logfile=None,
pidfile=None,
uid=None,
gid=None,
umask=None,
working_directory=None,
detach=False,
**kwargs):
from celery.events.snapshot import evcam
workdir = working_directory
self.set_process_status('cam')
kwargs['app'] = self.app
cam = partial(evcam,
camera,
logfile=logfile,
pidfile=pidfile,
**kwargs)
if detach:
with detached(logfile, pidfile, uid, gid, umask, workdir):
return cam()
else:
return cam()
def beat(ctx,
detach=False,
logfile=None,
pidfile=None,
uid=None,
gid=None,
umask=None,
workdir=None,
**kwargs):
"""Start the beat periodic task scheduler."""
app = ctx.obj.app
if ctx.args:
try:
app.config_from_cmdline(ctx.args)
except (KeyError, ValueError) as e:
# TODO: Improve the error messages
raise click.UsageError(
"Unable to parse extra configuration"
" from command line.\n"
f"Reason: {e}",
ctx=ctx)
if not detach:
maybe_drop_privileges(uid=uid, gid=gid)
beat = partial(app.Beat, logfile=logfile, pidfile=pidfile, **kwargs)
if detach:
with detached(logfile, pidfile, uid, gid, umask, workdir):
return beat().run()
else:
return beat().run()
def test_default(self, open, geteuid, maybe_drop, signals, pidlock):
geteuid.return_value = 0
context = detached(uid="user", gid="group")
self.assertIsInstance(context, DaemonContext)
signals.reset.assert_called_with("SIGCLD")
maybe_drop.assert_called_with(uid="user", gid="group")
open.return_value = Mock()
geteuid.return_value = 5001
context = detached(uid="user", gid="group", logfile="/foo/bar")
self.assertIsInstance(context, DaemonContext)
open.assert_called_with("/foo/bar", "a")
open.return_value.close.assert_called_with()
context = detached(pidfile="/foo/bar/pid")
self.assertIsInstance(context, DaemonContext)
pidlock.assert_called_with("/foo/bar/pid")
def test_default(self, open, geteuid, maybe_drop, signals, pidlock):
geteuid.return_value = 0
context = detached(uid="user", gid="group")
self.assertIsInstance(context, DaemonContext)
signals.reset.assert_called_with("SIGCLD")
maybe_drop.assert_called_with(uid="user", gid="group")
open.return_value = Mock()
geteuid.return_value = 5001
context = detached(uid="user", gid="group", logfile="/foo/bar")
self.assertIsInstance(context, DaemonContext)
open.assert_called_with("/foo/bar", "a")
open.return_value.close.assert_called_with()
context = detached(pidfile="/foo/bar/pid")
self.assertIsInstance(context, DaemonContext)
pidlock.assert_called_with("/foo/bar/pid")
def test_default(self, open, geteuid, maybe_drop, signals, pidlock):
geteuid.return_value = 0
context = detached(uid='user', gid='group')
self.assertIsInstance(context, DaemonContext)
signals.reset.assert_called_with('SIGCLD')
maybe_drop.assert_called_with(uid='user', gid='group')
open.return_value = Mock()
geteuid.return_value = 5001
context = detached(uid='user', gid='group', logfile='/foo/bar')
self.assertIsInstance(context, DaemonContext)
open.assert_called_with('/foo/bar', 'a')
open.return_value.close.assert_called_with()
context = detached(pidfile='/foo/bar/pid')
self.assertIsInstance(context, DaemonContext)
pidlock.assert_called_with('/foo/bar/pid')
def test_default(self, open, geteuid, maybe_drop, signals, pidlock):
geteuid.return_value = 0
context = detached(uid='user', gid='group')
self.assertIsInstance(context, DaemonContext)
signals.reset.assert_called_with('SIGCLD')
maybe_drop.assert_called_with(uid='user', gid='group')
open.return_value = Mock()
geteuid.return_value = 5001
context = detached(uid='user', gid='group', logfile='/foo/bar')
self.assertIsInstance(context, DaemonContext)
open.assert_called_with('/foo/bar', 'a')
open.return_value.close.assert_called_with()
context = detached(pidfile='/foo/bar/pid')
self.assertIsInstance(context, DaemonContext)
pidlock.assert_called_with('/foo/bar/pid')
def detach(path, argv, logfile=None, pidfile=None, uid=None,
gid=None, umask=0, working_directory=None, fake=False, ):
with detached(logfile, pidfile, uid, gid, umask, working_directory, fake):
try:
os.execv(path, [path] + argv)
except Exception:
from celery import current_app
logger = current_app.log.setup_logger("ERROR", logfile)
logger.critical("Can't exec %r", " ".join([path] + argv),
exc_info=True)
def _detach(self,
logfile=None,
pidfile=None,
uid=None,
gid=None,
umask=None,
working_directory=None,
**kwargs):
print('detaching... [pidfile=%s logfile=%s]' % (pidfile, logfile))
with detached(logfile, pidfile, uid, gid, umask, working_directory):
return self._start(pidfile=pidfile)
def detach(path, argv, logfile=None, pidfile=None, uid=None,
gid=None, umask=0, working_directory=None):
with detached(logfile, pidfile, uid, gid, umask, working_directory):
try:
os.execv(path, [path] + argv)
except Exception:
import logging
from ..log import setup_logger
logger = setup_logger(logfile=logfile, loglevel=logging.ERROR)
logger.critical("Can't exec %r", " ".join([path] + argv),
exc_info=sys.exc_info())
def test_with_locked_pidfile(self, open, geteuid, maybe_drop, signals,
pidlock, Pidfile):
context = detached(pidfile='/foo/bar/pid')
p = Pidfile.return_value
p.is_locked.return_value = True
assert isinstance(context, DaemonContext)
assert context.after_chdir
with pytest.raises(SystemExit):
context.after_chdir()
Pidfile.assert_called_with('/foo/bar/pid')
p.is_locked.assert_called_once()
pidlock.assert_not_called()
def run(self, detach=False, logfile=None, pidfile=None, uid=None,
gid=None, umask=None, working_directory=None, **kwargs):
kwargs.pop("app", None)
beat = partial(self.app.Beat,
logfile=logfile, pidfile=pidfile, **kwargs)
workdir = working_directory
if detach:
with detached(logfile, pidfile, uid, gid, umask, workdir):
return beat().run()
else:
return beat().run()
def detach(path, argv, logfile=None, pidfile=None, uid=None,
gid=None, umask=0, working_directory=None, fake=False, ):
fake = 1 if C_FAKEFORK else fake
with detached(logfile, pidfile, uid, gid, umask, working_directory, fake):
try:
os.execv(path, [path] + argv)
except Exception:
from celery import current_app
current_app.log.setup_logging_subsystem('ERROR', logfile)
logger.critical("Can't exec %r", ' '.join([path] + argv),
exc_info=True)
return EX_FAILURE
def run(self, detach=False, logfile=None, pidfile=None, uid=None,
gid=None, umask=None, working_directory=None, **kwargs):
workdir = working_directory
kwargs.pop("app", None)
beat = partial(self.app.Beat,
logfile=logfile, pidfile=pidfile, **kwargs)
if detach:
with detached(logfile, pidfile, uid, gid, umask, workdir):
return beat().run()
else:
return beat().run()
def run(self, detach=False, logfile=None, pidfile=None, uid=None,
gid=None, umask=None, workdir=None, **kwargs):
if not detach:
maybe_drop_privileges(uid=uid, gid=gid)
kwargs.pop('app', None)
beat = partial(self.app.Beat,
logfile=logfile, pidfile=pidfile, **kwargs)
if detach:
with detached(logfile, pidfile, uid, gid, umask, workdir):
return beat().run()
else:
return beat().run()
def run(self, detach=False, logfile=None, pidfile=None, uid=None,
gid=None, umask=None, workdir=None, **kwargs):
if not detach:
maybe_drop_privileges(uid=uid, gid=gid)
kwargs.pop('app', None)
beat = partial(self.app.Beat,
logfile=logfile, pidfile=pidfile, **kwargs)
if detach:
with detached(logfile, pidfile, uid, gid, umask, workdir):
return beat().run()
else:
return beat().run()
def run_evcam(self, camera, logfile=None, pidfile=None, uid=None,
gid=None, umask=None, workdir=None,
detach=False, **kwargs):
from celery.events.snapshot import evcam
self.set_process_status('cam')
kwargs['app'] = self.app
cam = partial(evcam, camera,
logfile=logfile, pidfile=pidfile, **kwargs)
if detach:
with detached(logfile, pidfile, uid, gid, umask, workdir):
return cam()
else:
return cam()
def test_default(self, open, geteuid, maybe_drop, signals, pidlock,
Pidfile):
geteuid.return_value = 0
context = detached(uid='user', gid='group')
assert isinstance(context, DaemonContext)
signals.reset.assert_called_with('SIGCLD')
maybe_drop.assert_called_with(uid='user', gid='group')
open.return_value = Mock()
geteuid.return_value = 5001
context = detached(uid='user', gid='group', logfile='/foo/bar')
assert isinstance(context, DaemonContext)
assert context.after_chdir
context.after_chdir()
open.assert_called_with('/foo/bar', 'a')
open.return_value.close.assert_called_with()
context = detached(pidfile='/foo/bar/pid')
assert isinstance(context, DaemonContext)
assert context.after_chdir
context.after_chdir()
Pidfile.assert_called_with('/foo/bar/pid')
Pidfile.return_value.acquire.assert_not_called()
pidlock.assert_not_called()
def run(self, loglevel='ERROR', logfile=None, http_port=8989,
http_address='', app=None, detach=False, pidfile=PID_FILE,
uid=None, gid=None, umask=None, working_directory=None, **kwargs):
print(('celerymon %s is starting.' % self.version))
app = self.app
workdir = working_directory
# Setup logging
if not isinstance(loglevel, int):
loglevel = LOG_LEVELS[loglevel.upper()]
# Dump configuration to screen so we have some basic information
# when users sends e-mails.
print((STARTUP_INFO_FMT % {
'http_port': http_port,
'http_address': http_address or 'localhost',
'conninfo': app.broker_connection().as_uri(),
}))
print('celerymon has started.')
set_process_title('celerymon', info=strargv(sys.argv))
def _run_monitor():
create_pidlock(pidfile)
app.log.setup_logging_subsystem(loglevel=loglevel,
logfile=logfile)
logger = app.log.get_default_logger(name='celery.mon')
monitor = MonitorService(logger=logger,
http_port=http_port,
http_address=http_address)
try:
monitor.start()
except Exception as exc:
logger.error('celerymon raised exception %r',
exc, exc_info=True)
except KeyboardInterrupt:
pass
if detach:
with detached(logfile, pidfile, uid, gid, umask, workdir):
_run_monitor()
else:
_run_monitor()
def detach(path,
argv,
logfile=None,
pidfile=None,
uid=None,
gid=None,
umask=None,
workdir=None,
fake=False,
app=None,
executable=None,
hostname=None):
"""Detach program by argv."""
fake = 1 if C_FAKEFORK else fake
# `detached()` will attempt to touch the logfile to confirm that error
# messages won't be lost after detaching stdout/err, but this means we need
# to pre-format it rather than relying on `setup_logging_subsystem()` like
# we can elsewhere.
logfile = node_format(logfile, hostname)
with detached(logfile,
pidfile,
uid,
gid,
umask,
workdir,
fake,
after_forkers=False):
try:
if executable is not None:
path = executable
os.execv(path, [path] + argv)
return EX_OK
except Exception: # pylint: disable=broad-except
if app is None:
from celery import current_app
app = current_app
app.log.setup_logging_subsystem('ERROR',
logfile,
hostname=hostname)
logger.critical("Can't exec %r",
' '.join([path] + argv),
exc_info=True)
return EX_FAILURE
def detach(
path,
argv,
logfile=None,
pidfile=None,
uid=None,
gid=None,
umask=None,
workdir=None,
fake=False,
app=None,
executable=None,
hostname=None,
):
"""Detach program by argv'."""
hostname = default_nodename(hostname)
logfile = node_format(logfile, hostname)
pidfile = node_format(pidfile, hostname)
fake = 1 if C_FAKEFORK else fake
with detached(logfile,
pidfile,
uid,
gid,
umask,
workdir,
fake,
after_forkers=False):
try:
if executable is not None:
path = executable
os.execv(path, [path] + argv)
except Exception: # pylint: disable=broad-except
if app is None:
from celery import current_app
app = current_app
app.log.setup_logging_subsystem("ERROR",
logfile,
hostname=hostname)
logger.critical("Can't exec %r",
" ".join([path] + argv),
exc_info=True)
return EX_FAILURE
def detach(
path,
argv,
logfile=None,
pidfile=None,
uid=None,
gid=None,
umask=0,
working_directory=None,
fake=False,
):
with detached(logfile, pidfile, uid, gid, umask, working_directory, fake):
try:
os.execv(path, [path] + argv)
except Exception:
from celery import current_app
current_app.log.setup_logging_subsystem('ERROR', logfile)
logger.critical("Can't exec %r",
' '.join([path] + argv),
exc_info=True)
return EX_FAILURE
def detach(path, argv, logfile=None, pidfile=None, uid=None,
gid=None, umask=None, workdir=None, fake=False, app=None,
executable=None, hostname=None):
"""Detach program by argv'."""
hostname = default_nodename(hostname)
logfile = node_format(logfile, hostname)
pidfile = node_format(pidfile, hostname)
fake = 1 if C_FAKEFORK else fake
with detached(logfile, pidfile, uid, gid, umask, workdir, fake,
after_forkers=False):
try:
if executable is not None:
path = executable
os.execv(path, [path] + argv)
except Exception: # pylint: disable=broad-except
if app is None:
from celery import current_app
app = current_app
app.log.setup_logging_subsystem(
'ERROR', logfile, hostname=hostname)
logger.critical("Can't exec %r", ' '.join([path] + argv),
exc_info=True)
return EX_FAILURE
def run(self, loglevel="ERROR", logfile=None, http_port=12201,
http_address='', app=None, detach=False, pidfile=None,
uid=None, gid=None, umask=None, working_directory=None,
plugins=None, storage=None, debug=False, **kwargs):
print("{0} {1} is starting.".format(self.namespace, self.version, ))
app = self.app
workdir = working_directory
# Setup logging
if not isinstance(loglevel, int):
loglevel = LOG_LEVELS[loglevel.upper()]
print("")
print("Using below's plugins.")
pprint(plugins)
# debug server
# print(STARTUP_INFO_FMT % {
# "http_port": http_port,
# "http_address": http_address or "localhost",
# "conninfo": app.broker_connection().as_uri(),
# })
print("")
print("{0} has started.".format(self.namespace))
set_process_title("{0}".format(self.namespace), info=strargv(sys.argv))
if detach:
with detached(logfile, pidfile, uid, gid, umask, workdir):
self._run_tracker(
loglevel, logfile, http_port,
http_address, plugins=plugins, storage=storage)
else:
self._run_tracker(
loglevel, logfile, http_port,
http_address, plugins=plugins, storage=storage)
logger = app.log.get_default_logger(name='celery.mon')
monitor = MonitorService(logger=logger,
http_port=http_port,
http_address=http_address)
try:
monitor.start()
except Exception, exc:
logger.error('celerymon raised exception %r',
exc,
exc_info=True)
except KeyboardInterrupt:
pass
if detach:
with detached(logfile, pidfile, uid, gid, umask, workdir):
_run_monitor()
else:
_run_monitor()
def prepare_preload_options(self, options):
workdir = options.get('working_directory')
if workdir:
os.chdir(workdir)
def get_options(self):
conf = self.app.conf
return (Option(
'-l',
'--loglevel',
default=conf.CELERYMON_LOG_LEVEL,
logfile=logfile)
logger = app.log.get_default_logger(name='celery.mon')
monitor = MonitorService(logger=logger,
http_port=http_port,
http_address=http_address)
try:
monitor.start()
except Exception, exc:
logger.error('celerymon raised exception %r',
exc, exc_info=True)
except KeyboardInterrupt:
pass
if detach:
with detached(logfile, pidfile, uid, gid, umask, workdir):
_run_monitor()
else:
_run_monitor()
def prepare_preload_options(self, options):
workdir = options.get('working_directory')
if workdir:
os.chdir(workdir)
def get_options(self):
conf = self.app.conf
return (
Option('-l', '--loglevel',
default=conf.CELERYMON_LOG_LEVEL,
help='Choose between DEBUG/INFO/WARNING/ERROR/CRITICAL.'),
def run(self, **kwargs):
sys.argv[1] = 'celeryd'
with detached(kwargs['logfile'],
kwargs['pidfile']):
os.execv(sys.argv[0], sys.argv)
def maybe_detached(*, detach, **kwargs):
return detached(**kwargs) if detach else nullcontext()
def _detach(self, logfile=None, pidfile=None, uid=None, gid=None,
umask=None, working_directory=None, **kwargs):
print("detaching... [pidfile=%s logfile=%s]" % (pidfile, logfile))
with detached(logfile, pidfile, uid, gid, umask, working_directory):
return self._start(pidfile=pidfile)