def evcam(camera,
freq=1.0,
maxrate=None,
loglevel=0,
logfile=None,
pidfile=None,
timer=None,
app=None):
app = app_or_default(app)
if pidfile:
platforms.create_pidlock(pidfile)
app.log.setup_logging_subsystem(loglevel, logfile)
print('-> evcam: Taking snapshots with {0} (every {1} secs.)'.format(
camera, freq))
state = app.events.State()
cam = instantiate(camera,
state,
app=app,
freq=freq,
maxrate=maxrate,
timer=timer)
cam.install()
conn = app.connection_for_read()
recv = app.events.Receiver(conn, handlers={'*': state.event})
try:
try:
recv.capture(limit=None)
except KeyboardInterrupt:
raise SystemExit
finally:
cam.cancel()
conn.close()
def start_scheduler(self):
c = self.colored
if self.pidfile:
platforms.create_pidlock(self.pidfile)
beat = self.Service(app=self.app,
max_interval=self.max_interval,
scheduler_cls=self.scheduler_cls,
schedule_filename=self.schedule)
print(str(c.blue('__ ', c.magenta('-'),
c.blue(' ... __ '), c.magenta('-'),
c.blue(' _\n'),
c.reset(self.startup_info(beat)))))
self.setup_logging()
if self.socket_timeout:
logger.debug('Setting default socket timeout to %r',
self.socket_timeout)
socket.setdefaulttimeout(self.socket_timeout)
try:
self.install_sync_handler(beat)
beat.start()
except Exception, exc:
logger.critical('celerybeat raised exception %s: %r',
exc.__class__, exc,
exc_info=True)
def start_scheduler(self):
c = self.colored
if self.pidfile:
platforms.create_pidlock(self.pidfile)
beat = self.Service(
app=self.app,
max_interval=self.max_interval,
scheduler_cls=self.scheduler_cls,
schedule_filename=self.schedule,
)
print(
text_t( # noqa (pyflakes chokes on print)
c.blue(
"__ ",
c.magenta("-"),
c.blue(" ... __ "),
c.magenta("-"),
c.blue(" _\n"),
c.reset(self.startup_info(beat)),
)
)
)
self.setup_logging()
if self.socket_timeout:
logger.debug("Setting default socket timeout to %r", self.socket_timeout)
socket.setdefaulttimeout(self.socket_timeout)
try:
self.install_sync_handler(beat)
beat.start()
except Exception as exc:
logger.critical("beat raised exception %s: %r", exc.__class__, exc, exc_info=True)
def evcam(camera, freq=1.0, maxrate=None, loglevel=0,
logfile=None, pidfile=None, timer=None, app=None):
"""Start snapshot recorder."""
app = app_or_default(app)
if pidfile:
platforms.create_pidlock(pidfile)
app.log.setup_logging_subsystem(loglevel, logfile)
print('-> evcam: Taking snapshots with {0} (every {1} secs.)'.format(
camera, freq))
state = app.events.State()
cam = instantiate(camera, state, app=app, freq=freq,
maxrate=maxrate, timer=timer)
cam.install()
conn = app.connection_for_read()
recv = app.events.Receiver(conn, handlers={'*': state.event})
try:
try:
recv.capture(limit=None)
except KeyboardInterrupt:
raise SystemExit
finally:
cam.cancel()
conn.close()
def start_scheduler(self):
if self.pidfile:
platforms.create_pidlock(self.pidfile)
service = self.Service(
app=self.app,
max_interval=self.max_interval,
scheduler_cls=self.scheduler_cls,
schedule_filename=self.schedule,
)
print(self.banner(service))
self.setup_logging()
if self.socket_timeout:
logger.debug('Setting default socket timeout to %r',
self.socket_timeout)
socket.setdefaulttimeout(self.socket_timeout)
try:
self.install_sync_handler(service)
service.start()
except Exception as exc:
logger.critical('beat raised exception %s: %r',
exc.__class__, exc,
exc_info=True)
raise
def evcam(camera, freq=1.0, maxrate=None, loglevel=0,
logfile=None, pidfile=None, timer=None, app=None):
app = app_or_default(app)
if pidfile:
pidlock = platforms.create_pidlock(pidfile).acquire()
atexit.register(pidlock.release)
if not isinstance(loglevel, int):
loglevel = LOG_LEVELS[loglevel.upper()]
logger = app.log.setup_logger(loglevel=loglevel,
logfile=logfile,
name="celery.evcam")
if pidfile:
pidlock = platforms.create_pidlock(pidfile).acquire()
atexit.register(pidlock.release)
logger.info(
"-> evcam: Taking snapshots with %s (every %s secs.)\n" % (
camera, freq))
state = app.events.State()
cam = instantiate(camera, state, app=app,
freq=freq, maxrate=maxrate, logger=logger,
timer=timer)
cam.install()
conn = app.broker_connection()
recv = app.events.Receiver(conn, handlers={"*": state.event})
try:
try:
recv.capture(limit=None)
except KeyboardInterrupt:
raise SystemExit
finally:
cam.cancel()
conn.close()
def start_scheduler(self):
c = self.colored
if self.pidfile:
platforms.create_pidlock(self.pidfile)
beat = self.Service(app=self.app,
max_interval=self.max_interval,
scheduler_cls=self.scheduler_cls,
schedule_filename=self.schedule)
print(
text_t( # noqa (pyflakes chokes on print)
c.blue('__ ', c.magenta('-'), c.blue(' ... __ '),
c.magenta('-'), c.blue(' _\n'),
c.reset(self.startup_info(beat))), ))
self.setup_logging()
if self.socket_timeout:
logger.debug('Setting default socket timeout to %r',
self.socket_timeout)
socket.setdefaulttimeout(self.socket_timeout)
try:
self.install_sync_handler(beat)
beat.start()
except Exception as exc:
logger.critical('beat raised exception %s: %r',
exc.__class__,
exc,
exc_info=True)
raise exc
def start_scheduler(self):
c = self.colored
if self.pidfile:
platforms.create_pidlock(self.pidfile)
beat = self.Service(app=self.app,
max_interval=self.max_interval,
scheduler_cls=self.scheduler_cls,
schedule_filename=self.schedule)
print(
str(
c.blue("__ ", c.magenta("-"), c.blue(" ... __ "),
c.magenta("-"), c.blue(" _\n"),
c.reset(self.startup_info(beat)))))
if self.socket_timeout:
logger.debug("Setting default socket timeout to %r",
self.socket_timeout)
socket.setdefaulttimeout(self.socket_timeout)
try:
self.install_sync_handler(beat)
beat.start()
except Exception, exc:
logger.critical("celerybeat raised exception %s: %r",
exc.__class__,
exc,
exc_info=True)
def start_scheduler(self):
if self.pidfile:
platforms.create_pidlock(self.pidfile)
service = self.Service(
app=self.app,
max_interval=self.max_interval,
scheduler_cls=self.scheduler_cls,
schedule_filename=self.schedule,
)
print(self.banner(service))
self.setup_logging()
if self.socket_timeout:
logger.debug('Setting default socket timeout to %r',
self.socket_timeout)
socket.setdefaulttimeout(self.socket_timeout)
try:
self.install_sync_handler(service)
service.start()
except Exception as exc:
logger.critical('beat raised exception %s: %r',
exc.__class__,
exc,
exc_info=True)
raise
def evcam(camera, freq=1.0, maxrate=None, loglevel=0,
logfile=None, pidfile=None, timer=None, app=None):
app = app_or_default(app)
if pidfile:
platforms.create_pidlock(pidfile)
app.log.setup_logging_subsystem(loglevel, logfile)
logger.info(
"-> evcam: Taking snapshots with %s (every %s secs.)\n" % (
camera, freq))
state = app.events.State()
cam = instantiate(camera, state, app=app, freq=freq,
maxrate=maxrate, timer=timer)
cam.install()
conn = app.broker_connection()
recv = app.events.Receiver(conn, handlers={"*": state.event})
try:
try:
recv.capture(limit=None)
except KeyboardInterrupt:
raise SystemExit
finally:
cam.cancel()
conn.close()
def test_create_pidlock(self, PIDFile):
p = PIDFile.return_value = Mock()
p.is_locked.return_value = True
p.remove_if_stale.return_value = False
with self.assertRaises(SystemExit):
create_pidlock('/var/pid')
p.remove_if_stale.return_value = True
ret = create_pidlock('/var/pid')
self.assertIs(ret, p)
def test_create_pidlock(self, PIDFile):
p = PIDFile.return_value = Mock()
p.is_locked.return_value = True
p.remove_if_stale.return_value = False
with self.assertRaises(SystemExit):
create_pidlock("/var/pid")
p.remove_if_stale.return_value = True
ret = create_pidlock("/var/pid")
self.assertIs(ret, p)
def test_create_pidlock(self, Pidfile):
p = Pidfile.return_value = Mock()
p.is_locked.return_value = True
with mock.stdouts() as (_, err):
with pytest.raises(SystemExit):
create_pidlock('/var/pid')
assert 'already exists' in err.getvalue()
p.is_locked.return_value = False
ret = create_pidlock('/var/pid')
assert ret is p
def run_worker(self):
if self.pidfile:
platforms.create_pidlock(self.pidfile)
worker = self.WorkController(app=self.app,
hostname=self.hostname,
ready_callback=self.on_consumer_ready, beat=self.beat,
autoscale=self.autoscale, autoreload=self.autoreload,
no_execv=self.no_execv,
**self.confopts_as_dict())
self.install_platform_tweaks(worker)
signals.worker_init.send(sender=worker)
worker.start()
def test_create_pidlock(self, Pidfile):
p = Pidfile.return_value = Mock()
p.is_locked.return_value = True
p.remove_if_stale.return_value = False
with override_stdouts() as (_, err):
with self.assertRaises(SystemExit):
create_pidlock('/var/pid')
self.assertIn('already exists', err.getvalue())
p.remove_if_stale.return_value = True
ret = create_pidlock('/var/pid')
self.assertIs(ret, p)
def test_create_pidlock(self, Pidfile):
p = Pidfile.return_value = Mock()
p.is_locked.return_value = True
p.remove_if_stale.return_value = False
with mock.stdouts() as (_, err):
with pytest.raises(SystemExit):
create_pidlock('/var/pid')
assert 'already exists' in err.getvalue()
p.remove_if_stale.return_value = True
ret = create_pidlock('/var/pid')
assert ret is p
def test_create_pidlock(self, Pidfile):
p = Pidfile.return_value = Mock()
p.is_locked.return_value = True
p.remove_if_stale.return_value = False
with override_stdouts() as (_, err):
with self.assertRaises(SystemExit):
create_pidlock('/var/pid')
self.assertIn('already exists', err.getvalue())
p.remove_if_stale.return_value = True
ret = create_pidlock('/var/pid')
self.assertIs(ret, p)
def _run_monitor():
create_pidlock(pidfile)
app.log.setup_logging_subsystem(loglevel=loglevel, logfile=logfile)
logger = app.log.get_default_logger(name='celery.mon')
monitor = MonitorService(logger=logger,
http_port=http_port,
http_address=http_address)
try:
monitor.start()
except Exception, exc:
logger.error('celerymon raised exception %r',
exc,
exc_info=True)
def _run_monitor():
create_pidlock(pidfile)
app.log.setup_logging_subsystem(loglevel=loglevel,
logfile=logfile)
logger = app.log.get_default_logger(name='celery.mon')
monitor = MonitorService(logger=logger,
http_port=http_port,
http_address=http_address)
try:
monitor.start()
except Exception, exc:
logger.error('celerymon raised exception %r',
exc, exc_info=True)
def start_scheduler(self, logger=None):
c = self.colored
if self.pidfile:
pidlock = platforms.create_pidlock(self.pidfile).acquire()
atexit.register(pidlock.release)
beat = self.Service(app=self.app,
logger=logger,
max_interval=self.max_interval,
scheduler_cls=self.scheduler_cls,
schedule_filename=self.schedule)
print(str(c.blue("__ ", c.magenta("-"),
c.blue(" ... __ "), c.magenta("-"),
c.blue(" _\n"),
c.reset(self.startup_info(beat)))))
if self.socket_timeout:
logger.debug("Setting default socket timeout to %r" % (
self.socket_timeout))
socket.setdefaulttimeout(self.socket_timeout)
try:
self.install_sync_handler(beat)
beat.start()
except Exception, exc:
logger.critical("celerybeat raised exception %s: %r\n%s" % (
exc.__class__, exc, traceback.format_exc()),
exc_info=sys.exc_info())
def start(self):
"""Starts the workers main loop."""
self.on_start()
self._state = self.RUN
if self.pidfile:
self.pidlock = platforms.create_pidlock(self.pidfile)
try:
for i, component in enumerate(self.components):
logger.debug('Starting %s...', qualname(component))
self._running = i + 1
if component:
component.start()
logger.debug('%s OK!', qualname(component))
except SystemTerminate:
self.terminate()
except Exception as exc:
logger.error('Unrecoverable error: %r', exc, exc_info=True)
self.stop()
except (KeyboardInterrupt, SystemExit):
self.stop()
try:
# Will only get here if running green,
# makes sure all greenthreads have exited.
self._shutdown_complete.wait()
except IGNORE_ERRORS:
pass
def start(self):
"""Starts the workers main loop."""
self.on_start()
self._state = self.RUN
if self.pidfile:
self.pidlock = platforms.create_pidlock(self.pidfile)
try:
for i, component in enumerate(self.components):
logger.debug('Starting %s...', qualname(component))
self._running = i + 1
if component:
component.start()
logger.debug('%s OK!', qualname(component))
except SystemTerminate:
self.terminate()
except Exception as exc:
logger.error('Unrecoverable error: %r', exc,
exc_info=True)
self.stop()
except (KeyboardInterrupt, SystemExit):
self.stop()
try:
# Will only get here if running green,
# makes sure all greenthreads have exited.
self._shutdown_complete.wait()
except IGNORE_ERRORS:
pass
def start_scheduler(self, logger=None):
c = self.colored
if self.pidfile:
pidlock = platforms.create_pidlock(self.pidfile).acquire()
atexit.register(pidlock.release)
beat = self.Service(app=self.app,
logger=logger,
max_interval=self.max_interval,
scheduler_cls=self.scheduler_cls,
schedule_filename=self.schedule)
print(str(c.blue("__ ", c.magenta("-"),
c.blue(" ... __ "), c.magenta("-"),
c.blue(" _\n"),
c.reset(self.startup_info(beat)))))
if self.socket_timeout:
logger.debug("Setting default socket timeout to %r" % (
self.socket_timeout))
socket.setdefaulttimeout(self.socket_timeout)
try:
self.install_sync_handler(beat)
beat.start()
except Exception, exc:
logger.critical("celerybeat raised exception %s: %r\n%s" % (
exc.__class__, exc, traceback.format_exc()),
exc_info=sys.exc_info())
def evcam(camera,
freq=1.0,
maxrate=None,
loglevel=0,
logfile=None,
pidfile=None,
timer=None,
app=None):
app = app_or_default(app)
if pidfile:
pidlock = platforms.create_pidlock(pidfile).acquire()
atexit.register(pidlock.release)
if not isinstance(loglevel, int):
loglevel = LOG_LEVELS[loglevel.upper()]
logger = app.log.setup_logger(loglevel=loglevel,
logfile=logfile,
name="celery.evcam")
if pidfile:
pidlock = platforms.create_pidlock(pidfile).acquire()
atexit.register(pidlock.release)
logger.info("-> evcam: Taking snapshots with %s (every %s secs.)\n" %
(camera, freq))
state = app.events.State()
cam = instantiate(camera,
state,
app=app,
freq=freq,
maxrate=maxrate,
logger=logger,
timer=timer)
cam.install()
conn = app.broker_connection()
recv = app.events.Receiver(conn, handlers={"*": state.event})
try:
try:
recv.capture(limit=None)
except KeyboardInterrupt:
raise SystemExit
finally:
cam.cancel()
conn.close()
def _start(self, pidfile=None, **kwargs):
self.setup_logging(logfile=self.logfile, loglevel=self.loglevel)
self.set_process_title('boot')
self.install_signal_handlers()
if pidfile:
pidlock = create_pidlock(pidfile).acquire()
atexit.register(pidlock.release)
try:
return self.branch.start().wait()
except SystemExit:
self.branch.stop()
def _start(self, pidfile=None, **kwargs):
self.setup_logging(logfile=self.logfile, loglevel=self.loglevel)
self.set_process_title("boot")
self.install_signal_handlers()
if pidfile:
pidlock = create_pidlock(pidfile).acquire()
atexit.register(pidlock.release)
try:
return self.branch.start().wait()
except SystemExit:
self.branch.stop()
def _run_monitor():
create_pidlock(pidfile)
logger = logging.getLogger('celery_stalker')
log_handler = logging.FileHandler(logfile)
formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
log_handler.setFormatter(formatter)
logger.addHandler(log_handler)
logger.setLevel(logging.INFO)
monitor = MonitorService(logger=logger)
try:
monitor.start()
except Exception, exc:
logger.error('celery_stalker raised exception %r',
exc, exc_info=True)
def run_worker(self):
if self.pidfile:
pidlock = platforms.create_pidlock(self.pidfile).acquire()
atexit.register(pidlock.release)
worker = self.WorkController(app=self.app,
hostname=self.hostname,
ready_callback=self.on_consumer_ready,
embed_clockservice=self.embed_clockservice,
autoscale=self.autoscale,
autoreload=self.autoreload,
**self.confopts_as_dict())
self.install_platform_tweaks(worker)
signals.worker_init.send(sender=worker)
worker.start()
def run_worker(self):
if self.pidfile:
pidlock = platforms.create_pidlock(self.pidfile).acquire()
atexit.register(pidlock.release)
worker = self.WorkController(
app=self.app,
hostname=self.hostname,
ready_callback=self.on_consumer_ready,
embed_clockservice=self.embed_clockservice,
autoscale=self.autoscale,
autoreload=self.autoreload,
**self.confopts_as_dict())
self.install_platform_tweaks(worker)
signals.worker_init.send(sender=worker)
worker.start()
def start(self):
"""Starts the workers main loop."""
self._state = self.RUN
if self.pidfile:
self.pidlock = platforms.create_pidlock(self.pidfile)
try:
for i, component in enumerate(self.components):
logger.debug("Starting %s...", qualname(component))
self._running = i + 1
component.start()
logger.debug("%s OK!", qualname(component))
except SystemTerminate:
self.terminate()
except Exception, exc:
logger.error("Unrecoverable error: %r", exc,
exc_info=True)
self.stop()
def start(self):
"""Starts the workers main loop."""
self._state = self.RUN
if self.pidfile:
self.pidlock = platforms.create_pidlock(self.pidfile)
try:
for i, component in enumerate(self.components):
logger.debug('Starting %s...', qualname(component))
self._running = i + 1
if component:
component.start()
logger.debug('%s OK!', qualname(component))
except SystemTerminate:
self.terminate()
except Exception, exc:
logger.error('Unrecoverable error: %r', exc, exc_info=True)
self.stop()
def run_worker(self):
if self.pidfile:
pidlock = platforms.create_pidlock(self.pidfile).acquire()
atexit.register(pidlock.release)
worker = self.WorkController(concurrency=self.concurrency,
loglevel=self.loglevel,
logfile=self.logfile,
hostname=self.hostname,
ready_callback=self.on_listener_ready,
embed_clockservice=self.run_clockservice,
schedule_filename=self.schedule,
send_events=self.events,
db=self.db,
max_tasks_per_child=self.max_tasks_per_child,
task_time_limit=self.task_time_limit,
task_soft_time_limit=self.task_soft_time_limit)
self.install_platform_tweaks(worker)
worker.start()
def run_worker(self):
if self.pidfile:
pidlock = platforms.create_pidlock(self.pidfile).acquire()
atexit.register(pidlock.release)
worker = self.WorkController(
concurrency=self.concurrency,
loglevel=self.loglevel,
logfile=self.logfile,
hostname=self.hostname,
ready_callback=self.on_listener_ready,
embed_clockservice=self.run_clockservice,
schedule_filename=self.schedule,
scheduler_cls=self.scheduler_cls,
send_events=self.events,
db=self.db,
max_tasks_per_child=self.max_tasks_per_child,
task_time_limit=self.task_time_limit,
task_soft_time_limit=self.task_soft_time_limit)
self.install_platform_tweaks(worker)
worker.start()
def on_start(self):
if self.pidfile:
self.pidlock = platforms.create_pidlock(self.pidfile)
def on_start(self):
if self.pidfile:
self.pidlock = create_pidlock(self.pidfile)
def on_start(self):
if self.pidfile:
self.pidlock = create_pidlock(self.pidfile)
def on_start(self):
if self.pidfile:
self.pidlock = platforms.create_pidlock(self.pidfile)
logger = app.log.get_default_logger(name="celery.mon")
monitor = MonitorService(logger=logger,
http_port=http_port,
http_address=http_address)
try:
monitor.start()
except Exception, exc:
logger.error("celerymon raised exception %r\n%s" % (
exc, traceback.format_exc()))
except KeyboardInterrupt:
pass
if detach:
with detached(logfile, pidfile, uid, gid, umask, workdir):
with create_pidlock(pidfile) as pidlock:
_run_monitor()
else:
_run_monitor()
def prepare_preload_options(self, options):
workdir = options.get("working_directory")
if workdir:
os.chdir(workdir)
def get_options(self):
conf = self.app.conf
return (Option('-l', '--loglevel',
default=conf.CELERYMON_LOG_LEVEL,
action="store", dest="loglevel",
help="Choose between DEBUG/INFO/WARNING/ERROR/CRITICAL."),