def scheduler(self): if self._scheduler is None: self._scheduler = instantiate(self.scheduler_cls, schedule=self.schedule, logger=self.logger, max_interval=self.max_interval) return self._scheduler
def evcam(camera, freq=1.0, maxrate=None, loglevel=0, logfile=None, app=None): app = app_or_default(app) if not isinstance(loglevel, int): loglevel = LOG_LEVELS[loglevel.upper()] logger = app.log.setup_logger(loglevel=loglevel, logfile=logfile, name="celery.evcam") logger.info( "-> evcam: Taking snapshots with %s (every %s secs.)\n" % ( camera, freq)) state = State() cam = instantiate(camera, state, app=app, freq=freq, maxrate=maxrate, logger=logger) cam.install() conn = app.broker_connection() recv = EventReceiver(conn, app=app, handlers={"*": state.event}) try: try: recv.capture(limit=None) except KeyboardInterrupt: raise SystemExit finally: cam.cancel() conn.close()
def scheduler(self): if self._scheduler is None: filename = self.schedule_filename self._scheduler = instantiate(self.scheduler_cls, schedule_filename=filename, logger=self.logger, max_interval=self.max_interval) self._scheduler.update_from_dict(self.schedule) return self._scheduler
def get_scheduler(self, lazy=False): filename = self.schedule_filename scheduler = instantiate(self.scheduler_cls, app=self.app, schedule_filename=filename, logger=self.logger, max_interval=self.max_interval, lazy=lazy) return scheduler
def get_scheduler(self, lazy=False): filename = self.schedule_filename scheduler = instantiate(self.scheduler_cls, schedule_filename=filename, logger=self.logger, max_interval=self.max_interval, lazy=lazy) if not lazy: scheduler.update_from_dict(self.schedule) return scheduler
def handle(self, *args, **kwargs): kwargs = self.prepare_options(**kwargs) self.loglevel = kwargs.get("loglevel") self.logfile = kwargs.get("logfile") self.enter_instance_dir() self.env.syncdb(interactive=False) self.install_cry_handler() self.install_rdb_handler() self.colored = celery.log.colored(kwargs.get("logfile")) self.branch = instantiate(self.branch_cls, *args, colored=self.colored, **kwargs) self.connect_signals() print(str(self.colored.cyan(self.banner()))) self.detached = kwargs.get("detach", False) return (self._detach if self.detached else self._start)(**kwargs)
def evcam(camera, freq=1.0, maxrate=None, loglevel=0, logfile=None, pidfile=None, timer=None, app=None): app = app_or_default(app) if pidfile: pidlock = platforms.create_pidlock(pidfile).acquire() atexit.register(pidlock.release) if not isinstance(loglevel, int): loglevel = LOG_LEVELS[loglevel.upper()] logger = app.log.setup_logger(loglevel=loglevel, logfile=logfile, name="celery.evcam") logger.info("-> evcam: Taking snapshots with %s (every %s secs.)\n" % (camera, freq)) state = app.events.State() cam = instantiate(camera, state, app=app, freq=freq, maxrate=maxrate, logger=logger, timer=timer) cam.install() conn = app.broker_connection() recv = app.events.Receiver(conn, handlers={"*": state.event}) try: try: recv.capture(limit=None) except KeyboardInterrupt: raise SystemExit finally: cam.cancel() conn.close()
def evcam(camera, freq=1.0, maxrate=None, loglevel=0, logfile=None): if not isinstance(loglevel, int): loglevel = conf.LOG_LEVELS[loglevel.upper()] logger = log.setup_logger(loglevel=loglevel, logfile=logfile, name="celery.evcam") logger.info( "-> evcam: Taking snapshots with %s (every %s secs.)\n" % ( camera, freq)) state = State() cam = instantiate(camera, state, freq=freq, maxrate=maxrate, logger=logger) cam.install() conn = establish_connection() recv = EventReceiver(conn, handlers={"*": state.event}) try: try: recv.capture(limit=None) except KeyboardInterrupt: raise SystemExit finally: cam.cancel() conn.close()
def events(self): """Sending/receiving events. See :class:`~celery.events.Events`. """ return instantiate(self.events_cls, app=self)
def amqp(self): """Sending/receiving messages. See :class:`~celery.app.amqp.AMQP`.""" return instantiate(self.amqp_cls, app=self)
def control(self): """Controlling worker nodes. See :class:`~celery.task.control.Control`.""" return instantiate(self.control_cls, app=self)
def log(self): """Logging utilities. See :class:`~celery.log.Logging`.""" return instantiate(self.log_cls, app=self)
def __init__(self, env=None, *args, **kwargs): if env is None: env = instantiate('cyme.bin.base.Env') self.setup_default_env(env) self.env = env
def Worker(self, **kwargs): """Create new :class:`~celery.apps.worker.Worker` instance.""" return instantiate("celery.apps.worker.Worker", app=self, **kwargs)
def __init__(self, concurrency=None, logfile=None, loglevel=None, send_events=conf.SEND_EVENTS, hostname=None, ready_callback=noop, embed_clockservice=False, pool_cls=conf.CELERYD_POOL, listener_cls=conf.CELERYD_LISTENER, mediator_cls=conf.CELERYD_MEDIATOR, eta_scheduler_cls=conf.CELERYD_ETA_SCHEDULER, schedule_filename=conf.CELERYBEAT_SCHEDULE_FILENAME, task_time_limit=conf.CELERYD_TASK_TIME_LIMIT, task_soft_time_limit=conf.CELERYD_TASK_SOFT_TIME_LIMIT, max_tasks_per_child=conf.CELERYD_MAX_TASKS_PER_CHILD, pool_putlocks=conf.CELERYD_POOL_PUTLOCKS, db=conf.CELERYD_STATE_DB): # Options self.loglevel = loglevel or self.loglevel self.concurrency = concurrency or self.concurrency self.logfile = logfile or self.logfile self.logger = setup_logger(loglevel, logfile) self.hostname = hostname or socket.gethostname() self.embed_clockservice = embed_clockservice self.ready_callback = ready_callback self.send_events = send_events self.task_time_limit = task_time_limit self.task_soft_time_limit = task_soft_time_limit self.max_tasks_per_child = max_tasks_per_child self.pool_putlocks = pool_putlocks self.db = db self._finalize = Finalize(self, self.stop, exitpriority=1) if self.db: persistence = state.Persistent(self.db) Finalize(persistence, persistence.save, exitpriority=5) # Queues if conf.DISABLE_RATE_LIMITS: self.ready_queue = FastQueue() else: self.ready_queue = TaskBucket(task_registry=registry.tasks) self.eta_schedule = Scheduler(self.ready_queue, logger=self.logger) self.logger.debug("Instantiating thread components...") # Threads + Pool + Consumer self.pool = instantiate(pool_cls, self.concurrency, logger=self.logger, initializer=process_initializer, maxtasksperchild=self.max_tasks_per_child, timeout=self.task_time_limit, soft_timeout=self.task_soft_time_limit, putlocks=self.pool_putlocks) self.mediator = instantiate(mediator_cls, self.ready_queue, callback=self.process_task, logger=self.logger) self.scheduler = instantiate(eta_scheduler_cls, self.eta_schedule, logger=self.logger) self.clockservice = None if self.embed_clockservice: self.clockservice = EmbeddedClockService( logger=self.logger, schedule_filename=schedule_filename) prefetch_count = self.concurrency * conf.CELERYD_PREFETCH_MULTIPLIER self.listener = instantiate(listener_cls, self.ready_queue, self.eta_schedule, logger=self.logger, hostname=self.hostname, send_events=self.send_events, init_callback=self.ready_callback, initial_prefetch_count=prefetch_count, pool=self.pool) # The order is important here; # the first in the list is the first to start, # and they must be stopped in reverse order. self.components = filter(None, (self.pool, self.mediator, self.scheduler, self.clockservice, self.listener))
def __init__(self, concurrency=None, logfile=None, loglevel=None, send_events=None, hostname=None, ready_callback=noop, embed_clockservice=False, pool_cls=None, consumer_cls=None, mediator_cls=None, eta_scheduler_cls=None, schedule_filename=None, task_time_limit=None, task_soft_time_limit=None, max_tasks_per_child=None, pool_putlocks=None, db=None, prefetch_multiplier=None, eta_scheduler_precision=None, queues=None, disable_rate_limits=None, autoscale=None, autoscaler_cls=None, scheduler_cls=None, app=None): self.app = app_or_default(app) conf = self.app.conf # Options self.loglevel = loglevel or self.loglevel self.concurrency = concurrency or conf.CELERYD_CONCURRENCY self.logfile = logfile or conf.CELERYD_LOG_FILE self.logger = self.app.log.get_default_logger() if send_events is None: send_events = conf.CELERY_SEND_EVENTS self.send_events = send_events self.pool_cls = pool_cls or conf.CELERYD_POOL self.consumer_cls = consumer_cls or conf.CELERYD_CONSUMER self.mediator_cls = mediator_cls or conf.CELERYD_MEDIATOR self.eta_scheduler_cls = eta_scheduler_cls or \ conf.CELERYD_ETA_SCHEDULER self.autoscaler_cls = autoscaler_cls or \ conf.CELERYD_AUTOSCALER self.schedule_filename = schedule_filename or \ conf.CELERYBEAT_SCHEDULE_FILENAME self.scheduler_cls = scheduler_cls or conf.CELERYBEAT_SCHEDULER self.hostname = hostname or socket.gethostname() self.embed_clockservice = embed_clockservice self.ready_callback = ready_callback self.task_time_limit = task_time_limit or \ conf.CELERYD_TASK_TIME_LIMIT self.task_soft_time_limit = task_soft_time_limit or \ conf.CELERYD_TASK_SOFT_TIME_LIMIT self.max_tasks_per_child = max_tasks_per_child or \ conf.CELERYD_MAX_TASKS_PER_CHILD self.pool_putlocks = pool_putlocks or \ conf.CELERYD_POOL_PUTLOCKS self.eta_scheduler_precision = eta_scheduler_precision or \ conf.CELERYD_ETA_SCHEDULER_PRECISION self.prefetch_multiplier = prefetch_multiplier or \ conf.CELERYD_PREFETCH_MULTIPLIER self.timer_debug = SilenceRepeated(self.logger.debug, max_iterations=10) self.db = db or conf.CELERYD_STATE_DB self.disable_rate_limits = disable_rate_limits or \ conf.CELERY_DISABLE_RATE_LIMITS self.queues = queues self._finalize = Finalize(self, self.stop, exitpriority=1) if self.db: persistence = state.Persistent(self.db) Finalize(persistence, persistence.save, exitpriority=5) # Queues if disable_rate_limits: self.ready_queue = FastQueue() self.ready_queue.put = self.process_task else: self.ready_queue = TaskBucket(task_registry=registry.tasks) self.logger.debug("Instantiating thread components...") # Threads + Pool + Consumer self.autoscaler = None max_concurrency = None min_concurrency = concurrency if autoscale: max_concurrency, min_concurrency = autoscale self.pool = instantiate(self.pool_cls, min_concurrency, logger=self.logger, initializer=process_initializer, initargs=(self.app, self.hostname), maxtasksperchild=self.max_tasks_per_child, timeout=self.task_time_limit, soft_timeout=self.task_soft_time_limit, putlocks=self.pool_putlocks) if autoscale: self.autoscaler = instantiate(self.autoscaler_cls, self.pool, max_concurrency=max_concurrency, min_concurrency=min_concurrency, logger=self.logger) self.mediator = None if not disable_rate_limits: self.mediator = instantiate(self.mediator_cls, self.ready_queue, app=self.app, callback=self.process_task, logger=self.logger) self.scheduler = instantiate(self.eta_scheduler_cls, precision=eta_scheduler_precision, on_error=self.on_timer_error, on_tick=self.on_timer_tick) self.beat = None if self.embed_clockservice: self.beat = beat.EmbeddedService(app=self.app, logger=self.logger, schedule_filename=self.schedule_filename, scheduler_cls=self.scheduler_cls) prefetch_count = self.concurrency * self.prefetch_multiplier self.consumer = instantiate(self.consumer_cls, self.ready_queue, self.scheduler, logger=self.logger, hostname=self.hostname, send_events=self.send_events, init_callback=self.ready_callback, initial_prefetch_count=prefetch_count, pool=self.pool, queues=self.queues, app=self.app) # The order is important here; # the first in the list is the first to start, # and they must be stopped in reverse order. self.components = filter(None, (self.pool, self.mediator, self.scheduler, self.beat, self.autoscaler, self.consumer))
def __init__(self, concurrency=None, logfile=None, loglevel=None, send_events=conf.SEND_EVENTS, hostname=None, ready_callback=noop, embed_clockservice=False, pool_cls=conf.CELERYD_POOL, listener_cls=conf.CELERYD_LISTENER, mediator_cls=conf.CELERYD_MEDIATOR, eta_scheduler_cls=conf.CELERYD_ETA_SCHEDULER, schedule_filename=conf.CELERYBEAT_SCHEDULE_FILENAME, task_time_limit=conf.CELERYD_TASK_TIME_LIMIT, task_soft_time_limit=conf.CELERYD_TASK_SOFT_TIME_LIMIT, max_tasks_per_child=conf.CELERYD_MAX_TASKS_PER_CHILD, pool_putlocks=conf.CELERYD_POOL_PUTLOCKS, db=conf.CELERYD_STATE_DB): # Options self.loglevel = loglevel or self.loglevel self.concurrency = concurrency or self.concurrency self.logfile = logfile or self.logfile self.logger = setup_logger(loglevel, logfile) self.hostname = hostname or socket.gethostname() self.embed_clockservice = embed_clockservice self.ready_callback = ready_callback self.send_events = send_events self.task_time_limit = task_time_limit self.task_soft_time_limit = task_soft_time_limit self.max_tasks_per_child = max_tasks_per_child self.pool_putlocks = pool_putlocks self.timer_debug = log.SilenceRepeated(self.logger.debug, max_iterations=10) self.db = db self._finalize = Finalize(self, self.stop, exitpriority=1) if self.db: persistence = state.Persistent(self.db) Finalize(persistence, persistence.save, exitpriority=5) # Queues if conf.DISABLE_RATE_LIMITS: self.ready_queue = FastQueue() else: self.ready_queue = TaskBucket(task_registry=registry.tasks) self.logger.debug("Instantiating thread components...") # Threads + Pool + Consumer self.pool = instantiate(pool_cls, self.concurrency, logger=self.logger, initializer=process_initializer, maxtasksperchild=self.max_tasks_per_child, timeout=self.task_time_limit, soft_timeout=self.task_soft_time_limit, putlocks=self.pool_putlocks) self.mediator = instantiate(mediator_cls, self.ready_queue, callback=self.process_task, logger=self.logger) self.scheduler = instantiate(eta_scheduler_cls, precision=conf.CELERYD_ETA_SCHEDULER_PRECISION, on_error=self.on_timer_error, on_tick=self.on_timer_tick) self.clockservice = None if self.embed_clockservice: self.clockservice = EmbeddedClockService(logger=self.logger, schedule_filename=schedule_filename) prefetch_count = self.concurrency * conf.CELERYD_PREFETCH_MULTIPLIER self.listener = instantiate(listener_cls, self.ready_queue, self.scheduler, logger=self.logger, hostname=self.hostname, send_events=self.send_events, init_callback=self.ready_callback, initial_prefetch_count=prefetch_count, pool=self.pool) # The order is important here; # the first in the list is the first to start, # and they must be stopped in reverse order. self.components = filter(None, (self.pool, self.mediator, self.scheduler, self.clockservice, self.listener))
def __init__(self, concurrency=None, logfile=None, loglevel=None, send_events=None, hostname=None, ready_callback=noop, embed_clockservice=False, pool_cls=None, consumer_cls=None, mediator_cls=None, eta_scheduler_cls=None, schedule_filename=None, task_time_limit=None, task_soft_time_limit=None, max_tasks_per_child=None, pool_putlocks=None, db=None, prefetch_multiplier=None, eta_scheduler_precision=None, disable_rate_limits=None, autoscale=None, autoscaler_cls=None, scheduler_cls=None, app=None): self.app = app_or_default(app) conf = self.app.conf # Options self.loglevel = loglevel or self.loglevel self.concurrency = concurrency or conf.CELERYD_CONCURRENCY self.logfile = logfile or conf.CELERYD_LOG_FILE self.logger = self.app.log.get_default_logger() if send_events is None: send_events = conf.CELERY_SEND_EVENTS self.send_events = send_events self.pool_cls = _concurrency.get_implementation(pool_cls or conf.CELERYD_POOL) self.consumer_cls = consumer_cls or conf.CELERYD_CONSUMER self.mediator_cls = mediator_cls or conf.CELERYD_MEDIATOR self.eta_scheduler_cls = eta_scheduler_cls or \ conf.CELERYD_ETA_SCHEDULER self.autoscaler_cls = autoscaler_cls or \ conf.CELERYD_AUTOSCALER self.schedule_filename = schedule_filename or \ conf.CELERYBEAT_SCHEDULE_FILENAME self.scheduler_cls = scheduler_cls or conf.CELERYBEAT_SCHEDULER self.hostname = hostname or socket.gethostname() self.embed_clockservice = embed_clockservice self.ready_callback = ready_callback self.task_time_limit = task_time_limit or \ conf.CELERYD_TASK_TIME_LIMIT self.task_soft_time_limit = task_soft_time_limit or \ conf.CELERYD_TASK_SOFT_TIME_LIMIT self.max_tasks_per_child = max_tasks_per_child or \ conf.CELERYD_MAX_TASKS_PER_CHILD self.pool_putlocks = pool_putlocks or \ conf.CELERYD_POOL_PUTLOCKS self.eta_scheduler_precision = eta_scheduler_precision or \ conf.CELERYD_ETA_SCHEDULER_PRECISION self.prefetch_multiplier = prefetch_multiplier or \ conf.CELERYD_PREFETCH_MULTIPLIER self.timer_debug = SilenceRepeated(self.logger.debug, max_iterations=10) self.db = db or conf.CELERYD_STATE_DB self.disable_rate_limits = disable_rate_limits or \ conf.CELERY_DISABLE_RATE_LIMITS self._finalize = Finalize(self, self.stop, exitpriority=1) self._finalize_db = None if self.db: persistence = state.Persistent(self.db) self._finalize_db = Finalize(persistence, persistence.save, exitpriority=5) # Queues if self.disable_rate_limits: self.ready_queue = FastQueue() self.ready_queue.put = self.process_task else: self.ready_queue = TaskBucket(task_registry=registry.tasks) self.logger.debug("Instantiating thread components...") # Threads + Pool + Consumer self.autoscaler = None max_concurrency = None min_concurrency = concurrency if autoscale: max_concurrency, min_concurrency = autoscale self.pool = instantiate(self.pool_cls, min_concurrency, logger=self.logger, initializer=process_initializer, initargs=(self.app, self.hostname), maxtasksperchild=self.max_tasks_per_child, timeout=self.task_time_limit, soft_timeout=self.task_soft_time_limit, putlocks=self.pool_putlocks) self.priority_timer = instantiate(self.pool.Timer) if not self.eta_scheduler_cls: # Default Timer is set by the pool, as e.g. eventlet # needs a custom implementation. self.eta_scheduler_cls = self.pool.Timer if autoscale: self.autoscaler = instantiate(self.autoscaler_cls, self.pool, max_concurrency=max_concurrency, min_concurrency=min_concurrency, logger=self.logger) self.mediator = None if not self.disable_rate_limits: self.mediator = instantiate(self.mediator_cls, self.ready_queue, app=self.app, callback=self.process_task, logger=self.logger) self.scheduler = instantiate(self.eta_scheduler_cls, precision=eta_scheduler_precision, on_error=self.on_timer_error, on_tick=self.on_timer_tick) self.beat = None if self.embed_clockservice: self.beat = beat.EmbeddedService( app=self.app, logger=self.logger, schedule_filename=self.schedule_filename, scheduler_cls=self.scheduler_cls) prefetch_count = self.concurrency * self.prefetch_multiplier self.consumer = instantiate(self.consumer_cls, self.ready_queue, self.scheduler, logger=self.logger, hostname=self.hostname, send_events=self.send_events, init_callback=self.ready_callback, initial_prefetch_count=prefetch_count, pool=self.pool, priority_timer=self.priority_timer, app=self.app) # The order is important here; # the first in the list is the first to start, # and they must be stopped in reverse order. self.components = filter(None, (self.pool, self.mediator, self.scheduler, self.beat, self.autoscaler, self.consumer))
def Beat(self, **kwargs): """Create new :class:`~celery.apps.beat.Beat` instance.""" return instantiate("celery.apps.beat.Beat", app=self, **kwargs)
__contact__ = "*****@*****.**" __homepage__ = "http://celeryproject.org" __docformat__ = "restructuredtext" if sys.version_info < (2, 5): import warnings warnings.warn( DeprecationWarning( """ Python 2.4 support is deprecated and only versions 2.5, 2.6, 2.7+ will be supported starting from Celery version 2.3. """ ) ) def Celery(*args, **kwargs): from celery.app import App return App(*args, **kwargs) if not os.environ.get("CELERY_NO_EVAL", False): from celery.utils import LocalProxy, instantiate current_app = LocalProxy(lambda: instantiate("celery.app.current_app"))
def __init__( self, concurrency=None, logfile=None, loglevel=None, send_events=conf.SEND_EVENTS, hostname=None, ready_callback=noop, embed_clockservice=False, pool_cls=conf.CELERYD_POOL, listener_cls=conf.CELERYD_LISTENER, mediator_cls=conf.CELERYD_MEDIATOR, eta_scheduler_cls=conf.CELERYD_ETA_SCHEDULER, schedule_filename=conf.CELERYBEAT_SCHEDULE_FILENAME, ): # Options self.loglevel = loglevel or self.loglevel self.concurrency = concurrency or self.concurrency self.logfile = logfile or self.logfile self.logger = setup_logger(loglevel, logfile) self.hostname = hostname or socket.gethostname() self.embed_clockservice = embed_clockservice self.ready_callback = ready_callback self.send_events = send_events self._finalize = Finalize(self, self.stop, exitpriority=20) # Queues if conf.DISABLE_RATE_LIMITS: self.ready_queue = Queue() else: self.ready_queue = TaskBucket(task_registry=registry.tasks) self.eta_schedule = Scheduler(self.ready_queue, logger=self.logger) self.logger.debug("Instantiating thread components...") # Threads + Pool + Consumer self.pool = instantiate(pool_cls, self.concurrency, logger=self.logger, initializer=process_initializer) self.mediator = instantiate(mediator_cls, self.ready_queue, callback=self.process_task, logger=self.logger) self.scheduler = instantiate(eta_scheduler_cls, self.eta_schedule, logger=self.logger) self.clockservice = None if self.embed_clockservice: self.clockservice = EmbeddedClockService(logger=self.logger, schedule_filename=schedule_filename) prefetch_count = self.concurrency * conf.CELERYD_PREFETCH_MULTIPLIER self.listener = instantiate( listener_cls, self.ready_queue, self.eta_schedule, logger=self.logger, hostname=self.hostname, send_events=self.send_events, init_callback=self.ready_callback, initial_prefetch_count=prefetch_count, ) # The order is important here; # the first in the list is the first to start, # and they must be stopped in reverse order. self.components = filter(None, (self.pool, self.mediator, self.scheduler, self.clockservice, self.listener))
def expand_route(route): if isinstance(route, dict): return MapRoute(route) if isinstance(route, basestring): return instantiate(route) return route