Ejemplo n.º 1
0
Archivo: beat.py Proyecto: clayg/celery
 def scheduler(self):
     if self._scheduler is None:
         self._scheduler = instantiate(self.scheduler_cls,
                                       schedule=self.schedule,
                                       logger=self.logger,
                                       max_interval=self.max_interval)
     return self._scheduler
Ejemplo n.º 2
0
def evcam(camera, freq=1.0, maxrate=None, loglevel=0,
        logfile=None, app=None):
    app = app_or_default(app)
    if not isinstance(loglevel, int):
        loglevel = LOG_LEVELS[loglevel.upper()]
    logger = app.log.setup_logger(loglevel=loglevel,
                                  logfile=logfile,
                                  name="celery.evcam")
    logger.info(
        "-> evcam: Taking snapshots with %s (every %s secs.)\n" % (
            camera, freq))
    state = State()
    cam = instantiate(camera, state, app=app,
                      freq=freq, maxrate=maxrate, logger=logger)
    cam.install()
    conn = app.broker_connection()
    recv = EventReceiver(conn, app=app, handlers={"*": state.event})
    try:
        try:
            recv.capture(limit=None)
        except KeyboardInterrupt:
            raise SystemExit
    finally:
        cam.cancel()
        conn.close()
Ejemplo n.º 3
0
 def scheduler(self):
     if self._scheduler is None:
         filename = self.schedule_filename
         self._scheduler = instantiate(self.scheduler_cls,
                                       schedule_filename=filename,
                                       logger=self.logger,
                                       max_interval=self.max_interval)
         self._scheduler.update_from_dict(self.schedule)
     return self._scheduler
Ejemplo n.º 4
0
 def get_scheduler(self, lazy=False):
     filename = self.schedule_filename
     scheduler = instantiate(self.scheduler_cls,
                             app=self.app,
                             schedule_filename=filename,
                             logger=self.logger,
                             max_interval=self.max_interval,
                             lazy=lazy)
     return scheduler
Ejemplo n.º 5
0
 def get_scheduler(self, lazy=False):
     filename = self.schedule_filename
     scheduler = instantiate(self.scheduler_cls,
                                   schedule_filename=filename,
                                   logger=self.logger,
                                   max_interval=self.max_interval,
                                   lazy=lazy)
     if not lazy:
         scheduler.update_from_dict(self.schedule)
     return scheduler
Ejemplo n.º 6
0
    def handle(self, *args, **kwargs):
        kwargs = self.prepare_options(**kwargs)
        self.loglevel = kwargs.get("loglevel")
        self.logfile = kwargs.get("logfile")
        self.enter_instance_dir()
        self.env.syncdb(interactive=False)
        self.install_cry_handler()
        self.install_rdb_handler()
        self.colored = celery.log.colored(kwargs.get("logfile"))
        self.branch = instantiate(self.branch_cls, *args,
                                 colored=self.colored, **kwargs)
        self.connect_signals()
        print(str(self.colored.cyan(self.banner())))

        self.detached = kwargs.get("detach", False)
        return (self._detach if self.detached else self._start)(**kwargs)
Ejemplo n.º 7
0
def evcam(camera,
          freq=1.0,
          maxrate=None,
          loglevel=0,
          logfile=None,
          pidfile=None,
          timer=None,
          app=None):
    app = app_or_default(app)

    if pidfile:
        pidlock = platforms.create_pidlock(pidfile).acquire()
        atexit.register(pidlock.release)

    if not isinstance(loglevel, int):
        loglevel = LOG_LEVELS[loglevel.upper()]
    logger = app.log.setup_logger(loglevel=loglevel,
                                  logfile=logfile,
                                  name="celery.evcam")

    logger.info("-> evcam: Taking snapshots with %s (every %s secs.)\n" %
                (camera, freq))
    state = app.events.State()
    cam = instantiate(camera,
                      state,
                      app=app,
                      freq=freq,
                      maxrate=maxrate,
                      logger=logger,
                      timer=timer)
    cam.install()
    conn = app.broker_connection()
    recv = app.events.Receiver(conn, handlers={"*": state.event})
    try:
        try:
            recv.capture(limit=None)
        except KeyboardInterrupt:
            raise SystemExit
    finally:
        cam.cancel()
        conn.close()
Ejemplo n.º 8
0
def evcam(camera, freq=1.0, maxrate=None, loglevel=0,
        logfile=None):
    if not isinstance(loglevel, int):
        loglevel = conf.LOG_LEVELS[loglevel.upper()]
    logger = log.setup_logger(loglevel=loglevel,
                              logfile=logfile,
                              name="celery.evcam")
    logger.info(
        "-> evcam: Taking snapshots with %s (every %s secs.)\n" % (
            camera, freq))
    state = State()
    cam = instantiate(camera, state,
                      freq=freq, maxrate=maxrate, logger=logger)
    cam.install()
    conn = establish_connection()
    recv = EventReceiver(conn, handlers={"*": state.event})
    try:
        try:
            recv.capture(limit=None)
        except KeyboardInterrupt:
            raise SystemExit
    finally:
        cam.cancel()
        conn.close()
Ejemplo n.º 9
0
Archivo: base.py Proyecto: frac/celery
 def events(self):
     """Sending/receiving events.  See :class:`~celery.events.Events`. """
     return instantiate(self.events_cls, app=self)
Ejemplo n.º 10
0
Archivo: base.py Proyecto: frac/celery
 def amqp(self):
     """Sending/receiving messages.  See :class:`~celery.app.amqp.AMQP`."""
     return instantiate(self.amqp_cls, app=self)
Ejemplo n.º 11
0
Archivo: base.py Proyecto: frac/celery
 def control(self):
     """Controlling worker nodes.  See
     :class:`~celery.task.control.Control`."""
     return instantiate(self.control_cls, app=self)
Ejemplo n.º 12
0
 def log(self):
     """Logging utilities.  See :class:`~celery.log.Logging`."""
     return instantiate(self.log_cls, app=self)
Ejemplo n.º 13
0
 def __init__(self, env=None, *args, **kwargs):
     if env is None:
         env = instantiate('cyme.bin.base.Env')
         self.setup_default_env(env)
     self.env = env
Ejemplo n.º 14
0
 def Worker(self, **kwargs):
     """Create new :class:`~celery.apps.worker.Worker` instance."""
     return instantiate("celery.apps.worker.Worker", app=self, **kwargs)
Ejemplo n.º 15
0
 def events(self):
     """Sending/receiving events.  See :class:`~celery.events.Events`. """
     return instantiate(self.events_cls, app=self)
    def __init__(self,
                 concurrency=None,
                 logfile=None,
                 loglevel=None,
                 send_events=conf.SEND_EVENTS,
                 hostname=None,
                 ready_callback=noop,
                 embed_clockservice=False,
                 pool_cls=conf.CELERYD_POOL,
                 listener_cls=conf.CELERYD_LISTENER,
                 mediator_cls=conf.CELERYD_MEDIATOR,
                 eta_scheduler_cls=conf.CELERYD_ETA_SCHEDULER,
                 schedule_filename=conf.CELERYBEAT_SCHEDULE_FILENAME,
                 task_time_limit=conf.CELERYD_TASK_TIME_LIMIT,
                 task_soft_time_limit=conf.CELERYD_TASK_SOFT_TIME_LIMIT,
                 max_tasks_per_child=conf.CELERYD_MAX_TASKS_PER_CHILD,
                 pool_putlocks=conf.CELERYD_POOL_PUTLOCKS,
                 db=conf.CELERYD_STATE_DB):

        # Options
        self.loglevel = loglevel or self.loglevel
        self.concurrency = concurrency or self.concurrency
        self.logfile = logfile or self.logfile
        self.logger = setup_logger(loglevel, logfile)
        self.hostname = hostname or socket.gethostname()
        self.embed_clockservice = embed_clockservice
        self.ready_callback = ready_callback
        self.send_events = send_events
        self.task_time_limit = task_time_limit
        self.task_soft_time_limit = task_soft_time_limit
        self.max_tasks_per_child = max_tasks_per_child
        self.pool_putlocks = pool_putlocks
        self.db = db
        self._finalize = Finalize(self, self.stop, exitpriority=1)

        if self.db:
            persistence = state.Persistent(self.db)
            Finalize(persistence, persistence.save, exitpriority=5)

        # Queues
        if conf.DISABLE_RATE_LIMITS:
            self.ready_queue = FastQueue()
        else:
            self.ready_queue = TaskBucket(task_registry=registry.tasks)
        self.eta_schedule = Scheduler(self.ready_queue, logger=self.logger)

        self.logger.debug("Instantiating thread components...")

        # Threads + Pool + Consumer
        self.pool = instantiate(pool_cls,
                                self.concurrency,
                                logger=self.logger,
                                initializer=process_initializer,
                                maxtasksperchild=self.max_tasks_per_child,
                                timeout=self.task_time_limit,
                                soft_timeout=self.task_soft_time_limit,
                                putlocks=self.pool_putlocks)
        self.mediator = instantiate(mediator_cls,
                                    self.ready_queue,
                                    callback=self.process_task,
                                    logger=self.logger)
        self.scheduler = instantiate(eta_scheduler_cls,
                                     self.eta_schedule,
                                     logger=self.logger)

        self.clockservice = None
        if self.embed_clockservice:
            self.clockservice = EmbeddedClockService(
                logger=self.logger, schedule_filename=schedule_filename)

        prefetch_count = self.concurrency * conf.CELERYD_PREFETCH_MULTIPLIER
        self.listener = instantiate(listener_cls,
                                    self.ready_queue,
                                    self.eta_schedule,
                                    logger=self.logger,
                                    hostname=self.hostname,
                                    send_events=self.send_events,
                                    init_callback=self.ready_callback,
                                    initial_prefetch_count=prefetch_count,
                                    pool=self.pool)

        # The order is important here;
        #   the first in the list is the first to start,
        # and they must be stopped in reverse order.
        self.components = filter(None,
                                 (self.pool, self.mediator, self.scheduler,
                                  self.clockservice, self.listener))
Ejemplo n.º 17
0
 def amqp(self):
     """Sending/receiving messages.  See :class:`~celery.app.amqp.AMQP`."""
     return instantiate(self.amqp_cls, app=self)
Ejemplo n.º 18
0
    def __init__(self, concurrency=None, logfile=None, loglevel=None,
            send_events=None, hostname=None, ready_callback=noop,
            embed_clockservice=False, pool_cls=None, consumer_cls=None,
            mediator_cls=None, eta_scheduler_cls=None,
            schedule_filename=None, task_time_limit=None,
            task_soft_time_limit=None, max_tasks_per_child=None,
            pool_putlocks=None, db=None, prefetch_multiplier=None,
            eta_scheduler_precision=None, queues=None,
            disable_rate_limits=None, autoscale=None,
            autoscaler_cls=None, scheduler_cls=None, app=None):

        self.app = app_or_default(app)
        conf = self.app.conf

        # Options
        self.loglevel = loglevel or self.loglevel
        self.concurrency = concurrency or conf.CELERYD_CONCURRENCY
        self.logfile = logfile or conf.CELERYD_LOG_FILE
        self.logger = self.app.log.get_default_logger()
        if send_events is None:
            send_events = conf.CELERY_SEND_EVENTS
        self.send_events = send_events
        self.pool_cls = pool_cls or conf.CELERYD_POOL
        self.consumer_cls = consumer_cls or conf.CELERYD_CONSUMER
        self.mediator_cls = mediator_cls or conf.CELERYD_MEDIATOR
        self.eta_scheduler_cls = eta_scheduler_cls or \
                                    conf.CELERYD_ETA_SCHEDULER
        self.autoscaler_cls = autoscaler_cls or \
                                    conf.CELERYD_AUTOSCALER
        self.schedule_filename = schedule_filename or \
                                    conf.CELERYBEAT_SCHEDULE_FILENAME
        self.scheduler_cls = scheduler_cls or conf.CELERYBEAT_SCHEDULER
        self.hostname = hostname or socket.gethostname()
        self.embed_clockservice = embed_clockservice
        self.ready_callback = ready_callback
        self.task_time_limit = task_time_limit or \
                                conf.CELERYD_TASK_TIME_LIMIT
        self.task_soft_time_limit = task_soft_time_limit or \
                                conf.CELERYD_TASK_SOFT_TIME_LIMIT
        self.max_tasks_per_child = max_tasks_per_child or \
                                conf.CELERYD_MAX_TASKS_PER_CHILD
        self.pool_putlocks = pool_putlocks or \
                                conf.CELERYD_POOL_PUTLOCKS
        self.eta_scheduler_precision = eta_scheduler_precision or \
                                conf.CELERYD_ETA_SCHEDULER_PRECISION
        self.prefetch_multiplier = prefetch_multiplier or \
                                conf.CELERYD_PREFETCH_MULTIPLIER
        self.timer_debug = SilenceRepeated(self.logger.debug,
                                           max_iterations=10)
        self.db = db or conf.CELERYD_STATE_DB
        self.disable_rate_limits = disable_rate_limits or \
                                conf.CELERY_DISABLE_RATE_LIMITS
        self.queues = queues

        self._finalize = Finalize(self, self.stop, exitpriority=1)

        if self.db:
            persistence = state.Persistent(self.db)
            Finalize(persistence, persistence.save, exitpriority=5)

        # Queues
        if disable_rate_limits:
            self.ready_queue = FastQueue()
            self.ready_queue.put = self.process_task
        else:
            self.ready_queue = TaskBucket(task_registry=registry.tasks)

        self.logger.debug("Instantiating thread components...")

        # Threads + Pool + Consumer
        self.autoscaler = None
        max_concurrency = None
        min_concurrency = concurrency
        if autoscale:
            max_concurrency, min_concurrency = autoscale

        self.pool = instantiate(self.pool_cls, min_concurrency,
                                logger=self.logger,
                                initializer=process_initializer,
                                initargs=(self.app, self.hostname),
                                maxtasksperchild=self.max_tasks_per_child,
                                timeout=self.task_time_limit,
                                soft_timeout=self.task_soft_time_limit,
                                putlocks=self.pool_putlocks)

        if autoscale:
            self.autoscaler = instantiate(self.autoscaler_cls, self.pool,
                                          max_concurrency=max_concurrency,
                                          min_concurrency=min_concurrency,
                                          logger=self.logger)

        self.mediator = None
        if not disable_rate_limits:
            self.mediator = instantiate(self.mediator_cls, self.ready_queue,
                                        app=self.app,
                                        callback=self.process_task,
                                        logger=self.logger)
        self.scheduler = instantiate(self.eta_scheduler_cls,
                                     precision=eta_scheduler_precision,
                                     on_error=self.on_timer_error,
                                     on_tick=self.on_timer_tick)

        self.beat = None
        if self.embed_clockservice:
            self.beat = beat.EmbeddedService(app=self.app,
                                logger=self.logger,
                                schedule_filename=self.schedule_filename,
                                scheduler_cls=self.scheduler_cls)

        prefetch_count = self.concurrency * self.prefetch_multiplier
        self.consumer = instantiate(self.consumer_cls,
                                    self.ready_queue,
                                    self.scheduler,
                                    logger=self.logger,
                                    hostname=self.hostname,
                                    send_events=self.send_events,
                                    init_callback=self.ready_callback,
                                    initial_prefetch_count=prefetch_count,
                                    pool=self.pool,
                                    queues=self.queues,
                                    app=self.app)

        # The order is important here;
        #   the first in the list is the first to start,
        # and they must be stopped in reverse order.
        self.components = filter(None, (self.pool,
                                        self.mediator,
                                        self.scheduler,
                                        self.beat,
                                        self.autoscaler,
                                        self.consumer))
Ejemplo n.º 19
0
    def __init__(self, concurrency=None, logfile=None, loglevel=None,
            send_events=conf.SEND_EVENTS, hostname=None,
            ready_callback=noop, embed_clockservice=False,
            pool_cls=conf.CELERYD_POOL, listener_cls=conf.CELERYD_LISTENER,
            mediator_cls=conf.CELERYD_MEDIATOR,
            eta_scheduler_cls=conf.CELERYD_ETA_SCHEDULER,
            schedule_filename=conf.CELERYBEAT_SCHEDULE_FILENAME,
            task_time_limit=conf.CELERYD_TASK_TIME_LIMIT,
            task_soft_time_limit=conf.CELERYD_TASK_SOFT_TIME_LIMIT,
            max_tasks_per_child=conf.CELERYD_MAX_TASKS_PER_CHILD,
            pool_putlocks=conf.CELERYD_POOL_PUTLOCKS,
            db=conf.CELERYD_STATE_DB):

        # Options
        self.loglevel = loglevel or self.loglevel
        self.concurrency = concurrency or self.concurrency
        self.logfile = logfile or self.logfile
        self.logger = setup_logger(loglevel, logfile)
        self.hostname = hostname or socket.gethostname()
        self.embed_clockservice = embed_clockservice
        self.ready_callback = ready_callback
        self.send_events = send_events
        self.task_time_limit = task_time_limit
        self.task_soft_time_limit = task_soft_time_limit
        self.max_tasks_per_child = max_tasks_per_child
        self.pool_putlocks = pool_putlocks
        self.timer_debug = log.SilenceRepeated(self.logger.debug,
                                               max_iterations=10)
        self.db = db
        self._finalize = Finalize(self, self.stop, exitpriority=1)

        if self.db:
            persistence = state.Persistent(self.db)
            Finalize(persistence, persistence.save, exitpriority=5)

        # Queues
        if conf.DISABLE_RATE_LIMITS:
            self.ready_queue = FastQueue()
        else:
            self.ready_queue = TaskBucket(task_registry=registry.tasks)

        self.logger.debug("Instantiating thread components...")

        # Threads + Pool + Consumer
        self.pool = instantiate(pool_cls, self.concurrency,
                                logger=self.logger,
                                initializer=process_initializer,
                                maxtasksperchild=self.max_tasks_per_child,
                                timeout=self.task_time_limit,
                                soft_timeout=self.task_soft_time_limit,
                                putlocks=self.pool_putlocks)
        self.mediator = instantiate(mediator_cls, self.ready_queue,
                                    callback=self.process_task,
                                    logger=self.logger)
        self.scheduler = instantiate(eta_scheduler_cls,
                               precision=conf.CELERYD_ETA_SCHEDULER_PRECISION,
                               on_error=self.on_timer_error,
                               on_tick=self.on_timer_tick)

        self.clockservice = None
        if self.embed_clockservice:
            self.clockservice = EmbeddedClockService(logger=self.logger,
                                    schedule_filename=schedule_filename)

        prefetch_count = self.concurrency * conf.CELERYD_PREFETCH_MULTIPLIER
        self.listener = instantiate(listener_cls,
                                    self.ready_queue,
                                    self.scheduler,
                                    logger=self.logger,
                                    hostname=self.hostname,
                                    send_events=self.send_events,
                                    init_callback=self.ready_callback,
                                    initial_prefetch_count=prefetch_count,
                                    pool=self.pool)

        # The order is important here;
        #   the first in the list is the first to start,
        # and they must be stopped in reverse order.
        self.components = filter(None, (self.pool,
                                        self.mediator,
                                        self.scheduler,
                                        self.clockservice,
                                        self.listener))
Ejemplo n.º 20
0
    def __init__(self,
                 concurrency=None,
                 logfile=None,
                 loglevel=None,
                 send_events=None,
                 hostname=None,
                 ready_callback=noop,
                 embed_clockservice=False,
                 pool_cls=None,
                 consumer_cls=None,
                 mediator_cls=None,
                 eta_scheduler_cls=None,
                 schedule_filename=None,
                 task_time_limit=None,
                 task_soft_time_limit=None,
                 max_tasks_per_child=None,
                 pool_putlocks=None,
                 db=None,
                 prefetch_multiplier=None,
                 eta_scheduler_precision=None,
                 disable_rate_limits=None,
                 autoscale=None,
                 autoscaler_cls=None,
                 scheduler_cls=None,
                 app=None):

        self.app = app_or_default(app)
        conf = self.app.conf

        # Options
        self.loglevel = loglevel or self.loglevel
        self.concurrency = concurrency or conf.CELERYD_CONCURRENCY
        self.logfile = logfile or conf.CELERYD_LOG_FILE
        self.logger = self.app.log.get_default_logger()
        if send_events is None:
            send_events = conf.CELERY_SEND_EVENTS
        self.send_events = send_events
        self.pool_cls = _concurrency.get_implementation(pool_cls
                                                        or conf.CELERYD_POOL)
        self.consumer_cls = consumer_cls or conf.CELERYD_CONSUMER
        self.mediator_cls = mediator_cls or conf.CELERYD_MEDIATOR
        self.eta_scheduler_cls = eta_scheduler_cls or \
                                    conf.CELERYD_ETA_SCHEDULER

        self.autoscaler_cls = autoscaler_cls or \
                                    conf.CELERYD_AUTOSCALER
        self.schedule_filename = schedule_filename or \
                                    conf.CELERYBEAT_SCHEDULE_FILENAME
        self.scheduler_cls = scheduler_cls or conf.CELERYBEAT_SCHEDULER
        self.hostname = hostname or socket.gethostname()
        self.embed_clockservice = embed_clockservice
        self.ready_callback = ready_callback
        self.task_time_limit = task_time_limit or \
                                conf.CELERYD_TASK_TIME_LIMIT
        self.task_soft_time_limit = task_soft_time_limit or \
                                conf.CELERYD_TASK_SOFT_TIME_LIMIT
        self.max_tasks_per_child = max_tasks_per_child or \
                                conf.CELERYD_MAX_TASKS_PER_CHILD
        self.pool_putlocks = pool_putlocks or \
                                conf.CELERYD_POOL_PUTLOCKS
        self.eta_scheduler_precision = eta_scheduler_precision or \
                                conf.CELERYD_ETA_SCHEDULER_PRECISION
        self.prefetch_multiplier = prefetch_multiplier or \
                                conf.CELERYD_PREFETCH_MULTIPLIER
        self.timer_debug = SilenceRepeated(self.logger.debug,
                                           max_iterations=10)
        self.db = db or conf.CELERYD_STATE_DB
        self.disable_rate_limits = disable_rate_limits or \
                                conf.CELERY_DISABLE_RATE_LIMITS
        self._finalize = Finalize(self, self.stop, exitpriority=1)
        self._finalize_db = None

        if self.db:
            persistence = state.Persistent(self.db)
            self._finalize_db = Finalize(persistence,
                                         persistence.save,
                                         exitpriority=5)

        # Queues
        if self.disable_rate_limits:
            self.ready_queue = FastQueue()
            self.ready_queue.put = self.process_task
        else:
            self.ready_queue = TaskBucket(task_registry=registry.tasks)

        self.logger.debug("Instantiating thread components...")

        # Threads + Pool + Consumer
        self.autoscaler = None
        max_concurrency = None
        min_concurrency = concurrency
        if autoscale:
            max_concurrency, min_concurrency = autoscale

        self.pool = instantiate(self.pool_cls,
                                min_concurrency,
                                logger=self.logger,
                                initializer=process_initializer,
                                initargs=(self.app, self.hostname),
                                maxtasksperchild=self.max_tasks_per_child,
                                timeout=self.task_time_limit,
                                soft_timeout=self.task_soft_time_limit,
                                putlocks=self.pool_putlocks)
        self.priority_timer = instantiate(self.pool.Timer)

        if not self.eta_scheduler_cls:
            # Default Timer is set by the pool, as e.g. eventlet
            # needs a custom implementation.
            self.eta_scheduler_cls = self.pool.Timer

        if autoscale:
            self.autoscaler = instantiate(self.autoscaler_cls,
                                          self.pool,
                                          max_concurrency=max_concurrency,
                                          min_concurrency=min_concurrency,
                                          logger=self.logger)

        self.mediator = None
        if not self.disable_rate_limits:
            self.mediator = instantiate(self.mediator_cls,
                                        self.ready_queue,
                                        app=self.app,
                                        callback=self.process_task,
                                        logger=self.logger)

        self.scheduler = instantiate(self.eta_scheduler_cls,
                                     precision=eta_scheduler_precision,
                                     on_error=self.on_timer_error,
                                     on_tick=self.on_timer_tick)

        self.beat = None
        if self.embed_clockservice:
            self.beat = beat.EmbeddedService(
                app=self.app,
                logger=self.logger,
                schedule_filename=self.schedule_filename,
                scheduler_cls=self.scheduler_cls)

        prefetch_count = self.concurrency * self.prefetch_multiplier
        self.consumer = instantiate(self.consumer_cls,
                                    self.ready_queue,
                                    self.scheduler,
                                    logger=self.logger,
                                    hostname=self.hostname,
                                    send_events=self.send_events,
                                    init_callback=self.ready_callback,
                                    initial_prefetch_count=prefetch_count,
                                    pool=self.pool,
                                    priority_timer=self.priority_timer,
                                    app=self.app)

        # The order is important here;
        #   the first in the list is the first to start,
        # and they must be stopped in reverse order.
        self.components = filter(None,
                                 (self.pool, self.mediator, self.scheduler,
                                  self.beat, self.autoscaler, self.consumer))
Ejemplo n.º 21
0
 def Beat(self, **kwargs):
     """Create new :class:`~celery.apps.beat.Beat` instance."""
     return instantiate("celery.apps.beat.Beat", app=self, **kwargs)
Ejemplo n.º 22
0
 def Worker(self, **kwargs):
     """Create new :class:`~celery.apps.worker.Worker` instance."""
     return instantiate("celery.apps.worker.Worker", app=self, **kwargs)
Ejemplo n.º 23
0
Archivo: base.py Proyecto: frac/celery
 def log(self):
     """Logging utilities.  See :class:`~celery.log.Logging`."""
     return instantiate(self.log_cls, app=self)
Ejemplo n.º 24
0
 def control(self):
     """Controlling worker nodes.  See
     :class:`~celery.task.control.Control`."""
     return instantiate(self.control_cls, app=self)
Ejemplo n.º 25
0
__contact__ = "*****@*****.**"
__homepage__ = "http://celeryproject.org"
__docformat__ = "restructuredtext"

if sys.version_info < (2, 5):
    import warnings

    warnings.warn(
        DeprecationWarning(
            """

Python 2.4 support is deprecated and only versions 2.5, 2.6, 2.7+
will be supported starting from Celery version 2.3.


"""
        )
    )


def Celery(*args, **kwargs):
    from celery.app import App

    return App(*args, **kwargs)


if not os.environ.get("CELERY_NO_EVAL", False):
    from celery.utils import LocalProxy, instantiate

    current_app = LocalProxy(lambda: instantiate("celery.app.current_app"))
Ejemplo n.º 26
0
 def Beat(self, **kwargs):
     """Create new :class:`~celery.apps.beat.Beat` instance."""
     return instantiate("celery.apps.beat.Beat", app=self, **kwargs)
Ejemplo n.º 27
0
    def __init__(
        self,
        concurrency=None,
        logfile=None,
        loglevel=None,
        send_events=conf.SEND_EVENTS,
        hostname=None,
        ready_callback=noop,
        embed_clockservice=False,
        pool_cls=conf.CELERYD_POOL,
        listener_cls=conf.CELERYD_LISTENER,
        mediator_cls=conf.CELERYD_MEDIATOR,
        eta_scheduler_cls=conf.CELERYD_ETA_SCHEDULER,
        schedule_filename=conf.CELERYBEAT_SCHEDULE_FILENAME,
    ):

        # Options
        self.loglevel = loglevel or self.loglevel
        self.concurrency = concurrency or self.concurrency
        self.logfile = logfile or self.logfile
        self.logger = setup_logger(loglevel, logfile)
        self.hostname = hostname or socket.gethostname()
        self.embed_clockservice = embed_clockservice
        self.ready_callback = ready_callback
        self.send_events = send_events
        self._finalize = Finalize(self, self.stop, exitpriority=20)

        # Queues
        if conf.DISABLE_RATE_LIMITS:
            self.ready_queue = Queue()
        else:
            self.ready_queue = TaskBucket(task_registry=registry.tasks)
        self.eta_schedule = Scheduler(self.ready_queue, logger=self.logger)

        self.logger.debug("Instantiating thread components...")

        # Threads + Pool + Consumer
        self.pool = instantiate(pool_cls, self.concurrency, logger=self.logger, initializer=process_initializer)
        self.mediator = instantiate(mediator_cls, self.ready_queue, callback=self.process_task, logger=self.logger)
        self.scheduler = instantiate(eta_scheduler_cls, self.eta_schedule, logger=self.logger)

        self.clockservice = None
        if self.embed_clockservice:
            self.clockservice = EmbeddedClockService(logger=self.logger, schedule_filename=schedule_filename)

        prefetch_count = self.concurrency * conf.CELERYD_PREFETCH_MULTIPLIER
        self.listener = instantiate(
            listener_cls,
            self.ready_queue,
            self.eta_schedule,
            logger=self.logger,
            hostname=self.hostname,
            send_events=self.send_events,
            init_callback=self.ready_callback,
            initial_prefetch_count=prefetch_count,
        )

        # The order is important here;
        #   the first in the list is the first to start,
        # and they must be stopped in reverse order.
        self.components = filter(None, (self.pool, self.mediator, self.scheduler, self.clockservice, self.listener))
Ejemplo n.º 28
0
 def expand_route(route):
     if isinstance(route, dict):
         return MapRoute(route)
     if isinstance(route, basestring):
         return instantiate(route)
     return route