def __init__(self, loglevel=None, hostname=None, ready_callback=noop, queues=None, app=None, pidfile=None, **kwargs): self.app = app_or_default(app or self.app) # all new threads start without a current app, so if an app is not # passed on to the thread it will fall back to the "default app", # which then could be the wrong app. So for the worker # we set this to always return our app. This is a hack, # and means that only a single app can be used for workers # running in the same process. set_default_app(self.app) self._shutdown_complete = Event() self.setup_defaults(kwargs, namespace="celeryd") self.app.select_queues(queues) # select queues subset. # Options self.loglevel = loglevel or self.loglevel self.hostname = hostname or socket.gethostname() self.ready_callback = ready_callback self._finalize = Finalize(self, self.stop, exitpriority=1) self.pidfile = pidfile self.pidlock = None self.use_eventloop = (detect_environment() == "default" and self.app.broker_connection().is_evented) # Initialize boot steps self.pool_cls = _concurrency.get_implementation(self.pool_cls) self.components = [] self.namespace = Namespace(app=self.app).apply(self, **kwargs)
def __init__(self, channel, handlers=None, routing_key='#', node_id=None, app=None, queue_prefix=None, accept=None, queue_ttl=None, queue_expires=None): self.app = app_or_default(app or self.app) self.channel = maybe_channel(channel) self.handlers = {} if handlers is None else handlers self.routing_key = routing_key self.node_id = node_id or uuid() self.queue_prefix = queue_prefix or self.app.conf.event_queue_prefix self.exchange = get_exchange( self.connection or self.app.connection_for_write()) if queue_ttl is None: queue_ttl = self.app.conf.event_queue_ttl if queue_expires is None: queue_expires = self.app.conf.event_queue_expires self.queue = Queue( '.'.join([self.queue_prefix, self.node_id]), exchange=self.exchange, routing_key=self.routing_key, auto_delete=True, durable=False, message_ttl=queue_ttl, expires=queue_expires, ) self.clock = self.app.clock self.adjust_clock = self.clock.adjust self.forward_clock = self.clock.forward if accept is None: accept = {self.app.conf.event_serializer, 'json'} self.accept = accept
def migrate_tasks(source, dest, timeout=1.0, app=None, migrate=None, callback=None): state = State() app = app_or_default(app) def update_state(body, message): state.count += 1 producer = app.amqp.TaskPublisher(dest) if migrate is None: migrate = partial(migrate_task, producer) if callback is not None: callback = partial(callback, state) consumer = app.amqp.get_task_consumer(source) consumer.register_callback(update_state) consumer.register_callback(callback) consumer.register_callback(migrate) # declare all queues on the new broker. for queue in consumer.queues: queue(producer.channel).declare() try: _, mcount, _ = queue(consumer.channel).queue_declare(passive=True) if mcount: state.total_apx += mcount except source.channel_errors + (StdChannelError, ): pass # start migrating messages. with consumer: try: for _ in eventloop(source, timeout=timeout): pass except socket.timeout: return
def policy_main(app=None, loglevel=0, logfile=None, **kwargs): """ Policy manager entry-point function. """ import logging import sys # Set process name that appears in logging messages. import multiprocessing multiprocessing.current_process().name = 'cm-policy-manager' app = app_or_default(app) if not isinstance(loglevel, int): loglevel = LOG_LEVELS[loglevel.upper()] logger = app.log.setup_logger(loglevel=loglevel, logfile=logfile, name="cm.policy") orig_ios = (sys.stdout, sys.stderr) logger.warning('-> cmpolicy: Loading policy manager...') conn = app.broker_connection() try: try: with PolicyMain(conn, logger, app=app) as pmain: pmain.loop() except (KeyboardInterrupt, SystemExit): pass except Exception: import traceback tb = traceback.format_exc() logger.error('\n'+tb) raise finally: conn.close() logger.warning('-> cmpolicy: Policy manager shut down.\n') sys.stdout, sys.stderr = orig_ios
def redis(app=None): app = app_or_default(app) if not hasattr(app, "redbeat_redis") or app.redbeat_redis is None: app.redbeat_redis = StrictRedis.from_url(app.conf.REDBEAT_REDIS_URL, decode_responses=True) return app.redbeat_redis
def handle(self, *args, **options): w = Worker(app_or_default()) options = { 'loglevel': 'DEBUG', 'traceback': True } w.run(**options)
def evcam(camera, freq=1.0, maxrate=None, loglevel=0, logfile=None, app=None): app = app_or_default(app) if not isinstance(loglevel, int): loglevel = LOG_LEVELS[loglevel.upper()] logger = app.log.setup_logger(loglevel=loglevel, logfile=logfile, name="celery.evcam") logger.info( "-> evcam: Taking snapshots with %s (every %s secs.)\n" % ( camera, freq)) state = State() cam = instantiate(camera, state, app=app, freq=freq, maxrate=maxrate, logger=logger) cam.install() conn = app.broker_connection() recv = EventReceiver(conn, app=app, handlers={"*": state.event}) try: try: recv.capture(limit=None) except KeyboardInterrupt: raise SystemExit finally: cam.cancel() conn.close()
def __init__(self, app=None, *args, **kwargs): super(CeleryProc, self).__init__(*args, **kwargs) if app is not None: self.app = app self.app = app_or_default(self.app) self.connection = self.app.connection() self.channel = self.connection.channel()
def __init__(self, concurrency=None, loglevel=None, logfile=None, hostname=None, discard=False, run_clockservice=False, schedule=None, task_time_limit=None, task_soft_time_limit=None, max_tasks_per_child=None, queues=None, events=False, db=None, include=None, app=None, pidfile=None, redirect_stdouts=None, redirect_stdouts_level=None, autoscale=None, scheduler_cls=None, pool=None, **kwargs): self.app = app = app_or_default(app) self.concurrency = (concurrency or app.conf.CELERYD_CONCURRENCY or multiprocessing.cpu_count()) self.loglevel = loglevel or app.conf.CELERYD_LOG_LEVEL self.logfile = logfile or app.conf.CELERYD_LOG_FILE self.hostname = hostname or socket.gethostname() self.discard = discard self.run_clockservice = run_clockservice if self.app.IS_WINDOWS and self.run_clockservice: self.die("-B option does not work on Windows. " "Please run celerybeat as a separate service.") self.schedule = schedule or app.conf.CELERYBEAT_SCHEDULE_FILENAME self.scheduler_cls = scheduler_cls or app.conf.CELERYBEAT_SCHEDULER self.events = events self.task_time_limit = (task_time_limit or app.conf.CELERYD_TASK_TIME_LIMIT) self.task_soft_time_limit = (task_soft_time_limit or app.conf.CELERYD_TASK_SOFT_TIME_LIMIT) self.max_tasks_per_child = (max_tasks_per_child or app.conf.CELERYD_MAX_TASKS_PER_CHILD) self.redirect_stdouts = (redirect_stdouts or app.conf.CELERY_REDIRECT_STDOUTS) self.redirect_stdouts_level = (redirect_stdouts_level or app.conf.CELERY_REDIRECT_STDOUTS_LEVEL) self.pool = (pool or app.conf.CELERYD_POOL) self.db = db self.use_queues = queues or [] self.queues = None self.include = include or [] self.pidfile = pidfile self.autoscale = None if autoscale: max_c, _, min_c = partition(autoscale, ",") self.autoscale = [int(max_c), min_c and int(min_c) or 0] self._isatty = sys.stdout.isatty() self.colored = app.log.colored(self.logfile) if isinstance(self.use_queues, basestring): self.use_queues = self.use_queues.split(",") if isinstance(self.include, basestring): self.include = self.include.split(",") if not isinstance(self.loglevel, int): try: self.loglevel = LOG_LEVELS[self.loglevel.upper()] except KeyError: self.die("Unknown level %r. Please use one of %s." % ( self.loglevel, "|".join(l for l in LOG_LEVELS.keys() if isinstance(l, basestring))))
def get_backend_cls(backend, loader=None): """Get backend class by name/alias""" loader = loader or app_or_default().loader if backend not in _backend_cache: aliases = dict(BACKEND_ALIASES, **loader.override_backends) _backend_cache[backend] = get_cls_by_name(backend, aliases) return _backend_cache[backend]
def move_tasks(conn, predicate, exchange, routing_key, app=None, **kwargs): """Find tasks by filtering them and move the tasks to a new queue. :param conn: Connection to use. :param predicate: Filter function with signature ``(body, message)``. :param exchange: Destination exchange. :param routing_key: Destination routing key. Also supports the same keyword arguments as :func:`start_filter`. To demonstrate, the :func:`move_task_by_id` operation can be implemented like this: .. code-block:: python def is_wanted_task(body, message): if body['id'] == wanted_id: return True move_tasks(conn, is_wanted_task, exchange, routing_key) """ app = app_or_default(app) producer = app.amqp.TaskProducer(conn) def on_task(body, message): if predicate(body, message): republish(producer, message, exchange=exchange, routing_key=routing_key) message.ack() return start_filter(app, conn, on_task, **kwargs)
def main(): def print_event(event): type = event.pop("type") s = "Event: {0}\n".format(type) keys = event.keys() keys.sort() for k in keys: v = event[k] s += " {0}: {1}\n".format(k, v) print s print "Initializing event listener." app = app_or_default(None) conn = app.broker_connection() recv = EventReceiver(conn, handlers={"*": print_event}, app=app) try: try: print "Listening for events... (use Ctrl-C to exit)" recv.capture(limit=None) except KeyboardInterrupt, SystemExit: raise SystemExit except Exception: import traceback print "Exception while listening for events:\n" traceback.print_exc()
def __init__(self, max_interval=None, app=None, socket_timeout=30, pidfile=None, no_color=None, loglevel=None, logfile=None, schedule=None, scheduler_cls=None, redirect_stdouts=None, redirect_stdouts_level=None, **kwargs): """Starts the beat task scheduler.""" self.app = app = app_or_default(app or self.app) self.loglevel = self._getopt('log_level', loglevel) self.logfile = self._getopt('log_file', logfile) self.schedule = self._getopt('schedule_filename', schedule) self.scheduler_cls = self._getopt('scheduler', scheduler_cls) self.redirect_stdouts = self._getopt( 'redirect_stdouts', redirect_stdouts, ) self.redirect_stdouts_level = self._getopt( 'redirect_stdouts_level', redirect_stdouts_level, ) self.max_interval = max_interval self.socket_timeout = socket_timeout self.no_color = no_color self.colored = app.log.colored( self.logfile, enabled=not no_color if no_color is not None else no_color, ) self.pidfile = pidfile if not isinstance(self.loglevel, int): self.loglevel = LOG_LEVELS[self.loglevel.upper()]
def __init__(self, task_name, task_id, args, kwargs, on_ack=noop, retries=0, delivery_info=None, hostname=None, email_subject=None, email_body=None, logger=None, eventer=None, eta=None, expires=None, app=None, taskset_id=None, **opts): self.app = app_or_default(app) self.task_name = task_name self.task_id = task_id self.taskset_id = taskset_id self.retries = retries self.args = args self.kwargs = kwargs self.eta = eta self.expires = expires self.on_ack = on_ack self.delivery_info = delivery_info or {} self.hostname = hostname or socket.gethostname() self.logger = logger or self.app.log.get_default_logger() self.eventer = eventer self.email_subject = email_subject or self.email_subject self.email_body = email_body or self.email_body self.task = tasks[self.task_name] self._store_errors = True if self.task.ignore_result: self._store_errors = self.task.store_errors_even_if_ignored
def evcam(camera, freq=1.0, maxrate=None, loglevel=0, logfile=None, pidfile=None, timer=None, app=None): """Start snapshot recorder.""" app = app_or_default(app) if pidfile: platforms.create_pidlock(pidfile) app.log.setup_logging_subsystem(loglevel, logfile) print('-> evcam: Taking snapshots with {0} (every {1} secs.)'.format( camera, freq)) state = app.events.State() cam = instantiate(camera, state, app=app, freq=freq, maxrate=maxrate, timer=timer) cam.install() conn = app.connection_for_read() recv = app.events.Receiver(conn, handlers={'*': state.event}) try: try: recv.capture(limit=None) except KeyboardInterrupt: raise SystemExit finally: cam.cancel() conn.close()
def test_send_email(self): app = app_or_default() old_mail_admins = app.mail_admins old_enable_mails = mytask.send_error_emails mail_sent = [False] def mock_mail_admins(*args, **kwargs): mail_sent[0] = True app.mail_admins = mock_mail_admins mytask.send_error_emails = True try: tw = TaskRequest(mytask.name, gen_unique_id(), [1], {"f": "x"}) try: raise KeyError("moofoobar") except: einfo = ExceptionInfo(sys.exc_info()) tw.on_failure(einfo) self.assertTrue(mail_sent[0]) mail_sent[0] = False mytask.send_error_emails = False tw.on_failure(einfo) self.assertFalse(mail_sent[0]) finally: app.mail_admins = old_mail_admins mytask.send_error_emails = old_enable_mails
def __init__(self, app=None, serializer=None, max_cached_results=None, **kwargs): from celery.app import app_or_default self.app = app_or_default(app) self.serializer = serializer or self.app.conf.CELERY_RESULT_SERIALIZER (self.content_type, self.content_encoding, self.encoder) = serialization.registry._encoders[self.serializer] self._cache = LRUCache(limit=max_cached_results or self.app.conf.CELERY_MAX_CACHED_RESULTS)
def __init__(self, connection=None, hostname=None, enabled=True, channel=None, buffer_while_offline=True, app=None, serializer=None, groups=None, delivery_mode=1): self.app = app_or_default(app or self.app) self.connection = connection self.channel = channel self.hostname = hostname or anon_nodename() self.buffer_while_offline = buffer_while_offline self.mutex = threading.Lock() self.producer = None self._outbound_buffer = deque() self.serializer = serializer or self.app.conf.CELERY_EVENT_SERIALIZER self.on_enabled = set() self.on_disabled = set() self.groups = set(groups or []) self.tzoffset = [-time.timezone, -time.altzone] self.clock = self.app.clock self.delivery_mode = delivery_mode if not connection and channel: self.connection = channel.connection.client self.enabled = enabled conninfo = self.connection or self.app.connection() self.exchange = get_exchange(conninfo) if conninfo.transport.driver_type in self.DISABLED_TRANSPORTS: self.enabled = False if self.enabled: self.enable() self.headers = {'hostname': self.hostname} self.pid = os.getpid()
def __init__(self, connection=None, hostname=None, enabled=True, channel=None, buffer_while_offline=True, app=None, serializer=None): self.app = app_or_default(app or self.app) self.connection = connection self.channel = channel self.hostname = hostname or socket.gethostname() self.buffer_while_offline = buffer_while_offline self.mutex = threading.Lock() self.producer = None self._outbound_buffer = deque() self.serializer = serializer or self.app.conf.CELERY_EVENT_SERIALIZER self.on_enabled = set() self.on_disabled = set() self.enabled = enabled if not connection and channel: self.connection = channel.connection.client self.enabled = enabled conninfo = self.connection or self.app.connection() self.exchange = get_exchange(conninfo) if conninfo.transport.driver_type in self.DISABLED_TRANSPORTS: self.enabled = False if self.enabled: self.enable()
def __init__( self, body, on_ack=noop, hostname=None, eventer=None, app=None, connection_errors=None, request_dict=None, delivery_info=None, task=None, **opts ): self.app = app or app_or_default(app) name = self.name = body["task"] self.id = body["id"] self.args = body.get("args", []) self.kwargs = body.get("kwargs", {}) try: self.kwargs.items except AttributeError: raise exceptions.InvalidTaskError("Task keyword arguments is not a mapping") if NEEDS_KWDICT: self.kwargs = kwdict(self.kwargs) eta = body.get("eta") expires = body.get("expires") utc = body.get("utc", False) self.on_ack = on_ack self.hostname = hostname or socket.gethostname() self.eventer = eventer self.connection_errors = connection_errors or () self.task = task or self.app.tasks[name] self.acknowledged = self._already_revoked = False self.time_start = self.worker_pid = self._terminate_on_ack = None self._tzlocal = None # timezone means the message is timezone-aware, and the only timezone # supported at this point is UTC. if eta is not None: tz = tz_utc if utc else self.tzlocal self.eta = tz_to_local(maybe_iso8601(eta), self.tzlocal, tz) else: self.eta = None if expires is not None: tz = tz_utc if utc else self.tzlocal self.expires = tz_to_local(maybe_iso8601(expires), self.tzlocal, tz) else: self.expires = None delivery_info = {} if delivery_info is None else delivery_info self.delivery_info = { "exchange": delivery_info.get("exchange"), "routing_key": delivery_info.get("routing_key"), "priority": delivery_info.get("priority"), } # amqplib transport adds the channel here for some reason, so need # to remove it. self.delivery_info.pop("channel", None) self.request_dict = body
def evcam(camera, freq=1.0, maxrate=None, loglevel=0, logfile=None, pidfile=None, timer=None, app=None): app = app_or_default(app) if pidfile: platforms.create_pidlock(pidfile) app.log.setup_logging_subsystem(loglevel, logfile) logger.info( "-> evcam: Taking snapshots with %s (every %s secs.)\n" % ( camera, freq)) state = app.events.State() cam = instantiate(camera, state, app=app, freq=freq, maxrate=maxrate, timer=timer) cam.install() conn = app.broker_connection() recv = app.events.Receiver(conn, handlers={"*": state.event}) try: try: recv.capture(limit=None) except KeyboardInterrupt: raise SystemExit finally: cam.cancel() conn.close()
def task(*args, **kwargs): """Decorator to create a task class out of any callable. .. admonition:: Examples .. code-block:: python @task() def refresh_feed(url): return Feed.objects.get(url=url).refresh() With setting extra options and using retry. .. code-block:: python @task(exchange="feeds") def refresh_feed(url, **kwargs): try: return Feed.objects.get(url=url).refresh() except socket.error, exc: refresh_feed.retry(args=[url], kwargs=kwargs, exc=exc) Calling the resulting task: >>> refresh_feed("http://example.com/rss") # Regular <Feed: http://example.com/rss> >>> refresh_feed.delay("http://example.com/rss") # Async <AsyncResult: 8998d0f4-da0b-4669-ba03-d5ab5ac6ad5d> """ kwargs.setdefault("accept_magic_kwargs", False) return app_or_default().task(*args, **kwargs)
def test_init_queues(self): app = app_or_default() c = app.conf p, app.amqp.queues = app.amqp.queues, { "celery": {"exchange": "celery", "binding_key": "celery"}, "video": {"exchange": "video", "binding_key": "video"}} try: worker = self.Worker(queues=["video"]) worker.init_queues() self.assertIn("video", worker.queues) self.assertNotIn("celery", worker.queues) c.CELERY_CREATE_MISSING_QUEUES = False self.assertRaises(ImproperlyConfigured, self.Worker(queues=["image"]).init_queues) c.CELERY_CREATE_MISSING_QUEUES = True worker = self.Worker(queues=["image"]) worker.init_queues() self.assertIn("image", worker.queues) self.assertDictContainsSubset({"exchange": "image", "routing_key": "image", "binding_key": "image", "exchange_type": "direct"}, worker.queues["image"]) finally: app.amqp.queues = p
def __init__(self, channel, handlers=None, routing_key='#', node_id=None, app=None, queue_prefix=None, accept=None, queue_ttl=None, queue_expires=None): self.app = app_or_default(app or self.app) self.channel = maybe_channel(channel) self.handlers = {} if handlers is None else handlers self.routing_key = routing_key self.node_id = node_id or uuid() self.queue_prefix = queue_prefix if queue_prefix: self.queue_prefix = queue_prefix else: self.queue_prefix = self.app.conf.CELERY_EVENT_QUEUE_PREFIX self.exchange = get_exchange(self.connection or self.app.connection()) self.queue = Queue( '.'.join([self.queue_prefix, self.node_id]), exchange=self.exchange, routing_key=self.routing_key, auto_delete=True, durable=False, queue_arguments=self._get_queue_arguments( ttl=queue_ttl, expires=queue_expires, ), ) self.clock = self.app.clock self.adjust_clock = self.clock.adjust self.forward_clock = self.clock.forward if accept is None: accept = {self.app.conf.CELERY_EVENT_SERIALIZER, 'json'} self.accept = accept
def test_missing_dburi_raises_ImproperlyConfigured(self): conf = app_or_default().conf prev, conf.CELERY_RESULT_DBURI = conf.CELERY_RESULT_DBURI, None try: self.assertRaises(ImproperlyConfigured, DatabaseBackend) finally: conf.CELERY_RESULT_DBURI = prev
def test_lookup_takes_first(self): R = routes.prepare(({"celery.ping": {"queue": "bar"}}, {"celery.ping": {"queue": "foo"}})) router = routes.Router(R, app_or_default().conf.CELERY_QUEUES) self.assertDictContainsSubset(b_queue, router.route({}, "celery.ping", args=[1, 2], kwargs={}))
def __init__(self, loglevel=None, hostname=None, ready_callback=noop, queues=None, app=None, pidfile=None, use_eventloop=None, **kwargs): self.app = app_or_default(app or self.app) self._shutdown_complete = Event() self.setup_defaults(kwargs, namespace='celeryd') self.app.select_queues(queues) # select queues subset. # Options self.loglevel = loglevel or self.loglevel self.hostname = hostname or socket.gethostname() self.ready_callback = ready_callback self._finalize = Finalize(self, self.stop, exitpriority=1) self.pidfile = pidfile self.pidlock = None # this connection is not established, only used for params self._conninfo = self.app.connection() self.use_eventloop = ( self.should_use_eventloop() if use_eventloop is None else use_eventloop ) # Update celery_include to have all known task modules, so that we # ensure all task modules are imported in case an execv happens. task_modules = set(task.__class__.__module__ for task in self.app.tasks.itervalues()) self.app.conf.CELERY_INCLUDE = tuple( set(self.app.conf.CELERY_INCLUDE) | task_modules, ) # Initialize boot steps self.pool_cls = _concurrency.get_implementation(self.pool_cls) self.components = [] self.namespace = Namespace(app=self.app).apply(self, **kwargs)
def __init__(self, tasks=None, app=None, Publisher=None): self.app = app_or_default(app or self.app) super(TaskSet, self).__init__( maybe_signature(t, app=self.app) for t in tasks or [] ) self.Publisher = Publisher or self.app.amqp.Producer self.total = len(self) # XXX compat
def __init__(self, *args, **kwargs): app = self.app = app_or_default(kwargs.get("app")) kwargs["exchange"] = kwargs.get("exchange") or \ app.conf.CELERY_BROADCAST_EXCHANGE kwargs["exchange_type"] = kwargs.get("exchange_type") or \ app.conf.CELERY_BROADCAST_EXCHANGE_TYPE super(BroadcastPublisher, self).__init__(*args, **kwargs)
def handle(self, *args, **options): b = Beat(app_or_default()) options = { 'loglevel': 'DEBUG', 'traceback': True } b.run(**options)
def redis(app=None): app = app_or_default(app) conf = ensure_conf(app) if not hasattr(app, 'redbeat_redis') or app.redbeat_redis is None: redis_options = conf.app.conf.get( 'REDBEAT_REDIS_OPTIONS', conf.app.conf.get('BROKER_TRANSPORT_OPTIONS', {})) if conf.redis_url.startswith( 'redis-sentinel') and 'sentinels' in redis_options: from redis.sentinel import Sentinel sentinel = Sentinel( redis_options['sentinels'], socket_timeout=redis_options.get('socket_timeout'), password=redis_options.get('password'), decode_responses=True) app.redbeat_redis = sentinel.master_for( redis_options.get('service_name', 'master')) else: app.redbeat_redis = StrictRedis.from_url(conf.redis_url, decode_responses=True) return app.redbeat_redis
def __init__(self, max_interval=None, app=None, socket_timeout=30, pidfile=None, no_color=None, **kwargs): """Starts the celerybeat task scheduler.""" self.app = app = app_or_default(app or self.app) self.setup_defaults(kwargs, namespace='celerybeat') self.max_interval = max_interval self.socket_timeout = socket_timeout self.no_color = no_color self.colored = app.log.colored( self.logfile, enabled=not no_color if no_color is not None else no_color, ) self.pidfile = pidfile if not isinstance(self.loglevel, int): self.loglevel = LOG_LEVELS[self.loglevel.upper()]
def migrate_tasks(source, dest, timeout=1.0, app=None, migrate=None, callback=None): state = State() app = app_or_default(app) def update_state(body, message): state.count += 1 producer = app.amqp.TaskProducer(dest) if migrate is None: migrate = partial(migrate_task, producer) consumer = app.amqp.TaskConsumer(source) consumer.register_callback(update_state) if callback is not None: callback = partial(callback, state) consumer.register_callback(callback) consumer.register_callback(migrate) # declare all queues on the new broker. for queue in consumer.queues: queue(producer.channel).declare() try: _, mcount, _ = queue(consumer.channel).queue_declare(passive=True) if mcount: state.total_apx += mcount except source.channel_errors + (StdChannelError, ): pass # start migrating messages. with consumer: try: for _ in eventloop(source, timeout=timeout): # pragma: no cover pass except socket.timeout: return
def __init__(self, channel, handlers=None, routing_key='#', node_id=None, app=None, queue_prefix=None, accept=None, queue_ttl=None, queue_expires=None): self.app = app_or_default(app or self.app) self.channel = maybe_channel(channel) self.handlers = {} if handlers is None else handlers self.routing_key = routing_key self.node_id = node_id or uuid() self.queue_prefix = queue_prefix or self.app.conf.event_queue_prefix self.exchange = get_exchange(self.connection or self.app.connection_for_write(), name=self.app.conf.event_exchange) if queue_ttl is None: queue_ttl = self.app.conf.event_queue_ttl if queue_expires is None: queue_expires = self.app.conf.event_queue_expires self.queue = Queue( '.'.join([self.queue_prefix, self.node_id]), exchange=self.exchange, routing_key=self.routing_key, auto_delete=True, durable=False, message_ttl=queue_ttl, expires=queue_expires, ) self.clock = self.app.clock self.adjust_clock = self.clock.adjust self.forward_clock = self.clock.forward if accept is None: accept = {self.app.conf.event_serializer, 'json'} self.accept = accept
def evcam( camera, freq=1.0, maxrate=None, loglevel=0, logfile=None, pidfile=None, timer=None, app=None, ): """Start snapshot recorder.""" app = app_or_default(app) if pidfile: platforms.create_pidlock(pidfile) app.log.setup_logging_subsystem(loglevel, logfile) print("-> evcam: Taking snapshots with {0} (every {1} secs.)".format( camera, freq)) state = app.events.State() cam = instantiate(camera, state, app=app, freq=freq, maxrate=maxrate, timer=timer) cam.install() conn = app.connection_for_read() recv = app.events.Receiver(conn, handlers={"*": state.event}) try: try: recv.capture(limit=None) except KeyboardInterrupt: raise SystemExit finally: cam.cancel() conn.close()
def _test_on_failure(self, exception): app = app_or_default() tid = uuid() tw = TaskRequest(mytask.name, tid, [4], {"f": "x"}) try: raise exception except Exception: exc_info = ExceptionInfo(sys.exc_info()) logfh = WhateverIO() tw.logger.handlers = [] tw.logger = setup_logger(logfile=logfh, loglevel=logging.INFO, root=False) app.conf.CELERY_SEND_TASK_ERROR_EMAILS = True tw.on_failure(exc_info) logvalue = logfh.getvalue() self.assertIn(mytask.name, logvalue) self.assertIn(tid, logvalue) self.assertIn("ERROR", logvalue) app.conf.CELERY_SEND_TASK_ERROR_EMAILS = False
def __init__(self, connection=None, hostname=None, enabled=True, channel=None, buffer_while_offline=True, app=None, serializer=None): self.app = app_or_default(app) self.connection = connection self.channel = channel self.hostname = hostname or socket.gethostname() self.buffer_while_offline = buffer_while_offline self.mutex = threading.Lock() self.publisher = None self._outbound_buffer = deque() self.serializer = serializer or self.app.conf.CELERY_EVENT_SERIALIZER self.on_enabled = set() self.on_disabled = set() self.enabled = enabled if self.enabled: self.enable()
def __init__(self, loglevel=None, hostname=None, ready_callback=noop, queues=None, app=None, pidfile=None, **kwargs): self.app = app_or_default(app or self.app) # all new threads start without a current app, so if an app is not # passed on to the thread it will fall back to the "default app", # which then could be the wrong app. So for the worker # we set this to always return our app. This is a hack, # and means that only a single app can be used for workers # running in the same process. set_default_app(self.app) self.app.finalize() trace._tasks = self.app._tasks self._shutdown_complete = Event() self.setup_defaults(kwargs, namespace="celeryd") self.app.select_queues(queues) # select queues subset. # Options self.loglevel = loglevel or self.loglevel self.hostname = hostname or socket.gethostname() self.ready_callback = ready_callback self._finalize = Finalize(self, self.stop, exitpriority=1) self.pidfile = pidfile self.pidlock = None self.use_eventloop = (detect_environment() == "default" and self.app.broker_connection().is_evented) # Initialize boot steps self.pool_cls = _concurrency.get_implementation(self.pool_cls) self.components = [] self.namespace = Namespace(app=self.app).apply(self, **kwargs)
def evtop(app=None): sys.stderr.write("-> evtop: starting capture...\n") app = app_or_default(app) state = app.events.State() conn = app.broker_connection() recv = app.events.Receiver(conn, handlers={"*": state.event}) capture = recv.itercapture() consumer = capture.next() display = CursesMonitor(state, app=app) display.init_screen() refresher = DisplayThread(display) refresher.start() try: capture.next() except Exception: refresher.shutdown = True refresher.join() display.resetscreen() raise except (KeyboardInterrupt, SystemExit): conn and conn.close() refresher.shutdown = True refresher.join() display.resetscreen()
def get_redis(app=None): app = app_or_default(app) conf = ensure_conf(app) if not hasattr(app, 'redbeat_redis') or app.redbeat_redis is None: redis_options = conf.app.conf.get( 'REDBEAT_REDIS_OPTIONS', conf.app.conf.get('BROKER_TRANSPORT_OPTIONS', {})) retry_period = redis_options.get('retry_period') if conf.redis_url.startswith( 'redis-sentinel') and 'sentinels' in redis_options: from redis.sentinel import Sentinel sentinel = Sentinel( redis_options['sentinels'], socket_timeout=redis_options.get('socket_timeout'), password=redis_options.get('password'), db=redis_options.get('db', 0), decode_responses=True) connection = sentinel.master_for( redis_options.get('service_name', 'master')) elif conf.redis_url.startswith('rediss'): ssl_options = {'ssl_cert_reqs': ssl.CERT_REQUIRED} if isinstance(conf.redis_use_ssl, dict): ssl_options.update(conf.redis_use_ssl) connection = StrictRedis.from_url(conf.redis_url, decode_responses=True, **ssl_options) else: connection = StrictRedis.from_url(conf.redis_url, decode_responses=True) if retry_period is None: app.redbeat_redis = connection else: app.redbeat_redis = RetryingConnection(retry_period, connection) return app.redbeat_redis
def evcam(camera, freq=1.0, maxrate=None, loglevel=0, logfile=None, pidfile=None, timer=None, app=None): app = app_or_default(app) if pidfile: pidlock = platforms.create_pidlock(pidfile).acquire() atexit.register(pidlock.release) app.log.setup_logging_subsystem(loglevel, logfile) logger.info("-> evcam: Taking snapshots with %s (every %s secs.)\n" % (camera, freq)) state = app.events.State() cam = instantiate(camera, state, app=app, freq=freq, maxrate=maxrate, timer=timer) cam.install() conn = app.broker_connection() recv = app.events.Receiver(conn, handlers={"*": state.event}) try: try: recv.capture(limit=None) except KeyboardInterrupt: raise SystemExit finally: cam.cancel() conn.close()
def __init__(self, task_id, backend=None, task_name=None, app=None): app = app_or_default(app) backend = backend or app.backend super(AsyncResult, self).__init__(task_id, backend, task_name=task_name, app=app)
from __future__ import absolute_import import six from django.core.urlresolvers import reverse from django.views.generic import DetailView, TemplateView from django.utils.translation import ugettext_lazy as _ from django.utils.encoding import force_text from django.template.defaultfilters import escape from celery import app, states celery_app = app.app_or_default() from django.contrib.contenttypes.models import ContentType from ..models import WooeyJob, WooeyFile, Favorite from .. import settings as wooey_settings from ..backend.utils import valid_user, get_file_previews from ..django_compat import JsonResponse from django.db.models import Q from django.views.generic import ListView from django.template.loader import render_to_string SPANBASE = "<span title='{}' class='glyphicon {}'></span> " MAXIMUM_JOBS_NAVBAR = 10 STATE_MAPPER = { # Default Primary Success Info Warning Danger WooeyJob.COMPLETED: SPANBASE.format(_('Success'), 'success glyphicon-ok'), WooeyJob.RUNNING: SPANBASE.format(_('Executing'), 'success glyphicon-refresh spinning'), states.PENDING: SPANBASE.format(_('Queued'), 'glyphicon-time'), states.REVOKED: SPANBASE.format(_('Halted'), 'danger glyphicon-stop'),
def test_parse_options(self): cmd = beat_bin.beat() cmd.app = app_or_default() options, args = cmd.parse_options('celery beat', ['-s', 'foo']) self.assertEqual(options.schedule, 'foo')
def test_detach(self): cmd = beat_bin.beat() cmd.app = app_or_default() cmd.run(detach=True) self.assertTrue(MockDaemonContext.opened) self.assertTrue(MockDaemonContext.closed)
def setUp(self): app = app_or_default() self.i = Control(app=app).inspect()
def setUp(self): self.app = app_or_default() self.app.events = self.MockEvents()
def __init__(self, ready_queue, callback, app=None, **kw): self.app = app_or_default(app) self.ready_queue = ready_queue self.callback = callback self._does_debug = logger.isEnabledFor(logging.DEBUG) super(Mediator, self).__init__()
def setUp(self): app = self.app = app_or_default() self.c = Control(app=app) self.prev, app.control = app.control, self.c self.i = self.c.inspect()
def __init__(self, handle_task, init_callback=noop, hostname=None, pool=None, app=None, timer=None, controller=None, hub=None, amqheartbeat=None, worker_options=None, disable_rate_limits=False, **kwargs): self.app = app_or_default(app) self.controller = controller self.init_callback = init_callback self.hostname = hostname or socket.gethostname() self.pid = os.getpid() self.pool = pool self.timer = timer or default_timer self.strategies = {} conninfo = self.app.connection() self.connection_errors = conninfo.connection_errors self.channel_errors = conninfo.channel_errors self._restart_state = restart_state(maxR=5, maxT=1) self._does_info = logger.isEnabledFor(logging.INFO) self.handle_task = handle_task self.amqheartbeat_rate = self.app.conf.BROKER_HEARTBEAT_CHECKRATE self.disable_rate_limits = disable_rate_limits # this contains a tokenbucket for each task type by name, used for # rate limits, or None if rate limits are disabled for that task. self.task_buckets = defaultdict(lambda: None) self.reset_rate_limits() if hub: self.amqheartbeat = amqheartbeat if self.amqheartbeat is None: self.amqheartbeat = self.app.conf.BROKER_HEARTBEAT self.hub = hub self.hub.on_init.append(self.on_poll_init) else: self.hub = None self.amqheartbeat = 0 if not hasattr(self, 'loop'): self.loop = loops.asynloop if hub else loops.synloop if _detect_environment() == 'gevent': # there's a gevent bug that causes timeouts to not be reset, # so if the connection timeout is exceeded once, it can NEVER # connect again. self.app.conf.BROKER_CONNECTION_TIMEOUT = None self.steps = [] self.blueprint = self.Blueprint( app=self.app, on_close=self.on_close, ) self.blueprint.apply(self, **dict(worker_options or {}, **kwargs))
def __init__(self, concurrency=None, loglevel=None, logfile=None, hostname=None, discard=False, run_clockservice=False, schedule=None, task_time_limit=None, task_soft_time_limit=None, max_tasks_per_child=None, queues=None, events=None, db=None, include=None, app=None, pidfile=None, redirect_stdouts=None, redirect_stdouts_level=None, autoscale=None, scheduler_cls=None, pool=None, **kwargs): self.app = app = app_or_default(app) conf = app.conf self.concurrency = (concurrency or conf.CELERYD_CONCURRENCY or cpu_count()) self.loglevel = loglevel or conf.CELERYD_LOG_LEVEL self.logfile = logfile or conf.CELERYD_LOG_FILE self.hostname = hostname or socket.gethostname() self.discard = discard self.run_clockservice = run_clockservice if self.app.IS_WINDOWS and self.run_clockservice: self.die("-B option does not work on Windows. " "Please run celerybeat as a separate service.") self.schedule = schedule or conf.CELERYBEAT_SCHEDULE_FILENAME self.scheduler_cls = scheduler_cls or conf.CELERYBEAT_SCHEDULER self.events = events if events is not None else conf.CELERY_SEND_EVENTS self.task_time_limit = (task_time_limit or conf.CELERYD_TASK_TIME_LIMIT) self.task_soft_time_limit = (task_soft_time_limit or conf.CELERYD_TASK_SOFT_TIME_LIMIT) self.max_tasks_per_child = (max_tasks_per_child or conf.CELERYD_MAX_TASKS_PER_CHILD) self.redirect_stdouts = (redirect_stdouts or conf.CELERY_REDIRECT_STDOUTS) self.redirect_stdouts_level = (redirect_stdouts_level or conf.CELERY_REDIRECT_STDOUTS_LEVEL) self.pool = pool or conf.CELERYD_POOL self.db = db self.use_queues = [] if queues is None else queues self.queues = None self.include = [] if include is None else include self.pidfile = pidfile self.autoscale = None if autoscale: max_c, _, min_c = autoscale.partition(",") self.autoscale = [int(max_c), min_c and int(min_c) or 0] self._isatty = sys.stdout.isatty() self.colored = app.log.colored(self.logfile) if isinstance(self.use_queues, basestring): self.use_queues = self.use_queues.split(",") if isinstance(self.include, basestring): self.include = self.include.split(",") if not isinstance(self.loglevel, int): try: self.loglevel = LOG_LEVELS[self.loglevel.upper()] except KeyError: self.die( "Unknown level %r. Please use one of %s." % (self.loglevel, "|".join(l for l in LOG_LEVELS.keys() if isinstance(l, basestring))))
def __init__(self, taskset_id, subtasks, app=None): self.taskset_id = taskset_id self.subtasks = subtasks self.app = app_or_default(app)
def setUp(self): self.app = app_or_default() self.state = self.app.events.State()
def __init__(self, app=None, **kwargs): from celery.app import app_or_default self.app = app_or_default(app) self.task_modules = set()
def __init__(self, app=None, no_color=False): self.app = app_or_default(app) self.colored = term.colored(enabled=not no_color)
def setUp(self): self.app = app_or_default() self.control = Control(app=self.app) self.app.control = self.control
def move(predicate, connection=None, exchange=None, routing_key=None, source=None, app=None, callback=None, limit=None, transform=None, **kwargs): """Find tasks by filtering them and move the tasks to a new queue. Arguments: predicate (Callable): Filter function used to decide the messages to move. Must accept the standard signature of ``(body, message)`` used by Kombu consumer callbacks. If the predicate wants the message to be moved it must return either: 1) a tuple of ``(exchange, routing_key)``, or 2) a :class:`~kombu.entity.Queue` instance, or 3) any other true value means the specified ``exchange`` and ``routing_key`` arguments will be used. connection (kombu.Connection): Custom connection to use. source: List[Union[str, kombu.Queue]]: Optional list of source queues to use instead of the default (queues in :setting:`task_queues`). This list can also contain :class:`~kombu.entity.Queue` instances. exchange (str, kombu.Exchange): Default destination exchange. routing_key (str): Default destination routing key. limit (int): Limit number of messages to filter. callback (Callable): Callback called after message moved, with signature ``(state, body, message)``. transform (Callable): Optional function to transform the return value (destination) of the filter function. Also supports the same keyword arguments as :func:`start_filter`. To demonstrate, the :func:`move_task_by_id` operation can be implemented like this: .. code-block:: python def is_wanted_task(body, message): if body['id'] == wanted_id: return Queue('foo', exchange=Exchange('foo'), routing_key='foo') move(is_wanted_task) or with a transform: .. code-block:: python def transform(value): if isinstance(value, str): return Queue(value, Exchange(value), value) return value move(is_wanted_task, transform=transform) Note: The predicate may also return a tuple of ``(exchange, routing_key)`` to specify the destination to where the task should be moved, or a :class:`~kombu.entity.Queue` instance. Any other true value means that the task will be moved to the default exchange/routing_key. """ app = app_or_default(app) queues = [_maybe_queue(app, queue) for queue in source or []] or None with app.connection_or_acquire(connection, pool=False) as conn: producer = app.amqp.Producer(conn) state = State() def on_task(body, message): ret = predicate(body, message) if ret: if transform: ret = transform(ret) if isinstance(ret, Queue): maybe_declare(ret, conn.default_channel) ex, rk = ret.exchange.name, ret.routing_key else: ex, rk = expand_dest(ret, exchange, routing_key) republish(producer, message, exchange=ex, routing_key=rk) message.ack() state.filtered += 1 if callback: callback(state, body, message) if limit and state.filtered >= limit: raise StopFiltering() return start_filter(app, conn, on_task, consume_from=queues, **kwargs)
def __init__(self, task_id, backend, task_name=None, app=None): self.task_id = task_id self.backend = backend self.task_name = task_name self.app = app_or_default(app)