Exemple #1
0
    def test_prepare(self):
        from celery.datastructures import LRUCache
        o = object()
        R = [{"foo": "bar"}, "celery.datastructures.LRUCache", o]
        p = routes.prepare(R)
        self.assertIsInstance(p[0], routes.MapRoute)
        self.assertIsInstance(maybe_promise(p[1]), LRUCache)
        self.assertIs(p[2], o)

        self.assertEqual(routes.prepare(o), [o])
Exemple #2
0
    def test_prepare(self):
        from celery.datastructures import LRUCache

        o = object()
        R = [{"foo": "bar"}, "celery.datastructures.LRUCache", o]
        p = routes.prepare(R)
        self.assertIsInstance(p[0], routes.MapRoute)
        self.assertIsInstance(maybe_promise(p[1]), LRUCache)
        self.assertIs(p[2], o)

        self.assertEqual(routes.prepare(o), [o])
Exemple #3
0
 def test_lookup_takes_first(self):
     R = routes.prepare(({"celery.ping": {"queue": "bar"}},
                         {"celery.ping": {"queue": "foo"}}))
     router = routes.Router(R, conf.QUEUES)
     self.assertDictContainsSubset(b_queue,
             router.route({}, "celery.ping",
                 args=[1, 2], kwargs={}))
Exemple #4
0
 def test_lookup_takes_first(self):
     R = routes.prepare(({"celery.ping": {"queue": "bar"}},
                         {"celery.ping": {"queue": "foo"}}))
     router = routes.Router(R, current_app.conf.CELERY_QUEUES)
     self.assertDictContainsSubset(b_queue,
             router.route({}, "celery.ping",
                 args=[1, 2], kwargs={}))
Exemple #5
0
 def test_lookup_paths_traversed(self):
     R = routes.prepare(({"celery.xaza": {"queue": "bar"}},
                         {"celery.ping": {"queue": "foo"}}))
     router = routes.Router(R, conf.QUEUES)
     self.assertDictContainsSubset(a_queue,
             router.route({}, "celery.ping",
                 args=[1, 2], kwargs={}))
     self.assertEqual(router.route({}, "celery.poza"), {})
Exemple #6
0
 def test_lookup_paths_traversed(self):
     R = routes.prepare(({"celery.xaza": {"queue": "bar"}},
                         {"celery.ping": {"queue": "foo"}}))
     router = routes.Router(R, conf.QUEUES)
     self.assertDictContainsSubset(a_queue,
             router.route({}, "celery.ping",
                 args=[1, 2], kwargs={}))
     self.assertEqual(router.route({}, "celery.poza"), {})
Exemple #7
0
 def test_lookup_paths_traversed(self):
     R = routes.prepare(({"celery.xaza": {"queue": "bar"}},
                         {"celery.ping": {"queue": "foo"}}))
     router = routes.Router(R, current_app.amqp.queues)
     self.assertDictContainsSubset(a_queue,
             router.route({}, "celery.ping",
                 args=[1, 2], kwargs={}))
     self.assertEqual(router.route({}, "celery.poza"),
             dict(d_queue, queue=current_app.conf.CELERY_DEFAULT_QUEUE))
Exemple #8
0
 def test_lookup_takes_first(self):
     R = routes.prepare(({
         mytask.name: {
             "queue": "bar"
         }
     }, {
         mytask.name: {
             "queue": "foo"
         }
     }))
     router = routes.Router(R, current_app.conf.CELERY_QUEUES)
     self.assertDictContainsSubset(
         b_queue, router.route({}, mytask.name, args=[1, 2], kwargs={}))
Exemple #9
0
 def test_expands_queue_in_options(self):
     R = routes.prepare(())
     router = routes.Router(R, current_app.conf.CELERY_QUEUES, create_missing=True)
     # apply_async forwards all arguments, even exchange=None etc,
     # so need to make sure it's merged correctly.
     route = router.route(
         {"queue": "testq", "exchange": None, "routing_key": None, "immediate": False},
         mytask.name,
         args=[1, 2],
         kwargs={},
     )
     self.assertDictContainsSubset({"exchange": "testq", "routing_key": "testq", "immediate": False}, route)
     self.assertIn("queue", route)
Exemple #10
0
 def test_lookup_paths_traversed(self):
     R = routes.prepare(({
         "celery.xaza": {
             "queue": "bar"
         }
     }, {
         mytask.name: {
             "queue": "foo"
         }
     }))
     router = routes.Router(R, current_app.amqp.queues)
     self.assertDictContainsSubset(
         a_queue, router.route({}, mytask.name, args=[1, 2], kwargs={}))
     self.assertEqual(
         router.route({}, "celery.poza"),
         dict(d_queue, queue=current_app.conf.CELERY_DEFAULT_QUEUE))
Exemple #11
0
 def test_expands_queue_in_options(self):
     R = routes.prepare(())
     router = routes.Router(R, conf.QUEUES, create_missing=True)
     # apply_async forwards all arguments, even exchange=None etc,
     # so need to make sure it's merged correctly.
     route = router.route({"queue": "testq",
                           "exchange": None,
                           "routing_key": None,
                           "immediate": False},
                          "celery.ping",
                          args=[1, 2], kwargs={})
     self.assertDictContainsSubset({"exchange": "testq",
                                    "routing_key": "testq",
                                    "immediate": False},
                                    route)
     self.assertNotIn("queue", route)
Exemple #12
0
 def post_config_merge(self, c):
     """Prepare configuration after it has been merged with the
     defaults."""
     if not c.get("CELERY_QUEUES"):
         c["CELERY_QUEUES"] = {
             c.CELERY_DEFAULT_QUEUE: {
                 "exchange": c.CELERY_DEFAULT_EXCHANGE,
                 "exchange_type": c.CELERY_DEFAULT_EXCHANGE_TYPE,
                 "binding_key": c.CELERY_DEFAULT_ROUTING_KEY}}
     c["CELERY_ROUTES"] = routes.prepare(c.get("CELERY_ROUTES") or {})
     if c.get("CELERYD_LOG_COLOR") is None:
         c["CELERYD_LOG_COLOR"] = not c.CELERYD_LOG_FILE and \
                                     isatty(sys.stderr)
         if self.IS_WINDOWS:  # windows console doesn't support ANSI colors
             c["CELERYD_LOG_COLOR"] = False
     if isinstance(c.CELERY_TASK_RESULT_EXPIRES, int):
         c["CELERY_TASK_RESULT_EXPIRES"] = timedelta(
                 seconds=c.CELERY_TASK_RESULT_EXPIRES)
     return c
Exemple #13
0
 def test_prepare_item_is_dict(self):
     R = {"foo": "bar"}
     p = routes.prepare(R)
     self.assertIsInstance(p[0], routes.MapRoute)
Exemple #14
0
 def flush_routes(self):
     self._rtable = _routes.prepare(self.app.conf.CELERY_ROUTES)
Exemple #15
0
 def flush_routes(self):
     self._rtable = _routes.prepare(self.app.conf.CELERY_ROUTES)
Exemple #16
0
 def test_prepare_item_is_dict(self):
     R = {"foo": "bar"}
     p = routes.prepare(R)
     self.assertIsInstance(p[0], routes.MapRoute)
Exemple #17
0
# <--- Message routing                             <-   --   --- - ----- -- #
DEFAULT_QUEUE = _get("CELERY_DEFAULT_QUEUE")
DEFAULT_ROUTING_KEY = _get("CELERY_DEFAULT_ROUTING_KEY")
DEFAULT_EXCHANGE = _get("CELERY_DEFAULT_EXCHANGE")
DEFAULT_EXCHANGE_TYPE = _get("CELERY_DEFAULT_EXCHANGE_TYPE")
DEFAULT_DELIVERY_MODE = _get("CELERY_DEFAULT_DELIVERY_MODE")
QUEUES = _get("CELERY_QUEUES") or {
    DEFAULT_QUEUE: {
        "exchange": DEFAULT_EXCHANGE,
        "exchange_type": DEFAULT_EXCHANGE_TYPE,
        "binding_key": DEFAULT_ROUTING_KEY
    }
}
CREATE_MISSING_QUEUES = _get("CELERY_CREATE_MISSING_QUEUES")
ROUTES = routes.prepare(_get("CELERY_ROUTES") or [])
# :--- Broadcast queue settings                     <-   --   --- - ----- -- #

BROADCAST_QUEUE = _get("CELERY_BROADCAST_QUEUE")
BROADCAST_EXCHANGE = _get("CELERY_BROADCAST_EXCHANGE")
BROADCAST_EXCHANGE_TYPE = _get("CELERY_BROADCAST_EXCHANGE_TYPE")

# :--- Event queue settings                         <-   --   --- - ----- -- #

EVENT_QUEUE = _get("CELERY_EVENT_QUEUE")
EVENT_EXCHANGE = _get("CELERY_EVENT_EXCHANGE")
EVENT_EXCHANGE_TYPE = _get("CELERY_EVENT_EXCHANGE_TYPE")
EVENT_ROUTING_KEY = _get("CELERY_EVENT_ROUTING_KEY")
EVENT_SERIALIZER = _get("CELERY_EVENT_SERIALIZER")

# :--- AMQP Backend settings                        <-   --   --- - ----- -- #
Exemple #18
0
def prepare(m, source=settings, defaults=_DEFAULTS):

    def _get(name, default=None, compat=None):
        compat = compat or []
        if default is None:
            default = defaults.get(name)
        compat = [name] + compat
        for i, alias in enumerate(compat):
            try:
                value = getattr(source, alias)
                i > 0 and warnings.warn(
                        DeprecationWarning(_DEPRECATION_FMT % (alias, name)))
                return value
            except AttributeError:
                pass
        return default

    # <--- Task                                    <-   --   --- - ----- -- #
    m.ALWAYS_EAGER = _get("CELERY_ALWAYS_EAGER")
    m.EAGER_PROPAGATES_EXCEPTIONS = _get("CELERY_EAGER_PROPAGATES_EXCEPTIONS")
    m.RESULT_BACKEND = _get("CELERY_RESULT_BACKEND", compat=["CELERY_BACKEND"])
    m.CELERY_BACKEND = m.RESULT_BACKEND              # FIXME Remove in 1.4
    m.CACHE_BACKEND = _get("CELERY_CACHE_BACKEND") or _get("CACHE_BACKEND")
    m.CACHE_BACKEND_OPTIONS = _get("CELERY_CACHE_BACKEND_OPTIONS") or {}
    m.TASK_SERIALIZER = _get("CELERY_TASK_SERIALIZER")
    m.TASK_RESULT_EXPIRES = _get("CELERY_TASK_RESULT_EXPIRES")
    m.AMQP_TASK_RESULT_EXPIRES = _get("CELERY_AMQP_TASK_RESULT_EXPIRES")
    m.IGNORE_RESULT = _get("CELERY_IGNORE_RESULT")
    m.TRACK_STARTED = _get("CELERY_TRACK_STARTED")
    m.ACKS_LATE = _get("CELERY_ACKS_LATE")
    m.REDIRECT_STDOUTS = _get("CELERY_REDIRECT_STDOUTS")
    m.REDIRECT_STDOUTS_LEVEL = _get("CELERY_REDIRECT_STDOUTS_LEVEL")

    # Make sure TASK_RESULT_EXPIRES is a timedelta.
    if isinstance(m.TASK_RESULT_EXPIRES, int):
        m.TASK_RESULT_EXPIRES = timedelta(seconds=m.TASK_RESULT_EXPIRES)

    # <--- SQLAlchemy                              <-   --   --- - ----- -- #
    m.RESULT_DBURI = _get("CELERY_RESULT_DBURI")
    m.RESULT_ENGINE_OPTIONS = _get("CELERY_RESULT_ENGINE_OPTIONS")

    # <--- Client                                  <-   --   --- - ----- -- #

    m.MAX_CACHED_RESULTS = _get("CELERY_MAX_CACHED_RESULTS")

    # <--- Worker                                  <-   --   --- - ----- -- #

    m.SEND_EVENTS = _get("CELERY_SEND_EVENTS")
    m.DEFAULT_RATE_LIMIT = _get("CELERY_DEFAULT_RATE_LIMIT")
    m.DISABLE_RATE_LIMITS = _get("CELERY_DISABLE_RATE_LIMITS")
    m.CELERYD_TASK_TIME_LIMIT = _get("CELERYD_TASK_TIME_LIMIT")
    m.CELERYD_TASK_SOFT_TIME_LIMIT = _get("CELERYD_TASK_SOFT_TIME_LIMIT")
    m.CELERYD_MAX_TASKS_PER_CHILD = _get("CELERYD_MAX_TASKS_PER_CHILD")
    m.STORE_ERRORS_EVEN_IF_IGNORED = \
            _get("CELERY_STORE_ERRORS_EVEN_IF_IGNORED")
    m.CELERY_SEND_TASK_ERROR_EMAILS = _get("CELERY_SEND_TASK_ERROR_EMAILS",
                                           False,
                                    compat=["SEND_CELERY_TASK_ERROR_EMAILS"])
    m.CELERY_TASK_ERROR_WHITELIST = _get("CELERY_TASK_ERROR_WHITELIST")
    m.CELERYD_LOG_FORMAT = _get("CELERYD_LOG_FORMAT",
                          compat=["CELERYD_DAEMON_LOG_FORMAT"])
    m.CELERYD_TASK_LOG_FORMAT = _get("CELERYD_TASK_LOG_FORMAT")
    m.CELERYD_LOG_FILE = _get("CELERYD_LOG_FILE")
    m.CELERYD_LOG_COLOR = _get("CELERYD_LOG_COLOR",
                       m.CELERYD_LOG_FILE is None and isatty(sys.stderr))
    if IS_WINDOWS:          # windows console does not support ANSI colors.
        m.CELERYD_LOG_COLOR = False

    m.CELERYD_LOG_LEVEL = _get("CELERYD_LOG_LEVEL",
                            compat=["CELERYD_DAEMON_LOG_LEVEL"])
    if not isinstance(m.CELERYD_LOG_LEVEL, int):
        m.CELERYD_LOG_LEVEL = LOG_LEVELS[m.CELERYD_LOG_LEVEL.upper()]
    m.CELERYD_HIJACK_ROOT_LOGGER = _get("CELERYD_HIJACK_ROOT_LOGGER")
    m.CELERYD_STATE_DB = _get("CELERYD_STATE_DB")
    m.CELERYD_CONCURRENCY = _get("CELERYD_CONCURRENCY")
    m.CELERYD_PREFETCH_MULTIPLIER = _get("CELERYD_PREFETCH_MULTIPLIER")
    m.CELERYD_POOL_PUTLOCKS = _get("CELERYD_POOL_PUTLOCKS")

    m.CELERYD_POOL = _get("CELERYD_POOL")
    m.CELERYD_LISTENER = _get("CELERYD_LISTENER")
    m.CELERYD_MEDIATOR = _get("CELERYD_MEDIATOR")
    m.CELERYD_ETA_SCHEDULER = _get("CELERYD_ETA_SCHEDULER")
    m.CELERYD_ETA_SCHEDULER_PRECISION = _get("CELERYD_ETA_SCHEDULER_PRECISION")

    # :--- Email settings                           <-   --   --- - ----- -- #
    m.ADMINS = _get("ADMINS")
    m.SERVER_EMAIL = _get("SERVER_EMAIL")
    m.EMAIL_HOST = _get("EMAIL_HOST")
    m.EMAIL_HOST_USER = _get("EMAIL_HOST_USER")
    m.EMAIL_HOST_PASSWORD = _get("EMAIL_HOST_PASSWORD")
    m.EMAIL_PORT = _get("EMAIL_PORT")

    # :--- Broker connections                       <-   --   --- - ----- -- #
    m.BROKER_HOST = _get("BROKER_HOST")
    m.BROKER_PORT = _get("BROKER_PORT")
    m.BROKER_USER = _get("BROKER_USER")
    m.BROKER_PASSWORD = _get("BROKER_PASSWORD")
    m.BROKER_VHOST = _get("BROKER_VHOST")
    m.BROKER_USE_SSL = _get("BROKER_USE_SSL")
    m.BROKER_INSIST = _get("BROKER_INSIST")
    m.BROKER_CONNECTION_TIMEOUT = _get("BROKER_CONNECTION_TIMEOUT",
                                compat=["CELERY_BROKER_CONNECTION_TIMEOUT"])
    m.BROKER_CONNECTION_RETRY = _get("BROKER_CONNECTION_RETRY",
                                compat=["CELERY_BROKER_CONNECTION_RETRY"])
    m.BROKER_CONNECTION_MAX_RETRIES = _get("BROKER_CONNECTION_MAX_RETRIES",
                            compat=["CELERY_BROKER_CONNECTION_MAX_RETRIES"])
    m.BROKER_BACKEND = _get("BROKER_TRANSPORT") or \
                            _get("BROKER_BACKEND") or \
                                _get("CARROT_BACKEND")

    # <--- Message routing                         <-   --   --- - ----- -- #
    m.DEFAULT_QUEUE = _get("CELERY_DEFAULT_QUEUE")
    m.DEFAULT_ROUTING_KEY = _get("CELERY_DEFAULT_ROUTING_KEY")
    m.DEFAULT_EXCHANGE = _get("CELERY_DEFAULT_EXCHANGE")
    m.DEFAULT_EXCHANGE_TYPE = _get("CELERY_DEFAULT_EXCHANGE_TYPE")
    m.DEFAULT_DELIVERY_MODE = _get("CELERY_DEFAULT_DELIVERY_MODE")
    m.QUEUES = _get("CELERY_QUEUES") or {
                    m.DEFAULT_QUEUE: {
                        "exchange": m.DEFAULT_EXCHANGE,
                        "exchange_type": m.DEFAULT_EXCHANGE_TYPE,
                        "binding_key": m.DEFAULT_ROUTING_KEY,
                    },
    }
    m.CREATE_MISSING_QUEUES = _get("CELERY_CREATE_MISSING_QUEUES")
    m.ROUTES = routes.prepare(_get("CELERY_ROUTES") or [])
    # :--- Broadcast queue settings                 <-   --   --- - ----- -- #

    m.BROADCAST_QUEUE = _get("CELERY_BROADCAST_QUEUE")
    m.BROADCAST_EXCHANGE = _get("CELERY_BROADCAST_EXCHANGE")
    m.BROADCAST_EXCHANGE_TYPE = _get("CELERY_BROADCAST_EXCHANGE_TYPE")

    # :--- Event queue settings                     <-   --   --- - ----- -- #

    m.EVENT_QUEUE = _get("CELERY_EVENT_QUEUE")
    m.EVENT_EXCHANGE = _get("CELERY_EVENT_EXCHANGE")
    m.EVENT_EXCHANGE_TYPE = _get("CELERY_EVENT_EXCHANGE_TYPE")
    m.EVENT_ROUTING_KEY = _get("CELERY_EVENT_ROUTING_KEY")
    m.EVENT_PERSISTENT = _get("CELERY_EVENT_PERSISTENT")
    m.EVENT_SERIALIZER = _get("CELERY_EVENT_SERIALIZER")

    # :--- AMQP Backend settings                    <-   --   --- - ----- -- #

    m.RESULT_EXCHANGE = _get("CELERY_RESULT_EXCHANGE")
    m.RESULT_EXCHANGE_TYPE = _get("CELERY_RESULT_EXCHANGE_TYPE")
    m.RESULT_SERIALIZER = _get("CELERY_RESULT_SERIALIZER")
    m.RESULT_PERSISTENT = _get("CELERY_RESULT_PERSISTENT")

    # :--- Celery Beat                              <-   --   --- - ----- -- #
    m.CELERYBEAT_LOG_LEVEL = _get("CELERYBEAT_LOG_LEVEL")
    m.CELERYBEAT_LOG_FILE = _get("CELERYBEAT_LOG_FILE")
    m.CELERYBEAT_SCHEDULE = _get("CELERYBEAT_SCHEDULE")
    m.CELERYBEAT_SCHEDULER = _get("CELERYBEAT_SCHEDULER")
    m.CELERYBEAT_SCHEDULE_FILENAME = _get("CELERYBEAT_SCHEDULE_FILENAME")
    m.CELERYBEAT_MAX_LOOP_INTERVAL = _get("CELERYBEAT_MAX_LOOP_INTERVAL")

    # :--- Celery Monitor                           <-   --   --- - ----- -- #
    m.CELERYMON_LOG_LEVEL = _get("CELERYMON_LOG_LEVEL")
    m.CELERYMON_LOG_FILE = _get("CELERYMON_LOG_FILE")
Exemple #19
0
BROKER_CONNECTION_MAX_RETRIES = _get("BROKER_CONNECTION_MAX_RETRIES",
                            compat=["CELERY_BROKER_CONNECTION_MAX_RETRIES"])
BROKER_BACKEND = _get("BROKER_BACKEND") or _get("CARROT_BACKEND")

# <--- Message routing                             <-   --   --- - ----- -- #
DEFAULT_QUEUE = _get("CELERY_DEFAULT_QUEUE")
DEFAULT_ROUTING_KEY = _get("CELERY_DEFAULT_ROUTING_KEY")
DEFAULT_EXCHANGE = _get("CELERY_DEFAULT_EXCHANGE")
DEFAULT_EXCHANGE_TYPE = _get("CELERY_DEFAULT_EXCHANGE_TYPE")
DEFAULT_DELIVERY_MODE = _get("CELERY_DEFAULT_DELIVERY_MODE")
QUEUES = _get("CELERY_QUEUES") or {DEFAULT_QUEUE: {
                                       "exchange": DEFAULT_EXCHANGE,
                                       "exchange_type": DEFAULT_EXCHANGE_TYPE,
                                       "binding_key": DEFAULT_ROUTING_KEY}}
CREATE_MISSING_QUEUES = _get("CELERY_CREATE_MISSING_QUEUES")
ROUTES = routes.prepare(_get("CELERY_ROUTES") or [])
# :--- Broadcast queue settings                     <-   --   --- - ----- -- #

BROADCAST_QUEUE = _get("CELERY_BROADCAST_QUEUE")
BROADCAST_EXCHANGE = _get("CELERY_BROADCAST_EXCHANGE")
BROADCAST_EXCHANGE_TYPE = _get("CELERY_BROADCAST_EXCHANGE_TYPE")

# :--- Event queue settings                         <-   --   --- - ----- -- #

EVENT_QUEUE = _get("CELERY_EVENT_QUEUE")
EVENT_EXCHANGE = _get("CELERY_EVENT_EXCHANGE")
EVENT_EXCHANGE_TYPE = _get("CELERY_EVENT_EXCHANGE_TYPE")
EVENT_ROUTING_KEY = _get("CELERY_EVENT_ROUTING_KEY")
EVENT_SERIALIZER = _get("CELERY_EVENT_SERIALIZER")

# :--- AMQP Backend settings                        <-   --   --- - ----- -- #