def cast(self, method, args={}, declare=None, retry=None, retry_policy=None, type=None, exchange=None, **props): """Send message to actor. Discarding replies.""" retry = self.retry if retry is None else retry body = {'class': self.name, 'method': method, 'args': args} _retry_policy = self.retry_policy if retry_policy: # merge default and custom policies. _retry_policy = dict(_retry_policy, **retry_policy) if type and type not in self.types: raise ValueError('Unsupported type: {0}'.format(type)) elif not type: type = ACTOR_TYPE.DIRECT props.setdefault('routing_key', self.routing_key) props.setdefault('serializer', self.serializer) exchange = exchange or self.type_to_exchange[type]() declare = (maybe_list(declare) or []) + [exchange] with producers[self._connection].acquire(block=True) as producer: return producer.publish(body, exchange=exchange, declare=declare, retry=retry, retry_policy=retry_policy, **props)
def select(self, include): """Sets :attr:`consume_from` by selecting a subset of the currently defined queues. :param include: Names of queues to consume from. Can be iterable or string. """ if include: self._consume_from = {name: self[name] for name in maybe_list(include)}
def select(self, include): """Select a subset of currently defined queues to consume from. Arguments: include (Sequence[str], str): Names of queues to consume from. """ if include: self._consume_from = { name: self[name] for name in maybe_list(include) }
def select(self, include): """Sets :attr:`consume_from` by selecting a subset of the currently defined queues. :param include: Names of queues to consume from. Can be iterable or string. """ if include: self._consume_from = dict( (name, self[name]) for name in maybe_list(include))
def traverse_subscribers(it, *args, **kwargs): stream = deque([it]) while stream: for node in maybe_list(stream.popleft()): if isinstance(node, string_types) and node.startswith('!'): node = symbol_by_name(node[1:]) if isinstance(node, Callable): node = node(*args, **kwargs) if is_list(node): stream.append(node) elif node: yield node
def deselect(self, exclude): """Deselect queues so that they won't be consumed from. Arguments: exclude (Sequence[str], str): Names of queues to avoid consuming from. """ if exclude: exclude = maybe_list(exclude) if self._consume_from is None: # using all queues return self.select(k for k in self if k not in exclude) # using selection for queue in exclude: self._consume_from.pop(queue, None)
def deselect(self, exclude): """Deselect queues so that they will not be consumed from. :param exclude: Names of queues to avoid consuming from. Can be iterable or string. """ if exclude: exclude = maybe_list(exclude) if self._consume_from is None: # using selection return self.select(k for k in self if k not in exclude) # using all queues for queue in exclude: self._consume_from.pop(queue, None)
def deselect(self, exclude): """Deselect queues so that they won't be consumed from. Arguments: exclude (Sequence[str], str): Names of queues to avoid consuming from. """ if exclude: exclude = maybe_list(exclude) if self._consume_from is None: # using selection return self.select(k for k in self if k not in exclude) # using all queues for queue in exclude: self._consume_from.pop(queue, None)
def test_maybe_list(obj, expected): assert maybe_list(obj) == expected
def perform(self, request, **kwargs): for req in maybe_list(request) or []: if not isinstance(req, self.Request): req = self.Request(req, **kwargs) self.add_request(req)
def send_task(self, name, args=None, kwargs=None, countdown=None, eta=None, task_id=None, producer=None, connection=None, router=None, result_cls=None, expires=None, publisher=None, link=None, link_error=None, add_to_parent=True, group_id=None, retries=0, chord=None, reply_to=None, time_limit=None, soft_time_limit=None, root_id=None, parent_id=None, route_name=None, shadow=None, chain=None, task_type=None, **options): """Send task by name. Supports the same arguments as :meth:`@-Task.apply_async`. Arguments: name (str): Name of task to call (e.g., `"tasks.add"`). result_cls (~@AsyncResult): Specify custom result class. """ # This is a copy of (the top of) celery.app.base.Celery.send_task parent = have_parent = None amqp = self.amqp task_id = task_id or uuid() producer = producer or publisher # XXX compat router = router or amqp.router conf = self.conf if conf.task_always_eager: # pragma: no cover warnings.warn(AlwaysEagerIgnored( 'task_always_eager has no effect on send_task', ), stacklevel=2) options = router.route( options, route_name or name, args, kwargs, task_type) # if not root_id or not parent_id: # parent = self.current_worker_task # if parent: # if not root_id: # root_id = parent.request.root_id or parent.request.id # if not parent_id: # parent_id = parent.request.id message = amqp.create_task_message( task_id, name, args, kwargs, countdown, eta, group_id, expires, retries, chord, maybe_list(link), maybe_list(link_error), reply_to or self.app.oid, time_limit, soft_time_limit, self.conf.task_send_sent_event, root_id, parent_id, shadow, chain, ) # if connection: # producer = amqp.Producer(connection, auto_declare=False) # self.backend.on_task_call(P, task_id) # with self.producer_or_acquire(producer) as P: # with P.connection._reraise_as_library_errors(): # if not ignored_result: # self.backend.on_task_call(P, task_id) # amqp.send_task_message(P, name, message, **options) yield self.channel.ensure_connected() # TODO Handle deferred returned. self.send_task_message(name, message, **options) result = defer.Deferred() self._sent_tasks[task_id] = result # result = (result_cls or self.AsyncResult)(task_id) # if add_to_parent: # if not have_parent: # parent, have_parent = self.current_worker_task, True # if parent: # parent.add_trail(result) # return result result = yield result defer.returnValue(result)
def as_task_v2(self, task_id, name, args=None, kwargs=None, countdown=None, eta=None, group_id=None, group_index=None, expires=None, retries=0, chord=None, callbacks=None, errbacks=None, reply_to=None, time_limit=None, soft_time_limit=None, create_sent_event=False, root_id=None, parent_id=None, shadow=None, chain=None, now=None, timezone=None, origin=None, ignore_result=False, argsrepr=None, kwargsrepr=None, stamped_headers=None, **options): args = args or () kwargs = kwargs or {} if not isinstance(args, (list, tuple)): raise TypeError('task args must be a list or tuple') if not isinstance(kwargs, Mapping): raise TypeError('task keyword arguments must be a mapping') if countdown: # convert countdown to ETA self._verify_seconds(countdown, 'countdown') now = now or self.app.now() timezone = timezone or self.app.timezone eta = maybe_make_aware( now + timedelta(seconds=countdown), tz=timezone, ) if isinstance(expires, numbers.Real): self._verify_seconds(expires, 'expires') now = now or self.app.now() timezone = timezone or self.app.timezone expires = maybe_make_aware( now + timedelta(seconds=expires), tz=timezone, ) if not isinstance(eta, str): eta = eta and eta.isoformat() # If we retry a task `expires` will already be ISO8601-formatted. if not isinstance(expires, str): expires = expires and expires.isoformat() if argsrepr is None: argsrepr = saferepr(args, self.argsrepr_maxsize) if kwargsrepr is None: kwargsrepr = saferepr(kwargs, self.kwargsrepr_maxsize) if not root_id: # empty root_id defaults to task_id root_id = task_id stamps = { header: maybe_list(options[header]) for header in stamped_headers or [] } headers = { 'lang': 'py', 'task': name, 'id': task_id, 'shadow': shadow, 'eta': eta, 'expires': expires, 'group': group_id, 'group_index': group_index, 'retries': retries, 'timelimit': [time_limit, soft_time_limit], 'root_id': root_id, 'parent_id': parent_id, 'argsrepr': argsrepr, 'kwargsrepr': kwargsrepr, 'origin': origin or anon_nodename(), 'ignore_result': ignore_result, 'stamped_headers': stamped_headers, 'stamps': stamps, } return task_message( headers=headers, properties={ 'correlation_id': task_id, 'reply_to': reply_to or '', }, body=( args, kwargs, { 'callbacks': callbacks, 'errbacks': errbacks, 'chain': chain, 'chord': chord, }, ), sent_event={ 'uuid': task_id, 'root_id': root_id, 'parent_id': parent_id, 'name': name, 'args': argsrepr, 'kwargs': kwargsrepr, 'retries': retries, 'eta': eta, 'expires': expires, } if create_sent_event else None, )
def test_maybe_list(self): self.assertIsNone(maybe_list(None)) self.assertEqual(maybe_list(1), [1]) self.assertEqual(maybe_list([1, 2, 3]), [1, 2, 3])
def add(self, fds, callback, flags, consolidate=False): for fd in maybe_list(fds, None): try: self._add(fd, callback, flags, consolidate) except ValueError: self._discard(fd)
CELERY_BROKER_HOSTNAME, CELERY_BROKER_VHOST, ) CELERY_MONGODB_BACKEND_SETTINGS = { "database": MONGODB_DB_NAME, "taskmeta_collection": "taskmeta_collection", } CELERY_RESULT_BACKEND = "mongodb://{}/".format(MONGODB_HOST) CELERY_RESULT_DB_TABLENAMES = { "task": "celery_edx_task", "group": "celery_edx_group" } CELERY_IMPORTS = locals().get("CELERY_IMPORTS", []) # XXX for some reason celery is not registering the bookmarks app CELERY_IMPORTS = list(maybe_list(CELERY_IMPORTS)) + [ "openedx.core.djangoapps.bookmarks.tasks", "openedx.core.djangoapps.content.course_overviews.tasks", ] CELERYBEAT_SCHEDULE = {} CELERY_ROUTES = "{}.celery.Router".format(SERVICE_VARIANT) CELERY_QUEUES = {"openedx.lms.default": {}, "openedx.cms.default": {}} CELERY_DEFAULT_EXCHANGE = "openedx.{}".format(SERVICE_VARIANT) CELERY_DEFAULT_QUEUE = "{}.default".format(CELERY_DEFAULT_EXCHANGE) HIGH_PRIORITY_QUEUE = CELERY_DEFAULT_QUEUE DEFAULT_PRIORITY_QUEUE = CELERY_DEFAULT_QUEUE HIGH_MEM_QUEUE = CELERY_DEFAULT_QUEUE CELERY_DEFAULT_ROUTING_KEY = DEFAULT_PRIORITY_QUEUE
# Load aws.py in plugins for reverse compatibility. This can be removed after aws.py # is officially removed. plugin_settings.add_plugins(__name__, PROJECT_TYPE, plugin_constants.SettingsType.AWS) # We continue to load production.py over aws.py plugin_settings.add_plugins(__name__, PROJECT_TYPE, plugin_constants.SettingsType.PRODUCTION) ####################### Celery fix ########################## # XXX for some reason celery is not registering the bookmarks app from kombu.utils.functional import maybe_list CELERY_IMPORTS = locals().get("CELERY_IMPORTS", []) CELERY_IMPORTS = list( maybe_list(CELERY_IMPORTS)) + ["openedx.core.djangoapps.bookmarks.tasks"] ########################## This container should never be exposed directly ####################### SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https") ########################## Derive Any Derived Settings ####################### ########################## Features ######################### FEATURES = locals().get("FEATURES", {}) FEATURES.update({ "ENABLE_COMBINED_LOGIN_REGISTRATION": True, "ENABLE_DISCUSSION_SERVICE": False }) derive_settings(__name__)
def _configured_for_event(self, name, **context): return self._traverse_subscribers( maybe_list(self.app.settings.THORN_SUBSCRIBERS.get(name)) or [], name, **context)