def cast(self,
method,
args={},
declare=None,
retry=None,
retry_policy=None,
type=None,
exchange=None,
**props):
"""Send message to actor. Discarding replies."""
retry = self.retry if retry is None else retry
body = {'class': self.name, 'method': method, 'args': args}
_retry_policy = self.retry_policy
if retry_policy: # merge default and custom policies.
_retry_policy = dict(_retry_policy, **retry_policy)
if type and type not in self.types:
raise ValueError('Unsupported type: {0}'.format(type))
elif not type:
type = ACTOR_TYPE.DIRECT
props.setdefault('routing_key', self.routing_key)
props.setdefault('serializer', self.serializer)
exchange = exchange or self.type_to_exchange[type]()
declare = (maybe_list(declare) or []) + [exchange]
with producers[self._connection].acquire(block=True) as producer:
return producer.publish(body,
exchange=exchange,
declare=declare,
retry=retry,
retry_policy=retry_policy,
**props)
def select(self, include):
"""Sets :attr:`consume_from` by selecting a subset of the
currently defined queues.
:param include: Names of queues to consume from.
Can be iterable or string.
"""
if include:
self._consume_from = {name: self[name] for name in maybe_list(include)}
def select(self, include):
"""Select a subset of currently defined queues to consume from.
Arguments:
include (Sequence[str], str): Names of queues to consume from.
"""
if include:
self._consume_from = {
name: self[name] for name in maybe_list(include)
}
def select(self, include):
"""Select a subset of currently defined queues to consume from.
Arguments:
include (Sequence[str], str): Names of queues to consume from.
"""
if include:
self._consume_from = {
name: self[name] for name in maybe_list(include)
}
def select(self, include):
"""Sets :attr:`consume_from` by selecting a subset of the
currently defined queues.
:param include: Names of queues to consume from.
Can be iterable or string.
"""
if include:
self._consume_from = dict(
(name, self[name]) for name in maybe_list(include))
def traverse_subscribers(it, *args, **kwargs):
stream = deque([it])
while stream:
for node in maybe_list(stream.popleft()):
if isinstance(node, string_types) and node.startswith('!'):
node = symbol_by_name(node[1:])
if isinstance(node, Callable):
node = node(*args, **kwargs)
if is_list(node):
stream.append(node)
elif node:
yield node
def deselect(self, exclude):
"""Deselect queues so that they won't be consumed from.
Arguments:
exclude (Sequence[str], str): Names of queues to avoid
consuming from.
"""
if exclude:
exclude = maybe_list(exclude)
if self._consume_from is None:
# using all queues
return self.select(k for k in self if k not in exclude)
# using selection
for queue in exclude:
self._consume_from.pop(queue, None)
def deselect(self, exclude):
"""Deselect queues so that they will not be consumed from.
:param exclude: Names of queues to avoid consuming from.
Can be iterable or string.
"""
if exclude:
exclude = maybe_list(exclude)
if self._consume_from is None:
# using selection
return self.select(k for k in self if k not in exclude)
# using all queues
for queue in exclude:
self._consume_from.pop(queue, None)
def deselect(self, exclude):
"""Deselect queues so that they won't be consumed from.
Arguments:
exclude (Sequence[str], str): Names of queues to avoid
consuming from.
"""
if exclude:
exclude = maybe_list(exclude)
if self._consume_from is None:
# using selection
return self.select(k for k in self if k not in exclude)
# using all queues
for queue in exclude:
self._consume_from.pop(queue, None)
def deselect(self, exclude):
"""Deselect queues so that they will not be consumed from.
:param exclude: Names of queues to avoid consuming from.
Can be iterable or string.
"""
if exclude:
exclude = maybe_list(exclude)
if self._consume_from is None:
# using selection
return self.select(k for k in self if k not in exclude)
# using all queues
for queue in exclude:
self._consume_from.pop(queue, None)
def cast(self, method, args={}, declare=None, retry=None,
retry_policy=None, type=None, exchange=None, **props):
"""Send message to actor. Discarding replies."""
retry = self.retry if retry is None else retry
body = {'class': self.name, 'method': method, 'args': args}
_retry_policy = self.retry_policy
if retry_policy: # merge default and custom policies.
_retry_policy = dict(_retry_policy, **retry_policy)
if type and type not in self.types:
raise ValueError('Unsupported type: {0}'.format(type))
elif not type:
type = ACTOR_TYPE.DIRECT
props.setdefault('routing_key', self.routing_key)
props.setdefault('serializer', self.serializer)
exchange = exchange or self.type_to_exchange[type]()
declare = (maybe_list(declare) or []) + [exchange]
with producers[self._connection].acquire(block=True) as producer:
return producer.publish(body, exchange=exchange, declare=declare,
retry=retry, retry_policy=retry_policy,
**props)
def test_maybe_list(obj, expected):
assert maybe_list(obj) == expected
def perform(self, request, **kwargs):
for req in maybe_list(request) or []:
if not isinstance(req, self.Request):
req = self.Request(req, **kwargs)
self.add_request(req)
def test_maybe_list(obj, expected):
assert maybe_list(obj) == expected
def send_task(self, name, args=None, kwargs=None, countdown=None,
eta=None, task_id=None, producer=None, connection=None,
router=None, result_cls=None, expires=None,
publisher=None, link=None, link_error=None,
add_to_parent=True, group_id=None, retries=0, chord=None,
reply_to=None, time_limit=None, soft_time_limit=None,
root_id=None, parent_id=None, route_name=None,
shadow=None, chain=None, task_type=None, **options):
"""Send task by name.
Supports the same arguments as :meth:`@-Task.apply_async`.
Arguments:
name (str): Name of task to call (e.g., `"tasks.add"`).
result_cls (~@AsyncResult): Specify custom result class.
"""
# This is a copy of (the top of) celery.app.base.Celery.send_task
parent = have_parent = None
amqp = self.amqp
task_id = task_id or uuid()
producer = producer or publisher # XXX compat
router = router or amqp.router
conf = self.conf
if conf.task_always_eager: # pragma: no cover
warnings.warn(AlwaysEagerIgnored(
'task_always_eager has no effect on send_task',
), stacklevel=2)
options = router.route(
options, route_name or name, args, kwargs, task_type)
# if not root_id or not parent_id:
# parent = self.current_worker_task
# if parent:
# if not root_id:
# root_id = parent.request.root_id or parent.request.id
# if not parent_id:
# parent_id = parent.request.id
message = amqp.create_task_message(
task_id, name, args, kwargs, countdown, eta, group_id,
expires, retries, chord,
maybe_list(link), maybe_list(link_error),
reply_to or self.app.oid, time_limit, soft_time_limit,
self.conf.task_send_sent_event,
root_id, parent_id, shadow, chain,
)
# if connection:
# producer = amqp.Producer(connection, auto_declare=False)
# self.backend.on_task_call(P, task_id)
# with self.producer_or_acquire(producer) as P:
# with P.connection._reraise_as_library_errors():
# if not ignored_result:
# self.backend.on_task_call(P, task_id)
# amqp.send_task_message(P, name, message, **options)
yield self.channel.ensure_connected()
# TODO Handle deferred returned.
self.send_task_message(name, message, **options)
result = defer.Deferred()
self._sent_tasks[task_id] = result
# result = (result_cls or self.AsyncResult)(task_id)
# if add_to_parent:
# if not have_parent:
# parent, have_parent = self.current_worker_task, True
# if parent:
# parent.add_trail(result)
# return result
result = yield result
defer.returnValue(result)
def as_task_v2(self,
task_id,
name,
args=None,
kwargs=None,
countdown=None,
eta=None,
group_id=None,
group_index=None,
expires=None,
retries=0,
chord=None,
callbacks=None,
errbacks=None,
reply_to=None,
time_limit=None,
soft_time_limit=None,
create_sent_event=False,
root_id=None,
parent_id=None,
shadow=None,
chain=None,
now=None,
timezone=None,
origin=None,
ignore_result=False,
argsrepr=None,
kwargsrepr=None,
stamped_headers=None,
**options):
args = args or ()
kwargs = kwargs or {}
if not isinstance(args, (list, tuple)):
raise TypeError('task args must be a list or tuple')
if not isinstance(kwargs, Mapping):
raise TypeError('task keyword arguments must be a mapping')
if countdown: # convert countdown to ETA
self._verify_seconds(countdown, 'countdown')
now = now or self.app.now()
timezone = timezone or self.app.timezone
eta = maybe_make_aware(
now + timedelta(seconds=countdown),
tz=timezone,
)
if isinstance(expires, numbers.Real):
self._verify_seconds(expires, 'expires')
now = now or self.app.now()
timezone = timezone or self.app.timezone
expires = maybe_make_aware(
now + timedelta(seconds=expires),
tz=timezone,
)
if not isinstance(eta, str):
eta = eta and eta.isoformat()
# If we retry a task `expires` will already be ISO8601-formatted.
if not isinstance(expires, str):
expires = expires and expires.isoformat()
if argsrepr is None:
argsrepr = saferepr(args, self.argsrepr_maxsize)
if kwargsrepr is None:
kwargsrepr = saferepr(kwargs, self.kwargsrepr_maxsize)
if not root_id: # empty root_id defaults to task_id
root_id = task_id
stamps = {
header: maybe_list(options[header])
for header in stamped_headers or []
}
headers = {
'lang': 'py',
'task': name,
'id': task_id,
'shadow': shadow,
'eta': eta,
'expires': expires,
'group': group_id,
'group_index': group_index,
'retries': retries,
'timelimit': [time_limit, soft_time_limit],
'root_id': root_id,
'parent_id': parent_id,
'argsrepr': argsrepr,
'kwargsrepr': kwargsrepr,
'origin': origin or anon_nodename(),
'ignore_result': ignore_result,
'stamped_headers': stamped_headers,
'stamps': stamps,
}
return task_message(
headers=headers,
properties={
'correlation_id': task_id,
'reply_to': reply_to or '',
},
body=(
args,
kwargs,
{
'callbacks': callbacks,
'errbacks': errbacks,
'chain': chain,
'chord': chord,
},
),
sent_event={
'uuid': task_id,
'root_id': root_id,
'parent_id': parent_id,
'name': name,
'args': argsrepr,
'kwargs': kwargsrepr,
'retries': retries,
'eta': eta,
'expires': expires,
} if create_sent_event else None,
)
def test_maybe_list(self):
self.assertIsNone(maybe_list(None))
self.assertEqual(maybe_list(1), [1])
self.assertEqual(maybe_list([1, 2, 3]), [1, 2, 3])
def perform(self, request, **kwargs):
for req in maybe_list(request) or []:
if not isinstance(req, self.Request):
req = self.Request(req, **kwargs)
self.add_request(req)
def add(self, fds, callback, flags, consolidate=False):
for fd in maybe_list(fds, None):
try:
self._add(fd, callback, flags, consolidate)
except ValueError:
self._discard(fd)
CELERY_BROKER_HOSTNAME,
CELERY_BROKER_VHOST,
)
CELERY_MONGODB_BACKEND_SETTINGS = {
"database": MONGODB_DB_NAME,
"taskmeta_collection": "taskmeta_collection",
}
CELERY_RESULT_BACKEND = "mongodb://{}/".format(MONGODB_HOST)
CELERY_RESULT_DB_TABLENAMES = {
"task": "celery_edx_task",
"group": "celery_edx_group"
}
CELERY_IMPORTS = locals().get("CELERY_IMPORTS", [])
# XXX for some reason celery is not registering the bookmarks app
CELERY_IMPORTS = list(maybe_list(CELERY_IMPORTS)) + [
"openedx.core.djangoapps.bookmarks.tasks",
"openedx.core.djangoapps.content.course_overviews.tasks",
]
CELERYBEAT_SCHEDULE = {}
CELERY_ROUTES = "{}.celery.Router".format(SERVICE_VARIANT)
CELERY_QUEUES = {"openedx.lms.default": {}, "openedx.cms.default": {}}
CELERY_DEFAULT_EXCHANGE = "openedx.{}".format(SERVICE_VARIANT)
CELERY_DEFAULT_QUEUE = "{}.default".format(CELERY_DEFAULT_EXCHANGE)
HIGH_PRIORITY_QUEUE = CELERY_DEFAULT_QUEUE
DEFAULT_PRIORITY_QUEUE = CELERY_DEFAULT_QUEUE
HIGH_MEM_QUEUE = CELERY_DEFAULT_QUEUE
CELERY_DEFAULT_ROUTING_KEY = DEFAULT_PRIORITY_QUEUE
# Load aws.py in plugins for reverse compatibility. This can be removed after aws.py
# is officially removed.
plugin_settings.add_plugins(__name__, PROJECT_TYPE,
plugin_constants.SettingsType.AWS)
# We continue to load production.py over aws.py
plugin_settings.add_plugins(__name__, PROJECT_TYPE,
plugin_constants.SettingsType.PRODUCTION)
####################### Celery fix ##########################
# XXX for some reason celery is not registering the bookmarks app
from kombu.utils.functional import maybe_list
CELERY_IMPORTS = locals().get("CELERY_IMPORTS", [])
CELERY_IMPORTS = list(
maybe_list(CELERY_IMPORTS)) + ["openedx.core.djangoapps.bookmarks.tasks"]
########################## This container should never be exposed directly #######################
SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https")
########################## Derive Any Derived Settings #######################
########################## Features #########################
FEATURES = locals().get("FEATURES", {})
FEATURES.update({
"ENABLE_COMBINED_LOGIN_REGISTRATION": True,
"ENABLE_DISCUSSION_SERVICE": False
})
derive_settings(__name__)
def _configured_for_event(self, name, **context):
return self._traverse_subscribers(
maybe_list(self.app.settings.THORN_SUBSCRIBERS.get(name)) or [],
name, **context)
