def test_accept_precedence(self):
# default is app.conf.accept_content
accept_content = self.app.conf.accept_content
b1 = BaseBackend(self.app)
assert prepare_accept_content(accept_content) == b1.accept
# accept parameter
b2 = BaseBackend(self.app, accept=['yaml'])
assert len(b2.accept) == 1
assert list(b2.accept)[0] == 'application/x-yaml'
assert prepare_accept_content(['yaml']) == b2.accept
# accept parameter over result_accept_content
self.app.conf.result_accept_content = ['json']
b3 = BaseBackend(self.app, accept=['yaml'])
assert len(b3.accept) == 1
assert list(b3.accept)[0] == 'application/x-yaml'
assert prepare_accept_content(['yaml']) == b3.accept
# conf.result_accept_content if specified
self.app.conf.result_accept_content = ['yaml']
b4 = BaseBackend(self.app)
assert len(b4.accept) == 1
assert list(b4.accept)[0] == 'application/x-yaml'
assert prepare_accept_content(['yaml']) == b4.accept
def test_accept_precedence(self):
# default is app.conf.accept_content
accept_content = self.app.conf.accept_content
b1 = BaseBackend(self.app)
assert prepare_accept_content(accept_content) == b1.accept
# accept parameter
b2 = BaseBackend(self.app, accept=['yaml'])
assert len(b2.accept) == 1
assert list(b2.accept)[0] == 'application/x-yaml'
assert prepare_accept_content(['yaml']) == b2.accept
# accept parameter over result_accept_content
self.app.conf.result_accept_content = ['json']
b3 = BaseBackend(self.app, accept=['yaml'])
assert len(b3.accept) == 1
assert list(b3.accept)[0] == 'application/x-yaml'
assert prepare_accept_content(['yaml']) == b3.accept
# conf.result_accept_content if specified
self.app.conf.result_accept_content = ['yaml']
b4 = BaseBackend(self.app)
assert len(b4.accept) == 1
assert list(b4.accept)[0] == 'application/x-yaml'
assert prepare_accept_content(['yaml']) == b4.accept
async def handle_message(self, message: IncomingMessage) -> None:
message.ack()
correlation_id = message.correlation_id
future = self._reply_futures.pop(correlation_id, None)
if future is not None:
try:
try:
accept = self.accept = prepare_accept_content(
self.cluster_proxy.accept)
body = loads(message.body,
content_type=message.content_type,
content_encoding=message.content_encoding,
accept=accept)
except Exception as e:
future.set_exception(e)
else:
future.set_result(body)
except asyncio.InvalidStateError as e:
# for catching the errors after the future obj was cancelled by the asyncio.wait_for timeout.
sys.stdout.write("{}, correlation id: {}".format(
e, correlation_id))
else:
sys.stdout.write(
"Unknown correlation id: {}".format(correlation_id))
def setup_worker_optimizations(app, hostname=None):
global trace_task_ret
hostname = hostname or socket.gethostname()
# make sure custom Task.__call__ methods that calls super
# will not mess up the request/task stack.
_install_stack_protection()
# all new threads start without a current app, so if an app is not
# passed on to the thread it will fall back to the "default app",
# which then could be the wrong app. So for the worker
# we set this to always return our app. This is a hack,
# and means that only a single app can be used for workers
# running in the same process.
app.set_current()
set_default_app(app)
# evaluate all task classes by finalizing the app.
app.finalize()
# set fast shortcut to task registry
_localized[:] = [
app._tasks,
prepare_accept_content(app.conf.CELERY_ACCEPT_CONTENT),
hostname,
]
trace_task_ret = _fast_trace_task
from celery.worker import request as request_module
request_module.trace_task_ret = _fast_trace_task
request_module.__optimize__()
def setup_worker_optimizations(app, hostname=None):
global trace_task_ret
hostname = hostname or gethostname()
# make sure custom Task.__call__ methods that calls super
# won't mess up the request/task stack.
_install_stack_protection()
# all new threads start without a current app, so if an app is not
# passed on to the thread it will fall back to the "default app",
# which then could be the wrong app. So for the worker
# we set this to always return our app. This is a hack,
# and means that only a single app can be used for workers
# running in the same process.
app.set_current()
set_default_app(app)
# evaluate all task classes by finalizing the app.
app.finalize()
# set fast shortcut to task registry
_localized[:] = [
app._tasks,
prepare_accept_content(app.conf.accept_content),
hostname,
]
trace_task_ret = _fast_trace_task
from celery.worker import request as request_module
request_module.trace_task_ret = _fast_trace_task
request_module.__optimize__()
def __init__(self, app, serializer=None, max_cached_results=None, accept=None, **kwargs):
self.app = app
conf = self.app.conf
self.serializer = serializer or conf.CELERY_RESULT_SERIALIZER
(self.content_type, self.content_encoding, self.encoder) = serializer_registry._encoders[self.serializer]
self._cache = LRUCache(limit=max_cached_results or conf.CELERY_MAX_CACHED_RESULTS)
self.accept = prepare_accept_content(conf.CELERY_ACCEPT_CONTENT if accept is None else accept)
def __init__(self,
app,
serializer=None,
max_cached_results=None,
accept=None,
expires=None,
expires_type=None,
url=None,
**kwargs):
self.app = app
conf = self.app.conf
self.serializer = serializer or conf.result_serializer
(self.content_type, self.content_encoding,
self.encoder) = serializer_registry._encoders[self.serializer]
cmax = max_cached_results or conf.result_cache_max
self._cache = _nulldict() if cmax == -1 else LRUCache(limit=cmax)
self.expires = self.prepare_expires(expires, expires_type)
# precedence: accept, conf.result_accept_content, conf.accept_content
self.accept = conf.result_accept_content if accept is None else accept
self.accept = conf.accept_content if self.accept is None else self.accept
self.accept = prepare_accept_content(self.accept)
self.always_retry = conf.get('result_backend_always_retry', False)
self.max_sleep_between_retries_ms = conf.get(
'result_backend_max_sleep_between_retries_ms', 10000)
self.base_sleep_between_retries_ms = conf.get(
'result_backend_base_sleep_between_retries_ms', 10)
self.max_retries = conf.get('result_backend_max_retries', float("inf"))
self._pending_results = pending_results_t({}, WeakValueDictionary())
self._pending_messages = BufferMap(MESSAGE_BUFFER_MAX)
self.url = url
def setup_worker_optimizations(app, hostname=None):
"""Setup worker related optimizations."""
hostname = hostname or gethostname()
# make sure custom Task.__call__ methods that calls super
# won't mess up the request/task stack.
_install_stack_protection()
# all new threads start without a current app, so if an app is not
# passed on to the thread it will fall back to the "default app",
# which then could be the wrong app. So for the worker
# we set this to always return our app. This is a hack,
# and means that only a single app can be used for workers
# running in the same process.
app.set_current()
app.set_default()
# evaluate all task classes by finalizing the app.
app.finalize()
# set fast shortcut to task registry
_localized[:] = [
app._tasks,
prepare_accept_content(app.conf.accept_content),
hostname,
]
app.use_fast_trace_task = True
def __init__(self,
app,
serializer=None,
max_cached_results=None,
accept=None,
expires=None,
expires_type=None,
url=None,
**kwargs):
self.app = app
conf = self.app.conf
self.serializer = serializer or conf.result_serializer
(self.content_type, self.content_encoding,
self.encoder) = serializer_registry._encoders[self.serializer]
cmax = max_cached_results or conf.result_cache_max
self._cache = _nulldict() if cmax == -1 else LRUCache(limit=cmax)
self.expires = self.prepare_expires(expires, expires_type)
# precedence: accept, conf.result_accept_content, conf.accept_content
self.accept = conf.result_accept_content if accept is None else accept
self.accept = conf.accept_content if self.accept is None else self.accept # noqa: E501
self.accept = prepare_accept_content(self.accept)
self._pending_results = pending_results_t({}, WeakValueDictionary())
self._pending_messages = BufferMap(MESSAGE_BUFFER_MAX)
self.url = url
def _trace_task_ret(name, uuid, request, body, content_type,
content_encoding, loads=loads_message, app=None,
**extra_request):
app = app or current_app._get_current_object()
accept = prepare_accept_content(app.conf.CELERY_ACCEPT_CONTENT)
args, kwargs = loads(body, content_type, content_encoding, accept=accept)
request.update(args=args, kwargs=kwargs, **extra_request)
R, I, T, Rstr = trace_task(app.tasks[name],
uuid, args, kwargs, request, app=app)
return (1, R, T) if I else (0, Rstr, T)
def __init__(self, app, serializer=None,
max_cached_results=None, accept=None, **kwargs):
self.app = app
conf = self.app.conf
self.serializer = serializer or conf.CELERY_RESULT_SERIALIZER
(self.content_type,
self.content_encoding,
self.encoder) = serializer_registry._encoders[self.serializer]
cmax = max_cached_results or conf.CELERY_MAX_CACHED_RESULTS
self._cache = _nulldict() if cmax == -1 else LRUCache(limit=cmax)
self.accept = prepare_accept_content(
conf.CELERY_ACCEPT_CONTENT if accept is None else accept,
)
def _trace_task_ret(
name, uuid, request, body, content_type, content_encoding, loads=loads_message, app=None, **extra_request
):
app = app or current_app._get_current_object()
embed = None
if content_type:
accept = prepare_accept_content(app.conf.accept_content)
args, kwargs, embed = loads(body, content_type, content_encoding, accept=accept)
else:
args, kwargs, embed = body
hostname = gethostname()
request.update({"args": args, "kwargs": kwargs, "hostname": hostname, "is_eager": False}, **embed or {})
R, I, T, Rstr = trace_task(app.tasks[name], uuid, args, kwargs, request, app=app)
return (1, R, T) if I else (0, Rstr, T)
def testAcceptJsonCeleryRpc(self):
""" Accepted content type are correct.
"""
from ..app import rpc
accept = serialization.prepare_accept_content(
rpc.conf.CELERY_ACCEPT_CONTENT)
source = ('a', 1, None)
for codec in ('json', 'x-json', 'x-rpc-json'):
content_type, encoding, result = serialization.dumps(source, codec)
restored = serialization.loads(result,
content_type,
encoding,
accept=accept)
self.assertEqual(list(source), restored)
def __init__(self, app,
serializer=None, max_cached_results=None, accept=None,
expires=None, expires_type=None, **kwargs):
self.app = app
conf = self.app.conf
self.serializer = serializer or conf.CELERY_RESULT_SERIALIZER
(self.content_type,
self.content_encoding,
self.encoder) = serializer_registry._encoders[self.serializer]
cmax = max_cached_results or conf.CELERY_MAX_CACHED_RESULTS
self._cache = _nulldict() if cmax == -1 else LRUCache(limit=cmax)
self.expires = self.prepare_expires(expires, expires_type)
self.accept = prepare_accept_content(
conf.CELERY_ACCEPT_CONTENT if accept is None else accept,
)
def __init__(self, app,
serializer=None, max_cached_results=None, accept=None,
expires=None, expires_type=None, **kwargs):
self.app = app
conf = self.app.conf
self.serializer = serializer or conf.result_serializer
(self.content_type,
self.content_encoding,
self.encoder) = serializer_registry._encoders[self.serializer]
cmax = max_cached_results or conf.result_cache_max
self._cache = _nulldict() if cmax == -1 else LRUCache(limit=cmax)
self.expires = self.prepare_expires(expires, expires_type)
self.accept = prepare_accept_content(
conf.accept_content if accept is None else accept,
)
def __init__(self, app,
serializer=None, max_cached_results=None, accept=None,
expires=None, expires_type=None, url=None, **kwargs):
self.app = app
conf = self.app.conf
self.serializer = serializer or conf.result_serializer
(self.content_type,
self.content_encoding,
self.encoder) = serializer_registry._encoders[self.serializer]
cmax = max_cached_results or conf.result_cache_max
self._cache = _nulldict() if cmax == -1 else LRUCache(limit=cmax)
self.expires = self.prepare_expires(expires, expires_type)
self.accept = prepare_accept_content(
conf.accept_content if accept is None else accept)
self._pending_results = pending_results_t({}, WeakValueDictionary())
self._pending_messages = BufferMap(MESSAGE_BUFFER_MAX)
self.url = url
def _trace_task_ret(name, uuid, request, body, content_type,
content_encoding, loads=loads_message, app=None,
**extra_request):
app = app or current_app._get_current_object()
embed = None
if content_type:
accept = prepare_accept_content(app.conf.accept_content)
args, kwargs, embed = loads(
body, content_type, content_encoding, accept=accept,
)
else:
args, kwargs, embed = body
hostname = gethostname()
request.update({
'args': args, 'kwargs': kwargs,
'hostname': hostname, 'is_eager': False,
}, **embed or {})
R, I, T, Rstr = trace_task(app.tasks[name],
uuid, args, kwargs, request, app=app)
return (1, R, T) if I else (0, Rstr, T)
def _trace_task_ret(name, uuid, request, body, content_type,
content_encoding, loads=loads_message, app=None,
**extra_request):
app = app or current_app._get_current_object()
embed = None
if content_type:
accept = prepare_accept_content(app.conf.CELERY_ACCEPT_CONTENT)
args, kwargs, embed = loads(
body, content_type, content_encoding, accept=accept,
)
else:
args, kwargs = body
hostname = socket.gethostname()
request.update({
'args': args, 'kwargs': kwargs,
'hostname': hostname, 'is_eager': False,
}, **embed or {})
R, I, T, Rstr = trace_task(app.tasks[name],
uuid, args, kwargs, request, app=app)
return (1, R, T) if I else (0, Rstr, T)
def test_prepare_accept_content_bad_serializer(self):
with pytest.raises(SerializerNotInstalled):
prepare_accept_content(['bad_serializer'])
def offline_orphaned_jobs(es_url, dry_run=False):
"""Set jobs with job-queued or job-started state to job-offline
if not synced with celery task state."""
# get redis connection
set_redis_pool()
global POOL
rd = StrictRedis(connection_pool=POOL)
# get celery task result serializer
content_type, content_encoding, encoder = registry._encoders[
app.conf.CELERY_RESULT_SERIALIZER]
accept = prepare_accept_content(app.conf.CELERY_ACCEPT_CONTENT)
logging.info("content_type: {}".format(content_type))
logging.info("content_encoding: {}".format(content_encoding))
logging.info("encoder: {}".format(encoder))
logging.info("accept: {}".format(accept))
# query
query = {
"query": {
"bool": {
"must": [{
"terms": {
"status": ["job-started", "job-queued"]
}
}]
}
},
"_source": ["status", "tags", "uuid"],
}
url_tmpl = "{}/job_status-current/_search?search_type=scan&scroll=10m&size=100"
r = requests.post(url_tmpl.format(es_url), data=json.dumps(query))
if r.status_code != 200:
logging.error("Failed to query ES. Got status code %d:\n%s" %
(r.status_code, json.dumps(query, indent=2)))
r.raise_for_status()
scan_result = r.json()
count = scan_result["hits"]["total"]
scroll_id = scan_result["_scroll_id"]
# get list of results
results = []
while True:
r = requests.post("%s/_search/scroll?scroll=10m" % es_url,
data=scroll_id)
res = r.json()
scroll_id = res["_scroll_id"]
if len(res["hits"]["hits"]) == 0:
break
for hit in res["hits"]["hits"]:
results.append(hit)
# check for celery state
for res in results:
id = res["_id"]
src = res.get("_source", {})
status = src["status"]
tags = src.get("tags", [])
task_id = src["uuid"]
# check celery task status in ES
task_query = {
"query": {
"term": {
"_id": task_id
}
},
"_source": ["status"]
}
r = requests.post("%s/task_status-current/task/_search" % es_url,
data=json.dumps(task_query))
if r.status_code != 200:
logging.error("Failed to query ES. Got status code %d:\n%s" %
(r.status_code, json.dumps(task_query, indent=2)))
continue
task_res = r.json()
if task_res["hits"]["total"] > 0:
task_info = task_res["hits"]["hits"][0]
if task_info["_source"]["status"] == "task-failed":
updated_status = "job-failed"
elif task_info["_source"]["status"] == "task-succeeded":
updated_status = "job-completed"
elif task_info["_source"]["status"] in ("task-sent",
"task-started"):
continue
else:
logging.error("Cannot handle task status %s for %s." %
(task_info["_source"]["status"], task_id))
continue
if dry_run:
logging.info("Would've update job status to %s for %s." %
(updated_status, task_id))
else:
new_doc = {
"doc": {
"status": updated_status
},
"doc_as_upsert": True
}
r = requests.post(
"%s/job_status-current/job/%s/_update" % (es_url, id),
data=json.dumps(new_doc),
)
result = r.json()
if r.status_code != 200:
logging.error(
"Failed to update tags for %s. Got status code %d:\n%s"
% (id, r.status_code, json.dumps(result, indent=2)))
r.raise_for_status()
logging.info("Set job %s to %s." % (id, updated_status))
continue
# get celery task metadata in redis
task_meta = loads(
rd.get("celery-task-meta-%s" % task_id),
content_type=content_type,
content_encoding=content_encoding,
accept=accept,
)
if task_meta is None:
updated_status = "job-offline"
if dry_run:
logging.info("Would've update job status to %s for %s." %
(updated_status, task_id))
else:
new_doc = {
"doc": {
"status": updated_status
},
"doc_as_upsert": True
}
r = requests.post(
"%s/job_status-current/job/%s/_update" % (es_url, id),
data=json.dumps(new_doc),
)
result = r.json()
if r.status_code != 200:
logging.error(
"Failed to update tags for %s. Got status code %d:\n%s"
% (id, r.status_code, json.dumps(result, indent=2)))
r.raise_for_status()
logging.info("Set job %s to %s." % (id, updated_status))
continue
def test_prepare_accept_content(self):
assert {'application/json'} == prepare_accept_content(['json'])
assert {'application/json'} == prepare_accept_content(
['application/json'])
