def test_accept_precedence(self): # default is app.conf.accept_content accept_content = self.app.conf.accept_content b1 = BaseBackend(self.app) assert prepare_accept_content(accept_content) == b1.accept # accept parameter b2 = BaseBackend(self.app, accept=['yaml']) assert len(b2.accept) == 1 assert list(b2.accept)[0] == 'application/x-yaml' assert prepare_accept_content(['yaml']) == b2.accept # accept parameter over result_accept_content self.app.conf.result_accept_content = ['json'] b3 = BaseBackend(self.app, accept=['yaml']) assert len(b3.accept) == 1 assert list(b3.accept)[0] == 'application/x-yaml' assert prepare_accept_content(['yaml']) == b3.accept # conf.result_accept_content if specified self.app.conf.result_accept_content = ['yaml'] b4 = BaseBackend(self.app) assert len(b4.accept) == 1 assert list(b4.accept)[0] == 'application/x-yaml' assert prepare_accept_content(['yaml']) == b4.accept
async def handle_message(self, message: IncomingMessage) -> None: message.ack() correlation_id = message.correlation_id future = self._reply_futures.pop(correlation_id, None) if future is not None: try: try: accept = self.accept = prepare_accept_content( self.cluster_proxy.accept) body = loads(message.body, content_type=message.content_type, content_encoding=message.content_encoding, accept=accept) except Exception as e: future.set_exception(e) else: future.set_result(body) except asyncio.InvalidStateError as e: # for catching the errors after the future obj was cancelled by the asyncio.wait_for timeout. sys.stdout.write("{}, correlation id: {}".format( e, correlation_id)) else: sys.stdout.write( "Unknown correlation id: {}".format(correlation_id))
def setup_worker_optimizations(app, hostname=None): global trace_task_ret hostname = hostname or socket.gethostname() # make sure custom Task.__call__ methods that calls super # will not mess up the request/task stack. _install_stack_protection() # all new threads start without a current app, so if an app is not # passed on to the thread it will fall back to the "default app", # which then could be the wrong app. So for the worker # we set this to always return our app. This is a hack, # and means that only a single app can be used for workers # running in the same process. app.set_current() set_default_app(app) # evaluate all task classes by finalizing the app. app.finalize() # set fast shortcut to task registry _localized[:] = [ app._tasks, prepare_accept_content(app.conf.CELERY_ACCEPT_CONTENT), hostname, ] trace_task_ret = _fast_trace_task from celery.worker import request as request_module request_module.trace_task_ret = _fast_trace_task request_module.__optimize__()
def setup_worker_optimizations(app, hostname=None): global trace_task_ret hostname = hostname or gethostname() # make sure custom Task.__call__ methods that calls super # won't mess up the request/task stack. _install_stack_protection() # all new threads start without a current app, so if an app is not # passed on to the thread it will fall back to the "default app", # which then could be the wrong app. So for the worker # we set this to always return our app. This is a hack, # and means that only a single app can be used for workers # running in the same process. app.set_current() set_default_app(app) # evaluate all task classes by finalizing the app. app.finalize() # set fast shortcut to task registry _localized[:] = [ app._tasks, prepare_accept_content(app.conf.accept_content), hostname, ] trace_task_ret = _fast_trace_task from celery.worker import request as request_module request_module.trace_task_ret = _fast_trace_task request_module.__optimize__()
def __init__(self, app, serializer=None, max_cached_results=None, accept=None, **kwargs): self.app = app conf = self.app.conf self.serializer = serializer or conf.CELERY_RESULT_SERIALIZER (self.content_type, self.content_encoding, self.encoder) = serializer_registry._encoders[self.serializer] self._cache = LRUCache(limit=max_cached_results or conf.CELERY_MAX_CACHED_RESULTS) self.accept = prepare_accept_content(conf.CELERY_ACCEPT_CONTENT if accept is None else accept)
def __init__(self, app, serializer=None, max_cached_results=None, accept=None, expires=None, expires_type=None, url=None, **kwargs): self.app = app conf = self.app.conf self.serializer = serializer or conf.result_serializer (self.content_type, self.content_encoding, self.encoder) = serializer_registry._encoders[self.serializer] cmax = max_cached_results or conf.result_cache_max self._cache = _nulldict() if cmax == -1 else LRUCache(limit=cmax) self.expires = self.prepare_expires(expires, expires_type) # precedence: accept, conf.result_accept_content, conf.accept_content self.accept = conf.result_accept_content if accept is None else accept self.accept = conf.accept_content if self.accept is None else self.accept self.accept = prepare_accept_content(self.accept) self.always_retry = conf.get('result_backend_always_retry', False) self.max_sleep_between_retries_ms = conf.get( 'result_backend_max_sleep_between_retries_ms', 10000) self.base_sleep_between_retries_ms = conf.get( 'result_backend_base_sleep_between_retries_ms', 10) self.max_retries = conf.get('result_backend_max_retries', float("inf")) self._pending_results = pending_results_t({}, WeakValueDictionary()) self._pending_messages = BufferMap(MESSAGE_BUFFER_MAX) self.url = url
def setup_worker_optimizations(app, hostname=None): """Setup worker related optimizations.""" hostname = hostname or gethostname() # make sure custom Task.__call__ methods that calls super # won't mess up the request/task stack. _install_stack_protection() # all new threads start without a current app, so if an app is not # passed on to the thread it will fall back to the "default app", # which then could be the wrong app. So for the worker # we set this to always return our app. This is a hack, # and means that only a single app can be used for workers # running in the same process. app.set_current() app.set_default() # evaluate all task classes by finalizing the app. app.finalize() # set fast shortcut to task registry _localized[:] = [ app._tasks, prepare_accept_content(app.conf.accept_content), hostname, ] app.use_fast_trace_task = True
def __init__(self, app, serializer=None, max_cached_results=None, accept=None, expires=None, expires_type=None, url=None, **kwargs): self.app = app conf = self.app.conf self.serializer = serializer or conf.result_serializer (self.content_type, self.content_encoding, self.encoder) = serializer_registry._encoders[self.serializer] cmax = max_cached_results or conf.result_cache_max self._cache = _nulldict() if cmax == -1 else LRUCache(limit=cmax) self.expires = self.prepare_expires(expires, expires_type) # precedence: accept, conf.result_accept_content, conf.accept_content self.accept = conf.result_accept_content if accept is None else accept self.accept = conf.accept_content if self.accept is None else self.accept # noqa: E501 self.accept = prepare_accept_content(self.accept) self._pending_results = pending_results_t({}, WeakValueDictionary()) self._pending_messages = BufferMap(MESSAGE_BUFFER_MAX) self.url = url
def _trace_task_ret(name, uuid, request, body, content_type, content_encoding, loads=loads_message, app=None, **extra_request): app = app or current_app._get_current_object() accept = prepare_accept_content(app.conf.CELERY_ACCEPT_CONTENT) args, kwargs = loads(body, content_type, content_encoding, accept=accept) request.update(args=args, kwargs=kwargs, **extra_request) R, I, T, Rstr = trace_task(app.tasks[name], uuid, args, kwargs, request, app=app) return (1, R, T) if I else (0, Rstr, T)
def __init__(self, app, serializer=None, max_cached_results=None, accept=None, **kwargs): self.app = app conf = self.app.conf self.serializer = serializer or conf.CELERY_RESULT_SERIALIZER (self.content_type, self.content_encoding, self.encoder) = serializer_registry._encoders[self.serializer] cmax = max_cached_results or conf.CELERY_MAX_CACHED_RESULTS self._cache = _nulldict() if cmax == -1 else LRUCache(limit=cmax) self.accept = prepare_accept_content( conf.CELERY_ACCEPT_CONTENT if accept is None else accept, )
def _trace_task_ret( name, uuid, request, body, content_type, content_encoding, loads=loads_message, app=None, **extra_request ): app = app or current_app._get_current_object() embed = None if content_type: accept = prepare_accept_content(app.conf.accept_content) args, kwargs, embed = loads(body, content_type, content_encoding, accept=accept) else: args, kwargs, embed = body hostname = gethostname() request.update({"args": args, "kwargs": kwargs, "hostname": hostname, "is_eager": False}, **embed or {}) R, I, T, Rstr = trace_task(app.tasks[name], uuid, args, kwargs, request, app=app) return (1, R, T) if I else (0, Rstr, T)
def testAcceptJsonCeleryRpc(self): """ Accepted content type are correct. """ from ..app import rpc accept = serialization.prepare_accept_content( rpc.conf.CELERY_ACCEPT_CONTENT) source = ('a', 1, None) for codec in ('json', 'x-json', 'x-rpc-json'): content_type, encoding, result = serialization.dumps(source, codec) restored = serialization.loads(result, content_type, encoding, accept=accept) self.assertEqual(list(source), restored)
def __init__(self, app, serializer=None, max_cached_results=None, accept=None, expires=None, expires_type=None, **kwargs): self.app = app conf = self.app.conf self.serializer = serializer or conf.CELERY_RESULT_SERIALIZER (self.content_type, self.content_encoding, self.encoder) = serializer_registry._encoders[self.serializer] cmax = max_cached_results or conf.CELERY_MAX_CACHED_RESULTS self._cache = _nulldict() if cmax == -1 else LRUCache(limit=cmax) self.expires = self.prepare_expires(expires, expires_type) self.accept = prepare_accept_content( conf.CELERY_ACCEPT_CONTENT if accept is None else accept, )
def __init__(self, app, serializer=None, max_cached_results=None, accept=None, expires=None, expires_type=None, **kwargs): self.app = app conf = self.app.conf self.serializer = serializer or conf.result_serializer (self.content_type, self.content_encoding, self.encoder) = serializer_registry._encoders[self.serializer] cmax = max_cached_results or conf.result_cache_max self._cache = _nulldict() if cmax == -1 else LRUCache(limit=cmax) self.expires = self.prepare_expires(expires, expires_type) self.accept = prepare_accept_content( conf.accept_content if accept is None else accept, )
def __init__(self, app, serializer=None, max_cached_results=None, accept=None, expires=None, expires_type=None, url=None, **kwargs): self.app = app conf = self.app.conf self.serializer = serializer or conf.result_serializer (self.content_type, self.content_encoding, self.encoder) = serializer_registry._encoders[self.serializer] cmax = max_cached_results or conf.result_cache_max self._cache = _nulldict() if cmax == -1 else LRUCache(limit=cmax) self.expires = self.prepare_expires(expires, expires_type) self.accept = prepare_accept_content( conf.accept_content if accept is None else accept) self._pending_results = pending_results_t({}, WeakValueDictionary()) self._pending_messages = BufferMap(MESSAGE_BUFFER_MAX) self.url = url
def _trace_task_ret(name, uuid, request, body, content_type, content_encoding, loads=loads_message, app=None, **extra_request): app = app or current_app._get_current_object() embed = None if content_type: accept = prepare_accept_content(app.conf.accept_content) args, kwargs, embed = loads( body, content_type, content_encoding, accept=accept, ) else: args, kwargs, embed = body hostname = gethostname() request.update({ 'args': args, 'kwargs': kwargs, 'hostname': hostname, 'is_eager': False, }, **embed or {}) R, I, T, Rstr = trace_task(app.tasks[name], uuid, args, kwargs, request, app=app) return (1, R, T) if I else (0, Rstr, T)
def _trace_task_ret(name, uuid, request, body, content_type, content_encoding, loads=loads_message, app=None, **extra_request): app = app or current_app._get_current_object() embed = None if content_type: accept = prepare_accept_content(app.conf.CELERY_ACCEPT_CONTENT) args, kwargs, embed = loads( body, content_type, content_encoding, accept=accept, ) else: args, kwargs = body hostname = socket.gethostname() request.update({ 'args': args, 'kwargs': kwargs, 'hostname': hostname, 'is_eager': False, }, **embed or {}) R, I, T, Rstr = trace_task(app.tasks[name], uuid, args, kwargs, request, app=app) return (1, R, T) if I else (0, Rstr, T)
def test_prepare_accept_content_bad_serializer(self): with pytest.raises(SerializerNotInstalled): prepare_accept_content(['bad_serializer'])
def offline_orphaned_jobs(es_url, dry_run=False): """Set jobs with job-queued or job-started state to job-offline if not synced with celery task state.""" # get redis connection set_redis_pool() global POOL rd = StrictRedis(connection_pool=POOL) # get celery task result serializer content_type, content_encoding, encoder = registry._encoders[ app.conf.CELERY_RESULT_SERIALIZER] accept = prepare_accept_content(app.conf.CELERY_ACCEPT_CONTENT) logging.info("content_type: {}".format(content_type)) logging.info("content_encoding: {}".format(content_encoding)) logging.info("encoder: {}".format(encoder)) logging.info("accept: {}".format(accept)) # query query = { "query": { "bool": { "must": [{ "terms": { "status": ["job-started", "job-queued"] } }] } }, "_source": ["status", "tags", "uuid"], } url_tmpl = "{}/job_status-current/_search?search_type=scan&scroll=10m&size=100" r = requests.post(url_tmpl.format(es_url), data=json.dumps(query)) if r.status_code != 200: logging.error("Failed to query ES. Got status code %d:\n%s" % (r.status_code, json.dumps(query, indent=2))) r.raise_for_status() scan_result = r.json() count = scan_result["hits"]["total"] scroll_id = scan_result["_scroll_id"] # get list of results results = [] while True: r = requests.post("%s/_search/scroll?scroll=10m" % es_url, data=scroll_id) res = r.json() scroll_id = res["_scroll_id"] if len(res["hits"]["hits"]) == 0: break for hit in res["hits"]["hits"]: results.append(hit) # check for celery state for res in results: id = res["_id"] src = res.get("_source", {}) status = src["status"] tags = src.get("tags", []) task_id = src["uuid"] # check celery task status in ES task_query = { "query": { "term": { "_id": task_id } }, "_source": ["status"] } r = requests.post("%s/task_status-current/task/_search" % es_url, data=json.dumps(task_query)) if r.status_code != 200: logging.error("Failed to query ES. Got status code %d:\n%s" % (r.status_code, json.dumps(task_query, indent=2))) continue task_res = r.json() if task_res["hits"]["total"] > 0: task_info = task_res["hits"]["hits"][0] if task_info["_source"]["status"] == "task-failed": updated_status = "job-failed" elif task_info["_source"]["status"] == "task-succeeded": updated_status = "job-completed" elif task_info["_source"]["status"] in ("task-sent", "task-started"): continue else: logging.error("Cannot handle task status %s for %s." % (task_info["_source"]["status"], task_id)) continue if dry_run: logging.info("Would've update job status to %s for %s." % (updated_status, task_id)) else: new_doc = { "doc": { "status": updated_status }, "doc_as_upsert": True } r = requests.post( "%s/job_status-current/job/%s/_update" % (es_url, id), data=json.dumps(new_doc), ) result = r.json() if r.status_code != 200: logging.error( "Failed to update tags for %s. Got status code %d:\n%s" % (id, r.status_code, json.dumps(result, indent=2))) r.raise_for_status() logging.info("Set job %s to %s." % (id, updated_status)) continue # get celery task metadata in redis task_meta = loads( rd.get("celery-task-meta-%s" % task_id), content_type=content_type, content_encoding=content_encoding, accept=accept, ) if task_meta is None: updated_status = "job-offline" if dry_run: logging.info("Would've update job status to %s for %s." % (updated_status, task_id)) else: new_doc = { "doc": { "status": updated_status }, "doc_as_upsert": True } r = requests.post( "%s/job_status-current/job/%s/_update" % (es_url, id), data=json.dumps(new_doc), ) result = r.json() if r.status_code != 200: logging.error( "Failed to update tags for %s. Got status code %d:\n%s" % (id, r.status_code, json.dumps(result, indent=2))) r.raise_for_status() logging.info("Set job %s to %s." % (id, updated_status)) continue
def test_prepare_accept_content(self): assert {'application/json'} == prepare_accept_content(['json']) assert {'application/json'} == prepare_accept_content( ['application/json'])