def startup_info(self, beat):
return STARTUP_INFO_FMT % {
"conninfo": info.format_broker_info(),
"logfile": self.logfile or "[stderr]",
"loglevel": LOG_LEVELS[self.loglevel],
"loader": get_full_cls_name(self.loader.__class__),
"scheduler": get_full_cls_name(beat.scheduler.__class__),
"scheduler_info": beat.scheduler.info,
"hmax_interval": humanize_seconds(beat.max_interval),
"max_interval": beat.max_interval,
}
def startup_info(self, beat):
scheduler = beat.get_scheduler(lazy=True)
return STARTUP_INFO_FMT % {
"conninfo": self.app.broker_connection().as_uri(),
"logfile": self.logfile or "[stderr]",
"loglevel": LOG_LEVELS[self.loglevel],
"loader": get_full_cls_name(self.app.loader.__class__),
"scheduler": get_full_cls_name(scheduler.__class__),
"scheduler_info": scheduler.info,
"hmax_interval": humanize_seconds(beat.max_interval),
"max_interval": beat.max_interval,
}
def startup_info(self, beat):
scheduler = beat.get_scheduler(lazy=True)
return STARTUP_INFO_FMT % {
"conninfo": self.app.broker_connection().as_uri(),
"logfile": self.logfile or "[stderr]",
"loglevel": LOG_LEVELS[self.loglevel],
"loader": get_full_cls_name(self.app.loader.__class__),
"scheduler": get_full_cls_name(scheduler.__class__),
"scheduler_info": scheduler.info,
"hmax_interval": humanize_seconds(beat.max_interval),
"max_interval": beat.max_interval,
}
class Celery(App):
flask_app = None
loader_cls = get_full_cls_name(FlaskLoader)
def __init__(self, flask_app, **kwargs):
self.flask_app = flask_app
super(Celery, self).__init__(**kwargs)
def detect_loader():
loader = os.environ.get("CELERY_LOADER")
if loader:
return get_loader_cls(loader)
loader = _detect_loader()
os.environ["CELERY_LOADER"] = get_full_cls_name(loader)
return loader
def task_status(task_id):
""" returns task status and result """
status = default_app.backend.get_status(task_id)
res = default_app.backend.get_result(task_id)
response_data = dict(id=task_id, status=status, result=res)
if status in default_app.backend.EXCEPTION_STATES:
traceback = default_app.backend.get_traceback(task_id)
response_data.update({"result": repr(res),
"exc": get_full_cls_name(res.__class__),
"traceback": traceback})
return {"task": response_data}
def get(self, task_id):
from app import celery
result = celery.AsyncResult(task_id)
state, retval = result.state, result.result
response_data = dict(id=task_id, status=state)
if state in states.EXCEPTION_STATES:
traceback = result.traceback
response_data.update({'exc': get_full_cls_name(retval.__class__),
'traceback': traceback})
return jsonify({'task': response_data})
def task_status(request, task_id):
"""Returns task status and result in JSON format."""
result = AsyncResult(task_id)
state, retval = result.state, result.result
response_data = dict(id=task_id, status=state, result=retval)
if state in states.EXCEPTION_STATES:
traceback = result.traceback
response_data.update({"result": safe_repr(retval),
"exc": get_full_cls_name(retval.__class__),
"traceback": traceback})
return JsonResponse({"task": response_data})
class Celery(App):
Pickler = FlaskAppPickler
flask_app = None
loader_cls = get_full_cls_name(FlaskLoader)
def __init__(self, flask_app=None, *args, **kwargs):
self.flask_app = flask_app
super(Celery, self).__init__(*args, **kwargs)
def __reduce_args__(self):
return (self.flask_app, ) + super(Celery, self).__reduce_args__()
def task_status(request, task_id):
"""Returns task status and result in JSON format."""
result = AsyncResult(task_id)
state, retval = result.state, result.result
response_data = {'id': task_id, 'status': state, 'result': retval}
if state in states.EXCEPTION_STATES:
traceback = result.traceback
response_data.update({'result': safe_repr(retval),
'exc': get_full_cls_name(retval.__class__),
'traceback': traceback})
return JsonResponse({'task': response_data})
def task_status(request, task_id):
"""Returns task status and result in JSON format."""
status = default_backend.get_status(task_id)
res = default_backend.get_result(task_id)
response_data = dict(id=task_id, status=status, result=res)
if status in default_backend.EXCEPTION_STATES:
traceback = default_backend.get_traceback(task_id)
response_data.update(
{"result": str(res.args[0]), "exc": get_full_cls_name(res.__class__), "traceback": traceback}
)
return HttpResponse(JSON_dump({"task": response_data}), mimetype="application/json")
def assertStatusForIs(self, status, res, traceback=None):
uuid = gen_unique_id()
default_backend.store_result(uuid, res, status, traceback=traceback)
json = self.client.get(task_status(task_id=uuid))
expect = dict(id=uuid, status=status, result=res)
if status in default_backend.EXCEPTION_STATES:
instore = default_backend.get_result(uuid)
self.assertEqual(str(instore.args), str(res.args))
expect["result"] = str(res.args[0])
expect["exc"] = get_full_cls_name(res.__class__)
expect["traceback"] = traceback
self.assertJSONEqual(json, dict(task=expect))
def get(self, task_id):
from app import celery
result = celery.AsyncResult(task_id)
state, retval = result.state, result.result
response_data = dict(id=task_id, status=state)
if state in states.EXCEPTION_STATES:
traceback = result.traceback
response_data.update({
'exc': get_full_cls_name(retval.__class__),
'traceback': traceback
})
return {'task': response_data}
def task_status(request, task_id):
"""Returns task status and result in JSON format."""
status = default_app.backend.get_status(task_id)
res = default_app.backend.get_result(task_id)
response_data = dict(id=task_id, status=status, result=res)
if status in default_app.backend.EXCEPTION_STATES:
traceback = default_app.backend.get_traceback(task_id)
response_data.update({"result": repr(res),
"exc": get_full_cls_name(res.__class__),
"traceback": traceback})
return HttpResponse(JSON_dump({"task": response_data}),
mimetype="application/json")
def get(self, id):
'''Get a tasks status given its ID'''
result = AsyncResult(id, app=celery)
status, retval = result.status, result.result
data = {'id': id, 'status': status, 'result': retval}
if status in states.EXCEPTION_STATES:
traceback = result.traceback
data.update({
'result': safe_repr(retval),
'exc': get_full_cls_name(retval.__class__),
'traceback': traceback,
})
return data
def assertStatusForIs(self, status, res, traceback=None):
uuid = gen_unique_id()
current_app.backend.store_result(uuid, res, status, traceback=traceback)
json = self.client.get(task_status(task_id=uuid))
expect = dict(id=uuid, status=status, result=res)
if status in current_app.backend.EXCEPTION_STATES:
instore = current_app.backend.get_result(uuid)
self.assertEqual(str(instore.args[0]), str(res.args[0]))
expect["result"] = repr(res)
expect["exc"] = get_full_cls_name(res.__class__)
expect["traceback"] = traceback
self.assertJSONEqual(json, dict(task=expect))
class Celery(App):
flask_app = None
loader_cls = get_full_cls_name(FlaskLoader)
def __init__(self, flask_app=None, **kwargs):
if flask_app is not None:
self.flask_app = flask_app
self.init_app(flask_app)
super(Celery, self).__init__(**kwargs)
def init_app(self, flask_app):
"""Initialize the celery using factory pattern"""
self.flask_app = flask_app
def assertStatusForIs(self, status, res, traceback=None):
uuid = gen_unique_id()
current_app.backend.store_result(uuid, res, status,
traceback=traceback)
json = self.client.get(task_status(task_id=uuid))
expect = dict(id=uuid, status=status, result=res)
if status in current_app.backend.EXCEPTION_STATES:
instore = current_app.backend.get_result(uuid)
self.assertEqual(str(instore.args[0]), str(res.args[0]))
expect['result'] = repr(res)
expect['exc'] = get_full_cls_name(res.__class__)
expect['traceback'] = traceback
self.assertJSONEqual(json, dict(task=expect))
def task_status(request, task_id):
response = json_response(status=200, text='ok')
status = default_backend.get_status(task_id)
res = default_backend.get_result(task_id)
response['data'] = {
'id': task_id,
'status': status,
'result': res
}
if status in default_backend.EXCEPTION_STATES:
traceback = default_backend.get_traceback(task_id)
response['data'].update({'result': str(res.args[0]),
'exc': get_full_cls_name(res.__class__),
'traceback': traceback})
return response
def task_progress(self):
task_id = self.task_id
if task_id is None:
return None
result = AsyncResult(task_id)
state, retval = result.state, result.result
data = {'id': task_id, 'status': state, 'result': retval}
if state in states.EXCEPTION_STATES:
traceback = result.traceback
data.update({
'result': safe_repr(retval),
'exc': get_full_cls_name(retval.__class__),
'traceback': traceback
})
return data
def startup_info(self):
concurrency = self.concurrency
if self.autoscale:
cmax, cmin = self.autoscale
concurrency = "{min=%s, max=%s}" % (cmin, cmax)
return BANNER % {
"hostname": self.hostname,
"version": __version__,
"conninfo": self.app.broker_connection().as_uri(),
"concurrency": concurrency,
"loglevel": LOG_LEVELS[self.loglevel],
"logfile": self.logfile or "[stderr]",
"celerybeat": self.run_clockservice and "ON" or "OFF",
"events": self.events and "ON" or "OFF",
"loader": get_full_cls_name(self.loader.__class__),
"queues": self.queues.format(indent=18, indent_first=False),
}
def managed(cls, hostname=None, caller=None):
hostname = hostname or socket.gethostname()
if caller:
hostname = ".".join([get_full_cls_name(caller), hostname])
else:
hostname += str(cls.next_worker_id())
worker = cls(hostname)
worker.ensure_started()
stack = traceback.format_stack()
@atexit.register
def _ensure_shutdown_once():
if not worker._shutdown_called:
say("-- Found worker not stopped at shutdown: %s\n%s" % (worker.hostname, "\n".join(stack)))
worker.ensure_shutdown()
return worker
def startup_info(self):
tasklist = ""
if self.loglevel <= logging.INFO:
include_builtins = self.loglevel <= logging.DEBUG
tasklist = self.tasklist(include_builtins=include_builtins)
return STARTUP_INFO_FMT % {
"conninfo": self.app.amqp.format_broker_info(),
"queues": self.queues.format(indent=8),
"concurrency": self.concurrency,
"loglevel": LOG_LEVELS[self.loglevel],
"logfile": self.logfile or "[stderr]",
"celerybeat": self.run_clockservice and "ON" or "OFF",
"events": self.events and "ON" or "OFF",
"tasks": tasklist,
"loader": get_full_cls_name(self.loader.__class__),
}
def startup_info(self):
app = self.app
concurrency = self.concurrency
if self.autoscale:
cmax, cmin = self.autoscale
concurrency = "{min=%s, max=%s}" % (cmin, cmax)
return BANNER % {
"hostname": self.hostname,
"version": __version__,
"conninfo": self.app.broker_connection().as_uri(),
"concurrency": concurrency,
"loglevel": LOG_LEVELS[self.loglevel],
"logfile": self.logfile or "[stderr]",
"celerybeat": "ON" if self.run_clockservice else "OFF",
"events": "ON" if self.events else "OFF",
"loader": get_full_cls_name(self.loader.__class__),
"queues": app.amqp.queues.format(indent=18, indent_first=False),
}
def managed(cls, hostname=None, caller=None):
hostname = hostname or socket.gethostname()
if caller:
hostname = ".".join([get_full_cls_name(caller), hostname])
else:
hostname += str(cls.next_worker_id())
worker = cls(hostname)
worker.ensure_started()
stack = traceback.format_stack()
@atexit.register
def _ensure_shutdown_once():
if not worker._shutdown_called:
say("-- Found worker not stopped at shutdown: %s\n%s" %
(worker.hostname, "\n".join(stack)))
worker.ensure_shutdown()
return worker
def task_status(request, task_id):
"""Returns task status and result in JSON format."""
result = AsyncResult(task_id)
#print dir(result)
print result.state
state, retval, success = result.state, False, result.successful()
response_data = dict(task_id=task_id,
status=state,
result=retval,
success=success)
if state in states.EXCEPTION_STATES:
traceback = result.traceback
response_data.update({
'result': safe_repr(retval),
'exc': get_full_cls_name(retval.__class__),
'traceback': traceback
})
return JsonResponse(response_data)
class WebCelery(App):
"""Celery wrapper for web.py app.
"""
loader_cls = get_full_cls_name(WebLoader)
def __init__(self, app, *args, **kwargs):
"""Initialize the WebCelery.
``app``
The web.py application.
``kwargs``
Keyword arguments. `config` argument must be passed which contains
the celery configuration.
"""
self.app = app
self._config = kwargs.pop('config', None)
super(WebCelery, self).__init__(app, *args, **kwargs)
@property
def config(self):
"""Setup the configuration variables and pass them to loader.
"""
return WebCelery.import_module(self._config)
@classmethod
def import_module(cls, filename, config_module='celeryconfig'):
"""The celery config file is imported and executed and passed to dict.
"""
module = imp.new_module(config_module)
module.__file__ = filename
execfile(filename, module.__dict__)
return WebCelery.config_to_dict(module)
@classmethod
def config_to_dict(cls, module):
"""Puts configuration constant variables to dict.
"""
mapp = {}
for key in dir(module):
mapp[key] = getattr(module, key)
return mapp
def startup_info(self):
tasklist = ""
if self.loglevel <= logging.INFO:
include_builtins = self.loglevel <= logging.DEBUG
tasklist = self.tasklist(include_builtins=include_builtins)
queues = self.defaults.get_queues()
return STARTUP_INFO_FMT % {
"conninfo": info.format_broker_info(),
"queues": info.format_queues(queues, indent=8),
"concurrency": self.concurrency,
"loglevel": LOG_LEVELS[self.loglevel],
"logfile": self.logfile or "[stderr]",
"celerybeat": self.run_clockservice and "ON" or "OFF",
"events": self.events and "ON" or "OFF",
"tasks": tasklist,
"loader": get_full_cls_name(self.loader.__class__),
}
def __init__(self):
if not self.__class__.name:
self.__class__.name = get_full_cls_name(self.__class__)
def test_get_full_cls_name(self):
Class = type("Fox", (object, ), {"__module__": "quick.brown"})
self.assertEqual(utils.get_full_cls_name(Class), "quick.brown.Fox")
def session_tasks( request ):
"""Gets task statuses from Celery API, appends to task dicts from session.
This function is used to generate the list of pending/successful/failed tasks
in the webapp page notification area.
@param request: A Django request object
@return tasks: a dict with task_id for key
"""
# basic tasks info from session:
# task_id, action ('name' argument of @task), start time, args
tasks = request.session.get(settings.CELERY_TASKS_SESSION_KEY, {})
# add entity URLs
for task_id in tasks.keys():
task = tasks.get(task_id, None)
if task and task['action'] in ['webui-file-new-master',
'webui-file-new-mezzanine',
'webui-file-new-access']:
# Add entity_url to task for newly-created file
repo,org,cid,eid = task['entity_id'].split('-')
task['entity_url'] = reverse('webui-entity', args=[repo,org,cid,eid])
# Hit the celery-task_status view for status updates on each task.
# get status, retval from celery
# TODO Don't create a new ctask/task dict here!!! >:-O
traceback = None
for task_id in tasks.keys():
# Skip the HTTP and get directly from Celery API
# djcelery.views.task_status
result = AsyncResult(task_id)
state, retval = result.state, result.result
response_data = {'id': task_id, 'status': state, 'result': retval}
if state in states.EXCEPTION_STATES:
traceback = result.traceback
response_data.update({'result': safe_repr(retval),
'exc': get_full_cls_name(retval.__class__),
'traceback': traceback})
# end djcelery.views.task_status
task = response_data
# construct collection/entity/file urls if possible
if task:
ctask = tasks[task['id']]
ctask['status'] = task.get('status', None)
ctask['result'] = task.get('result', None)
# try to convert 'result' into a collection/entity/file URL
if (ctask['status'] != 'FAILURE') and ctask['result']:
r = ctask['result']
if type(r) == type({}):
if r.get('id', None):
oid = Identifier(r['id'])
object_url = reverse('webui-%s' % oid.model, args=oid.parts.values())
ctask['%s_url' % oid.model] = object_url
tasks[task['id']] = ctask
# pretty status messages
for task_id in tasks.keys():
task = tasks[task_id]
action = task.get('action', None)
if action:
messages = TASK_STATUS_MESSAGES.get(action, None)
status = task.get('status', None)
template = None
if messages and status:
template = messages.get(status, None)
if template:
msg = template.format(**task)
task['message'] = msg
# indicate if task is dismiss or not
for task_id in tasks.keys():
task = tasks[task_id]
if task.get('status', None):
task['dismissable'] = (task['status'] in TASK_STATUSES_DISMISSABLE)
# include traceback in task if present
if traceback:
task['traceback'] = traceback
# done
return tasks
def test_get_full_cls_name(self):
Class = type("Fox", (object, ), {"__module__": "quick.brown"})
self.assertEqual(utils.get_full_cls_name(Class), "quick.brown.Fox")
import os
from functools import partial
from celery.datastructures import AttributeDict
from celery.loaders import default as _default
from celery.utils import get_full_cls_name
class FlaskLoader(_default.Loader):
def read_configuration(self):
self.configured = True
return self.setup_settings(_default.DEFAULT_UNCONFIGURED_SETTINGS)
os.environ.setdefault("CELERY_LOADER", get_full_cls_name(FlaskLoader))
class Celery(object):
def __init__(self, app):
self.app = app
self.conf = AttributeDict()
self.app.config.setdefault("CELERY_RESULT_BACKEND", "amqp")
from celery.conf import prepare
prepare(self.conf, AttributeDict(self.app.config))
def create_task_cls(self):
from celery.backends import default_backend, get_backend_cls
from celery.task.base import Task
class Hook(object):
"""A Web Hook Event.
:keyword name: See :attr:`name`.
:keyword provides_args: See :attr:`provides_args`.
:keyword config_form: See :attr:`config_form`.
:keyword timeout: See :attr:`timeout`.
:keyword async: See :attr:`async`.
:keyword retry: See :attr:`retry`.
:keyword max_retries: See :attr:`max_retries`.
:keyword fail_silently: See :attr:`fail_silently`.
:keyword task_cls: See :attr:`task_cls`.
:keyword match_forms: See :attr:`match_forms`
.. attribute:: name
The name of the hook.
If not provided this will be automatically generated using the class
module and name, if you want to use this feature you can't use
relative imports.
.. attribute:: provides_args
The list of arguments the event provides. This is the standard
list of arguments you are going to pass on to :meth:`send`, used to
generate the filter events form (:attr:`match_forms`).
.. attribute:: config_form
A Django form to save configuration for listeners attaching to this
form. The default form is :class:`durian.forms.HookConfigForm`, which
has the URL field.
.. attribute:: timeout
The timeout in seconds before we give up trying to dispatch the
event to a listener URL.
.. attribute:: async
If ``True``, signals are dispatched to celery workers via a mesage.
Otherwise dispatch happens locally (not a good idea in production).
.. attribute:: retry
Retry the task if it fails.
.. attribute:: max_retries
Maximum number of retries before we give up.
.. attribute:: fail_silently
Fail silently if the dispatch gives an HTTP error.
.. attribute:: task_cls
The :class:`celery.task.base.Task` class to use for dispatching
the event.
.. attribute:: match_forms
A list of forms to create an event filter. This is automatically
generated based on the :attr:`provides_args` attribute.
"""
name = None
verbose_name = None
task_cls = WebhookSignal
timeout = 4
async = True
retry = False
max_retries = 3
fail_silently = False
config_form = HookConfigForm
provides_args = set()
match_forms = None
def __init__(self,
name=None,
verbose_name=None,
task_cls=None,
timeout=None,
async=None,
retry=None,
max_retries=None,
fail_silently=False,
config_form=None,
provides_args=None,
match_forms=None,
**kwargs):
self.name = name or self.name or get_full_cls_name(self.__class__)
self.verbose_name = verbose_name or self.verbose_name or self.name
self.task_cls = task_cls or self.task_cls
if timeout is not None:
self.timeout = timeout
if async is not None:
self. async = async
