def handle_task(self, uuid_task, worker=None):
"""Handle snapshotted event."""
uuid, task = uuid_task
if task.worker and task.worker.hostname:
worker = self.handle_worker((task.worker.hostname, task.worker), )
defaults = {
'name': task.name,
'args': task.args,
'kwargs': task.kwargs,
'eta': maybe_make_aware(maybe_iso8601(task.eta)),
'expires': maybe_make_aware(maybe_iso8601(task.expires)),
'state': task.state,
'tstamp': aware_tstamp(task.timestamp),
'result': task.result or task.exception,
'traceback': task.traceback,
'runtime': task.runtime,
'worker': worker
}
# Some fields are only stored in the RECEIVED event,
# so we should remove these from default values,
# so that they are not overwritten by subsequent states.
[
defaults.pop(attr, None) for attr in NOT_SAVED_ATTRIBUTES
if defaults[attr] is None
]
return self.update_task(task.state, task_id=uuid, defaults=defaults)
def handle_task(self, uuid_task, worker=None):
"""Handle snapshotted event."""
(uuid, task) = uuid_task
if task.worker and task.worker.hostname:
worker = self.handle_worker(
(task.worker.hostname, task.worker),
)
defaults = {
'name': task.name,
'args': task.args,
'kwargs': task.kwargs,
'eta': maybe_make_aware(maybe_iso8601(task.eta)),
'expires': maybe_make_aware(maybe_iso8601(task.expires)),
'state': task.state,
'tstamp': aware_tstamp(task.timestamp),
'result': task.result or task.exception,
'traceback': task.traceback,
'runtime': task.runtime,
'worker': worker
}
# Some fields are only stored in the RECEIVED event,
# so we should remove these from default values,
# so that they are not overwritten by subsequent states.
[defaults.pop(attr, None) for attr in NOT_SAVED_ATTRIBUTES
if defaults[attr] is None]
return self.update_task(task.state,
task_id=uuid, defaults=defaults)
def from_message(cls, message, body, on_ack=noop, **kw):
"""Create request from a task message.
:raises UnknownTaskError: if the message does not describe a task,
the message is also rejected.
"""
_delivery_info = getattr(message, "delivery_info", {})
delivery_info = dict(
(key, _delivery_info.get(key)) for key in WANTED_DELIVERY_INFO)
kwargs = body["kwargs"]
if not hasattr(kwargs, "items"):
raise InvalidTaskError("Task keyword arguments is not a mapping.")
return cls(task_name=body["task"],
task_id=body["id"],
taskset_id=body.get("taskset", None),
args=body["args"],
kwargs=kwdict(kwargs),
retries=body.get("retries", 0),
eta=maybe_iso8601(body.get("eta")),
expires=maybe_iso8601(body.get("expires")),
on_ack=on_ack,
delivery_info=delivery_info,
**kw)
def from_message(cls, message, body, on_ack=noop, **kw):
"""Create request from a task message.
:raises UnknownTaskError: if the message does not describe a task,
the message is also rejected.
"""
_delivery_info = getattr(message, "delivery_info", {})
delivery_info = dict((key, _delivery_info.get(key))
for key in WANTED_DELIVERY_INFO)
kwargs = body["kwargs"]
if not hasattr(kwargs, "items"):
raise InvalidTaskError("Task keyword arguments is not a mapping.")
return cls(task_name=body["task"],
task_id=body["id"],
args=body["args"],
kwargs=kwdict(kwargs),
retries=body.get("retries", 0),
eta=maybe_iso8601(body.get("eta")),
expires=maybe_iso8601(body.get("expires")),
on_ack=on_ack,
delivery_info=delivery_info,
**kw)
def run(self, name, *_, **kw):
# Positional args.
args = kw.get('args') or ()
if isinstance(args, string_t):
args = anyjson.loads(args)
# Keyword args.
kwargs = kw.get('kwargs') or {}
if isinstance(kwargs, string_t):
kwargs = anyjson.loads(kwargs)
# Expires can be int/float.
expires = kw.get('expires') or None
try:
expires = float(expires)
except (TypeError, ValueError):
# or a string describing an ISO 8601 datetime.
try:
expires = maybe_iso8601(expires)
except (TypeError, ValueError):
raise
res = self.app.send_task(name,
args=args,
kwargs=kwargs,
countdown=kw.get('countdown'),
serializer=kw.get('serializer'),
queue=kw.get('queue'),
exchange=kw.get('exchange'),
routing_key=kw.get('routing_key'),
eta=maybe_iso8601(kw.get('eta')),
expires=expires)
self.out(res.id)
def __init__(self, body, on_ack=noop,
hostname=None, eventer=None, app=None,
connection_errors=None, request_dict=None,
delivery_info=None, task=None, on_reject=noop, **opts):
self.app = app
name = self.name = body['task']
self.id = body['id']
self.args = body.get('args', [])
self.kwargs = body.get('kwargs', {})
try:
self.kwargs.items
except AttributeError:
raise InvalidTaskError(
'Task keyword arguments is not a mapping')
if NEEDS_KWDICT:
self.kwargs = kwdict(self.kwargs)
eta = body.get('eta')
expires = body.get('expires')
utc = self.utc = body.get('utc', False)
self.on_ack = on_ack
self.on_reject = on_reject
self.hostname = hostname or socket.gethostname()
self.eventer = eventer
self.connection_errors = connection_errors or ()
self.task = task or self.app.tasks[name]
self.acknowledged = self._already_revoked = False
self.time_start = self.worker_pid = self._terminate_on_ack = None
self._tzlocal = None
# timezone means the message is timezone-aware, and the only timezone
# supported at this point is UTC.
if eta is not None:
try:
self.eta = maybe_iso8601(eta)
except (AttributeError, ValueError, TypeError) as exc:
raise InvalidTaskError(
'invalid eta value {0!r}: {1}'.format(eta, exc))
if utc:
self.eta = maybe_make_aware(self.eta, self.tzlocal)
else:
self.eta = None
if expires is not None:
try:
self.expires = maybe_iso8601(expires)
except (AttributeError, ValueError, TypeError) as exc:
raise InvalidTaskError(
'invalid expires value {0!r}: {1}'.format(expires, exc))
if utc:
self.expires = maybe_make_aware(self.expires, self.tzlocal)
else:
self.expires = None
delivery_info = {} if delivery_info is None else delivery_info
self.delivery_info = {
'exchange': delivery_info.get('exchange'),
'routing_key': delivery_info.get('routing_key'),
'priority': delivery_info.get('priority'),
'redelivered': delivery_info.get('redelivered'),
}
self.request_dict = body
def __init__(
self,
body,
on_ack=noop,
hostname=None,
eventer=None,
app=None,
connection_errors=None,
request_dict=None,
delivery_info=None,
task=None,
**opts
):
self.app = app or app_or_default(app)
name = self.name = body["task"]
self.id = body["id"]
self.args = body.get("args", [])
self.kwargs = body.get("kwargs", {})
try:
self.kwargs.items
except AttributeError:
raise exceptions.InvalidTaskError("Task keyword arguments is not a mapping")
if NEEDS_KWDICT:
self.kwargs = kwdict(self.kwargs)
eta = body.get("eta")
expires = body.get("expires")
utc = body.get("utc", False)
self.on_ack = on_ack
self.hostname = hostname or socket.gethostname()
self.eventer = eventer
self.connection_errors = connection_errors or ()
self.task = task or self.app.tasks[name]
self.acknowledged = self._already_revoked = False
self.time_start = self.worker_pid = self._terminate_on_ack = None
self._tzlocal = None
# timezone means the message is timezone-aware, and the only timezone
# supported at this point is UTC.
if eta is not None:
tz = tz_utc if utc else self.tzlocal
self.eta = tz_to_local(maybe_iso8601(eta), self.tzlocal, tz)
else:
self.eta = None
if expires is not None:
tz = tz_utc if utc else self.tzlocal
self.expires = tz_to_local(maybe_iso8601(expires), self.tzlocal, tz)
else:
self.expires = None
delivery_info = {} if delivery_info is None else delivery_info
self.delivery_info = {
"exchange": delivery_info.get("exchange"),
"routing_key": delivery_info.get("routing_key"),
"priority": delivery_info.get("priority"),
}
# amqplib transport adds the channel here for some reason, so need
# to remove it.
self.delivery_info.pop("channel", None)
self.request_dict = body
def __init__(self,
body,
on_ack=noop,
hostname=None,
eventer=None,
app=None,
connection_errors=None,
request_dict=None,
delivery_info=None,
task=None,
**opts):
self.app = app or app_or_default(app)
name = self.name = body['task']
self.id = body['id']
self.args = body.get('args', [])
self.kwargs = body.get('kwargs', {})
try:
self.kwargs.items
except AttributeError:
raise exceptions.InvalidTaskError(
'Task keyword arguments is not a mapping')
if NEEDS_KWDICT:
self.kwargs = kwdict(self.kwargs)
eta = body.get('eta')
expires = body.get('expires')
utc = body.get('utc', False)
self.on_ack = on_ack
self.hostname = hostname or socket.gethostname()
self.eventer = eventer
self.connection_errors = connection_errors or ()
self.task = task or self.app.tasks[name]
self.acknowledged = self._already_revoked = False
self.time_start = self.worker_pid = self._terminate_on_ack = None
self._tzlocal = None
# timezone means the message is timezone-aware, and the only timezone
# supported at this point is UTC.
if eta is not None:
tz = tz_utc if utc else self.tzlocal
self.eta = tz_to_local(maybe_iso8601(eta), self.tzlocal, tz)
else:
self.eta = None
if expires is not None:
tz = tz_utc if utc else self.tzlocal
self.expires = tz_to_local(maybe_iso8601(expires), self.tzlocal,
tz)
else:
self.expires = None
delivery_info = {} if delivery_info is None else delivery_info
self.delivery_info = {
'exchange': delivery_info.get('exchange'),
'routing_key': delivery_info.get('routing_key'),
'priority': delivery_info.get('priority'),
}
# amqplib transport adds the channel here for some reason, so need
# to remove it.
self.delivery_info.pop('channel', None)
self.request_dict = body
def run(self, name, *_, **kw):
# Positional args.
args = kw.get('args') or ()
if isinstance(args, string_t):
args = json.loads(args)
# Keyword args.
kwargs = kw.get('kwargs') or {}
if isinstance(kwargs, string_t):
kwargs = json.loads(kwargs)
# Expires can be int/float.
expires = kw.get('expires') or None
try:
expires = float(expires)
except (TypeError, ValueError):
# or a string describing an ISO 8601 datetime.
try:
expires = maybe_iso8601(expires)
except (TypeError, ValueError):
raise
res = self.app.send_task(name, args=args, kwargs=kwargs,
countdown=kw.get('countdown'),
serializer=kw.get('serializer'),
queue=kw.get('queue'),
exchange=kw.get('exchange'),
routing_key=kw.get('routing_key'),
eta=maybe_iso8601(kw.get('eta')),
expires=expires)
self.out(res.id)
def from_message(cls, message, message_data, logger=None, eventer=None,
hostname=None, app=None):
"""Create a :class:`TaskRequest` from a task message sent by
:class:`celery.app.amqp.TaskPublisher`.
:raises UnknownTaskError: if the message does not describe a task,
the message is also rejected.
:returns :class:`TaskRequest`:
"""
task_name = message_data["task"]
task_id = message_data["id"]
args = message_data["args"]
kwargs = message_data["kwargs"]
retries = message_data.get("retries", 0)
eta = maybe_iso8601(message_data.get("eta"))
expires = maybe_iso8601(message_data.get("expires"))
_delivery_info = getattr(message, "delivery_info", {})
delivery_info = dict((key, _delivery_info.get(key))
for key in WANTED_DELIVERY_INFO)
if not hasattr(kwargs, "items"):
raise InvalidTaskError("Task kwargs must be a dictionary.")
return cls(task_name, task_id, args, kwdict(kwargs),
retries=retries, on_ack=message.ack,
delivery_info=delivery_info, logger=logger,
eventer=eventer, hostname=hostname,
eta=eta, expires=expires, app=app)
def run(self, name, *_, **kw):
# Positional args.
args = kw.get("args") or ()
if isinstance(args, basestring):
args = deserialize(args)
# Keyword args.
kwargs = kw.get("kwargs") or {}
if isinstance(kwargs, basestring):
kwargs = deserialize(kwargs)
# Expires can be int/float.
expires = kw.get("expires") or None
try:
expires = float(expires)
except (TypeError, ValueError):
# or a string describing an ISO 8601 datetime.
try:
expires = maybe_iso8601(expires)
except (TypeError, ValueError):
pass
res = self.app.send_task(name, args=args, kwargs=kwargs,
countdown=kw.get("countdown"),
serializer=kw.get("serializer"),
queue=kw.get("queue"),
exchange=kw.get("exchange"),
routing_key=kw.get("routing_key"),
eta=maybe_iso8601(kw.get("eta")),
expires=expires)
self.out(res.task_id)
def run(self, name, *_, **kw):
# Positional args.
args = kw.get("args") or ()
if isinstance(args, basestring):
args = deserialize(args)
# Keyword args.
kwargs = kw.get("kwargs") or {}
if isinstance(kwargs, basestring):
kwargs = deserialize(kwargs)
# Expires can be int/float.
expires = kw.get("expires") or None
try:
expires = float(expires)
except (TypeError, ValueError):
# or a string describing an ISO 8601 datetime.
try:
expires = maybe_iso8601(expires)
except (TypeError, ValueError):
pass
res = self.app.send_task(name,
args=args,
kwargs=kwargs,
countdown=kw.get("countdown"),
serializer=kw.get("serializer"),
queue=kw.get("queue"),
exchange=kw.get("exchange"),
routing_key=kw.get("routing_key"),
eta=maybe_iso8601(kw.get("eta")),
expires=expires)
self.out(res.task_id)
def __init__(self,
body,
on_ack=noop,
hostname=None,
eventer=None,
app=None,
connection_errors=None,
request_dict=None,
delivery_info=None,
task=None,
**opts):
self.app = app or app_or_default(app)
name = self.name = body["task"]
self.id = body["id"]
self.args = body.get("args", [])
self.kwargs = body.get("kwargs", {})
try:
self.kwargs.items
except AttributeError:
raise exceptions.InvalidTaskError(
"Task keyword arguments is not a mapping")
if NEEDS_KWDICT:
self.kwargs = kwdict(self.kwargs)
eta = body.get("eta")
expires = body.get("expires")
utc = body.get("utc", False)
self.on_ack = on_ack
self.hostname = hostname or socket.gethostname()
self.eventer = eventer
self.connection_errors = connection_errors or ()
self.task = task or self.app.tasks[name]
self.acknowledged = self._already_revoked = False
self.time_start = self.worker_pid = self._terminate_on_ack = None
self._tzlocal = None
# timezone means the message is timezone-aware, and the only timezone
# supported at this point is UTC.
if eta is not None:
tz = tz_utc if utc else self.tzlocal
self.eta = tz_to_local(maybe_iso8601(eta), self.tzlocal, tz)
else:
self.eta = None
if expires is not None:
tz = tz_utc if utc else self.tzlocal
self.expires = tz_to_local(maybe_iso8601(expires), self.tzlocal,
tz)
else:
self.expires = None
delivery_info = {} if delivery_info is None else delivery_info
self.delivery_info = {
"exchange": delivery_info.get("exchange"),
"routing_key": delivery_info.get("routing_key"),
}
## shortcuts
self._does_debug = logger.isEnabledFor(logging.DEBUG)
self._does_info = logger.isEnabledFor(logging.INFO)
self.request_dict = body
def _send_task(self, name, args=None, kwargs=None,
countdown=None, serializer=None,
queue=None, exchange=None, routing_key=None,
eta=None, expires=None):
# arguments
args = loads(args) if isinstance(args, string_t) else args
kwargs = loads(kwargs) if isinstance(kwargs, string_t) else kwargs
# Expires can be int/float.
try:
expires = float(expires)
except (TypeError, ValueError):
# or a string describing an ISO 8601 datetime.
try:
expires = maybe_iso8601(expires)
except (TypeError, ValueError):
raise
# send the task and print the id.
self.out(self.app.send_task(
name,
args=args or (), kwargs=kwargs or {},
countdown=countdown,
serializer=serializer,
queue=queue,
exchange=exchange,
routing_key=routing_key,
eta=maybe_iso8601(eta),
expires=expires,
).id)
def __init__(self, body, on_ack=noop,
hostname=None, eventer=None, app=None,
connection_errors=None, request_dict=None,
delivery_info=None, task=None, **opts):
self.app = app or app_or_default(app)
name = self.name = body['task']
self.id = body['id']
self.args = body.get('args', [])
self.kwargs = body.get('kwargs', {})
try:
self.kwargs.items
except AttributeError:
raise exceptions.InvalidTaskError(
'Task keyword arguments is not a mapping')
if NEEDS_KWDICT:
self.kwargs = kwdict(self.kwargs)
eta = body.get('eta')
expires = body.get('expires')
utc = self.utc = body.get('utc', False)
self.on_ack = on_ack
self.hostname = hostname or socket.gethostname()
self.eventer = eventer
self.connection_errors = connection_errors or ()
self.task = task or self.app.tasks[name]
self.acknowledged = self._already_revoked = False
self.time_start = self.worker_pid = self._terminate_on_ack = None
self._tzlocal = None
# timezone means the message is timezone-aware, and the only timezone
# supported at this point is UTC.
if eta is not None:
self.eta = maybe_iso8601(eta)
if utc:
self.eta = maybe_make_aware(self.eta, self.tzlocal)
else:
self.eta = None
if expires is not None:
self.expires = maybe_iso8601(expires)
if utc:
self.expires = maybe_make_aware(self.expires, self.tzlocal)
else:
self.expires = None
delivery_info = {} if delivery_info is None else delivery_info
self.delivery_info = {
'exchange': delivery_info.get('exchange'),
'routing_key': delivery_info.get('routing_key'),
'priority': delivery_info.get('priority'),
}
# amqplib transport adds the channel here for some reason, so need
# to remove it.
self.delivery_info.pop('channel', None)
self.request_dict = body
def __init__(self, body, on_ack=noop,
hostname=None, eventer=None, app=None,
connection_errors=None, request_dict=None,
delivery_info=None, task=None, **opts):
self.app = app or app_or_default(app)
name = self.name = body['task']
self.id = body['id']
self.args = body.get('args', [])
self.kwargs = body.get('kwargs', {})
try:
self.kwargs.items
except AttributeError:
raise exceptions.InvalidTaskError(
'Task keyword arguments is not a mapping')
if NEEDS_KWDICT:
self.kwargs = kwdict(self.kwargs)
eta = body.get('eta')
expires = body.get('expires')
utc = body.get('utc', False)
self.on_ack = on_ack
self.hostname = hostname or socket.gethostname()
self.eventer = eventer
self.connection_errors = connection_errors or ()
self.task = task or self.app.tasks[name]
self.acknowledged = self._already_revoked = False
self.time_start = self.worker_pid = self._terminate_on_ack = None
self._tzlocal = None
# timezone means the message is timezone-aware, and the only timezone
# supported at this point is UTC.
if eta is not None:
tz = tz_utc if utc else self.tzlocal
self.eta = tz_to_local(maybe_iso8601(eta), self.tzlocal, tz)
else:
self.eta = None
if expires is not None:
tz = tz_utc if utc else self.tzlocal
self.expires = tz_to_local(maybe_iso8601(expires),
self.tzlocal, tz)
else:
self.expires = None
self.delivery_info = delivery_info or {}
# amqplib transport adds the channel here for some reason, so need
# to remove it. sqs also adds additional unpickleable attributes
# than need to be removed.
for key in ['channel', 'sqs_message', 'sqs_queue']:
self.delivery_info.pop(key, None)
self.request_dict = body
def sync_tasks(self, tasks):
for task_id, task in tasks.items():
row = db.celery_taskstate(task_id=task_id)
d = {
"name": task.name,
"args": task.args,
"kwargs": task.kwargs,
"eta": maybe_iso8601(task.eta),
"expires": maybe_iso8601(task.expires),
"state": task.state,
"tstamp": datetime.fromtimestamp(task.timestamp),
"result": task.result or task.exception,
"traceback": task.traceback,
"runtime": task.runtime,
"worker": task.worker.hostname
}
row = db.celery_taskstate(task_id=task_id)
if row: row.update_record(**d)
else: db.celery_taskstate.insert(task_id=task_id, **d)
db.commit()
def _send_task(self,
name,
args=None,
kwargs=None,
countdown=None,
serializer=None,
queue=None,
exchange=None,
routing_key=None,
eta=None,
expires=None):
# arguments
args = loads(args) if isinstance(args, string_t) else args
kwargs = loads(kwargs) if isinstance(kwargs, string_t) else kwargs
# Expires can be int/float.
try:
expires = float(expires)
except (TypeError, ValueError):
# or a string describing an ISO 8601 datetime.
try:
expires = maybe_iso8601(expires)
except (TypeError, ValueError):
raise
# send the task and print the id.
self.out(
self.app.send_task(
name,
args=args or (),
kwargs=kwargs or {},
countdown=countdown,
serializer=serializer,
queue=queue,
exchange=exchange,
routing_key=routing_key,
eta=maybe_iso8601(eta),
expires=expires,
).id)
def sync_tasks(self, tasks):
for task_id, task in tasks.items():
row = db.celery_taskstate(task_id=task_id)
d = {
"name": task.name,
"args": task.args,
"kwargs": task.kwargs,
"eta": maybe_iso8601(task.eta),
"expires": maybe_iso8601(task.expires),
"state": task.state,
"tstamp": datetime.fromtimestamp(task.timestamp),
"result": task.result or task.exception,
"traceback": task.traceback,
"runtime": task.runtime,
"worker": task.worker.hostname,
}
row = db.celery_taskstate(task_id=task_id)
if row:
row.update_record(**d)
else:
db.celery_taskstate.insert(task_id=task_id, **d)
db.commit()
def __init__(self,
body,
on_ack=noop,
hostname=None,
eventer=None,
app=None,
connection_errors=None,
request_dict=None,
delivery_info=None,
task=None,
**opts):
self.app = app or app_or_default(app)
name = self.name = body['task']
self.id = body['id']
self.args = body.get('args', [])
self.kwargs = body.get('kwargs', {})
try:
self.kwargs.items
except AttributeError:
raise InvalidTaskError('Task keyword arguments is not a mapping')
if NEEDS_KWDICT:
self.kwargs = kwdict(self.kwargs)
eta = body.get('eta')
expires = body.get('expires')
utc = self.utc = body.get('utc', False)
self.on_ack = on_ack
self.hostname = hostname or socket.gethostname()
self.eventer = eventer
self.connection_errors = connection_errors or ()
self.task = task or self.app.tasks[name]
self.acknowledged = self._already_revoked = False
self.time_start = self.worker_pid = self._terminate_on_ack = None
self._tzlocal = None
# timezone means the message is timezone-aware, and the only timezone
# supported at this point is UTC.
if eta is not None:
try:
self.eta = maybe_iso8601(eta)
except (AttributeError, ValueError), exc:
raise InvalidTaskError('invalid eta value %r: %s' % (
eta,
exc,
))
if utc:
self.eta = maybe_make_aware(self.eta, self.tzlocal)
def generate_indexing_docs(cls, tasks, project=None):
docs = []
for task in tasks:
worker = task.worker
project, worker_name = get_project_and_name(worker, project=project)
if project not in app.config['CELERIUM_PROJECTS']:
continue
doc = dict(task)
doc.pop('worker', None)
doc['id'] = doc.pop('uuid')
doc['project'] = project
if doc['name'].find('.') != -1:
doc['module'] = '.'.join(doc['name'].split('.')[:-1])
for field in cls.timestamp_fields:
doc[field] = try_fromtimestamp(doc[field])
for field in cls.iso8601_fields:
doc[field] = maybe_iso8601(doc[field])
if worker:
doc['worker_id'] = get_worker_id(project, worker_name)
doc['worker_name'] = worker_name
docs.append(doc)
return docs
def __init__(self, body, on_ack=noop,
hostname=None, eventer=None, app=None,
connection_errors=None, request_dict=None,
delivery_info=None, task=None, **opts):
self.app = app or app_or_default(app)
name = self.name = body['task']
self.id = body['id']
self.args = body.get('args', [])
self.kwargs = body.get('kwargs', {})
try:
self.kwargs.items
except AttributeError:
raise InvalidTaskError(
'Task keyword arguments is not a mapping')
if NEEDS_KWDICT:
self.kwargs = kwdict(self.kwargs)
eta = body.get('eta')
expires = body.get('expires')
utc = self.utc = body.get('utc', False)
self.on_ack = on_ack
self.hostname = hostname or socket.gethostname()
self.eventer = eventer
self.connection_errors = connection_errors or ()
self.task = task or self.app.tasks[name]
self.acknowledged = self._already_revoked = False
self.time_start = self.worker_pid = self._terminate_on_ack = None
self._tzlocal = None
# timezone means the message is timezone-aware, and the only timezone
# supported at this point is UTC.
if eta is not None:
try:
self.eta = maybe_iso8601(eta)
except (AttributeError, ValueError), exc:
raise InvalidTaskError(
'invalid eta value %r: %s' % (eta, exc, ))
if utc:
self.eta = maybe_make_aware(self.eta, self.tzlocal)
def __init__(
self,
message,
on_ack=noop,
hostname=None,
eventer=None,
app=None,
connection_errors=None,
request_dict=None,
task=None,
on_reject=noop,
body=None,
headers=None,
decoded=False,
utc=True,
maybe_make_aware=maybe_make_aware,
maybe_iso8601=maybe_iso8601,
**opts
):
if headers is None:
headers = message.headers
if body is None:
body = message.body
self.app = app
self.message = message
self.body = body
self.utc = utc
self._decoded = decoded
if decoded:
self.content_type = self.content_encoding = None
else:
self.content_type, self.content_encoding = (message.content_type, message.content_encoding)
self.id = headers["id"]
type = self.type = self.name = headers["task"]
self.root_id = headers.get("root_id")
self.parent_id = headers.get("parent_id")
if "shadow" in headers:
self.name = headers["shadow"] or self.name
if "timelimit" in headers:
self.time_limits = headers["timelimit"]
self.argsrepr = headers.get("argsrepr", "")
self.kwargsrepr = headers.get("kwargsrepr", "")
self.on_ack = on_ack
self.on_reject = on_reject
self.hostname = hostname or socket.gethostname()
self.eventer = eventer
self.connection_errors = connection_errors or ()
self.task = task or self.app.tasks[type]
# timezone means the message is timezone-aware, and the only timezone
# supported at this point is UTC.
eta = headers.get("eta")
if eta is not None:
try:
eta = maybe_iso8601(eta)
except (AttributeError, ValueError, TypeError) as exc:
raise InvalidTaskError("invalid eta value {0!r}: {1}".format(eta, exc))
self.eta = maybe_make_aware(eta, self.tzlocal)
else:
self.eta = None
expires = headers.get("expires")
if expires is not None:
try:
expires = maybe_iso8601(expires)
except (AttributeError, ValueError, TypeError) as exc:
raise InvalidTaskError("invalid expires value {0!r}: {1}".format(expires, exc))
self.expires = maybe_make_aware(expires, self.tzlocal)
else:
self.expires = None
delivery_info = message.delivery_info or {}
properties = message.properties or {}
headers.update(
{
"reply_to": properties.get("reply_to"),
"correlation_id": properties.get("correlation_id"),
"delivery_info": {
"exchange": delivery_info.get("exchange"),
"routing_key": delivery_info.get("routing_key"),
"priority": properties.get("priority"),
"redelivered": delivery_info.get("redelivered"),
},
}
)
self.request_dict = headers
def __init__(self, message, on_ack=noop,
hostname=None, eventer=None, app=None,
connection_errors=None, request_dict=None,
task=None, on_reject=noop, body=None,
headers=None, decoded=False, utc=True,
maybe_make_aware=maybe_make_aware,
maybe_iso8601=maybe_iso8601, **opts):
if headers is None:
headers = message.headers
if body is None:
body = message.body
self.app = app
self.message = message
self.body = body
self.utc = utc
self._decoded = decoded
if decoded:
self.content_type = self.content_encoding = None
else:
self.content_type, self.content_encoding = (
message.content_type, message.content_encoding,
)
self.id = headers['id']
type = self.type = self.name = headers['task']
self.root_id = headers.get('root_id')
self.parent_id = headers.get('parent_id')
if 'shadow' in headers:
self.name = headers['shadow'] or self.name
if 'timelimit' in headers:
self.time_limits = headers['timelimit']
self.argsrepr = headers.get('argsrepr', '')
self.kwargsrepr = headers.get('kwargsrepr', '')
self.on_ack = on_ack
self.on_reject = on_reject
self.hostname = hostname or gethostname()
self.eventer = eventer
self.connection_errors = connection_errors or ()
self.task = task or self.app.tasks[type]
# timezone means the message is timezone-aware, and the only timezone
# supported at this point is UTC.
eta = headers.get('eta')
if eta is not None:
try:
eta = maybe_iso8601(eta)
except (AttributeError, ValueError, TypeError) as exc:
raise InvalidTaskError(
'invalid eta value {0!r}: {1}'.format(eta, exc))
self.eta = maybe_make_aware(eta, self.tzlocal)
else:
self.eta = None
expires = headers.get('expires')
if expires is not None:
try:
expires = maybe_iso8601(expires)
except (AttributeError, ValueError, TypeError) as exc:
raise InvalidTaskError(
'invalid expires value {0!r}: {1}'.format(expires, exc))
self.expires = maybe_make_aware(expires, self.tzlocal)
else:
self.expires = None
delivery_info = message.delivery_info or {}
properties = message.properties or {}
headers.update({
'reply_to': properties.get('reply_to'),
'correlation_id': properties.get('correlation_id'),
'delivery_info': {
'exchange': delivery_info.get('exchange'),
'routing_key': delivery_info.get('routing_key'),
'priority': properties.get('priority'),
'redelivered': delivery_info.get('redelivered'),
}
})
self.request_dict = headers
def __init__(self,
body,
on_ack=noop,
hostname=None,
eventer=None,
app=None,
connection_errors=None,
request_dict=None,
message=None,
task=None,
on_reject=noop,
**opts):
self.app = app
name = self.name = body['task']
self.id = body['id']
self.args = body.get('args', [])
self.kwargs = body.get('kwargs', {})
try:
self.kwargs.items
except AttributeError:
raise InvalidTaskError('Task keyword arguments is not a mapping')
if NEEDS_KWDICT:
self.kwargs = kwdict(self.kwargs)
eta = body.get('eta')
expires = body.get('expires')
utc = self.utc = body.get('utc', False)
self.on_ack = on_ack
self.on_reject = on_reject
self.hostname = hostname or socket.gethostname()
self.eventer = eventer
self.connection_errors = connection_errors or ()
self.task = task or self.app.tasks[name]
self.acknowledged = self._already_revoked = False
self.time_start = self.worker_pid = self._terminate_on_ack = None
self._apply_result = None
self._tzlocal = None
# timezone means the message is timezone-aware, and the only timezone
# supported at this point is UTC.
if eta is not None:
try:
self.eta = maybe_iso8601(eta)
except (AttributeError, ValueError, TypeError) as exc:
raise InvalidTaskError('invalid eta value {0!r}: {1}'.format(
eta, exc))
if utc:
self.eta = maybe_make_aware(self.eta, self.tzlocal)
else:
self.eta = None
if expires is not None:
try:
self.expires = maybe_iso8601(expires)
except (AttributeError, ValueError, TypeError) as exc:
raise InvalidTaskError(
'invalid expires value {0!r}: {1}'.format(expires, exc))
if utc:
self.expires = maybe_make_aware(self.expires, self.tzlocal)
else:
self.expires = None
if message:
delivery_info = message.delivery_info or {}
properties = message.properties or {}
body.update({
'headers': message.headers,
'reply_to': properties.get('reply_to'),
'correlation_id': properties.get('correlation_id'),
'delivery_info': {
'exchange':
delivery_info.get('exchange'),
'routing_key':
delivery_info.get('routing_key'),
'priority':
properties.get('priority', delivery_info.get('priority')),
'redelivered':
delivery_info.get('redelivered'),
}
})
else:
body.update(DEFAULT_FIELDS)
self.request_dict = body
hostname=hostname,
defaults={"last_heartbeat":
self.get_heartbeat(worker)})
self._last_worker_write[hostname] = (time(), obj)
return obj
def handle_task(self, (uuid, task), worker=None):
"""Handle snapshotted event."""
if task.worker and task.worker.hostname:
worker = self.handle_worker((task.worker.hostname, task.worker))
NOT_SAVED_ATTRIBUTES = ('name', 'args', 'kwargs', 'eta')
defaults = {"name": task.name,
"args": task.args,
"kwargs": task.kwargs,
"eta": maybe_make_aware(maybe_iso8601(task.eta)),
"expires": maybe_make_aware(maybe_iso8601(task.expires)),
"state": task.state,
"tstamp": make_aware(datetime.fromtimestamp(
task.timestamp)),
"result": task.result or task.exception,
"traceback": task.traceback,
"runtime": task.runtime,
"worker": worker}
# If RECEIVED event is lost, some data is lost as it is stored
# only in RECEIVED event for efficiency. In this case we just
# do not overwrite this fields in TaskState instance row
[defaults.pop(attr) for attr in NOT_SAVED_ATTRIBUTES
if defaults[attr] is None]
return self.update_task(task.state, task_id=uuid, defaults=defaults)
return obj
def handle_task(self, (uuid, task), worker=None):
if task.worker and task.worker.hostname:
worker = self.handle_worker((task.worker.hostname, task.worker))
return self.update_task(task.state,
task_id=uuid,
defaults={
"name":
task.name,
"args":
task.args,
"kwargs":
task.kwargs,
"eta":
maybe_iso8601(task.eta),
"expires":
maybe_iso8601(task.expires),
"state":
task.state,
"tstamp":
datetime.fromtimestamp(task.timestamp),
"result":
task.result or task.exception,
"traceback":
task.traceback,
"runtime":
task.runtime,
"worker":
worker
})
def test_maybe_iso8601_datetime(self):
from celery.utils.timeutils import maybe_iso8601
from datetime import datetime
now = datetime.now()
self.assertIs(maybe_iso8601(now), now)
hostname=hostname,
defaults={"last_heartbeat": self.get_heartbeat(worker)})
self._last_worker_write[hostname] = (time(), obj)
return obj
def handle_task(self, (uuid, task), worker=None):
if task.worker and task.worker.hostname:
worker = self.handle_worker((task.worker.hostname, task.worker))
return self.update_task(
task.state,
task_id=uuid,
defaults={
"name": task.name,
"args": task.args,
"kwargs": task.kwargs,
"eta": maybe_iso8601(task.eta),
"expires": maybe_iso8601(task.expires),
"state": task.state,
"tstamp": make_aware(datetime.fromtimestamp(task.timestamp)),
"result": task.result or task.exception,
"traceback": task.traceback,
"runtime": task.runtime,
"worker": worker
})
def update_task(self, state, **kwargs):
objects = self.TaskState.objects
defaults = kwargs.pop("defaults", None) or {}
try:
obj = objects.get(**kwargs)
except ObjectDoesNotExist:
)
self._last_worker_write[hostname] = (time(), obj)
return obj
def handle_task(self, (uuid, task), worker=None):
"""Handle snapshotted event."""
if task.worker and task.worker.hostname:
worker = self.handle_worker(
(task.worker.hostname, task.worker),
)
defaults = {
'name': task.name,
'args': task.args,
'kwargs': task.kwargs,
'eta': maybe_make_aware(maybe_iso8601(task.eta)),
'expires': maybe_make_aware(maybe_iso8601(task.expires)),
'state': task.state,
'tstamp': aware_tstamp(task.timestamp),
'result': task.result or task.exception,
'traceback': task.traceback,
'runtime': task.runtime,
'worker': worker
}
# Some fields are only stored in the RECEIVED event,
# so we should remove these from default values,
# so that they are not overwritten by subsequent states.
[defaults.pop(attr, None) for attr in NOT_SAVED_ATTRIBUTES
if defaults[attr] is None]
return self.update_task(task.state,
task_id=uuid, defaults=defaults)
def test_maybe_iso8601_datetime(self):
now = datetime.now()
self.assertIs(timeutils.maybe_iso8601(now), now)
def __init__(
self,
body,
on_ack=noop,
hostname=None,
eventer=None,
app=None,
connection_errors=None,
request_dict=None,
message=None,
task=None,
on_reject=noop,
**opts
):
self.app = app
name = self.name = body["task"]
self.id = body["id"]
self.args = body.get("args", [])
self.kwargs = body.get("kwargs", {})
try:
self.kwargs.items
except AttributeError:
raise InvalidTaskError("Task keyword arguments is not a mapping")
if NEEDS_KWDICT:
self.kwargs = kwdict(self.kwargs)
eta = body.get("eta")
expires = body.get("expires")
utc = self.utc = body.get("utc", False)
self.on_ack = on_ack
self.on_reject = on_reject
self.hostname = hostname or socket.gethostname()
self.eventer = eventer
self.connection_errors = connection_errors or ()
self.task = task or self.app.tasks[name]
self.acknowledged = self._already_revoked = False
self.time_start = self.worker_pid = self._terminate_on_ack = None
self._apply_result = None
self._tzlocal = None
# timezone means the message is timezone-aware, and the only timezone
# supported at this point is UTC.
if eta is not None:
try:
self.eta = maybe_iso8601(eta)
except (AttributeError, ValueError, TypeError) as exc:
raise InvalidTaskError("invalid eta value {0!r}: {1}".format(eta, exc))
if utc:
self.eta = maybe_make_aware(self.eta, self.tzlocal)
else:
self.eta = None
if expires is not None:
try:
self.expires = maybe_iso8601(expires)
except (AttributeError, ValueError, TypeError) as exc:
raise InvalidTaskError("invalid expires value {0!r}: {1}".format(expires, exc))
if utc:
self.expires = maybe_make_aware(self.expires, self.tzlocal)
else:
self.expires = None
if message:
delivery_info = message.delivery_info or {}
properties = message.properties or {}
body.update(
{
"headers": message.headers,
"reply_to": properties.get("reply_to"),
"correlation_id": properties.get("correlation_id"),
"delivery_info": {
"exchange": delivery_info.get("exchange"),
"routing_key": delivery_info.get("routing_key"),
"priority": delivery_info.get("priority"),
"redelivered": delivery_info.get("redelivered"),
},
}
)
else:
body.update(DEFAULT_FIELDS)
self.request_dict = body
def test_maybe_iso8601_datetime(self):
from celery.utils.timeutils import maybe_iso8601
from datetime import datetime
now = datetime.now()
self.assertIs(maybe_iso8601(now), now)
obj = self.WorkerState.objects.update_or_create(
hostname=hostname,
defaults={"last_heartbeat": self.get_heartbeat(worker)})
self._last_worker_write[hostname] = (time(), obj)
return obj
def handle_task(self, (uuid, task), worker=None):
"""Handle snapshotted event."""
if task.worker and task.worker.hostname:
worker = self.handle_worker((task.worker.hostname, task.worker))
defaults = {
"name": task.name,
"args": task.args,
"kwargs": task.kwargs,
"eta": maybe_make_aware(maybe_iso8601(task.eta)),
"expires": maybe_make_aware(maybe_iso8601(task.expires)),
"state": task.state,
"tstamp": aware_tstamp(task.timestamp),
"result": task.result or task.exception,
"traceback": task.traceback,
"runtime": task.runtime,
"worker": worker
}
# If RECEIVED event is lost, some data is lost as it is stored
# only in RECEIVED event for efficiency. In this case we just
# do not overwrite this fields in TaskState instance row
[
defaults.pop(attr, None) for attr in NOT_SAVED_ATTRIBUTES
if defaults[attr] is None
]
def __init__(self, message, on_ack=noop,
hostname=None, eventer=None, app=None,
connection_errors=None, request_dict=None,
task=None, on_reject=noop, body=None,
headers=None, decoded=False, utc=True,
maybe_make_aware=maybe_make_aware,
maybe_iso8601=maybe_iso8601, **opts):
if headers is None:
headers = message.headers
if body is None:
body = message.body
self.app = app
self.message = message
self.body = body
self.utc = utc
self._decoded = decoded
if decoded:
self.content_type = self.content_encoding = None
else:
self.content_type, self.content_encoding = (
message.content_type, message.content_encoding,
)
self.id = headers['id']
type = self.type = self.name = headers['task']
self.root_id = headers.get('root_id')
self.parent_id = headers.get('parent_id')
if 'shadow' in headers:
self.name = headers['shadow'] or self.name
if 'timelimit' in headers:
self.time_limits = headers['timelimit']
self.argsrepr = headers.get('argsrepr', '')
self.kwargsrepr = headers.get('kwargsrepr', '')
self.on_ack = on_ack
self.on_reject = on_reject
self.hostname = hostname or gethostname()
self.eventer = eventer
self.connection_errors = connection_errors or ()
self.task = task or self.app.tasks[type]
# timezone means the message is timezone-aware, and the only timezone
# supported at this point is UTC.
eta = headers.get('eta')
if eta is not None:
try:
eta = maybe_iso8601(eta)
except (AttributeError, ValueError, TypeError) as exc:
raise InvalidTaskError(
'invalid eta value {0!r}: {1}'.format(eta, exc))
self.eta = maybe_make_aware(eta, self.tzlocal)
else:
self.eta = None
expires = headers.get('expires')
if expires is not None:
try:
expires = maybe_iso8601(expires)
except (AttributeError, ValueError, TypeError) as exc:
raise InvalidTaskError(
'invalid expires value {0!r}: {1}'.format(expires, exc))
self.expires = maybe_make_aware(expires, self.tzlocal)
else:
self.expires = None
delivery_info = message.delivery_info or {}
properties = message.properties or {}
headers.update({
'reply_to': properties.get('reply_to'),
'correlation_id': properties.get('correlation_id'),
'delivery_info': {
'exchange': delivery_info.get('exchange'),
'routing_key': delivery_info.get('routing_key'),
'priority': properties.get('priority'),
'redelivered': delivery_info.get('redelivered'),
}
})
self.request_dict = headers
# timezone means the message is timezone-aware, and the only timezone
# supported at this point is UTC.
if eta is not None:
try:
self.eta = maybe_iso8601(eta)
except (AttributeError, ValueError), exc:
raise InvalidTaskError(
'invalid eta value %r: %s' % (eta, exc, ))
if utc:
self.eta = maybe_make_aware(self.eta, self.tzlocal)
else:
self.eta = None
if expires is not None:
try:
self.expires = maybe_iso8601(expires)
except (AttributeError, ValueError), exc:
raise InvalidTaskError(
'invalid expires value %r: %s' % (expires, exc, ))
if utc:
self.expires = maybe_make_aware(self.expires, self.tzlocal)
else:
self.expires = None
delivery_info = {} if delivery_info is None else delivery_info
self.delivery_info = {
'exchange': delivery_info.get('exchange'),
'routing_key': delivery_info.get('routing_key'),
'priority': delivery_info.get('priority'),
}
hostname=hostname,
defaults={'last_heartbeat': self.get_heartbeat(worker)},
)
self._last_worker_write[hostname] = (monotonic(), obj)
return obj
def handle_task(self, (uuid, task), worker=None):
"""Handle snapshotted event."""
if task.worker and task.worker.hostname:
worker = self.handle_worker((task.worker.hostname, task.worker), )
defaults = {
'name': task.name,
'args': task.args,
'kwargs': task.kwargs,
'eta': maybe_make_aware(maybe_iso8601(task.eta)),
'expires': maybe_make_aware(maybe_iso8601(task.expires)),
'state': task.state,
'tstamp': aware_tstamp(task.timestamp),
'result': task.result or task.exception,
'traceback': task.traceback,
'runtime': task.runtime,
'worker': worker
}
# Some fields are only stored in the RECEIVED event,
# so we should remove these from default values,
# so that they are not overwritten by subsequent states.
[
defaults.pop(attr, None) for attr in NOT_SAVED_ATTRIBUTES
if defaults[attr] is None
]
if not last_write or time() - last_write > self.worker_update_freq:
obj = self.WorkerState.objects.update_or_create(
hostname=hostname,
defaults={"last_heartbeat":
self.get_heartbeat(worker)})
self._last_worker_write[hostname] = (time(), obj)
return obj
def handle_task(self, (uuid, task), worker=None):
if task.worker and task.worker.hostname:
worker = self.handle_worker((task.worker.hostname, task.worker))
return self.update_task(task.state, task_id=uuid,
defaults={"name": task.name,
"args": task.args,
"kwargs": task.kwargs,
"eta": maybe_iso8601(task.eta),
"expires": maybe_iso8601(task.expires),
"state": task.state,
"tstamp": datetime.fromtimestamp(task.timestamp),
"result": task.result or task.exception,
"traceback": task.traceback,
"runtime": task.runtime,
"worker": worker})
def update_task(self, state, **kwargs):
objects = self.TaskState.objects
defaults = kwargs.pop("defaults", None) or {}
try:
obj = objects.get(**kwargs)
except ObjectDoesNotExist:
if not defaults.get("name"):
def test_maybe_iso8601_datetime(self):
now = datetime.now()
self.assertIs(maybe_iso8601(now), now)
# supported at this point is UTC.
if eta is not None:
try:
self.eta = maybe_iso8601(eta)
except (AttributeError, ValueError), exc:
raise InvalidTaskError('invalid eta value %r: %s' % (
eta,
exc,
))
if utc:
self.eta = maybe_make_aware(self.eta, self.tzlocal)
else:
self.eta = None
if expires is not None:
try:
self.expires = maybe_iso8601(expires)
except (AttributeError, ValueError), exc:
raise InvalidTaskError('invalid expires value %r: %s' % (
expires,
exc,
))
if utc:
self.expires = maybe_make_aware(self.expires, self.tzlocal)
else:
self.expires = None
delivery_info = {} if delivery_info is None else delivery_info
self.delivery_info = {
'exchange': delivery_info.get('exchange'),
'routing_key': delivery_info.get('routing_key'),
'priority': delivery_info.get('priority'),
