def handle_task(self, uuid_task, worker=None):
"""Handle snapshotted event."""
uuid, task = uuid_task
if task.worker and task.worker.hostname:
worker = self.handle_worker((task.worker.hostname, task.worker), )
defaults = {
'name': task.name,
'args': task.args,
'kwargs': task.kwargs,
'eta': correct_awareness(maybe_iso8601(task.eta)),
'expires': correct_awareness(maybe_iso8601(task.expires)),
'state': task.state,
'tstamp': fromtimestamp(task.timestamp),
'result': task.result or task.exception,
'traceback': task.traceback,
'runtime': task.runtime,
'worker': worker
}
# Some fields are only stored in the RECEIVED event,
# so we should remove these from default values,
# so that they are not overwritten by subsequent states.
[
defaults.pop(attr, None) for attr in NOT_SAVED_ATTRIBUTES
if defaults[attr] is None
]
return self.update_task(task.state, task_id=uuid, defaults=defaults)
def _send_task(self, name, args=None, kwargs=None,
countdown=None, serializer=None,
queue=None, exchange=None, routing_key=None,
eta=None, expires=None, **_):
# arguments
args = loads(args) if isinstance(args, string_t) else args
kwargs = loads(kwargs) if isinstance(kwargs, string_t) else kwargs
# Expires can be int/float.
try:
expires = float(expires)
except (TypeError, ValueError):
# or a string describing an ISO 8601 datetime.
try:
expires = maybe_iso8601(expires)
except (TypeError, ValueError):
raise
# send the task and print the id.
self.out(self.app.send_task(
name,
args=args or (), kwargs=kwargs or {},
countdown=countdown,
serializer=serializer,
queue=queue,
exchange=exchange,
routing_key=routing_key,
eta=maybe_iso8601(eta),
expires=expires,
).id)
def _send_task(self, name, args=None, kwargs=None,
countdown=None, serializer=None,
queue=None, exchange=None, routing_key=None,
eta=None, expires=None):
# arguments
args = loads(args) if isinstance(args, string_t) else args
kwargs = loads(kwargs) if isinstance(kwargs, string_t) else kwargs
# Expires can be int/float.
try:
expires = float(expires)
except (TypeError, ValueError):
# or a string describing an ISO 8601 datetime.
try:
expires = maybe_iso8601(expires)
except (TypeError, ValueError):
raise
# send the task and print the id.
self.out(self.app.send_task(
name,
args=args or (), kwargs=kwargs or {},
countdown=countdown,
serializer=serializer,
queue=queue,
exchange=exchange,
routing_key=routing_key,
eta=maybe_iso8601(eta),
expires=expires,
).id)
def send_task(self, *args, **kwargs):
# HACK: This needs to be removed once Celery > 4.0.2 is out:
# see https://github.com/celery/celery/issues/3734
# and https://github.com/celery/celery/pull/3790
expires = kwargs.get("expires")
if isinstance(expires, string_t):
kwargs["expires"] = maybe_iso8601(expires)
return super().send_task(*args, **kwargs)
def convert(self, value, param, ctx):
try:
return float(value)
except (TypeError, ValueError):
pass
try:
return maybe_iso8601(value)
except (TypeError, ValueError) as e:
self.fail(e)
def on_task_start(self, task_id, args, kwargs):
super().on_task_start(task_id, args, kwargs)
run_logger = CeleryTaskRunLogger()
self.request.run_logger = run_logger
self.request.invocation_logger = CeleryInvocationLogger(
**deserialize_data(
self._get_header_from_request('invocation_logger_data')))
run_logger.log_started(
name=self.name,
queue_name=self.request.delivery_info.get('routing_key'),
task_args=args,
task_kwargs=kwargs,
celery_task_id=task_id,
retries=self.request.retries,
trigger_time=maybe_iso8601(
self._get_header_from_request('trigger_time')))
self.request.output_stream = LogStringIO(
flush_callback=lambda output_stream: run_logger.log_output_updated(
output_stream.getvalue()))
def __init__(self, message, on_ack=noop,
hostname=None, eventer=None, app=None,
connection_errors=None, request_dict=None,
task=None, on_reject=noop, body=None,
headers=None, decoded=False, utc=True,
maybe_make_aware=maybe_make_aware,
maybe_iso8601=maybe_iso8601, **opts):
if headers is None:
headers = message.headers
if body is None:
body = message.body
self.app = app
self.message = message
self.body = body
self.utc = utc
self._decoded = decoded
if decoded:
self.content_type = self.content_encoding = None
else:
self.content_type, self.content_encoding = (
message.content_type, message.content_encoding,
)
self.id = headers['id']
type = self.type = self.name = headers['task']
self.root_id = headers.get('root_id')
self.parent_id = headers.get('parent_id')
if 'shadow' in headers:
self.name = headers['shadow'] or self.name
timelimit = headers.get('timelimit', None)
if timelimit:
self.time_limits = timelimit
self.argsrepr = headers.get('argsrepr', '')
self.kwargsrepr = headers.get('kwargsrepr', '')
self.on_ack = on_ack
self.on_reject = on_reject
self.hostname = hostname or gethostname()
self.eventer = eventer
self.connection_errors = connection_errors or ()
self.task = task or self.app.tasks[type]
# timezone means the message is timezone-aware, and the only timezone
# supported at this point is UTC.
eta = headers.get('eta')
if eta is not None:
try:
eta = maybe_iso8601(eta)
except (AttributeError, ValueError, TypeError) as exc:
raise InvalidTaskError(
'invalid ETA value {0!r}: {1}'.format(eta, exc))
self.eta = maybe_make_aware(eta, self.tzlocal)
else:
self.eta = None
expires = headers.get('expires')
if expires is not None:
try:
expires = maybe_iso8601(expires)
except (AttributeError, ValueError, TypeError) as exc:
raise InvalidTaskError(
'invalid expires value {0!r}: {1}'.format(expires, exc))
self.expires = maybe_make_aware(expires, self.tzlocal)
else:
self.expires = None
delivery_info = message.delivery_info or {}
properties = message.properties or {}
headers.update({
'reply_to': properties.get('reply_to'),
'correlation_id': properties.get('correlation_id'),
'delivery_info': {
'exchange': delivery_info.get('exchange'),
'routing_key': delivery_info.get('routing_key'),
'priority': properties.get('priority'),
'redelivered': delivery_info.get('redelivered'),
}
})
self.request_dict = headers
def __init__(self,
message,
on_ack=noop,
hostname=None,
eventer=None,
app=None,
connection_errors=None,
request_dict=None,
task=None,
on_reject=noop,
body=None,
headers=None,
decoded=False,
utc=True,
maybe_make_aware=maybe_make_aware,
maybe_iso8601=maybe_iso8601,
**opts):
self._message = message
self._request_dict = (message.headers.copy()
if headers is None else headers.copy())
self._body = message.body if body is None else body
self._app = app
self._utc = utc
self._decoded = decoded
if decoded:
self._content_type = self._content_encoding = None
else:
self._content_type, self._content_encoding = (
message.content_type,
message.content_encoding,
)
self.__payload = self._body if self._decoded else message.payload
self.id = self._request_dict['id']
self._type = self.name = self._request_dict['task']
if 'shadow' in self._request_dict:
self.name = self._request_dict['shadow'] or self.name
self._root_id = self._request_dict.get('root_id')
self._parent_id = self._request_dict.get('parent_id')
timelimit = self._request_dict.get('timelimit', None)
if timelimit:
self.time_limits = timelimit
self._argsrepr = self._request_dict.get('argsrepr', '')
self._kwargsrepr = self._request_dict.get('kwargsrepr', '')
self._on_ack = on_ack
self._on_reject = on_reject
self._hostname = hostname or gethostname()
self._eventer = eventer
self._connection_errors = connection_errors or ()
self._task = task or self._app.tasks[self._type]
self._ignore_result = self._request_dict.get('ignore_result', False)
# timezone means the message is timezone-aware, and the only timezone
# supported at this point is UTC.
eta = self._request_dict.get('eta')
if eta is not None:
try:
eta = maybe_iso8601(eta)
except (AttributeError, ValueError, TypeError) as exc:
raise InvalidTaskError(f'invalid ETA value {eta!r}: {exc}')
self._eta = maybe_make_aware(eta, self.tzlocal)
else:
self._eta = None
expires = self._request_dict.get('expires')
if expires is not None:
try:
expires = maybe_iso8601(expires)
except (AttributeError, ValueError, TypeError) as exc:
raise InvalidTaskError(
f'invalid expires value {expires!r}: {exc}')
self._expires = maybe_make_aware(expires, self.tzlocal)
else:
self._expires = None
delivery_info = message.delivery_info or {}
properties = message.properties or {}
self._delivery_info = {
'exchange': delivery_info.get('exchange'),
'routing_key': delivery_info.get('routing_key'),
'priority': properties.get('priority'),
'redelivered': delivery_info.get('redelivered', False),
}
self._request_dict.update({
'properties':
properties,
'reply_to':
properties.get('reply_to'),
'correlation_id':
properties.get('correlation_id'),
'hostname':
self._hostname,
'delivery_info':
self._delivery_info
})
# this is a reference pass to avoid memory usage burst
self._request_dict['args'], self._request_dict[
'kwargs'], _ = self.__payload
self._args = self._request_dict['args']
self._kwargs = self._request_dict['kwargs']
def test_maybe_iso8601_datetime():
now = datetime.now()
assert maybe_iso8601(now) is now
def __init__(self,
message,
on_ack=noop,
hostname=None,
eventer=None,
app=None,
connection_errors=None,
request_dict=None,
task=None,
on_reject=noop,
body=None,
headers=None,
decoded=False,
utc=True,
maybe_make_aware=maybe_make_aware,
maybe_iso8601=maybe_iso8601,
**opts):
self._message = message
self._request_dict = message.headers if headers is None else headers
self._body = message.body if body is None else body
self._app = app
self._utc = utc
self._decoded = decoded
if decoded:
self._content_type = self._content_encoding = None
else:
self._content_type, self._content_encoding = (
message.content_type,
message.content_encoding,
)
self.__payload = self._body if self._decoded else message.payload
self.id = self._request_dict["id"]
self._type = self.name = self._request_dict["task"]
if "shadow" in self._request_dict:
self.name = self._request_dict["shadow"] or self.name
self._root_id = self._request_dict.get("root_id")
self._parent_id = self._request_dict.get("parent_id")
timelimit = self._request_dict.get("timelimit", None)
if timelimit:
self.time_limits = timelimit
self._argsrepr = self._request_dict.get("argsrepr", "")
self._kwargsrepr = self._request_dict.get("kwargsrepr", "")
self._on_ack = on_ack
self._on_reject = on_reject
self._hostname = hostname or gethostname()
self._eventer = eventer
self._connection_errors = connection_errors or ()
self._task = task or self._app.tasks[self._type]
# timezone means the message is timezone-aware, and the only timezone
# supported at this point is UTC.
eta = self._request_dict.get("eta")
if eta is not None:
try:
eta = maybe_iso8601(eta)
except (AttributeError, ValueError, TypeError) as exc:
raise InvalidTaskError("invalid ETA value {0!r}: {1}".format(
eta, exc))
self._eta = maybe_make_aware(eta, self.tzlocal)
else:
self._eta = None
expires = self._request_dict.get("expires")
if expires is not None:
try:
expires = maybe_iso8601(expires)
except (AttributeError, ValueError, TypeError) as exc:
raise InvalidTaskError(
"invalid expires value {0!r}: {1}".format(expires, exc))
self._expires = maybe_make_aware(expires, self.tzlocal)
else:
self._expires = None
delivery_info = message.delivery_info or {}
properties = message.properties or {}
self._delivery_info = {
"exchange": delivery_info.get("exchange"),
"routing_key": delivery_info.get("routing_key"),
"priority": properties.get("priority"),
"redelivered": delivery_info.get("redelivered"),
}
self._request_dict.update({
"reply_to":
properties.get("reply_to"),
"correlation_id":
properties.get("correlation_id"),
"hostname":
self._hostname,
"delivery_info":
self._delivery_info,
})
# this is a reference pass to avoid memory usage burst
self._request_dict["args"], self._request_dict[
"kwargs"], _ = self.__payload
self._args = self._request_dict["args"]
self._kwargs = self._request_dict["kwargs"]
def test_maybe_iso8601_datetime():
now = datetime.now()
assert maybe_iso8601(now) is now
def __init__(self, message, on_ack=noop,
hostname=None, eventer=None, app=None,
connection_errors=None, request_dict=None,
task=None, on_reject=noop, body=None,
headers=None, decoded=False, utc=True,
maybe_make_aware=maybe_make_aware,
maybe_iso8601=maybe_iso8601, **opts):
if headers is None:
headers = message.headers
if body is None:
body = message.body
self.app = app
self.message = message
self.body = body
self.utc = utc
self._decoded = decoded
if decoded:
self.content_type = self.content_encoding = None
else:
self.content_type, self.content_encoding = (
message.content_type, message.content_encoding,
)
self.id = headers['id']
type = self.type = self.name = headers['task']
self.root_id = headers.get('root_id')
self.parent_id = headers.get('parent_id')
if 'shadow' in headers:
self.name = headers['shadow'] or self.name
if 'timelimit' in headers:
self.time_limits = headers['timelimit']
self.argsrepr = headers.get('argsrepr', '')
self.kwargsrepr = headers.get('kwargsrepr', '')
self.on_ack = on_ack
self.on_reject = on_reject
self.hostname = hostname or gethostname()
self.eventer = eventer
self.connection_errors = connection_errors or ()
self.task = task or self.app.tasks[type]
# timezone means the message is timezone-aware, and the only timezone
# supported at this point is UTC.
eta = headers.get('eta')
if eta is not None:
try:
eta = maybe_iso8601(eta)
except (AttributeError, ValueError, TypeError) as exc:
raise InvalidTaskError(
'invalid ETA value {0!r}: {1}'.format(eta, exc))
self.eta = maybe_make_aware(eta, self.tzlocal)
else:
self.eta = None
expires = headers.get('expires')
if expires is not None:
try:
expires = maybe_iso8601(expires)
except (AttributeError, ValueError, TypeError) as exc:
raise InvalidTaskError(
'invalid expires value {0!r}: {1}'.format(expires, exc))
self.expires = maybe_make_aware(expires, self.tzlocal)
else:
self.expires = None
delivery_info = message.delivery_info or {}
properties = message.properties or {}
headers.update({
'reply_to': properties.get('reply_to'),
'correlation_id': properties.get('correlation_id'),
'delivery_info': {
'exchange': delivery_info.get('exchange'),
'routing_key': delivery_info.get('routing_key'),
'priority': properties.get('priority'),
'redelivered': delivery_info.get('redelivered'),
}
})
self.request_dict = headers
def __init__(
self,
message,
on_ack=noop,
hostname=None,
eventer=None,
app=None,
connection_errors=None,
request_dict=None,
task=None,
on_reject=noop,
body=None,
headers=None,
decoded=False,
utc=True,
maybe_make_aware=maybe_make_aware,
maybe_iso8601=maybe_iso8601,
**opts
):
if headers is None:
headers = message.headers
if body is None:
body = message.body
self.app = app
self.message = message
self.body = body
self.utc = utc
self._decoded = decoded
if decoded:
self.content_type = self.content_encoding = None
else:
self.content_type, self.content_encoding = (message.content_type, message.content_encoding)
self.id = headers["id"]
type = self.type = self.name = headers["task"]
self.root_id = headers.get("root_id")
self.parent_id = headers.get("parent_id")
if "shadow" in headers:
self.name = headers["shadow"] or self.name
if "timelimit" in headers:
self.time_limits = headers["timelimit"]
self.argsrepr = headers.get("argsrepr", "")
self.kwargsrepr = headers.get("kwargsrepr", "")
self.on_ack = on_ack
self.on_reject = on_reject
self.hostname = hostname or gethostname()
self.eventer = eventer
self.connection_errors = connection_errors or ()
self.task = task or self.app.tasks[type]
# timezone means the message is timezone-aware, and the only timezone
# supported at this point is UTC.
eta = headers.get("eta")
if eta is not None:
try:
eta = maybe_iso8601(eta)
except (AttributeError, ValueError, TypeError) as exc:
raise InvalidTaskError("invalid ETA value {0!r}: {1}".format(eta, exc))
self.eta = maybe_make_aware(eta, self.tzlocal)
else:
self.eta = None
expires = headers.get("expires")
if expires is not None:
try:
expires = maybe_iso8601(expires)
except (AttributeError, ValueError, TypeError) as exc:
raise InvalidTaskError("invalid expires value {0!r}: {1}".format(expires, exc))
self.expires = maybe_make_aware(expires, self.tzlocal)
else:
self.expires = None
delivery_info = message.delivery_info or {}
properties = message.properties or {}
headers.update(
{
"reply_to": properties.get("reply_to"),
"correlation_id": properties.get("correlation_id"),
"delivery_info": {
"exchange": delivery_info.get("exchange"),
"routing_key": delivery_info.get("routing_key"),
"priority": properties.get("priority"),
"redelivered": delivery_info.get("redelivered"),
},
}
)
self.request_dict = headers
def decode_datetime(s):
return maybe_iso8601(s) if s else None
def test_maybe_iso8601_datetime(self):
now = datetime.now()
self.assertIs(maybe_iso8601(now), now)
