def handle_task(self, uuid_task, worker=None): """Handle snapshotted event.""" uuid, task = uuid_task if task.worker and task.worker.hostname: worker = self.handle_worker((task.worker.hostname, task.worker), ) defaults = { 'name': task.name, 'args': task.args, 'kwargs': task.kwargs, 'eta': maybe_make_aware(maybe_iso8601(task.eta)), 'expires': maybe_make_aware(maybe_iso8601(task.expires)), 'state': task.state, 'tstamp': aware_tstamp(task.timestamp), 'result': task.result or task.exception, 'traceback': task.traceback, 'runtime': task.runtime, 'worker': worker } # Some fields are only stored in the RECEIVED event, # so we should remove these from default values, # so that they are not overwritten by subsequent states. [ defaults.pop(attr, None) for attr in NOT_SAVED_ATTRIBUTES if defaults[attr] is None ] return self.update_task(task.state, task_id=uuid, defaults=defaults)
def handle_task(self, uuid_task, worker=None): """Handle snapshotted event.""" (uuid, task) = uuid_task if task.worker and task.worker.hostname: worker = self.handle_worker( (task.worker.hostname, task.worker), ) defaults = { 'name': task.name, 'args': task.args, 'kwargs': task.kwargs, 'eta': maybe_make_aware(maybe_iso8601(task.eta)), 'expires': maybe_make_aware(maybe_iso8601(task.expires)), 'state': task.state, 'tstamp': aware_tstamp(task.timestamp), 'result': task.result or task.exception, 'traceback': task.traceback, 'runtime': task.runtime, 'worker': worker } # Some fields are only stored in the RECEIVED event, # so we should remove these from default values, # so that they are not overwritten by subsequent states. [defaults.pop(attr, None) for attr in NOT_SAVED_ATTRIBUTES if defaults[attr] is None] return self.update_task(task.state, task_id=uuid, defaults=defaults)
def from_message(cls, message, body, on_ack=noop, **kw): """Create request from a task message. :raises UnknownTaskError: if the message does not describe a task, the message is also rejected. """ _delivery_info = getattr(message, "delivery_info", {}) delivery_info = dict( (key, _delivery_info.get(key)) for key in WANTED_DELIVERY_INFO) kwargs = body["kwargs"] if not hasattr(kwargs, "items"): raise InvalidTaskError("Task keyword arguments is not a mapping.") return cls(task_name=body["task"], task_id=body["id"], taskset_id=body.get("taskset", None), args=body["args"], kwargs=kwdict(kwargs), retries=body.get("retries", 0), eta=maybe_iso8601(body.get("eta")), expires=maybe_iso8601(body.get("expires")), on_ack=on_ack, delivery_info=delivery_info, **kw)
def from_message(cls, message, body, on_ack=noop, **kw): """Create request from a task message. :raises UnknownTaskError: if the message does not describe a task, the message is also rejected. """ _delivery_info = getattr(message, "delivery_info", {}) delivery_info = dict((key, _delivery_info.get(key)) for key in WANTED_DELIVERY_INFO) kwargs = body["kwargs"] if not hasattr(kwargs, "items"): raise InvalidTaskError("Task keyword arguments is not a mapping.") return cls(task_name=body["task"], task_id=body["id"], args=body["args"], kwargs=kwdict(kwargs), retries=body.get("retries", 0), eta=maybe_iso8601(body.get("eta")), expires=maybe_iso8601(body.get("expires")), on_ack=on_ack, delivery_info=delivery_info, **kw)
def run(self, name, *_, **kw): # Positional args. args = kw.get('args') or () if isinstance(args, string_t): args = anyjson.loads(args) # Keyword args. kwargs = kw.get('kwargs') or {} if isinstance(kwargs, string_t): kwargs = anyjson.loads(kwargs) # Expires can be int/float. expires = kw.get('expires') or None try: expires = float(expires) except (TypeError, ValueError): # or a string describing an ISO 8601 datetime. try: expires = maybe_iso8601(expires) except (TypeError, ValueError): raise res = self.app.send_task(name, args=args, kwargs=kwargs, countdown=kw.get('countdown'), serializer=kw.get('serializer'), queue=kw.get('queue'), exchange=kw.get('exchange'), routing_key=kw.get('routing_key'), eta=maybe_iso8601(kw.get('eta')), expires=expires) self.out(res.id)
def __init__(self, body, on_ack=noop, hostname=None, eventer=None, app=None, connection_errors=None, request_dict=None, delivery_info=None, task=None, on_reject=noop, **opts): self.app = app name = self.name = body['task'] self.id = body['id'] self.args = body.get('args', []) self.kwargs = body.get('kwargs', {}) try: self.kwargs.items except AttributeError: raise InvalidTaskError( 'Task keyword arguments is not a mapping') if NEEDS_KWDICT: self.kwargs = kwdict(self.kwargs) eta = body.get('eta') expires = body.get('expires') utc = self.utc = body.get('utc', False) self.on_ack = on_ack self.on_reject = on_reject self.hostname = hostname or socket.gethostname() self.eventer = eventer self.connection_errors = connection_errors or () self.task = task or self.app.tasks[name] self.acknowledged = self._already_revoked = False self.time_start = self.worker_pid = self._terminate_on_ack = None self._tzlocal = None # timezone means the message is timezone-aware, and the only timezone # supported at this point is UTC. if eta is not None: try: self.eta = maybe_iso8601(eta) except (AttributeError, ValueError, TypeError) as exc: raise InvalidTaskError( 'invalid eta value {0!r}: {1}'.format(eta, exc)) if utc: self.eta = maybe_make_aware(self.eta, self.tzlocal) else: self.eta = None if expires is not None: try: self.expires = maybe_iso8601(expires) except (AttributeError, ValueError, TypeError) as exc: raise InvalidTaskError( 'invalid expires value {0!r}: {1}'.format(expires, exc)) if utc: self.expires = maybe_make_aware(self.expires, self.tzlocal) else: self.expires = None delivery_info = {} if delivery_info is None else delivery_info self.delivery_info = { 'exchange': delivery_info.get('exchange'), 'routing_key': delivery_info.get('routing_key'), 'priority': delivery_info.get('priority'), 'redelivered': delivery_info.get('redelivered'), } self.request_dict = body
def __init__( self, body, on_ack=noop, hostname=None, eventer=None, app=None, connection_errors=None, request_dict=None, delivery_info=None, task=None, **opts ): self.app = app or app_or_default(app) name = self.name = body["task"] self.id = body["id"] self.args = body.get("args", []) self.kwargs = body.get("kwargs", {}) try: self.kwargs.items except AttributeError: raise exceptions.InvalidTaskError("Task keyword arguments is not a mapping") if NEEDS_KWDICT: self.kwargs = kwdict(self.kwargs) eta = body.get("eta") expires = body.get("expires") utc = body.get("utc", False) self.on_ack = on_ack self.hostname = hostname or socket.gethostname() self.eventer = eventer self.connection_errors = connection_errors or () self.task = task or self.app.tasks[name] self.acknowledged = self._already_revoked = False self.time_start = self.worker_pid = self._terminate_on_ack = None self._tzlocal = None # timezone means the message is timezone-aware, and the only timezone # supported at this point is UTC. if eta is not None: tz = tz_utc if utc else self.tzlocal self.eta = tz_to_local(maybe_iso8601(eta), self.tzlocal, tz) else: self.eta = None if expires is not None: tz = tz_utc if utc else self.tzlocal self.expires = tz_to_local(maybe_iso8601(expires), self.tzlocal, tz) else: self.expires = None delivery_info = {} if delivery_info is None else delivery_info self.delivery_info = { "exchange": delivery_info.get("exchange"), "routing_key": delivery_info.get("routing_key"), "priority": delivery_info.get("priority"), } # amqplib transport adds the channel here for some reason, so need # to remove it. self.delivery_info.pop("channel", None) self.request_dict = body
def __init__(self, body, on_ack=noop, hostname=None, eventer=None, app=None, connection_errors=None, request_dict=None, delivery_info=None, task=None, **opts): self.app = app or app_or_default(app) name = self.name = body['task'] self.id = body['id'] self.args = body.get('args', []) self.kwargs = body.get('kwargs', {}) try: self.kwargs.items except AttributeError: raise exceptions.InvalidTaskError( 'Task keyword arguments is not a mapping') if NEEDS_KWDICT: self.kwargs = kwdict(self.kwargs) eta = body.get('eta') expires = body.get('expires') utc = body.get('utc', False) self.on_ack = on_ack self.hostname = hostname or socket.gethostname() self.eventer = eventer self.connection_errors = connection_errors or () self.task = task or self.app.tasks[name] self.acknowledged = self._already_revoked = False self.time_start = self.worker_pid = self._terminate_on_ack = None self._tzlocal = None # timezone means the message is timezone-aware, and the only timezone # supported at this point is UTC. if eta is not None: tz = tz_utc if utc else self.tzlocal self.eta = tz_to_local(maybe_iso8601(eta), self.tzlocal, tz) else: self.eta = None if expires is not None: tz = tz_utc if utc else self.tzlocal self.expires = tz_to_local(maybe_iso8601(expires), self.tzlocal, tz) else: self.expires = None delivery_info = {} if delivery_info is None else delivery_info self.delivery_info = { 'exchange': delivery_info.get('exchange'), 'routing_key': delivery_info.get('routing_key'), 'priority': delivery_info.get('priority'), } # amqplib transport adds the channel here for some reason, so need # to remove it. self.delivery_info.pop('channel', None) self.request_dict = body
def run(self, name, *_, **kw): # Positional args. args = kw.get('args') or () if isinstance(args, string_t): args = json.loads(args) # Keyword args. kwargs = kw.get('kwargs') or {} if isinstance(kwargs, string_t): kwargs = json.loads(kwargs) # Expires can be int/float. expires = kw.get('expires') or None try: expires = float(expires) except (TypeError, ValueError): # or a string describing an ISO 8601 datetime. try: expires = maybe_iso8601(expires) except (TypeError, ValueError): raise res = self.app.send_task(name, args=args, kwargs=kwargs, countdown=kw.get('countdown'), serializer=kw.get('serializer'), queue=kw.get('queue'), exchange=kw.get('exchange'), routing_key=kw.get('routing_key'), eta=maybe_iso8601(kw.get('eta')), expires=expires) self.out(res.id)
def from_message(cls, message, message_data, logger=None, eventer=None, hostname=None, app=None): """Create a :class:`TaskRequest` from a task message sent by :class:`celery.app.amqp.TaskPublisher`. :raises UnknownTaskError: if the message does not describe a task, the message is also rejected. :returns :class:`TaskRequest`: """ task_name = message_data["task"] task_id = message_data["id"] args = message_data["args"] kwargs = message_data["kwargs"] retries = message_data.get("retries", 0) eta = maybe_iso8601(message_data.get("eta")) expires = maybe_iso8601(message_data.get("expires")) _delivery_info = getattr(message, "delivery_info", {}) delivery_info = dict((key, _delivery_info.get(key)) for key in WANTED_DELIVERY_INFO) if not hasattr(kwargs, "items"): raise InvalidTaskError("Task kwargs must be a dictionary.") return cls(task_name, task_id, args, kwdict(kwargs), retries=retries, on_ack=message.ack, delivery_info=delivery_info, logger=logger, eventer=eventer, hostname=hostname, eta=eta, expires=expires, app=app)
def run(self, name, *_, **kw): # Positional args. args = kw.get("args") or () if isinstance(args, basestring): args = deserialize(args) # Keyword args. kwargs = kw.get("kwargs") or {} if isinstance(kwargs, basestring): kwargs = deserialize(kwargs) # Expires can be int/float. expires = kw.get("expires") or None try: expires = float(expires) except (TypeError, ValueError): # or a string describing an ISO 8601 datetime. try: expires = maybe_iso8601(expires) except (TypeError, ValueError): pass res = self.app.send_task(name, args=args, kwargs=kwargs, countdown=kw.get("countdown"), serializer=kw.get("serializer"), queue=kw.get("queue"), exchange=kw.get("exchange"), routing_key=kw.get("routing_key"), eta=maybe_iso8601(kw.get("eta")), expires=expires) self.out(res.task_id)
def __init__(self, body, on_ack=noop, hostname=None, eventer=None, app=None, connection_errors=None, request_dict=None, delivery_info=None, task=None, **opts): self.app = app or app_or_default(app) name = self.name = body["task"] self.id = body["id"] self.args = body.get("args", []) self.kwargs = body.get("kwargs", {}) try: self.kwargs.items except AttributeError: raise exceptions.InvalidTaskError( "Task keyword arguments is not a mapping") if NEEDS_KWDICT: self.kwargs = kwdict(self.kwargs) eta = body.get("eta") expires = body.get("expires") utc = body.get("utc", False) self.on_ack = on_ack self.hostname = hostname or socket.gethostname() self.eventer = eventer self.connection_errors = connection_errors or () self.task = task or self.app.tasks[name] self.acknowledged = self._already_revoked = False self.time_start = self.worker_pid = self._terminate_on_ack = None self._tzlocal = None # timezone means the message is timezone-aware, and the only timezone # supported at this point is UTC. if eta is not None: tz = tz_utc if utc else self.tzlocal self.eta = tz_to_local(maybe_iso8601(eta), self.tzlocal, tz) else: self.eta = None if expires is not None: tz = tz_utc if utc else self.tzlocal self.expires = tz_to_local(maybe_iso8601(expires), self.tzlocal, tz) else: self.expires = None delivery_info = {} if delivery_info is None else delivery_info self.delivery_info = { "exchange": delivery_info.get("exchange"), "routing_key": delivery_info.get("routing_key"), } ## shortcuts self._does_debug = logger.isEnabledFor(logging.DEBUG) self._does_info = logger.isEnabledFor(logging.INFO) self.request_dict = body
def _send_task(self, name, args=None, kwargs=None, countdown=None, serializer=None, queue=None, exchange=None, routing_key=None, eta=None, expires=None): # arguments args = loads(args) if isinstance(args, string_t) else args kwargs = loads(kwargs) if isinstance(kwargs, string_t) else kwargs # Expires can be int/float. try: expires = float(expires) except (TypeError, ValueError): # or a string describing an ISO 8601 datetime. try: expires = maybe_iso8601(expires) except (TypeError, ValueError): raise # send the task and print the id. self.out(self.app.send_task( name, args=args or (), kwargs=kwargs or {}, countdown=countdown, serializer=serializer, queue=queue, exchange=exchange, routing_key=routing_key, eta=maybe_iso8601(eta), expires=expires, ).id)
def __init__(self, body, on_ack=noop, hostname=None, eventer=None, app=None, connection_errors=None, request_dict=None, delivery_info=None, task=None, **opts): self.app = app or app_or_default(app) name = self.name = body['task'] self.id = body['id'] self.args = body.get('args', []) self.kwargs = body.get('kwargs', {}) try: self.kwargs.items except AttributeError: raise exceptions.InvalidTaskError( 'Task keyword arguments is not a mapping') if NEEDS_KWDICT: self.kwargs = kwdict(self.kwargs) eta = body.get('eta') expires = body.get('expires') utc = self.utc = body.get('utc', False) self.on_ack = on_ack self.hostname = hostname or socket.gethostname() self.eventer = eventer self.connection_errors = connection_errors or () self.task = task or self.app.tasks[name] self.acknowledged = self._already_revoked = False self.time_start = self.worker_pid = self._terminate_on_ack = None self._tzlocal = None # timezone means the message is timezone-aware, and the only timezone # supported at this point is UTC. if eta is not None: self.eta = maybe_iso8601(eta) if utc: self.eta = maybe_make_aware(self.eta, self.tzlocal) else: self.eta = None if expires is not None: self.expires = maybe_iso8601(expires) if utc: self.expires = maybe_make_aware(self.expires, self.tzlocal) else: self.expires = None delivery_info = {} if delivery_info is None else delivery_info self.delivery_info = { 'exchange': delivery_info.get('exchange'), 'routing_key': delivery_info.get('routing_key'), 'priority': delivery_info.get('priority'), } # amqplib transport adds the channel here for some reason, so need # to remove it. self.delivery_info.pop('channel', None) self.request_dict = body
def __init__(self, body, on_ack=noop, hostname=None, eventer=None, app=None, connection_errors=None, request_dict=None, delivery_info=None, task=None, **opts): self.app = app or app_or_default(app) name = self.name = body['task'] self.id = body['id'] self.args = body.get('args', []) self.kwargs = body.get('kwargs', {}) try: self.kwargs.items except AttributeError: raise exceptions.InvalidTaskError( 'Task keyword arguments is not a mapping') if NEEDS_KWDICT: self.kwargs = kwdict(self.kwargs) eta = body.get('eta') expires = body.get('expires') utc = body.get('utc', False) self.on_ack = on_ack self.hostname = hostname or socket.gethostname() self.eventer = eventer self.connection_errors = connection_errors or () self.task = task or self.app.tasks[name] self.acknowledged = self._already_revoked = False self.time_start = self.worker_pid = self._terminate_on_ack = None self._tzlocal = None # timezone means the message is timezone-aware, and the only timezone # supported at this point is UTC. if eta is not None: tz = tz_utc if utc else self.tzlocal self.eta = tz_to_local(maybe_iso8601(eta), self.tzlocal, tz) else: self.eta = None if expires is not None: tz = tz_utc if utc else self.tzlocal self.expires = tz_to_local(maybe_iso8601(expires), self.tzlocal, tz) else: self.expires = None self.delivery_info = delivery_info or {} # amqplib transport adds the channel here for some reason, so need # to remove it. sqs also adds additional unpickleable attributes # than need to be removed. for key in ['channel', 'sqs_message', 'sqs_queue']: self.delivery_info.pop(key, None) self.request_dict = body
def sync_tasks(self, tasks): for task_id, task in tasks.items(): row = db.celery_taskstate(task_id=task_id) d = { "name": task.name, "args": task.args, "kwargs": task.kwargs, "eta": maybe_iso8601(task.eta), "expires": maybe_iso8601(task.expires), "state": task.state, "tstamp": datetime.fromtimestamp(task.timestamp), "result": task.result or task.exception, "traceback": task.traceback, "runtime": task.runtime, "worker": task.worker.hostname } row = db.celery_taskstate(task_id=task_id) if row: row.update_record(**d) else: db.celery_taskstate.insert(task_id=task_id, **d) db.commit()
def _send_task(self, name, args=None, kwargs=None, countdown=None, serializer=None, queue=None, exchange=None, routing_key=None, eta=None, expires=None): # arguments args = loads(args) if isinstance(args, string_t) else args kwargs = loads(kwargs) if isinstance(kwargs, string_t) else kwargs # Expires can be int/float. try: expires = float(expires) except (TypeError, ValueError): # or a string describing an ISO 8601 datetime. try: expires = maybe_iso8601(expires) except (TypeError, ValueError): raise # send the task and print the id. self.out( self.app.send_task( name, args=args or (), kwargs=kwargs or {}, countdown=countdown, serializer=serializer, queue=queue, exchange=exchange, routing_key=routing_key, eta=maybe_iso8601(eta), expires=expires, ).id)
def sync_tasks(self, tasks): for task_id, task in tasks.items(): row = db.celery_taskstate(task_id=task_id) d = { "name": task.name, "args": task.args, "kwargs": task.kwargs, "eta": maybe_iso8601(task.eta), "expires": maybe_iso8601(task.expires), "state": task.state, "tstamp": datetime.fromtimestamp(task.timestamp), "result": task.result or task.exception, "traceback": task.traceback, "runtime": task.runtime, "worker": task.worker.hostname, } row = db.celery_taskstate(task_id=task_id) if row: row.update_record(**d) else: db.celery_taskstate.insert(task_id=task_id, **d) db.commit()
def __init__(self, body, on_ack=noop, hostname=None, eventer=None, app=None, connection_errors=None, request_dict=None, delivery_info=None, task=None, **opts): self.app = app or app_or_default(app) name = self.name = body['task'] self.id = body['id'] self.args = body.get('args', []) self.kwargs = body.get('kwargs', {}) try: self.kwargs.items except AttributeError: raise InvalidTaskError('Task keyword arguments is not a mapping') if NEEDS_KWDICT: self.kwargs = kwdict(self.kwargs) eta = body.get('eta') expires = body.get('expires') utc = self.utc = body.get('utc', False) self.on_ack = on_ack self.hostname = hostname or socket.gethostname() self.eventer = eventer self.connection_errors = connection_errors or () self.task = task or self.app.tasks[name] self.acknowledged = self._already_revoked = False self.time_start = self.worker_pid = self._terminate_on_ack = None self._tzlocal = None # timezone means the message is timezone-aware, and the only timezone # supported at this point is UTC. if eta is not None: try: self.eta = maybe_iso8601(eta) except (AttributeError, ValueError), exc: raise InvalidTaskError('invalid eta value %r: %s' % ( eta, exc, )) if utc: self.eta = maybe_make_aware(self.eta, self.tzlocal)
def generate_indexing_docs(cls, tasks, project=None): docs = [] for task in tasks: worker = task.worker project, worker_name = get_project_and_name(worker, project=project) if project not in app.config['CELERIUM_PROJECTS']: continue doc = dict(task) doc.pop('worker', None) doc['id'] = doc.pop('uuid') doc['project'] = project if doc['name'].find('.') != -1: doc['module'] = '.'.join(doc['name'].split('.')[:-1]) for field in cls.timestamp_fields: doc[field] = try_fromtimestamp(doc[field]) for field in cls.iso8601_fields: doc[field] = maybe_iso8601(doc[field]) if worker: doc['worker_id'] = get_worker_id(project, worker_name) doc['worker_name'] = worker_name docs.append(doc) return docs
def __init__(self, body, on_ack=noop, hostname=None, eventer=None, app=None, connection_errors=None, request_dict=None, delivery_info=None, task=None, **opts): self.app = app or app_or_default(app) name = self.name = body['task'] self.id = body['id'] self.args = body.get('args', []) self.kwargs = body.get('kwargs', {}) try: self.kwargs.items except AttributeError: raise InvalidTaskError( 'Task keyword arguments is not a mapping') if NEEDS_KWDICT: self.kwargs = kwdict(self.kwargs) eta = body.get('eta') expires = body.get('expires') utc = self.utc = body.get('utc', False) self.on_ack = on_ack self.hostname = hostname or socket.gethostname() self.eventer = eventer self.connection_errors = connection_errors or () self.task = task or self.app.tasks[name] self.acknowledged = self._already_revoked = False self.time_start = self.worker_pid = self._terminate_on_ack = None self._tzlocal = None # timezone means the message is timezone-aware, and the only timezone # supported at this point is UTC. if eta is not None: try: self.eta = maybe_iso8601(eta) except (AttributeError, ValueError), exc: raise InvalidTaskError( 'invalid eta value %r: %s' % (eta, exc, )) if utc: self.eta = maybe_make_aware(self.eta, self.tzlocal)
def __init__( self, message, on_ack=noop, hostname=None, eventer=None, app=None, connection_errors=None, request_dict=None, task=None, on_reject=noop, body=None, headers=None, decoded=False, utc=True, maybe_make_aware=maybe_make_aware, maybe_iso8601=maybe_iso8601, **opts ): if headers is None: headers = message.headers if body is None: body = message.body self.app = app self.message = message self.body = body self.utc = utc self._decoded = decoded if decoded: self.content_type = self.content_encoding = None else: self.content_type, self.content_encoding = (message.content_type, message.content_encoding) self.id = headers["id"] type = self.type = self.name = headers["task"] self.root_id = headers.get("root_id") self.parent_id = headers.get("parent_id") if "shadow" in headers: self.name = headers["shadow"] or self.name if "timelimit" in headers: self.time_limits = headers["timelimit"] self.argsrepr = headers.get("argsrepr", "") self.kwargsrepr = headers.get("kwargsrepr", "") self.on_ack = on_ack self.on_reject = on_reject self.hostname = hostname or socket.gethostname() self.eventer = eventer self.connection_errors = connection_errors or () self.task = task or self.app.tasks[type] # timezone means the message is timezone-aware, and the only timezone # supported at this point is UTC. eta = headers.get("eta") if eta is not None: try: eta = maybe_iso8601(eta) except (AttributeError, ValueError, TypeError) as exc: raise InvalidTaskError("invalid eta value {0!r}: {1}".format(eta, exc)) self.eta = maybe_make_aware(eta, self.tzlocal) else: self.eta = None expires = headers.get("expires") if expires is not None: try: expires = maybe_iso8601(expires) except (AttributeError, ValueError, TypeError) as exc: raise InvalidTaskError("invalid expires value {0!r}: {1}".format(expires, exc)) self.expires = maybe_make_aware(expires, self.tzlocal) else: self.expires = None delivery_info = message.delivery_info or {} properties = message.properties or {} headers.update( { "reply_to": properties.get("reply_to"), "correlation_id": properties.get("correlation_id"), "delivery_info": { "exchange": delivery_info.get("exchange"), "routing_key": delivery_info.get("routing_key"), "priority": properties.get("priority"), "redelivered": delivery_info.get("redelivered"), }, } ) self.request_dict = headers
def __init__(self, message, on_ack=noop, hostname=None, eventer=None, app=None, connection_errors=None, request_dict=None, task=None, on_reject=noop, body=None, headers=None, decoded=False, utc=True, maybe_make_aware=maybe_make_aware, maybe_iso8601=maybe_iso8601, **opts): if headers is None: headers = message.headers if body is None: body = message.body self.app = app self.message = message self.body = body self.utc = utc self._decoded = decoded if decoded: self.content_type = self.content_encoding = None else: self.content_type, self.content_encoding = ( message.content_type, message.content_encoding, ) self.id = headers['id'] type = self.type = self.name = headers['task'] self.root_id = headers.get('root_id') self.parent_id = headers.get('parent_id') if 'shadow' in headers: self.name = headers['shadow'] or self.name if 'timelimit' in headers: self.time_limits = headers['timelimit'] self.argsrepr = headers.get('argsrepr', '') self.kwargsrepr = headers.get('kwargsrepr', '') self.on_ack = on_ack self.on_reject = on_reject self.hostname = hostname or gethostname() self.eventer = eventer self.connection_errors = connection_errors or () self.task = task or self.app.tasks[type] # timezone means the message is timezone-aware, and the only timezone # supported at this point is UTC. eta = headers.get('eta') if eta is not None: try: eta = maybe_iso8601(eta) except (AttributeError, ValueError, TypeError) as exc: raise InvalidTaskError( 'invalid eta value {0!r}: {1}'.format(eta, exc)) self.eta = maybe_make_aware(eta, self.tzlocal) else: self.eta = None expires = headers.get('expires') if expires is not None: try: expires = maybe_iso8601(expires) except (AttributeError, ValueError, TypeError) as exc: raise InvalidTaskError( 'invalid expires value {0!r}: {1}'.format(expires, exc)) self.expires = maybe_make_aware(expires, self.tzlocal) else: self.expires = None delivery_info = message.delivery_info or {} properties = message.properties or {} headers.update({ 'reply_to': properties.get('reply_to'), 'correlation_id': properties.get('correlation_id'), 'delivery_info': { 'exchange': delivery_info.get('exchange'), 'routing_key': delivery_info.get('routing_key'), 'priority': properties.get('priority'), 'redelivered': delivery_info.get('redelivered'), } }) self.request_dict = headers
def __init__(self, body, on_ack=noop, hostname=None, eventer=None, app=None, connection_errors=None, request_dict=None, message=None, task=None, on_reject=noop, **opts): self.app = app name = self.name = body['task'] self.id = body['id'] self.args = body.get('args', []) self.kwargs = body.get('kwargs', {}) try: self.kwargs.items except AttributeError: raise InvalidTaskError('Task keyword arguments is not a mapping') if NEEDS_KWDICT: self.kwargs = kwdict(self.kwargs) eta = body.get('eta') expires = body.get('expires') utc = self.utc = body.get('utc', False) self.on_ack = on_ack self.on_reject = on_reject self.hostname = hostname or socket.gethostname() self.eventer = eventer self.connection_errors = connection_errors or () self.task = task or self.app.tasks[name] self.acknowledged = self._already_revoked = False self.time_start = self.worker_pid = self._terminate_on_ack = None self._apply_result = None self._tzlocal = None # timezone means the message is timezone-aware, and the only timezone # supported at this point is UTC. if eta is not None: try: self.eta = maybe_iso8601(eta) except (AttributeError, ValueError, TypeError) as exc: raise InvalidTaskError('invalid eta value {0!r}: {1}'.format( eta, exc)) if utc: self.eta = maybe_make_aware(self.eta, self.tzlocal) else: self.eta = None if expires is not None: try: self.expires = maybe_iso8601(expires) except (AttributeError, ValueError, TypeError) as exc: raise InvalidTaskError( 'invalid expires value {0!r}: {1}'.format(expires, exc)) if utc: self.expires = maybe_make_aware(self.expires, self.tzlocal) else: self.expires = None if message: delivery_info = message.delivery_info or {} properties = message.properties or {} body.update({ 'headers': message.headers, 'reply_to': properties.get('reply_to'), 'correlation_id': properties.get('correlation_id'), 'delivery_info': { 'exchange': delivery_info.get('exchange'), 'routing_key': delivery_info.get('routing_key'), 'priority': properties.get('priority', delivery_info.get('priority')), 'redelivered': delivery_info.get('redelivered'), } }) else: body.update(DEFAULT_FIELDS) self.request_dict = body
hostname=hostname, defaults={"last_heartbeat": self.get_heartbeat(worker)}) self._last_worker_write[hostname] = (time(), obj) return obj def handle_task(self, (uuid, task), worker=None): """Handle snapshotted event.""" if task.worker and task.worker.hostname: worker = self.handle_worker((task.worker.hostname, task.worker)) NOT_SAVED_ATTRIBUTES = ('name', 'args', 'kwargs', 'eta') defaults = {"name": task.name, "args": task.args, "kwargs": task.kwargs, "eta": maybe_make_aware(maybe_iso8601(task.eta)), "expires": maybe_make_aware(maybe_iso8601(task.expires)), "state": task.state, "tstamp": make_aware(datetime.fromtimestamp( task.timestamp)), "result": task.result or task.exception, "traceback": task.traceback, "runtime": task.runtime, "worker": worker} # If RECEIVED event is lost, some data is lost as it is stored # only in RECEIVED event for efficiency. In this case we just # do not overwrite this fields in TaskState instance row [defaults.pop(attr) for attr in NOT_SAVED_ATTRIBUTES if defaults[attr] is None] return self.update_task(task.state, task_id=uuid, defaults=defaults)
return obj def handle_task(self, (uuid, task), worker=None): if task.worker and task.worker.hostname: worker = self.handle_worker((task.worker.hostname, task.worker)) return self.update_task(task.state, task_id=uuid, defaults={ "name": task.name, "args": task.args, "kwargs": task.kwargs, "eta": maybe_iso8601(task.eta), "expires": maybe_iso8601(task.expires), "state": task.state, "tstamp": datetime.fromtimestamp(task.timestamp), "result": task.result or task.exception, "traceback": task.traceback, "runtime": task.runtime, "worker": worker })
def test_maybe_iso8601_datetime(self): from celery.utils.timeutils import maybe_iso8601 from datetime import datetime now = datetime.now() self.assertIs(maybe_iso8601(now), now)
hostname=hostname, defaults={"last_heartbeat": self.get_heartbeat(worker)}) self._last_worker_write[hostname] = (time(), obj) return obj def handle_task(self, (uuid, task), worker=None): if task.worker and task.worker.hostname: worker = self.handle_worker((task.worker.hostname, task.worker)) return self.update_task( task.state, task_id=uuid, defaults={ "name": task.name, "args": task.args, "kwargs": task.kwargs, "eta": maybe_iso8601(task.eta), "expires": maybe_iso8601(task.expires), "state": task.state, "tstamp": make_aware(datetime.fromtimestamp(task.timestamp)), "result": task.result or task.exception, "traceback": task.traceback, "runtime": task.runtime, "worker": worker }) def update_task(self, state, **kwargs): objects = self.TaskState.objects defaults = kwargs.pop("defaults", None) or {} try: obj = objects.get(**kwargs) except ObjectDoesNotExist:
) self._last_worker_write[hostname] = (time(), obj) return obj def handle_task(self, (uuid, task), worker=None): """Handle snapshotted event.""" if task.worker and task.worker.hostname: worker = self.handle_worker( (task.worker.hostname, task.worker), ) defaults = { 'name': task.name, 'args': task.args, 'kwargs': task.kwargs, 'eta': maybe_make_aware(maybe_iso8601(task.eta)), 'expires': maybe_make_aware(maybe_iso8601(task.expires)), 'state': task.state, 'tstamp': aware_tstamp(task.timestamp), 'result': task.result or task.exception, 'traceback': task.traceback, 'runtime': task.runtime, 'worker': worker } # Some fields are only stored in the RECEIVED event, # so we should remove these from default values, # so that they are not overwritten by subsequent states. [defaults.pop(attr, None) for attr in NOT_SAVED_ATTRIBUTES if defaults[attr] is None] return self.update_task(task.state, task_id=uuid, defaults=defaults)
def test_maybe_iso8601_datetime(self): now = datetime.now() self.assertIs(timeutils.maybe_iso8601(now), now)
def __init__( self, body, on_ack=noop, hostname=None, eventer=None, app=None, connection_errors=None, request_dict=None, message=None, task=None, on_reject=noop, **opts ): self.app = app name = self.name = body["task"] self.id = body["id"] self.args = body.get("args", []) self.kwargs = body.get("kwargs", {}) try: self.kwargs.items except AttributeError: raise InvalidTaskError("Task keyword arguments is not a mapping") if NEEDS_KWDICT: self.kwargs = kwdict(self.kwargs) eta = body.get("eta") expires = body.get("expires") utc = self.utc = body.get("utc", False) self.on_ack = on_ack self.on_reject = on_reject self.hostname = hostname or socket.gethostname() self.eventer = eventer self.connection_errors = connection_errors or () self.task = task or self.app.tasks[name] self.acknowledged = self._already_revoked = False self.time_start = self.worker_pid = self._terminate_on_ack = None self._apply_result = None self._tzlocal = None # timezone means the message is timezone-aware, and the only timezone # supported at this point is UTC. if eta is not None: try: self.eta = maybe_iso8601(eta) except (AttributeError, ValueError, TypeError) as exc: raise InvalidTaskError("invalid eta value {0!r}: {1}".format(eta, exc)) if utc: self.eta = maybe_make_aware(self.eta, self.tzlocal) else: self.eta = None if expires is not None: try: self.expires = maybe_iso8601(expires) except (AttributeError, ValueError, TypeError) as exc: raise InvalidTaskError("invalid expires value {0!r}: {1}".format(expires, exc)) if utc: self.expires = maybe_make_aware(self.expires, self.tzlocal) else: self.expires = None if message: delivery_info = message.delivery_info or {} properties = message.properties or {} body.update( { "headers": message.headers, "reply_to": properties.get("reply_to"), "correlation_id": properties.get("correlation_id"), "delivery_info": { "exchange": delivery_info.get("exchange"), "routing_key": delivery_info.get("routing_key"), "priority": delivery_info.get("priority"), "redelivered": delivery_info.get("redelivered"), }, } ) else: body.update(DEFAULT_FIELDS) self.request_dict = body
obj = self.WorkerState.objects.update_or_create( hostname=hostname, defaults={"last_heartbeat": self.get_heartbeat(worker)}) self._last_worker_write[hostname] = (time(), obj) return obj def handle_task(self, (uuid, task), worker=None): """Handle snapshotted event.""" if task.worker and task.worker.hostname: worker = self.handle_worker((task.worker.hostname, task.worker)) defaults = { "name": task.name, "args": task.args, "kwargs": task.kwargs, "eta": maybe_make_aware(maybe_iso8601(task.eta)), "expires": maybe_make_aware(maybe_iso8601(task.expires)), "state": task.state, "tstamp": aware_tstamp(task.timestamp), "result": task.result or task.exception, "traceback": task.traceback, "runtime": task.runtime, "worker": worker } # If RECEIVED event is lost, some data is lost as it is stored # only in RECEIVED event for efficiency. In this case we just # do not overwrite this fields in TaskState instance row [ defaults.pop(attr, None) for attr in NOT_SAVED_ATTRIBUTES if defaults[attr] is None ]
# timezone means the message is timezone-aware, and the only timezone # supported at this point is UTC. if eta is not None: try: self.eta = maybe_iso8601(eta) except (AttributeError, ValueError), exc: raise InvalidTaskError( 'invalid eta value %r: %s' % (eta, exc, )) if utc: self.eta = maybe_make_aware(self.eta, self.tzlocal) else: self.eta = None if expires is not None: try: self.expires = maybe_iso8601(expires) except (AttributeError, ValueError), exc: raise InvalidTaskError( 'invalid expires value %r: %s' % (expires, exc, )) if utc: self.expires = maybe_make_aware(self.expires, self.tzlocal) else: self.expires = None delivery_info = {} if delivery_info is None else delivery_info self.delivery_info = { 'exchange': delivery_info.get('exchange'), 'routing_key': delivery_info.get('routing_key'), 'priority': delivery_info.get('priority'), }
hostname=hostname, defaults={'last_heartbeat': self.get_heartbeat(worker)}, ) self._last_worker_write[hostname] = (monotonic(), obj) return obj def handle_task(self, (uuid, task), worker=None): """Handle snapshotted event.""" if task.worker and task.worker.hostname: worker = self.handle_worker((task.worker.hostname, task.worker), ) defaults = { 'name': task.name, 'args': task.args, 'kwargs': task.kwargs, 'eta': maybe_make_aware(maybe_iso8601(task.eta)), 'expires': maybe_make_aware(maybe_iso8601(task.expires)), 'state': task.state, 'tstamp': aware_tstamp(task.timestamp), 'result': task.result or task.exception, 'traceback': task.traceback, 'runtime': task.runtime, 'worker': worker } # Some fields are only stored in the RECEIVED event, # so we should remove these from default values, # so that they are not overwritten by subsequent states. [ defaults.pop(attr, None) for attr in NOT_SAVED_ATTRIBUTES if defaults[attr] is None ]
if not last_write or time() - last_write > self.worker_update_freq: obj = self.WorkerState.objects.update_or_create( hostname=hostname, defaults={"last_heartbeat": self.get_heartbeat(worker)}) self._last_worker_write[hostname] = (time(), obj) return obj def handle_task(self, (uuid, task), worker=None): if task.worker and task.worker.hostname: worker = self.handle_worker((task.worker.hostname, task.worker)) return self.update_task(task.state, task_id=uuid, defaults={"name": task.name, "args": task.args, "kwargs": task.kwargs, "eta": maybe_iso8601(task.eta), "expires": maybe_iso8601(task.expires), "state": task.state, "tstamp": datetime.fromtimestamp(task.timestamp), "result": task.result or task.exception, "traceback": task.traceback, "runtime": task.runtime, "worker": worker}) def update_task(self, state, **kwargs): objects = self.TaskState.objects defaults = kwargs.pop("defaults", None) or {} try: obj = objects.get(**kwargs) except ObjectDoesNotExist: if not defaults.get("name"):
def test_maybe_iso8601_datetime(self): now = datetime.now() self.assertIs(maybe_iso8601(now), now)
# supported at this point is UTC. if eta is not None: try: self.eta = maybe_iso8601(eta) except (AttributeError, ValueError), exc: raise InvalidTaskError('invalid eta value %r: %s' % ( eta, exc, )) if utc: self.eta = maybe_make_aware(self.eta, self.tzlocal) else: self.eta = None if expires is not None: try: self.expires = maybe_iso8601(expires) except (AttributeError, ValueError), exc: raise InvalidTaskError('invalid expires value %r: %s' % ( expires, exc, )) if utc: self.expires = maybe_make_aware(self.expires, self.tzlocal) else: self.expires = None delivery_info = {} if delivery_info is None else delivery_info self.delivery_info = { 'exchange': delivery_info.get('exchange'), 'routing_key': delivery_info.get('routing_key'), 'priority': delivery_info.get('priority'),