def is_due(self):
if not self.model.enabled:
# 5 second delay for re-enable.
return schedules.schedstate(False, 5.0)
# START DATE: only run after the `start_time`, if one exists.
if self.model.start_time is not None:
now = self._default_now()
if getattr(settings, 'DJANGO_CELERY_BEAT_TZ_AWARE', True):
now = maybe_make_aware(self._default_now())
if now < self.model.start_time:
# The datetime is before the start date - don't run.
# send a delay to retry on start_time
delay = math.ceil(
(self.model.start_time - now).total_seconds())
return schedules.schedstate(False, delay)
# ONE OFF TASK: Disable one off tasks after they've ran once
if self.model.one_off and self.model.enabled \
and self.model.total_run_count > 0:
self.model.enabled = False
self.model.total_run_count = 0 # Reset
self.model.no_changes = False # Mark the model entry as changed
self.model.save()
# Don't recheck
return schedules.schedstate(False, NEVER_CHECK_TIMEOUT)
# CAUTION: make_aware assumes settings.TIME_ZONE for naive datetimes,
# while maybe_make_aware assumes utc for naive datetimes
tz = self.app.timezone
last_run_at_in_tz = maybe_make_aware(self.last_run_at).astimezone(tz)
return self.schedule.is_due(last_run_at_in_tz)
def __init__(self, model, Session, app=None, **kw):
"""Initialize the model entry."""
self.app = app or current_app._get_current_object()
self.session = kw.get('session')
self.Session = Session
self.model = model
self.name = model.name
self.task = model.task
try:
self.schedule = model.schedule
logger.debug('schedule: {}'.format(self.schedule))
except Exception as e:
logger.error(e)
logger.error(
'Disabling schedule %s that was removed from database',
self.name,
)
self._disable(model)
try:
self.args = loads(model.args or '[]')
self.kwargs = loads(model.kwargs or '{}')
except ValueError as exc:
logger.exception(
'Removing schedule %s for argument deseralization error: %r',
self.name,
exc,
)
self._disable(model)
self.options = {}
for option in [
'queue', 'exchange', 'routing_key', 'expires', 'priority'
]:
value = getattr(model, option)
if value is None:
continue
self.options[option] = value
self.total_run_count = model.total_run_count
self.enabled = model.enabled
if not model.last_run_at:
model.last_run_at = self._default_now()
self.last_run_at = model.last_run_at
timezone = self.app.timezone
# 因为从数据库读取的 last_run_at 可能没有时区信息，所以这里必须加上时区信息
self.last_run_at = maybe_make_aware(self.last_run_at, timezone)
# self.options['expires'] 同理
if 'expires' in self.options:
expires = self.options['expires']
self.options['expires'] = maybe_make_aware(expires, timezone)
def test_maybe_make_aware(self):
aware = datetime.utcnow().replace(tzinfo=timezone.utc)
assert maybe_make_aware(aware)
naive = datetime.utcnow()
assert maybe_make_aware(naive)
assert maybe_make_aware(naive).tzinfo is pytz.utc
tz = pytz.timezone('US/Eastern')
eastern = datetime.utcnow().replace(tzinfo=tz)
assert maybe_make_aware(eastern).tzinfo is tz
utcnow = datetime.utcnow()
assert maybe_make_aware(utcnow, 'UTC').tzinfo is pytz.utc
def test_maybe_make_aware(self):
aware = datetime.utcnow().replace(tzinfo=timezone.utc)
assert maybe_make_aware(aware)
naive = datetime.utcnow()
assert maybe_make_aware(naive)
assert maybe_make_aware(naive).tzinfo is pytz.utc
tz = pytz.timezone('US/Eastern')
eastern = datetime.utcnow().replace(tzinfo=tz)
assert maybe_make_aware(eastern).tzinfo is tz
utcnow = datetime.utcnow()
assert maybe_make_aware(utcnow, 'UTC').tzinfo is pytz.utc
def is_due(self):
if not self.model.enabled:
# 5 second delay for re-enable.
return schedules.schedstate(False, 5.0)
# START DATE: only run after the `start_time`, if one exists.
if self.model.start_time is not None:
now = maybe_make_aware(self._default_now())
start_time = self.model.start_time.replace(
tzinfo=self.app.timezone)
if now < start_time:
# The datetime is before the start date - don't run.
_, delay = self.schedule.is_due(self.last_run_at)
# use original delay for re-check
return schedules.schedstate(False, delay)
# ONE OFF TASK: Disable one off tasks after they've ran once
if self.model.one_off and self.model.enabled \
and self.model.total_run_count > 0:
self.model.enabled = False # disable
self.model.total_run_count = 0 # Reset
self.model.no_changes = False # Mark the model entry as changed
save_fields = ('enabled', ) # the additional fields to save
self.save(save_fields)
return schedules.schedstate(False, None) # Don't recheck
return self.schedule.is_due(self.last_run_at)
def __init__(self, clocked_time, enabled=True,
model=None, nowfun=None, app=None):
"""Initialize clocked."""
self.clocked_time = maybe_make_aware(clocked_time)
self.enabled = enabled
self.model = model
super(clocked, self).__init__(nowfun=nowfun, app=app)
def is_due(self):
if not self.model.enabled:
# 5 second delay for re-enable.
return schedules.schedstate(False, 5.0)
# START DATE: only run after the `start_time`, if one exists.
if self.model.start_time is not None:
now = maybe_make_aware(self._default_now())
if now < self.model.start_time:
# The datetime is before the start date - don't run.
# send a delay to retry on start_time
delay = math.ceil(
(self.model.start_time - now).total_seconds())
return schedules.schedstate(False, delay)
# ONE OFF TASK: Disable one off tasks after they've ran once
if self.model.one_off and self.model.enabled \
and self.model.total_run_count > 0:
self.model.enabled = False
self.model.total_run_count = 0 # Reset
self.model.no_changes = False # Mark the model entry as changed
self.model.save()
return schedules.schedstate(False, None) # Don't recheck
return self.schedule.is_due(self.last_run_at)
def is_due(self):
if not self.model.enabled:
# 5 second delay for re-enable.
return schedules.schedstate(False, 5.0)
# START DATE: only run after the `start_time`, if one exists.
if self.model.start_time is not None:
now = self._default_now()
if getattr(settings, 'DJANGO_CELERY_BEAT_TZ_AWARE', True):
now = maybe_make_aware(self._default_now())
if now < self.model.start_time:
# The datetime is before the start date - don't run.
# send a delay to retry on start_time
delay = math.ceil(
(self.model.start_time - now).total_seconds())
return schedules.schedstate(False, delay)
# ONE OFF TASK: Disable one off tasks after they've ran once
if self.model.one_off and self.model.enabled \
and self.model.total_run_count > 0:
self.model.enabled = False
self.model.total_run_count = 0 # Reset
self.model.no_changes = False # Mark the model entry as changed
self.model.save()
return schedules.schedstate(False, None) # Don't recheck
# When Django settings USE_TZ is False and Django settings TIME_ZONE is set
# value of TIME_ZINE is the time zone in which Django will store all datetimes.
# Because of that if datetime is naive we should use it as source timezone
# celery.utils.time.maybe_make_aware - always convert naive datetime to UTC
# which may be wrong in Django sicase.
django_timezone = getattr(settings, 'TIME_ZONE', None)
django_use_tz = getattr(settings, 'USE_TZ', None)
if (not django_use_tz and django_timezone
and self.last_run_at.tzinfo is None and self.app.timezone):
source_timezone = pytz.timezone(django_timezone)
last_run_at_django_tz = source_timezone.localize(self.last_run_at)
self.last_run_at = last_run_at_django_tz.astimezone(
self.app.timezone)
# CAUTION: make_aware assumes settings.TIME_ZONE for naive datetimes,
# while maybe_make_aware assumes utc for naive datetimes
tz = self.app.timezone
last_run_at_in_tz = maybe_make_aware(self.last_run_at).astimezone(tz)
return self.schedule.is_due(last_run_at_in_tz)
def get_countdown_and_eta(self, options):
now = self.app.now()
countdown, eta = options.get("countdown"), options.get("eta")
if countdown:
return countdown, now + timedelta(seconds=countdown)
elif eta:
aware_eta = maybe_make_aware(eta, self.app.timezone)
return (aware_eta - now).total_seconds(), aware_eta
else:
return 0, now
def test_roundtrip(self):
now = self.app.now()
# 3.x returns naive, but 4.x returns aware
now = maybe_make_aware(now)
roundtripped = from_timestamp(to_timestamp(now))
# we lose microseconds in the roundtrip, so we need to ignore them
now = now.replace(microsecond=0)
self.assertEqual(now, roundtripped)
def is_due(self, last_run_at):
# actually last run at is useless
last_run_at = maybe_make_aware(last_run_at)
rem_delta = self.remaining_estimate(last_run_at)
remaining_s = max(rem_delta.total_seconds(), 0)
if not self.enabled:
return schedstate(is_due=False, next=None)
if remaining_s == 0:
if self.model:
self.model.enabled = False
self.model.save()
return schedstate(is_due=True, next=None)
return schedstate(is_due=False, next=remaining_s)
def test_score(self):
run_every = 61 * 60
entry = self.create_entry(run_every=run_every)
entry = entry._next_instance()
score = entry.score
expected = entry.last_run_at + timedelta(seconds=run_every)
expected = expected.replace(
microsecond=0) # discard microseconds, lost in timestamp
# 3.x returns naive, but 4.x returns aware
expected = maybe_make_aware(expected)
self.assertEqual(score, to_timestamp(expected))
self.assertEqual(expected, from_timestamp(score))
def is_due(self):
if not self.model.enabled:
return False, 5.0 # 5 second delay for re-enable.
# START DATE: only run after the `start_time`, if one exists.
if self.model.start_time is not None:
if maybe_make_aware(self._default_now()) < self.model.start_time:
# The datetime is before the start date - don't run.
_, delay = self.schedule.is_due(self.last_run_at)
return False, delay # use original delay for re-check
# ONE OFF TASK: Disable one off tasks after they've ran once
if self.model.one_off and self.model.enabled \
and self.model.total_run_count > 0:
self.model.enabled = False
self.model.total_run_count = 0 # Reset
self.model.save()
return False, None # Don't recheck
return self.schedule.is_due(self.last_run_at)
def test_next(self):
initial = self.create_entry().save()
now = self.app.now()
# 3.x is naive but 4.x is aware
now = maybe_make_aware(now)
n = initial.next(last_run_at=now)
self.assertIsNotNone(now.tzinfo)
self.assertEqual(n.last_run_at, now)
self.assertEqual(initial.total_run_count + 1, n.total_run_count)
# updated meta was stored into redis
loaded = RedBeatSchedulerEntry.from_key(initial.key, app=self.app)
self.assertEqual(loaded.last_run_at, now)
self.assertEqual(loaded.total_run_count, initial.total_run_count + 1)
# new entry updated the schedule
redis = self.app.redbeat_redis
self.assertEqual(
redis.zscore(self.app.redbeat_conf.schedule_key, n.key), n.score)
def is_due(self):
if not self.model.enabled:
# 5 second delay for re-enable.
return schedules.schedstate(False, 5.0)
# START DATE: only run after the `start_time`, if one exists.
if self.model.start_time is not None:
if maybe_make_aware(self._default_now()) < self.model.start_time:
# The datetime is before the start date - don't run.
_, delay = self.schedule.is_due(self.last_run_at)
# use original delay for re-check
return schedules.schedstate(False, delay)
# ONE OFF TASK: Disable one off tasks after they've ran once
if self.model.one_off and self.model.enabled \
and self.model.total_run_count > 0:
self.model.enabled = False
self.model.total_run_count = 0 # Reset
self.model.no_changes = False # Mark the model entry as changed
self.model.save()
return schedules.schedstate(False, None) # Don't recheck
return self.schedule.is_due(self.last_run_at)
def as_task_v2(self, task_id, name, args=None, kwargs=None,
countdown=None, eta=None, group_id=None,
expires=None, retries=0, chord=None,
callbacks=None, errbacks=None, reply_to=None,
time_limit=None, soft_time_limit=None,
create_sent_event=False, root_id=None, parent_id=None,
shadow=None, chain=None, now=None, timezone=None,
origin=None, argsrepr=None, kwargsrepr=None):
args = args or ()
kwargs = kwargs or {}
if not isinstance(args, (list, tuple)):
raise TypeError('task args must be a list or tuple')
if not isinstance(kwargs, Mapping):
raise TypeError('task keyword arguments must be a mapping')
if countdown: # convert countdown to ETA
self._verify_seconds(countdown, 'countdown')
now = now or self.app.now()
timezone = timezone or self.app.timezone
eta = maybe_make_aware(
now + timedelta(seconds=countdown), tz=timezone,
)
if isinstance(expires, numbers.Real):
self._verify_seconds(expires, 'expires')
now = now or self.app.now()
timezone = timezone or self.app.timezone
expires = maybe_make_aware(
now + timedelta(seconds=expires), tz=timezone,
)
eta = eta and eta.isoformat()
expires = expires and expires.isoformat()
if argsrepr is None:
argsrepr = saferepr(args, self.argsrepr_maxsize)
if kwargsrepr is None:
kwargsrepr = saferepr(kwargs, self.kwargsrepr_maxsize)
if JSON_NEEDS_UNICODE_KEYS: # pragma: no cover
if callbacks:
callbacks = [utf8dict(callback) for callback in callbacks]
if errbacks:
errbacks = [utf8dict(errback) for errback in errbacks]
if chord:
chord = utf8dict(chord)
return task_message(
headers={
'lang': 'py',
'task': name,
'id': task_id,
'eta': eta,
'expires': expires,
'group': group_id,
'retries': retries,
'timelimit': [time_limit, soft_time_limit],
'root_id': root_id,
'parent_id': parent_id,
'argsrepr': argsrepr,
'kwargsrepr': kwargsrepr,
'origin': origin or anon_nodename()
},
properties={
'correlation_id': task_id,
'reply_to': reply_to or '',
},
body=(
args, kwargs, {
'callbacks': callbacks,
'errbacks': errbacks,
'chain': chain,
'chord': chord,
},
),
sent_event={
'uuid': task_id,
'root_id': root_id,
'parent_id': parent_id,
'name': name,
'args': argsrepr,
'kwargs': kwargsrepr,
'retries': retries,
'eta': eta,
'expires': expires,
} if create_sent_event else None,
)
def as_task_v2(self,
task_id,
name,
args=None,
kwargs=None,
countdown=None,
eta=None,
group_id=None,
expires=None,
retries=0,
chord=None,
callbacks=None,
errbacks=None,
reply_to=None,
time_limit=None,
soft_time_limit=None,
create_sent_event=False,
root_id=None,
parent_id=None,
shadow=None,
chain=None,
now=None,
timezone=None,
origin=None,
argsrepr=None,
kwargsrepr=None):
args = args or ()
kwargs = kwargs or {}
if not isinstance(args, (list, tuple)):
raise TypeError('task args must be a list or tuple')
if not isinstance(kwargs, Mapping):
raise TypeError('task keyword arguments must be a mapping')
if countdown: # convert countdown to ETA
self._verify_seconds(countdown, 'countdown')
now = now or self.app.now()
timezone = timezone or self.app.timezone
eta = maybe_make_aware(
now + timedelta(seconds=countdown),
tz=timezone,
)
if isinstance(expires, numbers.Real):
self._verify_seconds(expires, 'expires')
now = now or self.app.now()
timezone = timezone or self.app.timezone
expires = maybe_make_aware(
now + timedelta(seconds=expires),
tz=timezone,
)
if not isinstance(eta, string_t):
eta = eta and eta.isoformat()
# If we retry a task `expires` will already be ISO8601-formatted.
if not isinstance(expires, string_t):
expires = expires and expires.isoformat()
if argsrepr is None:
argsrepr = saferepr(args, self.argsrepr_maxsize)
if kwargsrepr is None:
kwargsrepr = saferepr(kwargs, self.kwargsrepr_maxsize)
if JSON_NEEDS_UNICODE_KEYS: # pragma: no cover
if callbacks:
callbacks = [utf8dict(callback) for callback in callbacks]
if errbacks:
errbacks = [utf8dict(errback) for errback in errbacks]
if chord:
chord = utf8dict(chord)
if not root_id: # empty root_id defaults to task_id
root_id = task_id
return task_message(
headers={
'lang': 'py',
'task': name,
'id': task_id,
'shadow': shadow,
'eta': eta,
'expires': expires,
'group': group_id,
'retries': retries,
'timelimit': [time_limit, soft_time_limit],
'root_id': root_id,
'parent_id': parent_id,
'argsrepr': argsrepr,
'kwargsrepr': kwargsrepr,
'origin': origin or anon_nodename()
},
properties={
'correlation_id': task_id,
'reply_to': reply_to or '',
},
body=(
args,
kwargs,
{
'callbacks': callbacks,
'errbacks': errbacks,
'chain': chain,
'chord': chord,
},
),
sent_event={
'uuid': task_id,
'root_id': root_id,
'parent_id': parent_id,
'name': name,
'args': argsrepr,
'kwargs': kwargsrepr,
'retries': retries,
'eta': eta,
'expires': expires,
} if create_sent_event else None,
)
def __init__(self, clocked_time, nowfun=None, app=None):
"""Initialize clocked."""
self.clocked_time = maybe_make_aware(clocked_time)
super().__init__(nowfun=nowfun, app=app)
def test_maybe_make_aware(self):
aware = datetime.utcnow().replace(tzinfo=timezone.utc)
self.assertTrue(maybe_make_aware(aware), timezone.utc)
naive = datetime.utcnow()
self.assertTrue(maybe_make_aware(naive))
def __init__(self, message, on_ack=noop,
hostname=None, eventer=None, app=None,
connection_errors=None, request_dict=None,
task=None, on_reject=noop, body=None,
headers=None, decoded=False, utc=True,
maybe_make_aware=maybe_make_aware,
maybe_iso8601=maybe_iso8601, **opts):
if headers is None:
headers = message.headers
if body is None:
body = message.body
self.app = app
self.message = message
self.body = body
self.utc = utc
self._decoded = decoded
if decoded:
self.content_type = self.content_encoding = None
else:
self.content_type, self.content_encoding = (
message.content_type, message.content_encoding,
)
self.id = headers['id']
type = self.type = self.name = headers['task']
self.root_id = headers.get('root_id')
self.parent_id = headers.get('parent_id')
if 'shadow' in headers:
self.name = headers['shadow'] or self.name
timelimit = headers.get('timelimit', None)
if timelimit:
self.time_limits = timelimit
self.argsrepr = headers.get('argsrepr', '')
self.kwargsrepr = headers.get('kwargsrepr', '')
self.on_ack = on_ack
self.on_reject = on_reject
self.hostname = hostname or gethostname()
self.eventer = eventer
self.connection_errors = connection_errors or ()
self.task = task or self.app.tasks[type]
# timezone means the message is timezone-aware, and the only timezone
# supported at this point is UTC.
eta = headers.get('eta')
if eta is not None:
try:
eta = maybe_iso8601(eta)
except (AttributeError, ValueError, TypeError) as exc:
raise InvalidTaskError(
'invalid ETA value {0!r}: {1}'.format(eta, exc))
self.eta = maybe_make_aware(eta, self.tzlocal)
else:
self.eta = None
expires = headers.get('expires')
if expires is not None:
try:
expires = maybe_iso8601(expires)
except (AttributeError, ValueError, TypeError) as exc:
raise InvalidTaskError(
'invalid expires value {0!r}: {1}'.format(expires, exc))
self.expires = maybe_make_aware(expires, self.tzlocal)
else:
self.expires = None
delivery_info = message.delivery_info or {}
properties = message.properties or {}
headers.update({
'reply_to': properties.get('reply_to'),
'correlation_id': properties.get('correlation_id'),
'delivery_info': {
'exchange': delivery_info.get('exchange'),
'routing_key': delivery_info.get('routing_key'),
'priority': properties.get('priority'),
'redelivered': delivery_info.get('redelivered'),
}
})
self.request_dict = headers
def __init__(self,
message,
on_ack=noop,
hostname=None,
eventer=None,
app=None,
connection_errors=None,
request_dict=None,
task=None,
on_reject=noop,
body=None,
headers=None,
decoded=False,
utc=True,
maybe_make_aware=maybe_make_aware,
maybe_iso8601=maybe_iso8601,
**opts):
self._message = message
self._request_dict = (message.headers.copy()
if headers is None else headers.copy())
self._body = message.body if body is None else body
self._app = app
self._utc = utc
self._decoded = decoded
if decoded:
self._content_type = self._content_encoding = None
else:
self._content_type, self._content_encoding = (
message.content_type,
message.content_encoding,
)
self.__payload = self._body if self._decoded else message.payload
self.id = self._request_dict['id']
self._type = self.name = self._request_dict['task']
if 'shadow' in self._request_dict:
self.name = self._request_dict['shadow'] or self.name
self._root_id = self._request_dict.get('root_id')
self._parent_id = self._request_dict.get('parent_id')
timelimit = self._request_dict.get('timelimit', None)
if timelimit:
self.time_limits = timelimit
self._argsrepr = self._request_dict.get('argsrepr', '')
self._kwargsrepr = self._request_dict.get('kwargsrepr', '')
self._on_ack = on_ack
self._on_reject = on_reject
self._hostname = hostname or gethostname()
self._eventer = eventer
self._connection_errors = connection_errors or ()
self._task = task or self._app.tasks[self._type]
self._ignore_result = self._request_dict.get('ignore_result', False)
# timezone means the message is timezone-aware, and the only timezone
# supported at this point is UTC.
eta = self._request_dict.get('eta')
if eta is not None:
try:
eta = maybe_iso8601(eta)
except (AttributeError, ValueError, TypeError) as exc:
raise InvalidTaskError(f'invalid ETA value {eta!r}: {exc}')
self._eta = maybe_make_aware(eta, self.tzlocal)
else:
self._eta = None
expires = self._request_dict.get('expires')
if expires is not None:
try:
expires = maybe_iso8601(expires)
except (AttributeError, ValueError, TypeError) as exc:
raise InvalidTaskError(
f'invalid expires value {expires!r}: {exc}')
self._expires = maybe_make_aware(expires, self.tzlocal)
else:
self._expires = None
delivery_info = message.delivery_info or {}
properties = message.properties or {}
self._delivery_info = {
'exchange': delivery_info.get('exchange'),
'routing_key': delivery_info.get('routing_key'),
'priority': properties.get('priority'),
'redelivered': delivery_info.get('redelivered', False),
}
self._request_dict.update({
'properties':
properties,
'reply_to':
properties.get('reply_to'),
'correlation_id':
properties.get('correlation_id'),
'hostname':
self._hostname,
'delivery_info':
self._delivery_info
})
# this is a reference pass to avoid memory usage burst
self._request_dict['args'], self._request_dict[
'kwargs'], _ = self.__payload
self._args = self._request_dict['args']
self._kwargs = self._request_dict['kwargs']
def maybe_make_aware(self, dt):
return maybe_make_aware(dt, self.tz)
def as_task_v2(
self,
task_id,
name,
args=None,
kwargs=None,
countdown=None,
eta=None,
group_id=None,
group_index=None,
expires=None,
retries=0,
chord=None,
callbacks=None,
errbacks=None,
reply_to=None,
time_limit=None,
soft_time_limit=None,
create_sent_event=False,
root_id=None,
parent_id=None,
shadow=None,
chain=None,
now=None,
timezone=None,
origin=None,
argsrepr=None,
kwargsrepr=None,
):
args = args or ()
kwargs = kwargs or {}
if not isinstance(args, (list, tuple)):
raise TypeError("task args must be a list or tuple")
if not isinstance(kwargs, Mapping):
raise TypeError("task keyword arguments must be a mapping")
if countdown: # convert countdown to ETA
self._verify_seconds(countdown, "countdown")
now = now or self.app.now()
timezone = timezone or self.app.timezone
eta = maybe_make_aware(
now + timedelta(seconds=countdown),
tz=timezone,
)
if isinstance(expires, numbers.Real):
self._verify_seconds(expires, "expires")
now = now or self.app.now()
timezone = timezone or self.app.timezone
expires = maybe_make_aware(
now + timedelta(seconds=expires),
tz=timezone,
)
if not isinstance(eta, string_t):
eta = eta and eta.isoformat()
# If we retry a task `expires` will already be ISO8601-formatted.
if not isinstance(expires, string_t):
expires = expires and expires.isoformat()
if argsrepr is None:
argsrepr = saferepr(args, self.argsrepr_maxsize)
if kwargsrepr is None:
kwargsrepr = saferepr(kwargs, self.kwargsrepr_maxsize)
if JSON_NEEDS_UNICODE_KEYS: # pragma: no cover
if callbacks:
callbacks = [utf8dict(callback) for callback in callbacks]
if errbacks:
errbacks = [utf8dict(errback) for errback in errbacks]
if chord:
chord = utf8dict(chord)
if not root_id: # empty root_id defaults to task_id
root_id = task_id
return task_message(
headers={
"lang": "py",
"task": name,
"id": task_id,
"shadow": shadow,
"eta": eta,
"expires": expires,
"group": group_id,
"group_index": group_index,
"retries": retries,
"timelimit": [time_limit, soft_time_limit],
"root_id": root_id,
"parent_id": parent_id,
"argsrepr": argsrepr,
"kwargsrepr": kwargsrepr,
"origin": origin or anon_nodename(),
},
properties={
"correlation_id": task_id,
"reply_to": reply_to or "",
},
body=(
args,
kwargs,
{
"callbacks": callbacks,
"errbacks": errbacks,
"chain": chain,
"chord": chord,
},
),
sent_event={
"uuid": task_id,
"root_id": root_id,
"parent_id": parent_id,
"name": name,
"args": argsrepr,
"kwargs": kwargsrepr,
"retries": retries,
"eta": eta,
"expires": expires,
} if create_sent_event else None,
)
def test_maybe_make_aware(self):
aware = datetime.utcnow().replace(tzinfo=timezone.utc)
assert maybe_make_aware(aware)
naive = datetime.utcnow()
assert maybe_make_aware(naive)
def __init__(
self,
message,
on_ack=noop,
hostname=None,
eventer=None,
app=None,
connection_errors=None,
request_dict=None,
task=None,
on_reject=noop,
body=None,
headers=None,
decoded=False,
utc=True,
maybe_make_aware=maybe_make_aware,
maybe_iso8601=maybe_iso8601,
**opts
):
if headers is None:
headers = message.headers
if body is None:
body = message.body
self.app = app
self.message = message
self.body = body
self.utc = utc
self._decoded = decoded
if decoded:
self.content_type = self.content_encoding = None
else:
self.content_type, self.content_encoding = (message.content_type, message.content_encoding)
self.id = headers["id"]
type = self.type = self.name = headers["task"]
self.root_id = headers.get("root_id")
self.parent_id = headers.get("parent_id")
if "shadow" in headers:
self.name = headers["shadow"] or self.name
if "timelimit" in headers:
self.time_limits = headers["timelimit"]
self.argsrepr = headers.get("argsrepr", "")
self.kwargsrepr = headers.get("kwargsrepr", "")
self.on_ack = on_ack
self.on_reject = on_reject
self.hostname = hostname or gethostname()
self.eventer = eventer
self.connection_errors = connection_errors or ()
self.task = task or self.app.tasks[type]
# timezone means the message is timezone-aware, and the only timezone
# supported at this point is UTC.
eta = headers.get("eta")
if eta is not None:
try:
eta = maybe_iso8601(eta)
except (AttributeError, ValueError, TypeError) as exc:
raise InvalidTaskError("invalid ETA value {0!r}: {1}".format(eta, exc))
self.eta = maybe_make_aware(eta, self.tzlocal)
else:
self.eta = None
expires = headers.get("expires")
if expires is not None:
try:
expires = maybe_iso8601(expires)
except (AttributeError, ValueError, TypeError) as exc:
raise InvalidTaskError("invalid expires value {0!r}: {1}".format(expires, exc))
self.expires = maybe_make_aware(expires, self.tzlocal)
else:
self.expires = None
delivery_info = message.delivery_info or {}
properties = message.properties or {}
headers.update(
{
"reply_to": properties.get("reply_to"),
"correlation_id": properties.get("correlation_id"),
"delivery_info": {
"exchange": delivery_info.get("exchange"),
"routing_key": delivery_info.get("routing_key"),
"priority": properties.get("priority"),
"redelivered": delivery_info.get("redelivered"),
},
}
)
self.request_dict = headers
def __init__(self,
message,
on_ack=noop,
hostname=None,
eventer=None,
app=None,
connection_errors=None,
request_dict=None,
task=None,
on_reject=noop,
body=None,
headers=None,
decoded=False,
utc=True,
maybe_make_aware=maybe_make_aware,
maybe_iso8601=maybe_iso8601,
**opts):
self._message = message
self._request_dict = message.headers if headers is None else headers
self._body = message.body if body is None else body
self._app = app
self._utc = utc
self._decoded = decoded
if decoded:
self._content_type = self._content_encoding = None
else:
self._content_type, self._content_encoding = (
message.content_type,
message.content_encoding,
)
self.__payload = self._body if self._decoded else message.payload
self.id = self._request_dict["id"]
self._type = self.name = self._request_dict["task"]
if "shadow" in self._request_dict:
self.name = self._request_dict["shadow"] or self.name
self._root_id = self._request_dict.get("root_id")
self._parent_id = self._request_dict.get("parent_id")
timelimit = self._request_dict.get("timelimit", None)
if timelimit:
self.time_limits = timelimit
self._argsrepr = self._request_dict.get("argsrepr", "")
self._kwargsrepr = self._request_dict.get("kwargsrepr", "")
self._on_ack = on_ack
self._on_reject = on_reject
self._hostname = hostname or gethostname()
self._eventer = eventer
self._connection_errors = connection_errors or ()
self._task = task or self._app.tasks[self._type]
# timezone means the message is timezone-aware, and the only timezone
# supported at this point is UTC.
eta = self._request_dict.get("eta")
if eta is not None:
try:
eta = maybe_iso8601(eta)
except (AttributeError, ValueError, TypeError) as exc:
raise InvalidTaskError("invalid ETA value {0!r}: {1}".format(
eta, exc))
self._eta = maybe_make_aware(eta, self.tzlocal)
else:
self._eta = None
expires = self._request_dict.get("expires")
if expires is not None:
try:
expires = maybe_iso8601(expires)
except (AttributeError, ValueError, TypeError) as exc:
raise InvalidTaskError(
"invalid expires value {0!r}: {1}".format(expires, exc))
self._expires = maybe_make_aware(expires, self.tzlocal)
else:
self._expires = None
delivery_info = message.delivery_info or {}
properties = message.properties or {}
self._delivery_info = {
"exchange": delivery_info.get("exchange"),
"routing_key": delivery_info.get("routing_key"),
"priority": properties.get("priority"),
"redelivered": delivery_info.get("redelivered"),
}
self._request_dict.update({
"reply_to":
properties.get("reply_to"),
"correlation_id":
properties.get("correlation_id"),
"hostname":
self._hostname,
"delivery_info":
self._delivery_info,
})
# this is a reference pass to avoid memory usage burst
self._request_dict["args"], self._request_dict[
"kwargs"], _ = self.__payload
self._args = self._request_dict["args"]
self._kwargs = self._request_dict["kwargs"]
def send_task(self, name, args=None, kwargs=None, countdown=None,
eta=None, task_id=None, producer=None, connection=None,
router=None, result_cls=None, expires=None,
publisher=None, link=None, link_error=None,
add_to_parent=True, group_id=None, group_index=None,
retries=0, chord=None,
reply_to=None, time_limit=None, soft_time_limit=None,
root_id=None, parent_id=None, route_name=None,
shadow=None, chain=None, task_type=None, **options):
"""Send task by name.
Supports the same arguments as :meth:`@-Task.apply_async`.
Arguments:
name (str): Name of task to call (e.g., `"tasks.add"`).
result_cls (AsyncResult): Specify custom result class.
"""
parent = have_parent = None
amqp = self.amqp
task_id = task_id or uuid()
producer = producer or publisher # XXX compat
router = router or amqp.router
conf = self.conf
if conf.task_always_eager: # pragma: no cover
warnings.warn(AlwaysEagerIgnored(
'task_always_eager has no effect on send_task',
), stacklevel=2)
ignore_result = options.pop('ignore_result', False)
options = router.route(
options, route_name or name, args, kwargs, task_type)
if expires is not None:
if isinstance(expires, datetime):
expires_s = (maybe_make_aware(expires) - self.now()).total_seconds()
else:
expires_s = expires
if expires_s < 0:
logger.warning(
f"{task_id} has an expiration date in the past ({-expires_s}s ago).\n"
"We assume this is intended and so we have set the "
"expiration date to 0 instead.\n"
"According to RabbitMQ's documentation:\n"
"\"Setting the TTL to 0 causes messages to be expired upon "
"reaching a queue unless they can be delivered to a "
"consumer immediately.\"\n"
"If this was unintended, please check the code which "
"published this task."
)
expires_s = 0
options["expiration"] = expires_s
if not root_id or not parent_id:
parent = self.current_worker_task
if parent:
if not root_id:
root_id = parent.request.root_id or parent.request.id
if not parent_id:
parent_id = parent.request.id
if conf.task_inherit_parent_priority:
options.setdefault('priority',
parent.request.delivery_info.get('priority'))
message = amqp.create_task_message(
task_id, name, args, kwargs, countdown, eta, group_id, group_index,
expires, retries, chord,
maybe_list(link), maybe_list(link_error),
reply_to or self.thread_oid, time_limit, soft_time_limit,
self.conf.task_send_sent_event,
root_id, parent_id, shadow, chain,
ignore_result=ignore_result,
argsrepr=options.get('argsrepr'),
kwargsrepr=options.get('kwargsrepr'),
)
if connection:
producer = amqp.Producer(connection, auto_declare=False)
with self.producer_or_acquire(producer) as P:
with P.connection._reraise_as_library_errors():
if not ignore_result:
self.backend.on_task_call(P, task_id)
amqp.send_task_message(P, name, message, **options)
result = (result_cls or self.AsyncResult)(task_id)
# We avoid using the constructor since a custom result class
# can be used, in which case the constructor may still use
# the old signature.
result.ignored = ignore_result
if add_to_parent:
if not have_parent:
parent, have_parent = self.current_worker_task, True
if parent:
parent.add_trail(result)
return result
def __init__(self, message, on_ack=noop,
hostname=None, eventer=None, app=None,
connection_errors=None, request_dict=None,
task=None, on_reject=noop, body=None,
headers=None, decoded=False, utc=True,
maybe_make_aware=maybe_make_aware,
maybe_iso8601=maybe_iso8601, **opts):
if headers is None:
headers = message.headers
if body is None:
body = message.body
self.app = app
self.message = message
self.body = body
self.utc = utc
self._decoded = decoded
if decoded:
self.content_type = self.content_encoding = None
else:
self.content_type, self.content_encoding = (
message.content_type, message.content_encoding,
)
self.id = headers['id']
type = self.type = self.name = headers['task']
self.root_id = headers.get('root_id')
self.parent_id = headers.get('parent_id')
if 'shadow' in headers:
self.name = headers['shadow'] or self.name
if 'timelimit' in headers:
self.time_limits = headers['timelimit']
self.argsrepr = headers.get('argsrepr', '')
self.kwargsrepr = headers.get('kwargsrepr', '')
self.on_ack = on_ack
self.on_reject = on_reject
self.hostname = hostname or gethostname()
self.eventer = eventer
self.connection_errors = connection_errors or ()
self.task = task or self.app.tasks[type]
# timezone means the message is timezone-aware, and the only timezone
# supported at this point is UTC.
eta = headers.get('eta')
if eta is not None:
try:
eta = maybe_iso8601(eta)
except (AttributeError, ValueError, TypeError) as exc:
raise InvalidTaskError(
'invalid ETA value {0!r}: {1}'.format(eta, exc))
self.eta = maybe_make_aware(eta, self.tzlocal)
else:
self.eta = None
expires = headers.get('expires')
if expires is not None:
try:
expires = maybe_iso8601(expires)
except (AttributeError, ValueError, TypeError) as exc:
raise InvalidTaskError(
'invalid expires value {0!r}: {1}'.format(expires, exc))
self.expires = maybe_make_aware(expires, self.tzlocal)
else:
self.expires = None
delivery_info = message.delivery_info or {}
properties = message.properties or {}
headers.update({
'reply_to': properties.get('reply_to'),
'correlation_id': properties.get('correlation_id'),
'delivery_info': {
'exchange': delivery_info.get('exchange'),
'routing_key': delivery_info.get('routing_key'),
'priority': properties.get('priority'),
'redelivered': delivery_info.get('redelivered'),
}
})
self.request_dict = headers
def task_message_handler(message,
body,
ack,
reject,
callbacks,
to_timestamp=to_timestamp):
# print('crawl_task_message_handler %s %s' % (task_name, repr(body)))
body, headers, decoded, utc = (
message.body,
message.headers,
False,
True,
)
if not body_can_be_buffer:
body = bytes(body) if isinstance(body, buffer_t) else body
req = BaseReq(
message,
on_ack=ack,
on_reject=reject,
app=app,
hostname=hostname,
eventer=eventer,
task=task,
connection_errors=connection_errors,
body=body,
headers=headers,
decoded=decoded,
utc=utc,
)
# if _does_info:
meta = req.task_info()
taskinfo = {'meta': meta}
_info(u'收到任务', extra=taskinfo)
if (req.expires
or req.id in controller_revoked_tasks) and req.revoked():
return
# req_args, req_kwargs, req_embed = req._payload
if task_sends_events:
send_event(
'task-received',
uuid=req.id,
name=req.name,
args=req.argsrepr,
kwargs=req.kwargsrepr,
root_id=req.root_id,
parent_id=req.parent_id,
retries=req.request_dict.get('retries', 0),
eta=req.eta and req.eta.isoformat(),
expires=req.expires and req.expires.isoformat(),
)
# 保存
# ti = get_task_info(req._args, req._kwargs)
fields = dict(
name=req.name,
# project=req._project, page=req._page, url=req._url,
kwargs=json.dumps(req._kwargs),
# args=req_args, kwargs=req_kwargs,
root_id=req.root_id,
parent_id=req.parent_id,
retries=req.request_dict.get('retries', 0),
eta=req.eta and req.eta.isoformat(),
expires=req.expires and req.expires.isoformat(),
meta=meta)
save_task_status('task-received', req.id, fields)
# 限速
if req._kwargs.get('__limit__'):
try:
key = 'rate:%s' % meta['project']
pending = get_expected_time(key)
# print '----Rate limit pending: %s %r' % (req.id, pending)
if pending > 0:
req.eta = maybe_make_aware(datetime.utcnow() +
timedelta(seconds=pending))
info('Rate Limit [%s.%s] %s', meta['project'],
meta['page'], pending)
except Exception:
error('Rate limit. Task: %r',
req.info(safe=True),
exc_info=True)
if req.eta:
try:
if req.utc:
eta = to_timestamp(to_system_tz(req.eta))
else:
eta = to_timestamp(req.eta, timezone.local)
except (OverflowError, ValueError):
error("Couldn't convert ETA %r to timestamp. Task: %r",
req.eta,
req.info(safe=True),
exc_info=True)
req.reject(requeue=False)
else:
consumer.qos.increment_eventually()
call_at(eta, apply_eta_task, (req, ), priority=6)
else:
if rate_limits_enabled:
bucket = get_bucket(task.name)
if bucket:
return limit_task(req, bucket, 1)
task_reserved(req)
if callbacks:
[callback(req) for callback in callbacks]
handle(req)
def test_maybe_make_aware(self):
aware = datetime.utcnow().replace(tzinfo=timezone.utc)
assert maybe_make_aware(aware)
naive = datetime.utcnow()
assert maybe_make_aware(naive)