def get_session_key():
"""
returns a 7 character string that varies with the current "session" (task or request)
"""
session_id = None
current_task = get_current_task()
if current_task:
# at least in tests, current_task may have the same id between different tasks!
session_id = current_task.request.id
if not session_id:
request = get_request()
if request:
session_id = str(id(get_request()))
else:
session_id = None
if session_id:
session_id = session_id.encode('utf-8')
# hash it so that similar numbers end up very different (esp. because we're truncating)
return hashlib.md5(session_id).hexdigest()[:7]
else:
# quickcache catches this and skips the cache
# this happens during tests (outside a fake task/request context)
# and during management commands
raise ForceSkipCache("Not part of a session")
def schedule_task(data):
""" 这个函数的作用是根据定时任务去完成定时任务的配置
:param data: 前端提交上来的数据
:return:
"""
# 获取任务的id
task_uuid = get_current_task().request.id
# 配置定时任务
cron_data = data.get('cron_data', {})
schedule, _ = CrontabSchedule.objects.get_or_create(
minute=cron_data.get('minute', '*'),
hour=cron_data.get('hour', '*'),
day_of_week=cron_data.get('day_of_week', '*'),
day_of_month=cron_data.get('day_of_month', '*'),
month_of_year=cron_data.get('month_of_year', '*'),
timezone=pytz.timezone(CELERY_TIMEZONE))
# 配置任务的一些信息
# (1)获取任务名称
# (1)配置所需要接收的参数
task_obj = TaskModel.objects.filter(id=data.get("task_id")).first()
PeriodicTask.objects.create(crontab=schedule,
name="{}##{}".format(task_uuid, task_obj.name),
task='task.tasks.now_exec_task',
kwargs=json.dumps({"data": data}),
description=task_obj.describe)
return True
def format(self, record):
task = get_current_task()
if task and task.request:
record.__dict__.update(task_id=task.request.id, task_name=task.name)
else:
record.__dict__.setdefault("task_name", "???")
record.__dict__.setdefault("task_id", "???")
return ColorFormatter.format(self, record)
def filter(self, record):
task = get_current_task()
if task and task.request:
record.task_id = task.request.id
record.task_name = task.name
else:
record.task_id = record.task_name = '-'
return True
def format(self, record):
task = get_current_task()
if task and task.request:
record.__dict__.update(task_id=task.request.id, task_name=task.name)
else:
record.__dict__.setdefault('task_name', '???')
record.__dict__.setdefault('task_id', '???')
return logging.Formatter.format(self, record)
def format(self, record):
task = get_current_task()
if task and task.request:
name = task.name.split('.')[-1]
record.__dict__.update(task_id=task.request.id, task_name=name)
else:
record.__dict__.update(task_id='xxx', task_name='xxx')
return ColorFormatter.format(self, record)
def __scripts__import__(name,
globals_=None,
locals_=None,
fromlist=None,
level=-1):
'''from __scripts__ import xxx OR from __scripts__.xxx import yyy'''
attrs, project = None, None
if name.startswith('__project__'):
env = globals_.get('environ')
if env:
project = env.get('project')
if not project:
task = get_current_task()
if not task:
raise ImportError('No module named %s (current task is None)' %
name)
b = task.brief()
project = b.get('project')
if not project:
raise ImportError('No module named %s (project is None)' %
name)
attrs = name.split('.', 1)
attrs.pop(0)
elif name.startswith('__scripts__'):
attrs = name.split('.', 1)
attrs.pop(0)
else:
return __import__original(name, globals_, locals_, fromlist, level)
if attrs:
for attr in attrs:
mod_name = '%s_%s' % (project if project else '', attr)
if hasattr(__scripts__, mod_name):
submod = getattr(__scripts__, mod_name)
if submod.__created__ - datetime.now() < MAX_LIVE:
return submod
submod = get_script(attr, globals_, project=project)
setattr(__scripts__, mod_name, submod)
return submod
elif fromlist:
obj = FakeModule()
for attr in fromlist:
mod_name = '%s_%s' % (project if project else '', attr)
if hasattr(__scripts__, mod_name):
submod = getattr(__scripts__, mod_name)
if submod.__created__ - datetime.now() < MAX_LIVE:
setattr(obj, attr, submod)
continue
submod = get_script(attr, globals_, project=project)
setattr(__scripts__, mod_name, submod)
setattr(obj, attr, submod)
return obj
else:
raise ImportError('No module named %s (__scripts__import__)' % name)
def format(self, record):
task = get_current_task()
if task and task.request:
record.__dict__.update(task_id=task.request.id,
task_name=task.name)
else:
record.__dict__.setdefault('task_name', '???')
record.__dict__.setdefault('task_id', '???')
return ColorFormatter.format(self, record)
def format(self, record):
task = get_current_task()
if task:
record.__dict__.update(task_id=task.request.id,
task_name=task.name)
else:
record.__dict__.setdefault('task_name', '')
record.__dict__.setdefault('task_id', '')
return super().format(record)
def to_dict(self, record: logging.LogRecord):
data = super(JsonTaskFormatter, self).to_dict(record)
try:
from celery._state import get_current_task
task = get_current_task()
if task and task.request:
data['task_id'] = task.request.id
data['task_name'] = task.name
except ImportError:
pass
return data
def to_dict(self, record: logging.LogRecord):
data = super(JsonTaskFormatter, self).to_dict(record)
try:
from celery._state import get_current_task
task = get_current_task()
if task and task.request:
data['task_id'] = task.request.id
data['task_name'] = task.name
except ImportError:
pass
return data
def format(self, record):
self.use_color = False
task = get_current_task()
if task and task.request:
record.__dict__.update(root_id=task.request.root_id,
task_id=task.request.id,
task_name=task.name)
else:
record.__dict__.setdefault('task_name', None)
record.__dict__.setdefault('task_id', None)
record.__dict__.setdefault('root_id', None)
return ColorFormatter.format(self, record)
def filter(self, record):
task = get_current_task()
if task and task.request:
record.task_id = task.request.id
record.request_id = get_celery_request_id(task.request)
record.task_name = task.name
else:
record.task_id = None
record.request_id = None
record.task_name = None
return record
def get_celery_task_attributes():
result = dict(task_name=None, task_id=None, meta=None, meta_meta=None)
try:
# See celery.app.log.TaskFormatter
from celery._state import get_current_task
task = get_current_task()
if not task:
return dict(result, meta='no_task')
if not task.request:
return dict(result, meta='no_task_request')
return dict(result, task_name=task.name, task_id=task.request.id)
except Exception as exc:
return dict(result, meta='error', meta_meta=exc)
def get_celery_task_attributes():
result = dict(task_name=None, task_id=None, meta=None, meta_meta=None)
try:
# See celery.app.log.TaskFormatter
from celery._state import get_current_task
task = get_current_task()
if not task:
return dict(result, meta='no_task')
if not task.request:
return dict(result, meta='no_task_request')
return dict(result, task_name=task.name, task_id=task.request.id)
except Exception as exc:
return dict(result, meta='error', meta_meta=exc)
def makeRecord(self,
name,
level,
fn,
lno,
msg,
args,
exc_info,
func=None,
extra=None,
sinfo=None):
record = super(Logger,
self).makeRecord(name, level, fn, lno, msg, args,
exc_info, func, extra, sinfo)
if extra and 'local_only' in extra:
return record
record_dict = record.__dict__
meta = record_dict.setdefault('meta', {})
task = get_current_task()
if task and task.request and hasattr(task, 'brief'):
if 'project' not in meta:
meta.update(task.brief())
rq = task.request
meta.update(worker=rq.hostname)
meta.pop('task_id', None)
meta.pop('task_name', None)
record_dict.update(task_id=rq.id, task_name=task.name)
else:
record_dict['task_id'] = meta.pop('task_id', '??')
record_dict['task_name'] = meta.pop('task_name', '??')
if meta:
task_info = []
if 'job' in meta:
task_info.append(
'[%s/%s/%s]' %
(meta['project'], meta['job'], meta.get('page', '??')))
if 'action' in meta:
task_info.append('[%s/%s]' % (meta['project'], meta['action']))
if 'batch_id' in meta:
task_info.append('[%s]' % (meta['batch_id'], ))
record_dict.update({'task_info': ''.join(task_info)})
else:
record_dict.setdefault('task_info', '')
return record
def format(self, record):
task = get_current_task()
if task and task.request:
# For gevent pool, task_id will be something like `7ab44cb4-aacf-444e-bc20-4cbaa2a7b082`. For logs
# is better to get it short
task_id = task.request.id[:8]
# Task name usually has all the package, better cut the first part for logging
task_name = task.name.split('.')[-1]
record.__dict__.update(task_id=task_id, task_name=task_name)
else:
record.__dict__.setdefault('task_name', '???')
record.__dict__.setdefault('task_id', '???')
return ColorFormatter.format(self, record)
def get_session_key():
"""
returns a 7 character string that varies with the current "session" (task or request)
"""
current_task = get_current_task()
if current_task is not None:
return _quickcache_id(current_task.request)
request = get_request()
if request is not None:
return _quickcache_id(request)
# quickcache catches this and skips the cache
# this happens during tests (outside a fake task/request context)
# and during management commands
raise ForceSkipCache("Not part of a session")
def get_celery_request_id():
"""Return the request ID from the current Celery request
If there is no request then return None.
Importing this module will register this getter with ``get_request_id``.
Returns:
str or None: the current request ID
"""
task = get_current_task()
if task and task.request:
return task.request.get(TS_REQUEST_ID)
return None
def get_celery_request_id(request=None):
"""Return the request ID from the current Celery request
If there is no request then return None.
Importing this module will register this getter with ``get_request_id``.
Returns:
str or None: the current request ID
"""
if not request:
task = get_current_task()
if task and task.request:
request = task.request
if request:
return request.get(_CELERY_X_HEADER, None)
def filter(self, record):
"""
Checks for a currently executing celery task and adds the name and
id to the log record.
:param record:
:return:
"""
task = get_current_task()
if task and hasattr(task, 'request') and task.request:
record.__dict__.update(task_id=task.request.id,
task_name=task.name)
else:
record.__dict__.setdefault('task_name', '')
record.__dict__.setdefault('task_id', '')
return True
def format(self, record):
"""Append task-related values to the record upon format."""
task = get_current_task()
if task and task.request:
record.__dict__.update(
task_id=task.request.id,
task_name=task.name,
task_root_id=task.request.root_id,
task_parent_id=task.request.parent_id,
)
else:
record.__dict__.setdefault("task_name", "None")
record.__dict__.setdefault("task_id", "None")
record.__dict__.setdefault("task_root_id", "None")
record.__dict__.setdefault("task_parent_id", "None")
return ColorFormatter.format(self, record)
def now_exec_task(data, *args, **kwargs):
""" 这个函数的作用是用来立即执行同步任务
步骤
（1）取到 task 的信息
（2）根据 task.id 取到需要发送邮件的用户信息
:param data:
:return:
"""
print(kwargs)
# 获取任务的id
task_uuid = get_current_task().request.id
# 保存任务对应的需要发送的用户
save_task_user.delay(data=data, task_uuid=task_uuid)
# 执行相应任务
exec_task.delay(data=data, task_uuid=task_uuid)
return True
def _on_task_finished(self, task_id, value, receiver, callback=None):
if isinstance(value, Exception):
self.logger.error('Failed to save result, task_id: %s',
task_id,
exc_info=value)
elif isinstance(value, GroupResult):
self.logger.debug('Received GroupResult, task_id: %s', task_id)
self._progress_manager.update_progress_total(len(value))
self._handle_result(value, receiver=receiver)
elif self._is_groupresult_meta(value):
self.logger.debug('Received GroupResult META, task_id: %s',
task_id)
for results in gen_chunk(
self._gen_result_from_groupresult_meta(value),
self._batch_size):
self._progress_manager.update_progress_total(len(results))
for result in results:
self._handle_result(result,
receiver=receiver,
other_worker=True)
else:
self.logger.debug(
'Received common result, task_id: %s, size: %d bytes', task_id,
total_size(value))
root_id = current_task.request.root_id
if subtask_success.receivers:
task = _state.get_current_task()
subtask_success.send(sender=task,
root_id=root_id,
task_id=task_id,
retval=value)
if receiver:
try:
receiver(root_id=root_id, task_id=task_id, retval=value)
except Exception as exc:
self.logger.exception('Failed to save result, task_id: %s',
task_id)
self._progress_manager.update_progress_completed(callback=callback)
def __add_log_source_to_dict(logger, _, event_dict):
try:
from celery._state import get_current_task
get_current_task = get_current_task
except ImportError:
get_current_task = lambda: None
task = get_current_task()
if task and task.request and 'fields' in event_dict:
event_dict['fields']['celery_task_id'] = task.request.id
event_dict['fields']['celery_task_name'] = task.name
if 'task_id' in event_dict:
event_dict['correlationId'] = event_dict['task_id']
del event_dict['task_id']
# If by any chance the record already contains a `source` key,
# (very rare) move that into a 'source_original' key
if 'source' in event_dict:
event_dict['source_original'] = event_dict['source']
f, name = _find_first_app_frame_and_name(additional_ignores=[
"logging",
__name__
])
if not f:
return event_dict
filename = inspect.getfile(f)
frameinfo = inspect.getframeinfo(f)
if not frameinfo:
return event_dict
if frameinfo:
event_dict['source'] = '{}:{}:{}'.format(
filename,
frameinfo.function,
frameinfo.lineno,
)
return event_dict
def get_task_extra():
task = get_current_task()
request = task.request
indexes = getattr(request, "formatter_omit_indexes", [])
task_args = [
value if index + 1 not in indexes else OMITTED_STR
for index, value in enumerate(request.args)
]
names = getattr(request, "formatter_omit_names", [])
task_kwargs = {
key: value if key not in names else OMITTED_STR
for key, value in request.kwargs.items()
}
extra = dict(
TASK_ID=request.id,
TASK_NAME=task.name,
TASK_ARGS=task_args,
TASK_KWARGS=task_kwargs,
TASK_RETRIES=request.retries,
)
return extra
def current_task_children(self, request=None):
request = request or getattr(get_current_task(), 'request', None)
if request:
return [r.as_tuple() for r in getattr(request, 'children', [])]
def current_task_children(self, request=None):
request = request or getattr(get_current_task(), 'request', None)
if request:
return [r.as_tuple() for r in getattr(request, 'children', [])]
