def get_session_key(): """ returns a 7 character string that varies with the current "session" (task or request) """ session_id = None current_task = get_current_task() if current_task: # at least in tests, current_task may have the same id between different tasks! session_id = current_task.request.id if not session_id: request = get_request() if request: session_id = str(id(get_request())) else: session_id = None if session_id: session_id = session_id.encode('utf-8') # hash it so that similar numbers end up very different (esp. because we're truncating) return hashlib.md5(session_id).hexdigest()[:7] else: # quickcache catches this and skips the cache # this happens during tests (outside a fake task/request context) # and during management commands raise ForceSkipCache("Not part of a session")
def schedule_task(data): """ 这个函数的作用是根据定时任务去完成定时任务的配置 :param data: 前端提交上来的数据 :return: """ # 获取任务的id task_uuid = get_current_task().request.id # 配置定时任务 cron_data = data.get('cron_data', {}) schedule, _ = CrontabSchedule.objects.get_or_create( minute=cron_data.get('minute', '*'), hour=cron_data.get('hour', '*'), day_of_week=cron_data.get('day_of_week', '*'), day_of_month=cron_data.get('day_of_month', '*'), month_of_year=cron_data.get('month_of_year', '*'), timezone=pytz.timezone(CELERY_TIMEZONE)) # 配置任务的一些信息 # (1)获取任务名称 # (1)配置所需要接收的参数 task_obj = TaskModel.objects.filter(id=data.get("task_id")).first() PeriodicTask.objects.create(crontab=schedule, name="{}##{}".format(task_uuid, task_obj.name), task='task.tasks.now_exec_task', kwargs=json.dumps({"data": data}), description=task_obj.describe) return True
def format(self, record): task = get_current_task() if task and task.request: record.__dict__.update(task_id=task.request.id, task_name=task.name) else: record.__dict__.setdefault("task_name", "???") record.__dict__.setdefault("task_id", "???") return ColorFormatter.format(self, record)
def filter(self, record): task = get_current_task() if task and task.request: record.task_id = task.request.id record.task_name = task.name else: record.task_id = record.task_name = '-' return True
def format(self, record): task = get_current_task() if task and task.request: record.__dict__.update(task_id=task.request.id, task_name=task.name) else: record.__dict__.setdefault('task_name', '???') record.__dict__.setdefault('task_id', '???') return logging.Formatter.format(self, record)
def format(self, record): task = get_current_task() if task and task.request: name = task.name.split('.')[-1] record.__dict__.update(task_id=task.request.id, task_name=name) else: record.__dict__.update(task_id='xxx', task_name='xxx') return ColorFormatter.format(self, record)
def __scripts__import__(name, globals_=None, locals_=None, fromlist=None, level=-1): '''from __scripts__ import xxx OR from __scripts__.xxx import yyy''' attrs, project = None, None if name.startswith('__project__'): env = globals_.get('environ') if env: project = env.get('project') if not project: task = get_current_task() if not task: raise ImportError('No module named %s (current task is None)' % name) b = task.brief() project = b.get('project') if not project: raise ImportError('No module named %s (project is None)' % name) attrs = name.split('.', 1) attrs.pop(0) elif name.startswith('__scripts__'): attrs = name.split('.', 1) attrs.pop(0) else: return __import__original(name, globals_, locals_, fromlist, level) if attrs: for attr in attrs: mod_name = '%s_%s' % (project if project else '', attr) if hasattr(__scripts__, mod_name): submod = getattr(__scripts__, mod_name) if submod.__created__ - datetime.now() < MAX_LIVE: return submod submod = get_script(attr, globals_, project=project) setattr(__scripts__, mod_name, submod) return submod elif fromlist: obj = FakeModule() for attr in fromlist: mod_name = '%s_%s' % (project if project else '', attr) if hasattr(__scripts__, mod_name): submod = getattr(__scripts__, mod_name) if submod.__created__ - datetime.now() < MAX_LIVE: setattr(obj, attr, submod) continue submod = get_script(attr, globals_, project=project) setattr(__scripts__, mod_name, submod) setattr(obj, attr, submod) return obj else: raise ImportError('No module named %s (__scripts__import__)' % name)
def format(self, record): task = get_current_task() if task and task.request: record.__dict__.update(task_id=task.request.id, task_name=task.name) else: record.__dict__.setdefault('task_name', '???') record.__dict__.setdefault('task_id', '???') return ColorFormatter.format(self, record)
def format(self, record): task = get_current_task() if task: record.__dict__.update(task_id=task.request.id, task_name=task.name) else: record.__dict__.setdefault('task_name', '') record.__dict__.setdefault('task_id', '') return super().format(record)
def to_dict(self, record: logging.LogRecord): data = super(JsonTaskFormatter, self).to_dict(record) try: from celery._state import get_current_task task = get_current_task() if task and task.request: data['task_id'] = task.request.id data['task_name'] = task.name except ImportError: pass return data
def format(self, record): self.use_color = False task = get_current_task() if task and task.request: record.__dict__.update(root_id=task.request.root_id, task_id=task.request.id, task_name=task.name) else: record.__dict__.setdefault('task_name', None) record.__dict__.setdefault('task_id', None) record.__dict__.setdefault('root_id', None) return ColorFormatter.format(self, record)
def filter(self, record): task = get_current_task() if task and task.request: record.task_id = task.request.id record.request_id = get_celery_request_id(task.request) record.task_name = task.name else: record.task_id = None record.request_id = None record.task_name = None return record
def get_celery_task_attributes(): result = dict(task_name=None, task_id=None, meta=None, meta_meta=None) try: # See celery.app.log.TaskFormatter from celery._state import get_current_task task = get_current_task() if not task: return dict(result, meta='no_task') if not task.request: return dict(result, meta='no_task_request') return dict(result, task_name=task.name, task_id=task.request.id) except Exception as exc: return dict(result, meta='error', meta_meta=exc)
def makeRecord(self, name, level, fn, lno, msg, args, exc_info, func=None, extra=None, sinfo=None): record = super(Logger, self).makeRecord(name, level, fn, lno, msg, args, exc_info, func, extra, sinfo) if extra and 'local_only' in extra: return record record_dict = record.__dict__ meta = record_dict.setdefault('meta', {}) task = get_current_task() if task and task.request and hasattr(task, 'brief'): if 'project' not in meta: meta.update(task.brief()) rq = task.request meta.update(worker=rq.hostname) meta.pop('task_id', None) meta.pop('task_name', None) record_dict.update(task_id=rq.id, task_name=task.name) else: record_dict['task_id'] = meta.pop('task_id', '??') record_dict['task_name'] = meta.pop('task_name', '??') if meta: task_info = [] if 'job' in meta: task_info.append( '[%s/%s/%s]' % (meta['project'], meta['job'], meta.get('page', '??'))) if 'action' in meta: task_info.append('[%s/%s]' % (meta['project'], meta['action'])) if 'batch_id' in meta: task_info.append('[%s]' % (meta['batch_id'], )) record_dict.update({'task_info': ''.join(task_info)}) else: record_dict.setdefault('task_info', '') return record
def format(self, record): task = get_current_task() if task and task.request: # For gevent pool, task_id will be something like `7ab44cb4-aacf-444e-bc20-4cbaa2a7b082`. For logs # is better to get it short task_id = task.request.id[:8] # Task name usually has all the package, better cut the first part for logging task_name = task.name.split('.')[-1] record.__dict__.update(task_id=task_id, task_name=task_name) else: record.__dict__.setdefault('task_name', '???') record.__dict__.setdefault('task_id', '???') return ColorFormatter.format(self, record)
def get_session_key(): """ returns a 7 character string that varies with the current "session" (task or request) """ current_task = get_current_task() if current_task is not None: return _quickcache_id(current_task.request) request = get_request() if request is not None: return _quickcache_id(request) # quickcache catches this and skips the cache # this happens during tests (outside a fake task/request context) # and during management commands raise ForceSkipCache("Not part of a session")
def get_celery_request_id(): """Return the request ID from the current Celery request If there is no request then return None. Importing this module will register this getter with ``get_request_id``. Returns: str or None: the current request ID """ task = get_current_task() if task and task.request: return task.request.get(TS_REQUEST_ID) return None
def get_celery_request_id(request=None): """Return the request ID from the current Celery request If there is no request then return None. Importing this module will register this getter with ``get_request_id``. Returns: str or None: the current request ID """ if not request: task = get_current_task() if task and task.request: request = task.request if request: return request.get(_CELERY_X_HEADER, None)
def filter(self, record): """ Checks for a currently executing celery task and adds the name and id to the log record. :param record: :return: """ task = get_current_task() if task and hasattr(task, 'request') and task.request: record.__dict__.update(task_id=task.request.id, task_name=task.name) else: record.__dict__.setdefault('task_name', '') record.__dict__.setdefault('task_id', '') return True
def format(self, record): """Append task-related values to the record upon format.""" task = get_current_task() if task and task.request: record.__dict__.update( task_id=task.request.id, task_name=task.name, task_root_id=task.request.root_id, task_parent_id=task.request.parent_id, ) else: record.__dict__.setdefault("task_name", "None") record.__dict__.setdefault("task_id", "None") record.__dict__.setdefault("task_root_id", "None") record.__dict__.setdefault("task_parent_id", "None") return ColorFormatter.format(self, record)
def now_exec_task(data, *args, **kwargs): """ 这个函数的作用是用来立即执行同步任务 步骤 (1)取到 task 的信息 (2)根据 task.id 取到需要发送邮件的用户信息 :param data: :return: """ print(kwargs) # 获取任务的id task_uuid = get_current_task().request.id # 保存任务对应的需要发送的用户 save_task_user.delay(data=data, task_uuid=task_uuid) # 执行相应任务 exec_task.delay(data=data, task_uuid=task_uuid) return True
def _on_task_finished(self, task_id, value, receiver, callback=None): if isinstance(value, Exception): self.logger.error('Failed to save result, task_id: %s', task_id, exc_info=value) elif isinstance(value, GroupResult): self.logger.debug('Received GroupResult, task_id: %s', task_id) self._progress_manager.update_progress_total(len(value)) self._handle_result(value, receiver=receiver) elif self._is_groupresult_meta(value): self.logger.debug('Received GroupResult META, task_id: %s', task_id) for results in gen_chunk( self._gen_result_from_groupresult_meta(value), self._batch_size): self._progress_manager.update_progress_total(len(results)) for result in results: self._handle_result(result, receiver=receiver, other_worker=True) else: self.logger.debug( 'Received common result, task_id: %s, size: %d bytes', task_id, total_size(value)) root_id = current_task.request.root_id if subtask_success.receivers: task = _state.get_current_task() subtask_success.send(sender=task, root_id=root_id, task_id=task_id, retval=value) if receiver: try: receiver(root_id=root_id, task_id=task_id, retval=value) except Exception as exc: self.logger.exception('Failed to save result, task_id: %s', task_id) self._progress_manager.update_progress_completed(callback=callback)
def __add_log_source_to_dict(logger, _, event_dict): try: from celery._state import get_current_task get_current_task = get_current_task except ImportError: get_current_task = lambda: None task = get_current_task() if task and task.request and 'fields' in event_dict: event_dict['fields']['celery_task_id'] = task.request.id event_dict['fields']['celery_task_name'] = task.name if 'task_id' in event_dict: event_dict['correlationId'] = event_dict['task_id'] del event_dict['task_id'] # If by any chance the record already contains a `source` key, # (very rare) move that into a 'source_original' key if 'source' in event_dict: event_dict['source_original'] = event_dict['source'] f, name = _find_first_app_frame_and_name(additional_ignores=[ "logging", __name__ ]) if not f: return event_dict filename = inspect.getfile(f) frameinfo = inspect.getframeinfo(f) if not frameinfo: return event_dict if frameinfo: event_dict['source'] = '{}:{}:{}'.format( filename, frameinfo.function, frameinfo.lineno, ) return event_dict
def get_task_extra(): task = get_current_task() request = task.request indexes = getattr(request, "formatter_omit_indexes", []) task_args = [ value if index + 1 not in indexes else OMITTED_STR for index, value in enumerate(request.args) ] names = getattr(request, "formatter_omit_names", []) task_kwargs = { key: value if key not in names else OMITTED_STR for key, value in request.kwargs.items() } extra = dict( TASK_ID=request.id, TASK_NAME=task.name, TASK_ARGS=task_args, TASK_KWARGS=task_kwargs, TASK_RETRIES=request.retries, ) return extra
def current_task_children(self, request=None): request = request or getattr(get_current_task(), 'request', None) if request: return [r.as_tuple() for r in getattr(request, 'children', [])]