def push_to_orcid(sender, record, *args, **kwargs):
"""If needed, queue the push of the new changes to ORCID.
"""
if not is_hep(record):
return
def _get_orcid(author_ids):
for author_id in author_ids:
if author_id.get('schema', '').lower() == 'orcid':
return author_id['value']
task_name = current_app.config['ORCID_PUSH_TASK_ENDPOINT']
authors = record.get('authors', ())
for author in authors:
orcid = _get_orcid(author.get('ids', ()))
if not orcid:
continue
token = get_push_access_token(orcid)
if token is None:
continue
push_to_orcid_task = Task()
push_to_orcid_task.name = task_name
push_to_orcid_task.apply_async(
queue='orcid_push',
kwargs={
'orcid': orcid,
'rec_id': record['control_number'],
'token': token,
},
)
def create_for_task(cls, celecry_task: Task, a_id, task_name=''):
"""创建关联任务并设置初始进度"""
task_name = task_name or getattr(celecry_task.request, 'task')
a_task_result = AssociatedTaskResult.create(a_type_name=task_name,
a_id=a_id)
a_task_result.add_task_result(celecry_task.request.id)
celecry_task.update_state(state=states.PROGRESS, request=celecry_task.request)
def __call__(self, *args, **kwargs):
"""Execute task."""
# If an "app_context" has already been loaded, just pass through
if flask._app_ctx_stack.top is not None:
return Task.__call__(self, *args, **kwargs)
with self.app.flask_app.app_context():
return Task.__call__(self, *args, **kwargs)
def push_to_orcid(sender, record, *args, **kwargs):
"""If needed, queue the push of the new changes to ORCID."""
if not is_hep(record) or not current_app.config[
'FEATURE_FLAG_ENABLE_ORCID_PUSH']:
return
# Ensure there is a control number. This is not always the case because of broken store_record.
if 'control_number' not in record:
return
task_name = current_app.config['ORCID_PUSH_TASK_ENDPOINT']
orcids = get_orcids_for_push(record)
orcids_and_tokens = get_push_access_tokens(orcids)
for orcid, access_token in orcids_and_tokens:
push_to_orcid_task = Task()
push_to_orcid_task.name = task_name
push_to_orcid_task.apply_async(
queue='orcid_push',
kwargs={
'orcid': orcid,
'rec_id': record['control_number'],
'oauth_token': access_token,
},
)
def wrapper(task: Task, *args):
task.max_retries = times
try:
result = func(task, *args)
except Exception as exc:
raise task.retry(exc=exc, countdown=5)
else:
return result
def __init__(self):
Task.__init__(self)
# Shared namespace
self.worker_tracker = {
'last_req_at': datetime.now(),
'last_req_time': 0,
'throughput': 0
}
def __init__(self, *args, **kwargs):
Task.__init__(self, *args, **kwargs)
self.logger = logger
try:
self._redis = self.app.api_manager.redis_taskmanager.conn
except BaseException:
self._redis = None
def __init__(self, *args, **kwargs):
Task.__init__(self, *args, **kwargs)
#task_short_name = {u'task_name':self.name.split(u'.')[-1]}
#logger = logging.LoggerAdapter(get_task_logger(__name__), task_short_name)
self.logger = logger
try:
self._redis = self.app.api_manager.redis_taskmanager.conn
except:
self._redis = None
def map_task_per_tenants(task: Task, sequential: bool = False) -> None:
# todo: filtering by tenant status
tenants_ids = list(
TenantModel.objects.exclude(
schema_name=public_schema_name).values_list('id', flat=True))
if sequential:
logger.info("Submitting tasks for tenants: %s" %
str(task.map(tenants_ids).apply_async()))
else:
logger.info("Submitting tasks for tenants: %s" % ', '.join(
[str(task.delay(tenant_id)) for tenant_id in tenants_ids]))
def __call__(self, *args, **kwargs):
# from windar_api.db_util.feature_flags import get_all_flags_from_celery_task
# fflags = get_all_flags_from_celery_task(self)
rq = self.request
with app.test_request_context():
log.info('Start zadania %s (id: %s)', self.name, rq.id)
log.debug('Parametry zadania %s\nargs: %r\nkwargs: %r', rq.id, args, kwargs)
# if fflags:
# log.debug('Feature flags wymuszone przez headers: %r', fflags)
try:
ret = Task.__call__(self, *args, **kwargs)
db.session.commit()
# db_kplus.session.commit()
except Retry:
db.session.rollback()
# db_kplus.session.rollback()
raise
except Exception:
db.session.rollback()
# db_kplus.session.rollback()
log.error('Blad w trakcie wykonywania zadania %s (id: %s)', self.name, rq.id, exc_info=True)
raise
else:
log.info('Koniec zadania %s (id: %s)', self.name, rq.id)
return ret
def retry_task(
task: Task,
exception: Optional[Exception] = None,
max_attempts: Optional[int] = None,
) -> None:
"""
Retry celery tasks.
Retries the specified task using a "backing off countdown",
meaning that the interval between retries grows exponentially
with every retry.
:param task: The task to retry.
:param exception: Optionally, the exception that caused the retry.
:param max_attempts: Optionally, number of maximum attempts to retry the task.
"""
def backoff(attempts: int) -> int:
return 5 * (2 ** attempts)
kwargs: dict = {
"countdown": backoff(task.request.retries),
}
if exception:
kwargs["exc"] = exception
if not max_attempts or task.request.retries < max_attempts:
raise task.retry(**kwargs)
def __call__(self, *args, **kwargs):
"""Execute task."""
with self.app.flask_app.test_request_context():
self.app.flask_app.try_trigger_before_first_request_functions()
self.app.flask_app.preprocess_request()
res = Task.__call__(self, *args, **kwargs)
self.app.flask_app.process_response(
self.app.flask_app.response_class())
return res
def __call__(self, *args, **kwargs):
"""Execute task."""
with self.app.flask_app.test_request_context():
self.app.flask_app.try_trigger_before_first_request_functions()
self.app.flask_app.preprocess_request()
res = Task.__call__(self, *args, **kwargs)
self.app.flask_app.process_response(
self.app.flask_app.response_class())
return res
def __init__(self, *args, **kwargs):
Task.__init__(self, *args, **kwargs)
self.name = self.__class__.__module__ + '.' + self.name
self.logger = logger
self.steps = []
self.task_result = TaskResult()
self.objid = None
self.objtype = None
self.objdef = None
self.op = None
self.opid = None
self.delta = None
self.user = None
self.api_id = None
if self.entity_class is not None:
self.objtype = self.entity_class.objtype
self.objdef = self.entity_class.objdef
def send(self, message):
# 生产者配置
app = Celery(broker=celery_config.BROKER_URL)
task = Task()
task.name = celery_config.REQUEST_TASK
task.bind(app)
task.apply_async([message],
exchange=celery_config.REQUEST_NAME,
routing_key=celery_config.ROUNTING_KEY,
expires=60 * 60 * 24)
def update_progress(task: Task, new_status: int):
""" Set the current progress for the task object (celery)
Args:
task (Task):
new_status (int): The progress bar
Returns:
"""
if new_status < 0 or new_status > 100:
raise ValueError("new_status must be in range [0, 100]")
try:
pending_task = PendingTask.objects.get(task_id=task.request.id)
pending_task.progress = new_status
pending_task.save()
except ObjectDoesNotExist:
pass
task.update_state(state='PROGRESS',
meta={
'current': new_status,
'total': 100,
})
def run_task_on_commit(task: Task, args: Iterable = [], kwargs: Mapping = {}):
"""Run a celery task after the next database commit (if we are in a transaction)"""
transaction.on_commit(lambda: task.delay(*args, **kwargs))
def __call__(self, *args, **kwargs):
"""Execute task."""
with self.app.flask_app.app_context():
return Task.__call__(self, *args, **kwargs)
def __call__(self, *args: list, **kwargs: dict) -> Any:
with app.app_context():
return Task.__call__(self, *args, **kwargs)
def __call__(self, *args, **kwargs):
if flask_current_app:
return Task.__call__(self, *args, **kwargs)
with self.app.flask_app.app_context():
return Task.__call__(self, *args, **kwargs)
def __init__(self):
Task.__init__(self)
self._handlers: Handlers = None
self._name: str = None
self.core: PC2Core = None
self._handler: Handler = None
def __init__(self):
Task.__init__(self)
def apply_async(self, *args, **kwargs):
kwargs.update(self._gen_task_id())
return Task.apply_async(self, *args, **kwargs)
def __call__(self, *args, **kwargs):
"""Execute task."""
with self.app.flask_app.app_context():
return Task.__call__(self, *args, **kwargs)
def schedule(force=False):
for site in Site.objects.all():
if site.needs_check:
Task.delay(check, site.id)
def __call__(self, *args, **kwargs):
if flask_current_app:
return Task.__call__(self, *args, **kwargs)
with self.app.flask_app.app_context():
return Task.__call__(self, *args, **kwargs)
def __init__(self, request=None):
Task.__init__(self)
if request:
self.req = ProcessedRequest(request)
def __init__(self):
Task.__init__(self)
# Shared namespace
self.worker_tracker = {'last_req_at': datetime.now(), 'last_req_time': 0, 'throughput': 0}
def __call__(self, *args, **kwargs):
with app.app_context():
return Task.__call__(self, *args, **kwargs)
def __call__(self, *args, **kwargs):
with app.app_context():
return Task.__call__(self, *args, **kwargs)