Example #1
0
def push_to_orcid(sender, record, *args, **kwargs):
    """If needed, queue the push of the new changes to ORCID.
    """
    if not is_hep(record):
        return

    def _get_orcid(author_ids):
        for author_id in author_ids:
            if author_id.get('schema', '').lower() == 'orcid':
                return author_id['value']

    task_name = current_app.config['ORCID_PUSH_TASK_ENDPOINT']
    authors = record.get('authors', ())

    for author in authors:
        orcid = _get_orcid(author.get('ids', ()))
        if not orcid:
            continue

        token = get_push_access_token(orcid)
        if token is None:
            continue

        push_to_orcid_task = Task()
        push_to_orcid_task.name = task_name
        push_to_orcid_task.apply_async(
            queue='orcid_push',
            kwargs={
                'orcid': orcid,
                'rec_id': record['control_number'],
                'token': token,
            },
        )
Example #2
0
 def create_for_task(cls, celecry_task: Task, a_id, task_name=''):
     """创建关联任务并设置初始进度"""
     task_name = task_name or getattr(celecry_task.request, 'task')
     a_task_result = AssociatedTaskResult.create(a_type_name=task_name,
                                                 a_id=a_id)
     a_task_result.add_task_result(celecry_task.request.id)
     celecry_task.update_state(state=states.PROGRESS, request=celecry_task.request)
Example #3
0
 def __call__(self, *args, **kwargs):
     """Execute task."""
     # If an "app_context" has already been loaded, just pass through
     if flask._app_ctx_stack.top is not None:
         return Task.__call__(self, *args, **kwargs)
     with self.app.flask_app.app_context():
         return Task.__call__(self, *args, **kwargs)
Example #4
0
def push_to_orcid(sender, record, *args, **kwargs):
    """If needed, queue the push of the new changes to ORCID."""
    if not is_hep(record) or not current_app.config[
            'FEATURE_FLAG_ENABLE_ORCID_PUSH']:
        return

    # Ensure there is a control number. This is not always the case because of broken store_record.
    if 'control_number' not in record:
        return

    task_name = current_app.config['ORCID_PUSH_TASK_ENDPOINT']

    orcids = get_orcids_for_push(record)
    orcids_and_tokens = get_push_access_tokens(orcids)
    for orcid, access_token in orcids_and_tokens:
        push_to_orcid_task = Task()
        push_to_orcid_task.name = task_name
        push_to_orcid_task.apply_async(
            queue='orcid_push',
            kwargs={
                'orcid': orcid,
                'rec_id': record['control_number'],
                'oauth_token': access_token,
            },
        )
Example #5
0
 def wrapper(task: Task, *args):
     task.max_retries = times
     try:
         result = func(task, *args)
     except Exception as exc:
         raise task.retry(exc=exc, countdown=5)
     else:
         return result
Example #6
0
    def __init__(self):
        Task.__init__(self)

        # Shared namespace
        self.worker_tracker = {
            'last_req_at': datetime.now(),
            'last_req_time': 0,
            'throughput': 0
        }
Example #7
0
    def __init__(self, *args, **kwargs):
        Task.__init__(self, *args, **kwargs)

        self.logger = logger

        try:
            self._redis = self.app.api_manager.redis_taskmanager.conn
        except BaseException:
            self._redis = None
Example #8
0
 def __init__(self, *args, **kwargs):
     Task.__init__(self, *args, **kwargs)
     #task_short_name = {u'task_name':self.name.split(u'.')[-1]}
     #logger = logging.LoggerAdapter(get_task_logger(__name__), task_short_name)
     self.logger = logger
     
     try:
         self._redis = self.app.api_manager.redis_taskmanager.conn
     except:
         self._redis = None
Example #9
0
def map_task_per_tenants(task: Task, sequential: bool = False) -> None:
    # todo: filtering by tenant status
    tenants_ids = list(
        TenantModel.objects.exclude(
            schema_name=public_schema_name).values_list('id', flat=True))
    if sequential:
        logger.info("Submitting tasks for tenants: %s" %
                    str(task.map(tenants_ids).apply_async()))
    else:
        logger.info("Submitting tasks for tenants: %s" % ', '.join(
            [str(task.delay(tenant_id)) for tenant_id in tenants_ids]))
Example #10
0
 def __call__(self, *args, **kwargs):
     # from windar_api.db_util.feature_flags import get_all_flags_from_celery_task
     # fflags = get_all_flags_from_celery_task(self)
     rq = self.request
     with app.test_request_context():
         log.info('Start zadania %s (id: %s)', self.name, rq.id)
         log.debug('Parametry zadania %s\nargs: %r\nkwargs: %r', rq.id, args, kwargs)
         # if fflags:
         #     log.debug('Feature flags wymuszone przez headers: %r', fflags)
         try:
             ret = Task.__call__(self, *args, **kwargs)
             db.session.commit()
             # db_kplus.session.commit()
         except Retry:
             db.session.rollback()
             # db_kplus.session.rollback()
             raise
         except Exception:
             db.session.rollback()
             # db_kplus.session.rollback()
             log.error('Blad w trakcie wykonywania zadania %s (id: %s)', self.name, rq.id, exc_info=True)
             raise
         else:
             log.info('Koniec zadania %s (id: %s)', self.name, rq.id)
             return ret
def retry_task(
        task: Task,
        exception: Optional[Exception] = None,
        max_attempts: Optional[int] = None,
) -> None:
    """
    Retry celery tasks.

    Retries the specified task using a "backing off countdown",
    meaning that the interval between retries grows exponentially
    with every retry.

    :param task: The task to retry.
    :param exception: Optionally, the exception that caused the retry.
    :param max_attempts: Optionally, number of maximum attempts to retry the task.
    """

    def backoff(attempts: int) -> int:
        return 5 * (2 ** attempts)

    kwargs: dict = {
        "countdown": backoff(task.request.retries),
    }

    if exception:
        kwargs["exc"] = exception

    if not max_attempts or task.request.retries < max_attempts:
        raise task.retry(**kwargs)
Example #12
0
 def __call__(self, *args, **kwargs):
     """Execute task."""
     with self.app.flask_app.test_request_context():
         self.app.flask_app.try_trigger_before_first_request_functions()
         self.app.flask_app.preprocess_request()
         res = Task.__call__(self, *args, **kwargs)
         self.app.flask_app.process_response(
             self.app.flask_app.response_class())
         return res
Example #13
0
 def __call__(self, *args, **kwargs):
     """Execute task."""
     with self.app.flask_app.test_request_context():
         self.app.flask_app.try_trigger_before_first_request_functions()
         self.app.flask_app.preprocess_request()
         res = Task.__call__(self, *args, **kwargs)
         self.app.flask_app.process_response(
             self.app.flask_app.response_class())
         return res
Example #14
0
    def __init__(self, *args, **kwargs):
        Task.__init__(self, *args, **kwargs)

        self.name = self.__class__.__module__ + '.' + self.name

        self.logger = logger
        self.steps = []
        self.task_result = TaskResult()

        self.objid = None
        self.objtype = None
        self.objdef = None
        self.op = None
        self.opid = None
        self.delta = None
        self.user = None
        self.api_id = None

        if self.entity_class is not None:
            self.objtype = self.entity_class.objtype
            self.objdef = self.entity_class.objdef
Example #15
0
 def send(self, message):
     # 生产者配置
     app = Celery(broker=celery_config.BROKER_URL)
     task = Task()
     task.name = celery_config.REQUEST_TASK
     task.bind(app)
     task.apply_async([message],
                      exchange=celery_config.REQUEST_NAME,
                      routing_key=celery_config.ROUNTING_KEY,
                      expires=60 * 60 * 24)
Example #16
0
def update_progress(task: Task, new_status: int):
    """ Set the current progress for the task object (celery)

    Args:
        task (Task):
        new_status (int): The progress bar
    Returns:

    """
    if new_status < 0 or new_status > 100:
        raise ValueError("new_status must be in range [0, 100]")

    try:
        pending_task = PendingTask.objects.get(task_id=task.request.id)
        pending_task.progress = new_status
        pending_task.save()
    except ObjectDoesNotExist:
        pass

    task.update_state(state='PROGRESS',
                      meta={
                          'current': new_status,
                          'total': 100,
                      })
def run_task_on_commit(task: Task, args: Iterable = [], kwargs: Mapping = {}):
    """Run a celery task after the next database commit (if we are in a transaction)"""
    transaction.on_commit(lambda: task.delay(*args, **kwargs))
Example #18
0
 def __call__(self, *args, **kwargs):
     """Execute task."""
     with self.app.flask_app.app_context():
         return Task.__call__(self, *args, **kwargs)
Example #19
0
 def __call__(self, *args: list, **kwargs: dict) -> Any:
     with app.app_context():
         return Task.__call__(self, *args, **kwargs)
Example #20
0
 def __call__(self, *args, **kwargs):
     if flask_current_app:
         return Task.__call__(self, *args, **kwargs)
     with self.app.flask_app.app_context():
         return Task.__call__(self, *args, **kwargs)
Example #21
0
 def __init__(self):
     Task.__init__(self)
     self._handlers: Handlers = None
     self._name: str = None
     self.core: PC2Core = None
     self._handler: Handler = None
Example #22
0
 def __init__(self):
     Task.__init__(self)
Example #23
0
 def apply_async(self, *args, **kwargs):
     kwargs.update(self._gen_task_id())
     return Task.apply_async(self, *args, **kwargs)
Example #24
0
 def __call__(self, *args, **kwargs):
     """Execute task."""
     with self.app.flask_app.app_context():
         return Task.__call__(self, *args, **kwargs)
Example #25
0
def schedule(force=False):
    for site in Site.objects.all():
        if site.needs_check:
            Task.delay(check, site.id)
Example #26
0
 def __call__(self, *args, **kwargs):
     if flask_current_app:
         return Task.__call__(self, *args, **kwargs)
     with self.app.flask_app.app_context():
         return Task.__call__(self, *args, **kwargs)
Example #27
0
 def __init__(self, request=None):
     Task.__init__(self)
     if request:
         self.req = ProcessedRequest(request)
Example #28
0
    def __init__(self):
        Task.__init__(self)

        # Shared namespace
        self.worker_tracker = {'last_req_at': datetime.now(), 'last_req_time': 0, 'throughput': 0}
Example #29
0
 def __call__(self, *args, **kwargs):
     with app.app_context():
         return Task.__call__(self, *args, **kwargs)
Example #30
0
 def __call__(self, *args, **kwargs):
     with app.app_context():
         return Task.__call__(self, *args, **kwargs)