def ready(self): from django.db.models.signals import post_delete, post_save from bread.utils.celery import RepeatedTask post_save.connect(save_handler, dispatch_uid="trigger_save") post_delete.connect(delete_handler, dispatch_uid="trigger_delete") shared_task(base=RepeatedTask, run_every=TRIGGER_PERIOD)(periodic_trigger)
def short_doctask(*args, **kwargs): return shared_task(*args, bind=True, on_failure=on_failure, soft_time_limit=60, hard_time_limit=90, **kwargs)
def __init__( self, *, signal: Type[Signal], func: Callable, parameters: FrozenSet[str], coalesce: bool, queue: Optional[str], lock_timeout: int, ): self.signal = signal self.func = func self.parameters = parameters self.coalesce = coalesce self.lock_timeout = lock_timeout autoretry_for = (LockError, ) if lock_timeout > 0 else None self._task: Task = shared_task( queue=queue, name= f"{signal.__name__}.receiver.{func.__module__}.{func.__qualname__}", typing=False, autoretry_for=autoretry_for, )(self.task)
def task(t): t = shared_task(t) def _task(*args, **kwargs): return t.delay(*args, **kwargs) return _task
def f(task): @wraps(task) def wrapped_task(self, *args, **kwargs): self.logger = create_logger(task) # Create hash of all args digest_base = "/".join(str(arg) for arg in args) keys = sorted(kwargs.keys()) digest_base += "//" + "/".join("{}={}".format(key, kwargs[key]) for key in keys) digest = hashlib.sha256(digest_base.encode('utf-8')).hexdigest() lock_id = '{0}-lock-{1}'.format(self.name, digest) if cache.add(lock_id, 'true', LOCK_EXPIRE): try: return task(self, *args, **kwargs) except Exception as e: self.logger.error( "An exception occured when executing with args: %r kwargs: %r", args, kwargs) self.logger.exception(e) raise finally: cache.delete(lock_id) elif wait: self.logger.info( "Waiting for another instance of the task to end.") self.retry(args=args, countdown=wait_countdown, max_retries=wait_retries) return shared_task(*args, **kwargs)(wrapped_task)
def retriable_task( *args, bind=False, start=None, increment=0, retry_on=(Exception,), max=3600 * 12, min=0, exp_base=2, strategy=None, **kwargs, ): if strategy is None: strategy = retry_strategy( forward_self=bind, start=start, increment=increment, retry_on=retry_on, max=max, min=min, exp_base=exp_base, ) taskifier = shared_task(bind=True, **kwargs) def decorate(f): return taskifier(strategy(f)) if len(args) == 1: return decorate(args[0]) return decorate
def create_task(name): def task(*args, **options): r = call_command(name, *args, **options) return json.loads(r) task.__name__ = name return shared_task(task)
def _get_task(self, options: dict) -> Task: """ This creates and returns a shared task. """ def closure(*args, instance: Message, **kwargs) -> Dict[str, Any]: """ This is the actual task, we need a closure so that ``self`` does not get serialized. """ """ The idea itself is that the ``self`` object that consume the task is not the same. """ if self.owner: instance = cattr.structure(instance, self.owner) data = {} for name, callback in self.__callbacks.items(): data[name] = callback(instance, *args, **kwargs) return data else: raise AttributeError( f'No owner registered! ' f'Probably no callback was registered for ' f'{self.routing_key}') task = shared_task(**options)(closure) return task
def _retriable_shared_task(function): return shared_task(function, autoretry_for=autoretry_for, max_retries=max_retries, retry_backoff=retry_backoff, retry_jitter=retry_jitter, retry_backoff_max=retry_backoff_max, **kwargs)
def wrapper(wrapped): if task_name is None: name = '.'.join([wrapped.__module__, wrapped.__name__]) else: name = task_name func = transaction_handler(klass, wrapped) task = shared_task(name=name, base=MorpTask)(func) klass._celery_subscribe(signal)(task) return task
def wrapper(wrapped): func = periodic_transaction_handler(klass, wrapped) task_name = '.'.join([wrapped.__module__, wrapped.__name__]) task = shared_task(name=task_name, base=MorpTask)(func) klass.celery.conf.beat_schedule[name] = { 'task': task_name, 'schedule': seconds } return task
def extended_shared_task_decorator(func): func = shared_task(func, *decorator_args, **decorator_kwargs) def _wrapped(*args, **kwargs): if settings.USE_CELERY: return func.delay(*args, **kwargs) return func(*args, **kwargs) return _wrapped
def __init__(self, target): if settings.ASYNC_TASK == AsyncTask.CELERY.value: try: import celery except ImportError: raise ImportError('Async task is enabled and set to use celery,' ' but it is not installed.') self.target = celery.shared_task(target) else: self.target = target
def block_structure_task(**kwargs): """ Decorator for block structure tasks. """ return shared_task( default_retry_delay=settings. BLOCK_STRUCTURES_SETTINGS['TASK_DEFAULT_RETRY_DELAY'], max_retries=settings.BLOCK_STRUCTURES_SETTINGS['TASK_MAX_RETRIES'], bind=True, **kwargs)
def task(func, *args, **kwargs): """ A task decorator that uses the django-cereal pickler as the default serializer. """ # Note we must import here to avoid recursion issue with kombu entry points registration from celery import shared_task if 'serializer' not in kwargs: kwargs['serializer'] = DJANGO_CEREAL_PICKLE return shared_task(func, *args, **kwargs)
def __init__(self, target): if settings.ASYNC_TASK == AsyncTask.CELERY.value: try: import celery except ImportError: raise ImportError( 'Async task is enabled and set to use celery,' ' but it is not installed.') self.target = celery.shared_task(target) else: self.target = target
def task(func, *args, **kwargs): """ A task decorator that uses the django-cereal pickler as the default serializer. """ # Note we must import here to avoid recursion issue with kombu entry points registration from celery import shared_task if "serializer" not in kwargs: kwargs["serializer"] = DJANGO_CEREAL_PICKLE return shared_task(func, *args, **kwargs)
def task_decorator(func): # todo: what about routing keys? exchanges? exchange = kwargs.pop('exchange', None) queue = kwargs.pop('queue', None) routing_key = kwargs.pop('routing_key', None) # todo: use celery.current_app.task? func = celery.shared_task(*args, **kwargs)(func) self._tasks_to_register[func.name] = (queue, exchange, routing_key) return func
def wrapper(wrapped): func = periodic_transaction_handler(klass, wrapped) task_name = '.'.join([wrapped.__module__, wrapped.__name__]) task = shared_task(name=task_name, base=MorpTask)(func) klass.celery.conf.beat_schedule[name] = { 'task': task_name, 'schedule': crontab(minute=minute, hour=hour, day_of_week=day_of_week, day_of_month=day_of_month, month_of_year=month_of_year) } return task
def __call__(self, func): @functools.wraps(func) def wrapper_func(*args, **kwargs): try: with transaction.atomic(): return func(*args, **kwargs) except Exception as e: # task_func.request.retries raise task_func.retry(exc=e, countdown=5) task_func = shared_task(*self.task_args, **self.task_kwargs)(wrapper_func) return task_func
def create_geocoder(model): def geocode_model(pk): try: item = model.objects.get(pk=pk) except model.DoesNotExist: return if geocode_element(item): item.save() geocode_model.__name__ = "geocode_{}".format(model.__name__.lower()) return shared_task(geocode_model)
def f(task): @wraps(task) def wrapped_task(self, *args, **kwargs): self.logger = create_logger(task) try: return task(self, *args, **kwargs) except Exception as e: self.logger.error( "An exception occured when executing with args: %r kwargs: %r", args, kwargs) self.logger.exception(e) raise return shared_task(*args, **new_kwargs)(wrapped_task)
def ready(self): from django.db.models.signals import post_save from .models import Address, Phone, Vocabulary shared_task(base=RepeatedTask, run_every=timedelta(hours=6))(update_addresses) def saveperson(sender, instance, **kwargs): instance.person.save() post_save.connect(saveperson, Phone, dispatch_uid="save_person_phone") post_save.connect(saveperson, Address, dispatch_uid="save_person_address") pre_installed_vocabulary = { "tag": _("Tags"), "title": _("Titles"), "salutation": _("Salutations"), "form_of_address": _("Forms Of Address"), "gender": _("Genders"), "naturaltype": _("Natural Person Types"), "legaltype": _("Legal Person Types"), "associationtype": _("Association Person Types"), "addressstatus": _("Address statuses"), "emailtype": _("Email Types"), "urltype": _("URL Types"), "phonetype": _("Phone Types"), "addresstype": _("Address Types"), } try: for slug, name in pre_installed_vocabulary.items(): Vocabulary.objects.get_or_create(slug=slug, defaults={"name": name}) except Exception: # nosec this is trivial, no service interuption pass
def shared_task_email(func): """ Replacement for @shared_task decorator that emails admins if an exception is raised. """ @wraps(func) def new_func(*args, **kwargs): try: return func(*args, **kwargs) except: subject = "Celery task failure" message = traceback.format_exc() mail_admins(subject, message) raise return shared_task(new_func)
def _decorator(func): # Convert normal function to celery task func_celery = shared_task(func, **kwargs) # Connect to a signal if isinstance(signal, (list, tuple)): for s in signal: # Weak is false as func_celery doesn't exists outside the closure scope. So cannot # be referenced weakly and will be erased by garbage collector s.connect(func_celery.delay, sender=sender) else: signal.connect(func_celery.delay, sender=sender) # To let celery recognize normal function as celery task return func_celery
def test_get_task_key(task_key_name, task_key_args, task_key_kwargs, task_name, task_args, task_kwargs, task_key): def task_func(*args, **kwargs): pass task_class = shared_task( base=LockedTask, name=task_name, task_key_name=task_key_name, task_key_args=task_key_args, task_key_kwargs=task_key_kwargs, )(task_func) generated_task_key = task_class._get_task_key(*task_args, **task_kwargs) assert generated_task_key == task_key
def create_geocoder(model): def geocode_model(self, pk): try: item = model.objects.get(pk=pk) except model.DoesNotExist: return try: geocode_element(item) item.save() except (ValueError, requests.RequestException) as exc: self.retry(countdown=60, exc=exc) geocode_model.__name__ = "geocode_{}".format(model.__name__.lower()) return shared_task(geocode_model, bind=True)
def __call__(self, func): @functools.wraps(func) def wrapper_func(*args, **kwargs): try: with transaction.atomic(): # add Django db transaction support return func(*args, **kwargs) except self.EXCEPTION_BLOCK_LIST: # do not retry for those exceptions raise except Exception as e: # here we add Exponential Backoff just like Celery countdown = self._get_retry_countdown(task_func) raise task_func.retry(exc=e, countdown=countdown) task_func = shared_task(*self.task_args, **self.task_kwargs)(wrapper_func) return task_func
def adagetask(func): """ Decorator that adds a 's' attribute to a function The attribute can be used to fully define a function call to be executed at a later time. The result will be a zero-argument callable """ try: from celery import shared_task func.celery = shared_task(func) except ImportError: pass def partial(*args,**kwargs): return functools.partial(func,*args,**kwargs) func.s = partial return func
def decorator(func): if settings.NO_CELERY: setattr(func, 'delay', func) return func # Get the real decorator dec = shared_task(func, **kwargs) # Create a stub to apply a countdown and run def patched(*args, **kwargs): return dec.apply_async(args=args, kwargs=kwargs, countdown=settings.CELERY_DELAY) # Substitute the delay method dec.delay = patched return dec
def adagetask(func): """ Decorator that adds a 's' attribute to a function The attribute can be used to fully define a function call to be executed at a later time. The result will be a zero-argument callable """ try: from celery import shared_task func.celery = shared_task(func) except ImportError: pass def partial(*args, **kwargs): return functools.partial(func, *args, **kwargs) func.s = partial return func
def __init__(self, app, queue, broadcast_task_base=None): def _broadcast(**kwargs): """Broadcasts an event by calling the registered tasks.""" app_name = kwargs.pop('app_name', None) event_name = kwargs.pop('event_name', None) if app_name and event_name: event = app.registry.event(app_name, event_name, raise_does_not_exist=True) app.update_local_event(event) for task in event.tasks: signature(task.name, kwargs=kwargs, queue=task.queue).delay() self.queue = queue self.broadcast_func = _broadcast self.broadcast_task = shared_task(base=broadcast_task_base)(_broadcast)
def wrapper(func): task_name = func.__name__ wrapped_task = shared_task(name='publication.utils.%s' % task_name, ignore_result=True, filter=task_method, **task_kwargs)(publication_command_internal(func)) globals()[task_name] = wrapped_task @wraps(func) def wrapped(self, async=False, *args, **kwargs): if change_status_to is not None or async: self.rebind_or_create_publication() if change_status_to is not None: self.update(status=change_status_to) if not async: return func(self, *args, **kwargs) else: queue_name = getattr(self.driver, '%s_queue' % queue, 'publisher') return wrapped_task.apply_async(args=((self,) + args), kwargs=kwargs, queue=queue_name)
def _nonoverlapping_task(task_func): def wrapper(*exec_args, **exec_kwargs): r = get_redis_connection() task_name = task_kwargs.get('name', task_func.__name__) # lock key can be provided or defaults to celery-task-lock:<task_name> lock_key = task_kwargs.pop('lock_key', 'celery-task-lock:' + task_name) # lock timeout can be provided or defaults to task hard time limit lock_timeout = task_kwargs.pop('lock_timeout', None) if lock_timeout is None: lock_timeout = task_kwargs.get('time_limit', DEFAULT_TASK_LOCK_TIMEOUT) if r.get(lock_key): print("Skipping task %s to prevent overlapping" % task_name) else: with r.lock(lock_key, timeout=lock_timeout): task_func(*exec_args, **exec_kwargs) return shared_task(*task_args, **task_kwargs)(wrapper)
# tasks.py # author: andrew young # email: [email protected] import functools try: from celery import shared_task except ImportError: # user hasn't installed celery... no biggy def shared_task(fn): @functools.wraps(fn) def wrap(*args, **kwargs): return fn(*args, **kwargs) return wrap def _queued_task(fn): @functools.wraps(fn) def wrap(*args, **kwargs): return fn(*args, **kwargs) return wrap indexing_task = shared_task(_queued_task) bulk_indexing_task = shared_task(_queued_task)
def wrapped_task(*args, **kwargs): if settings.USE_CELERY: from celery import shared_task return shared_task(func).apply_async(args, kwargs) else: return func(*args, **kwargs)
issue.save() # Fetches the production team group production_team = get_production_team_group() if production_team is None: return # Now fetches the issue submissions that will soon be deleted. The production team must be # informed that the deletion will occur in a few days. email_limit_dt = now_dt - dt.timedelta(days=editor_settings.ARCHIVE_DAY_OFFSET - 5) email_limit_dt_range = [ email_limit_dt.replace(hour=0, minute=0, second=0, microsecond=0), email_limit_dt.replace(hour=23, minute=59, second=59, microsecond=999999), ] issue_submissions_to_email = IssueSubmission.objects.filter( status=IssueSubmission.VALID, date_modified__range=email_limit_dt_range) if issue_submissions_to_email.exists(): emails = production_team.user_set.values_list('email', flat=True) if not emails: return email = Email( list(emails), html_template='emails/editor/issue_files_deletion_content.html', subject_template='emails/editor/issue_files_deletion_subject.html', extra_context={'issue_submissions': issue_submissions_to_email}) email.send() handle_issuesubmission_files_removal = shared_task(_handle_issuesubmission_files_removal)
def managed_task(*args, **kwargs): return shared_task(base=ManagedTask, *args, **kwargs)
def import_social_content(): """ Iterate over all active social accounts and import post data with matching social service class. """ for social_account in SocialAccount.active.all(): try: service = get_service_class_by_name(social_account.social_content_type) except ServiceDoesNotExist, e: logger.error(str(e), exc_info=True) continue importer = service(social_account.raw_identifier or social_account.identifier, social_account_id=social_account.pk) # Tell importer to do magic. try: num_models_created = importer.import_posts() logger.info('Created %s %s social posts for id: %s' % (num_models_created, social_account.social_content_type, social_account.identifier,)) except HTTPError, e: logger.error('Social import error for %s account with identifier %s: %s' % (social_account.social_content_type, social_account.identifier, str(e)), exc_info=True) # If error, deactivate the account and store the error message social_account.status = SocialAccount.STATUS.inactive social_account.last_import_error = str(e) social_account.save() if shared_task: import_social_content = shared_task(import_social_content)
def _org_task(task_func): def _decorator(org_id): org = apps.get_model("orgs", "Org").objects.get(pk=org_id) maybe_run_for_org(org, task_func, task_key, lock_timeout) return shared_task(wraps(task_func)(_decorator))
def doctask(*args, **kwargs): return shared_task(*args, bind=True, on_failure=on_failure, **kwargs)