def ready(self):
from django.db.models.signals import post_delete, post_save
from bread.utils.celery import RepeatedTask
post_save.connect(save_handler, dispatch_uid="trigger_save")
post_delete.connect(delete_handler, dispatch_uid="trigger_delete")
shared_task(base=RepeatedTask,
run_every=TRIGGER_PERIOD)(periodic_trigger)
def short_doctask(*args, **kwargs):
return shared_task(*args,
bind=True,
on_failure=on_failure,
soft_time_limit=60,
hard_time_limit=90,
**kwargs)
def __init__(
self,
*,
signal: Type[Signal],
func: Callable,
parameters: FrozenSet[str],
coalesce: bool,
queue: Optional[str],
lock_timeout: int,
):
self.signal = signal
self.func = func
self.parameters = parameters
self.coalesce = coalesce
self.lock_timeout = lock_timeout
autoretry_for = (LockError, ) if lock_timeout > 0 else None
self._task: Task = shared_task(
queue=queue,
name=
f"{signal.__name__}.receiver.{func.__module__}.{func.__qualname__}",
typing=False,
autoretry_for=autoretry_for,
)(self.task)
def task(t):
t = shared_task(t)
def _task(*args, **kwargs):
return t.delay(*args, **kwargs)
return _task
def f(task):
@wraps(task)
def wrapped_task(self, *args, **kwargs):
self.logger = create_logger(task)
# Create hash of all args
digest_base = "/".join(str(arg) for arg in args)
keys = sorted(kwargs.keys())
digest_base += "//" + "/".join("{}={}".format(key, kwargs[key])
for key in keys)
digest = hashlib.sha256(digest_base.encode('utf-8')).hexdigest()
lock_id = '{0}-lock-{1}'.format(self.name, digest)
if cache.add(lock_id, 'true', LOCK_EXPIRE):
try:
return task(self, *args, **kwargs)
except Exception as e:
self.logger.error(
"An exception occured when executing with args: %r kwargs: %r",
args, kwargs)
self.logger.exception(e)
raise
finally:
cache.delete(lock_id)
elif wait:
self.logger.info(
"Waiting for another instance of the task to end.")
self.retry(args=args,
countdown=wait_countdown,
max_retries=wait_retries)
return shared_task(*args, **kwargs)(wrapped_task)
def retriable_task(
*args,
bind=False,
start=None,
increment=0,
retry_on=(Exception,),
max=3600 * 12,
min=0,
exp_base=2,
strategy=None,
**kwargs,
):
if strategy is None:
strategy = retry_strategy(
forward_self=bind,
start=start,
increment=increment,
retry_on=retry_on,
max=max,
min=min,
exp_base=exp_base,
)
taskifier = shared_task(bind=True, **kwargs)
def decorate(f):
return taskifier(strategy(f))
if len(args) == 1:
return decorate(args[0])
return decorate
def create_task(name):
def task(*args, **options):
r = call_command(name, *args, **options)
return json.loads(r)
task.__name__ = name
return shared_task(task)
def _get_task(self, options: dict) -> Task:
"""
This creates and returns a shared task.
"""
def closure(*args, instance: Message, **kwargs) -> Dict[str, Any]:
"""
This is the actual task, we need a closure so that
``self`` does not get serialized.
"""
"""
The idea itself is that the ``self`` object that consume the task
is not the same.
"""
if self.owner:
instance = cattr.structure(instance, self.owner)
data = {}
for name, callback in self.__callbacks.items():
data[name] = callback(instance, *args, **kwargs)
return data
else:
raise AttributeError(
f'No owner registered! '
f'Probably no callback was registered for '
f'{self.routing_key}')
task = shared_task(**options)(closure)
return task
def _retriable_shared_task(function):
return shared_task(function,
autoretry_for=autoretry_for,
max_retries=max_retries,
retry_backoff=retry_backoff,
retry_jitter=retry_jitter,
retry_backoff_max=retry_backoff_max,
**kwargs)
def wrapper(wrapped):
if task_name is None:
name = '.'.join([wrapped.__module__, wrapped.__name__])
else:
name = task_name
func = transaction_handler(klass, wrapped)
task = shared_task(name=name, base=MorpTask)(func)
klass._celery_subscribe(signal)(task)
return task
def wrapper(wrapped):
func = periodic_transaction_handler(klass, wrapped)
task_name = '.'.join([wrapped.__module__, wrapped.__name__])
task = shared_task(name=task_name, base=MorpTask)(func)
klass.celery.conf.beat_schedule[name] = {
'task': task_name,
'schedule': seconds
}
return task
def extended_shared_task_decorator(func):
func = shared_task(func, *decorator_args, **decorator_kwargs)
def _wrapped(*args, **kwargs):
if settings.USE_CELERY:
return func.delay(*args, **kwargs)
return func(*args, **kwargs)
return _wrapped
def __init__(self, target):
if settings.ASYNC_TASK == AsyncTask.CELERY.value:
try:
import celery
except ImportError:
raise ImportError('Async task is enabled and set to use celery,'
' but it is not installed.')
self.target = celery.shared_task(target)
else:
self.target = target
def block_structure_task(**kwargs):
"""
Decorator for block structure tasks.
"""
return shared_task(
default_retry_delay=settings.
BLOCK_STRUCTURES_SETTINGS['TASK_DEFAULT_RETRY_DELAY'],
max_retries=settings.BLOCK_STRUCTURES_SETTINGS['TASK_MAX_RETRIES'],
bind=True,
**kwargs)
def task(func, *args, **kwargs):
"""
A task decorator that uses the django-cereal pickler as the default serializer.
"""
# Note we must import here to avoid recursion issue with kombu entry points registration
from celery import shared_task
if 'serializer' not in kwargs:
kwargs['serializer'] = DJANGO_CEREAL_PICKLE
return shared_task(func, *args, **kwargs)
def __init__(self, target):
if settings.ASYNC_TASK == AsyncTask.CELERY.value:
try:
import celery
except ImportError:
raise ImportError(
'Async task is enabled and set to use celery,'
' but it is not installed.')
self.target = celery.shared_task(target)
else:
self.target = target
def task(func, *args, **kwargs):
"""
A task decorator that uses the django-cereal pickler as the default serializer.
"""
# Note we must import here to avoid recursion issue with kombu entry points registration
from celery import shared_task
if "serializer" not in kwargs:
kwargs["serializer"] = DJANGO_CEREAL_PICKLE
return shared_task(func, *args, **kwargs)
def task_decorator(func):
# todo: what about routing keys? exchanges?
exchange = kwargs.pop('exchange', None)
queue = kwargs.pop('queue', None)
routing_key = kwargs.pop('routing_key', None)
# todo: use celery.current_app.task?
func = celery.shared_task(*args, **kwargs)(func)
self._tasks_to_register[func.name] = (queue, exchange, routing_key)
return func
def wrapper(wrapped):
func = periodic_transaction_handler(klass, wrapped)
task_name = '.'.join([wrapped.__module__, wrapped.__name__])
task = shared_task(name=task_name, base=MorpTask)(func)
klass.celery.conf.beat_schedule[name] = {
'task': task_name,
'schedule': crontab(minute=minute, hour=hour,
day_of_week=day_of_week,
day_of_month=day_of_month,
month_of_year=month_of_year)
}
return task
def __call__(self, func):
@functools.wraps(func)
def wrapper_func(*args, **kwargs):
try:
with transaction.atomic():
return func(*args, **kwargs)
except Exception as e:
# task_func.request.retries
raise task_func.retry(exc=e, countdown=5)
task_func = shared_task(*self.task_args,
**self.task_kwargs)(wrapper_func)
return task_func
def create_geocoder(model):
def geocode_model(pk):
try:
item = model.objects.get(pk=pk)
except model.DoesNotExist:
return
if geocode_element(item):
item.save()
geocode_model.__name__ = "geocode_{}".format(model.__name__.lower())
return shared_task(geocode_model)
def f(task):
@wraps(task)
def wrapped_task(self, *args, **kwargs):
self.logger = create_logger(task)
try:
return task(self, *args, **kwargs)
except Exception as e:
self.logger.error(
"An exception occured when executing with args: %r kwargs: %r",
args, kwargs)
self.logger.exception(e)
raise
return shared_task(*args, **new_kwargs)(wrapped_task)
def ready(self):
from django.db.models.signals import post_save
from .models import Address, Phone, Vocabulary
shared_task(base=RepeatedTask,
run_every=timedelta(hours=6))(update_addresses)
def saveperson(sender, instance, **kwargs):
instance.person.save()
post_save.connect(saveperson, Phone, dispatch_uid="save_person_phone")
post_save.connect(saveperson,
Address,
dispatch_uid="save_person_address")
pre_installed_vocabulary = {
"tag": _("Tags"),
"title": _("Titles"),
"salutation": _("Salutations"),
"form_of_address": _("Forms Of Address"),
"gender": _("Genders"),
"naturaltype": _("Natural Person Types"),
"legaltype": _("Legal Person Types"),
"associationtype": _("Association Person Types"),
"addressstatus": _("Address statuses"),
"emailtype": _("Email Types"),
"urltype": _("URL Types"),
"phonetype": _("Phone Types"),
"addresstype": _("Address Types"),
}
try:
for slug, name in pre_installed_vocabulary.items():
Vocabulary.objects.get_or_create(slug=slug,
defaults={"name": name})
except Exception: # nosec this is trivial, no service interuption
pass
def shared_task_email(func):
"""
Replacement for @shared_task decorator that emails admins if an exception is raised.
"""
@wraps(func)
def new_func(*args, **kwargs):
try:
return func(*args, **kwargs)
except:
subject = "Celery task failure"
message = traceback.format_exc()
mail_admins(subject, message)
raise
return shared_task(new_func)
def _decorator(func):
# Convert normal function to celery task
func_celery = shared_task(func, **kwargs)
# Connect to a signal
if isinstance(signal, (list, tuple)):
for s in signal:
# Weak is false as func_celery doesn't exists outside the closure scope. So cannot
# be referenced weakly and will be erased by garbage collector
s.connect(func_celery.delay, sender=sender)
else:
signal.connect(func_celery.delay, sender=sender)
# To let celery recognize normal function as celery task
return func_celery
def test_get_task_key(task_key_name, task_key_args, task_key_kwargs, task_name,
task_args, task_kwargs, task_key):
def task_func(*args, **kwargs):
pass
task_class = shared_task(
base=LockedTask,
name=task_name,
task_key_name=task_key_name,
task_key_args=task_key_args,
task_key_kwargs=task_key_kwargs,
)(task_func)
generated_task_key = task_class._get_task_key(*task_args, **task_kwargs)
assert generated_task_key == task_key
def create_geocoder(model):
def geocode_model(self, pk):
try:
item = model.objects.get(pk=pk)
except model.DoesNotExist:
return
try:
geocode_element(item)
item.save()
except (ValueError, requests.RequestException) as exc:
self.retry(countdown=60, exc=exc)
geocode_model.__name__ = "geocode_{}".format(model.__name__.lower())
return shared_task(geocode_model, bind=True)
def __call__(self, func):
@functools.wraps(func)
def wrapper_func(*args, **kwargs):
try:
with transaction.atomic():
# add Django db transaction support
return func(*args, **kwargs)
except self.EXCEPTION_BLOCK_LIST:
# do not retry for those exceptions
raise
except Exception as e:
# here we add Exponential Backoff just like Celery
countdown = self._get_retry_countdown(task_func)
raise task_func.retry(exc=e, countdown=countdown)
task_func = shared_task(*self.task_args, **self.task_kwargs)(wrapper_func)
return task_func
def adagetask(func):
"""
Decorator that adds a 's' attribute to a function
The attribute can be used to fully define a function
call to be executed at a later time. The result
will be a zero-argument callable
"""
try:
from celery import shared_task
func.celery = shared_task(func)
except ImportError:
pass
def partial(*args,**kwargs):
return functools.partial(func,*args,**kwargs)
func.s = partial
return func
def decorator(func):
if settings.NO_CELERY:
setattr(func, 'delay', func)
return func
# Get the real decorator
dec = shared_task(func, **kwargs)
# Create a stub to apply a countdown and run
def patched(*args, **kwargs):
return dec.apply_async(args=args,
kwargs=kwargs,
countdown=settings.CELERY_DELAY)
# Substitute the delay method
dec.delay = patched
return dec
def adagetask(func):
"""
Decorator that adds a 's' attribute to a function
The attribute can be used to fully define a function
call to be executed at a later time. The result
will be a zero-argument callable
"""
try:
from celery import shared_task
func.celery = shared_task(func)
except ImportError:
pass
def partial(*args, **kwargs):
return functools.partial(func, *args, **kwargs)
func.s = partial
return func
def __init__(self, app, queue, broadcast_task_base=None):
def _broadcast(**kwargs):
"""Broadcasts an event by calling the registered tasks."""
app_name = kwargs.pop('app_name', None)
event_name = kwargs.pop('event_name', None)
if app_name and event_name:
event = app.registry.event(app_name,
event_name,
raise_does_not_exist=True)
app.update_local_event(event)
for task in event.tasks:
signature(task.name, kwargs=kwargs,
queue=task.queue).delay()
self.queue = queue
self.broadcast_func = _broadcast
self.broadcast_task = shared_task(base=broadcast_task_base)(_broadcast)
def wrapper(func):
task_name = func.__name__
wrapped_task = shared_task(name='publication.utils.%s' % task_name, ignore_result=True, filter=task_method, **task_kwargs)(publication_command_internal(func))
globals()[task_name] = wrapped_task
@wraps(func)
def wrapped(self, async=False, *args, **kwargs):
if change_status_to is not None or async:
self.rebind_or_create_publication()
if change_status_to is not None:
self.update(status=change_status_to)
if not async:
return func(self, *args, **kwargs)
else:
queue_name = getattr(self.driver, '%s_queue' % queue, 'publisher')
return wrapped_task.apply_async(args=((self,) + args), kwargs=kwargs, queue=queue_name)
def _nonoverlapping_task(task_func):
def wrapper(*exec_args, **exec_kwargs):
r = get_redis_connection()
task_name = task_kwargs.get('name', task_func.__name__)
# lock key can be provided or defaults to celery-task-lock:<task_name>
lock_key = task_kwargs.pop('lock_key', 'celery-task-lock:' + task_name)
# lock timeout can be provided or defaults to task hard time limit
lock_timeout = task_kwargs.pop('lock_timeout', None)
if lock_timeout is None:
lock_timeout = task_kwargs.get('time_limit', DEFAULT_TASK_LOCK_TIMEOUT)
if r.get(lock_key):
print("Skipping task %s to prevent overlapping" % task_name)
else:
with r.lock(lock_key, timeout=lock_timeout):
task_func(*exec_args, **exec_kwargs)
return shared_task(*task_args, **task_kwargs)(wrapper)
def _nonoverlapping_task(task_func):
def wrapper(*exec_args, **exec_kwargs):
r = get_redis_connection()
task_name = task_kwargs.get('name', task_func.__name__)
# lock key can be provided or defaults to celery-task-lock:<task_name>
lock_key = task_kwargs.pop('lock_key', 'celery-task-lock:' + task_name)
# lock timeout can be provided or defaults to task hard time limit
lock_timeout = task_kwargs.pop('lock_timeout', None)
if lock_timeout is None:
lock_timeout = task_kwargs.get('time_limit', DEFAULT_TASK_LOCK_TIMEOUT)
if r.get(lock_key):
print("Skipping task %s to prevent overlapping" % task_name)
else:
with r.lock(lock_key, timeout=lock_timeout):
task_func(*exec_args, **exec_kwargs)
return shared_task(*task_args, **task_kwargs)(wrapper)
# tasks.py # author: andrew young # email: [email protected] import functools try: from celery import shared_task except ImportError: # user hasn't installed celery... no biggy def shared_task(fn): @functools.wraps(fn) def wrap(*args, **kwargs): return fn(*args, **kwargs) return wrap def _queued_task(fn): @functools.wraps(fn) def wrap(*args, **kwargs): return fn(*args, **kwargs) return wrap indexing_task = shared_task(_queued_task) bulk_indexing_task = shared_task(_queued_task)
def wrapped_task(*args, **kwargs):
if settings.USE_CELERY:
from celery import shared_task
return shared_task(func).apply_async(args, kwargs)
else:
return func(*args, **kwargs)
issue.save()
# Fetches the production team group
production_team = get_production_team_group()
if production_team is None:
return
# Now fetches the issue submissions that will soon be deleted. The production team must be
# informed that the deletion will occur in a few days.
email_limit_dt = now_dt - dt.timedelta(days=editor_settings.ARCHIVE_DAY_OFFSET - 5)
email_limit_dt_range = [
email_limit_dt.replace(hour=0, minute=0, second=0, microsecond=0),
email_limit_dt.replace(hour=23, minute=59, second=59, microsecond=999999),
]
issue_submissions_to_email = IssueSubmission.objects.filter(
status=IssueSubmission.VALID, date_modified__range=email_limit_dt_range)
if issue_submissions_to_email.exists():
emails = production_team.user_set.values_list('email', flat=True)
if not emails:
return
email = Email(
list(emails),
html_template='emails/editor/issue_files_deletion_content.html',
subject_template='emails/editor/issue_files_deletion_subject.html',
extra_context={'issue_submissions': issue_submissions_to_email})
email.send()
handle_issuesubmission_files_removal = shared_task(_handle_issuesubmission_files_removal)
def managed_task(*args, **kwargs):
return shared_task(base=ManagedTask, *args, **kwargs)
def import_social_content():
"""
Iterate over all active social accounts and import post data with matching
social service class.
"""
for social_account in SocialAccount.active.all():
try:
service = get_service_class_by_name(social_account.social_content_type)
except ServiceDoesNotExist, e:
logger.error(str(e), exc_info=True)
continue
importer = service(social_account.raw_identifier or social_account.identifier, social_account_id=social_account.pk)
# Tell importer to do magic.
try:
num_models_created = importer.import_posts()
logger.info('Created %s %s social posts for id: %s' % (num_models_created, social_account.social_content_type, social_account.identifier,))
except HTTPError, e:
logger.error('Social import error for %s account with identifier %s: %s' % (social_account.social_content_type, social_account.identifier, str(e)), exc_info=True)
# If error, deactivate the account and store the error message
social_account.status = SocialAccount.STATUS.inactive
social_account.last_import_error = str(e)
social_account.save()
if shared_task:
import_social_content = shared_task(import_social_content)
def _org_task(task_func):
def _decorator(org_id):
org = apps.get_model("orgs", "Org").objects.get(pk=org_id)
maybe_run_for_org(org, task_func, task_key, lock_timeout)
return shared_task(wraps(task_func)(_decorator))
def doctask(*args, **kwargs):
return shared_task(*args, bind=True, on_failure=on_failure, **kwargs)
