Ejemplo n.º 1
0
def update_zabbix_templates(device, templates=None, remote=False):
    hostname = device.name
    role = device.venture_role
    try:
        ip = device.ipaddress_set.get(hostname=hostname)
    except IPAddress.DoesNotExist:
        return
    if templates is None:
        templates = _calculate_zabbix_templates(role)
    if remote:
        task = zabbix_update_task.delay
    else:
        task = zabbix_update_task
    task(hostname, str(ip.address), templates)
Ejemplo n.º 2
0
def update_mangopay_pay_out(id):
    payout = MangoPayPayOut.objects.get(id=id, mangopay_id__isnull=False)
    try:
        payout = payout.get()
    except ResponseException as exc:
        raise update_mangopay_pay_out.retry((), {"id": id}, exc=exc)
    if not payout.status or payout.status == "CREATED":
        eta = next_weekday()
        update_mangopay_pay_out.apply_async((), {"id": id}, eta=eta)
    elif payout.status == "SUCCEEDED":
        task = getattr(settings, 'MANGOPAY_PAYOUT_SUCCEEDED_TASK', None)
        if task:
            task().run(payout_id=payout.id)
    else:
        logger.error("Payout %i could not be process successfully" % payout.id)
Ejemplo n.º 3
0
def update_mangopay_pay_out(id):
    payout = MangoPayPayOut.objects.get(id=id, mangopay_id__isnull=False)
    try:
        payout = payout.get()
    except ResponseException as exc:
        raise update_mangopay_pay_out.retry((), {"id": id}, exc=exc)
    if not payout.status or payout.status == "CREATED":
        eta = next_weekday()
        update_mangopay_pay_out.apply_async((), {"id": id}, eta=eta)
    elif payout.status == "SUCCEEDED":
        task = getattr(settings, 'MANGOPAY_PAYOUT_SUCCEEDED_TASK', None)
        if task:
            task().run(payout_id=payout.id)
    else:
        logger.error("Payout %i could not be process successfully" % payout.id)
Ejemplo n.º 4
0
 def __init__(self):
     try:
         from celery.task import task
     except:
         raise InvalidImageCacheBackendError("Celery image cache backend requires the 'celery' library")
     if not getattr(CeleryImageCacheBackend, '_task', None):
         CeleryImageCacheBackend._task = task(generate)
Ejemplo n.º 5
0
 def litetask_celery(task_func):
     """
     Wrapper around celery.task
     """
     tsk = task(litetask_futures(task_func), queue='celery')
     tsk.__func__ = tsk
     tsk.task_func = task_func
     return tsk
Ejemplo n.º 6
0
 def __init__(self):
     try:
         from celery.task import task
     except:
         raise InvalidImageCacheBackendError(
             "Celery image cache backend requires the 'celery' library")
     if not getattr(CeleryImageCacheBackend, '_task', None):
         CeleryImageCacheBackend._task = task(generate)
Ejemplo n.º 7
0
 def oqtask(task_func):
     """
     Wrapper around celery.task and parallel.litetask
     """
     tsk = task(litetask(task_func), queue=celery_queue)
     tsk.__func__ = tsk
     tsk.task_func = task_func
     return tsk
Ejemplo n.º 8
0
 def litetask_celery(task_func):
     """
     Wrapper around celery.task
     """
     tsk = task(litetask_futures(task_func), queue='celery')
     tsk.__func__ = tsk
     tsk.task_func = task_func
     return tsk
Ejemplo n.º 9
0
 def oqtask(task_func):
     """
     Wrapper around celery.task and parallel.litetask
     """
     tsk = task(litetask(task_func), queue=celery_queue)
     tsk.__func__ = tsk
     tsk.task_func = task_func
     return tsk
Ejemplo n.º 10
0
    def decorator(taskfunc):
        def db_constructor(job_id, *args, **kwargs):
            session = db_constructor.backend.ResultSession()
            zoomdb = ZoomDatabase(session.bind, job_id)
            return taskfunc(job_id, zoomdb, *args, **kwargs)

        db_constructor = task(**celery_kwargs)(db_constructor)

        return db_constructor
Ejemplo n.º 11
0
def test_task(func):
    @functools.wraps(func)
    def wrapper(*args, **kwargs):
        try:
            return func(*args, **kwargs), None
        except:
            exctype, exc, _tb = sys.exc_info()
            return str(exc), exctype
    return task(wrapper)
Ejemplo n.º 12
0
def oqtask(task_func):
    """
    Task function decorator which sets up logging and catches (and logs) any
    errors which occur inside the task. Also checks to make sure the job is
    actually still running. If it is not running, the task doesn't get
    executed, so we don't do useless computation.
    """

    @wraps(task_func)
    def wrapped(*args):
        """
        Initialize logs, make sure the job is still running, and run the task
        code surrounded by a try-except. If any error occurs, log it as a
        critical failure.
        """
        # job_id is always assumed to be the first argument
        job_id = args[0]
        job = models.OqJob.objects.get(id=job_id)
        if job.is_running is False:
            # the job was killed, it is useless to run the task
            return

        # it is important to save the task id soon, so that
        # the revoke functionality can work
        EnginePerformanceMonitor.store_task_id(job_id, tsk)

        with EnginePerformanceMonitor(
                'total ' + task_func.__name__, job_id, tsk, flush=True):

            with EnginePerformanceMonitor(
                    'loading calculation object', job_id, tsk, flush=True):
                calculation = job.calculation

            # tasks write on the celery log file
            logs.init_logs(
                level=job.log_level,
                calc_domain='hazard' if isinstance(
                    calculation, models.HazardCalculation) else'risk',
                calc_id=calculation.id)
            try:
                return task_func(*args), None
            except:
                etype, exc, tb = sys.exc_info()
                tb_str = ''.join(traceback.format_tb(tb))
                return '%s\n%s' % (exc, tb_str), etype
            finally:
                CacheInserter.flushall()
                # the task finished, we can remove from the performance
                # table the associated row 'storing task id'
                models.Performance.objects.filter(
                    oq_job=job,
                    operation='storing task id',
                    task_id=tsk.request.id).delete()
    celery_queue = config.get('amqp', 'celery_queue')
    tsk = task(wrapped, queue=celery_queue)
    tsk.task_func = task_func
    return tsk
Ejemplo n.º 13
0
 def wrapped(func):
     @wraps(func)
     def _wrapped(*args, **kwargs):
         statsd_key = 'jobs.duration.{name}'.format(name=name)
         if stat_suffix:
             statsd_key += '.{key}'.format(key=stat_suffix(*args, **kwargs))
         with statsd.timer(statsd_key):
             result = func(*args, **kwargs)
         return result
     return task(name=name, **kwargs)(_wrapped)
Ejemplo n.º 14
0
def block_structure_task(**kwargs):
    """
    Decorator for block structure tasks.
    """
    return task(
        default_retry_delay=settings.
        BLOCK_STRUCTURES_SETTINGS['TASK_DEFAULT_RETRY_DELAY'],
        max_retries=settings.BLOCK_STRUCTURES_SETTINGS['TASK_MAX_RETRIES'],
        bind=True,
        **kwargs)
Ejemplo n.º 15
0
def block_structure_task(**kwargs):
    """
    Decorator for block structure tasks.
    """
    return task(
        default_retry_delay=settings.BLOCK_STRUCTURES_SETTINGS['TASK_DEFAULT_RETRY_DELAY'],
        max_retries=settings.BLOCK_STRUCTURES_SETTINGS['TASK_MAX_RETRIES'],
        bind=True,
        **kwargs
    )
Ejemplo n.º 16
0
    def wrapped(func):
        @wraps(func)
        def _wrapped(*args, **kwargs):
            key = 'jobs.duration.{name}'.format(name=name)
            if stat_suffix:
                key += '.{key}'.format(key=stat_suffix(*args, **kwargs))
            with metrics.timer(key):
                result = func(*args, **kwargs)
            return result

        return task(name=name, **kwargs)(_wrapped)
Ejemplo n.º 17
0
def oqtask(task_func):
    """
    Task function decorator which sets up logging and catches (and logs) any
    errors which occur inside the task. Also checks to make sure the job is
    actually still running. If it is not running, the task doesn't get
    executed, so we don't do useless computation.

    :param task_func: the function to decorate
    """

    def wrapped(*args):
        """
        Initialize logs, make sure the job is still running, and run the task
        code surrounded by a try-except. If any error occurs, log it as a
        critical failure.
        """
        # the last argument is assumed to be a monitor
        monitor = args[-1]
        job = models.OqJob.objects.get(id=monitor.job_id)
        if job.is_running is False:
            # the job was killed, it is useless to run the task
            raise JobNotRunning(monitor.job_id)

        # it is important to save the task id soon, so that
        # the revoke functionality can work
        with monitor("storing task id", task=tsk, autoflush=True):
            pass

        with logs.handle(job):
            check_mem_usage()  # warn if too much memory is used
            # run the task
            try:
                total = "total " + task_func.__name__
                with monitor(total, task=tsk):
                    with GroundShakingIntensityModel.forbid_instantiation():
                        return task_func(*args)
            finally:
                # save on the db
                CacheInserter.flushall()
                # the task finished, we can remove from the performance
                # table the associated row 'storing task id'
                models.Performance.objects.filter(
                    oq_job=job, operation="storing task id", task_id=tsk.request.id
                ).delete()

    celery_queue = config.get("amqp", "celery_queue")
    f = lambda *args: safely_call(wrapped, args, pickle=True)
    f.__name__ = task_func.__name__
    f.__module__ = task_func.__module__
    tsk = task(f, queue=celery_queue)
    tsk.__func__ = tsk
    tsk.task_func = task_func
    return tsk
def task_with_callbacks(func, **options):
    """ decorator "task with callbacks"

    Callback or list of callbacks which go to function in "callbacks" kwarg,
    will be executed after the function, regardless of the subtask's return
    status.

    If subtask (function) result is an object, then a property named
    "async_result" will be added to that object so that it will be possible to
    join() for that result.
    """
    return task(run_with_callbacks(func), **options)
Ejemplo n.º 19
0
def test_task(func):
    @functools.wraps(func)
    def wrapper(*args):
        try:
            res = func(*[a.unpickle() for a in args])
            return Pickled((res, None))
        except:
            exctype, exc, tb = sys.exc_info()
            tb_str = ''.join(traceback.format_tb(tb))
            err_msg = '\n%s%s: %s' % (tb_str, exctype.__name__, exc)
            return Pickled((err_msg, exctype))
    return task(wrapper)
Ejemplo n.º 20
0
def task_with_callbacks(func, **options):
    """ decorator "task with callbacks"

    Callback or list of callbacks which go to function in "callbacks" kwarg,
    will be executed after the function, regardless of the subtask's return
    status.

    If subtask (function) result is an object, then a property named
    "async_result" will be added to that object so that it will be possible to
    join() for that result.
    """
    return task(run_with_callbacks(func), **options)
Ejemplo n.º 21
0
def oqtask(task_func):
    """
    Task function decorator which sets up logging and catches (and logs) any
    errors which occur inside the task. Also checks to make sure the job is
    actually still running. If it is not running, the task doesn't get
    executed, so we don't do useless computation.

    :param task_func: the function to decorate
    """
    def wrapped(*args):
        """
        Initialize logs, make sure the job is still running, and run the task
        code surrounded by a try-except. If any error occurs, log it as a
        critical failure.
        """
        # the last argument is assumed to be a monitor
        monitor = args[-1]
        job = models.OqJob.objects.get(id=monitor.job_id)
        if job.is_running is False:
            # the job was killed, it is useless to run the task
            raise JobNotRunning(monitor.job_id)

        # it is important to save the task id soon, so that
        # the revoke functionality can work
        with monitor('storing task id', task=tsk, autoflush=True):
            pass

        with logs.handle(job):
            check_mem_usage()  # warn if too much memory is used
            # run the task
            try:
                total = 'total ' + task_func.__name__
                with monitor(total, task=tsk, autoflush=True):
                    return task_func(*args)
            finally:
                # save on the db
                CacheInserter.flushall()
                # the task finished, we can remove from the performance
                # table the associated row 'storing task id'
                models.Performance.objects.filter(
                    oq_job=job,
                    operation='storing task id',
                    task_id=tsk.request.id).delete()

    celery_queue = config.get('amqp', 'celery_queue')
    f = lambda *args: safely_call(wrapped, args, pickle=True)
    f.__name__ = task_func.__name__
    f.__module__ = task_func.__module__
    tsk = task(f, queue=celery_queue)
    tsk.__func__ = tsk
    tsk.task_func = task_func
    return tsk
Ejemplo n.º 22
0
def oqtask(task_func):
    """
    Task function decorator which sets up logging and catches (and logs) any
    errors which occur inside the task. Also checks to make sure the job is
    actually still running. If it is not running, the task doesn't get
    executed, so we don't do useless computation.

    :param task_func: the function to decorate
    """
    def wrapped(*args):
        """
        Initialize logs, make sure the job is still running, and run the task
        code surrounded by a try-except. If any error occurs, log it as a
        critical failure.
        """
        # job_id is always assumed to be the first argument
        job_id = args[0]
        job = models.OqJob.objects.get(id=job_id)
        if job.is_running is False:
            # the job was killed, it is useless to run the task
            raise JobNotRunning(job_id)

        # it is important to save the task id soon, so that
        # the revoke functionality can work
        EnginePerformanceMonitor.store_task_id(job_id, tsk)

        with EnginePerformanceMonitor(
                'total ' + task_func.__name__, job_id, tsk, flush=True):
            # tasks write on the celery log file
            logs.set_level(job.log_level)
            try:
                # log a warning if too much memory is used
                check_mem_usage(SOFT_MEM_LIMIT, HARD_MEM_LIMIT)
                # run the task
                return task_func(*args)
            finally:
                # save on the db
                CacheInserter.flushall()
                # the task finished, we can remove from the performance
                # table the associated row 'storing task id'
                models.Performance.objects.filter(
                    oq_job=job,
                    operation='storing task id',
                    task_id=tsk.request.id).delete()
    celery_queue = config.get('amqp', 'celery_queue')
    f = lambda *args: safely_call(wrapped, args, pickle=True)
    f.__name__ = task_func.__name__
    tsk = task(f, queue=celery_queue)
    tsk.task_func = task_func
    return tsk
Ejemplo n.º 23
0
def test_task(func):
    @functools.wraps(func)
    def wrapper(*args):
        try:
            res = func(*[a.unpickle() for a in args])
            return Pickled((res, None))
        except:
            exctype, exc, tb = sys.exc_info()
            tb_str = ''.join(traceback.format_tb(tb))
            err_msg = '\n%s%s: %s' % (tb_str, exctype.__name__, exc)
            return Pickled((err_msg, exctype))

    tsk = task(wrapper)
    tsk.task_func = func
    return tsk
Ejemplo n.º 24
0
 def createTask(self, name):
     return task.task(__module__=self.__module__, name=name)(return_True)
Ejemplo n.º 25
0
from celery.utils import uuid
from celery.utils.timeutils import parse_iso8601, timedelta_seconds

from celery.tests.utils import Case, with_eager_tasks, WhateverIO


def now():
    return current_app.now()


def return_True(*args, **kwargs):
    # Task run functions can't be closures/lambdas, as they're pickled.
    return True


return_True_task = task()(return_True)


def raise_exception(self, **kwargs):
    raise Exception('%s error' % self.__class__)


class MockApplyTask(Task):
    applied = 0

    def run(self, x, y):
        return x * y

    @classmethod
    def apply_async(self, *args, **kwargs):
        self.applied += 1
Ejemplo n.º 26
0
                logs.init_logs_amqp_send(level=job.log_level,
                                         calc_domain='hazard',
                                         calc_id=calculation.id)
            else:
                logs.init_logs_amqp_send(level=job.log_level,
                                         calc_domain='risk',
                                         calc_id=calculation.id)

            try:
                # Tasks can be used in the `execute` or `post-process` phase
                if job.is_running is False:
                    raise JobCompletedError('Job %d was killed' % job_id)
                elif job.status not in ('executing', 'post_processing'):
                    raise JobCompletedError(
                        'The status of job %d is %s, should be executing or '
                        'post_processing' % (job_id, job.status))
                # else continue with task execution
                res = task_func(*args, **kwargs)
            # TODO: should we do something different with JobCompletedError?
            except Exception, err:
                logs.LOG.critical('Error occurred in task: %s', err)
                logs.LOG.exception(err)
                raise
            else:
                return res
            finally:
                CacheInserter.flushall()
    celery_queue = config.get('amqp', 'celery_queue')
    tsk = task(wrapped, ignore_result=True, queue=celery_queue)
    return tsk
Ejemplo n.º 27
0
    def run(self, obj, field_name):
        """Just run `build` on FieldFile, can't pass it directly to Celery."""
        return self.build(getattr(obj, field_name))

    def build(self, fieldfile):
        book = fieldfile.instance
        out = self.transform(book.wldocument(), fieldfile)
        fieldfile.save(None, File(open(out.get_filename())), save=False)
        if book.pk is not None:
            type(book).objects.filter(pk=book.pk).update(**{
                fieldfile.field.attname: fieldfile
            })
        if fieldfile.field.format_name in app_settings.FORMAT_ZIPS:
            remove_zip(app_settings.FORMAT_ZIPS[fieldfile.field.format_name])
# Don't decorate BuildEbook, because we want to subclass it.
BuildEbookTask = task(BuildEbook, ignore_result=True)


@BuildEbook.register('txt')
@task(ignore_result=True)
class BuildTxt(BuildEbook):
    @staticmethod
    def transform(wldoc, fieldfile):
        return wldoc.as_text()


@BuildEbook.register('pdf')
@task(ignore_result=True)
class BuildPdf(BuildEbook):
    @staticmethod
    def transform(wldoc, fieldfile):
Ejemplo n.º 28
0
def task(*args, **kwargs):  # ✞
    kwargs.setdefault("accept_magic_kwargs", True)
    return _task.task(*args, **kwargs)
Ejemplo n.º 29
0
def test_task(func, *args, **kwargs):
    kwargs['queue'] = config.get('amqp', 'celery_queue')
    return task(func, *args, **kwargs)
Ejemplo n.º 30
0
from celery.execute import send_task
from celery.five import items, range, string_t
from celery.result import EagerResult
from celery.schedules import crontab, crontab_parser, ParseException
from celery.utils import uuid
from celery.utils.timeutils import parse_iso8601, timedelta_seconds

from celery.tests.case import AppCase, eager_tasks, WhateverIO


def return_True(*args, **kwargs):
    # Task run functions can't be closures/lambdas, as they're pickled.
    return True


return_True_task = task()(return_True)


def raise_exception(self, **kwargs):
    raise Exception('%s error' % self.__class__)


class MockApplyTask(Task):
    applied = 0

    def run(self, x, y):
        return x * y

    @classmethod
    def apply_async(self, *args, **kwargs):
        self.applied += 1
Ejemplo n.º 31
0
#@task
def _send_notification(type='email', **kwargs):
    """ sends notification to user """
    kwargs.update({'fail_silently': True})
    if type == 'email':
        send_mail(**kwargs)
        return 0
    else:
        return 0

def send_void_mail(*args, **kwargs):
    if settings.DEBUG_MESSAGES:
        logger.info('no email sent, because SEND_EMAIL is disabled')
    if 'message' in kwargs:
        if settings.DEBUG_MESSAGES:
            logger.info("message: %s" % kwargs['message'])
    return

#SET SETTINGS, DISABLING SENDING NOTIFICATIONS OR ENABLING THEM
if settings.SMS_ENABLE:
    send_sms = task(_send_sms)
else:
    send_sms = lambda x,y: x
    send_sms.delay = send_void_sms

if settings.SEND_EMAIL:
    send_notification = task(_send_notification)
else:
    send_notification = lambda x: x
    send_notification.delay = send_void_mail
Ejemplo n.º 32
0
def task(*args, **kwargs):  # ✞
    kwargs.setdefault("accept_magic_kwargs", True)
    return _task.task(*args, **kwargs)
Ejemplo n.º 33
0
 def decorator(func):
     return task(**kwargs)(func)
Ejemplo n.º 34
0
 def decorator(func):
     return task(**kwargs)(func)
Ejemplo n.º 35
0
    @wraps(task_func)
    def wrapped(*args, **kwargs):
        """
        Initialize logs, make sure the job is still running, and run the task
        code surrounded by a try-except. If any error occurs, log it as a
        critical failure.
        """
        # job_id is always assumed to be the first arugment passed to a task
        # this is the only required argument
        job_id = args[0]
        # Set up logging via amqp.
        try:
            # check if the job is still running
            job = models.OqJob.objects.get(id=job_id)
            if not job.status == 'executing' and not job.is_running:
                # the job is not running
                raise JobCompletedError(job_id)
            # The job is running.
            # Setup task logging, via AMQP ...
            logs.init_logs_amqp_send(level=job.log_level, job_id=job_id)
            # ... and continue with task execution.
            task_func(*args, **kwargs)
        # TODO: should we do something different with the JobCompletedError?
        except Exception, err:
            logs.LOG.critical('Error occurred in task: %s' % str(err))
            logs.LOG.exception(err)
            raise

    return task(wrapped, ignore_result=True)
Ejemplo n.º 36
0
        """Just run `build` on FieldFile, can't pass it directly to Celery."""
        return self.build(getattr(obj, field_name))

    def build(self, fieldfile):
        book = fieldfile.instance
        out = self.transform(book.wldocument(), fieldfile)
        fieldfile.save(None, File(open(out.get_filename())), save=False)
        if book.pk is not None:
            type(book).objects.filter(pk=book.pk).update(
                **{fieldfile.field.attname: fieldfile})
        if fieldfile.field.format_name in app_settings.FORMAT_ZIPS:
            remove_zip(app_settings.FORMAT_ZIPS[fieldfile.field.format_name])


# Don't decorate BuildEbook, because we want to subclass it.
BuildEbookTask = task(BuildEbook, ignore_result=True)


@BuildEbook.register('txt')
@task(ignore_result=True)
class BuildTxt(BuildEbook):
    @staticmethod
    def transform(wldoc, fieldfile):
        return wldoc.as_text()


@BuildEbook.register('pdf')
@task(ignore_result=True)
class BuildPdf(BuildEbook):
    @staticmethod
    def transform(wldoc, fieldfile):
Ejemplo n.º 37
0
 def wrapper(fcn):
     return task(*args, **kwargs)(fcn)
Ejemplo n.º 38
0
 def _get_task(self):
     return task(lambda: None).delay()
Ejemplo n.º 39
0
 def createTask(self, name):
     return task(__module__=self.__module__, name=name)(return_True)
Ejemplo n.º 40
0
 def wrapper(fcn):
     return task(*args, **kwargs)(fcn)
Ejemplo n.º 41
0
                stats.put("ol.celery.task_run_time", run_time * 1000)
                stats.put("ol.celery.%s_run_time"%fn.__name__, run_time * 1000)
            except:
                pass
            raise ExceptionWrapper(e, d)
        log = s.getvalue()
        d = dict(largs = json.dumps(largs),
                 kargs = json.dumps(kargs),
                 command = fn.__name__,
                 enqueued_at = enqueue_time,
                 started_at = started_at,
                 log = log,
                 result = ret,
                 context = task_context,
                 parent_task = parent_task)
        logging.root.removeHandler(h)
        task_context = {}
        try:
            end_time = calendar.timegm(datetime.datetime.utcnow().timetuple())
            run_time = end_time - started_at
            stats.put("ol.celery.task_run_time", run_time * 1000)
            stats.put("ol.celery.%s_run_time"%fn.__name__, run_time * 1000)
        except:
            pass
        return d
    retval = task(wrapped)
    patched_delay = create_patched_delay(retval)
    retval.original_delay = retval.delay
    retval.delay = patched_delay
    return retval
Ejemplo n.º 42
0
                                  'Update failure for {0}'.format(shop.domain))
        send_html_email.delay('mail/customers/update_report.html', result_dict,
                              start_char + _('Update report for %s') % shop.domain,
                              email=[user.email])
    except SoftTimeLimitExceeded:
        # send email
        send_email.delay('mail/feedback', {'message': unicode(shop.domain), 'email': user.email},
                         'Time limit exceeded')
        send_email.delay('mail/customers/time_limit_exceeded.html', {'language': language},
                         _('Time limit exceeded'), email=[user.email])
    status.update_status = 'success'
    status.save()


def checked_update_presta_data_task(shop, *args, **kwargs):
    if shop.user.is_paid():
        checked_update_presta_data.delay(shop, *args, **kwargs)
    else:
        checked_update_presta_data_free.delay(shop, *args, **kwargs)


checked_update_presta_data_free = task(ignore_result=True,
                                       name="presta.tasks.checked_update_presta_data_free",
                                       soft_time_limit=300,
                                       time_limit=305)(_checked_update_presta_data)

checked_update_presta_data = task(ignore_result=True,
                                  name="presta.tasks.checked_update_presta_data",
                                  time_limit=3600)(_checked_update_presta_data)

Ejemplo n.º 43
0
def cleanBlanksFromList(datalist):
    return [x for x in datalist if x]

@task()
Ejemplo n.º 44
0
 def _get_task(self):
     return task(lambda : None).delay()
Ejemplo n.º 45
0
            # Set up logging via amqp.
            if isinstance(calculation, models.HazardCalculation):
                logs.init_logs_amqp_send(level=job.log_level,
                                         calc_domain='hazard',
                                         calc_id=calculation.id)
            else:
                logs.init_logs_amqp_send(level=job.log_level,
                                         calc_domain='risk',
                                         calc_id=calculation.id)
            try:
                res = task_func(*args, **kwargs)
            except Exception, err:
                logs.LOG.critical('Error occurred in task: %s', err)
                logs.LOG.exception(err)
                raise
            else:
                return res
            finally:
                CacheInserter.flushall()
                # the task finished, we can remove from the performance
                # table the associated row 'storing task id', then the
                # supervisor will not try revoke it without need
                models.Performance.objects.filter(
                    oq_job=job,
                    operation='storing task id',
                    task_id=tsk.request.id).delete()

    celery_queue = config.get('amqp', 'celery_queue')
    tsk = task(wrapped, ignore_result=True, queue=celery_queue)
    return tsk
Ejemplo n.º 46
0
def mltshp_task(*args, **options):
    # This is how celery's periodic_task decorator customizes the class, so try it here too.
    return task(**dict({"base": MltshpTask}, **options))
Ejemplo n.º 47
0
        raise RuntimeError('Monitor(%r).flush() must not be called '
                           'by %s!' % (self.monitor.operation, self.taskname))


def rec_delattr(mon, name):
    """
    Delete attribute from a monitor recursively
    """
    for child in mon.children:
        rec_delattr(child, name)
    if name in vars(mon):
        delattr(mon, name)


if OQ_DISTRIBUTE == 'celery':
    safe_task = task(safely_call,  queue='celery')


def _wakeup(sec):
    """Waiting functions, used to wake up the process pool"""
    try:
        import prctl
    except ImportError:
        pass
    else:
        # if the parent dies, the children die
        prctl.set_pdeathsig(signal.SIGKILL)
    time.sleep(sec)
    return os.getpid()

Ejemplo n.º 48
0
                          run_time * 1000)
            except:
                pass
            raise ExceptionWrapper(e, d)
        log = s.getvalue()
        d = dict(largs=json.dumps(largs),
                 kargs=json.dumps(kargs),
                 command=fn.__name__,
                 enqueued_at=enqueue_time,
                 started_at=started_at,
                 log=log,
                 result=ret,
                 context=task_context,
                 parent_task=parent_task)
        logging.root.removeHandler(h)
        task_context = {}
        try:
            end_time = calendar.timegm(datetime.datetime.utcnow().timetuple())
            run_time = end_time - started_at
            stats.put("ol.celery.task_run_time", run_time * 1000)
            stats.put("ol.celery.%s_run_time" % fn.__name__, run_time * 1000)
        except:
            pass
        return d

    retval = task(wrapped)
    patched_delay = create_patched_delay(retval)
    retval.original_delay = retval.delay
    retval.delay = patched_delay
    return retval
Ejemplo n.º 49
0
    })

    mail = EmailMultiAlternatives(subject=subject,
                                  body=text,
                                  from_email=from_email,
                                  to=to)

    # html
    html = render_to_string(
        "lfs/mail/review_added_mail.html", {
            "site": "http://%s" % Site.objects.get(id=settings.SITE_ID),
            "review": review,
            "product": product,
        })

    mail.attach_alternative(html, "text/html")
    mail.send(fail_silently=True)


# celery
try:
    from celery.task import task
except ImportError:
    pass
else:
    _send_customer_added = task(_send_customer_added)
    _send_order_paid_mail = task(_send_order_paid_mail)
    _send_order_received_mail = task(_send_order_received_mail)
    _send_order_sent_mail = task(_send_order_sent_mail)
    _send_review_added = task(_send_review_added)
Ejemplo n.º 50
0
                import pynliner
                body = pynliner.fromString(body)
            msg.attach_alternative(body, "text/html")
        except TemplateDoesNotExist:
            logging.info("Email sent without HTML, since %s not found" %
                         html_path)

        msg.send(fail_silently=fail_silently)

        # reset environment to original language
        if isinstance(recipient, User):
            activate(current_language)


if use_celery:
    _send_task = task(_send)


def get_users_language(user):
    """
    Returns site-specific language for this user. Raises
    LanguageStoreNotAvailable if this site does not use translated
    notifications.
    """
    if getattr(settings, 'NOTIFICATION_LANGUAGE_MODULE', False):
        try:
            app_label, model_name = settings.NOTIFICATION_LANGUAGE_MODULE.split(
                '.')
            model = models.get_model(app_label, model_name)
            language_model = model._default_manager.get(
                user__id__exact=user.id)
Ejemplo n.º 51
0
        args[-1].weight = getattr(args[0], 'weight', 1.)


def do_not_aggregate(acc, value):
    """
    Do nothing aggregation function.

    :param acc: the accumulator
    :param value: the value to accumulate
    :returns: the accumulator unchanged
    """
    return acc


if OQ_DISTRIBUTE == 'celery':
    safe_task = task(safely_call, queue='celery')


def _wakeup(sec):
    """Waiting functions, used to wake up the process pool"""
    try:
        import prctl
    except ImportError:
        pass
    else:
        # if the parent dies, the children die
        prctl.set_pdeathsig(signal.SIGKILL)
    time.sleep(sec)
    return os.getpid()

Ejemplo n.º 52
0
    text = render_to_string("lfs/mail/review_added_mail.txt", {
        "review": review,
        "product": product,
    })

    mail = EmailMultiAlternatives(
        subject=subject, body=text, from_email=from_email, to=to)

    # html
    html = render_to_string("lfs/mail/review_added_mail.html", {
        "site": "http://%s" % Site.objects.get(id=settings.SITE_ID),
        "review": review,
        "product": product,
    })

    mail.attach_alternative(html, "text/html")
    mail.send(fail_silently=True)


# celery
try:
    from celery.task import task
except ImportError:
    pass
else:
    _send_customer_added = task(_send_customer_added)
    _send_order_paid_mail = task(_send_order_paid_mail)
    _send_order_received_mail = task(_send_order_received_mail)
    _send_order_sent_mail = task(_send_order_sent_mail)
    _send_review_added = task(_send_review_added)
Ejemplo n.º 53
0
        # try to attach the html variant
        try:
            body = render_to_string(html_path, context)
            if pynliner:
                body = pynliner.fromString(body)
            msg.attach_alternative(body, "text/html")
        except TemplateDoesNotExist:
            logging.info("Email sent without HTML, since %s not found" % html_path)

        msg.send(fail_silently=fail_silently)

        # reset environment to original language
        if isinstance(recipient, User):
            activate(current_language)
if use_celery:
    _send_task = task(_send)


def get_users_language(user):
    """
    Returns site-specific language for this user. Raises
    LanguageStoreNotAvailable if this site does not use translated
    notifications.
    """
    if getattr(settings, 'NOTIFICATION_LANGUAGE_MODULE', False):
        try:
            app_label, model_name = settings.NOTIFICATION_LANGUAGE_MODULE.split('.')
            model = models.get_model(app_label, model_name)
            language_model = model._default_manager.get(user__id__exact=user.id)
            if hasattr(language_model, 'language'):
                return language_model.language
Ejemplo n.º 54
0
    backurl = getattr(mon, 'backurl', None)
    zsocket = (
        Socket(backurl, zmq.PUSH, 'connect') if backurl else mock.MagicMock()
    )  # do nothing
    with zsocket:
        zsocket.send(res)
    return zsocket.num_sent if backurl else res


if OQ_DISTRIBUTE.startswith('celery'):
    from celery.result import ResultSet
    from celery import Celery
    from celery.task import task
    app = Celery('openquake')
    app.config_from_object('openquake.engine.celeryconfig')
    safetask = task(safely_call, queue='celery')  # has to be global


class IterResult(object):
    """
    :param iresults:
        an iterator over Result objects
    :param taskname:
        the name of the task
    :param num_tasks:
        the total number of expected tasks
    :param progress:
        a logging function for the progress report
    :param sent:
        the number of bytes sent (0 if OQ_DISTRIBUTE=no)
    """