def wrapped(*args): """ Initialize logs, make sure the job is still running, and run the task code surrounded by a try-except. If any error occurs, log it as a critical failure. """ # job_id is always assumed to be the first argument job_id = args[0] job = models.OqJob.objects.get(id=job_id) if job.is_running is False: # the job was killed, it is useless to run the task raise JobNotRunning(job_id) # it is important to save the task id soon, so that # the revoke functionality can work EnginePerformanceMonitor.store_task_id(job_id, tsk) with EnginePerformanceMonitor( 'total ' + task_func.__name__, job_id, tsk, flush=True): # tasks write on the celery log file logs.set_level(job.log_level) check_mem_usage() # log a warning if too much memory is used try: # run the task return task_func(*args) finally: # save on the db CacheInserter.flushall() # the task finished, we can remove from the performance # table the associated row 'storing task id' models.Performance.objects.filter( oq_job=job, operation='storing task id', task_id=tsk.request.id).delete()
def wrapped(*args, **kwargs): """ Initialize logs, make sure the job is still running, and run the task code surrounded by a try-except. If any error occurs, log it as a critical failure. """ # job_id is always assumed to be the first argument passed to # the task, or a keyword argument # this is the only required argument job_id = kwargs.get('job_id') or args[0] with EnginePerformanceMonitor( 'totals per task', job_id, tsk, flush=True): job = models.OqJob.objects.get(id=job_id) # it is important to save the task ids soon, so that # the revoke functionality implemented in supervisor.py can work EnginePerformanceMonitor.store_task_id(job_id, tsk) with EnginePerformanceMonitor( 'loading calculation object', job_id, tsk, flush=True): calculation = job.calculation # Set up logging via amqp. if isinstance(calculation, models.HazardCalculation): logs.init_logs_amqp_send(level=job.log_level, calc_domain='hazard', calc_id=calculation.id) else: logs.init_logs_amqp_send(level=job.log_level, calc_domain='risk', calc_id=calculation.id) try: # Tasks can be used in the `execute` or `post-process` phase if job.is_running is False: raise JobCompletedError('Job %d was killed' % job_id) elif job.status not in ('executing', 'post_processing'): raise JobCompletedError( 'The status of job %d is %s, should be executing or ' 'post_processing' % (job_id, job.status)) # else continue with task execution res = task_func(*args, **kwargs) # TODO: should we do something different with JobCompletedError? except Exception, err: logs.LOG.critical('Error occurred in task: %s', err) logs.LOG.exception(err) raise else:
def wrapped(*args, **kwargs): """ Initialize logs, make sure the job is still running, and run the task code surrounded by a try-except. If any error occurs, log it as a critical failure. """ # job_id is always assumed to be the first argument passed to # the task, or a keyword argument # this is the only required argument job_id = kwargs.get('job_id') or args[0] job = models.OqJob.objects.get(id=job_id) if job.is_running is False: # the job was killed, it is useless to run the task return # it is important to save the task ids soon, so that # the revoke functionality implemented in supervisor.py can work EnginePerformanceMonitor.store_task_id(job_id, tsk) with EnginePerformanceMonitor('total ' + task_func.__name__, job_id, tsk, flush=True): with EnginePerformanceMonitor('loading calculation object', job_id, tsk, flush=True): calculation = job.calculation # Set up logging via amqp. if isinstance(calculation, models.HazardCalculation): logs.init_logs_amqp_send(level=job.log_level, calc_domain='hazard', calc_id=calculation.id) else: logs.init_logs_amqp_send(level=job.log_level, calc_domain='risk', calc_id=calculation.id) try: res = task_func(*args, **kwargs) except Exception, err: logs.LOG.critical('Error occurred in task: %s', err) logs.LOG.exception(err) raise else:
def wrapped(*args): """ Initialize logs, make sure the job is still running, and run the task code surrounded by a try-except. If any error occurs, log it as a critical failure. """ # job_id is always assumed to be the first argument job_id = args[0] job = models.OqJob.objects.get(id=job_id) if job.is_running is False: # the job was killed, it is useless to run the task return # it is important to save the task id soon, so that # the revoke functionality can work EnginePerformanceMonitor.store_task_id(job_id, tsk) with EnginePerformanceMonitor( 'total ' + task_func.__name__, job_id, tsk, flush=True): with EnginePerformanceMonitor( 'loading calculation object', job_id, tsk, flush=True): calculation = job.calculation # tasks write on the celery log file logs.init_logs( level=job.log_level, calc_domain='hazard' if isinstance( calculation, models.HazardCalculation) else'risk', calc_id=calculation.id) try: return task_func(*args), None except: etype, exc, tb = sys.exc_info() tb_str = ''.join(traceback.format_tb(tb)) return '%s\n%s' % (exc, tb_str), etype finally: CacheInserter.flushall() # the task finished, we can remove from the performance # table the associated row 'storing task id' models.Performance.objects.filter( oq_job=job, operation='storing task id', task_id=tsk.request.id).delete()
def wrapped(*args, **kwargs): """ Initialize logs, make sure the job is still running, and run the task code surrounded by a try-except. If any error occurs, log it as a critical failure. """ # job_id is always assumed to be the first argument passed to # the task, or a keyword argument # this is the only required argument job_id = kwargs.get('job_id') or args[0] job = models.OqJob.objects.get(id=job_id) if job.is_running is False: # the job was killed, it is useless to run the task return # it is important to save the task ids soon, so that # the revoke functionality implemented in supervisor.py can work EnginePerformanceMonitor.store_task_id(job_id, tsk) with EnginePerformanceMonitor( 'total ' + task_func.__name__, job_id, tsk, flush=True): with EnginePerformanceMonitor( 'loading calculation object', job_id, tsk, flush=True): calculation = job.calculation # Set up logging via amqp. if isinstance(calculation, models.HazardCalculation): logs.init_logs_amqp_send(level=job.log_level, calc_domain='hazard', calc_id=calculation.id) else: logs.init_logs_amqp_send(level=job.log_level, calc_domain='risk', calc_id=calculation.id) try: res = task_func(*args, **kwargs) except Exception, err: logs.LOG.critical('Error occurred in task: %s', err) logs.LOG.exception(err) raise else: