def wrapped(*args):
"""
Initialize logs, make sure the job is still running, and run the task
code surrounded by a try-except. If any error occurs, log it as a
critical failure.
"""
# the last argument is assumed to be a monitor
monitor = args[-1]
job = models.OqJob.objects.get(id=monitor.job_id)
if job.is_running is False:
# the job was killed, it is useless to run the task
raise JobNotRunning(monitor.job_id)
# it is important to save the task id soon, so that
# the revoke functionality can work
with monitor("storing task id", task=tsk, autoflush=True):
pass
with logs.handle(job):
check_mem_usage() # warn if too much memory is used
# run the task
try:
total = "total " + task_func.__name__
with monitor(total, task=tsk):
with GroundShakingIntensityModel.forbid_instantiation():
return task_func(*args)
finally:
# save on the db
CacheInserter.flushall()
# the task finished, we can remove from the performance
# table the associated row 'storing task id'
models.Performance.objects.filter(
oq_job=job, operation="storing task id", task_id=tsk.request.id
).delete()
def _do_run_calc(calc, exports):
"""
Step through all of the phases of a calculation, updating the job
status at each phase.
:param calc:
An :class:`~openquake.engine.calculators.base.Calculator` instance.
:param exports:
a (potentially empty) comma-separated string of export targets
"""
job = calc.job
log_status(job, "pre_executing")
calc.pre_execute()
log_status(job, "executing")
calc.execute()
log_status(job, "post_executing")
calc.post_execute()
log_status(job, "post_processing")
calc.post_process()
log_status(job, "export")
calc.export(exports=exports)
log_status(job, "clean_up")
calc.clean_up()
CacheInserter.flushall() # flush caches into the db
log_status(job, "complete")
def wrapped(*args):
"""
Initialize logs, make sure the job is still running, and run the task
code surrounded by a try-except. If any error occurs, log it as a
critical failure.
"""
# job_id is always assumed to be the first argument
job_id = args[0]
job = models.OqJob.objects.get(id=job_id)
if job.is_running is False:
# the job was killed, it is useless to run the task
raise JobNotRunning(job_id)
# it is important to save the task id soon, so that
# the revoke functionality can work
EnginePerformanceMonitor.store_task_id(job_id, tsk)
with EnginePerformanceMonitor(
'total ' + task_func.__name__, job_id, tsk, flush=True):
# tasks write on the celery log file
logs.set_level(job.log_level)
check_mem_usage() # log a warning if too much memory is used
try:
# run the task
return task_func(*args)
finally:
# save on the db
CacheInserter.flushall()
# the task finished, we can remove from the performance
# table the associated row 'storing task id'
models.Performance.objects.filter(
oq_job=job,
operation='storing task id',
task_id=tsk.request.id).delete()
def _do_run_calc(calc, exports):
"""
Step through all of the phases of a calculation, updating the job
status at each phase.
:param calc:
An :class:`~openquake.engine.calculators.base.Calculator` instance.
:param exports:
a (potentially empty) comma-separated string of export targets
"""
job = calc.job
log_status(job, "pre_executing")
calc.pre_execute()
log_status(job, "executing")
result = calc.execute()
log_status(job, "post_executing")
calc.post_execute(result)
log_status(job, "post_processing")
calc.post_process()
log_status(job, "export")
calc.export(exports=exports)
log_status(job, "clean_up")
calc.clean_up()
CacheInserter.flushall() # flush caches into the db
log_status(job, "complete")
def wrapped(*args):
"""
Initialize logs, make sure the job is still running, and run the task
code surrounded by a try-except. If any error occurs, log it as a
critical failure.
"""
# the last argument is assumed to be a monitor
monitor = args[-1]
job = models.OqJob.objects.get(id=monitor.job_id)
if job.is_running is False:
# the job was killed, it is useless to run the task
raise JobNotRunning(monitor.job_id)
# it is important to save the task id soon, so that
# the revoke functionality can work
with monitor('storing task id', task=tsk, autoflush=True):
pass
with logs.handle(job):
check_mem_usage() # warn if too much memory is used
# run the task
try:
total = 'total ' + task_func.__name__
with monitor(total, task=tsk, autoflush=True):
return task_func(*args)
finally:
# save on the db
CacheInserter.flushall()
# the task finished, we can remove from the performance
# table the associated row 'storing task id'
models.Performance.objects.filter(
oq_job=job,
operation='storing task id',
task_id=tsk.request.id).delete()
def wrapped(*args):
"""
Initialize logs, make sure the job is still running, and run the task
code surrounded by a try-except. If any error occurs, log it as a
critical failure.
"""
# job_id is always assumed to be the first argument
job_id = args[0]
job = models.OqJob.objects.get(id=job_id)
if job.is_running is False:
# the job was killed, it is useless to run the task
return
# it is important to save the task id soon, so that
# the revoke functionality can work
EnginePerformanceMonitor.store_task_id(job_id, tsk)
with EnginePerformanceMonitor(
'total ' + task_func.__name__, job_id, tsk, flush=True):
with EnginePerformanceMonitor(
'loading calculation object', job_id, tsk, flush=True):
calculation = job.calculation
# tasks write on the celery log file
logs.init_logs(
level=job.log_level,
calc_domain='hazard' if isinstance(
calculation, models.HazardCalculation) else'risk',
calc_id=calculation.id)
try:
return task_func(*args), None
except:
etype, exc, tb = sys.exc_info()
tb_str = ''.join(traceback.format_tb(tb))
return '%s\n%s' % (exc, tb_str), etype
finally:
CacheInserter.flushall()
# the task finished, we can remove from the performance
# table the associated row 'storing task id'
models.Performance.objects.filter(
oq_job=job,
operation='storing task id',
task_id=tsk.request.id).delete()
def _do_run_calc(job, exports, calc, job_type):
"""
Step through all of the phases of a calculation, updating the job
status at each phase.
:param job:
An :class:`~openquake.engine.db.models.OqJob` instance.
:param list exports:
a (potentially empty) list of export targets, currently only "xml" is
supported
:returns:
The input job object when the calculation completes.
"""
_switch_to_job_phase(job, job_type, "pre_executing")
calc.progress_handler("pre_executing", calc.hc)
calc.pre_execute()
_switch_to_job_phase(job, job_type, "executing")
calc.progress_handler("executing", calc.hc)
calc.execute()
_switch_to_job_phase(job, job_type, "post_executing")
calc.progress_handler("post_executing", calc.hc)
calc.post_execute()
_switch_to_job_phase(job, job_type, "post_processing")
calc.progress_handler("post_processing", calc.hc)
calc.post_process()
_switch_to_job_phase(job, job_type, "export")
calc.export(exports=exports)
_switch_to_job_phase(job, job_type, "clean_up")
calc.clean_up()
CacheInserter.flushall() # flush caches into the db
_switch_to_job_phase(job, job_type, "complete")
calc.progress_handler("calculation complete", calc.hc)
logs.LOG.debug("*> complete")
return job
def _do_run_calc(job, exports, calc, job_type):
"""
Step through all of the phases of a calculation, updating the job
status at each phase.
:param job:
An :class:`~openquake.engine.db.models.OqJob` instance.
:param list exports:
a (potentially empty) list of export targets, currently only "xml" is
supported
:returns:
The input job object when the calculation completes.
"""
# - Run the calculation
_switch_to_job_phase(job, job_type, "pre_executing")
calc.pre_execute()
_switch_to_job_phase(job, job_type, "executing")
calc.execute()
_switch_to_job_phase(job, job_type, "post_executing")
calc.post_execute()
_switch_to_job_phase(job, job_type, "post_processing")
calc.post_process()
_switch_to_job_phase(job, job_type, "export")
calc.export(exports=exports)
_switch_to_job_phase(job, job_type, "clean_up")
calc.clean_up()
CacheInserter.flushall() # flush caches into the db
_switch_to_job_phase(job, job_type, "complete")
logs.LOG.debug("*> complete")
return job
def _do_run_calc(calc, exports, job_type):
"""
Step through all of the phases of a calculation, updating the job
status at each phase.
:param calc:
An :class:`~openquake.engine.calculators.base.Calculator` instance.
:param list exports:
a (potentially empty) list of export targets, currently only "xml" is
supported
:param str job_type:
calculation type (hazard|risk)
"""
job = calc.job
_switch_to_job_phase(job, job_type, "pre_executing")
calc.pre_execute()
_switch_to_job_phase(job, job_type, "executing")
calc.execute()
_switch_to_job_phase(job, job_type, "post_executing")
calc.post_execute()
_switch_to_job_phase(job, job_type, "post_processing")
calc.post_process()
_switch_to_job_phase(job, job_type, "export")
calc.export(exports=exports)
_switch_to_job_phase(job, job_type, "clean_up")
calc.clean_up()
CacheInserter.flushall() # flush caches into the db
_switch_to_job_phase(job, job_type, "complete")
logs.LOG.debug("*> complete")
logs.init_logs_amqp_send(level=job.log_level,
calc_domain='hazard',
calc_id=calculation.id)
else:
logs.init_logs_amqp_send(level=job.log_level,
calc_domain='risk',
calc_id=calculation.id)
try:
# Tasks can be used in the `execute` or `post-process` phase
if job.is_running is False:
raise JobCompletedError('Job %d was killed' % job_id)
elif job.status not in ('executing', 'post_processing'):
raise JobCompletedError(
'The status of job %d is %s, should be executing or '
'post_processing' % (job_id, job.status))
# else continue with task execution
res = task_func(*args, **kwargs)
# TODO: should we do something different with JobCompletedError?
except Exception, err:
logs.LOG.critical('Error occurred in task: %s', err)
logs.LOG.exception(err)
raise
else:
return res
finally:
CacheInserter.flushall()
celery_queue = config.get('amqp', 'celery_queue')
tsk = task(wrapped, ignore_result=True, queue=celery_queue)
return tsk
# Set up logging via amqp.
if isinstance(calculation, models.HazardCalculation):
logs.init_logs_amqp_send(level=job.log_level,
calc_domain='hazard',
calc_id=calculation.id)
else:
logs.init_logs_amqp_send(level=job.log_level,
calc_domain='risk',
calc_id=calculation.id)
try:
res = task_func(*args, **kwargs)
except Exception, err:
logs.LOG.critical('Error occurred in task: %s', err)
logs.LOG.exception(err)
raise
else:
return res
finally:
CacheInserter.flushall()
# the task finished, we can remove from the performance
# table the associated row 'storing task id', then the
# supervisor will not try revoke it without need
models.Performance.objects.filter(
oq_job=job,
operation='storing task id',
task_id=tsk.request.id).delete()
celery_queue = config.get('amqp', 'celery_queue')
tsk = task(wrapped, ignore_result=True, queue=celery_queue)
return tsk
