def update(self,
status,
ex=None,
traceback=None,
result=None,
msg=None,
start_time=None):
"""Update job and jobtask status
:param status: jobtask status
:param ex: exception raised [optional]
:param traceback: exception trace [optional]
:param result: task result. None otherwise task status is SUCCESS [optional]
:param msg: update message [optional]
:param start_time: job task start time [optional]
"""
# get variables from shared area
params = self.get_shared_data()
# get job start time
job_start_time = params[u'start-time']
# get current time
current_time = time()
# get elapsed
elapsed = current_time - float(job_start_time)
# update job
self.update_job(params, status, current_time, ex, traceback, result,
msg)
# log message
if msg is not None:
if status == u'FAILURE':
#logger.error(msg, exc_info=1)
logger.error(msg)
msg = u'ERROR: %s' % (msg)
else:
logger.debug(msg)
# update jobtask result
task_id = self.request.id
TaskResult.store(task_id,
status=status,
traceback=traceback,
retval=result,
msg=msg,
stop_time=current_time,
start_time=start_time)
#if status == u'PROGRESS' and not self.request.called_directly:
# self.update_state(state=u'PROGRESS', meta={u'elapsed': elapsed})
# send event
self.send_job_event(status, elapsed, ex, msg)
def update_job(self, params=None, status=None, current_time=None, ex=None, traceback=None, result=None, msg=None,
start_time=None):
"""Update job status
:param params: variables in shared area [optional]
:param status: job current status [optional]
:param start_time: job start time [optional]
:param current_time: current time [optional]
:param ex: exception raised [optional]
:param traceback: exception trace [optional]
:param result: task result. None otherwise task status is SUCCESS [optional]
:param msg: update message [optional]
"""
# get actual job
job = TaskResult.get(task_local.opid)
if job['status'] is not None and job['status'] == 'FAILURE':
return None
params = self.get_shared_data()
# if job is finished exit to avoid wrong status change
if params.get('is-finished', False) is True:
return None
if 'start-time' not in params.keys():
params['start-time'] = job.get('start_time')
self.set_shared_data(params)
# if status == 'STARTED':
# params['start-time'] = time()
# self.set_shared_data(params)
# set result if status is SUCCESS
retval = None
if status == 'SUCCESS':
# set flag for SUCCESS job
params['is-finished'] = True
self.set_shared_data(params)
if 'result' in params:
retval = params['result']
# remove shared area
# self.remove_shared_area()
# remove stack
# self.remove_stack()
# store job data
msg = None
counter = int(job.get('counter', 0)) + 1
TaskResult.store(task_local.opid, status=status, retval=retval, inner_type='JOB', traceback=traceback,
stop_time=current_time, msg=msg, counter=counter)
if status == 'FAILURE':
logger.error('JOB %s status change to %s' % (task_local.opid, status))
else:
logger.info('JOB %s status change to %s' % (task_local.opid, status))
def wait_for_job_complete(self, task_id):
"""Query celery job and wait until status is not SUCCESS or FAILURE
**Parameters**:
* **task_id**: celery task id
**Return**:
task results
"""
try:
# append job to task jobs list
self.update('PROGRESS', job=task_id)
# get celery task
inner_task = TaskResult.get(task_id)
start = time()
# loop until inner_task finish with success or error
status = inner_task.get('status')
start_counter = inner_task.get('counter', 0)
while status != 'SUCCESS' and status != 'FAILURE':
sleep(task_local.delta)
inner_task = TaskResult.get(task_id)
counter = inner_task.get('counter', 0)
elapsed = time() - start
# verify job is stalled
if counter - start_counter == 0 and elapsed > 240:
raise JobError('Job %s is stalled' % task_id)
self.update('PROGRESS', msg='Job %s status %s after %ss' % (task_id, status, elapsed))
status = inner_task.get('status')
elapsed = time() - start
if status == 'FAILURE':
err = inner_task.get('traceback')[-1]
logger.error('Job %s error after %ss' % (task_id, elapsed))
self.update('PROGRESS', msg='Job %s status %s after %ss' % (task_id, status, elapsed))
raise JobError(err)
elif status == 'SUCCESS':
self.update('PROGRESS', msg='Job %s success after %ss' % (task_id, elapsed))
res = inner_task
else:
logger.error('Job %s unknown error after %ss' % (task_id, elapsed))
self.update('PROGRESS', msg='Job %s status %s after %ss' % (task_id, 'UNKNONWN', elapsed))
raise JobError('Unknown error')
return res
except Exception as ex:
logger.error(ex)
raise
def update(self, status, ex=None, traceback=None, result=None, msg=None, start_time=None, job=None):
"""Update job and jobtask status
:param status: jobtask status
:param ex: exception raised [optional]
:param traceback: exception trace [optional]
:param result: task result. None otherwise task status is SUCCESS [optional]
:param msg: update message [optional]
:param start_time: job task start time [optional]
:param job: job id to add to tasks jobs list [optional]
"""
# get variables from shared area
params = self.get_shared_data()
# get current time
current_time = time()
# log message
if msg is not None:
# msg = str(msg)
if status == 'FAILURE':
# logger.error(msg, exc_info=1)
logger.error(msg)
# msg = 'ERROR: %s' % msg
else:
logger.debug(truncate(msg))
# update jobtask result
task_id = self.request.id
jobs = None
if job is not None:
jobs = [job]
TaskResult.store(task_id, status=status, traceback=traceback, retval=result, msg=msg, stop_time=current_time,
start_time=start_time, inner_type='JOBTASK', jobs=jobs)
# get job start time
job_start_time = params.get('start-time', 0)
# update job only if job_start_time is not 0. job_start_time=0 if job already finished and shared area is empty
# don't update job when task status is SUCCESS to avoid async on_success of a task to overwrite job status
# already written by a following task
if job_start_time != 0 and status != 'SUCCESS':
# get elapsed
elapsed = current_time - float(job_start_time)
# update job
self.update_job(current_time=time(), status='PROGRESS')
# send event
self.send_job_event(status, elapsed, ex, msg)
def wait_for_job_complete(self, task_id):
"""Query celery job and wait until status is not SUCCESS or FAILURE
:param task: celery task id
:return: task results
"""
try:
# get celery task
#inner_task = AsyncResult(task_id, app=task_manager)
inner_task = TaskResult.get(task_id)
res = None
start = time()
# loop until inner_task finish with success or error
#while inner_task.status != u'SUCCESS' and inner_task.status != u'FAILURE':
status = inner_task.get(u'status')
while status != u'SUCCESS' and status != u'FAILURE':
sleep(task_local.delta)
#inner_task = AsyncResult(task_id, app=task_manager)
inner_task = TaskResult.get(task_id)
elapsed = time() - start
self.update(u'PROGRESS',
msg=u'Task %s status %s after %ss' %
(task_id, status, elapsed))
status = inner_task.get(u'status')
elapsed = time() - start
if status == u'FAILURE':
logger.error(u'Task %s error after %ss' % (task_id, elapsed))
raise JobError(u'Task %s error' % task_id)
elif status == u'SUCCESS':
logger.debug(u'Task %s success after %ss' % (task_id, elapsed))
res = inner_task
else:
logger.error(u'Task %s unknown error after %ss' %
(task_id, elapsed))
raise JobError(u'Unknown error with task %s' % task_id)
return res
except Exception as ex:
#logger.error(ex, exc_info=1)
logger.error(ex)
raise
def apply_async(self, args=(), kwargs={}, route_name=None, **options):
"""Apply this task asynchronously.
Arguments:
args (Tuple): Partial args to be prepended to the existing args.
kwargs (Dict): Partial kwargs to be merged with existing kwargs.
options (Dict): Partial options to be merged
with existing options.
Returns:
~@AsyncResult: promise of future evaluation.
See also:
:meth:`[email protected]_async` and the :ref:`guide-calling` guide.
"""
jobid = CelerySignature.apply_async(self, args, kwargs, route_name,
**options)
task = TaskResult.task_pending(str(jobid))
logger.debug('Create new task: %s' % task)
return jobid
def update_job(self,
params,
status,
current_time,
ex=None,
traceback=None,
result=None,
msg=None,
start_time=None):
"""Update job status
:param params: variables in shared area
:param status: job current status
:param start_time: job start time [optional]
:param current_time: current time
:param ex: exception raised [optional]
:param traceback: exception trace [optional]
:param result: task result. None otherwise task status is SUCCESS [optional]
:param msg: update message [optional]
"""
'''
# update job status only if it is not already in FAILURE status
job_status = get_value(params, u'job-status', u'PROGRESS')
if job_status != u'FAILURE':
# set result if status is SUCCESS
retval = None
if status == u'SUCCESS':
if u'result' in params:
retval = params[u'result']
# remove shared area
self.remove_shared_area()
# remove stack
self.remove_stack()
else:
# set job status
params[u'job-status'] = status
# save data in shared area
self.set_shared_data(params)
# store job data
msg = None
TaskResult.store(task_local.opid, status=status, retval=retval,
inner_type=u'JOB', traceback=traceback,
stop_time=current_time, start_time=start_time,
msg=msg)
if status == u'FAILURE':
logger.error(u'JOB %s status change to %s' %
(task_local.opid, status))
else:
logger.info(u'JOB %s status change to %s' %
(task_local.opid, status))'''
# set result if status is SUCCESS
retval = None
if status == u'SUCCESS':
if u'result' in params:
retval = params[u'result']
# remove shared area
self.remove_shared_area()
# remove stack
self.remove_stack()
# store job data
msg = None
TaskResult.store(task_local.opid,
status=status,
retval=retval,
inner_type=u'JOB',
traceback=traceback,
stop_time=current_time,
start_time=start_time,
msg=msg)
if status == u'FAILURE':
logger.error(u'JOB %s status change to %s' %
(task_local.opid, status))
else:
logger.info(u'JOB %s status change to %s' %
(task_local.opid, status))