def _ProcessJob(job_id, queue_status, configuration):
job = job_model.JobFromId(job_id)
if not job:
logging.error('Failed to load job with id: %s', job_id)
scheduler.Remove(configuration, job_id)
return False
logging.info('Job "%s" status: "%s" queue: "%s"', job_id, job.status,
configuration)
if queue_status == 'Running':
if job.status in {'Failed', 'Completed'}:
scheduler.Complete(job)
return True # Continue processing this queue.
if queue_status == 'Queued':
job.Start()
# TODO(dberris): Remove this when the execution engine is the default.
if job.use_execution_engine:
logging.info(
'Skipping Job that uses the experimental execution engine: %s (%s)',
job.job_id, job.url)
scheduler.Complete(job)
return False
def Fail(self, exception=None):
tb = traceback.format_exc() or ''
title = _CRYING_CAT_FACE + ' Pinpoint job stopped with an error.'
exc_info = sys.exc_info()
if exception is None:
if exc_info[1] is None:
# We've been called without a exception in sys.exc_info or in our args.
# This should not happen.
exception = errors.JobError('Unknown job error')
exception.category = 'pinpoint'
else:
exception = exc_info[1]
exc_message = exception.message
category = None
if isinstance(exception, errors.JobError):
category = exception.category
self.exception_details = {
'message': exc_message,
'traceback': tb,
'category': category,
}
self.task = None
comment = '\n'.join((title, self.url, '', exc_message))
deferred.defer(_PostBugCommentDeferred,
self.bug_id,
comment,
_retry_options=RETRY_OPTIONS)
scheduler.Complete(self)
def Fail(self, exception=None):
if exception:
self.exception = exception
tb = traceback.format_exc()
if tb:
self.exception += '\n%s' % tb
else:
self.exception = traceback.format_exc()
title = _CRYING_CAT_FACE + ' Pinpoint job stopped with an error.'
exc_info = sys.exc_info()
exc_message = ''
if exc_info[1]:
exc_message = sys.exc_info()[1].message
elif self.exception:
exc_message = self.exception.splitlines()[-1]
self.task = None
comment = '\n'.join((title, self.url, '', exc_message))
deferred.defer(_PostBugCommentDeferred,
self.bug_id,
comment,
_retry_options=RETRY_OPTIONS)
scheduler.Complete(self)
def _Complete(self):
logging.debug('Job [%s]: Completed', self.job_id)
if self.use_execution_engine:
scheduler.Complete(self)
if not self._IsTryJob():
self.difference_count = len(self.state.Differences())
try:
# TODO(dberris): Migrate results2 generation to tasks and evaluators.
results2.ScheduleResults2Generation(self)
except taskqueue.Error as e:
logging.debug('Failed ScheduleResults2Generation: %s', str(e))
self._FormatAndPostBugCommentOnComplete()
self._UpdateGerritIfNeeded()
scheduler.Complete(self)
def _Complete(self):
if self.comparison_mode:
self.difference_count = len(self.state.Differences())
try:
results2.ScheduleResults2Generation(self)
except taskqueue.Error:
pass
self._FormatAndPostBugCommentOnComplete()
self._UpdateGerritIfNeeded()
scheduler.Complete(self)
def _Complete(self):
logging.debug('Job [%s]: Completed', self.job_id)
if not self._IsTryJob():
self.difference_count = len(self.state.Differences())
try:
results2.ScheduleResults2Generation(self)
except taskqueue.Error as e:
logging.debug('Failed ScheduleResults2Generation: %s', str(e))
self._FormatAndPostBugCommentOnComplete()
self._UpdateGerritIfNeeded()
scheduler.Complete(self)
def _Complete(self):
logging.debug('Job [%s]: Completed', self.job_id)
if self.use_execution_engine:
scheduler.Complete(self)
# Only proceed if this thread/process is the one that successfully marked
# a job done.
if not MarkDone(self.job_id):
return
logging.debug('Job [%s]: Marked done', self.job_id)
if not self._IsTryJob() and not self.use_execution_engine:
self.difference_count = len(self.state.Differences())
try:
# TODO(dberris): Migrate results2 generation to tasks and evaluators.
results2.ScheduleResults2Generation(self)
except taskqueue.Error as e:
logging.debug('Failed ScheduleResults2Generation: %s', str(e))
self._FormatAndPostBugCommentOnComplete()
self._UpdateGerritIfNeeded()
scheduler.Complete(self)
def _ProcessJob(job_id, queue_status, configuration):
job = job_model.JobFromId(job_id)
if not job:
logging.error('Failed to load job with id: %s', job_id)
scheduler.Remove(configuration, job_id)
return False
logging.info('Job "%s" status: "%s" queue: "%s"', job_id, job.status,
configuration)
if queue_status == 'Running':
logging.debug('Job details: %r', job)
if job.status in {'Failed', 'Completed'}:
scheduler.Complete(job)
return True # Continue processing this queue.
if queue_status == 'Queued':
job.Start()
return False
def Fail(self, exception=None):
tb = traceback.format_exc() or ''
title = _CRYING_CAT_FACE + ' Pinpoint job stopped with an error.'
exc_info = sys.exc_info()
exc_message = ''
if exception:
exc_message = exception
elif exc_info[1]:
exc_message = sys.exc_info()[1].message
self.exception_details = {
'message': exc_message,
'traceback': tb,
}
self.task = None
comment = '\n'.join((title, self.url, '', exc_message))
deferred.defer(
_PostBugCommentDeferred, self.bug_id, comment,
_retry_options=RETRY_OPTIONS)
scheduler.Complete(self)
def Fail(self, exception=None):
tb = traceback.format_exc() or ''
title = _CRYING_CAT_FACE + ' Pinpoint job stopped with an error.'
exc_info = sys.exc_info()
if exception is None:
if exc_info[1] is None:
# We've been called without a exception in sys.exc_info or in our args.
# This should not happen.
exception = errors.JobError('Unknown job error')
exception.category = 'pinpoint'
else:
exception = exc_info[1]
exc_message = exception.message
category = None
if isinstance(exception, errors.JobError):
category = exception.category
self.exception_details = {
'message': exc_message,
'traceback': tb,
'category': category,
}
self.task = None
comment = '\n'.join((title, self.url, '', exc_message))
# Short-circuit jobs failure updates when we are not the first one to mark a
# job done.
if self.use_execution_engine and not MarkDone(self.job_id):
return
deferred.defer(_PostBugCommentDeferred,
self.bug_id,
comment,
project=self.project,
labels=job_bug_update.ComputeLabelUpdates(
['Pinpoint-Job-Failed']),
send_email=True,
_retry_options=RETRY_OPTIONS)
scheduler.Complete(self)