def testReraiseACompoundFailure(self):
"""Tests that the list of ExceptInfo objects are copied over."""
tb1 = 'Dummy traceback1'
tb2 = 'Dummy traceback2'
org_infos = failures_lib.CreateExceptInfo(ValueError('No taco.'), tb1) + \
failures_lib.CreateExceptInfo(OSError('No salsa'), tb2)
try:
self._GetFunction(self.SubparLunch,
self.TacoNotTasty,
exc_infos=org_infos)()
except Exception as e:
self.assertTrue(isinstance(e, self.SubparLunch))
# The orignal exceptions stored in exc_infos are preserved.
self.assertEqual(e.exc_infos, org_infos)
# All essential inforamtion should be included in the message of
# the new excpetion.
self.assertTrue(tb1 in str(e))
self.assertTrue(tb2 in str(e))
self.assertTrue(str(ValueError) in str(e))
self.assertTrue(str(OSError) in str(e))
self.assertTrue(str('No taco') in str(e))
self.assertTrue(str('No salsa') in str(e))
# Assert that summary does not contain the textual tracebacks.
self.assertFalse(tb1 in e.ToSummaryString())
self.assertFalse(tb2 in e.ToSummaryString())
def _Run(self):
"""Internal method for running the list of steps."""
# Register a handler for a signal that is rarely used.
def trigger_bt(_sig_num, frame):
logging.error('pre-kill notification (SIGXCPU); traceback:\n%s',
''.join(traceback.format_stack(frame)))
signal.signal(signal.SIGXCPU, trigger_bt)
sys.stdout.flush()
sys.stderr.flush()
errors = []
# Send all output to a named temporary file.
with open(self._output.name, 'w', 0) as output:
# Back up sys.std{err,out}. These aren't used, but we keep a copy so
# that they aren't garbage collected. We intentionally don't restore
# the old stdout and stderr at the end, because we want shutdown errors
# to also be sent to the same log file.
_orig_stdout, _orig_stderr = sys.stdout, sys.stderr
# Replace std{out,err} with unbuffered file objects.
os.dup2(output.fileno(), sys.__stdout__.fileno())
os.dup2(output.fileno(), sys.__stderr__.fileno())
sys.stdout = os.fdopen(sys.__stdout__.fileno(), 'w', 0)
sys.stderr = os.fdopen(sys.__stderr__.fileno(), 'w', 0)
try:
self._started.set()
results_lib.Results.Clear()
# Reduce the silent timeout by the prescribed amount.
cls = self.__class__
cls.SILENT_TIMEOUT -= cls.SILENT_TIMEOUT_STEP
# Actually launch the task.
self._task(*self._task_args, **self._task_kwargs)
except failures_lib.StepFailure as ex:
errors.extend(failures_lib.CreateExceptInfo(
ex, traceback.format_exc()))
except BaseException as ex:
errors.extend(failures_lib.CreateExceptInfo(
ex, traceback.format_exc()))
if self._killing.is_set():
traceback.print_exc()
finally:
sys.stdout.flush()
sys.stderr.flush()
return errors
def _CreateExceptInfos(self, cls, message='', traceback='', num=1):
"""A helper function to create a list of ExceptInfo objects."""
exc_infos = []
for _ in xrange(num):
exc_infos.extend(
failures_lib.CreateExceptInfo(cls(message), traceback))
return exc_infos
def testConvertToExceptInfo(self):
"""Tests converting an exception to an ExceptInfo object."""
traceback = 'Dummy traceback'
message = 'Taco is not a valid option!'
except_infos = failures_lib.CreateExceptInfo(ValueError(message),
traceback)
self.assertEqual(except_infos[0].type, ValueError)
self.assertEqual(except_infos[0].str, message)
self.assertEqual(except_infos[0].traceback, traceback)
def TaskRunner(queue, task, onexit=None, task_args=None, task_kwargs=None):
"""Run task(*input) for each input in the queue.
Returns when it encounters an _AllTasksComplete object on the queue.
If exceptions occur, save them off and re-raise them as a
BackgroundFailure once we've finished processing the items in the queue.
Args:
queue: A queue of tasks to run. Add tasks to this queue, and they will
be run.
task: Function to run on each queued input.
onexit: Function to run after all inputs are processed.
task_args: A list of args to pass to the |task|.
task_kwargs: A dict of optional args to pass to the |task|.
"""
if task_args is None:
task_args = []
elif not isinstance(task_args, list):
task_args = list(task_args)
if task_kwargs is None:
task_kwargs = {}
errors = []
while True:
# Wait for a new item to show up on the queue. This is a blocking wait,
# so if there's nothing to do, we just sit here.
x = queue.get()
if isinstance(x, _AllTasksComplete):
# All tasks are complete, so we should exit.
break
elif not isinstance(x, list):
x = task_args + list(x)
else:
x = task_args + x
# If no tasks failed yet, process the remaining tasks.
if not errors:
try:
task(*x, **task_kwargs)
except BaseException as ex:
errors.extend(
failures_lib.CreateExceptInfo(ex, traceback.format_exc()))
# Run exit handlers.
if onexit:
onexit()
# Propagate any exceptions.
if errors:
raise BackgroundFailure(exc_infos=errors)
def testReportStageFailureToCIDB(self):
"""Tests that the reporting fuction reports all included exceptions."""
fake_db = fake_cidb.FakeCIDBConnection()
inner_exception_1 = failures_lib.TestLabFailure()
inner_exception_2 = TypeError()
exc_infos = failures_lib.CreateExceptInfo(inner_exception_1, None)
exc_infos += failures_lib.CreateExceptInfo(inner_exception_2, None)
outer_exception = failures_lib.GoBFailure(exc_infos=exc_infos)
mock_build_stage_id = 9345
failures_lib.ReportStageFailureToCIDB(fake_db, mock_build_stage_id,
outer_exception)
self.assertEqual(3, len(fake_db.failureTable))
self.assertEqual(
set([mock_build_stage_id]),
set([x['build_stage_id'] for x in fake_db.failureTable.values()]))
self.assertEqual(
set([
constants.EXCEPTION_CATEGORY_INFRA,
constants.EXCEPTION_CATEGORY_UNKNOWN,
constants.EXCEPTION_CATEGORY_LAB
]),
set([
x['exception_category'] for x in fake_db.failureTable.values()
]))
# Find the outer failure id.
for failure_id, failure in fake_db.failureTable.iteritems():
if failure['outer_failure_id'] is None:
outer_failure_id = failure_id
break
# Now verify inner failures reference this failure.
for failure_id, failure in fake_db.failureTable.iteritems():
if failure_id != outer_failure_id:
self.assertEqual(outer_failure_id, failure['outer_failure_id'])
def run(self):
"""Run the list of steps."""
if self._semaphore is not None:
self._semaphore.acquire()
errors = failures_lib.CreateExceptInfo(
UnexpectedException('Unexpected exception in %r' % self), '')
pid = os.getpid()
try:
errors = self._Run()
finally:
if not self._killing.is_set() and os.getpid() == pid:
results = results_lib.Results.Get()
self._queue.put((errors, results))
if self._semaphore is not None:
self._semaphore.release()
def Wait(self):
"""Wait for the task to complete.
Output from the task is printed as it runs.
If an exception occurs, return a string containing the traceback.
"""
try:
# Flush stdout and stderr to be sure no output is interleaved.
sys.stdout.flush()
sys.stderr.flush()
# File position pointers are shared across processes, so we must open
# our own file descriptor to ensure output is not lost.
self._WaitForStartup()
silent_death_time = time.time() + self.SILENT_TIMEOUT
results = []
with open(self._output.name, 'r') as output:
pos = 0
running, exited_cleanly, task_errors, all_errors = (True, False, [], [])
while running:
# Check whether the process is still alive.
running = self.is_alive()
try:
errors, results = \
self._queue.get(True, self.PRINT_INTERVAL)
if errors:
task_errors.extend(errors)
all_errors.extend(errors)
running = False
exited_cleanly = True
except Queue.Empty:
pass
if not running:
# Wait for the process to actually exit. If the child doesn't exit
# in a timely fashion, kill it.
self.join(self.EXIT_TIMEOUT)
if self.exitcode is None:
msg = '%r hung for %r seconds' % (self, self.EXIT_TIMEOUT)
all_errors.extend(
failures_lib.CreateExceptInfo(ProcessExitTimeout(msg), ''))
self._KillChildren([self])
elif not exited_cleanly:
msg = ('%r exited unexpectedly with code %s'
% (self, self.exitcode))
all_errors.extend(
failures_lib.CreateExceptInfo(ProcessUnexpectedExit(msg), ''))
# Read output from process.
output.seek(pos)
buf = output.read(_BUFSIZE)
if len(buf) > 0:
silent_death_time = time.time() + self.SILENT_TIMEOUT
elif running and time.time() > silent_death_time:
msg = ('No output from %r for %r seconds' %
(self, self.SILENT_TIMEOUT))
all_errors.extend(
failures_lib.CreateExceptInfo(ProcessSilentTimeout(msg), ''))
self._KillChildren([self])
# Read remaining output from the process.
output.seek(pos)
buf = output.read(_BUFSIZE)
running = False
# Print output so far.
while len(buf) > 0:
sys.stdout.write(buf)
pos += len(buf)
if len(buf) < _BUFSIZE:
break
buf = output.read(_BUFSIZE)
# Print error messages if anything exceptional occurred.
if len(all_errors) > len(task_errors):
logging.PrintBuildbotStepFailure()
msg = '\n'.join(x.str for x in all_errors if x)
logging.warning(msg)
traceback.print_stack()
sys.stdout.flush()
sys.stderr.flush()
# Propagate any results.
for result in results:
results_lib.Results.Record(*result)
finally:
self.Cleanup(silent=True)
# If an error occurred, return it.
return all_errors
