def _CreateExceptInfos(self, cls, message='', traceback='', num=1):
"""A helper function to create a list of ExceptInfo objects."""
exc_infos = []
for _ in xrange(num):
exc_infos.extend(
failures_lib.CreateExceptInfo(cls(message), traceback))
return exc_infos
def _RunVMTests(self, builder_run, board):
"""Run VM test stages for the specified board.
Args:
builder_run: BuilderRun object for stages.
board: String containing board name.
"""
config = builder_run.config
except_infos = []
try:
if config.vm_test_runs > 1:
# Run the VMTests multiple times to see if they fail.
self._RunStage(generic_stages.RepeatStage,
config.vm_test_runs,
vm_test_stages.VMTestStage,
board,
builder_run=builder_run)
else:
# Retry VM-based tests in case failures are flaky.
self._RunStage(generic_stages.RetryStage,
constants.VM_NUM_RETRIES,
vm_test_stages.VMTestStage,
board,
builder_run=builder_run)
except Exception as e:
except_infos.extend(
failures_lib.CreateExceptInfo(e, traceback.format_exc()))
# Run stages serially to avoid issues encountered when running VMs (or the
# devserver) in parallel: https://crbug.com/779267
if config.tast_vm_tests:
try:
self._RunStage(generic_stages.RetryStage,
constants.VM_NUM_RETRIES,
tast_test_stages.TastVMTestStage,
board,
builder_run=builder_run)
except Exception as e:
except_infos.extend(
failures_lib.CreateExceptInfo(e, traceback.format_exc()))
if except_infos:
raise failures_lib.CompoundFailure('VM tests failed', except_infos)
def testConvertToExceptInfo(self):
"""Tests converting an exception to an ExceptInfo object."""
traceback = 'Dummy traceback'
message = 'Taco is not a valid option!'
except_infos = failures_lib.CreateExceptInfo(ValueError(message),
traceback)
self.assertEqual(except_infos[0].type, ValueError)
self.assertEqual(except_infos[0].str, message)
self.assertEqual(except_infos[0].traceback, traceback)
def TaskRunner(queue, task, onexit=None, task_args=None, task_kwargs=None):
"""Run task(*input) for each input in the queue.
Returns when it encounters an _AllTasksComplete object on the queue.
If exceptions occur, save them off and re-raise them as a
BackgroundFailure once we've finished processing the items in the queue.
Args:
queue: A queue of tasks to run. Add tasks to this queue, and they will
be run.
task: Function to run on each queued input.
onexit: Function to run after all inputs are processed.
task_args: A list of args to pass to the |task|.
task_kwargs: A dict of optional args to pass to the |task|.
"""
if task_args is None:
task_args = []
elif not isinstance(task_args, list):
task_args = list(task_args)
if task_kwargs is None:
task_kwargs = {}
errors = []
while True:
# Wait for a new item to show up on the queue. This is a blocking wait,
# so if there's nothing to do, we just sit here.
x = queue.get()
if isinstance(x, _AllTasksComplete):
# All tasks are complete, so we should exit.
break
elif not isinstance(x, list):
x = task_args + list(x)
else:
x = task_args + x
# If no tasks failed yet, process the remaining tasks.
if not errors:
try:
task(*x, **task_kwargs)
except BaseException as ex:
errors.extend(
failures_lib.CreateExceptInfo(ex,
traceback.format_exc()))
# Run exit handlers.
if onexit:
onexit()
# Propagate any exceptions.
if errors:
raise BackgroundFailure(exc_infos=errors)
def testReportStageFailureToCIDB(self):
"""Tests that the reporting fuction reports all included exceptions."""
fake_db = fake_cidb.FakeCIDBConnection()
inner_exception_1 = failures_lib.TestLabFailure()
inner_exception_2 = TypeError()
exc_infos = failures_lib.CreateExceptInfo(inner_exception_1, None)
exc_infos += failures_lib.CreateExceptInfo(inner_exception_2, None)
outer_exception = failures_lib.GoBFailure(exc_infos=exc_infos)
mock_build_stage_id = 9345
failures_lib.ReportStageFailureToCIDB(fake_db, mock_build_stage_id,
outer_exception)
self.assertEqual(3, len(fake_db.failureTable))
self.assertEqual(
set([mock_build_stage_id]),
set([x['build_stage_id'] for x in fake_db.failureTable.values()]))
self.assertEqual(
set([
constants.EXCEPTION_CATEGORY_INFRA,
constants.EXCEPTION_CATEGORY_UNKNOWN,
constants.EXCEPTION_CATEGORY_LAB
]),
set([
x['exception_category'] for x in fake_db.failureTable.values()
]))
# Find the outer failure id.
for failure_id, failure in fake_db.failureTable.iteritems():
if failure['outer_failure_id'] is None:
outer_failure_id = failure_id
break
# Now verify inner failures reference this failure.
for failure_id, failure in fake_db.failureTable.iteritems():
if failure_id != outer_failure_id:
self.assertEqual(outer_failure_id, failure['outer_failure_id'])
def run(self):
"""Run the list of steps."""
if self._semaphore is not None:
self._semaphore.acquire()
errors = failures_lib.CreateExceptInfo(
UnexpectedException('Unexpected exception in %r' % self), '')
pid = os.getpid()
try:
errors = self._Run()
finally:
if not self._killing.is_set() and os.getpid() == pid:
results = results_lib.Results.Get()
self._queue.put((errors, results))
if self._semaphore is not None:
self._semaphore.release()
def Wait(self):
"""Wait for the task to complete.
Output from the task is printed as it runs.
If an exception occurs, return a string containing the traceback.
"""
try:
# Flush stdout and stderr to be sure no output is interleaved.
sys.stdout.flush()
sys.stderr.flush()
# File position pointers are shared across processes, so we must open
# our own file descriptor to ensure output is not lost.
self._WaitForStartup()
silent_death_time = time.time() + self.SILENT_TIMEOUT
results = []
with open(self._output.name, 'r') as output:
pos = 0
running, exited_cleanly, task_errors, run_errors = (True,
False, [],
[])
while running:
# Check whether the process is still alive.
running = self.is_alive()
try:
errors, results = \
self._queue.get(True, self.PRINT_INTERVAL)
if errors:
task_errors.extend(errors)
running = False
exited_cleanly = True
except Queue.Empty:
pass
if not running:
# Wait for the process to actually exit. If the child doesn't exit
# in a timely fashion, kill it.
self.join(self.EXIT_TIMEOUT)
if self.exitcode is None:
msg = '%r hung for %r seconds' % (
self, self.EXIT_TIMEOUT)
run_errors.extend(
failures_lib.CreateExceptInfo(
ProcessExitTimeout(msg), ''))
self._KillChildren([self])
elif not exited_cleanly:
msg = ('%r exited unexpectedly with code %s' %
(self, self.exitcode))
run_errors.extend(
failures_lib.CreateExceptInfo(
ProcessUnexpectedExit(msg), ''))
# Read output from process.
output.seek(pos)
buf = output.read(_BUFSIZE)
if buf:
silent_death_time = time.time() + self.SILENT_TIMEOUT
elif running and time.time() > silent_death_time:
msg = ('No output from %r for %r seconds' %
(self, self.SILENT_TIMEOUT))
run_errors.extend(
failures_lib.CreateExceptInfo(
ProcessSilentTimeout(msg), ''))
self._KillChildren([self])
# Read remaining output from the process.
output.seek(pos)
buf = output.read(_BUFSIZE)
running = False
# Print output so far.
while buf:
sys.stdout.write(buf)
pos += len(buf)
if len(buf) < _BUFSIZE:
break
buf = output.read(_BUFSIZE)
# Print error messages if anything exceptional occurred.
if run_errors:
logging.PrintBuildbotStepFailure()
traceback.print_stack()
logging.warning('\n'.join(x.str for x in run_errors
if x))
logging.info('\n'.join(x.str for x in task_errors
if x))
sys.stdout.flush()
sys.stderr.flush()
# Propagate any results.
for result in results:
results_lib.Results.Record(*result)
finally:
self.Cleanup(silent=True)
# If an error occurred, return it.
return run_errors + task_errors
