def _CreateExceptInfos(self, cls, message='', traceback='', num=1): """A helper function to create a list of ExceptInfo objects.""" exc_infos = [] for _ in xrange(num): exc_infos.extend( failures_lib.CreateExceptInfo(cls(message), traceback)) return exc_infos
def _RunVMTests(self, builder_run, board): """Run VM test stages for the specified board. Args: builder_run: BuilderRun object for stages. board: String containing board name. """ config = builder_run.config except_infos = [] try: if config.vm_test_runs > 1: # Run the VMTests multiple times to see if they fail. self._RunStage(generic_stages.RepeatStage, config.vm_test_runs, vm_test_stages.VMTestStage, board, builder_run=builder_run) else: # Retry VM-based tests in case failures are flaky. self._RunStage(generic_stages.RetryStage, constants.VM_NUM_RETRIES, vm_test_stages.VMTestStage, board, builder_run=builder_run) except Exception as e: except_infos.extend( failures_lib.CreateExceptInfo(e, traceback.format_exc())) # Run stages serially to avoid issues encountered when running VMs (or the # devserver) in parallel: https://crbug.com/779267 if config.tast_vm_tests: try: self._RunStage(generic_stages.RetryStage, constants.VM_NUM_RETRIES, tast_test_stages.TastVMTestStage, board, builder_run=builder_run) except Exception as e: except_infos.extend( failures_lib.CreateExceptInfo(e, traceback.format_exc())) if except_infos: raise failures_lib.CompoundFailure('VM tests failed', except_infos)
def testConvertToExceptInfo(self): """Tests converting an exception to an ExceptInfo object.""" traceback = 'Dummy traceback' message = 'Taco is not a valid option!' except_infos = failures_lib.CreateExceptInfo(ValueError(message), traceback) self.assertEqual(except_infos[0].type, ValueError) self.assertEqual(except_infos[0].str, message) self.assertEqual(except_infos[0].traceback, traceback)
def TaskRunner(queue, task, onexit=None, task_args=None, task_kwargs=None): """Run task(*input) for each input in the queue. Returns when it encounters an _AllTasksComplete object on the queue. If exceptions occur, save them off and re-raise them as a BackgroundFailure once we've finished processing the items in the queue. Args: queue: A queue of tasks to run. Add tasks to this queue, and they will be run. task: Function to run on each queued input. onexit: Function to run after all inputs are processed. task_args: A list of args to pass to the |task|. task_kwargs: A dict of optional args to pass to the |task|. """ if task_args is None: task_args = [] elif not isinstance(task_args, list): task_args = list(task_args) if task_kwargs is None: task_kwargs = {} errors = [] while True: # Wait for a new item to show up on the queue. This is a blocking wait, # so if there's nothing to do, we just sit here. x = queue.get() if isinstance(x, _AllTasksComplete): # All tasks are complete, so we should exit. break elif not isinstance(x, list): x = task_args + list(x) else: x = task_args + x # If no tasks failed yet, process the remaining tasks. if not errors: try: task(*x, **task_kwargs) except BaseException as ex: errors.extend( failures_lib.CreateExceptInfo(ex, traceback.format_exc())) # Run exit handlers. if onexit: onexit() # Propagate any exceptions. if errors: raise BackgroundFailure(exc_infos=errors)
def testReportStageFailureToCIDB(self): """Tests that the reporting fuction reports all included exceptions.""" fake_db = fake_cidb.FakeCIDBConnection() inner_exception_1 = failures_lib.TestLabFailure() inner_exception_2 = TypeError() exc_infos = failures_lib.CreateExceptInfo(inner_exception_1, None) exc_infos += failures_lib.CreateExceptInfo(inner_exception_2, None) outer_exception = failures_lib.GoBFailure(exc_infos=exc_infos) mock_build_stage_id = 9345 failures_lib.ReportStageFailureToCIDB(fake_db, mock_build_stage_id, outer_exception) self.assertEqual(3, len(fake_db.failureTable)) self.assertEqual( set([mock_build_stage_id]), set([x['build_stage_id'] for x in fake_db.failureTable.values()])) self.assertEqual( set([ constants.EXCEPTION_CATEGORY_INFRA, constants.EXCEPTION_CATEGORY_UNKNOWN, constants.EXCEPTION_CATEGORY_LAB ]), set([ x['exception_category'] for x in fake_db.failureTable.values() ])) # Find the outer failure id. for failure_id, failure in fake_db.failureTable.iteritems(): if failure['outer_failure_id'] is None: outer_failure_id = failure_id break # Now verify inner failures reference this failure. for failure_id, failure in fake_db.failureTable.iteritems(): if failure_id != outer_failure_id: self.assertEqual(outer_failure_id, failure['outer_failure_id'])
def run(self): """Run the list of steps.""" if self._semaphore is not None: self._semaphore.acquire() errors = failures_lib.CreateExceptInfo( UnexpectedException('Unexpected exception in %r' % self), '') pid = os.getpid() try: errors = self._Run() finally: if not self._killing.is_set() and os.getpid() == pid: results = results_lib.Results.Get() self._queue.put((errors, results)) if self._semaphore is not None: self._semaphore.release()
def Wait(self): """Wait for the task to complete. Output from the task is printed as it runs. If an exception occurs, return a string containing the traceback. """ try: # Flush stdout and stderr to be sure no output is interleaved. sys.stdout.flush() sys.stderr.flush() # File position pointers are shared across processes, so we must open # our own file descriptor to ensure output is not lost. self._WaitForStartup() silent_death_time = time.time() + self.SILENT_TIMEOUT results = [] with open(self._output.name, 'r') as output: pos = 0 running, exited_cleanly, task_errors, run_errors = (True, False, [], []) while running: # Check whether the process is still alive. running = self.is_alive() try: errors, results = \ self._queue.get(True, self.PRINT_INTERVAL) if errors: task_errors.extend(errors) running = False exited_cleanly = True except Queue.Empty: pass if not running: # Wait for the process to actually exit. If the child doesn't exit # in a timely fashion, kill it. self.join(self.EXIT_TIMEOUT) if self.exitcode is None: msg = '%r hung for %r seconds' % ( self, self.EXIT_TIMEOUT) run_errors.extend( failures_lib.CreateExceptInfo( ProcessExitTimeout(msg), '')) self._KillChildren([self]) elif not exited_cleanly: msg = ('%r exited unexpectedly with code %s' % (self, self.exitcode)) run_errors.extend( failures_lib.CreateExceptInfo( ProcessUnexpectedExit(msg), '')) # Read output from process. output.seek(pos) buf = output.read(_BUFSIZE) if buf: silent_death_time = time.time() + self.SILENT_TIMEOUT elif running and time.time() > silent_death_time: msg = ('No output from %r for %r seconds' % (self, self.SILENT_TIMEOUT)) run_errors.extend( failures_lib.CreateExceptInfo( ProcessSilentTimeout(msg), '')) self._KillChildren([self]) # Read remaining output from the process. output.seek(pos) buf = output.read(_BUFSIZE) running = False # Print output so far. while buf: sys.stdout.write(buf) pos += len(buf) if len(buf) < _BUFSIZE: break buf = output.read(_BUFSIZE) # Print error messages if anything exceptional occurred. if run_errors: logging.PrintBuildbotStepFailure() traceback.print_stack() logging.warning('\n'.join(x.str for x in run_errors if x)) logging.info('\n'.join(x.str for x in task_errors if x)) sys.stdout.flush() sys.stderr.flush() # Propagate any results. for result in results: results_lib.Results.Record(*result) finally: self.Cleanup(silent=True) # If an error occurred, return it. return run_errors + task_errors