def testReraiseACompoundFailure(self): """Tests that the list of ExceptInfo objects are copied over.""" tb1 = 'Dummy traceback1' tb2 = 'Dummy traceback2' org_infos = failures_lib.CreateExceptInfo(ValueError('No taco.'), tb1) + \ failures_lib.CreateExceptInfo(OSError('No salsa'), tb2) try: self._GetFunction(self.SubparLunch, self.TacoNotTasty, exc_infos=org_infos)() except Exception as e: self.assertTrue(isinstance(e, self.SubparLunch)) # The orignal exceptions stored in exc_infos are preserved. self.assertEqual(e.exc_infos, org_infos) # All essential inforamtion should be included in the message of # the new excpetion. self.assertTrue(tb1 in str(e)) self.assertTrue(tb2 in str(e)) self.assertTrue(str(ValueError) in str(e)) self.assertTrue(str(OSError) in str(e)) self.assertTrue(str('No taco') in str(e)) self.assertTrue(str('No salsa') in str(e)) # Assert that summary does not contain the textual tracebacks. self.assertFalse(tb1 in e.ToSummaryString()) self.assertFalse(tb2 in e.ToSummaryString())
def _Run(self): """Internal method for running the list of steps.""" # Register a handler for a signal that is rarely used. def trigger_bt(_sig_num, frame): logging.error('pre-kill notification (SIGXCPU); traceback:\n%s', ''.join(traceback.format_stack(frame))) signal.signal(signal.SIGXCPU, trigger_bt) sys.stdout.flush() sys.stderr.flush() errors = [] # Send all output to a named temporary file. with open(self._output.name, 'w', 0) as output: # Back up sys.std{err,out}. These aren't used, but we keep a copy so # that they aren't garbage collected. We intentionally don't restore # the old stdout and stderr at the end, because we want shutdown errors # to also be sent to the same log file. _orig_stdout, _orig_stderr = sys.stdout, sys.stderr # Replace std{out,err} with unbuffered file objects. os.dup2(output.fileno(), sys.__stdout__.fileno()) os.dup2(output.fileno(), sys.__stderr__.fileno()) sys.stdout = os.fdopen(sys.__stdout__.fileno(), 'w', 0) sys.stderr = os.fdopen(sys.__stderr__.fileno(), 'w', 0) try: self._started.set() results_lib.Results.Clear() # Reduce the silent timeout by the prescribed amount. cls = self.__class__ cls.SILENT_TIMEOUT -= cls.SILENT_TIMEOUT_STEP # Actually launch the task. self._task(*self._task_args, **self._task_kwargs) except failures_lib.StepFailure as ex: errors.extend(failures_lib.CreateExceptInfo( ex, traceback.format_exc())) except BaseException as ex: errors.extend(failures_lib.CreateExceptInfo( ex, traceback.format_exc())) if self._killing.is_set(): traceback.print_exc() finally: sys.stdout.flush() sys.stderr.flush() return errors
def _CreateExceptInfos(self, cls, message='', traceback='', num=1): """A helper function to create a list of ExceptInfo objects.""" exc_infos = [] for _ in xrange(num): exc_infos.extend( failures_lib.CreateExceptInfo(cls(message), traceback)) return exc_infos
def testConvertToExceptInfo(self): """Tests converting an exception to an ExceptInfo object.""" traceback = 'Dummy traceback' message = 'Taco is not a valid option!' except_infos = failures_lib.CreateExceptInfo(ValueError(message), traceback) self.assertEqual(except_infos[0].type, ValueError) self.assertEqual(except_infos[0].str, message) self.assertEqual(except_infos[0].traceback, traceback)
def TaskRunner(queue, task, onexit=None, task_args=None, task_kwargs=None): """Run task(*input) for each input in the queue. Returns when it encounters an _AllTasksComplete object on the queue. If exceptions occur, save them off and re-raise them as a BackgroundFailure once we've finished processing the items in the queue. Args: queue: A queue of tasks to run. Add tasks to this queue, and they will be run. task: Function to run on each queued input. onexit: Function to run after all inputs are processed. task_args: A list of args to pass to the |task|. task_kwargs: A dict of optional args to pass to the |task|. """ if task_args is None: task_args = [] elif not isinstance(task_args, list): task_args = list(task_args) if task_kwargs is None: task_kwargs = {} errors = [] while True: # Wait for a new item to show up on the queue. This is a blocking wait, # so if there's nothing to do, we just sit here. x = queue.get() if isinstance(x, _AllTasksComplete): # All tasks are complete, so we should exit. break elif not isinstance(x, list): x = task_args + list(x) else: x = task_args + x # If no tasks failed yet, process the remaining tasks. if not errors: try: task(*x, **task_kwargs) except BaseException as ex: errors.extend( failures_lib.CreateExceptInfo(ex, traceback.format_exc())) # Run exit handlers. if onexit: onexit() # Propagate any exceptions. if errors: raise BackgroundFailure(exc_infos=errors)
def testReportStageFailureToCIDB(self): """Tests that the reporting fuction reports all included exceptions.""" fake_db = fake_cidb.FakeCIDBConnection() inner_exception_1 = failures_lib.TestLabFailure() inner_exception_2 = TypeError() exc_infos = failures_lib.CreateExceptInfo(inner_exception_1, None) exc_infos += failures_lib.CreateExceptInfo(inner_exception_2, None) outer_exception = failures_lib.GoBFailure(exc_infos=exc_infos) mock_build_stage_id = 9345 failures_lib.ReportStageFailureToCIDB(fake_db, mock_build_stage_id, outer_exception) self.assertEqual(3, len(fake_db.failureTable)) self.assertEqual( set([mock_build_stage_id]), set([x['build_stage_id'] for x in fake_db.failureTable.values()])) self.assertEqual( set([ constants.EXCEPTION_CATEGORY_INFRA, constants.EXCEPTION_CATEGORY_UNKNOWN, constants.EXCEPTION_CATEGORY_LAB ]), set([ x['exception_category'] for x in fake_db.failureTable.values() ])) # Find the outer failure id. for failure_id, failure in fake_db.failureTable.iteritems(): if failure['outer_failure_id'] is None: outer_failure_id = failure_id break # Now verify inner failures reference this failure. for failure_id, failure in fake_db.failureTable.iteritems(): if failure_id != outer_failure_id: self.assertEqual(outer_failure_id, failure['outer_failure_id'])
def run(self): """Run the list of steps.""" if self._semaphore is not None: self._semaphore.acquire() errors = failures_lib.CreateExceptInfo( UnexpectedException('Unexpected exception in %r' % self), '') pid = os.getpid() try: errors = self._Run() finally: if not self._killing.is_set() and os.getpid() == pid: results = results_lib.Results.Get() self._queue.put((errors, results)) if self._semaphore is not None: self._semaphore.release()
def Wait(self): """Wait for the task to complete. Output from the task is printed as it runs. If an exception occurs, return a string containing the traceback. """ try: # Flush stdout and stderr to be sure no output is interleaved. sys.stdout.flush() sys.stderr.flush() # File position pointers are shared across processes, so we must open # our own file descriptor to ensure output is not lost. self._WaitForStartup() silent_death_time = time.time() + self.SILENT_TIMEOUT results = [] with open(self._output.name, 'r') as output: pos = 0 running, exited_cleanly, task_errors, all_errors = (True, False, [], []) while running: # Check whether the process is still alive. running = self.is_alive() try: errors, results = \ self._queue.get(True, self.PRINT_INTERVAL) if errors: task_errors.extend(errors) all_errors.extend(errors) running = False exited_cleanly = True except Queue.Empty: pass if not running: # Wait for the process to actually exit. If the child doesn't exit # in a timely fashion, kill it. self.join(self.EXIT_TIMEOUT) if self.exitcode is None: msg = '%r hung for %r seconds' % (self, self.EXIT_TIMEOUT) all_errors.extend( failures_lib.CreateExceptInfo(ProcessExitTimeout(msg), '')) self._KillChildren([self]) elif not exited_cleanly: msg = ('%r exited unexpectedly with code %s' % (self, self.exitcode)) all_errors.extend( failures_lib.CreateExceptInfo(ProcessUnexpectedExit(msg), '')) # Read output from process. output.seek(pos) buf = output.read(_BUFSIZE) if len(buf) > 0: silent_death_time = time.time() + self.SILENT_TIMEOUT elif running and time.time() > silent_death_time: msg = ('No output from %r for %r seconds' % (self, self.SILENT_TIMEOUT)) all_errors.extend( failures_lib.CreateExceptInfo(ProcessSilentTimeout(msg), '')) self._KillChildren([self]) # Read remaining output from the process. output.seek(pos) buf = output.read(_BUFSIZE) running = False # Print output so far. while len(buf) > 0: sys.stdout.write(buf) pos += len(buf) if len(buf) < _BUFSIZE: break buf = output.read(_BUFSIZE) # Print error messages if anything exceptional occurred. if len(all_errors) > len(task_errors): logging.PrintBuildbotStepFailure() msg = '\n'.join(x.str for x in all_errors if x) logging.warning(msg) traceback.print_stack() sys.stdout.flush() sys.stderr.flush() # Propagate any results. for result in results: results_lib.Results.Record(*result) finally: self.Cleanup(silent=True) # If an error occurred, return it. return all_errors