def addError(self, test, err): super(InteractiveFailureResult, self).addError(test, err) log.error(self._exc_info_to_string(err, test)) log.error("Error in test '{0}', going interactive".format( self.getDescription(test) )) interactive.task(ctx=None, config=None)
def nested(*managers): """ Like contextlib.nested but takes callables returning context managers, to avoid the major reason why contextlib.nested was deprecated. This version also logs any exceptions early, much like run_tasks, to ease debugging. TODO combine nested and run_tasks. """ exits = [] vars = [] exc = (None, None, None) try: for mgr_fn in managers: mgr = mgr_fn() exit = mgr.__exit__ enter = mgr.__enter__ vars.append(enter()) exits.append(exit) yield vars except Exception: log.exception('Saw exception from nested tasks') exc = sys.exc_info() # FIXME this needs to be more generic if config.ctx and config.ctx.config.get('interactive-on-error'): config.ctx.config['interactive-on-error'] = False from teuthology.task import interactive log.warning('Saw failure, going into interactive mode...') interactive.task(ctx=config.ctx, config=None) finally: while exits: exit = exits.pop() try: if exit(*exc): exc = (None, None, None) except Exception: exc = sys.exc_info() if exc != (None, None, None): # Don't rely on sys.exc_info() still containing # the right information. Another exception may # have been raised and caught by an exit method reraise(*exc)
def addError(self, test, err): log.error(self._exc_info_to_string(err, test)) log.error("Error in test '{0}', going interactive".format( self.getDescription(test))) interactive.task(ctx=self.ctx, config=None)
def run_tasks(tasks, ctx): archive_path = ctx.config.get('archive_path') if archive_path: timer = Timer( path=os.path.join(archive_path, 'timing.yaml'), sync=True, ) else: timer = Timer() stack = [] try: for taskdict in tasks: try: ((taskname, config), ) = taskdict.items() except (ValueError, AttributeError): raise RuntimeError('Invalid task definition: %s' % taskdict) log.info('Running task %s...', taskname) timer.mark('%s enter' % taskname) manager = run_one_task(taskname, ctx=ctx, config=config) if hasattr(manager, '__enter__'): stack.append((taskname, manager)) manager.__enter__() except BaseException as e: if isinstance(e, ConnectionLostError): # Prevent connection issues being flagged as failures set_status(ctx.summary, 'dead') else: # the status may have been set to dead, leave it as-is if so if not ctx.summary.get('status', '') == 'dead': set_status(ctx.summary, 'fail') if 'failure_reason' not in ctx.summary: ctx.summary['failure_reason'] = str(e) log.exception('Saw exception from tasks.') if teuth_config.sentry_dsn: sentry_sdk.init(teuth_config.sentry_dsn) config = deepcopy(ctx.config) tags = { 'task': taskname, 'owner': ctx.owner, } optional_tags = ('teuthology_branch', 'branch', 'suite', 'machine_type', 'os_type', 'os_version') for tag in optional_tags: if tag in config: tags[tag] = config[tag] # Remove ssh keys from reported config if 'targets' in config: targets = config['targets'] for host in targets.keys(): targets[host] = '<redacted>' job_id = ctx.config.get('job_id') archive_path = ctx.config.get('archive_path') extras = dict(config=config, ) if job_id: extras['logs'] = get_http_log_path(archive_path, job_id) fingerprint = e.fingerprint() if hasattr(e, 'fingerprint') else None exc_id = sentry_sdk.capture_exception( error=e, tags=tags, extras=extras, fingerprint=fingerprint, ) event_url = "{server}/?query={id}".format( server=teuth_config.sentry_server.strip('/'), id=exc_id) log.exception(" Sentry event: %s" % event_url) ctx.summary['sentry_event'] = event_url if ctx.config.get('interactive-on-error'): ctx.config['interactive-on-error'] = False from teuthology.task import interactive log.warning( 'Saw failure during task execution, going into interactive mode...' ) interactive.task(ctx=ctx, config=None) # Throughout teuthology, (x,) = y has been used to assign values # from yaml files where only one entry of type y is correct. This # causes failures with 'too many values to unpack.' We want to # fail as before, but with easier to understand error indicators. if isinstance(e, ValueError): if str(e) == 'too many values to unpack': emsg = 'Possible configuration error in yaml file' log.error(emsg) ctx.summary['failure_info'] = emsg finally: try: exc_info = sys.exc_info() sleep_before_teardown = ctx.config.get('sleep_before_teardown') if sleep_before_teardown: log.info('Sleeping for {} seconds before unwinding because' ' --sleep-before-teardown was given...'.format( sleep_before_teardown)) notify_sleep_before_teardown(ctx, stack, sleep_before_teardown) time.sleep(sleep_before_teardown) while stack: taskname, manager = stack.pop() log.debug('Unwinding manager %s', taskname) timer.mark('%s exit' % taskname) try: suppress = manager.__exit__(*exc_info) except Exception as e: if isinstance(e, ConnectionLostError): # Prevent connection issues being flagged as failures set_status(ctx.summary, 'dead') else: set_status(ctx.summary, 'fail') if 'failure_reason' not in ctx.summary: ctx.summary['failure_reason'] = str(e) log.exception('Manager failed: %s', taskname) if exc_info == (None, None, None): # if first failure is in an __exit__, we don't # have exc_info set yet exc_info = sys.exc_info() if ctx.config.get('interactive-on-error'): from teuthology.task import interactive log.warning( 'Saw failure during task cleanup, going into interactive mode...' ) interactive.task(ctx=ctx, config=None) else: if suppress: exc_info = (None, None, None) if exc_info != (None, None, None): log.debug('Exception was not quenched, exiting: %s: %s', exc_info[0].__name__, exc_info[1]) raise SystemExit(1) finally: # be careful about cyclic references del exc_info timer.mark("tasks complete")
def addError(self, test, err): log.error(self._exc_info_to_string(err, test)) log.error("Error in test '{0}', going interactive".format( self.getDescription(test) )) interactive.task(ctx=self.ctx, config=None)