def addError(self, test, err):
super(InteractiveFailureResult, self).addError(test, err)
log.error(self._exc_info_to_string(err, test))
log.error("Error in test '{0}', going interactive".format(
self.getDescription(test)
))
interactive.task(ctx=None, config=None)
def addError(self, test, err):
super(InteractiveFailureResult, self).addError(test, err)
log.error(self._exc_info_to_string(err, test))
log.error("Error in test '{0}', going interactive".format(
self.getDescription(test)
))
interactive.task(ctx=None, config=None)
def nested(*managers):
"""
Like contextlib.nested but takes callables returning context
managers, to avoid the major reason why contextlib.nested was
deprecated.
This version also logs any exceptions early, much like run_tasks,
to ease debugging. TODO combine nested and run_tasks.
"""
exits = []
vars = []
exc = (None, None, None)
try:
for mgr_fn in managers:
mgr = mgr_fn()
exit = mgr.__exit__
enter = mgr.__enter__
vars.append(enter())
exits.append(exit)
yield vars
except Exception:
log.exception('Saw exception from nested tasks')
exc = sys.exc_info()
# FIXME this needs to be more generic
if config.ctx and config.ctx.config.get('interactive-on-error'):
config.ctx.config['interactive-on-error'] = False
from teuthology.task import interactive
log.warning('Saw failure, going into interactive mode...')
interactive.task(ctx=config.ctx, config=None)
finally:
while exits:
exit = exits.pop()
try:
if exit(*exc):
exc = (None, None, None)
except Exception:
exc = sys.exc_info()
if exc != (None, None, None):
# Don't rely on sys.exc_info() still containing
# the right information. Another exception may
# have been raised and caught by an exit method
reraise(*exc)
def addError(self, test, err):
log.error(self._exc_info_to_string(err, test))
log.error("Error in test '{0}', going interactive".format(
self.getDescription(test)))
interactive.task(ctx=self.ctx, config=None)
def run_tasks(tasks, ctx):
archive_path = ctx.config.get('archive_path')
if archive_path:
timer = Timer(
path=os.path.join(archive_path, 'timing.yaml'),
sync=True,
)
else:
timer = Timer()
stack = []
try:
for taskdict in tasks:
try:
((taskname, config), ) = taskdict.items()
except (ValueError, AttributeError):
raise RuntimeError('Invalid task definition: %s' % taskdict)
log.info('Running task %s...', taskname)
timer.mark('%s enter' % taskname)
manager = run_one_task(taskname, ctx=ctx, config=config)
if hasattr(manager, '__enter__'):
stack.append((taskname, manager))
manager.__enter__()
except BaseException as e:
if isinstance(e, ConnectionLostError):
# Prevent connection issues being flagged as failures
set_status(ctx.summary, 'dead')
else:
# the status may have been set to dead, leave it as-is if so
if not ctx.summary.get('status', '') == 'dead':
set_status(ctx.summary, 'fail')
if 'failure_reason' not in ctx.summary:
ctx.summary['failure_reason'] = str(e)
log.exception('Saw exception from tasks.')
if teuth_config.sentry_dsn:
sentry_sdk.init(teuth_config.sentry_dsn)
config = deepcopy(ctx.config)
tags = {
'task': taskname,
'owner': ctx.owner,
}
optional_tags = ('teuthology_branch', 'branch', 'suite',
'machine_type', 'os_type', 'os_version')
for tag in optional_tags:
if tag in config:
tags[tag] = config[tag]
# Remove ssh keys from reported config
if 'targets' in config:
targets = config['targets']
for host in targets.keys():
targets[host] = '<redacted>'
job_id = ctx.config.get('job_id')
archive_path = ctx.config.get('archive_path')
extras = dict(config=config, )
if job_id:
extras['logs'] = get_http_log_path(archive_path, job_id)
fingerprint = e.fingerprint() if hasattr(e,
'fingerprint') else None
exc_id = sentry_sdk.capture_exception(
error=e,
tags=tags,
extras=extras,
fingerprint=fingerprint,
)
event_url = "{server}/?query={id}".format(
server=teuth_config.sentry_server.strip('/'), id=exc_id)
log.exception(" Sentry event: %s" % event_url)
ctx.summary['sentry_event'] = event_url
if ctx.config.get('interactive-on-error'):
ctx.config['interactive-on-error'] = False
from teuthology.task import interactive
log.warning(
'Saw failure during task execution, going into interactive mode...'
)
interactive.task(ctx=ctx, config=None)
# Throughout teuthology, (x,) = y has been used to assign values
# from yaml files where only one entry of type y is correct. This
# causes failures with 'too many values to unpack.' We want to
# fail as before, but with easier to understand error indicators.
if isinstance(e, ValueError):
if str(e) == 'too many values to unpack':
emsg = 'Possible configuration error in yaml file'
log.error(emsg)
ctx.summary['failure_info'] = emsg
finally:
try:
exc_info = sys.exc_info()
sleep_before_teardown = ctx.config.get('sleep_before_teardown')
if sleep_before_teardown:
log.info('Sleeping for {} seconds before unwinding because'
' --sleep-before-teardown was given...'.format(
sleep_before_teardown))
notify_sleep_before_teardown(ctx, stack, sleep_before_teardown)
time.sleep(sleep_before_teardown)
while stack:
taskname, manager = stack.pop()
log.debug('Unwinding manager %s', taskname)
timer.mark('%s exit' % taskname)
try:
suppress = manager.__exit__(*exc_info)
except Exception as e:
if isinstance(e, ConnectionLostError):
# Prevent connection issues being flagged as failures
set_status(ctx.summary, 'dead')
else:
set_status(ctx.summary, 'fail')
if 'failure_reason' not in ctx.summary:
ctx.summary['failure_reason'] = str(e)
log.exception('Manager failed: %s', taskname)
if exc_info == (None, None, None):
# if first failure is in an __exit__, we don't
# have exc_info set yet
exc_info = sys.exc_info()
if ctx.config.get('interactive-on-error'):
from teuthology.task import interactive
log.warning(
'Saw failure during task cleanup, going into interactive mode...'
)
interactive.task(ctx=ctx, config=None)
else:
if suppress:
exc_info = (None, None, None)
if exc_info != (None, None, None):
log.debug('Exception was not quenched, exiting: %s: %s',
exc_info[0].__name__, exc_info[1])
raise SystemExit(1)
finally:
# be careful about cyclic references
del exc_info
timer.mark("tasks complete")
def addError(self, test, err):
log.error(self._exc_info_to_string(err, test))
log.error("Error in test '{0}', going interactive".format(
self.getDescription(test)
))
interactive.task(ctx=self.ctx, config=None)