def build_email_body(ctx, stack, sleep_time_sec): email_template_path = os.path.dirname(__file__) + \ '/templates/email-sleep-before-teardown.jinja2' with open(email_template_path) as f: template_text = f.read() email_template = jinja2.Template(template_text) archive_path = ctx.config.get('archive_path') job_id = ctx.config.get('job_id') status = get_status(ctx.summary) stack_path = '/'.join(task for task, _ in stack) suite_name = ctx.config.get('suite') sleep_date = time.time() sleep_date_str = time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(sleep_date)) body = email_template.render( sleep_time=format_timespan(sleep_time_sec), sleep_time_sec=sleep_time_sec, sleep_date=sleep_date_str, owner=ctx.owner, run_name=ctx.name, job_id=ctx.config.get('job_id'), job_info=get_results_url(ctx.name), job_logs=get_http_log_path(archive_path, job_id), suite_name=suite_name, status=status, task_stack=stack_path, ) subject = ('teuthology job {run}/{job} has fallen asleep at {date}'.format( run=ctx.name, job=job_id, date=sleep_date_str)) return (subject.strip(), body.strip())
def build_rocketchat_message(ctx, stack, sleep_time_sec, template_path=None): message_template_path = template_path or os.path.dirname(__file__) + \ '/templates/rocketchat-sleep-before-teardown.jinja2' with open(message_template_path) as f: template_text = f.read() template = jinja2.Template(template_text) archive_path = ctx.config.get('archive_path') job_id = ctx.config.get('job_id') status = get_status(ctx.summary) stack_path = ' -> '.join(task for task, _ in stack) suite_name = ctx.config.get('suite') sleep_date = time.time() sleep_date_str = time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(sleep_date)) message = template.render( sleep_time=format_timespan(sleep_time_sec), sleep_time_sec=sleep_time_sec, sleep_date=sleep_date_str, owner=ctx.owner, run_name=ctx.name, job_id=ctx.config.get('job_id'), job_desc=ctx.config.get('description'), job_info=get_results_url(ctx.name, job_id), job_logs=get_http_log_path(archive_path, job_id), suite_name=suite_name, status=status, task_stack=stack_path, ) return message
def test_get_http_log_path(): # Fake configuration archive_server = "http://example.com/server_root" config.archive_server = archive_server archive_dir = "/var/www/archives" path = misc.get_http_log_path(archive_dir) assert path == "http://example.com/server_root/archives/" job_id = '12345' path = misc.get_http_log_path(archive_dir, job_id) assert path == "http://example.com/server_root/archives/12345/" # Inktank configuration archive_server = "http://qa-proxy.ceph.com/teuthology/" config.archive_server = archive_server archive_dir = "/var/lib/teuthworker/archive/teuthology-2013-09-12_11:49:50-ceph-deploy-master-testing-basic-vps" job_id = 31087 path = misc.get_http_log_path(archive_dir, job_id) assert path == "http://qa-proxy.ceph.com/teuthology/teuthology-2013-09-12_11:49:50-ceph-deploy-master-testing-basic-vps/31087/" path = misc.get_http_log_path(archive_dir) assert path == "http://qa-proxy.ceph.com/teuthology/teuthology-2013-09-12_11:49:50-ceph-deploy-master-testing-basic-vps/"
def build_email_body(name, archive_dir, timeout): failed = {} hung = {} passed = {} for job in get_jobs(archive_dir): job_dir = os.path.join(archive_dir, job) summary_file = os.path.join(job_dir, 'summary.yaml') # Unfinished jobs will have no summary.yaml if not os.path.exists(summary_file): info_file = os.path.join(job_dir, 'info.yaml') desc = '' if os.path.exists(info_file): with file(info_file) as f: info = yaml.safe_load(f) desc = info['description'] hung[job] = email_templates['hung_templ'].format( job_id=job, desc=desc, ) continue with file(summary_file) as f: summary = yaml.safe_load(f) if summary['success']: passed[job] = email_templates['pass_templ'].format( job_id=job, desc=summary.get('description'), time=int(summary.get('duration', 0)), ) else: log = misc.get_http_log_path(archive_dir, job) if log: log_line = email_templates['fail_log_templ'].format(log=log) else: log_line = '' sentry_events = summary.get('sentry_events') if sentry_events: sentry_line = email_templates['fail_sentry_templ'].format( sentries='\n '.join(sentry_events)) else: sentry_line = '' # 'fill' is from the textwrap module and it collapses a given # string into multiple lines of a maximum width as specified. We # want 75 characters here so that when we indent by 4 on the next # line, we have 79-character exception paragraphs. reason = fill(summary.get('failure_reason'), 75) reason = '\n'.join((' ') + line for line in reason.splitlines()) failed[job] = email_templates['fail_templ'].format( job_id=job, desc=summary.get('description'), time=int(summary.get('duration', 0)), reason=reason, log_line=log_line, sentry_line=sentry_line, ) maybe_comma = lambda s: ', ' if s else ' ' subject = '' fail_sect = '' hung_sect = '' pass_sect = '' if failed: subject += '{num_failed} failed{sep}'.format( num_failed=len(failed), sep=maybe_comma(hung or passed) ) fail_sect = email_templates['sect_templ'].format( title='Failed', jobs=''.join(failed.values()) ) if hung: subject += '{num_hung} hung{sep}'.format( num_hung=len(hung), sep=maybe_comma(passed), ) hung_sect = email_templates['sect_templ'].format( title='Hung', jobs=''.join(hung.values()), ) if passed: subject += '%s passed ' % len(passed) pass_sect = email_templates['sect_templ'].format( title='Passed', jobs=''.join(passed.values()), ) body = email_templates['body_templ'].format( name=name, log_root=misc.get_http_log_path(archive_dir), fail_count=len(failed), hung_count=len(hung), pass_count=len(passed), fail_sect=fail_sect, hung_sect=hung_sect, pass_sect=pass_sect, ) subject += 'in {suite}'.format(suite=name) return (subject.strip(), body.strip())
def build_email_body(name, archive_dir, timeout): failed = {} hung = {} passed = {} for job in get_jobs(archive_dir): job_dir = os.path.join(archive_dir, job) summary_file = os.path.join(job_dir, 'summary.yaml') # Unfinished jobs will have no summary.yaml if not os.path.exists(summary_file): info_file = os.path.join(job_dir, 'info.yaml') desc = '' if os.path.exists(info_file): with file(info_file) as f: info = yaml.safe_load(f) desc = info['description'] hung[job] = email_templates['hung_templ'].format( job_id=job, desc=desc, ) continue with file(summary_file) as f: summary = yaml.safe_load(f) if summary['success']: passed[job] = email_templates['pass_templ'].format( job_id=job, desc=summary.get('description'), time=int(summary.get('duration', 0)), ) else: log = misc.get_http_log_path(archive_dir, job) if log: log_line = email_templates['fail_log_templ'].format(log=log) else: log_line = '' sentry_events = summary.get('sentry_events') if sentry_events: sentry_line = email_templates['fail_sentry_templ'].format( sentries='\n '.join(sentry_events)) else: sentry_line = '' # 'fill' is from the textwrap module and it collapses a given # string into multiple lines of a maximum width as specified. We # want 75 characters here so that when we indent by 4 on the next # line, we have 79-character exception paragraphs. reason = fill(summary.get('failure_reason'), 75) reason = '\n'.join((' ') + line for line in reason.splitlines()) failed[job] = email_templates['fail_templ'].format( job_id=job, desc=summary.get('description'), time=int(summary.get('duration', 0)), reason=reason, log_line=log_line, sentry_line=sentry_line, ) maybe_comma = lambda s: ', ' if s else ' ' subject = '' fail_sect = '' hung_sect = '' pass_sect = '' if failed: subject += '{num_failed} failed{sep}'.format(num_failed=len(failed), sep=maybe_comma( hung or passed)) fail_sect = email_templates['sect_templ'].format(title='Failed', jobs=''.join( failed.values())) if hung: subject += '{num_hung} hung{sep}'.format( num_hung=len(hung), sep=maybe_comma(passed), ) hung_sect = email_templates['sect_templ'].format( title='Hung', jobs=''.join(hung.values()), ) if passed: subject += '%s passed ' % len(passed) pass_sect = email_templates['sect_templ'].format( title='Passed', jobs=''.join(passed.values()), ) body = email_templates['body_templ'].format( name=name, log_root=misc.get_http_log_path(archive_dir), fail_count=len(failed), hung_count=len(hung), pass_count=len(passed), fail_sect=fail_sect, hung_sect=hung_sect, pass_sect=pass_sect, ) subject += 'in {suite}'.format(suite=name) return (subject.strip(), body.strip())
def run_tasks(tasks, ctx): archive_path = ctx.config.get('archive_path') if archive_path: timer = Timer( path=os.path.join(archive_path, 'timing.yaml'), sync=True, ) else: timer = Timer() stack = [] try: for taskdict in tasks: try: ((taskname, config), ) = taskdict.items() except (ValueError, AttributeError): raise RuntimeError('Invalid task definition: %s' % taskdict) log.info('Running task %s...', taskname) timer.mark('%s enter' % taskname) manager = run_one_task(taskname, ctx=ctx, config=config) if hasattr(manager, '__enter__'): stack.append((taskname, manager)) manager.__enter__() except BaseException as e: if isinstance(e, ConnectionLostError): # Prevent connection issues being flagged as failures set_status(ctx.summary, 'dead') else: # the status may have been set to dead, leave it as-is if so if not ctx.summary.get('status', '') == 'dead': set_status(ctx.summary, 'fail') if 'failure_reason' not in ctx.summary: ctx.summary['failure_reason'] = str(e) log.exception('Saw exception from tasks.') if teuth_config.sentry_dsn: sentry_sdk.init(teuth_config.sentry_dsn) config = deepcopy(ctx.config) tags = { 'task': taskname, 'owner': ctx.owner, } optional_tags = ('teuthology_branch', 'branch', 'suite', 'machine_type', 'os_type', 'os_version') for tag in optional_tags: if tag in config: tags[tag] = config[tag] # Remove ssh keys from reported config if 'targets' in config: targets = config['targets'] for host in targets.keys(): targets[host] = '<redacted>' job_id = ctx.config.get('job_id') archive_path = ctx.config.get('archive_path') extras = dict(config=config, ) if job_id: extras['logs'] = get_http_log_path(archive_path, job_id) fingerprint = e.fingerprint() if hasattr(e, 'fingerprint') else None exc_id = sentry_sdk.capture_exception( error=e, tags=tags, extras=extras, fingerprint=fingerprint, ) event_url = "{server}/?query={id}".format( server=teuth_config.sentry_server.strip('/'), id=exc_id) log.exception(" Sentry event: %s" % event_url) ctx.summary['sentry_event'] = event_url if ctx.config.get('interactive-on-error'): ctx.config['interactive-on-error'] = False from teuthology.task import interactive log.warning( 'Saw failure during task execution, going into interactive mode...' ) interactive.task(ctx=ctx, config=None) # Throughout teuthology, (x,) = y has been used to assign values # from yaml files where only one entry of type y is correct. This # causes failures with 'too many values to unpack.' We want to # fail as before, but with easier to understand error indicators. if isinstance(e, ValueError): if str(e) == 'too many values to unpack': emsg = 'Possible configuration error in yaml file' log.error(emsg) ctx.summary['failure_info'] = emsg finally: try: exc_info = sys.exc_info() sleep_before_teardown = ctx.config.get('sleep_before_teardown') if sleep_before_teardown: log.info('Sleeping for {} seconds before unwinding because' ' --sleep-before-teardown was given...'.format( sleep_before_teardown)) notify_sleep_before_teardown(ctx, stack, sleep_before_teardown) time.sleep(sleep_before_teardown) while stack: taskname, manager = stack.pop() log.debug('Unwinding manager %s', taskname) timer.mark('%s exit' % taskname) try: suppress = manager.__exit__(*exc_info) except Exception as e: if isinstance(e, ConnectionLostError): # Prevent connection issues being flagged as failures set_status(ctx.summary, 'dead') else: set_status(ctx.summary, 'fail') if 'failure_reason' not in ctx.summary: ctx.summary['failure_reason'] = str(e) log.exception('Manager failed: %s', taskname) if exc_info == (None, None, None): # if first failure is in an __exit__, we don't # have exc_info set yet exc_info = sys.exc_info() if ctx.config.get('interactive-on-error'): from teuthology.task import interactive log.warning( 'Saw failure during task cleanup, going into interactive mode...' ) interactive.task(ctx=ctx, config=None) else: if suppress: exc_info = (None, None, None) if exc_info != (None, None, None): log.debug('Exception was not quenched, exiting: %s: %s', exc_info[0].__name__, exc_info[1]) raise SystemExit(1) finally: # be careful about cyclic references del exc_info timer.mark("tasks complete")