def failed_experiment_field(exp, stage_statuses, stage, notify=None): message = 'Failed at stage {}:\n{}'.format( stage, textwrap.shorten(stage_statuses[stage]['message'], width=280)) if notify is not None: message += '\nATTN: {}'.format(generate_ping_list(notify)) return build_field(title=exp, value=message)
def main(config_dir, home_dir, output_dir): config = read_config(config_dir) if 'channel_id' not in config: write_status(output_dir, False, 'No channel token given') return 1 channel = config['channel_id'] success, msg, client = new_client(config) if not success: write_status(output_dir, False, msg) return 1 info = DashboardInfo(home_dir) failed_subsys = [] reports = [] failed_reports = [] for subsys in info.all_present_subsystems(): # ignore self if subsys == 'subsys_reporter': continue if not info.subsys_active(subsys): continue status = info.subsys_stage_status(subsys, 'run') if not status['success']: failed_subsys.append(failed_subsys_field(subsys, status)) continue report_present = check_file_exists(info.subsys_output_dir(subsys), 'report.json') if not report_present: continue try: report = read_json(info.subsys_output_dir(subsys), 'report.json') reports.append( build_field(title=report['title'], value=report['value'])) except Exception: failed_reports.append(subsys_name) attachments = [] if reports: attachments.append(build_attachment(title='Reports', fields=reports)) if failed_reports or failed_subsys: failure_text = '' if failed_reports: failure_text = 'Failed to parse reports: {}'.format( ', '.join(failed_reports)) attachments.append( build_attachment(title='Errors', text=failure_text, color='#fa0000', fields=failed_subsys)) if not attachments: write_status(output_dir, True, 'Nothing to report') return 0 success, _, msg = post_message( client, channel, build_message(text='Subsystem Results', attachments=attachments)) write_status(output_dir, success, msg)
def failed_subsys_field(subsys, status): message = 'Failed to run:\n{}'.format( textwrap.shorten(status['message'], width=280)) return build_field(title=subsys, value=message)
def main(config_dir, home_dir, output_dir): config = read_config(config_dir) if 'webhook_url' not in config: write_status(output_dir, False, 'No webhook URL given') return 1 webhook = config['webhook_url'] if 'deadlines' not in config: write_status(output_dir, True, 'No deadlines to report') return 1 deadlines = config['deadlines'] if not isinstance(deadlines, dict): write_status(output_dir, False, 'Invalid deadlines structure') return 0 attachments = [] present = datetime.datetime.now() for (name, info) in deadlines.items(): if 'date' not in info: write_status(output_dir, False, 'Date missing in entry {} under {}'.format(info, name)) return 1 date = None try: date = datetime.datetime.strptime(info['date'], '%Y-%m-%d %H:%M:%S') except Exception as e: write_status(output_dir, False, 'Could not parse date {}'.format(info['date'])) return 1 diff = date - present days_left = diff.days if days_left < 0: # elapsed, so forget it continue alert = days_left <= 7 time_left_msg = '{} days, {:.2f} hours left'.format(diff.days, diff.seconds/3600) fields = [build_field(value=time_left_msg)] if alert and 'ping' in info: pings = generate_ping_list(info['ping']) fields.append(build_field(value='Beware {}!'.format(pings))) attachments.append(build_attachment( title=name, text='Deadline: {}'.format(info['date']), fields=fields, color='#fa0000' if alert else '#0fbf24')) if not attachments: write_status(output_dir, True, 'All deadlines elapsed') return 0 success, report = post_message( webhook, build_message( text='*Upcoming Deadlines*', attachments=attachments)) write_status(output_dir, success, report)
def main(config_dir, home_dir, output_dir): config = read_config(config_dir) if 'webhook_url' not in config: write_status(output_dir, False, 'No webhook URL given') return 1 webhook = config['webhook_url'] description = '' if 'description' in config: description = config['description'] info = DashboardInfo(home_dir) inactive_experiments = [] # list of titles failed_experiments = [] # list of slack fields successful_experiments = [] # list of slack fields failed_graphs = [] # list of titles for exp_name in info.all_present_experiments(): stage_statuses = info.exp_stage_statuses(exp_name) if not stage_statuses['precheck']['success']: failed_experiments.append( failed_experiment_field(exp_name, stage_statuses, 'precheck')) continue exp_conf = info.read_exp_config(exp_name) exp_title = exp_name if 'title' not in exp_conf else exp_conf['title'] notify = exp_conf['notify'] if not exp_conf['active']: inactive_experiments.append(exp_title) continue failure = False for stage in ['setup', 'run', 'analysis', 'summary']: if stage not in stage_statuses: # setup is the only stage that's optional assert stage == 'setup' continue if not stage_statuses[stage]['success']: failed_experiments.append( failed_experiment_field(exp_title, stage_statuses, stage, notify)) failure = True break if failure: continue # failure to visualize is not as big a deal as failing to # run or analyze the experiment, so we only report it but # don't fail to report the summary if not stage_statuses['visualization']['success']: failed_graphs.append(exp_title) summary = info.read_exp_summary(exp_name) successful_experiments.append( build_field(summary['title'], summary['value'])) # produce messages attachments = [] if successful_experiments: attachments.append( build_attachment( title='Successful benchmarks', pretext=description, fields=successful_experiments)) if failed_experiments: attachments.append( build_attachment( color='#fa0000', title='Failed benchmarks', fields=failed_experiments)) if inactive_experiments: attachments.append( build_attachment( color='#616161', title='Inactive benchmarks', text=', '.join(inactive_experiments))) if failed_graphs: attachments.append( build_attachment( color='#fa0000', title='Failed to Visualize', text=', '.join(failed_graphs))) success, report = post_message( webhook, build_message( text='Dashboard Results', attachments=attachments)) write_status(output_dir, success, report)
def main(config_dir, home_dir, output_dir): config = read_config(config_dir) if 'channel_id' not in config: write_status(output_dir, False, 'No channel token given') return 1 success, msg, client = new_client(config) info = DashboardInfo(home_dir) if not success: write_status(output_dir, False, msg) return 1 slack_channel = config['channel_id'] description = '' if 'description' in config: description = config['description'] info = DashboardInfo(home_dir) inactive_experiments = [] # list of titles failed_experiments = [] # list of slack fields successful_experiments = [] # list of slack fields failed_graphs = [] # list of titles for exp_name in info.all_present_experiments(): stage_statuses = info.exp_stage_statuses(exp_name) if not stage_statuses['precheck']['success']: failed_experiments.append( failed_experiment_field(exp_name, stage_statuses, 'precheck')) continue exp_conf = info.read_exp_config(exp_name) exp_status = info.exp_status_dir(exp_name) run_status = validate_json(exp_status, 'time_delta', filename='run.json') exp_title = exp_name if 'title' not in exp_conf else exp_conf['title'] notify = exp_conf['notify'] if not exp_conf['active']: inactive_experiments.append(exp_title) continue failure = False for stage in ['setup', 'run', 'analysis', 'summary']: if stage not in stage_statuses: # setup is the only stage that's optional assert stage == 'setup' continue if not stage_statuses[stage]['success']: failed_experiments.append( failed_experiment_field( exp_title, stage_statuses, stage, duration=run_status.get('time_delta'), notify=notify)) failure = True break if failure: continue # failure to visualize is not as big a deal as failing to # run or analyze the experiment, so we only report it but # don't fail to report the summary if not stage_statuses['visualization']['success']: failed_graphs.append(exp_title) summary = info.read_exp_summary(exp_name) successful_experiments.append( build_field( summary['title'], attach_duration(summary['value'], run_status.get('time_delta')))) # produce messages attachments = [] if successful_experiments: attachments.append( build_attachment(title='Successful benchmarks', fields=successful_experiments)) if failed_experiments: attachments.append( build_attachment(color='#fa0000', title='Failed benchmarks', fields=failed_experiments)) if inactive_experiments: attachments.append( build_attachment(color='#616161', title='Inactive benchmarks', text=', '.join(inactive_experiments))) if failed_graphs: attachments.append( build_attachment(color='#fa0000', title='Failed to Visualize', text=', '.join(failed_graphs))) success, _, report = post_message( client, slack_channel, build_message(text='*Dashboard Results*{}'.format( '\n' + description if description != '' else ''), attachments=attachments)) if config.get('report_images', False): success, msg = send_graphs(config, info, client, output_dir) if not success: write_status(output_dir, False, msg) return 1 write_status(output_dir, success, report)