def failed_experiment_field(exp, stage_statuses, stage, notify=None):
message = 'Failed at stage {}:\n{}'.format(
stage,
textwrap.shorten(stage_statuses[stage]['message'], width=280))
if notify is not None:
message += '\nATTN: {}'.format(generate_ping_list(notify))
return build_field(title=exp, value=message)
def main(config_dir, home_dir, output_dir):
config = read_config(config_dir)
if 'channel_id' not in config:
write_status(output_dir, False, 'No channel token given')
return 1
channel = config['channel_id']
success, msg, client = new_client(config)
if not success:
write_status(output_dir, False, msg)
return 1
info = DashboardInfo(home_dir)
failed_subsys = []
reports = []
failed_reports = []
for subsys in info.all_present_subsystems():
# ignore self
if subsys == 'subsys_reporter':
continue
if not info.subsys_active(subsys):
continue
status = info.subsys_stage_status(subsys, 'run')
if not status['success']:
failed_subsys.append(failed_subsys_field(subsys, status))
continue
report_present = check_file_exists(info.subsys_output_dir(subsys),
'report.json')
if not report_present:
continue
try:
report = read_json(info.subsys_output_dir(subsys), 'report.json')
reports.append(
build_field(title=report['title'], value=report['value']))
except Exception:
failed_reports.append(subsys_name)
attachments = []
if reports:
attachments.append(build_attachment(title='Reports', fields=reports))
if failed_reports or failed_subsys:
failure_text = ''
if failed_reports:
failure_text = 'Failed to parse reports: {}'.format(
', '.join(failed_reports))
attachments.append(
build_attachment(title='Errors',
text=failure_text,
color='#fa0000',
fields=failed_subsys))
if not attachments:
write_status(output_dir, True, 'Nothing to report')
return 0
success, _, msg = post_message(
client, channel,
build_message(text='Subsystem Results', attachments=attachments))
write_status(output_dir, success, msg)
def failed_subsys_field(subsys, status):
message = 'Failed to run:\n{}'.format(
textwrap.shorten(status['message'], width=280))
return build_field(title=subsys, value=message)
def main(config_dir, home_dir, output_dir):
config = read_config(config_dir)
if 'webhook_url' not in config:
write_status(output_dir, False, 'No webhook URL given')
return 1
webhook = config['webhook_url']
if 'deadlines' not in config:
write_status(output_dir, True, 'No deadlines to report')
return 1
deadlines = config['deadlines']
if not isinstance(deadlines, dict):
write_status(output_dir, False, 'Invalid deadlines structure')
return 0
attachments = []
present = datetime.datetime.now()
for (name, info) in deadlines.items():
if 'date' not in info:
write_status(output_dir, False,
'Date missing in entry {} under {}'.format(info, name))
return 1
date = None
try:
date = datetime.datetime.strptime(info['date'], '%Y-%m-%d %H:%M:%S')
except Exception as e:
write_status(output_dir, False,
'Could not parse date {}'.format(info['date']))
return 1
diff = date - present
days_left = diff.days
if days_left < 0:
# elapsed, so forget it
continue
alert = days_left <= 7
time_left_msg = '{} days, {:.2f} hours left'.format(diff.days, diff.seconds/3600)
fields = [build_field(value=time_left_msg)]
if alert and 'ping' in info:
pings = generate_ping_list(info['ping'])
fields.append(build_field(value='Beware {}!'.format(pings)))
attachments.append(build_attachment(
title=name,
text='Deadline: {}'.format(info['date']),
fields=fields,
color='#fa0000' if alert else '#0fbf24'))
if not attachments:
write_status(output_dir, True, 'All deadlines elapsed')
return 0
success, report = post_message(
webhook,
build_message(
text='*Upcoming Deadlines*',
attachments=attachments))
write_status(output_dir, success, report)
def main(config_dir, home_dir, output_dir):
config = read_config(config_dir)
if 'webhook_url' not in config:
write_status(output_dir, False, 'No webhook URL given')
return 1
webhook = config['webhook_url']
description = ''
if 'description' in config:
description = config['description']
info = DashboardInfo(home_dir)
inactive_experiments = [] # list of titles
failed_experiments = [] # list of slack fields
successful_experiments = [] # list of slack fields
failed_graphs = [] # list of titles
for exp_name in info.all_present_experiments():
stage_statuses = info.exp_stage_statuses(exp_name)
if not stage_statuses['precheck']['success']:
failed_experiments.append(
failed_experiment_field(exp_name, stage_statuses, 'precheck'))
continue
exp_conf = info.read_exp_config(exp_name)
exp_title = exp_name if 'title' not in exp_conf else exp_conf['title']
notify = exp_conf['notify']
if not exp_conf['active']:
inactive_experiments.append(exp_title)
continue
failure = False
for stage in ['setup', 'run', 'analysis', 'summary']:
if stage not in stage_statuses:
# setup is the only stage that's optional
assert stage == 'setup'
continue
if not stage_statuses[stage]['success']:
failed_experiments.append(
failed_experiment_field(exp_title, stage_statuses,
stage, notify))
failure = True
break
if failure:
continue
# failure to visualize is not as big a deal as failing to
# run or analyze the experiment, so we only report it but
# don't fail to report the summary
if not stage_statuses['visualization']['success']:
failed_graphs.append(exp_title)
summary = info.read_exp_summary(exp_name)
successful_experiments.append(
build_field(summary['title'], summary['value']))
# produce messages
attachments = []
if successful_experiments:
attachments.append(
build_attachment(
title='Successful benchmarks',
pretext=description,
fields=successful_experiments))
if failed_experiments:
attachments.append(
build_attachment(
color='#fa0000',
title='Failed benchmarks',
fields=failed_experiments))
if inactive_experiments:
attachments.append(
build_attachment(
color='#616161',
title='Inactive benchmarks',
text=', '.join(inactive_experiments)))
if failed_graphs:
attachments.append(
build_attachment(
color='#fa0000',
title='Failed to Visualize',
text=', '.join(failed_graphs)))
success, report = post_message(
webhook,
build_message(
text='Dashboard Results',
attachments=attachments))
write_status(output_dir, success, report)
def main(config_dir, home_dir, output_dir):
config = read_config(config_dir)
if 'channel_id' not in config:
write_status(output_dir, False, 'No channel token given')
return 1
success, msg, client = new_client(config)
info = DashboardInfo(home_dir)
if not success:
write_status(output_dir, False, msg)
return 1
slack_channel = config['channel_id']
description = ''
if 'description' in config:
description = config['description']
info = DashboardInfo(home_dir)
inactive_experiments = [] # list of titles
failed_experiments = [] # list of slack fields
successful_experiments = [] # list of slack fields
failed_graphs = [] # list of titles
for exp_name in info.all_present_experiments():
stage_statuses = info.exp_stage_statuses(exp_name)
if not stage_statuses['precheck']['success']:
failed_experiments.append(
failed_experiment_field(exp_name, stage_statuses, 'precheck'))
continue
exp_conf = info.read_exp_config(exp_name)
exp_status = info.exp_status_dir(exp_name)
run_status = validate_json(exp_status,
'time_delta',
filename='run.json')
exp_title = exp_name if 'title' not in exp_conf else exp_conf['title']
notify = exp_conf['notify']
if not exp_conf['active']:
inactive_experiments.append(exp_title)
continue
failure = False
for stage in ['setup', 'run', 'analysis', 'summary']:
if stage not in stage_statuses:
# setup is the only stage that's optional
assert stage == 'setup'
continue
if not stage_statuses[stage]['success']:
failed_experiments.append(
failed_experiment_field(
exp_title,
stage_statuses,
stage,
duration=run_status.get('time_delta'),
notify=notify))
failure = True
break
if failure:
continue
# failure to visualize is not as big a deal as failing to
# run or analyze the experiment, so we only report it but
# don't fail to report the summary
if not stage_statuses['visualization']['success']:
failed_graphs.append(exp_title)
summary = info.read_exp_summary(exp_name)
successful_experiments.append(
build_field(
summary['title'],
attach_duration(summary['value'],
run_status.get('time_delta'))))
# produce messages
attachments = []
if successful_experiments:
attachments.append(
build_attachment(title='Successful benchmarks',
fields=successful_experiments))
if failed_experiments:
attachments.append(
build_attachment(color='#fa0000',
title='Failed benchmarks',
fields=failed_experiments))
if inactive_experiments:
attachments.append(
build_attachment(color='#616161',
title='Inactive benchmarks',
text=', '.join(inactive_experiments)))
if failed_graphs:
attachments.append(
build_attachment(color='#fa0000',
title='Failed to Visualize',
text=', '.join(failed_graphs)))
success, _, report = post_message(
client, slack_channel,
build_message(text='*Dashboard Results*{}'.format(
'\n' + description if description != '' else ''),
attachments=attachments))
if config.get('report_images', False):
success, msg = send_graphs(config, info, client, output_dir)
if not success:
write_status(output_dir, False, msg)
return 1
write_status(output_dir, success, report)
