def main(): parser = argparse.ArgumentParser() parser.add_argument('path', action=ReadableDir) parser.add_argument('--timeouts', action='store_true') parser.add_argument('--tablefmt', choices=tabulate.tabulate_formats, default='pipe') args = parser.parse_args() scenario_data = AclibResult.merge_several_runs(args.path) timeouts = 0 runtime_data = [] # (runtime, label) for scenario, data in scenario_data.iteritems(): response = np.array([float(r['Response Value (y)']) for r in data.results]) if not args.timeouts and np.median(response) == 300: timeouts += 1 continue runtime_data.append((response, scenario)) if not args.timeouts: print('Filtered %d timeouts' % timeouts) runtime_data = sorted(runtime_data, key=lambda d: np.median(d[0])) def ratio_outliers(data, m = 2.): if len(data) == 1: return 0.0 d = np.abs(data - np.median(data)) mdev = np.median(d) s = d/mdev if mdev else 0. return len(data[s>m]) / len(data) header = ['Scenario', 'Median', 'Mean', 'Std', 'Ratio Outliers'] table = [[name, '%.2f' % np.median(data), '%.2f' % np.mean(data), '%.2f' % np.std(data), '%.2f' % ratio_outliers(data)] for data, name in runtime_data] print np.corrcoef([np.std(d) for d,_ in runtime_data],[np.mean(d) for d,_ in runtime_data]) print np.average(np.array([np.std(d) for d,_ in runtime_data])/np.array([np.mean(d) for d,_ in runtime_data])) print(tabulate.tabulate(table, header, tablefmt=args.tablefmt)) fig = plt.figure(figsize=(8.22, 5.08), dpi=300) boxplot = fig.add_subplot(111) boxplot.boxplot([time for (time, label) in runtime_data], vert=False) boxplot.yaxis.set_ticklabels( [label for (time, label) in runtime_data]) boxplot.yaxis.set_label_text('scenario') boxplot.set_xscale('log') boxplot.xaxis.set_label_text('runtime [s]') plt.show()
def compare_single_lingeling_runs(paths): run_datas = [] common_scenarios = set() for path in paths: result = AclibResult.merge_several_runs(path) run_datas.append(result) if not common_scenarios: common_scenarios = set(result.keys()) common_scenarios.intersection_update(result.keys()) for scenario in common_scenarios: print(scenario) means = [] stds = [] for result in run_datas: response = np.array([float(r['Response Value (y)']) for r in result[scenario].results]) means.append(np.mean(response)) stds.append(np.std(response)) difference = np.abs(means[0] - means[1]) / (stds[0] * stds[1]) if difference > 30: print(difference)