def main():
parser = argparse.ArgumentParser()
parser.add_argument('path', action=ReadableDir)
parser.add_argument('--timeouts', action='store_true')
parser.add_argument('--tablefmt', choices=tabulate.tabulate_formats, default='pipe')
args = parser.parse_args()
scenario_data = AclibResult.merge_several_runs(args.path)
timeouts = 0
runtime_data = [] # (runtime, label)
for scenario, data in scenario_data.iteritems():
response = np.array([float(r['Response Value (y)'])
for r in data.results])
if not args.timeouts and np.median(response) == 300:
timeouts += 1
continue
runtime_data.append((response, scenario))
if not args.timeouts:
print('Filtered %d timeouts' % timeouts)
runtime_data = sorted(runtime_data, key=lambda d: np.median(d[0]))
def ratio_outliers(data, m = 2.):
if len(data) == 1:
return 0.0
d = np.abs(data - np.median(data))
mdev = np.median(d)
s = d/mdev if mdev else 0.
return len(data[s>m]) / len(data)
header = ['Scenario', 'Median', 'Mean', 'Std', 'Ratio Outliers']
table = [[name, '%.2f' % np.median(data), '%.2f' % np.mean(data), '%.2f' % np.std(data), '%.2f' % ratio_outliers(data)] for data, name in runtime_data]
print np.corrcoef([np.std(d) for d,_ in runtime_data],[np.mean(d) for d,_ in runtime_data])
print np.average(np.array([np.std(d) for d,_ in runtime_data])/np.array([np.mean(d) for d,_ in runtime_data]))
print(tabulate.tabulate(table, header, tablefmt=args.tablefmt))
fig = plt.figure(figsize=(8.22, 5.08), dpi=300)
boxplot = fig.add_subplot(111)
boxplot.boxplot([time for (time, label) in runtime_data], vert=False)
boxplot.yaxis.set_ticklabels(
[label for (time, label) in runtime_data])
boxplot.yaxis.set_label_text('scenario')
boxplot.set_xscale('log')
boxplot.xaxis.set_label_text('runtime [s]')
plt.show()
def compare_single_lingeling_runs(paths):
run_datas = []
common_scenarios = set()
for path in paths:
result = AclibResult.merge_several_runs(path)
run_datas.append(result)
if not common_scenarios:
common_scenarios = set(result.keys())
common_scenarios.intersection_update(result.keys())
for scenario in common_scenarios:
print(scenario)
means = []
stds = []
for result in run_datas:
response = np.array([float(r['Response Value (y)'])
for r in result[scenario].results])
means.append(np.mean(response))
stds.append(np.std(response))
difference = np.abs(means[0] - means[1]) / (stds[0] * stds[1])
if difference > 30:
print(difference)
