def test_percentile(self):
# randomized range(10)
values = [4, 6, 9, 7, 5, 8, 3, 0, 1, 2]
self.assertEqual(utils.percentile(values, 0.00), 0)
self.assertEqual(utils.percentile(values, 0.25), 2.25)
self.assertEqual(utils.percentile(values, 0.50), 4.5)
self.assertEqual(utils.percentile(values, 0.75), 6.75)
self.assertEqual(utils.percentile(values, 1.00), 9)
def percentile(self, p):
if not (0 <= p <= 100):
raise ValueError("p must be in the range [0; 100]")
return percentile(self.get_values(), p / 100.0)
def test_calibrate_warmups(self, nwarmup, unit):
half = nwarmup + (len(self.warmups) - nwarmup) // 2
sample1 = [value for loops, value in self.warmups[nwarmup:half]]
sample2 = [value for loops, value in self.warmups[half:]]
first_value = sample1[0]
# test if the first value is an outlier
values = sample1[1:] + sample2
q1 = percentile(values, 0.25)
q3 = percentile(values, 0.75)
iqr = q3 - q1
outlier_max = (q3 + 1.5 * iqr)
# only check maximum, not minimum
outlier = not(first_value <= outlier_max)
mean1 = statistics.mean(sample1)
mean2 = statistics.mean(sample2)
mean_diff = (mean1 - mean2) / float(mean2)
s1_q1 = percentile(sample1, 0.25)
s2_q1 = percentile(sample2, 0.25)
s1_q3 = percentile(sample1, 0.75)
s2_q3 = percentile(sample2, 0.75)
q1_diff = (s1_q1 - s2_q1) / float(s2_q1)
q3_diff = (s1_q3 - s2_q3) / float(s2_q3)
mad1 = median_abs_dev(sample1)
mad2 = median_abs_dev(sample2)
# FIXME: handle division by zero
mad_diff = (mad1 - mad2) / float(mad2)
if self.args.verbose:
stdev1 = statistics.stdev(sample1)
stdev2 = statistics.stdev(sample2)
stdev_diff = (stdev1 - stdev2) / float(stdev2)
sample1_str = format_values(unit, (s1_q1, mean1, s1_q3, stdev1, mad1))
sample2_str = format_values(unit, (s2_q1, mean2, s2_q3, stdev2, mad2))
print("Calibration: warmups=%s" % format_number(nwarmup))
print(" first value: %s, outlier? %s (max: %s)"
% (format_value(unit, first_value), outlier,
format_value(unit, outlier_max)))
print(" sample1(%s): Q1=%s mean=%s Q3=%s stdev=%s MAD=%s"
% (len(sample1),
sample1_str[0],
sample1_str[1],
sample1_str[2],
sample1_str[3],
sample1_str[4]))
print(" sample2(%s): Q1=%s mean=%s Q3=%s stdev=%s MAD=%s"
% (len(sample2),
sample2_str[0],
sample2_str[1],
sample2_str[2],
sample2_str[3],
sample2_str[4]))
print(" diff: Q1=%+.0f%% mean=%+.0f%% Q3=%+.0f%% stdev=%+.0f%% MAD=%+.0f%%"
% (q1_diff * 100,
mean_diff * 100,
q3_diff * 100,
stdev_diff * 100,
mad_diff * 100))
if outlier:
return False
if not(-0.5 <= mean_diff <= 0.10):
return False
if abs(mad_diff) > 0.10:
return False
if abs(q1_diff) > 0.05:
return False
if abs(q3_diff) > 0.05:
return False
return True