def check_perf_change_significance(row, ref_row):
# Cast values to the proper types
ref_stddev = 0.0 if ref_row[STDDEV_IDX] == 'N/A' else float(
ref_row[STDDEV_IDX])
stddev = 0.0 if row[STDDEV_IDX] == 'N/A' else float(row[STDDEV_IDX])
avg, ref_avg = map(float, [row[AVG_IDX], ref_row[AVG_IDX]])
iters, ref_iters = map(int, [row[NUM_ITERS_IDX], ref_row[NUM_ITERS_IDX]])
stddevs_are_zero = (ref_stddev == 0.0) and (stddev == 0.0)
percent_difference = abs(ref_avg - avg) * 100 / ref_avg
# If result is within min_percent_change_threshold of the baseline,
# mark it as insignificant and ignore the t-test.
if percent_difference < options.min_percent_change_threshold:
return False, False
# If the average is more than max_percent_change_threshold of the baseline,
# ignore the t-test and mark it as significant.
elif percent_difference > options.max_percent_change_threshold:
return True, ref_avg - avg < 0
# If both stddev and ref_stddev are 0, the t-test is meaningless, and causes a divide
# by zero exception.
elif options.tval_threshold and not stddevs_are_zero:
tval = calculate_tval(avg, stddev, iters, ref_avg, ref_stddev,
ref_iters)
# TODO: Currently, this doesn't take into account the degrees of freedom
# (number of iterations). In the future the regression threshold could be updated to
# specify the confidence interval, and based on the tval result we can lookup whether
# we are in/not in that interval.
return abs(
tval) > options.tval_threshold, tval > options.tval_threshold
return False, False
def check_perf_change_significance(row, ref_row):
# Cast values to the proper types
ref_stddev = 0.0 if ref_row[STDDEV_IDX] == 'N/A' else float(ref_row[STDDEV_IDX])
stddev = 0.0 if row[STDDEV_IDX] == 'N/A' else float(row[STDDEV_IDX])
avg, ref_avg = map(float, [row[AVG_IDX], ref_row[AVG_IDX]])
iters, ref_iters = map(int, [row[NUM_ITERS_IDX], ref_row[NUM_ITERS_IDX]])
stddevs_are_zero = (ref_stddev == 0.0) and (stddev == 0.0)
percent_difference = abs(ref_avg - avg) * 100 / ref_avg
# If result is within min_percent_change_threshold of the baseline,
# mark it as insignificant and ignore the t-test.
if percent_difference < options.min_percent_change_threshold:
return False, False
# If the average is more than max_percent_change_threshold of the baseline,
# ignore the t-test and mark it as significant.
elif percent_difference > options.max_percent_change_threshold:
return True, ref_avg - avg < 0
# If both stddev and ref_stddev are 0, the t-test is meaningless, and causes a divide
# by zero exception.
elif options.tval_threshold and not stddevs_are_zero:
tval = calculate_tval(avg, stddev, iters, ref_avg, ref_stddev, ref_iters)
# TODO: Currently, this doesn't take into account the degrees of freedom
# (number of iterations). In the future the regression threshold could be updated to
# specify the confidence interval, and based on the tval result we can lookup whether
# we are in/not in that interval.
return abs(tval) > options.tval_threshold, tval > options.tval_threshold
return False, False
def __check_perf_change_significance(self, stat, ref_stat):
zval = calculate_mwu(stat[SORTED], ref_stat[SORTED])
tval = calculate_tval(stat[AVG], stat[STDDEV], stat[ITERATIONS],
ref_stat[AVG], ref_stat[STDDEV], ref_stat[ITERATIONS])
try:
percent_difference = abs(ref_stat[AVG] - stat[AVG]) * 100 / ref_stat[AVG]
except ZeroDivisionError:
percent_difference = 0.0
absolute_difference = abs(ref_stat[AVG] - stat[AVG])
if absolute_difference < options.allowed_latency_diff_secs:
return False, zval, tval
if percent_difference < options.min_percent_change_threshold:
return False, zval, tval
return (abs(zval) > options.zval_threshold or abs(tval) > options.tval_threshold,
zval, tval)
def __check_perf_change_significance(self, stat, ref_stat):
absolute_difference = abs(ref_stat[AVG] - stat[AVG])
try:
percent_difference = abs(ref_stat[AVG] - stat[AVG]) * 100 / ref_stat[AVG]
except ZeroDivisionError:
percent_difference = 0.0
stddevs_are_zero = (ref_stat[STDDEV] == 0) and (stat[STDDEV] == 0)
if absolute_difference < options.allowed_latency_diff_secs:
return False, False
if percent_difference < options.min_percent_change_threshold:
return False, False
if percent_difference > options.max_percent_change_threshold:
return True, ref_stat[AVG] < stat[AVG]
if options.tval_threshold and not stddevs_are_zero:
tval = calculate_tval(stat[AVG], stat[STDDEV], stat[ITERATIONS],
ref_stat[AVG], ref_stat[STDDEV], ref_stat[ITERATIONS])
return abs(tval) > options.tval_threshold, tval > options.tval_threshold
return False, False
def __check_perf_change_significance(self, stat, ref_stat):
zval = calculate_mwu(stat[SORTED], ref_stat[SORTED])
try:
tval = calculate_tval(stat[AVG], stat[STDDEV],
stat[ITERATIONS], ref_stat[AVG],
ref_stat[STDDEV], ref_stat[ITERATIONS])
except ZeroDivisionError:
# t-test cannot be performed if both standard deviations are 0
tval = float('nan')
Report.invalid_t_tests = True
try:
percent_difference = abs(ref_stat[AVG] -
stat[AVG]) * 100 / ref_stat[AVG]
except ZeroDivisionError:
percent_difference = 0.0
absolute_difference = abs(ref_stat[AVG] - stat[AVG])
if absolute_difference < options.allowed_latency_diff_secs:
return False, zval, tval
if percent_difference < options.min_percent_change_threshold:
return False, zval, tval
return (abs(zval) > options.zval_threshold
or abs(tval) > options.tval_threshold, zval, tval)
