def check_perf_change_significance(row, ref_row): # Cast values to the proper types ref_stddev = 0.0 if ref_row[STDDEV_IDX] == 'N/A' else float( ref_row[STDDEV_IDX]) stddev = 0.0 if row[STDDEV_IDX] == 'N/A' else float(row[STDDEV_IDX]) avg, ref_avg = map(float, [row[AVG_IDX], ref_row[AVG_IDX]]) iters, ref_iters = map(int, [row[NUM_ITERS_IDX], ref_row[NUM_ITERS_IDX]]) stddevs_are_zero = (ref_stddev == 0.0) and (stddev == 0.0) percent_difference = abs(ref_avg - avg) * 100 / ref_avg # If result is within min_percent_change_threshold of the baseline, # mark it as insignificant and ignore the t-test. if percent_difference < options.min_percent_change_threshold: return False, False # If the average is more than max_percent_change_threshold of the baseline, # ignore the t-test and mark it as significant. elif percent_difference > options.max_percent_change_threshold: return True, ref_avg - avg < 0 # If both stddev and ref_stddev are 0, the t-test is meaningless, and causes a divide # by zero exception. elif options.tval_threshold and not stddevs_are_zero: tval = calculate_tval(avg, stddev, iters, ref_avg, ref_stddev, ref_iters) # TODO: Currently, this doesn't take into account the degrees of freedom # (number of iterations). In the future the regression threshold could be updated to # specify the confidence interval, and based on the tval result we can lookup whether # we are in/not in that interval. return abs( tval) > options.tval_threshold, tval > options.tval_threshold return False, False
def check_perf_change_significance(row, ref_row): # Cast values to the proper types ref_stddev = 0.0 if ref_row[STDDEV_IDX] == 'N/A' else float(ref_row[STDDEV_IDX]) stddev = 0.0 if row[STDDEV_IDX] == 'N/A' else float(row[STDDEV_IDX]) avg, ref_avg = map(float, [row[AVG_IDX], ref_row[AVG_IDX]]) iters, ref_iters = map(int, [row[NUM_ITERS_IDX], ref_row[NUM_ITERS_IDX]]) stddevs_are_zero = (ref_stddev == 0.0) and (stddev == 0.0) percent_difference = abs(ref_avg - avg) * 100 / ref_avg # If result is within min_percent_change_threshold of the baseline, # mark it as insignificant and ignore the t-test. if percent_difference < options.min_percent_change_threshold: return False, False # If the average is more than max_percent_change_threshold of the baseline, # ignore the t-test and mark it as significant. elif percent_difference > options.max_percent_change_threshold: return True, ref_avg - avg < 0 # If both stddev and ref_stddev are 0, the t-test is meaningless, and causes a divide # by zero exception. elif options.tval_threshold and not stddevs_are_zero: tval = calculate_tval(avg, stddev, iters, ref_avg, ref_stddev, ref_iters) # TODO: Currently, this doesn't take into account the degrees of freedom # (number of iterations). In the future the regression threshold could be updated to # specify the confidence interval, and based on the tval result we can lookup whether # we are in/not in that interval. return abs(tval) > options.tval_threshold, tval > options.tval_threshold return False, False
def __check_perf_change_significance(self, stat, ref_stat): zval = calculate_mwu(stat[SORTED], ref_stat[SORTED]) tval = calculate_tval(stat[AVG], stat[STDDEV], stat[ITERATIONS], ref_stat[AVG], ref_stat[STDDEV], ref_stat[ITERATIONS]) try: percent_difference = abs(ref_stat[AVG] - stat[AVG]) * 100 / ref_stat[AVG] except ZeroDivisionError: percent_difference = 0.0 absolute_difference = abs(ref_stat[AVG] - stat[AVG]) if absolute_difference < options.allowed_latency_diff_secs: return False, zval, tval if percent_difference < options.min_percent_change_threshold: return False, zval, tval return (abs(zval) > options.zval_threshold or abs(tval) > options.tval_threshold, zval, tval)
def __check_perf_change_significance(self, stat, ref_stat): absolute_difference = abs(ref_stat[AVG] - stat[AVG]) try: percent_difference = abs(ref_stat[AVG] - stat[AVG]) * 100 / ref_stat[AVG] except ZeroDivisionError: percent_difference = 0.0 stddevs_are_zero = (ref_stat[STDDEV] == 0) and (stat[STDDEV] == 0) if absolute_difference < options.allowed_latency_diff_secs: return False, False if percent_difference < options.min_percent_change_threshold: return False, False if percent_difference > options.max_percent_change_threshold: return True, ref_stat[AVG] < stat[AVG] if options.tval_threshold and not stddevs_are_zero: tval = calculate_tval(stat[AVG], stat[STDDEV], stat[ITERATIONS], ref_stat[AVG], ref_stat[STDDEV], ref_stat[ITERATIONS]) return abs(tval) > options.tval_threshold, tval > options.tval_threshold return False, False
def __check_perf_change_significance(self, stat, ref_stat): zval = calculate_mwu(stat[SORTED], ref_stat[SORTED]) try: tval = calculate_tval(stat[AVG], stat[STDDEV], stat[ITERATIONS], ref_stat[AVG], ref_stat[STDDEV], ref_stat[ITERATIONS]) except ZeroDivisionError: # t-test cannot be performed if both standard deviations are 0 tval = float('nan') Report.invalid_t_tests = True try: percent_difference = abs(ref_stat[AVG] - stat[AVG]) * 100 / ref_stat[AVG] except ZeroDivisionError: percent_difference = 0.0 absolute_difference = abs(ref_stat[AVG] - stat[AVG]) if absolute_difference < options.allowed_latency_diff_secs: return False, zval, tval if percent_difference < options.min_percent_change_threshold: return False, zval, tval return (abs(zval) > options.zval_threshold or abs(tval) > options.tval_threshold, zval, tval)