def check_stats_change(actual: PerfStat, baseline: Baseline, tolerance_dev, allowed_perf_changes: Dict[TestName, List[AllowedPerfChange]] = {}, force_print = False ) -> Tuple[MetricChange, Any]: expected_val = baseline.perfStat.value full_name = actual.test + ' (' + actual.way + ')' lowerBound = trunc( int(expected_val) * ((100 - float(tolerance_dev))/100)) upperBound = trunc(0.5 + ceil(int(expected_val) * ((100 + float(tolerance_dev))/100))) actual_dev = round(((float(actual.value) * 100)/ int(expected_val)) - 100, 1) # Find the direction of change. change = MetricChange.NoChange if actual.value < lowerBound: change = MetricChange.Decrease elif actual.value > upperBound: change = MetricChange.Increase # Is the change allowed? allowed_change_directions = [MetricChange.NoChange] + [ allow_stmt.direction for allow_stmt in allowed_perf_changes.get(actual.test, []) # List of metrics are not specified or the metric is in the list of metrics. if not allow_stmt.metrics or actual.metric in allow_stmt.metrics # way/test are not specified, or match the actual way/test. if ((not 'way' in allow_stmt.opts.keys()) or actual.way == allow_stmt.opts['way']) if ((not 'test_env' in allow_stmt.opts.keys()) or actual.test_env == allow_stmt.opts['test_env']) ] change_allowed = change in allowed_change_directions # Print errors and create pass/fail object. result = passed() if not change_allowed: error = str(change) + ' from ' + baseline.perfStat.test_env + \ ' baseline @ %s' % baseline.commit print(actual.metric, error + ':') result = failBecause('stat ' + error, tag='stat') if not change_allowed or force_print: length = max(len(str(x)) for x in [expected_val, lowerBound, upperBound, actual.value]) def display(descr, val, extra): print(descr, str(val).rjust(length), extra) display(' Expected ' + full_name + ' ' + actual.metric + ':', expected_val, '+/-' + str(tolerance_dev) + '%') display(' Lower bound ' + full_name + ' ' + actual.metric + ':', lowerBound, '') display(' Upper bound ' + full_name + ' ' + actual.metric + ':', upperBound, '') display(' Actual ' + full_name + ' ' + actual.metric + ':', actual.value, '') if actual.value != expected_val: display(' Deviation ' + full_name + ' ' + actual.metric + ':', actual_dev, '%') return (change, result)
def check_stats_change(actual, baseline, tolerance_dev, allowed_perf_changes = {}, force_print = False): expected_val = baseline.perfStat.value full_name = actual.test + ' (' + actual.way + ')' lowerBound = trunc( int(expected_val) * ((100 - float(tolerance_dev))/100)) upperBound = trunc(0.5 + ceil(int(expected_val) * ((100 + float(tolerance_dev))/100))) actual_dev = round(((float(actual.value) * 100)/ int(expected_val)) - 100, 1) # Find the direction of change. change = MetricChange.NoChange if actual.value < lowerBound: change = MetricChange.Decrease elif actual.value > upperBound: change = MetricChange.Increase # Is the change allowed? allowed_change_directions = [MetricChange.NoChange] + [ allow_stmt['direction'] for allow_stmt in allowed_perf_changes.get(actual.test, []) # List of metrics are not specified or the metric is in the list of metrics. if not allow_stmt['metrics'] or actual.metric in allow_stmt['metrics'] # way/test are not specified, or match the actual way/test. if ((not 'way' in allow_stmt['opts'].keys()) or actual.way == allow_stmt['opts']['way']) if ((not 'test_env' in allow_stmt['opts'].keys()) or actual.test_env == allow_stmt['opts']['test_env']) ] change_allowed = change in allowed_change_directions # Print errors and create pass/fail object. result = passed() if not change_allowed: error = change + ' from ' + baseline.perfStat.test_env + \ ' baseline @ HEAD~' + str(baseline.commitDepth) print(actual.metric, error + ':') result = failBecause('stat ' + error, tag='stat') if not change_allowed or force_print: length = max(len(str(x)) for x in [expected_val, lowerBound, upperBound, actual.value]) def display(descr, val, extra): print(descr, str(val).rjust(length), extra) display(' Expected ' + full_name + ' ' + actual.metric + ':', expected_val, '+/-' + str(tolerance_dev) + '%') display(' Lower bound ' + full_name + ' ' + actual.metric + ':', lowerBound, '') display(' Upper bound ' + full_name + ' ' + actual.metric + ':', upperBound, '') display(' Actual ' + full_name + ' ' + actual.metric + ':', actual.value, '') if actual.value != expected_val: display(' Deviation ' + full_name + ' ' + actual.metric + ':', actual_dev, '%') return (change, result)