config.use_threads = False for oneTest in aloneTests: if stopping(): break oneTest(watcher) except KeyboardInterrupt: pass # flush everything before we continue sys.stdout.flush() # Warn if had to force skip perf tests (see Note force skip perf tests). spacing = " " if forceSkipPerfTests and not args.skip_perf_tests: print() print(str_warn('Skipping All Performance Tests') + ' `git` exited with non-zero exit code.') print(spacing + 'Git is required because performance test results are compared with ancestor git commits\' results (stored with git notes).') print(spacing + 'You can still run the tests without git by specifying an output file with --metrics-file FILE.') # Warn of new metrics. new_metrics = [metric for (change, metric) in t.metrics if change == MetricChange.NewMetric] if any(new_metrics): if inside_git_repo(): reason = 'a baseline (expected value) cannot be recovered from' + \ ' previous git commits. This may be due to HEAD having' + \ ' new tests or having expected changes, the presence of' + \ ' expected changes since the last run of the tests, and/or' + \ ' the latest test run being too old.' fix = 'If the tests exist on the previous' + \ ' commit (And are configured to run with the same ways),' + \ ' then check out that commit and run the tests to generate' + \
heading = 'Metrics: %s' % metric_name print() print(heading) print('-' * len(heading)) print() tabulate_metrics(stats) else: print("\nNone collected.") print("") # Warn if had to force skip perf tests (see Note force skip perf tests). spacing = " " if forceSkipPerfTests and not args.skip_perf_tests: print() print( str_warn('Skipping All Performance Tests') + ' `git` exited with non-zero exit code.') print( spacing + 'Git is required because performance test results are compared with ancestor git commits\' results (stored with git notes).' ) print( spacing + 'You can still run the tests without git by specifying an output file with --metrics-file FILE.' ) # Warn of new metrics. new_metrics = [ metric for (change, metric, baseline) in t.metrics if change == MetricChange.NewMetric ]
config.use_threads = False for oneTest in aloneTests: if stopping(): break oneTest(watcher) except KeyboardInterrupt: pass # flush everything before we continue sys.stdout.flush() # Warn if had to force skip perf tests (see Note force skip perf tests). spacing = " " if forceSkipPerfTests and not args.skip_perf_tests: print() print(str_warn('Skipping All Performance Tests') + ' `git` exited with non-zero exit code.') print(spacing + 'Git is required because performance test results are compared with ancestor git commits\' results (stored with git notes).') print(spacing + 'You can still run the tests without git by specifying an output file with --metrics-file FILE.') # Warn of new metrics. new_metrics = [metric for (change, metric) in t.metrics if change == MetricChange.NewMetric] if any(new_metrics): if inside_git_repo(): reason = 'a baseline (expected value) cannot be recovered from' + \ ' previous git commits. This may be due to HEAD having' + \ ' new tests or having expected changes, the presence of' + \ ' expected changes since the last run of the tests, and/or' + \ ' the latest test run being too old.' fix = 'If the tests exist on the previous' + \ ' commit (And are configured to run with the same ways),' + \ ' then check out that commit and run the tests to generate' + \
for oneTest in aloneTests: if stopping(): break oneTest(watcher) except KeyboardInterrupt: pass # flush everything before we continue sys.stdout.flush() # Warn if had to force skip perf tests (see Note force skip perf tests). spacing = " " if forceSkipPerfTests and not args.skip_perf_tests: print() print( str_warn('Skipping All Performance Tests') + ' `git status` exited with non-zero exit code.') print( spacing + 'Git is required because performance test results are compared with the previous git commit\'s results (stored with git notes).' ) print( spacing + 'You can still run the tests without git by specifying an output file with --metrics-file FILE.' ) # Warn of new metrics. new_metrics = [ metric for (change, metric) in t.metrics if change == MetricChange.NewMetric ]
config.use_threads = False for oneTest in aloneTests: if stopping(): break oneTest(watcher) except KeyboardInterrupt: pass # flush everything before we continue sys.stdout.flush() # Warn if had to force skip perf tests (see Note force skip perf tests). spacing = " " if forceSkipPerfTests and not args.skip_perf_tests: print() print(str_warn('Skipping All Performance Tests') + ' `git status` exited with non-zero exit code.') print(spacing + 'Git is required because performance test results are compared with the previous git commit\'s results (stored with git notes).') print(spacing + 'You can still run the tests without git by specifying an output file with --metrics-file FILE.') # Warn of new metrics. new_metrics = [metric for (change, metric) in t.metrics if change == MetricChange.NewMetric] if any(new_metrics): if canGitStatus: reason = 'the previous git commit doesn\'t have recorded metrics for the following tests.' + \ ' If the tests exist on the previous commit, then check it out and run the tests to generate the missing metrics.' else: reason = 'this is not a git repo so the previous git commit\'s metrics cannot be loaded from git notes:' print() print(str_warn('New Metrics') + ' these metrics trivially pass because ' + reason) print(spacing + ('\n' + spacing).join(set([metric.test for metric in new_metrics])))
config.use_threads = False for oneTest in aloneTests: if stopping(): break oneTest(watcher) except KeyboardInterrupt: pass # flush everything before we continue sys.stdout.flush() # Warn if had to force skip perf tests (see Note force skip perf tests). spacing = " " if forceSkipPerfTests and not args.skip_perf_tests: print() print(str_warn('Skipping All Performance Tests') + ' `git status` exited with non-zero exit code.') print(spacing + 'Git is required because performance test results are compared with the previous git commit\'s results (stored with git notes).') print(spacing + 'You can still run the tests without git by specifying an output file with --metrics-file FILE.') # Warn of new metrics. new_metrics = [metric for (change, metric) in t.metrics if change == MetricChange.NewMetric] if any(new_metrics): if canGitStatus: reason = 'the previous git commit doesn\'t have recorded metrics for the following tests.' + \ ' If the tests exist on the previous commit, then check it out and run the tests to generate the missing metrics.' else: reason = 'this is not a git repo so the previous git commit\'s metrics cannot be loaded from git notes:' print() print(str_warn('New Metrics') + ' these metrics trivially pass because ' + reason) print(spacing + ('\n' + spacing).join(set([metric.test for metric in new_metrics])))
except KeyboardInterrupt: pass # flush everything before we continue sys.stdout.flush() # Warn of new metrics. new_metrics = [ metric for (change, metric) in t.metrics if change == MetricChange.NewMetric ] spacing = " " if any(new_metrics): print() print( str_warn('New Metrics') + ' the previous git commit doesn\'t have metrics for the following tests:' ) print(spacing + ('\n' + spacing).join(set([metric.test for metric in new_metrics]))) # Inform of how to accept metric changes. if (len(t.unexpected_stat_failures) > 0): print() print( str_info("Some stats have changed") + " If this is expected, allow changes by appending the git commit message with this:" ) print('-' * 25) print(Perf.allow_changes_string(t.metrics))