def _update_or_add_new_baselines(self, driver_output, failures):
"""Updates or adds new baselines for the test if necessary."""
if (test_failures.has_failure_type(test_failures.FailureTimeout,
failures)
or test_failures.has_failure_type(test_failures.FailureCrash,
failures)):
return
self._save_baseline_data(
driver_output.text, '.txt',
test_failures.has_failure_type(test_failures.FailureMissingResult,
failures))
self._save_baseline_data(
driver_output.audio, '.wav',
test_failures.has_failure_type(test_failures.FailureMissingAudio,
failures))
self._save_baseline_data(
driver_output.image, '.png',
test_failures.has_failure_type(test_failures.FailureMissingImage,
failures))
def _update_or_add_new_baselines(self, driver_output, failures):
"""Updates or adds new baselines for the test if necessary."""
if (test_failures.has_failure_type(test_failures.FailureTimeout, failures) or
test_failures.has_failure_type(test_failures.FailureCrash, failures)):
return
# We usually don't want to create a new baseline if there isn't one
# existing (which usually means this baseline isn't necessary, e.g.
# an image-only test without text expectation files). However, in the
# following cases, we do:
# 1. The failure is MISSING; a baseline is apparently needed.
# 2. A testharness.js test fails assertions: testharness.js tests
# without baselines are implicitly expected to pass all assertions;
# if there are failed assertions we need to create a new baseline.
# Note that the created baseline might be redundant, but users can
# optimize them later with optimize-baselines.
if self._is_all_pass_testharness_text_not_needing_baseline(driver_output.text):
driver_output.text = None
self._save_baseline_data(
driver_output.text, '.txt',
test_failures.has_failure_type(test_failures.FailureMissingResult, failures) or
test_failures.has_failure_type(test_failures.FailureTestHarnessAssertion, failures))
self._save_baseline_data(
driver_output.audio, '.wav',
test_failures.has_failure_type(test_failures.FailureMissingAudio, failures))
expected_png = driver_output.image
if self._reference_files:
_log.warning('Can not rebaseline the image baseline of reftest %s', self._test_name)
# Let _save_baseline_data remove the '-expected.png' if it exists.
expected_png = None
self._save_baseline_data(
expected_png, '.png',
test_failures.has_failure_type(test_failures.FailureMissingImage, failures))
def _update_or_add_new_baselines(self, driver_output, failures):
"""Updates or adds new baselines for the test if necessary."""
if (test_failures.has_failure_type(test_failures.FailureTimeout,
failures)
or test_failures.has_failure_type(test_failures.FailureCrash,
failures)):
return
# We usually don't want to create a new baseline if there isn't one
# existing (which usually means this baseline isn't necessary, e.g.
# an image-first test without text expectation files). However, in the
# following cases, we do:
# 1. The failure is MISSING; a baseline is apparently needed.
# 2. A testharness.js test fails assertions: testharness.js tests
# without baselines are implicitly expected to pass all assertions;
# if there are failed assertions we need to create a new baseline.
# Note that the created baseline might be redundant, but users can
# optimize them later with optimize-baselines.
self._save_baseline_data(
driver_output.text, '.txt',
test_failures.has_failure_type(test_failures.FailureMissingResult,
failures)
or test_failures.has_failure_type(
test_failures.FailureTestHarnessAssertion, failures))
self._save_baseline_data(
driver_output.audio, '.wav',
test_failures.has_failure_type(test_failures.FailureMissingAudio,
failures))
self._save_baseline_data(
driver_output.image, '.png',
test_failures.has_failure_type(test_failures.FailureMissingImage,
failures))
def summarize_results(port_obj,
expectations,
initial_results,
all_retry_results,
only_include_failing=False):
"""Returns a dictionary containing a summary of the test runs, with the following fields:
'version': a version indicator
'fixable': The number of fixable tests (NOW - PASS)
'skipped': The number of skipped tests (NOW & SKIPPED)
'num_regressions': The number of non-flaky failures
'num_flaky': The number of flaky failures
'num_passes': The number of expected and unexpected passes
'tests': a dict of tests -> {'expected': '...', 'actual': '...'}
"""
results = {}
results['version'] = 3
all_retry_results = all_retry_results or []
tbe = initial_results.tests_by_expectation
results['skipped'] = len(tbe[ResultType.Skip])
# TODO(dpranke): Some or all of these counters can be removed.
num_passes = 0
num_flaky = 0
num_regressions = 0
# Calculate the number of failures by types (only in initial results).
num_failures_by_type = {}
for expected_result in initial_results.tests_by_expectation:
tests = initial_results.tests_by_expectation[expected_result]
num_failures_by_type[expected_result] = len(tests)
results['num_failures_by_type'] = num_failures_by_type
# Combine all iterations and retries together into a dictionary with the
# following structure:
# { test_name: [ (result, is_unexpected), ... ], ... }
# where result is a single TestResult, is_unexpected is a boolean
# representing whether the result is unexpected in that run.
merged_results_by_name = collections.defaultdict(list)
for test_run_results in [initial_results] + all_retry_results:
# all_results does not include SKIP, so we need results_by_name.
for test_name, result in test_run_results.results_by_name.iteritems():
if result.type == ResultType.Skip:
is_unexpected = test_name in test_run_results.unexpected_results_by_name
merged_results_by_name[test_name].append(
(result, is_unexpected))
# results_by_name only includes the last result, so we need all_results.
for result in test_run_results.all_results:
test_name = result.test_name
is_unexpected = test_name in test_run_results.unexpected_results_by_name
merged_results_by_name[test_name].append((result, is_unexpected))
# Finally, compute the tests dict.
tests = {}
for test_name, merged_results in merged_results_by_name.iteritems():
initial_result = merged_results[0][0]
if only_include_failing and initial_result.type == ResultType.Skip:
continue
exp = expectations.get_expectations(test_name)
expected_results, bugs = exp.results, exp.reason
expected = ' '.join(expected_results)
actual = []
actual_types = []
crash_sites = []
all_pass = True
has_expected = False
has_unexpected = False
has_unexpected_pass = False
has_stderr = False
for result, is_unexpected in merged_results:
actual.append(result.type)
actual_types.append(result.type)
crash_sites.append(result.crash_site)
if result.type != ResultType.Pass:
all_pass = False
if result.has_stderr:
has_stderr = True
if is_unexpected:
has_unexpected = True
if result.type == ResultType.Pass:
has_unexpected_pass = True
else:
has_expected = True
# TODO(crbug.com/855255): This code calls a test flaky if it has both
# expected and unexpected runs (NOT pass and failure); this is generally
# wrong (really it should just be if there are multiple kinds of results),
# but this works in the normal case because a test will only be retried
# if a result is unexpected, and if you get an expected result on the
# retry, then you did get multiple results. This fails if you get
# one kind of unexpected failure initially and another kind of
# unexpected failure on the retry (e.g., TIMEOUT CRASH), or if you
# explicitly run a test multiple times and get multiple expected results.
is_flaky = has_expected and has_unexpected
test_dict = {}
test_dict['expected'] = expected
test_dict['actual'] = ' '.join(actual)
# If a flag was added then add flag specific test expectations to the per test field
flag_exp = expectations.get_flag_expectations(test_name)
if flag_exp:
base_exp = expectations.get_base_expectations(test_name)
test_dict['flag_expectations'] = list(flag_exp.results)
test_dict['base_expectations'] = list(base_exp.results)
# Fields below are optional. To avoid bloating the output results json
# too much, only add them when they are True or non-empty.
if is_flaky:
num_flaky += 1
test_dict['is_flaky'] = True
elif all_pass or has_unexpected_pass:
# We count two situations as a "pass":
# 1. All test runs pass (which is obviously non-flaky, but does not
# imply whether the runs are expected, e.g. they can be all
# unexpected passes).
# 2. The test isn't flaky and has at least one unexpected pass
# (which implies all runs are unexpected). One tricky example
# that doesn't satisfy #1 is that if a test is expected to
# crash but in fact fails and then passes, it will be counted
# as "pass".
num_passes += 1
if not has_stderr and only_include_failing:
continue
elif has_unexpected:
# Either no retries or all retries failed unexpectedly.
num_regressions += 1
rounded_run_time = round(initial_result.test_run_time, 1)
if rounded_run_time:
test_dict['time'] = rounded_run_time
if exp.is_slow_test:
test_dict['is_slow_test'] = True
if has_stderr:
test_dict['has_stderr'] = True
if bugs:
test_dict['bugs'] = bugs.split()
if initial_result.reftest_type:
test_dict.update(reftest_type=list(initial_result.reftest_type))
crash_sites = [site for site in crash_sites if site]
if len(crash_sites) > 0:
test_dict['crash_site'] = crash_sites[0]
if test_failures.has_failure_type(test_failures.FailureTextMismatch,
initial_result.failures):
for failure in initial_result.failures:
if isinstance(failure, test_failures.FailureTextMismatch):
test_dict['text_mismatch'] = \
failure.text_mismatch_category()
break
# Note: is_unexpected and is_regression are intended to reflect the
# *last* result. In the normal use case (stop retrying failures
# once they pass), this is equivalent to saying that all of the
# results were unexpected failures.
last_result = actual_types[-1]
if not expectations.matches_an_expected_result(test_name, last_result):
test_dict['is_unexpected'] = True
if last_result != ResultType.Pass:
test_dict['is_regression'] = True
if initial_result.has_repaint_overlay:
test_dict['has_repaint_overlay'] = True
test_dict.update(_interpret_test_failures(initial_result.failures))
for retry_result, is_unexpected in merged_results[1:]:
# TODO(robertma): Why do we only update unexpected retry failures?
if is_unexpected:
test_dict.update(
_interpret_test_failures(retry_result.failures))
for test_result, _ in merged_results:
for artifact_name, artifacts in \
test_result.artifacts.artifacts.items():
artifact_dict = test_dict.setdefault('artifacts', {})
artifact_dict.setdefault(artifact_name, []).extend(artifacts)
# Store test hierarchically by directory. e.g.
# foo/bar/baz.html: test_dict
# foo/bar/baz1.html: test_dict
#
# becomes
# foo: {
# bar: {
# baz.html: test_dict,
# baz1.html: test_dict
# }
# }
parts = test_name.split('/')
current_map = tests
for i, part in enumerate(parts):
if i == (len(parts) - 1):
current_map[part] = test_dict
break
if part not in current_map:
current_map[part] = {}
current_map = current_map[part]
results['tests'] = tests
results['num_passes'] = num_passes
results['num_flaky'] = num_flaky
results['num_regressions'] = num_regressions
# Does results.html have enough information to compute this itself? (by
# checking total number of results vs. total number of tests?)
results['interrupted'] = initial_results.interrupted
results['layout_tests_dir'] = port_obj.web_tests_dir()
results['seconds_since_epoch'] = int(time.time())
results['build_number'] = port_obj.get_option('build_number')
results['builder_name'] = port_obj.get_option('builder_name')
if port_obj.get_option('order') == 'random':
results['random_order_seed'] = port_obj.get_option('seed')
results['path_delimiter'] = '/'
# If there is a flag name then add the flag name field
if expectations.flag_name:
results['flag_name'] = expectations.flag_name
# Don't do this by default since it takes >100ms.
# It's only used for rebaselining and uploading data to the flakiness dashboard.
results['chromium_revision'] = ''
if port_obj.get_option('builder_name'):
path = port_obj.repository_path()
git = port_obj.host.git(path=path)
if git:
results['chromium_revision'] = str(git.commit_position(path))
else:
_log.warning(
'Failed to determine chromium commit position for %s, '
'leaving "chromium_revision" key blank in full_results.json.',
path)
return results
def summarize_results(port_obj,
expectations,
initial_results,
all_retry_results,
enabled_pixel_tests_in_retry,
only_include_failing=False):
"""Returns a dictionary containing a summary of the test runs, with the following fields:
'version': a version indicator
'fixable': The number of fixable tests (NOW - PASS)
'skipped': The number of skipped tests (NOW & SKIPPED)
'num_regressions': The number of non-flaky failures
'num_flaky': The number of flaky failures
'num_passes': The number of expected and unexpected passes
'tests': a dict of tests -> {'expected': '...', 'actual': '...'}
"""
results = {}
results['version'] = 3
all_retry_results = all_retry_results or []
tbe = initial_results.tests_by_expectation
tbt = initial_results.tests_by_timeline
results['fixable'] = len(tbt[test_expectations.NOW] -
tbe[test_expectations.PASS])
# FIXME: Remove this. It is redundant with results['num_failures_by_type'].
results['skipped'] = len(tbt[test_expectations.NOW]
& tbe[test_expectations.SKIP])
# TODO(dpranke): Some or all of these counters can be removed.
num_passes = 0
num_flaky = 0
num_regressions = 0
keywords = test_expectations.TestExpectations.EXPECTATIONS_TO_STRING
# Calculate the number of failures by types (only in initial results).
num_failures_by_type = {}
for expectation in initial_results.tests_by_expectation:
tests = initial_results.tests_by_expectation[expectation]
if expectation != test_expectations.WONTFIX:
tests &= tbt[test_expectations.NOW]
num_failures_by_type[keywords[expectation]] = len(tests)
results['num_failures_by_type'] = num_failures_by_type
# Combine all iterations and retries together into a dictionary with the
# following structure:
# { test_name: [ (result, is_unexpected), ... ], ... }
# where result is a single TestResult, is_unexpected is a boolean
# representing whether the result is unexpected in that run.
merged_results_by_name = collections.defaultdict(list)
for test_run_results in [initial_results] + all_retry_results:
# all_results does not include SKIP, so we need results_by_name.
for test_name, result in test_run_results.results_by_name.iteritems():
if result.type == test_expectations.SKIP:
is_unexpected = test_name in test_run_results.unexpected_results_by_name
merged_results_by_name[test_name].append(
(result, is_unexpected))
# results_by_name only includes the last result, so we need all_results.
for result in test_run_results.all_results:
test_name = result.test_name
is_unexpected = test_name in test_run_results.unexpected_results_by_name
merged_results_by_name[test_name].append((result, is_unexpected))
# Finally, compute the tests dict.
tests = {}
for test_name, merged_results in merged_results_by_name.iteritems():
initial_result = merged_results[0][0]
if only_include_failing and initial_result.type == test_expectations.SKIP:
continue
expected = expectations.get_expectations_string(test_name)
actual = []
actual_types = []
crash_sites = []
all_pass = True
has_expected = False
has_unexpected = False
has_unexpected_pass = False
has_stderr = False
for result, is_unexpected in merged_results:
actual.append(keywords[result.type])
actual_types.append(result.type)
crash_sites.append(result.crash_site)
if result.type != test_expectations.PASS:
all_pass = False
if result.has_stderr:
has_stderr = True
if is_unexpected:
has_unexpected = True
if result.type == test_expectations.PASS:
has_unexpected_pass = True
else:
has_expected = True
# A test is flaky if it has both expected and unexpected runs (NOT pass
# and failure).
is_flaky = has_expected and has_unexpected
if len(set(actual)) == 1:
actual = [actual[0]]
actual_types = [actual_types[0]]
if is_flaky:
num_flaky += 1
elif all_pass or has_unexpected_pass:
# We count two situations as a "pass":
# 1. All test runs pass (which is obviously non-flaky, but does not
# imply whether the runs are expected, e.g. they can be all
# unexpected passes).
# 2. The test isn't flaky and has at least one unexpected pass
# (which implies all runs are unexpected). One tricky example
# that doesn't satisfy #1 is that if a test is expected to
# crash but in fact fails and then passes, it will be counted
# as "pass".
num_passes += 1
if not has_stderr and only_include_failing:
continue
elif has_unexpected and result.type != test_expectations.SKIP:
# Either no retries or all retries failed unexpectedly.
# TODO(robertma): When will there be unexpected skip? Do we really
# want to ignore them when counting regressions?
num_regressions += 1
test_dict = {}
test_dict['expected'] = expected
test_dict['actual'] = ' '.join(actual)
# Fields below are optional. To avoid bloating the output results json
# too much, only add them when they are True or non-empty.
rounded_run_time = round(initial_result.test_run_time, 1)
if rounded_run_time:
test_dict['time'] = rounded_run_time
if has_stderr:
test_dict['has_stderr'] = True
expectation_line = expectations.model().get_expectation_line(test_name)
bugs = expectation_line.bugs
if bugs:
test_dict['bugs'] = bugs
if expectation_line.flag_expectations:
test_dict['flag_expectations'] = expectation_line.flag_expectations
base_expectations = expectation_line.base_expectations
if base_expectations:
test_dict['base_expectations'] = base_expectations
if initial_result.reftest_type:
test_dict.update(reftest_type=list(initial_result.reftest_type))
crash_sites = [site for site in crash_sites if site]
if len(crash_sites) > 0:
test_dict['crash_site'] = crash_sites[0]
if test_failures.has_failure_type(test_failures.FailureTextMismatch,
initial_result.failures):
for failure in initial_result.failures:
if isinstance(failure, test_failures.FailureTextMismatch):
test_dict[
'text_mismatch'] = failure.text_mismatch_category()
break
def is_expected(actual_result):
return expectations.matches_an_expected_result(
test_name, actual_result,
port_obj.get_option('pixel_tests')
or initial_result.reftest_type,
port_obj.get_option('enable_sanitizer'))
# Note: is_unexpected is intended to capture the *last* result. In the
# normal use case (stop retrying failures once they pass), this is
# equivalent to checking if none of the results is expected.
if not any(
is_expected(actual_result) for actual_result in actual_types):
test_dict['is_unexpected'] = True
if initial_result.has_repaint_overlay:
test_dict['has_repaint_overlay'] = True
test_dict.update(_interpret_test_failures(initial_result.failures))
for retry_result, is_unexpected in merged_results[1:]:
# TODO(robertma): Why do we only update unexpected retry failures?
if is_unexpected:
test_dict.update(
_interpret_test_failures(retry_result.failures))
# Store test hierarchically by directory. e.g.
# foo/bar/baz.html: test_dict
# foo/bar/baz1.html: test_dict
#
# becomes
# foo: {
# bar: {
# baz.html: test_dict,
# baz1.html: test_dict
# }
# }
parts = test_name.split('/')
current_map = tests
for i, part in enumerate(parts):
if i == (len(parts) - 1):
current_map[part] = test_dict
break
if part not in current_map:
current_map[part] = {}
current_map = current_map[part]
results['tests'] = tests
results['num_passes'] = num_passes
results['num_flaky'] = num_flaky
results['num_regressions'] = num_regressions
# Does results.html have enough information to compute this itself? (by
# checking total number of results vs. total number of tests?)
results['interrupted'] = initial_results.interrupted
results['layout_tests_dir'] = port_obj.layout_tests_dir()
results['pixel_tests_enabled'] = port_obj.get_option('pixel_tests')
results['seconds_since_epoch'] = int(time.time())
results['build_number'] = port_obj.get_option('build_number')
results['builder_name'] = port_obj.get_option('builder_name')
if port_obj.get_option('order') == 'random':
results['random_order_seed'] = port_obj.get_option('seed')
results['path_delimiter'] = '/'
results['flag_name'] = expectations.model().get_flag_name()
# Don't do this by default since it takes >100ms.
# It's only used for rebaselining and uploading data to the flakiness dashboard.
results['chromium_revision'] = ''
if port_obj.get_option('builder_name'):
path = port_obj.repository_path()
git = port_obj.host.git(path=path)
if git:
results['chromium_revision'] = str(git.commit_position(path))
else:
_log.warning(
'Failed to determine chromium commit position for %s, '
'leaving "chromium_revision" key blank in full_results.json.',
path)
return results
