def testOutputResultsUnsupportedFormat(self):
"""Tests that passing in an unsupported format is an error."""
with self.assertRaises(RuntimeError):
result_output.OutputResults(data_types.TestExpectationMap(),
data_types.TestExpectationMap(),
data_types.TestExpectationMap(), {}, [],
'asdf')
def testUnusedButNotEmpty(self):
"""Tests filtering if there is an unused expectation but no empty tests."""
expectation_map = data_types.TestExpectationMap({
'foo/test':
data_types.ExpectationBuilderMap({
data_types.Expectation('foo/test', ['win'], ['Failure']):
data_types.BuilderStepMap({
'SomeBuilder':
data_types.StepBuildStatsMap(),
}),
data_types.Expectation('foo/test', ['linux'], ['Failure']):
data_types.BuilderStepMap(),
})
})
expected_expectation_map = data_types.TestExpectationMap({
'foo/test':
data_types.ExpectationBuilderMap({
data_types.Expectation('foo/test', ['win'], ['Failure']):
data_types.BuilderStepMap({
'SomeBuilder':
data_types.StepBuildStatsMap(),
}),
}),
})
unused_expectations = expectations.FilterOutUnusedExpectations(
expectation_map)
self.assertEqual(
unused_expectations,
[data_types.Expectation('foo/test', ['linux'], ['Failure'])])
self.assertEqual(expectation_map, expected_expectation_map)
def testRemoveExpectation(self):
"""Tests that specifying to remove an expectation does so."""
self._input_mock.return_value = 'r'
# yapf: disable
test_expectation_map = data_types.TestExpectationMap({
'foo/test':
data_types.ExpectationBuilderMap({
data_types.Expectation(
'foo/test', ['win'], 'Failure', 'crbug.com/1234'):
data_types.BuilderStepMap(),
}),
})
# yapf: enable
modified_urls = expectations.ModifySemiStaleExpectations(
test_expectation_map, self.filename)
self.assertEqual(modified_urls, set(['crbug.com/1234']))
expected_file_contents = """\
# tags: [ win linux ]
# results: [ Failure RetryOnFailure Skip ]
[ linux ] foo/test [ Failure ]
crbug.com/2345 [ linux ] bar/* [ RetryOnFailure ]
crbug.com/3456 [ linux ] some/bad/test [ Skip ]
"""
with open(self.filename) as f:
self.assertEqual(f.read(), expected_file_contents)
def testResultMatchFailingExisting(self):
"""Test adding a failing result when results for a builder exist."""
r = data_types.Result('some/test/case', ['win', 'win10'], 'Failure',
'pixel_tests', 'build_id')
e = data_types.Expectation('some/test/*', ['win10'], 'Failure')
stats = data_types.BuildStats()
stats.AddPassedBuild()
expectation_map = data_types.TestExpectationMap({
'some/test/*':
data_types.ExpectationBuilderMap({
e:
data_types.BuilderStepMap({
'builder':
data_types.StepBuildStatsMap({
'pixel_tests': stats,
}),
}),
}),
})
found_matching = expectations._AddResultToMap(r, 'builder',
expectation_map)
self.assertTrue(found_matching)
stats = data_types.BuildStats()
stats.AddFailedBuild('build_id')
stats.AddPassedBuild()
expected_expectation_map = {
'some/test/*': {
e: {
'builder': {
'pixel_tests': stats,
},
},
},
}
self.assertEqual(expectation_map, expected_expectation_map)
def testResultMatchMultiMatch(self):
"""Test adding a passing result when multiple expectations match."""
r = data_types.Result('some/test/case', ['win', 'win10'], 'Pass',
'pixel_tests', 'build_id')
e = data_types.Expectation('some/test/*', ['win10'], 'Failure')
e2 = data_types.Expectation('some/test/case', ['win10'], 'Failure')
expectation_map = data_types.TestExpectationMap({
'some/test/*':
data_types.ExpectationBuilderMap({
e: data_types.BuilderStepMap(),
e2: data_types.BuilderStepMap(),
}),
})
found_matching = expectations._AddResultToMap(r, 'builder',
expectation_map)
self.assertTrue(found_matching)
stats = data_types.BuildStats()
stats.AddPassedBuild()
expected_expectation_map = {
'some/test/*': {
e: {
'builder': {
'pixel_tests': stats,
},
},
e2: {
'builder': {
'pixel_tests': stats,
},
}
}
}
self.assertEqual(expectation_map, expected_expectation_map)
def _GetSampleTestExpectationMap(self):
build_stats = self._GetSampleBuildStats()
return data_types.TestExpectationMap({
'foo':
data_types.ExpectationBuilderMap({
data_types.Expectation('foo', ['tag'], ['Failure']):
data_types.BuilderStepMap({
'builder1':
data_types.StepBuildStatsMap({
'step1': build_stats[0],
'step2': build_stats[1],
}),
'builder2':
data_types.StepBuildStatsMap({
'step3': build_stats[2],
'step4': build_stats[3],
}),
}),
data_types.Expectation('foo', ['tag2'], ['Failure']):
data_types.BuilderStepMap({
'builder3':
data_types.StepBuildStatsMap({
'step5': build_stats[4],
'step6': build_stats[5],
}),
'builder4':
data_types.StepBuildStatsMap({
'step7': build_stats[6],
'step8': build_stats[7],
}),
}),
}),
})
def testEmptyExpectationMap(self):
"""Tests that an empty expectation map results in a no-op."""
modified_urls = expectations.ModifySemiStaleExpectations(
data_types.TestExpectationMap(), self.filename)
self.assertEqual(modified_urls, set())
self._input_mock.assert_not_called()
with open(self.filename) as f:
self.assertEqual(f.read(), FAKE_EXPECTATION_FILE_CONTENTS)
def GetEmptyMapForGenericFailureExpectation(self):
foo_expectation = self.GetGenericFailureExpectation()
return data_types.TestExpectationMap({
'foo/test':
data_types.ExpectationBuilderMap({
foo_expectation:
data_types.BuilderStepMap(),
}),
})
def testSemiStaleExpectations(self):
"""Tests output when only semi-stale expectations are provided."""
expectation_map = data_types.TestExpectationMap({
'foo':
data_types.ExpectationBuilderMap({
data_types.Expectation('foo', ['win'], ['Failure']):
data_types.BuilderStepMap({
'foo_builder':
data_types.StepBuildStatsMap({
'step1':
uu.CreateStatsWithPassFails(1, 0),
'step2':
uu.CreateStatsWithPassFails(2, 2),
}),
'bar_builder':
data_types.StepBuildStatsMap({
'step1':
uu.CreateStatsWithPassFails(3, 0),
'step2':
uu.CreateStatsWithPassFails(0, 4)
}),
}),
data_types.Expectation('foo', ['linux'], ['RetryOnFailure']):
data_types.BuilderStepMap({
'foo_builder':
data_types.StepBuildStatsMap({
'step1':
uu.CreateStatsWithPassFails(5, 0),
'step2':
uu.CreateStatsWithPassFails(6, 6),
}),
}),
}),
'bar':
data_types.ExpectationBuilderMap({
data_types.Expectation('bar', ['win'], ['Failure']):
data_types.BuilderStepMap({
'foo_builder':
data_types.StepBuildStatsMap({
'step1':
uu.CreateStatsWithPassFails(7, 0),
}),
'bar_builder':
data_types.StepBuildStatsMap({
'step1':
uu.CreateStatsWithPassFails(0, 8),
}),
}),
}),
})
expected_semi_stale_dict = copy.deepcopy(expectation_map)
stale_dict, semi_stale_dict, active_dict =\
expectations.SplitExpectationsByStaleness(expectation_map)
self.assertEqual(stale_dict, {})
self.assertEqual(semi_stale_dict, expected_semi_stale_dict)
self.assertEqual(active_dict, {})
def testMergeBuildStats(self):
"""Tests that BuildStats for the same step are merged properly."""
base_map = data_types.TestExpectationMap({
'foo':
data_types.ExpectationBuilderMap({
data_types.Expectation('foo', ['win'], 'Failure'):
data_types.BuilderStepMap({
'builder':
data_types.StepBuildStatsMap({
'step':
data_types.BuildStats(),
}),
}),
}),
})
merge_stats = data_types.BuildStats()
merge_stats.AddFailedBuild('1')
merge_map = data_types.TestExpectationMap({
'foo':
data_types.ExpectationBuilderMap({
data_types.Expectation('foo', ['win'], 'Failure'):
data_types.BuilderStepMap({
'builder':
data_types.StepBuildStatsMap({
'step': merge_stats,
}),
}),
}),
})
expected_stats = data_types.BuildStats()
expected_stats.AddFailedBuild('1')
expected_base_map = {
'foo': {
data_types.Expectation('foo', ['win'], 'Failure'): {
'builder': {
'step': expected_stats,
},
},
},
}
expectations.MergeExpectationMaps(base_map, merge_map)
self.assertEqual(base_map, expected_base_map)
def testEmptyInput(self):
"""Tests that nothing blows up with empty input."""
stale_dict, semi_stale_dict, active_dict =\
expectations.SplitExpectationsByStaleness(
data_types.TestExpectationMap())
self.assertEqual(stale_dict, {})
self.assertEqual(semi_stale_dict, {})
self.assertEqual(active_dict, {})
self.assertIsInstance(stale_dict, data_types.TestExpectationMap)
self.assertIsInstance(semi_stale_dict, data_types.TestExpectationMap)
self.assertIsInstance(active_dict, data_types.TestExpectationMap)
def testEmptyMergeMap(self):
"""Tests that a merge with an empty merge map is a no-op."""
base_map = data_types.TestExpectationMap({
'foo':
data_types.ExpectationBuilderMap({
data_types.Expectation('foo', ['win'], 'Failure'):
data_types.BuilderStepMap({
'builder':
data_types.StepBuildStatsMap({
'step':
data_types.BuildStats(),
}),
}),
}),
})
merge_map = data_types.TestExpectationMap()
original_base_map = copy.deepcopy(base_map)
expectations.MergeExpectationMaps(base_map, merge_map)
self.assertEqual(base_map, original_base_map)
self.assertEqual(merge_map, {})
def GetMapForExpectationAndStats(self, expectation, stats):
return data_types.TestExpectationMap({
expectation.test:
data_types.ExpectationBuilderMap({
expectation:
data_types.BuilderStepMap({
'builder':
data_types.StepBuildStatsMap({
'pixel_tests': stats,
}),
}),
}),
})
def _FillExpectationMapForBuilders(self, expectation_map, builders,
builder_type):
"""Fills |expectation_map| with results from |builders|.
Args:
expectation_map: A data_types.TestExpectationMap. Will be modified
in-place.
builders: A list of strings containing the names of builders to query.
builder_type: A string containing the type of builder to query, either
"ci" or "try".
Returns:
A dict containing any results that were retrieved that did not have a
matching expectation in |expectation_map| in the following format:
{
|builder_type|:|builder_name| (str): [
result1 (data_types.Result),
result2 (data_types.Result),
...
],
}
"""
assert isinstance(expectation_map, data_types.TestExpectationMap)
# Spin up a separate process for each query/add step. This is wasteful in
# the sense that we'll have a bunch of idle processes once faster steps
# start finishing, but ensures that we start slow queries early and avoids
# the overhead of passing large amounts of data between processes. See
# crbug.com/1182459 for more information on performance considerations.
process_pool = multiprocessing_utils.GetProcessPool(
nodes=len(builders))
args = [(b, builder_type, expectation_map) for b in builders]
results = process_pool.map(self._QueryAddCombined, args)
tmp_expectation_map = data_types.TestExpectationMap()
all_unmatched_results = {}
for (unmatched_results, prefixed_builder_name, merge_map) in results:
expectations.MergeExpectationMaps(tmp_expectation_map, merge_map,
expectation_map)
if unmatched_results:
all_unmatched_results[
prefixed_builder_name] = unmatched_results
expectation_map.clear()
expectation_map.update(tmp_expectation_map)
return all_unmatched_results
def testUnusedAndEmpty(self):
"""Tests filtering if there is an expectation that causes an empty test."""
expectation_map = data_types.TestExpectationMap({
'foo/test':
data_types.ExpectationBuilderMap({
data_types.Expectation('foo/test', ['win'], ['Failure']):
data_types.BuilderStepMap(),
}),
})
unused_expectations = expectations.FilterOutUnusedExpectations(
expectation_map)
self.assertEqual(
unused_expectations,
[data_types.Expectation('foo/test', ['win'], ['Failure'])])
self.assertEqual(expectation_map, {})
def testResultNoMatch(self):
"""Tests that a result is not added if no match is found."""
r = data_types.Result('some/test/case', ['win', 'win10'], 'Failure',
'pixel_tests', 'build_id')
e = data_types.Expectation('some/test/*', ['win10', 'foo'], 'Failure')
expectation_map = data_types.TestExpectationMap({
'some/test/*':
data_types.ExpectationBuilderMap({
e: data_types.BuilderStepMap(),
})
})
found_matching = expectations._AddResultToMap(r, 'builder',
expectation_map)
self.assertFalse(found_matching)
expected_expectation_map = {'some/test/*': {e: {}}}
self.assertEqual(expectation_map, expected_expectation_map)
def testNoUnused(self):
"""Tests that filtering is a no-op if there are no unused expectations."""
expectation_map = data_types.TestExpectationMap({
'foo/test':
data_types.ExpectationBuilderMap({
data_types.Expectation('foo/test', ['win'], ['Failure']):
data_types.BuilderStepMap({
'SomeBuilder':
data_types.StepBuildStatsMap(),
}),
})
})
expected_expectation_map = copy.deepcopy(expectation_map)
unused_expectations = expectations.FilterOutUnusedExpectations(
expectation_map)
self.assertEqual(len(unused_expectations), 0)
self.assertEqual(expectation_map, expected_expectation_map)
def testValidResults(self):
"""Tests functionality when valid results are returned by the query."""
def SideEffect(builder, *args):
del args
if builder == 'matched_builder':
return [
data_types.Result('foo', ['win'], 'Pass', 'step_name',
'build_id')
]
else:
return [
data_types.Result('bar', [], 'Pass', 'step_name',
'build_id')
]
self._query_mock.side_effect = SideEffect
expectation = data_types.Expectation('foo', ['win'], 'RetryOnFailure')
expectation_map = data_types.TestExpectationMap({
'foo':
data_types.ExpectationBuilderMap({
expectation:
data_types.BuilderStepMap(),
}),
})
unmatched_results = self._querier._FillExpectationMapForBuilders(
expectation_map, ['matched_builder', 'unmatched_builder'], 'ci')
stats = data_types.BuildStats()
stats.AddPassedBuild()
expected_expectation_map = {
'foo': {
expectation: {
'ci:matched_builder': {
'step_name': stats,
},
},
},
}
self.assertEqual(expectation_map, expected_expectation_map)
self.assertEqual(
unmatched_results, {
'ci:unmatched_builder': [
data_types.Result('bar', [], 'Pass', 'step_name',
'build_id'),
],
})
def testIgnoreExpectation(self):
"""Tests that specifying to ignore an expectation does nothing."""
self._input_mock.return_value = 'i'
# yapf: disable
test_expectation_map = data_types.TestExpectationMap({
'foo/test':
data_types.ExpectationBuilderMap({
data_types.Expectation(
'foo/test', ['win'], 'Failure', 'crbug.com/1234'):
data_types.BuilderStepMap(),
}),
})
# yapf: enable
modified_urls = expectations.ModifySemiStaleExpectations(
test_expectation_map, self.filename)
self.assertEqual(modified_urls, set())
with open(self.filename) as f:
self.assertEqual(f.read(), FAKE_EXPECTATION_FILE_CONTENTS)
def CreateTestExpectationMap(expectation_file, tests):
"""Creates an expectation map based off a file or list of tests.
Args:
expectation_file: A filepath to an expectation file to read from, or None.
If a filepath is specified, |tests| must be None.
tests: An iterable of strings containing test names to check. If specified,
|expectation_file| must be None.
Returns:
A data_types.TestExpectationMap, although all its BuilderStepMap contents
will be empty.
"""
logging.info('Creating test expectation map')
assert expectation_file or tests
assert not (expectation_file and tests)
if expectation_file:
with open(expectation_file) as f:
content = f.read()
else:
content = '# results: [ RetryOnFailure ]\n'
for t in tests:
content += '%s [ RetryOnFailure ]\n' % t
list_parser = expectations_parser.TaggedTestListParser(content)
expectation_map = data_types.TestExpectationMap()
logging.debug('Parsed %d expectations', len(list_parser.expectations))
for e in list_parser.expectations:
if 'Skip' in e.raw_results:
continue
expectation = data_types.Expectation(e.test, e.tags, e.raw_results,
e.reason)
expectations_for_test = expectation_map.setdefault(
e.test, data_types.ExpectationBuilderMap())
assert expectation not in expectations_for_test
expectations_for_test[expectation] = data_types.BuilderStepMap()
return expectation_map
def SplitExpectationsByStaleness(test_expectation_map):
"""Separates |test_expectation_map| based on expectation staleness.
Args:
test_expectation_map: A data_types.TestExpectationMap with any unused
expectations already filtered out.
Returns:
Three data_types.TestExpectationMaps (stale_dict, semi_stale_dict,
active_dict). All three combined contain the information of
|test_expectation_map|. |stale_dict| contains entries for expectations that
are no longer being helpful, |semi_stale_dict| contains entries for
expectations that might be removable or modifiable, but have at least one
failed test run. |active_dict| contains entries for expectations that are
preventing failures on all builders they're active on, and thus shouldn't be
removed.
"""
assert isinstance(test_expectation_map, data_types.TestExpectationMap)
stale_dict = data_types.TestExpectationMap()
semi_stale_dict = data_types.TestExpectationMap()
active_dict = data_types.TestExpectationMap()
# This initially looks like a good target for using
# data_types.TestExpectationMap's iterators since there are many nested loops.
# However, we need to reset state in different loops, and the alternative of
# keeping all the state outside the loop and resetting under certain
# conditions ends up being less readable than just using nested loops.
for test_name, expectation_map in test_expectation_map.iteritems():
for expectation, builder_map in expectation_map.iteritems():
# A temporary map to hold data so we can later determine whether an
# expectation is stale, semi-stale, or active.
tmp_map = {
FULL_PASS: data_types.BuilderStepMap(),
NEVER_PASS: data_types.BuilderStepMap(),
PARTIAL_PASS: data_types.BuilderStepMap(),
}
split_stats_map = builder_map.SplitBuildStatsByPass()
for builder_name, (
fully_passed, never_passed,
partially_passed) in split_stats_map.iteritems():
if fully_passed:
tmp_map[FULL_PASS][builder_name] = fully_passed
if never_passed:
tmp_map[NEVER_PASS][builder_name] = never_passed
if partially_passed:
tmp_map[PARTIAL_PASS][builder_name] = partially_passed
def _CopyPassesIntoBuilderMap(builder_map, pass_types):
for pt in pass_types:
for builder, steps in tmp_map[pt].iteritems():
builder_map.setdefault(
builder,
data_types.StepBuildStatsMap()).update(steps)
# Handle the case of a stale expectation.
if not (tmp_map[NEVER_PASS] or tmp_map[PARTIAL_PASS]):
builder_map = stale_dict.setdefault(
test_name, data_types.ExpectationBuilderMap()).setdefault(
expectation, data_types.BuilderStepMap())
_CopyPassesIntoBuilderMap(builder_map, [FULL_PASS])
# Handle the case of an active expectation.
elif not tmp_map[FULL_PASS]:
builder_map = active_dict.setdefault(
test_name, data_types.ExpectationBuilderMap()).setdefault(
expectation, data_types.BuilderStepMap())
_CopyPassesIntoBuilderMap(builder_map,
[NEVER_PASS, PARTIAL_PASS])
# Handle the case of a semi-stale expectation.
else:
# TODO(crbug.com/998329): Sort by pass percentage so it's easier to find
# problematic builders without highlighting.
builder_map = semi_stale_dict.setdefault(
test_name, data_types.ExpectationBuilderMap()).setdefault(
expectation, data_types.BuilderStepMap())
_CopyPassesIntoBuilderMap(
builder_map, [FULL_PASS, PARTIAL_PASS, NEVER_PASS])
return stale_dict, semi_stale_dict, active_dict
def testSemiStaleMap(self):
"""Tests that everything functions when regular data is provided."""
expectation_map = data_types.TestExpectationMap({
'foo':
data_types.ExpectationBuilderMap({
data_types.Expectation('foo', ['win', 'intel'], ['RetryOnFailure']):
data_types.BuilderStepMap({
'builder':
data_types.StepBuildStatsMap({
'all_pass':
uu.CreateStatsWithPassFails(2, 0),
'all_fail':
uu.CreateStatsWithPassFails(0, 2),
'some_pass':
uu.CreateStatsWithPassFails(1, 1),
}),
}),
data_types.Expectation('foo', ['linux', 'intel'], [
'RetryOnFailure'
]):
data_types.BuilderStepMap({
'builder':
data_types.StepBuildStatsMap({
'all_pass':
uu.CreateStatsWithPassFails(2, 0),
}),
}),
data_types.Expectation('foo', ['mac', 'intel'], ['RetryOnFailure']):
data_types.BuilderStepMap({
'builder':
data_types.StepBuildStatsMap({
'all_fail':
uu.CreateStatsWithPassFails(0, 2),
}),
}),
}),
})
expected_ouput = {
'foo': {
'"RetryOnFailure" expectation on "win intel"': {
'builder': {
'Fully passed in the following': [
'all_pass (2/2)',
],
'Never passed in the following': [
'all_fail (0/2)',
],
'Partially passed in the following': {
'some_pass (1/2)': [
data_types.BuildLinkFromBuildId('build_id0'),
],
},
},
},
'"RetryOnFailure" expectation on "intel linux"': {
'builder': {
'Fully passed in the following': [
'all_pass (2/2)',
],
},
},
'"RetryOnFailure" expectation on "mac intel"': {
'builder': {
'Never passed in the following': [
'all_fail (0/2)',
],
},
},
},
}
str_dict = result_output._ConvertTestExpectationMapToStringDict(
expectation_map)
self.assertEqual(str_dict, expected_ouput)
def testIterToNoSuchValue(self):
"""Tests that iterating to a type that has no data works as expected."""
test_expectation_map = data_types.TestExpectationMap()
# This should neither break nor return any data.
for _, __, ___ in test_expectation_map.IterBuilderStepMaps():
self.fail()
def SplitExpectationsByStaleness(test_expectation_map):
"""Separates |test_expectation_map| based on expectation staleness.
Args:
test_expectation_map: A data_types.TestExpectationMap with any unused
expectations already filtered out.
Returns:
Three data_types.TestExpectationMaps (stale_dict, semi_stale_dict,
active_dict). All three combined contain the information of
|test_expectation_map|. |stale_dict| contains entries for expectations that
are no longer being helpful, |semi_stale_dict| contains entries for
expectations that might be removable or modifiable, but have at least one
failed test run. |active_dict| contains entries for expectations that are
preventing failures on all builders they're active on, and thus shouldn't be
removed.
"""
assert isinstance(test_expectation_map, data_types.TestExpectationMap)
FULL_PASS = 1
NEVER_PASS = 2
PARTIAL_PASS = 3
stale_dict = data_types.TestExpectationMap()
semi_stale_dict = data_types.TestExpectationMap()
active_dict = data_types.TestExpectationMap()
for test_name, expectation_map in test_expectation_map.iteritems():
for expectation, builder_map in expectation_map.iteritems():
# A temporary map to hold data so we can later determine whether an
# expectation is stale, semi-stale, or active.
tmp_map = {
FULL_PASS: data_types.BuilderStepMap(),
NEVER_PASS: data_types.BuilderStepMap(),
PARTIAL_PASS: data_types.BuilderStepMap(),
}
for builder_name, step_map in builder_map.iteritems():
fully_passed = data_types.StepBuildStatsMap()
partially_passed = data_types.StepBuildStatsMap()
never_passed = data_types.StepBuildStatsMap()
for step_name, stats in step_map.iteritems():
if stats.passed_builds == stats.total_builds:
assert step_name not in fully_passed
fully_passed[step_name] = stats
elif stats.failed_builds == stats.total_builds:
assert step_name not in never_passed
never_passed[step_name] = stats
else:
assert step_name not in partially_passed
partially_passed[step_name] = stats
if fully_passed:
tmp_map[FULL_PASS][builder_name] = fully_passed
if never_passed:
tmp_map[NEVER_PASS][builder_name] = never_passed
if partially_passed:
tmp_map[PARTIAL_PASS][builder_name] = partially_passed
def _CopyPassesIntoBuilderMap(builder_map, pass_types):
for pt in pass_types:
for builder, steps in tmp_map[pt].iteritems():
builder_map.setdefault(
builder,
data_types.StepBuildStatsMap()).update(steps)
# Handle the case of a stale expectation.
if not (tmp_map[NEVER_PASS] or tmp_map[PARTIAL_PASS]):
builder_map = stale_dict.setdefault(
test_name, data_types.ExpectationBuilderMap()).setdefault(
expectation, data_types.BuilderStepMap())
_CopyPassesIntoBuilderMap(builder_map, [FULL_PASS])
# Handle the case of an active expectation.
elif not tmp_map[FULL_PASS]:
builder_map = active_dict.setdefault(
test_name, data_types.ExpectationBuilderMap()).setdefault(
expectation, data_types.BuilderStepMap())
_CopyPassesIntoBuilderMap(builder_map,
[NEVER_PASS, PARTIAL_PASS])
# Handle the case of a semi-stale expectation.
else:
# TODO(crbug.com/998329): Sort by pass percentage so it's easier to find
# problematic builders without highlighting.
builder_map = semi_stale_dict.setdefault(
test_name, data_types.ExpectationBuilderMap()).setdefault(
expectation, data_types.BuilderStepMap())
_CopyPassesIntoBuilderMap(
builder_map, [FULL_PASS, PARTIAL_PASS, NEVER_PASS])
return stale_dict, semi_stale_dict, active_dict
def testMissingKeys(self):
"""Tests that missing keys are properly copied to the base map."""
base_map = data_types.TestExpectationMap({
'foo':
data_types.ExpectationBuilderMap({
data_types.Expectation('foo', ['win'], 'Failure'):
data_types.BuilderStepMap({
'builder':
data_types.StepBuildStatsMap({
'step':
data_types.BuildStats(),
}),
}),
}),
})
merge_map = data_types.TestExpectationMap({
'foo':
data_types.ExpectationBuilderMap({
data_types.Expectation('foo', ['win'], 'Failure'):
data_types.BuilderStepMap({
'builder':
data_types.StepBuildStatsMap({
'step2':
data_types.BuildStats(),
}),
'builder2':
data_types.StepBuildStatsMap({
'step':
data_types.BuildStats(),
}),
}),
data_types.Expectation('foo', ['mac'], 'Failure'):
data_types.BuilderStepMap({
'builder':
data_types.StepBuildStatsMap({
'step':
data_types.BuildStats(),
})
})
}),
'bar':
data_types.ExpectationBuilderMap({
data_types.Expectation('bar', ['win'], 'Failure'):
data_types.BuilderStepMap({
'builder':
data_types.StepBuildStatsMap({
'step':
data_types.BuildStats(),
}),
}),
}),
})
expected_base_map = {
'foo': {
data_types.Expectation('foo', ['win'], 'Failure'): {
'builder': {
'step': data_types.BuildStats(),
'step2': data_types.BuildStats(),
},
'builder2': {
'step': data_types.BuildStats(),
},
},
data_types.Expectation('foo', ['mac'], 'Failure'): {
'builder': {
'step': data_types.BuildStats(),
}
}
},
'bar': {
data_types.Expectation('bar', ['win'], 'Failure'): {
'builder': {
'step': data_types.BuildStats(),
},
},
},
}
expectations.MergeExpectationMaps(base_map, merge_map)
self.assertEqual(base_map, expected_base_map)
def testEmptyMap(self):
"""Tests that providing an empty map is a no-op."""
self.assertEqual(
result_output._ConvertTestExpectationMapToStringDict(
data_types.TestExpectationMap()), {})
def testAllExpectations(self):
"""Tests output when all three types of expectations are provided."""
expectation_map = data_types.TestExpectationMap({
'foo':
data_types.ExpectationBuilderMap({
data_types.Expectation('foo', ['stale'], 'Failure'):
data_types.BuilderStepMap({
'foo_builder':
data_types.StepBuildStatsMap({
'step1':
uu.CreateStatsWithPassFails(1, 0),
'step2':
uu.CreateStatsWithPassFails(2, 0),
}),
'bar_builder':
data_types.StepBuildStatsMap({
'step1':
uu.CreateStatsWithPassFails(3, 0),
'step2':
uu.CreateStatsWithPassFails(4, 0)
}),
}),
data_types.Expectation('foo', ['semistale'], 'Failure'):
data_types.BuilderStepMap({
'foo_builder':
data_types.StepBuildStatsMap({
'step1':
uu.CreateStatsWithPassFails(1, 0),
'step2':
uu.CreateStatsWithPassFails(2, 2),
}),
'bar_builder':
data_types.StepBuildStatsMap({
'step1':
uu.CreateStatsWithPassFails(3, 0),
'step2':
uu.CreateStatsWithPassFails(0, 4)
}),
}),
data_types.Expectation('foo', ['active'], 'Failure'):
data_types.BuilderStepMap({
'foo_builder':
data_types.StepBuildStatsMap({
'step1':
uu.CreateStatsWithPassFails(1, 1),
'step2':
uu.CreateStatsWithPassFails(2, 2),
}),
'bar_builder':
data_types.StepBuildStatsMap({
'step1':
uu.CreateStatsWithPassFails(3, 3),
'step2':
uu.CreateStatsWithPassFails(0, 4)
}),
}),
}),
})
expected_stale = {
'foo': {
data_types.Expectation('foo', ['stale'], 'Failure'): {
'foo_builder': {
'step1': uu.CreateStatsWithPassFails(1, 0),
'step2': uu.CreateStatsWithPassFails(2, 0),
},
'bar_builder': {
'step1': uu.CreateStatsWithPassFails(3, 0),
'step2': uu.CreateStatsWithPassFails(4, 0)
},
},
},
}
expected_semi_stale = {
'foo': {
data_types.Expectation('foo', ['semistale'], 'Failure'): {
'foo_builder': {
'step1': uu.CreateStatsWithPassFails(1, 0),
'step2': uu.CreateStatsWithPassFails(2, 2),
},
'bar_builder': {
'step1': uu.CreateStatsWithPassFails(3, 0),
'step2': uu.CreateStatsWithPassFails(0, 4)
},
},
},
}
expected_active = {
'foo': {
data_types.Expectation('foo', ['active'], 'Failure'): {
'foo_builder': {
'step1': uu.CreateStatsWithPassFails(1, 1),
'step2': uu.CreateStatsWithPassFails(2, 2),
},
'bar_builder': {
'step1': uu.CreateStatsWithPassFails(3, 3),
'step2': uu.CreateStatsWithPassFails(0, 4)
},
},
},
}
stale_dict, semi_stale_dict, active_dict =\
expectations.SplitExpectationsByStaleness(expectation_map)
self.assertEqual(stale_dict, expected_stale)
self.assertEqual(semi_stale_dict, expected_semi_stale)
self.assertEqual(active_dict, expected_active)
def testOutputResultsSmoketest(self):
"""Test that nothing blows up when outputting."""
expectation_map = data_types.TestExpectationMap({
'foo':
data_types.ExpectationBuilderMap({
data_types.Expectation('foo', ['win', 'intel'], 'RetryOnFailure'):
data_types.BuilderStepMap({
'stale':
data_types.StepBuildStatsMap({
'all_pass':
uu.CreateStatsWithPassFails(2, 0),
}),
}),
data_types.Expectation('foo', ['linux'], 'Failure'):
data_types.BuilderStepMap({
'semi_stale':
data_types.StepBuildStatsMap({
'all_pass':
uu.CreateStatsWithPassFails(2, 0),
'some_pass':
uu.CreateStatsWithPassFails(1, 1),
'none_pass':
uu.CreateStatsWithPassFails(0, 2),
}),
}),
data_types.Expectation('foo', ['mac'], 'Failure'):
data_types.BuilderStepMap({
'active':
data_types.StepBuildStatsMap({
'none_pass':
uu.CreateStatsWithPassFails(0, 2),
}),
}),
}),
})
unmatched_results = {
'builder': [
data_types.Result('foo', ['win', 'intel'], 'Failure', 'step_name',
'build_id'),
],
}
unmatched_expectations = [
data_types.Expectation('foo', ['linux'], 'RetryOnFailure')
]
stale, semi_stale, active = expectations.SplitExpectationsByStaleness(
expectation_map)
result_output.OutputResults(stale, semi_stale, active, {}, [], 'print',
self._file_handle)
result_output.OutputResults(stale, semi_stale, active, unmatched_results,
[], 'print', self._file_handle)
result_output.OutputResults(stale, semi_stale, active, {},
unmatched_expectations, 'print',
self._file_handle)
result_output.OutputResults(stale, semi_stale, active, unmatched_results,
unmatched_expectations, 'print',
self._file_handle)
result_output.OutputResults(stale, semi_stale, active, {}, [], 'html',
self._file_handle)
result_output.OutputResults(stale, semi_stale, active, unmatched_results,
[], 'html', self._file_handle)
result_output.OutputResults(stale, semi_stale, active, {},
unmatched_expectations, 'html',
self._file_handle)
result_output.OutputResults(stale, semi_stale, active, unmatched_results,
unmatched_expectations, 'html',
self._file_handle)
def testQueryFailureIsSurfaced(self):
"""Tests that a query failure is properly surfaced despite being async."""
self._query_mock.side_effect = IndexError('failure')
with self.assertRaises(IndexError):
self._querier._FillExpectationMapForBuilders(
data_types.TestExpectationMap(), ['matched_builder'], 'ci')
