def _GetSampleTestExpectationMap(self):
build_stats = self._GetSampleBuildStats()
return data_types.TestExpectationMap({
'foo':
data_types.ExpectationBuilderMap({
data_types.Expectation('foo', ['tag'], ['Failure']):
data_types.BuilderStepMap({
'builder1':
data_types.StepBuildStatsMap({
'step1': build_stats[0],
'step2': build_stats[1],
}),
'builder2':
data_types.StepBuildStatsMap({
'step3': build_stats[2],
'step4': build_stats[3],
}),
}),
data_types.Expectation('foo', ['tag2'], ['Failure']):
data_types.BuilderStepMap({
'builder3':
data_types.StepBuildStatsMap({
'step5': build_stats[4],
'step6': build_stats[5],
}),
'builder4':
data_types.StepBuildStatsMap({
'step7': build_stats[6],
'step8': build_stats[7],
}),
}),
}),
})
def testUnusedButNotEmpty(self):
"""Tests filtering if there is an unused expectation but no empty tests."""
expectation_map = data_types.TestExpectationMap({
'foo/test':
data_types.ExpectationBuilderMap({
data_types.Expectation('foo/test', ['win'], ['Failure']):
data_types.BuilderStepMap({
'SomeBuilder':
data_types.StepBuildStatsMap(),
}),
data_types.Expectation('foo/test', ['linux'], ['Failure']):
data_types.BuilderStepMap(),
})
})
expected_expectation_map = data_types.TestExpectationMap({
'foo/test':
data_types.ExpectationBuilderMap({
data_types.Expectation('foo/test', ['win'], ['Failure']):
data_types.BuilderStepMap({
'SomeBuilder':
data_types.StepBuildStatsMap(),
}),
}),
})
unused_expectations = expectations.FilterOutUnusedExpectations(
expectation_map)
self.assertEqual(
unused_expectations,
[data_types.Expectation('foo/test', ['linux'], ['Failure'])])
self.assertEqual(expectation_map, expected_expectation_map)
def testResultMatchMultiMatch(self):
"""Test adding a passing result when multiple expectations match."""
r = data_types.Result('some/test/case', ['win', 'win10'], 'Pass',
'pixel_tests', 'build_id')
e = data_types.Expectation('some/test/*', ['win10'], 'Failure')
e2 = data_types.Expectation('some/test/case', ['win10'], 'Failure')
expectation_map = data_types.TestExpectationMap({
'some/test/*':
data_types.ExpectationBuilderMap({
e: data_types.BuilderStepMap(),
e2: data_types.BuilderStepMap(),
}),
})
found_matching = expectations._AddResultToMap(r, 'builder',
expectation_map)
self.assertTrue(found_matching)
stats = data_types.BuildStats()
stats.AddPassedBuild()
expected_expectation_map = {
'some/test/*': {
e: {
'builder': {
'pixel_tests': stats,
},
},
e2: {
'builder': {
'pixel_tests': stats,
},
}
}
}
self.assertEqual(expectation_map, expected_expectation_map)
def testSemiStaleExpectations(self):
"""Tests output when only semi-stale expectations are provided."""
expectation_map = data_types.TestExpectationMap({
'foo':
data_types.ExpectationBuilderMap({
data_types.Expectation('foo', ['win'], ['Failure']):
data_types.BuilderStepMap({
'foo_builder':
data_types.StepBuildStatsMap({
'step1':
uu.CreateStatsWithPassFails(1, 0),
'step2':
uu.CreateStatsWithPassFails(2, 2),
}),
'bar_builder':
data_types.StepBuildStatsMap({
'step1':
uu.CreateStatsWithPassFails(3, 0),
'step2':
uu.CreateStatsWithPassFails(0, 4)
}),
}),
data_types.Expectation('foo', ['linux'], ['RetryOnFailure']):
data_types.BuilderStepMap({
'foo_builder':
data_types.StepBuildStatsMap({
'step1':
uu.CreateStatsWithPassFails(5, 0),
'step2':
uu.CreateStatsWithPassFails(6, 6),
}),
}),
}),
'bar':
data_types.ExpectationBuilderMap({
data_types.Expectation('bar', ['win'], ['Failure']):
data_types.BuilderStepMap({
'foo_builder':
data_types.StepBuildStatsMap({
'step1':
uu.CreateStatsWithPassFails(7, 0),
}),
'bar_builder':
data_types.StepBuildStatsMap({
'step1':
uu.CreateStatsWithPassFails(0, 8),
}),
}),
}),
})
expected_semi_stale_dict = copy.deepcopy(expectation_map)
stale_dict, semi_stale_dict, active_dict =\
expectations.SplitExpectationsByStaleness(expectation_map)
self.assertEqual(stale_dict, {})
self.assertEqual(semi_stale_dict, expected_semi_stale_dict)
self.assertEqual(active_dict, {})
def testExpectationBuilderMap(self):
"""Tests ExpectationBuilderMap's type enforcement."""
m = data_types.ExpectationBuilderMap()
e = data_types.Expectation('test', ['tag'], 'Failure')
with self.assertRaises(AssertionError):
m[1] = data_types.BuilderStepMap()
with self.assertRaises(AssertionError):
m[e] = 2
m[e] = data_types.BuilderStepMap()
self.assertEqual(m, {e: data_types.BuilderStepMap()})
def testResultMatchFailingExisting(self):
"""Test adding a failing result when results for a builder exist."""
r = data_types.Result('some/test/case', ['win', 'win10'], 'Failure',
'pixel_tests', 'build_id')
e = data_types.Expectation('some/test/*', ['win10'], 'Failure')
stats = data_types.BuildStats()
stats.AddPassedBuild()
expectation_map = data_types.TestExpectationMap({
'some/test/*':
data_types.ExpectationBuilderMap({
e:
data_types.BuilderStepMap({
'builder':
data_types.StepBuildStatsMap({
'pixel_tests': stats,
}),
}),
}),
})
found_matching = expectations._AddResultToMap(r, 'builder',
expectation_map)
self.assertTrue(found_matching)
stats = data_types.BuildStats()
stats.AddFailedBuild('build_id')
stats.AddPassedBuild()
expected_expectation_map = {
'some/test/*': {
e: {
'builder': {
'pixel_tests': stats,
},
},
},
}
self.assertEqual(expectation_map, expected_expectation_map)
def testRemoveExpectation(self):
"""Tests that specifying to remove an expectation does so."""
self._input_mock.return_value = 'r'
# yapf: disable
test_expectation_map = data_types.TestExpectationMap({
'foo/test':
data_types.ExpectationBuilderMap({
data_types.Expectation(
'foo/test', ['win'], 'Failure', 'crbug.com/1234'):
data_types.BuilderStepMap(),
}),
})
# yapf: enable
modified_urls = expectations.ModifySemiStaleExpectations(
test_expectation_map, self.filename)
self.assertEqual(modified_urls, set(['crbug.com/1234']))
expected_file_contents = """\
# tags: [ win linux ]
# results: [ Failure RetryOnFailure Skip ]
[ linux ] foo/test [ Failure ]
crbug.com/2345 [ linux ] bar/* [ RetryOnFailure ]
crbug.com/3456 [ linux ] some/bad/test [ Skip ]
"""
with open(self.filename) as f:
self.assertEqual(f.read(), expected_file_contents)
def testEmptyInput(self):
"""Tests that an empty input doesn't cause breakage."""
output = expectations._ConvertBuilderMapToPassOrderedStringDict(
data_types.BuilderStepMap())
expected_output = collections.OrderedDict()
expected_output[result_output.FULL_PASS] = {}
expected_output[result_output.NEVER_PASS] = {}
expected_output[result_output.PARTIAL_PASS] = {}
self.assertEqual(output, expected_output)
def GetEmptyMapForGenericFailureExpectation(self):
foo_expectation = self.GetGenericFailureExpectation()
return data_types.TestExpectationMap({
'foo/test':
data_types.ExpectationBuilderMap({
foo_expectation:
data_types.BuilderStepMap(),
}),
})
def testMergeBuildStats(self):
"""Tests that BuildStats for the same step are merged properly."""
base_map = data_types.TestExpectationMap({
'foo':
data_types.ExpectationBuilderMap({
data_types.Expectation('foo', ['win'], 'Failure'):
data_types.BuilderStepMap({
'builder':
data_types.StepBuildStatsMap({
'step':
data_types.BuildStats(),
}),
}),
}),
})
merge_stats = data_types.BuildStats()
merge_stats.AddFailedBuild('1')
merge_map = data_types.TestExpectationMap({
'foo':
data_types.ExpectationBuilderMap({
data_types.Expectation('foo', ['win'], 'Failure'):
data_types.BuilderStepMap({
'builder':
data_types.StepBuildStatsMap({
'step': merge_stats,
}),
}),
}),
})
expected_stats = data_types.BuildStats()
expected_stats.AddFailedBuild('1')
expected_base_map = {
'foo': {
data_types.Expectation('foo', ['win'], 'Failure'): {
'builder': {
'step': expected_stats,
},
},
},
}
expectations.MergeExpectationMaps(base_map, merge_map)
self.assertEqual(base_map, expected_base_map)
def GetMapForExpectationAndStats(self, expectation, stats):
return data_types.TestExpectationMap({
expectation.test:
data_types.ExpectationBuilderMap({
expectation:
data_types.BuilderStepMap({
'builder':
data_types.StepBuildStatsMap({
'pixel_tests': stats,
}),
}),
}),
})
def testUnusedAndEmpty(self):
"""Tests filtering if there is an expectation that causes an empty test."""
expectation_map = data_types.TestExpectationMap({
'foo/test':
data_types.ExpectationBuilderMap({
data_types.Expectation('foo/test', ['win'], ['Failure']):
data_types.BuilderStepMap(),
}),
})
unused_expectations = expectations.FilterOutUnusedExpectations(
expectation_map)
self.assertEqual(
unused_expectations,
[data_types.Expectation('foo/test', ['win'], ['Failure'])])
self.assertEqual(expectation_map, {})
def testResultNoMatch(self):
"""Tests that a result is not added if no match is found."""
r = data_types.Result('some/test/case', ['win', 'win10'], 'Failure',
'pixel_tests', 'build_id')
e = data_types.Expectation('some/test/*', ['win10', 'foo'], 'Failure')
expectation_map = data_types.TestExpectationMap({
'some/test/*':
data_types.ExpectationBuilderMap({
e: data_types.BuilderStepMap(),
})
})
found_matching = expectations._AddResultToMap(r, 'builder',
expectation_map)
self.assertFalse(found_matching)
expected_expectation_map = {'some/test/*': {e: {}}}
self.assertEqual(expectation_map, expected_expectation_map)
def testNoUnused(self):
"""Tests that filtering is a no-op if there are no unused expectations."""
expectation_map = data_types.TestExpectationMap({
'foo/test':
data_types.ExpectationBuilderMap({
data_types.Expectation('foo/test', ['win'], ['Failure']):
data_types.BuilderStepMap({
'SomeBuilder':
data_types.StepBuildStatsMap(),
}),
})
})
expected_expectation_map = copy.deepcopy(expectation_map)
unused_expectations = expectations.FilterOutUnusedExpectations(
expectation_map)
self.assertEqual(len(unused_expectations), 0)
self.assertEqual(expectation_map, expected_expectation_map)
def testValidResults(self):
"""Tests functionality when valid results are returned by the query."""
def SideEffect(builder, *args):
del args
if builder == 'matched_builder':
return [
data_types.Result('foo', ['win'], 'Pass', 'step_name',
'build_id')
]
else:
return [
data_types.Result('bar', [], 'Pass', 'step_name',
'build_id')
]
self._query_mock.side_effect = SideEffect
expectation = data_types.Expectation('foo', ['win'], 'RetryOnFailure')
expectation_map = data_types.TestExpectationMap({
'foo':
data_types.ExpectationBuilderMap({
expectation:
data_types.BuilderStepMap(),
}),
})
unmatched_results = self._querier._FillExpectationMapForBuilders(
expectation_map, ['matched_builder', 'unmatched_builder'], 'ci')
stats = data_types.BuildStats()
stats.AddPassedBuild()
expected_expectation_map = {
'foo': {
expectation: {
'ci:matched_builder': {
'step_name': stats,
},
},
},
}
self.assertEqual(expectation_map, expected_expectation_map)
self.assertEqual(
unmatched_results, {
'ci:unmatched_builder': [
data_types.Result('bar', [], 'Pass', 'step_name',
'build_id'),
],
})
def testIgnoreExpectation(self):
"""Tests that specifying to ignore an expectation does nothing."""
self._input_mock.return_value = 'i'
# yapf: disable
test_expectation_map = data_types.TestExpectationMap({
'foo/test':
data_types.ExpectationBuilderMap({
data_types.Expectation(
'foo/test', ['win'], 'Failure', 'crbug.com/1234'):
data_types.BuilderStepMap(),
}),
})
# yapf: enable
modified_urls = expectations.ModifySemiStaleExpectations(
test_expectation_map, self.filename)
self.assertEqual(modified_urls, set())
with open(self.filename) as f:
self.assertEqual(f.read(), FAKE_EXPECTATION_FILE_CONTENTS)
def testEmptyMergeMap(self):
"""Tests that a merge with an empty merge map is a no-op."""
base_map = data_types.TestExpectationMap({
'foo':
data_types.ExpectationBuilderMap({
data_types.Expectation('foo', ['win'], 'Failure'):
data_types.BuilderStepMap({
'builder':
data_types.StepBuildStatsMap({
'step':
data_types.BuildStats(),
}),
}),
}),
})
merge_map = data_types.TestExpectationMap()
original_base_map = copy.deepcopy(base_map)
expectations.MergeExpectationMaps(base_map, merge_map)
self.assertEqual(base_map, original_base_map)
self.assertEqual(merge_map, {})
def CreateTestExpectationMap(expectation_file, tests):
"""Creates an expectation map based off a file or list of tests.
Args:
expectation_file: A filepath to an expectation file to read from, or None.
If a filepath is specified, |tests| must be None.
tests: An iterable of strings containing test names to check. If specified,
|expectation_file| must be None.
Returns:
A data_types.TestExpectationMap, although all its BuilderStepMap contents
will be empty.
"""
logging.info('Creating test expectation map')
assert expectation_file or tests
assert not (expectation_file and tests)
if expectation_file:
with open(expectation_file) as f:
content = f.read()
else:
content = '# results: [ RetryOnFailure ]\n'
for t in tests:
content += '%s [ RetryOnFailure ]\n' % t
list_parser = expectations_parser.TaggedTestListParser(content)
expectation_map = data_types.TestExpectationMap()
logging.debug('Parsed %d expectations', len(list_parser.expectations))
for e in list_parser.expectations:
if 'Skip' in e.raw_results:
continue
expectation = data_types.Expectation(e.test, e.tags, e.raw_results,
e.reason)
expectations_for_test = expectation_map.setdefault(
e.test, data_types.ExpectationBuilderMap())
assert expectation not in expectations_for_test
expectations_for_test[expectation] = data_types.BuilderStepMap()
return expectation_map
def SplitExpectationsByStaleness(test_expectation_map):
"""Separates |test_expectation_map| based on expectation staleness.
Args:
test_expectation_map: A data_types.TestExpectationMap with any unused
expectations already filtered out.
Returns:
Three data_types.TestExpectationMaps (stale_dict, semi_stale_dict,
active_dict). All three combined contain the information of
|test_expectation_map|. |stale_dict| contains entries for expectations that
are no longer being helpful, |semi_stale_dict| contains entries for
expectations that might be removable or modifiable, but have at least one
failed test run. |active_dict| contains entries for expectations that are
preventing failures on all builders they're active on, and thus shouldn't be
removed.
"""
assert isinstance(test_expectation_map, data_types.TestExpectationMap)
stale_dict = data_types.TestExpectationMap()
semi_stale_dict = data_types.TestExpectationMap()
active_dict = data_types.TestExpectationMap()
# This initially looks like a good target for using
# data_types.TestExpectationMap's iterators since there are many nested loops.
# However, we need to reset state in different loops, and the alternative of
# keeping all the state outside the loop and resetting under certain
# conditions ends up being less readable than just using nested loops.
for test_name, expectation_map in test_expectation_map.iteritems():
for expectation, builder_map in expectation_map.iteritems():
# A temporary map to hold data so we can later determine whether an
# expectation is stale, semi-stale, or active.
tmp_map = {
FULL_PASS: data_types.BuilderStepMap(),
NEVER_PASS: data_types.BuilderStepMap(),
PARTIAL_PASS: data_types.BuilderStepMap(),
}
split_stats_map = builder_map.SplitBuildStatsByPass()
for builder_name, (
fully_passed, never_passed,
partially_passed) in split_stats_map.iteritems():
if fully_passed:
tmp_map[FULL_PASS][builder_name] = fully_passed
if never_passed:
tmp_map[NEVER_PASS][builder_name] = never_passed
if partially_passed:
tmp_map[PARTIAL_PASS][builder_name] = partially_passed
def _CopyPassesIntoBuilderMap(builder_map, pass_types):
for pt in pass_types:
for builder, steps in tmp_map[pt].iteritems():
builder_map.setdefault(
builder,
data_types.StepBuildStatsMap()).update(steps)
# Handle the case of a stale expectation.
if not (tmp_map[NEVER_PASS] or tmp_map[PARTIAL_PASS]):
builder_map = stale_dict.setdefault(
test_name, data_types.ExpectationBuilderMap()).setdefault(
expectation, data_types.BuilderStepMap())
_CopyPassesIntoBuilderMap(builder_map, [FULL_PASS])
# Handle the case of an active expectation.
elif not tmp_map[FULL_PASS]:
builder_map = active_dict.setdefault(
test_name, data_types.ExpectationBuilderMap()).setdefault(
expectation, data_types.BuilderStepMap())
_CopyPassesIntoBuilderMap(builder_map,
[NEVER_PASS, PARTIAL_PASS])
# Handle the case of a semi-stale expectation.
else:
# TODO(crbug.com/998329): Sort by pass percentage so it's easier to find
# problematic builders without highlighting.
builder_map = semi_stale_dict.setdefault(
test_name, data_types.ExpectationBuilderMap()).setdefault(
expectation, data_types.BuilderStepMap())
_CopyPassesIntoBuilderMap(
builder_map, [FULL_PASS, PARTIAL_PASS, NEVER_PASS])
return stale_dict, semi_stale_dict, active_dict
def testSemiStaleMap(self):
"""Tests that everything functions when regular data is provided."""
expectation_map = data_types.TestExpectationMap({
'foo':
data_types.ExpectationBuilderMap({
data_types.Expectation('foo', ['win', 'intel'], ['RetryOnFailure']):
data_types.BuilderStepMap({
'builder':
data_types.StepBuildStatsMap({
'all_pass':
uu.CreateStatsWithPassFails(2, 0),
'all_fail':
uu.CreateStatsWithPassFails(0, 2),
'some_pass':
uu.CreateStatsWithPassFails(1, 1),
}),
}),
data_types.Expectation('foo', ['linux', 'intel'], [
'RetryOnFailure'
]):
data_types.BuilderStepMap({
'builder':
data_types.StepBuildStatsMap({
'all_pass':
uu.CreateStatsWithPassFails(2, 0),
}),
}),
data_types.Expectation('foo', ['mac', 'intel'], ['RetryOnFailure']):
data_types.BuilderStepMap({
'builder':
data_types.StepBuildStatsMap({
'all_fail':
uu.CreateStatsWithPassFails(0, 2),
}),
}),
}),
})
expected_ouput = {
'foo': {
'"RetryOnFailure" expectation on "win intel"': {
'builder': {
'Fully passed in the following': [
'all_pass (2/2)',
],
'Never passed in the following': [
'all_fail (0/2)',
],
'Partially passed in the following': {
'some_pass (1/2)': [
data_types.BuildLinkFromBuildId('build_id0'),
],
},
},
},
'"RetryOnFailure" expectation on "intel linux"': {
'builder': {
'Fully passed in the following': [
'all_pass (2/2)',
],
},
},
'"RetryOnFailure" expectation on "mac intel"': {
'builder': {
'Never passed in the following': [
'all_fail (0/2)',
],
},
},
},
}
str_dict = result_output._ConvertTestExpectationMapToStringDict(
expectation_map)
self.assertEqual(str_dict, expected_ouput)
def SplitExpectationsByStaleness(test_expectation_map):
"""Separates |test_expectation_map| based on expectation staleness.
Args:
test_expectation_map: A data_types.TestExpectationMap with any unused
expectations already filtered out.
Returns:
Three data_types.TestExpectationMaps (stale_dict, semi_stale_dict,
active_dict). All three combined contain the information of
|test_expectation_map|. |stale_dict| contains entries for expectations that
are no longer being helpful, |semi_stale_dict| contains entries for
expectations that might be removable or modifiable, but have at least one
failed test run. |active_dict| contains entries for expectations that are
preventing failures on all builders they're active on, and thus shouldn't be
removed.
"""
assert isinstance(test_expectation_map, data_types.TestExpectationMap)
FULL_PASS = 1
NEVER_PASS = 2
PARTIAL_PASS = 3
stale_dict = data_types.TestExpectationMap()
semi_stale_dict = data_types.TestExpectationMap()
active_dict = data_types.TestExpectationMap()
for test_name, expectation_map in test_expectation_map.iteritems():
for expectation, builder_map in expectation_map.iteritems():
# A temporary map to hold data so we can later determine whether an
# expectation is stale, semi-stale, or active.
tmp_map = {
FULL_PASS: data_types.BuilderStepMap(),
NEVER_PASS: data_types.BuilderStepMap(),
PARTIAL_PASS: data_types.BuilderStepMap(),
}
for builder_name, step_map in builder_map.iteritems():
fully_passed = data_types.StepBuildStatsMap()
partially_passed = data_types.StepBuildStatsMap()
never_passed = data_types.StepBuildStatsMap()
for step_name, stats in step_map.iteritems():
if stats.passed_builds == stats.total_builds:
assert step_name not in fully_passed
fully_passed[step_name] = stats
elif stats.failed_builds == stats.total_builds:
assert step_name not in never_passed
never_passed[step_name] = stats
else:
assert step_name not in partially_passed
partially_passed[step_name] = stats
if fully_passed:
tmp_map[FULL_PASS][builder_name] = fully_passed
if never_passed:
tmp_map[NEVER_PASS][builder_name] = never_passed
if partially_passed:
tmp_map[PARTIAL_PASS][builder_name] = partially_passed
def _CopyPassesIntoBuilderMap(builder_map, pass_types):
for pt in pass_types:
for builder, steps in tmp_map[pt].iteritems():
builder_map.setdefault(
builder,
data_types.StepBuildStatsMap()).update(steps)
# Handle the case of a stale expectation.
if not (tmp_map[NEVER_PASS] or tmp_map[PARTIAL_PASS]):
builder_map = stale_dict.setdefault(
test_name, data_types.ExpectationBuilderMap()).setdefault(
expectation, data_types.BuilderStepMap())
_CopyPassesIntoBuilderMap(builder_map, [FULL_PASS])
# Handle the case of an active expectation.
elif not tmp_map[FULL_PASS]:
builder_map = active_dict.setdefault(
test_name, data_types.ExpectationBuilderMap()).setdefault(
expectation, data_types.BuilderStepMap())
_CopyPassesIntoBuilderMap(builder_map,
[NEVER_PASS, PARTIAL_PASS])
# Handle the case of a semi-stale expectation.
else:
# TODO(crbug.com/998329): Sort by pass percentage so it's easier to find
# problematic builders without highlighting.
builder_map = semi_stale_dict.setdefault(
test_name, data_types.ExpectationBuilderMap()).setdefault(
expectation, data_types.BuilderStepMap())
_CopyPassesIntoBuilderMap(
builder_map, [FULL_PASS, PARTIAL_PASS, NEVER_PASS])
return stale_dict, semi_stale_dict, active_dict
def testMissingKeys(self):
"""Tests that missing keys are properly copied to the base map."""
base_map = data_types.TestExpectationMap({
'foo':
data_types.ExpectationBuilderMap({
data_types.Expectation('foo', ['win'], 'Failure'):
data_types.BuilderStepMap({
'builder':
data_types.StepBuildStatsMap({
'step':
data_types.BuildStats(),
}),
}),
}),
})
merge_map = data_types.TestExpectationMap({
'foo':
data_types.ExpectationBuilderMap({
data_types.Expectation('foo', ['win'], 'Failure'):
data_types.BuilderStepMap({
'builder':
data_types.StepBuildStatsMap({
'step2':
data_types.BuildStats(),
}),
'builder2':
data_types.StepBuildStatsMap({
'step':
data_types.BuildStats(),
}),
}),
data_types.Expectation('foo', ['mac'], 'Failure'):
data_types.BuilderStepMap({
'builder':
data_types.StepBuildStatsMap({
'step':
data_types.BuildStats(),
})
})
}),
'bar':
data_types.ExpectationBuilderMap({
data_types.Expectation('bar', ['win'], 'Failure'):
data_types.BuilderStepMap({
'builder':
data_types.StepBuildStatsMap({
'step':
data_types.BuildStats(),
}),
}),
}),
})
expected_base_map = {
'foo': {
data_types.Expectation('foo', ['win'], 'Failure'): {
'builder': {
'step': data_types.BuildStats(),
'step2': data_types.BuildStats(),
},
'builder2': {
'step': data_types.BuildStats(),
},
},
data_types.Expectation('foo', ['mac'], 'Failure'): {
'builder': {
'step': data_types.BuildStats(),
}
}
},
'bar': {
data_types.Expectation('bar', ['win'], 'Failure'): {
'builder': {
'step': data_types.BuildStats(),
},
},
},
}
expectations.MergeExpectationMaps(base_map, merge_map)
self.assertEqual(base_map, expected_base_map)
def testBasic(self):
"""Tests that a map is properly converted."""
builder_map = data_types.BuilderStepMap({
'fully pass':
data_types.StepBuildStatsMap({
'step1':
uu.CreateStatsWithPassFails(1, 0),
}),
'never pass':
data_types.StepBuildStatsMap({
'step3':
uu.CreateStatsWithPassFails(0, 1),
}),
'partial pass':
data_types.StepBuildStatsMap({
'step5':
uu.CreateStatsWithPassFails(1, 1),
}),
'mixed':
data_types.StepBuildStatsMap({
'step7':
uu.CreateStatsWithPassFails(1, 0),
'step8':
uu.CreateStatsWithPassFails(0, 1),
'step9':
uu.CreateStatsWithPassFails(1, 1),
}),
})
output = expectations._ConvertBuilderMapToPassOrderedStringDict(
builder_map)
expected_output = collections.OrderedDict()
expected_output[result_output.FULL_PASS] = {
'fully pass': [
'step1 (1/1)',
],
'mixed': [
'step7 (1/1)',
],
}
expected_output[result_output.NEVER_PASS] = {
'never pass': [
'step3 (0/1)',
],
'mixed': [
'step8 (0/1)',
],
}
expected_output[result_output.PARTIAL_PASS] = {
'partial pass': {
'step5 (1/2)': [
'http://ci.chromium.org/b/build_id0',
],
},
'mixed': {
'step9 (1/2)': [
'http://ci.chromium.org/b/build_id0',
],
},
}
self.assertEqual(output, expected_output)
def testOutputResultsSmoketest(self):
"""Test that nothing blows up when outputting."""
expectation_map = data_types.TestExpectationMap({
'foo':
data_types.ExpectationBuilderMap({
data_types.Expectation('foo', ['win', 'intel'], 'RetryOnFailure'):
data_types.BuilderStepMap({
'stale':
data_types.StepBuildStatsMap({
'all_pass':
uu.CreateStatsWithPassFails(2, 0),
}),
}),
data_types.Expectation('foo', ['linux'], 'Failure'):
data_types.BuilderStepMap({
'semi_stale':
data_types.StepBuildStatsMap({
'all_pass':
uu.CreateStatsWithPassFails(2, 0),
'some_pass':
uu.CreateStatsWithPassFails(1, 1),
'none_pass':
uu.CreateStatsWithPassFails(0, 2),
}),
}),
data_types.Expectation('foo', ['mac'], 'Failure'):
data_types.BuilderStepMap({
'active':
data_types.StepBuildStatsMap({
'none_pass':
uu.CreateStatsWithPassFails(0, 2),
}),
}),
}),
})
unmatched_results = {
'builder': [
data_types.Result('foo', ['win', 'intel'], 'Failure', 'step_name',
'build_id'),
],
}
unmatched_expectations = [
data_types.Expectation('foo', ['linux'], 'RetryOnFailure')
]
stale, semi_stale, active = expectations.SplitExpectationsByStaleness(
expectation_map)
result_output.OutputResults(stale, semi_stale, active, {}, [], 'print',
self._file_handle)
result_output.OutputResults(stale, semi_stale, active, unmatched_results,
[], 'print', self._file_handle)
result_output.OutputResults(stale, semi_stale, active, {},
unmatched_expectations, 'print',
self._file_handle)
result_output.OutputResults(stale, semi_stale, active, unmatched_results,
unmatched_expectations, 'print',
self._file_handle)
result_output.OutputResults(stale, semi_stale, active, {}, [], 'html',
self._file_handle)
result_output.OutputResults(stale, semi_stale, active, unmatched_results,
[], 'html', self._file_handle)
result_output.OutputResults(stale, semi_stale, active, {},
unmatched_expectations, 'html',
self._file_handle)
result_output.OutputResults(stale, semi_stale, active, unmatched_results,
unmatched_expectations, 'html',
self._file_handle)
def testAllExpectations(self):
"""Tests output when all three types of expectations are provided."""
expectation_map = data_types.TestExpectationMap({
'foo':
data_types.ExpectationBuilderMap({
data_types.Expectation('foo', ['stale'], 'Failure'):
data_types.BuilderStepMap({
'foo_builder':
data_types.StepBuildStatsMap({
'step1':
uu.CreateStatsWithPassFails(1, 0),
'step2':
uu.CreateStatsWithPassFails(2, 0),
}),
'bar_builder':
data_types.StepBuildStatsMap({
'step1':
uu.CreateStatsWithPassFails(3, 0),
'step2':
uu.CreateStatsWithPassFails(4, 0)
}),
}),
data_types.Expectation('foo', ['semistale'], 'Failure'):
data_types.BuilderStepMap({
'foo_builder':
data_types.StepBuildStatsMap({
'step1':
uu.CreateStatsWithPassFails(1, 0),
'step2':
uu.CreateStatsWithPassFails(2, 2),
}),
'bar_builder':
data_types.StepBuildStatsMap({
'step1':
uu.CreateStatsWithPassFails(3, 0),
'step2':
uu.CreateStatsWithPassFails(0, 4)
}),
}),
data_types.Expectation('foo', ['active'], 'Failure'):
data_types.BuilderStepMap({
'foo_builder':
data_types.StepBuildStatsMap({
'step1':
uu.CreateStatsWithPassFails(1, 1),
'step2':
uu.CreateStatsWithPassFails(2, 2),
}),
'bar_builder':
data_types.StepBuildStatsMap({
'step1':
uu.CreateStatsWithPassFails(3, 3),
'step2':
uu.CreateStatsWithPassFails(0, 4)
}),
}),
}),
})
expected_stale = {
'foo': {
data_types.Expectation('foo', ['stale'], 'Failure'): {
'foo_builder': {
'step1': uu.CreateStatsWithPassFails(1, 0),
'step2': uu.CreateStatsWithPassFails(2, 0),
},
'bar_builder': {
'step1': uu.CreateStatsWithPassFails(3, 0),
'step2': uu.CreateStatsWithPassFails(4, 0)
},
},
},
}
expected_semi_stale = {
'foo': {
data_types.Expectation('foo', ['semistale'], 'Failure'): {
'foo_builder': {
'step1': uu.CreateStatsWithPassFails(1, 0),
'step2': uu.CreateStatsWithPassFails(2, 2),
},
'bar_builder': {
'step1': uu.CreateStatsWithPassFails(3, 0),
'step2': uu.CreateStatsWithPassFails(0, 4)
},
},
},
}
expected_active = {
'foo': {
data_types.Expectation('foo', ['active'], 'Failure'): {
'foo_builder': {
'step1': uu.CreateStatsWithPassFails(1, 1),
'step2': uu.CreateStatsWithPassFails(2, 2),
},
'bar_builder': {
'step1': uu.CreateStatsWithPassFails(3, 3),
'step2': uu.CreateStatsWithPassFails(0, 4)
},
},
},
}
stale_dict, semi_stale_dict, active_dict =\
expectations.SplitExpectationsByStaleness(expectation_map)
self.assertEqual(stale_dict, expected_stale)
self.assertEqual(semi_stale_dict, expected_semi_stale)
self.assertEqual(active_dict, expected_active)