def testResultMatchFailingExisting(self):
"""Test adding a failing result when results for a builder exist."""
r = data_types.Result('some/test/case', ['win', 'win10'], 'Failure',
'pixel_tests', 'build_id')
e = data_types.Expectation('some/test/*', ['win10'], 'Failure')
stats = data_types.BuildStats()
stats.AddPassedBuild()
expectation_map = data_types.TestExpectationMap({
'some/test/*':
data_types.ExpectationBuilderMap({
e:
data_types.BuilderStepMap({
'builder':
data_types.StepBuildStatsMap({
'pixel_tests': stats,
}),
}),
}),
})
found_matching = expectations._AddResultToMap(r, 'builder',
expectation_map)
self.assertTrue(found_matching)
stats = data_types.BuildStats()
stats.AddFailedBuild('build_id')
stats.AddPassedBuild()
expected_expectation_map = {
'some/test/*': {
e: {
'builder': {
'pixel_tests': stats,
},
},
},
}
self.assertEqual(expectation_map, expected_expectation_map)
def testMergeBuildStats(self):
"""Tests that BuildStats for the same step are merged properly."""
base_map = {
'foo': {
data_types.Expectation('foo', ['win'], 'Failure'): {
'builder': {
'step': data_types.BuildStats(),
},
},
},
}
merge_stats = data_types.BuildStats()
merge_stats.AddFailedBuild('1')
merge_map = {
'foo': {
data_types.Expectation('foo', ['win'], 'Failure'): {
'builder': {
'step': merge_stats,
},
},
},
}
expected_stats = data_types.BuildStats()
expected_stats.AddFailedBuild('1')
expected_base_map = {
'foo': {
data_types.Expectation('foo', ['win'], 'Failure'): {
'builder': {
'step': expected_stats,
},
},
},
}
expectations.MergeExpectationMaps(base_map, merge_map)
self.assertEqual(base_map, expected_base_map)
def testInvalidMerge(self):
"""Tests that updating a BuildStats instance twice is an error."""
base_map = {
'foo': {
data_types.Expectation('foo', ['win'], 'Failure'): {
'builder': {
'step': data_types.BuildStats(),
},
},
},
}
merge_stats = data_types.BuildStats()
merge_stats.AddFailedBuild('1')
merge_map = {
'foo': {
data_types.Expectation('foo', ['win'], 'Failure'): {
'builder': {
'step': merge_stats,
},
},
},
}
original_base_map = copy.deepcopy(base_map)
expectations.MergeExpectationMaps(base_map, merge_map,
original_base_map)
with self.assertRaises(AssertionError):
expectations.MergeExpectationMaps(base_map, merge_map,
original_base_map)
def testMapConstructor(self):
"""Tests that constructors enforce type."""
# We only use one map type since they all share the same implementation for
# this logic.
with self.assertRaises(AssertionError):
data_types.StepBuildStatsMap({1: 2})
m = data_types.StepBuildStatsMap({'step': data_types.BuildStats()})
self.assertEqual(m, {'step': data_types.BuildStats()})
def testMapSetdefault(self):
"""Tests that setdefault() enforces type."""
# We only use one map type since they all share the same implementation for
# this logic.
m = data_types.StepBuildStatsMap()
with self.assertRaises(AssertionError):
m.setdefault(1, data_types.BuildStats())
with self.assertRaises(AssertionError):
m.setdefault('1', 2)
m.setdefault('1', data_types.BuildStats())
self.assertEqual(m, {'1': data_types.BuildStats()})
def testMapUpdate(self):
"""Tests that update() enforces type."""
# We only use one map type since they all share the same implementation for
# this logic.
m = data_types.StepBuildStatsMap({'step': data_types.BuildStats()})
with self.assertRaises(AssertionError):
m.update({1: 2})
with self.assertRaises(AssertionError):
m.update(step2=1)
m.update(step=data_types.BuildStats())
self.assertEqual(m, {'step': data_types.BuildStats()})
def testResultMatchPassingNew(self):
"""Test adding a passing result when no results for a builder exist."""
r = data_types.Result('some/test/case', ['win', 'win10'], 'Pass',
'pixel_tests', 'build_id')
e = data_types.Expectation('some/test/*', ['win10'], 'Failure')
expectation_map = {
'some/test/*': {
e: {},
},
}
found_matching = expectations._AddResultToMap(r, 'builder',
expectation_map)
self.assertTrue(found_matching)
stats = data_types.BuildStats()
stats.AddPassedBuild()
expected_expectation_map = {
'some/test/*': {
e: {
'builder': {
'pixel_tests': stats,
},
},
},
}
self.assertEqual(expectation_map, expected_expectation_map)
def testAddFailedBuild(self):
s = data_types.BuildStats()
s.AddFailedBuild('build_id')
self.assertEqual(s.total_builds, 1)
self.assertEqual(s.failed_builds, 1)
self.assertEqual(s.failure_links,
frozenset(['http://ci.chromium.org/b/build_id']))
def _AddResultToMap(result, builder, test_expectation_map):
"""Adds a single |result| to |test_expectation_map|.
Args:
result: A data_types.Result object to add.
builder: A string containing the name of the builder |result| came from.
test_expectation_map: A data_types.TestExpectationMap. Will be modified
in-place.
Returns:
True if an expectation in |test_expectation_map| applied to |result|,
otherwise False.
"""
assert isinstance(test_expectation_map, data_types.TestExpectationMap)
found_matching_expectation = False
# We need to use fnmatch since wildcards are supported, so there's no point in
# checking the test name key right now. The AppliesToResult check already does
# an fnmatch check.
for _, expectation, builder_map in test_expectation_map.IterBuilderStepMaps(
):
if expectation.AppliesToResult(result):
found_matching_expectation = True
step_map = builder_map.setdefault(builder,
data_types.StepBuildStatsMap())
stats = step_map.setdefault(result.step, data_types.BuildStats())
if result.actual_result == 'Pass':
stats.AddPassedBuild()
else:
stats.AddFailedBuild(result.build_id)
return found_matching_expectation
def CreateStatsWithPassFails(passes, fails):
stats = data_types.BuildStats()
for _ in xrange(passes):
stats.AddPassedBuild()
for i in xrange(fails):
stats.AddFailedBuild('build_id%d' % i)
return stats
def testResultMatchMultiMatch(self):
"""Test adding a passing result when multiple expectations match."""
r = data_types.Result('some/test/case', ['win', 'win10'], 'Pass',
'pixel_tests', 'build_id')
e = data_types.Expectation('some/test/*', ['win10'], 'Failure')
e2 = data_types.Expectation('some/test/case', ['win10'], 'Failure')
expectation_map = data_types.TestExpectationMap({
'some/test/*':
data_types.ExpectationBuilderMap({
e: data_types.BuilderStepMap(),
e2: data_types.BuilderStepMap(),
}),
})
found_matching = expectations._AddResultToMap(r, 'builder',
expectation_map)
self.assertTrue(found_matching)
stats = data_types.BuildStats()
stats.AddPassedBuild()
expected_expectation_map = {
'some/test/*': {
e: {
'builder': {
'pixel_tests': stats,
},
},
e2: {
'builder': {
'pixel_tests': stats,
},
}
}
}
self.assertEqual(expectation_map, expected_expectation_map)
def _AddResultToMap(result, builder, expectation_map):
"""Adds a single |result| to |expectation_map|.
Args:
result: A data_types.Result object to add.
builder: A string containing the name of the builder |result| came from.
expectation_map: A dict in the format returned by
expectations.CreateTestExpectationMap(). Will be modified in-place.
Returns:
True if an expectation in |expectation_map| applied to |result|, otherwise
False.
"""
found_matching_expectation = False
# We need to use fnmatch since wildcards are supported, so there's no point in
# checking the test name key right now. The AppliesToResult check already does
# an fnmatch check.
for expectations in expectation_map.itervalues():
for e, builder_map in expectations.iteritems():
if e.AppliesToResult(result):
found_matching_expectation = True
step_map = builder_map.setdefault(builder, {})
stats = step_map.setdefault(result.step,
data_types.BuildStats())
if result.actual_result == 'Pass':
stats.AddPassedBuild()
else:
stats.AddFailedBuild(result.build_id)
return found_matching_expectation
def _GetSampleBuildStats(self):
build_stats = []
for i in xrange(8):
bs = data_types.BuildStats()
for _ in xrange(i):
bs.AddPassedBuild()
build_stats.append(bs)
return build_stats
def testEmptyBaseMap(self):
"""Tests that a merge with an empty base map copies the merge map."""
base_map = {}
merge_map = {
'foo': {
data_types.Expectation('foo', ['win'], 'Failure'): {
'builder': {
'step': data_types.BuildStats(),
},
},
},
}
original_merge_map = copy.deepcopy(merge_map)
queries._MergeExpectationMaps(base_map, merge_map)
self.assertEqual(base_map, merge_map)
self.assertEqual(merge_map, original_merge_map)
def testEmptyMergeMap(self):
"""Tests that a merge with an empty merge map is a no-op."""
base_map = {
'foo': {
data_types.Expectation('foo', ['win'], 'Failure'): {
'builder': {
'step': data_types.BuildStats(),
},
},
},
}
merge_map = {}
original_base_map = copy.deepcopy(base_map)
expectations.MergeExpectationMaps(base_map, merge_map)
self.assertEqual(base_map, original_base_map)
self.assertEqual(merge_map, {})
def testValidResults(self):
"""Tests functionality when valid results are returned by the query."""
def SideEffect(builder, *args):
del args
if builder == 'matched_builder':
return [
data_types.Result('foo', ['win'], 'Pass', 'step_name',
'build_id')
]
else:
return [
data_types.Result('bar', [], 'Pass', 'step_name',
'build_id')
]
self._query_mock.side_effect = SideEffect
expectation = data_types.Expectation('foo', ['win'], 'RetryOnFailure')
expectation_map = data_types.TestExpectationMap({
'foo':
data_types.ExpectationBuilderMap({
expectation:
data_types.BuilderStepMap(),
}),
})
unmatched_results = self._querier._FillExpectationMapForBuilders(
expectation_map, ['matched_builder', 'unmatched_builder'], 'ci')
stats = data_types.BuildStats()
stats.AddPassedBuild()
expected_expectation_map = {
'foo': {
expectation: {
'ci:matched_builder': {
'step_name': stats,
},
},
},
}
self.assertEqual(expectation_map, expected_expectation_map)
self.assertEqual(
unmatched_results, {
'ci:unmatched_builder': [
data_types.Result('bar', [], 'Pass', 'step_name',
'build_id'),
],
})
def testEmptyBaseMap(self):
"""Tests that a merge with an empty base map copies the merge map."""
base_map = data_types.TestExpectationMap()
merge_map = data_types.TestExpectationMap({
'foo':
data_types.ExpectationBuilderMap({
data_types.Expectation('foo', ['win'], 'Failure'):
data_types.BuilderStepMap({
'builder':
data_types.StepBuildStatsMap({
'step':
data_types.BuildStats(),
}),
}),
}),
})
original_merge_map = copy.deepcopy(merge_map)
expectations.MergeExpectationMaps(base_map, merge_map)
self.assertEqual(base_map, merge_map)
self.assertEqual(merge_map, original_merge_map)
def testMissingKeys(self):
"""Tests that missing keys are properly copied to the base map."""
base_map = data_types.TestExpectationMap({
'foo':
data_types.ExpectationBuilderMap({
data_types.Expectation('foo', ['win'], 'Failure'):
data_types.BuilderStepMap({
'builder':
data_types.StepBuildStatsMap({
'step':
data_types.BuildStats(),
}),
}),
}),
})
merge_map = data_types.TestExpectationMap({
'foo':
data_types.ExpectationBuilderMap({
data_types.Expectation('foo', ['win'], 'Failure'):
data_types.BuilderStepMap({
'builder':
data_types.StepBuildStatsMap({
'step2':
data_types.BuildStats(),
}),
'builder2':
data_types.StepBuildStatsMap({
'step':
data_types.BuildStats(),
}),
}),
data_types.Expectation('foo', ['mac'], 'Failure'):
data_types.BuilderStepMap({
'builder':
data_types.StepBuildStatsMap({
'step':
data_types.BuildStats(),
})
})
}),
'bar':
data_types.ExpectationBuilderMap({
data_types.Expectation('bar', ['win'], 'Failure'):
data_types.BuilderStepMap({
'builder':
data_types.StepBuildStatsMap({
'step':
data_types.BuildStats(),
}),
}),
}),
})
expected_base_map = {
'foo': {
data_types.Expectation('foo', ['win'], 'Failure'): {
'builder': {
'step': data_types.BuildStats(),
'step2': data_types.BuildStats(),
},
'builder2': {
'step': data_types.BuildStats(),
},
},
data_types.Expectation('foo', ['mac'], 'Failure'): {
'builder': {
'step': data_types.BuildStats(),
}
}
},
'bar': {
data_types.Expectation('bar', ['win'], 'Failure'): {
'builder': {
'step': data_types.BuildStats(),
},
},
},
}
expectations.MergeExpectationMaps(base_map, merge_map)
self.assertEqual(base_map, expected_base_map)
def GetFailedMapForExpectation(self, expectation):
stats = data_types.BuildStats()
stats.AddFailedBuild('build_id')
return self.GetMapForExpectationAndStats(expectation, stats)
def GetPassedMapForExpectation(self, expectation):
stats = data_types.BuildStats()
stats.AddPassedBuild()
return self.GetMapForExpectationAndStats(expectation, stats)
def CreateGenericBuildStats(self):
stats = data_types.BuildStats()
stats.AddPassedBuild()
stats.AddFailedBuild('')
return stats
