def testResultMatchFailingExisting(self): """Test adding a failing result when results for a builder exist.""" r = data_types.Result('some/test/case', ['win', 'win10'], 'Failure', 'pixel_tests', 'build_id') e = data_types.Expectation('some/test/*', ['win10'], 'Failure') stats = data_types.BuildStats() stats.AddPassedBuild() expectation_map = data_types.TestExpectationMap({ 'some/test/*': data_types.ExpectationBuilderMap({ e: data_types.BuilderStepMap({ 'builder': data_types.StepBuildStatsMap({ 'pixel_tests': stats, }), }), }), }) found_matching = expectations._AddResultToMap(r, 'builder', expectation_map) self.assertTrue(found_matching) stats = data_types.BuildStats() stats.AddFailedBuild('build_id') stats.AddPassedBuild() expected_expectation_map = { 'some/test/*': { e: { 'builder': { 'pixel_tests': stats, }, }, }, } self.assertEqual(expectation_map, expected_expectation_map)
def testMergeBuildStats(self): """Tests that BuildStats for the same step are merged properly.""" base_map = { 'foo': { data_types.Expectation('foo', ['win'], 'Failure'): { 'builder': { 'step': data_types.BuildStats(), }, }, }, } merge_stats = data_types.BuildStats() merge_stats.AddFailedBuild('1') merge_map = { 'foo': { data_types.Expectation('foo', ['win'], 'Failure'): { 'builder': { 'step': merge_stats, }, }, }, } expected_stats = data_types.BuildStats() expected_stats.AddFailedBuild('1') expected_base_map = { 'foo': { data_types.Expectation('foo', ['win'], 'Failure'): { 'builder': { 'step': expected_stats, }, }, }, } expectations.MergeExpectationMaps(base_map, merge_map) self.assertEqual(base_map, expected_base_map)
def testInvalidMerge(self): """Tests that updating a BuildStats instance twice is an error.""" base_map = { 'foo': { data_types.Expectation('foo', ['win'], 'Failure'): { 'builder': { 'step': data_types.BuildStats(), }, }, }, } merge_stats = data_types.BuildStats() merge_stats.AddFailedBuild('1') merge_map = { 'foo': { data_types.Expectation('foo', ['win'], 'Failure'): { 'builder': { 'step': merge_stats, }, }, }, } original_base_map = copy.deepcopy(base_map) expectations.MergeExpectationMaps(base_map, merge_map, original_base_map) with self.assertRaises(AssertionError): expectations.MergeExpectationMaps(base_map, merge_map, original_base_map)
def testMapConstructor(self): """Tests that constructors enforce type.""" # We only use one map type since they all share the same implementation for # this logic. with self.assertRaises(AssertionError): data_types.StepBuildStatsMap({1: 2}) m = data_types.StepBuildStatsMap({'step': data_types.BuildStats()}) self.assertEqual(m, {'step': data_types.BuildStats()})
def testMapSetdefault(self): """Tests that setdefault() enforces type.""" # We only use one map type since they all share the same implementation for # this logic. m = data_types.StepBuildStatsMap() with self.assertRaises(AssertionError): m.setdefault(1, data_types.BuildStats()) with self.assertRaises(AssertionError): m.setdefault('1', 2) m.setdefault('1', data_types.BuildStats()) self.assertEqual(m, {'1': data_types.BuildStats()})
def testMapUpdate(self): """Tests that update() enforces type.""" # We only use one map type since they all share the same implementation for # this logic. m = data_types.StepBuildStatsMap({'step': data_types.BuildStats()}) with self.assertRaises(AssertionError): m.update({1: 2}) with self.assertRaises(AssertionError): m.update(step2=1) m.update(step=data_types.BuildStats()) self.assertEqual(m, {'step': data_types.BuildStats()})
def testResultMatchPassingNew(self): """Test adding a passing result when no results for a builder exist.""" r = data_types.Result('some/test/case', ['win', 'win10'], 'Pass', 'pixel_tests', 'build_id') e = data_types.Expectation('some/test/*', ['win10'], 'Failure') expectation_map = { 'some/test/*': { e: {}, }, } found_matching = expectations._AddResultToMap(r, 'builder', expectation_map) self.assertTrue(found_matching) stats = data_types.BuildStats() stats.AddPassedBuild() expected_expectation_map = { 'some/test/*': { e: { 'builder': { 'pixel_tests': stats, }, }, }, } self.assertEqual(expectation_map, expected_expectation_map)
def testAddFailedBuild(self): s = data_types.BuildStats() s.AddFailedBuild('build_id') self.assertEqual(s.total_builds, 1) self.assertEqual(s.failed_builds, 1) self.assertEqual(s.failure_links, frozenset(['http://ci.chromium.org/b/build_id']))
def _AddResultToMap(result, builder, test_expectation_map): """Adds a single |result| to |test_expectation_map|. Args: result: A data_types.Result object to add. builder: A string containing the name of the builder |result| came from. test_expectation_map: A data_types.TestExpectationMap. Will be modified in-place. Returns: True if an expectation in |test_expectation_map| applied to |result|, otherwise False. """ assert isinstance(test_expectation_map, data_types.TestExpectationMap) found_matching_expectation = False # We need to use fnmatch since wildcards are supported, so there's no point in # checking the test name key right now. The AppliesToResult check already does # an fnmatch check. for _, expectation, builder_map in test_expectation_map.IterBuilderStepMaps( ): if expectation.AppliesToResult(result): found_matching_expectation = True step_map = builder_map.setdefault(builder, data_types.StepBuildStatsMap()) stats = step_map.setdefault(result.step, data_types.BuildStats()) if result.actual_result == 'Pass': stats.AddPassedBuild() else: stats.AddFailedBuild(result.build_id) return found_matching_expectation
def CreateStatsWithPassFails(passes, fails): stats = data_types.BuildStats() for _ in xrange(passes): stats.AddPassedBuild() for i in xrange(fails): stats.AddFailedBuild('build_id%d' % i) return stats
def testResultMatchMultiMatch(self): """Test adding a passing result when multiple expectations match.""" r = data_types.Result('some/test/case', ['win', 'win10'], 'Pass', 'pixel_tests', 'build_id') e = data_types.Expectation('some/test/*', ['win10'], 'Failure') e2 = data_types.Expectation('some/test/case', ['win10'], 'Failure') expectation_map = data_types.TestExpectationMap({ 'some/test/*': data_types.ExpectationBuilderMap({ e: data_types.BuilderStepMap(), e2: data_types.BuilderStepMap(), }), }) found_matching = expectations._AddResultToMap(r, 'builder', expectation_map) self.assertTrue(found_matching) stats = data_types.BuildStats() stats.AddPassedBuild() expected_expectation_map = { 'some/test/*': { e: { 'builder': { 'pixel_tests': stats, }, }, e2: { 'builder': { 'pixel_tests': stats, }, } } } self.assertEqual(expectation_map, expected_expectation_map)
def _AddResultToMap(result, builder, expectation_map): """Adds a single |result| to |expectation_map|. Args: result: A data_types.Result object to add. builder: A string containing the name of the builder |result| came from. expectation_map: A dict in the format returned by expectations.CreateTestExpectationMap(). Will be modified in-place. Returns: True if an expectation in |expectation_map| applied to |result|, otherwise False. """ found_matching_expectation = False # We need to use fnmatch since wildcards are supported, so there's no point in # checking the test name key right now. The AppliesToResult check already does # an fnmatch check. for expectations in expectation_map.itervalues(): for e, builder_map in expectations.iteritems(): if e.AppliesToResult(result): found_matching_expectation = True step_map = builder_map.setdefault(builder, {}) stats = step_map.setdefault(result.step, data_types.BuildStats()) if result.actual_result == 'Pass': stats.AddPassedBuild() else: stats.AddFailedBuild(result.build_id) return found_matching_expectation
def _GetSampleBuildStats(self): build_stats = [] for i in xrange(8): bs = data_types.BuildStats() for _ in xrange(i): bs.AddPassedBuild() build_stats.append(bs) return build_stats
def testEmptyBaseMap(self): """Tests that a merge with an empty base map copies the merge map.""" base_map = {} merge_map = { 'foo': { data_types.Expectation('foo', ['win'], 'Failure'): { 'builder': { 'step': data_types.BuildStats(), }, }, }, } original_merge_map = copy.deepcopy(merge_map) queries._MergeExpectationMaps(base_map, merge_map) self.assertEqual(base_map, merge_map) self.assertEqual(merge_map, original_merge_map)
def testEmptyMergeMap(self): """Tests that a merge with an empty merge map is a no-op.""" base_map = { 'foo': { data_types.Expectation('foo', ['win'], 'Failure'): { 'builder': { 'step': data_types.BuildStats(), }, }, }, } merge_map = {} original_base_map = copy.deepcopy(base_map) expectations.MergeExpectationMaps(base_map, merge_map) self.assertEqual(base_map, original_base_map) self.assertEqual(merge_map, {})
def testValidResults(self): """Tests functionality when valid results are returned by the query.""" def SideEffect(builder, *args): del args if builder == 'matched_builder': return [ data_types.Result('foo', ['win'], 'Pass', 'step_name', 'build_id') ] else: return [ data_types.Result('bar', [], 'Pass', 'step_name', 'build_id') ] self._query_mock.side_effect = SideEffect expectation = data_types.Expectation('foo', ['win'], 'RetryOnFailure') expectation_map = data_types.TestExpectationMap({ 'foo': data_types.ExpectationBuilderMap({ expectation: data_types.BuilderStepMap(), }), }) unmatched_results = self._querier._FillExpectationMapForBuilders( expectation_map, ['matched_builder', 'unmatched_builder'], 'ci') stats = data_types.BuildStats() stats.AddPassedBuild() expected_expectation_map = { 'foo': { expectation: { 'ci:matched_builder': { 'step_name': stats, }, }, }, } self.assertEqual(expectation_map, expected_expectation_map) self.assertEqual( unmatched_results, { 'ci:unmatched_builder': [ data_types.Result('bar', [], 'Pass', 'step_name', 'build_id'), ], })
def testEmptyBaseMap(self): """Tests that a merge with an empty base map copies the merge map.""" base_map = data_types.TestExpectationMap() merge_map = data_types.TestExpectationMap({ 'foo': data_types.ExpectationBuilderMap({ data_types.Expectation('foo', ['win'], 'Failure'): data_types.BuilderStepMap({ 'builder': data_types.StepBuildStatsMap({ 'step': data_types.BuildStats(), }), }), }), }) original_merge_map = copy.deepcopy(merge_map) expectations.MergeExpectationMaps(base_map, merge_map) self.assertEqual(base_map, merge_map) self.assertEqual(merge_map, original_merge_map)
def testMissingKeys(self): """Tests that missing keys are properly copied to the base map.""" base_map = data_types.TestExpectationMap({ 'foo': data_types.ExpectationBuilderMap({ data_types.Expectation('foo', ['win'], 'Failure'): data_types.BuilderStepMap({ 'builder': data_types.StepBuildStatsMap({ 'step': data_types.BuildStats(), }), }), }), }) merge_map = data_types.TestExpectationMap({ 'foo': data_types.ExpectationBuilderMap({ data_types.Expectation('foo', ['win'], 'Failure'): data_types.BuilderStepMap({ 'builder': data_types.StepBuildStatsMap({ 'step2': data_types.BuildStats(), }), 'builder2': data_types.StepBuildStatsMap({ 'step': data_types.BuildStats(), }), }), data_types.Expectation('foo', ['mac'], 'Failure'): data_types.BuilderStepMap({ 'builder': data_types.StepBuildStatsMap({ 'step': data_types.BuildStats(), }) }) }), 'bar': data_types.ExpectationBuilderMap({ data_types.Expectation('bar', ['win'], 'Failure'): data_types.BuilderStepMap({ 'builder': data_types.StepBuildStatsMap({ 'step': data_types.BuildStats(), }), }), }), }) expected_base_map = { 'foo': { data_types.Expectation('foo', ['win'], 'Failure'): { 'builder': { 'step': data_types.BuildStats(), 'step2': data_types.BuildStats(), }, 'builder2': { 'step': data_types.BuildStats(), }, }, data_types.Expectation('foo', ['mac'], 'Failure'): { 'builder': { 'step': data_types.BuildStats(), } } }, 'bar': { data_types.Expectation('bar', ['win'], 'Failure'): { 'builder': { 'step': data_types.BuildStats(), }, }, }, } expectations.MergeExpectationMaps(base_map, merge_map) self.assertEqual(base_map, expected_base_map)
def GetFailedMapForExpectation(self, expectation): stats = data_types.BuildStats() stats.AddFailedBuild('build_id') return self.GetMapForExpectationAndStats(expectation, stats)
def GetPassedMapForExpectation(self, expectation): stats = data_types.BuildStats() stats.AddPassedBuild() return self.GetMapForExpectationAndStats(expectation, stats)
def CreateGenericBuildStats(self): stats = data_types.BuildStats() stats.AddPassedBuild() stats.AddFailedBuild('') return stats