def testWebGlVersion(self): """Tests that only results for the correct WebGL version are returned.""" query_results = [ { 'id': 'build-1234', 'test_id': ('ninja://chrome/test:telemetry_gpu_integration_test/' 'gpu_tests.webgl_conformance_integration_test.' 'WebGLConformanceIntegrationTest.test_name'), 'status': 'FAIL', 'typ_expectations': [ 'RetryOnFailure', ], 'typ_tags': [ 'webgl-version-1', ], 'step_name': 'step_name', }, { 'id': 'build-2345', 'test_id': ('ninja://chrome/test:telemetry_gpu_integration_test/' 'gpu_tests.webgl_conformance_integration_test.' 'WebGLConformanceIntegrationTest.test_name'), 'status': 'FAIL', 'typ_expectations': [ 'RetryOnFailure', ], 'typ_tags': [ 'webgl-version-2', ], 'step_name': 'step_name', }, ] querier = unittest_utils.CreateGenericQuerier( suite='webgl_conformance1') self._popen_mock.return_value = unittest_utils.FakeProcess( stdout=json.dumps(query_results)) results = querier.QueryBuilder('builder', 'ci') self.assertEqual(len(results), 1) self.assertEqual( results[0], data_types.Result('test_name', ['webgl-version-1'], 'Failure', 'step_name', '1234')) querier = unittest_utils.CreateGenericQuerier( suite='webgl_conformance2') results = querier.QueryBuilder('builder', 'ci') self.assertEqual(len(results), 1) self.assertEqual( results[0], data_types.Result('test_name', ['webgl-version-2'], 'Failure', 'step_name', '2345'))
def SideEffect(builder, *args): del args if builder == 'matched_builder': return [ data_types.Result('foo', ['win'], 'Pass', 'step_name', 'build_id') ] else: return [data_types.Result('bar', [], 'Pass', 'step_name', 'build_id')]
def testRetryFailThenPassMatching(self): """Tests when there are pass and fail results for retry expectations.""" foo_fail_result = data_types.Result('foo/test', ['win10'], 'Failure', 'pixel_tests', 'build_id') foo_pass_result = data_types.Result('foo/test', ['win10'], 'Pass', 'pixel_tests', 'build_id') expectation_map = self.GetEmptyMapForGenericRetryExpectation() unmatched_results = expectations.AddResultListToMap( expectation_map, 'builder', [foo_fail_result, foo_pass_result]) self.assertEqual(unmatched_results, []) expected_expectation_map = self.GetFailedMapForExpectation( self.GetGenericRetryExpectation()) self.assertEqual(expectation_map, expected_expectation_map)
def testResultMatchFailingExisting(self): """Test adding a failing result when results for a builder exist.""" r = data_types.Result('some/test/case', ['win', 'win10'], 'Failure', 'pixel_tests', 'build_id') e = data_types.Expectation('some/test/*', ['win10'], 'Failure') stats = data_types.BuildStats() stats.AddPassedBuild() expectation_map = data_types.TestExpectationMap({ 'some/test/*': data_types.ExpectationBuilderMap({ e: data_types.BuilderStepMap({ 'builder': data_types.StepBuildStatsMap({ 'pixel_tests': stats, }), }), }), }) found_matching = expectations._AddResultToMap(r, 'builder', expectation_map) self.assertTrue(found_matching) stats = data_types.BuildStats() stats.AddFailedBuild('build_id') stats.AddPassedBuild() expected_expectation_map = { 'some/test/*': { e: { 'builder': { 'pixel_tests': stats, }, }, }, } self.assertEqual(expectation_map, expected_expectation_map)
def testResultMatchMultiMatch(self): """Test adding a passing result when multiple expectations match.""" r = data_types.Result('some/test/case', ['win', 'win10'], 'Pass', 'pixel_tests', 'build_id') e = data_types.Expectation('some/test/*', ['win10'], 'Failure') e2 = data_types.Expectation('some/test/case', ['win10'], 'Failure') expectation_map = data_types.TestExpectationMap({ 'some/test/*': data_types.ExpectationBuilderMap({ e: data_types.BuilderStepMap(), e2: data_types.BuilderStepMap(), }), }) found_matching = expectations._AddResultToMap(r, 'builder', expectation_map) self.assertTrue(found_matching) stats = data_types.BuildStats() stats.AddPassedBuild() expected_expectation_map = { 'some/test/*': { e: { 'builder': { 'pixel_tests': stats, }, }, e2: { 'builder': { 'pixel_tests': stats, }, } } } self.assertEqual(expectation_map, expected_expectation_map)
def testValidResults(self): """Tests functionality when valid results are returned.""" query_results = [ { 'id': 'build-1234', 'test_id': ('ninja://chrome/test:telemetry_gpu_integration_test/' 'gpu_tests.pixel_integration_test.' 'PixelIntegrationTest.test_name'), 'status': 'FAIL', 'typ_expectations': [ 'RetryOnFailure', ], 'typ_tags': [ 'win', 'intel', ], 'step_name': 'step_name', }, ] self._popen_mock.return_value = unittest_utils.FakeProcess( stdout=json.dumps(query_results)) results = self._querier.QueryBuilder('builder', 'ci') self.assertEqual(len(results), 1) self.assertEqual( results[0], data_types.Result('test_name', ['win', 'intel'], 'Failure', 'step_name', '1234'))
def testOutputResultsSmoketest(self): """Test that nothing blows up when outputting.""" expectation_map = { 'foo': { data_types.Expectation('foo', ['win', 'intel'], 'RetryOnFailure'): { 'stale': { 'all_pass': uu.CreateStatsWithPassFails(2, 0), }, }, data_types.Expectation('foo', ['linux'], 'Failure'): { 'semi_stale': { 'all_pass': uu.CreateStatsWithPassFails(2, 0), 'some_pass': uu.CreateStatsWithPassFails(1, 1), 'none_pass': uu.CreateStatsWithPassFails(0, 2), }, }, data_types.Expectation('foo', ['mac'], 'Failure'): { 'active': { 'none_pass': uu.CreateStatsWithPassFails(0, 2), }, }, }, } unmatched_results = { 'builder': [ data_types.Result('foo', ['win', 'intel'], 'Failure', 'step_name', 'build_id'), ], } unmatched_expectations = [ data_types.Expectation('foo', ['linux'], 'RetryOnFailure') ] stale, semi_stale, active = expectations.SplitExpectationsByStaleness( expectation_map) result_output.OutputResults(stale, semi_stale, active, {}, [], 'print', self._file_handle) result_output.OutputResults(stale, semi_stale, active, unmatched_results, [], 'print', self._file_handle) result_output.OutputResults(stale, semi_stale, active, {}, unmatched_expectations, 'print', self._file_handle) result_output.OutputResults(stale, semi_stale, active, unmatched_results, unmatched_expectations, 'print', self._file_handle) result_output.OutputResults(stale, semi_stale, active, {}, [], 'html', self._file_handle) result_output.OutputResults(stale, semi_stale, active, unmatched_results, [], 'html', self._file_handle) result_output.OutputResults(stale, semi_stale, active, {}, unmatched_expectations, 'html', self._file_handle) result_output.OutputResults(stale, semi_stale, active, unmatched_results, unmatched_expectations, 'html', self._file_handle)
def testResultMatchPassingNew(self): """Test adding a passing result when no results for a builder exist.""" r = data_types.Result('some/test/case', ['win', 'win10'], 'Pass', 'pixel_tests', 'build_id') e = data_types.Expectation('some/test/*', ['win10'], 'Failure') expectation_map = { 'some/test/*': { e: {}, }, } found_matching = expectations._AddResultToMap(r, 'builder', expectation_map) self.assertTrue(found_matching) stats = data_types.BuildStats() stats.AddPassedBuild() expected_expectation_map = { 'some/test/*': { e: { 'builder': { 'pixel_tests': stats, }, }, }, } self.assertEqual(expectation_map, expected_expectation_map)
def QueryBuilder(self, builder, builder_type): """Queries ResultDB for results from |builder|. Args: builder: A string containing the name of the builder to query. builder_type: A string containing the type of builder to query, either "ci" or "try". Returns: The results returned by the query converted into a list of data_types.Resultobjects. """ test_filter_clause = self._GetTestFilterClauseForBuilder( builder, builder_type) if test_filter_clause is None: # No affected tests on this builder, so early return. return [] query = GPU_BQ_QUERY_TEMPLATE.format( builder_type=builder_type, test_filter_clause=test_filter_clause, suite=self._suite) query_results = self._RunBigQueryCommandForJsonOutput( query, { '': { 'builder_name': builder }, 'INT64': { 'num_builds': self._num_samples } }) results = [] if not query_results: # Don't bother logging if we know this is a fake CI builder. if not (builder_type == 'ci' and builder in builders_module.FAKE_CI_BUILDERS): logging.warning( 'Did not get results for "%s", but this may be because its results ' 'do not apply to any expectations for this suite.', builder) return results for r in query_results: if not self._check_webgl_version(r['typ_tags']): continue build_id = _StripPrefixFromBuildId(r['id']) test_name = _StripPrefixFromTestId(r['test_id']) actual_result = _ConvertActualResultToExpectationFileFormat( r['status']) tags = r['typ_tags'] step = r['step_name'] results.append( data_types.Result(test_name, tags, actual_result, step, build_id)) logging.debug('Got %d results for %s builder %s', len(results), builder_type, builder) return results
def testEquality(self): r = GENERIC_RESULT other = data_types.Result('test', ['tag1', 'tag2'], 'Pass', 'pixel_tests', 'build_id') self.assertEqual(r, other) other = data_types.Result('test2', ['tag1', 'tag2'], 'Pass', 'pixel_tests', 'build_id') self.assertNotEqual(r, other) other = data_types.Result('test', ['tag1'], 'Pass', 'pixel_tests', 'build_id') self.assertNotEqual(r, other) other = data_types.Result('test', ['tag1', 'tag2'], 'Pass', 'pixel_tests', 'build_id') self.assertNotEqual(r, other) other = data_types.Result('test', ['tag1', 'tag2'], 'Failure', 'pixel_tests', 'build_id') self.assertNotEqual(r, other) other = data_types.Result('test', ['tag1', 'tag2'], 'Pass', 'webgl_tests', 'build_id') self.assertNotEqual(r, other) other = data_types.Result('test', ['tag1', 'tag2'], 'Pass', 'pixel_tests', 'other_build_id') self.assertNotEqual(r, other) other = data_types.Expectation('test', ['tag1', 'tag2'], 'Pass') self.assertNotEqual(r, other)
def testAppliesToResultApplies(self): r = data_types.Result('test', ['tag1', 'tag2'], 'Pass', 'pixel_tests', 'build_id') # Exact name match, exact tag match. e = GENERIC_EXPECTATION self.assertTrue(e.AppliesToResult(r)) # Glob name match, exact tag match. e = data_types.Expectation('te*', ['tag1', 'tag2'], 'Pass') self.assertTrue(e.AppliesToResult(r)) # Exact name match, tag subset match. e = data_types.Expectation('test', ['tag1'], 'Pass') self.assertTrue(e.AppliesToResult(r)) # Expected result subset match. r = data_types.Result('test', ['tag1', 'tag2'], 'Pass', 'pixel_tests', 'build_id') e = GENERIC_EXPECTATION self.assertTrue(e.AppliesToResult(r)) e = data_types.Expectation('test', ['tag1', 'tag2'], ['RetryOnFailure']) self.assertTrue(e.AppliesToResult(r))
def testResultNoMatch(self): """Tests that a result is not added if no match is found.""" r = data_types.Result('some/test/case', ['win', 'win10'], 'Failure', 'pixel_tests', 'build_id') e = data_types.Expectation('some/test/*', ['win10', 'foo'], 'Failure') expectation_map = {'some/test/*': {e: {}}} found_matching = queries._AddResultToMap(r, 'builder', expectation_map) self.assertFalse(found_matching) expected_expectation_map = {'some/test/*': {e: {}}} self.assertEqual(expectation_map, expected_expectation_map)
def testMismatches(self): """Tests that unmatched results get returned.""" foo_match_result = data_types.Result('foo/test', ['win10'], 'Pass', 'pixel_tests', 'build_id') foo_mismatch_result = data_types.Result('foo/not_a_test', ['win10'], 'Failure', 'pixel_tests', 'build_id') bar_result = data_types.Result('bar/test', ['win10'], 'Pass', 'pixel_tests', 'build_id') expectation_map = self.GetEmptyMapForGenericFailureExpectation() unmatched_results = expectations.AddResultListToMap( expectation_map, 'builder', [foo_match_result, foo_mismatch_result, bar_result]) self.assertEqual(len(set(unmatched_results)), 2) self.assertEqual(set(unmatched_results), set([foo_mismatch_result, bar_result])) expected_expectation_map = self.GetPassedMapForExpectation( self.GetGenericFailureExpectation()) self.assertEqual(expectation_map, expected_expectation_map)
def testAppliesToResultDoesNotApply(self): r = data_types.Result('test', ['tag1', 'tag2'], 'Pass', 'pixel_tests', 'build_id') # Exact name mismatch. e = data_types.Expectation('te', ['tag1', 'tag2'], 'Pass') self.assertFalse(e.AppliesToResult(r)) # Glob name mismatch. e = data_types.Expectation('ta*', ['tag1', 'tag2'], 'Pass') self.assertFalse(e.AppliesToResult(r)) # Tags subset mismatch. e = data_types.Expectation('test', ['tag3'], 'Pass') self.assertFalse(e.AppliesToResult(r))
def testFailureFailureMatching(self): """Tests when there are failure results for failure expectations.""" foo_result = data_types.Result('foo/test', ['win10'], 'Failure', 'pixel_tests', 'build_id') expectation_map = self.GetEmptyMapForGenericFailureExpectation() unmatched_results = expectations.AddResultListToMap( expectation_map, 'builder', [foo_result]) self.assertEqual(unmatched_results, []) expected_expectation_map = self.GetFailedMapForExpectation( self.GetGenericFailureExpectation()) self.assertEqual(expectation_map, expected_expectation_map)
def testRetryOnlyFailMatching(self): """Tests when the only tests are retry expectations that fail and match.""" foo_result = data_types.Result('foo/test', ['win10'], 'Failure', 'pixel_tests', 'build_id') expectation_map = self.GetEmptyMapForGenericRetryExpectation() unmatched_results = expectations.AddResultListToMap( expectation_map, 'builder', [foo_result]) self.assertEqual(unmatched_results, []) expected_expectation_map = self.GetFailedMapForExpectation( self.GetGenericRetryExpectation()) self.assertEqual(expectation_map, expected_expectation_map)
def testEquality(self): e = GENERIC_EXPECTATION other = data_types.Expectation('test', ['tag1', 'tag2'], 'Pass') self.assertEqual(e, other) other = data_types.Expectation('test2', ['tag1', 'tag2'], 'Pass') self.assertNotEqual(e, other) other = data_types.Expectation('test', ['tag1'], 'Pass') self.assertNotEqual(e, other) other = data_types.Expectation('test', ['tag1', 'tag2'], 'Failure') self.assertNotEqual(e, other) other = data_types.Result('test', ['tag1', 'tag2'], 'Pass', 'pixel_tests', 'build_id') self.assertNotEqual(e, other)
def testValidResults(self): """Tests functionality when valid results are returned by the query.""" def SideEffect(builder, *args): del args if builder == 'matched_builder': return [ data_types.Result('foo', ['win'], 'Pass', 'step_name', 'build_id') ] else: return [ data_types.Result('bar', [], 'Pass', 'step_name', 'build_id') ] self._query_mock.side_effect = SideEffect expectation = data_types.Expectation('foo', ['win'], 'RetryOnFailure') expectation_map = data_types.TestExpectationMap({ 'foo': data_types.ExpectationBuilderMap({ expectation: data_types.BuilderStepMap(), }), }) unmatched_results = self._querier._FillExpectationMapForBuilders( expectation_map, ['matched_builder', 'unmatched_builder'], 'ci') stats = data_types.BuildStats() stats.AddPassedBuild() expected_expectation_map = { 'foo': { expectation: { 'ci:matched_builder': { 'step_name': stats, }, }, }, } self.assertEqual(expectation_map, expected_expectation_map) self.assertEqual( unmatched_results, { 'ci:unmatched_builder': [ data_types.Result('bar', [], 'Pass', 'step_name', 'build_id'), ], })
def testMinimalData(self): """Tests that everything functions when minimal data is provided.""" unmatched_results = { 'builder': [ data_types.Result('foo', [], 'Failure', None, 'build_id'), ], } expected_output = { 'foo': { 'builder': { None: [ 'Got "Failure" on http://ci.chromium.org/b/build_id with ' 'tags []', ], }, }, } output = result_output._ConvertUnmatchedResultsToStringDict( unmatched_results) self.assertEqual(output, expected_output)
def testRegularData(self): """Tests that everything functions when regular data is provided.""" unmatched_results = { 'builder': [ data_types.Result('foo', ['win', 'intel'], 'Failure', 'step_name', 'build_id') ], } expected_output = { 'foo': { 'builder': { 'step_name': [ 'Got "Failure" on http://ci.chromium.org/b/build_id with ' 'tags [win intel]', ] } } } output = result_output._ConvertUnmatchedResultsToStringDict( unmatched_results) self.assertEqual(output, expected_output)
def AddResultListToMap(expectation_map, builder, results): """Adds |results| to |expectation_map|. Args: expectation_map: A data_types.TestExpectationMap. Will be modified in-place. builder: A string containing the builder |results| came from. Should be prefixed with something to distinguish between identically named CI and try builders. results: A list of data_types.Result objects corresponding to the ResultDB data queried for |builder|. Returns: A list of data_types.Result objects who did not have a matching expectation in |expectation_map|. """ assert isinstance(expectation_map, data_types.TestExpectationMap) failure_results = set() pass_results = set() unmatched_results = [] for r in results: if r.actual_result == 'Pass': pass_results.add(r) else: failure_results.add(r) # Remove any cases of failure -> pass from the passing set. If a test is # flaky, we get both pass and failure results for it, so we need to remove the # any cases of a pass result having a corresponding, earlier failure result. modified_failing_retry_results = set() for r in failure_results: modified_failing_retry_results.add( data_types.Result(r.test, r.tags, 'Pass', r.step, r.build_id)) pass_results -= modified_failing_retry_results for r in pass_results | failure_results: found_matching = _AddResultToMap(r, builder, expectation_map) if not found_matching: unmatched_results.append(r) return unmatched_results
# Copyright 2020 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. import unittest import mock from unexpected_passes import data_types GENERIC_EXPECTATION = data_types.Expectation('test', ['tag1', 'tag2'], ['Pass']) GENERIC_RESULT = data_types.Result('test', ['tag1', 'tag2'], 'Pass', 'pixel_tests', 'build_id') class ExpectationUnittest(unittest.TestCase): def testEquality(self): e = GENERIC_EXPECTATION other = data_types.Expectation('test', ['tag1', 'tag2'], 'Pass') self.assertEqual(e, other) other = data_types.Expectation('test2', ['tag1', 'tag2'], 'Pass') self.assertNotEqual(e, other) other = data_types.Expectation('test', ['tag1'], 'Pass') self.assertNotEqual(e, other) other = data_types.Expectation('test', ['tag1', 'tag2'], 'Failure') self.assertNotEqual(e, other) other = data_types.Result('test', ['tag1', 'tag2'], 'Pass', 'pixel_tests', 'build_id') self.assertNotEqual(e, other)
def QueryBuilder(self, builder, builder_type): """Queries ResultDB for results from |builder|. Args: builder: A string containing the name of the builder to query. builder_type: A string containing the type of builder to query, either "ci" or "try". Returns: The results returned by the query converted into a list of data_types.Resultobjects. """ test_filter = self._GetTestFilterForBuilder(builder, builder_type) if not test_filter: # No affected tests on this builder, so early return. return [] # Query for the test data from the builder, splitting the query if we run # into the BigQuery hard memory limit. Even if we keep failing, this will # eventually stop due to getting a QuerySplitError when we can't split the # query any further. query_results = None while query_results is None: try: queries = [] for tfc in test_filter.GetClauses(): query = GPU_BQ_QUERY_TEMPLATE.format( builder_type=builder_type, test_filter_clause=tfc, suite=self._suite) queries.append(query) query_results = self._RunBigQueryCommandsForJsonOutput( queries, { '': { 'builder_name': builder }, 'INT64': { 'num_builds': self._num_samples } }) except MemoryLimitError: logging.warning( 'Query to builder %s hit BigQuery hard memory limit, trying again ' 'with more query splitting.', builder) test_filter.SplitFilter() results = [] if not query_results: # Don't bother logging if we know this is a fake CI builder. if not (builder_type == 'ci' and builder in builders_module.FAKE_CI_BUILDERS): logging.warning( 'Did not get results for "%s", but this may be because its ' 'results do not apply to any expectations for this suite.', builder) return results for r in query_results: if not self._check_webgl_version(r['typ_tags']): continue build_id = _StripPrefixFromBuildId(r['id']) test_name = _StripPrefixFromTestId(r['test_id']) actual_result = _ConvertActualResultToExpectationFileFormat( r['status']) tags = r['typ_tags'] step = r['step_name'] results.append( data_types.Result(test_name, tags, actual_result, step, build_id)) logging.debug('Got %d results for %s builder %s', len(results), builder_type, builder) return results
def testWildcardsDisallowed(self): with self.assertRaises(AssertionError): data_types.Result('*', ['tag1'], 'Pass', 'pixel_tests', 'build_id')
def QueryBuilder(builder, builder_type, suite, project, num_samples): """Queries ResultDB for results from |builder|. Args: builder: A string containing the name of the builder to query. builder_type: A string containing the type of builder to query, either "ci" or "try". suite: A string containing the name of the suite that is being queried for. project: A string containing the billing project to use for BigQuery. num_samples: An integer containing the number of builds to pull results from. Returns: A tuple (builder, results). |builder| is simply the value of the input |builder| argument, returned to facilitate parallel execution. |results| is the results returned by the query converted into a list of data_types.Result objects. """ num_samples = num_samples or DEFAULT_NUM_SAMPLES assert num_samples > 0 # WebGL 1 and 2 tests are technically the same suite, but have different # expectation files. This leads to us getting both WebGL 1 and 2 results when # we only have expectations for one of them, which causes all the results from # the other to be reported as not having a matching expectation. # TODO(crbug.com/1140283): Remove this once WebGL expectations are merged # and there's no need to differentiate them. if 'webgl_conformance' in suite: webgl_version = suite[-1] suite = 'webgl_conformance' check_webgl_version =\ lambda tags: 'webgl-version-%s' % webgl_version in tags else: check_webgl_version = lambda tags: True # Most test names are |suite|_integration_test, but there are several that # are not reported that way in typ, and by extension ResultDB, so adjust that # here. suite = TELEMETRY_SUITE_TO_RDB_SUITE_EXCEPTION_MAP.get( suite, suite + '_integration_test') query = GPU_BQ_QUERY_TEMPLATE.format(builder_type=builder_type, suite=suite) cmd = [ 'bq', 'query', '--max_rows=%d' % MAX_ROWS, '--format=json', '--project_id=%s' % project, '--use_legacy_sql=false', '--parameter=builder_name::%s' % builder, '--parameter=num_builds:INT64:%d' % num_samples, query, ] with open(os.devnull, 'w') as devnull: try: stdout = subprocess.check_output(cmd, stderr=devnull) except subprocess.CalledProcessError as e: logging.error(e.output) raise query_results = json.loads(stdout) results = [] if not query_results: # Don't bother logging if we know this is a fake CI builder. if not (builder_type == 'ci' and builder in builders_module.FAKE_CI_BUILDERS): logging.warning( 'Did not get results for "%s", but this may be because its results ' 'do not apply to any expectations for this suite.', builder) return (builder, results) for r in query_results: if not check_webgl_version(r['typ_tags']): continue build_id = _StripPrefixFromBuildId(r['id']) test_name = _StripPrefixFromTestId(r['test_id']) actual_result = _ConvertActualResultToExpectationFileFormat( r['status']) tags = r['typ_tags'] step = r['step_name'] results.append( data_types.Result(test_name, tags, actual_result, step, build_id)) logging.debug('Got %d results for %s builder %s', len(results), builder_type, builder) return (builder, results)
def QueryBuilder(builder, builder_type, suite, project, num_samples, large_query_mode): """Queries ResultDB for results from |builder|. Args: builder: A string containing the name of the builder to query. builder_type: A string containing the type of builder to query, either "ci" or "try". suite: A string containing the name of the suite that is being queried for. project: A string containing the billing project to use for BigQuery. num_samples: An integer containing the number of builds to pull results from. large_query_mode: A boolean indicating whether large query mode should be used. In this mode, an initial, smaller query is made and its results are used to perform additional filtering on a second, larger query in BigQuery. This works around hitting a hard memory limit when running the ORDER BY clause. Returns: A tuple (builder, results). |builder| is simply the value of the input |builder| argument, returned to facilitate parallel execution. |results| is the results returned by the query converted into a list of data_types.Result objects. """ num_samples = num_samples or DEFAULT_NUM_SAMPLES assert num_samples > 0 # WebGL 1 and 2 tests are technically the same suite, but have different # expectation files. This leads to us getting both WebGL 1 and 2 results when # we only have expectations for one of them, which causes all the results from # the other to be reported as not having a matching expectation. # TODO(crbug.com/1140283): Remove this once WebGL expectations are merged # and there's no need to differentiate them. if 'webgl_conformance' in suite: webgl_version = suite[-1] suite = 'webgl_conformance' check_webgl_version =\ lambda tags: 'webgl-version-%s' % webgl_version in tags else: check_webgl_version = lambda tags: True # Most test names are |suite|_integration_test, but there are several that # are not reported that way in typ, and by extension ResultDB, so adjust that # here. suite = TELEMETRY_SUITE_TO_RDB_SUITE_EXCEPTION_MAP.get( suite, suite + '_integration_test') test_filter_clause = _GetTestFilterClauseForBuilder( builder, builder_type, suite, project, large_query_mode) if test_filter_clause is None: # No affected tests on this builder, so early return. return (builder, []) query = GPU_BQ_QUERY_TEMPLATE.format(builder_type=builder_type, test_filter_clause=test_filter_clause, suite=suite) query_results = _RunBigQueryCommandForJsonOutput(query, project, { '': { 'builder_name': builder }, 'INT64': { 'num_builds': num_samples } }) results = [] if not query_results: # Don't bother logging if we know this is a fake CI builder. if not (builder_type == 'ci' and builder in builders_module.FAKE_CI_BUILDERS): logging.warning( 'Did not get results for "%s", but this may be because its results ' 'do not apply to any expectations for this suite.', builder) return (builder, results) for r in query_results: if not check_webgl_version(r['typ_tags']): continue build_id = _StripPrefixFromBuildId(r['id']) test_name = _StripPrefixFromTestId(r['test_id']) actual_result = _ConvertActualResultToExpectationFileFormat( r['status']) tags = r['typ_tags'] step = r['step_name'] results.append( data_types.Result(test_name, tags, actual_result, step, build_id)) logging.debug('Got %d results for %s builder %s', len(results), builder_type, builder) return (builder, results)