def test_basic_upload(self): master = master_config.getMaster('chromium.chromiumos') builder = 'test-builder' test_type = 'test-type' test_data = { 'tests': { 'Test1.testproc1': { 'expected': 'PASS', 'actual': 'FAIL', 'time': 1, } }, 'build_number': '123', 'version': JSON_RESULTS_HIERARCHICAL_VERSION, 'builder_name': builder, 'blink_revision': '12345', 'seconds_since_epoch': 1406123456, 'num_failures_by_type': { 'FAIL': 0, 'SKIP': 0, 'PASS': 1 }, 'chromium_revision': '67890', } params = collections.OrderedDict([ (testfilehandler.PARAM_BUILDER, builder), (testfilehandler.PARAM_MASTER, master['url_name']), (testfilehandler.PARAM_TEST_TYPE, test_type), ]) upload_files = [('file', 'full_results.json', json.JSONEncoder().encode(test_data))] response = self.test_app.post('/testfile/upload', params=params, upload_files=upload_files) self.assertEqual(response.status_int, 200) # test aggregated results.json got generated params = collections.OrderedDict([ (testfilehandler.PARAM_BUILDER, builder), (testfilehandler.PARAM_MASTER, master['url_name']), (testfilehandler.PARAM_TEST_TYPE, test_type), (testfilehandler.PARAM_NAME, 'results.json') ]) response = self.test_app.get('/testfile', params=params) self.assertEqual(response.status_int, 200) response_json = json.loads(response.normal_body) self.assertEqual(response_json[builder]['tests']['Test1.testproc1'], { 'results': [[1, 'Q']], 'times': [[1, 1]] }) # test testlistjson=1 params[testfilehandler.PARAM_TEST_LIST_JSON] = '1' response = self.test_app.get('/testfile', params=params) self.assertEqual(response.status_int, 200) response_json = json.loads(response.normal_body) self.assertEqual(response_json[builder]['tests']['Test1.testproc1'], {})
def test_basic_upload(self): master = master_config.getMaster('chromium.chromiumos') builder = 'test-builder' test_type = 'test-type' test_data = { 'tests': { 'Test1.testproc1': { 'expected': 'PASS', 'actual': 'FAIL', 'time': 1, } }, 'build_number': '123', 'version': JSON_RESULTS_HIERARCHICAL_VERSION, 'builder_name': builder, 'blink_revision': '12345', 'seconds_since_epoch': 1406123456, 'num_failures_by_type': { 'FAIL': 0, 'SKIP': 0, 'PASS': 1 }, 'chromium_revision': '67890', } params = collections.OrderedDict([ (testfilehandler.PARAM_BUILDER, builder), (testfilehandler.PARAM_MASTER, master['url_name']), (testfilehandler.PARAM_TEST_TYPE, test_type), ]) upload_files = [ ('file', 'full_results.json', json.JSONEncoder().encode(test_data))] response = self.test_app.post( '/testfile/upload', params=params, upload_files=upload_files) self.assertEqual(response.status_int, 200) # test aggregated results.json got generated params = collections.OrderedDict([ (testfilehandler.PARAM_BUILDER, builder), (testfilehandler.PARAM_MASTER, master['url_name']), (testfilehandler.PARAM_TEST_TYPE, test_type), (testfilehandler.PARAM_NAME, 'results.json') ]) response = self.test_app.get('/testfile', params=params) self.assertEqual(response.status_int, 200) response_json = json.loads(response.normal_body) self.assertEqual(response_json[builder]['tests']['Test1.testproc1'], {'results': [[1, 'Q']], 'times': [[1, 1]]}) # test testlistjson=1 params[testfilehandler.PARAM_TEST_LIST_JSON] = '1' response = self.test_app.get('/testfile', params=params) self.assertEqual(response.status_int, 200) response_json = json.loads(response.normal_body) self.assertEqual(response_json[builder]['tests']['Test1.testproc1'], {})
def get(self): # pragma: no cover key = self.request.get(PARAM_KEY) master = self.request.get(PARAM_MASTER) builder = self.request.get(PARAM_BUILDER) test_type = self.request.get(PARAM_TEST_TYPE) build_number = self.request.get(PARAM_BUILD_NUMBER, default_value=None) name = self.request.get(PARAM_NAME) before = self.request.get(PARAM_BEFORE) num_files = self.request.get(PARAM_NUM_FILES) test_list_json = self.request.get(PARAM_TEST_LIST_JSON) callback_name = self.request.get(PARAM_CALLBACK) logging.debug(("Getting files, master %s, builder: %s, test_type: %s, " "build_number: %s, name: %s, before: %s."), master, builder, test_type, build_number, name, before) if key: json, date = self._get_file_content_from_key(key) elif (num_files or not master or not builder or not test_type or (not build_number and not JsonResults.is_aggregate_file(name)) or not name): limit = int(num_files) if num_files else 100 self._get_file_list(master, builder, test_type, build_number, name, before, limit, callback_name) return else: # FIXME: Stop using the old master name style after all files have been # updated. master_data = master_config.getMaster(master) if not master_data: master_data = master_config.getMasterByMasterName(master) if not master_data: self.response.headers["Access-Control-Allow-Origin"] = "*" self.response.set_status(404) return json, date = self._get_file_content(master_data['url_name'], builder, test_type, build_number, name) if json is None: json, date = self._get_file_content(master_data['name'], builder, test_type, build_number, name) if json and test_list_json: json = JsonResults.get_test_list(builder, json) if json: json = _replace_jsonp_callback(json, callback_name) self._serve_json(json, date)
def get(self): # pragma: no cover key = self.request.get(PARAM_KEY) master = self.request.get(PARAM_MASTER) builder = self.request.get(PARAM_BUILDER) test_type = self.request.get(PARAM_TEST_TYPE) build_number = self.request.get(PARAM_BUILD_NUMBER, default_value=None) name = self.request.get(PARAM_NAME) before = self.request.get(PARAM_BEFORE) num_files = self.request.get(PARAM_NUM_FILES) test_list_json = self.request.get(PARAM_TEST_LIST_JSON) callback_name = self.request.get(PARAM_CALLBACK) logging.debug(("Getting files, master %s, builder: %s, test_type: %s, " "build_number: %s, name: %s, before: %s."), master, builder, test_type, build_number, name, before) if key: json, date = self._get_file_content_from_key(key) elif (num_files or not master or not builder or not test_type or (not build_number and not JsonResults.is_aggregate_file(name)) or not name): limit = int(num_files) if num_files else 100 self._get_file_list(master, builder, test_type, build_number, name, before, limit, callback_name) return else: # FIXME: Stop using the old master name style after all files have been # updated. master_data = master_config.getMaster(master) if not master_data: master_data = master_config.getMasterByMasterName(master) if not master_data: self.response.headers["Access-Control-Allow-Origin"] = "*" self.response.set_status(404) return json, date = self._get_file_content( master_data['url_name'], builder, test_type, build_number, name) if json is None: json, date = self._get_file_content( master_data['name'], builder, test_type, build_number, name) if json and test_list_json: json = JsonResults.get_test_list(builder, json) if json: json = _replace_jsonp_callback(json, callback_name) self._serve_json(json, date)
def test_get_nonexistant_results(self): master = master_config.getMaster('chromium.chromiumos') builder = 'test-builder' test_type = 'test-type' params = collections.OrderedDict([ (testfilehandler.PARAM_BUILDER, builder), (testfilehandler.PARAM_MASTER, master['url_name']), (testfilehandler.PARAM_TEST_TYPE, test_type), (testfilehandler.PARAM_NAME, 'results.json') ]) self.test_app.get('/testfile', params=params, status=404) params[testfilehandler.PARAM_TEST_LIST_JSON] = '1' self.test_app.get('/testfile', params=params, status=404)
def fetch_buildbot_data(masters=None): # pragma: no cover start_time = datetime.datetime.now() all_masters_data = [] if masters: masters = [master_config.getMaster(m) for m in masters] else: masters = master_config.getAllMasters() for master_data in masters: all_masters_data.append(master_data) url_name = master_data['url_name'] master_url = MASTER_URL_TEMPLATE % url_name builders = fetch_json(master_url) if not builders: msg = 'Aborting fetch. Could not fetch builders from %s' % master_url logging.warning(msg) raise FetchBuildersException(msg) tests_object = master_data.setdefault('tests', {}) for builder in builders['builders'].keys(): build = fetch_json(BUILDS_URL_TEMPLATE % (urllib2.quote(builder), url_name)) if not build: logging.info('Skipping builder %s on master %s due to empty data.', builder, url_name) continue if not build['builds']: logging.info( 'Skipping builder %s on master %s due to empty builds list.', builder, url_name) continue for step in build['builds'][0]['steps']: step_name = step['name'] if 'test' not in step_name: continue if any(name in step_name for name in NON_TEST_STEP_NAMES): continue if re.search('_only|_ignore|_perf$', step_name): continue # This is just triggering and collecting tests on swarming, not the # actual test types. if (step_name.startswith('[trigger]') or step_name.startswith('[collect]') or step_name.startswith('[skipped]')): continue # Get the test type from the test-results uploading step name. match = GTEST_UPLOADER_STEP_REGEX.match(step_name) if match: step_name = match.group(1) # Aggressively skip all steps that dont have test in the first # word (before first space) and remove all platform cruft. This rule # is based on a manual audit of valid and invalid test types populated # in the dashboard in Q4 2015. # Note: Normalization also happens at upload time to ensure known and # actual test types match. # TODO: Remove NON_TEST_STEP_NAMES since this rule should remove all # of them already. step_name = util.normalize_test_type(step_name) if 'test' not in step_name: continue tests_object.setdefault(step_name, {'builders': set()}) tests_object[step_name]['builders'].add(builder) for builders in tests_object.values(): builders['builders'] = sorted(builders['builders']) output_data = {'masters': all_masters_data, 'no_upload_test_types': TEST_STEPS_THAT_DO_NOT_UPLOAD_YET} delta = datetime.datetime.now() - start_time logging.info('Fetched buildbot data in %s seconds.', delta.seconds) return dump_json(output_data)
def test_deprecated_master_name(self): """Verify that a file uploaded with a deprecated master name can be downloaded by the corresponding new-style master name. """ master = master_config.getMaster('chromium.chromiumos') builder = 'test-builder' test_type = 'test-type' test_data = { 'tests': { 'Test1.testproc1': { 'expected': 'PASS', 'actual': 'PASS', 'time': 1, } }, 'build_number': '123', 'version': JSON_RESULTS_HIERARCHICAL_VERSION, 'builder_name': builder, 'blink_revision': '12345', 'seconds_since_epoch': 1406123456, 'num_failures_by_type': { 'FAIL': 0, 'SKIP': 0, 'PASS': 1 }, 'chromium_revision': '67890', } # Upload file using deprecated master name. params = collections.OrderedDict([ (testfilehandler.PARAM_BUILDER, builder), (testfilehandler.PARAM_MASTER, master['name']), (testfilehandler.PARAM_TEST_TYPE, test_type), ]) upload_files = [ ('file', 'full_results.json', json.JSONEncoder().encode(test_data))] response = self.test_app.post( '/testfile/upload', params=params, upload_files=upload_files) self.assertEqual(response.status_int, 200) # Download file using deprecated master name. params = collections.OrderedDict([ (testfilehandler.PARAM_BUILDER, builder), (testfilehandler.PARAM_MASTER, master['name']), (testfilehandler.PARAM_TEST_TYPE, test_type), (testfilehandler.PARAM_BUILD_NUMBER, '123'), (testfilehandler.PARAM_NAME, 'full_results.json') ]) response = self.test_app.get('/testfile', params=params) self.assertEqual(response.status_int, 200) response_json = json.loads(response.normal_body) self.assertEqual(response_json['chromium_revision'], '67890') # Download file using new-style name. params = collections.OrderedDict([ (testfilehandler.PARAM_BUILDER, builder), (testfilehandler.PARAM_MASTER, master['url_name']), (testfilehandler.PARAM_TEST_TYPE, test_type), (testfilehandler.PARAM_BUILD_NUMBER, '123'), (testfilehandler.PARAM_NAME, 'full_results.json') ]) response = self.test_app.get('/testfile', params=params) self.assertEqual(response.status_int, 200) response_json = json.loads(response.normal_body) self.assertEqual(response_json['chromium_revision'], '67890')
def fetch_buildbot_data(masters=None): # pragma: no cover start_time = datetime.datetime.now() all_masters_data = [] if masters: masters = [master_config.getMaster(m) for m in masters] else: masters = master_config.getAllMasters() for master_data in masters: all_masters_data.append(master_data) url_name = master_data['url_name'] master_url = MASTER_URL_TEMPLATE % url_name builders = fetch_json(master_url) if not builders: msg = 'Aborting fetch. Could not fetch builders from %s' % master_url logging.warning(msg) raise FetchBuildersException(msg) tests_object = master_data.setdefault('tests', {}) for builder in builders['builders'].keys(): build = fetch_json(BUILDS_URL_TEMPLATE % (urllib2.quote(builder), url_name)) if not build: logging.info('Skipping builder %s on master %s due to empty data.', builder, url_name) continue if not build['builds']: logging.info( 'Skipping builder %s on master %s due to empty builds list.', builder, url_name) continue for step in build['builds'][0]['steps']: step_name = step['name'] if not 'test' in step_name: continue if any(name in step_name for name in NON_TEST_STEP_NAMES): continue if re.search('_only|_ignore|_perf$', step_name): continue if step_name == 'webkit_tests': step_name = 'layout-tests' tests_object.setdefault(step_name, {'builders': []}) tests_object[step_name]['builders'].append(builder) for builders in tests_object.values(): builders['builders'].sort() output_data = {'masters': all_masters_data, 'no_upload_test_types': TEST_STEPS_THAT_DO_NOT_UPLOAD_YET} delta = datetime.datetime.now() - start_time logging.info('Fetched buildbot data in %s seconds.', delta.seconds) return dump_json(output_data)
def fetch_buildbot_data(masters=None): # pragma: no cover start_time = datetime.datetime.now() all_masters_data = [] if masters: masters = [master_config.getMaster(m) for m in masters] else: masters = master_config.getAllMasters() for master_data in masters: all_masters_data.append(master_data) url_name = master_data['url_name'] master_url = MASTER_URL_TEMPLATE % url_name builders = fetch_json(master_url) if not builders: msg = 'Aborting fetch. Could not fetch builders from %s' % master_url logging.warning(msg) raise FetchBuildersException(msg) tests_object = master_data.setdefault('tests', {}) for builder in builders['builders'].keys(): build = fetch_json(BUILDS_URL_TEMPLATE % (urllib2.quote(builder), url_name)) if not build: logging.info( 'Skipping builder %s on master %s due to empty data.', builder, url_name) continue if not build['builds']: logging.info( 'Skipping builder %s on master %s due to empty builds list.', builder, url_name) continue for step in build['builds'][0]['steps']: step_name = step['name'] if not 'test' in step_name: continue if any(name in step_name for name in NON_TEST_STEP_NAMES): continue if re.search('_only|_ignore|_perf$', step_name): continue if step_name == 'webkit_tests': step_name = 'layout-tests' tests_object.setdefault(step_name, {'builders': []}) tests_object[step_name]['builders'].append(builder) for builders in tests_object.values(): builders['builders'].sort() output_data = { 'masters': all_masters_data, 'no_upload_test_types': TEST_STEPS_THAT_DO_NOT_UPLOAD_YET } delta = datetime.datetime.now() - start_time logging.info('Fetched buildbot data in %s seconds.', delta.seconds) return dump_json(output_data)
def test_deprecated_master_name(self): """Verify that a file uploaded with a deprecated master name can be downloaded by the corresponding new-style master name. """ master = master_config.getMaster('chromium.chromiumos') builder = 'test-builder' test_type = 'test-type' test_data = { 'tests': { 'Test1.testproc1': { 'expected': 'PASS', 'actual': 'PASS', 'time': 1, } }, 'build_number': '123', 'version': JSON_RESULTS_HIERARCHICAL_VERSION, 'builder_name': builder, 'blink_revision': '12345', 'seconds_since_epoch': 1406123456, 'num_failures_by_type': { 'FAIL': 0, 'SKIP': 0, 'PASS': 1 }, 'chromium_revision': '67890', } # Upload file using deprecated master name. params = collections.OrderedDict([ (testfilehandler.PARAM_BUILDER, builder), (testfilehandler.PARAM_MASTER, master['name']), (testfilehandler.PARAM_TEST_TYPE, test_type), ]) upload_files = [('file', 'full_results.json', json.JSONEncoder().encode(test_data))] response = self.test_app.post('/testfile/upload', params=params, upload_files=upload_files) self.assertEqual(response.status_int, 200) # Download file using deprecated master name. params = collections.OrderedDict([ (testfilehandler.PARAM_BUILDER, builder), (testfilehandler.PARAM_MASTER, master['name']), (testfilehandler.PARAM_TEST_TYPE, test_type), (testfilehandler.PARAM_BUILD_NUMBER, '123'), (testfilehandler.PARAM_NAME, 'full_results.json') ]) response = self.test_app.get('/testfile', params=params) self.assertEqual(response.status_int, 200) response_json = json.loads(response.normal_body) self.assertEqual(response_json['chromium_revision'], '67890') # Download file using new-style name. params = collections.OrderedDict([ (testfilehandler.PARAM_BUILDER, builder), (testfilehandler.PARAM_MASTER, master['url_name']), (testfilehandler.PARAM_TEST_TYPE, test_type), (testfilehandler.PARAM_BUILD_NUMBER, '123'), (testfilehandler.PARAM_NAME, 'full_results.json') ]) response = self.test_app.get('/testfile', params=params) self.assertEqual(response.status_int, 200) response_json = json.loads(response.normal_body) self.assertEqual(response_json['chromium_revision'], '67890')
def test_deprecated_master_name(self): tb = testbed.Testbed() tb.activate() tb.init_datastore_v3_stub() tb.init_blobstore_stub() master = master_config.getMaster('chromium.chromiumos') builder = 'test-builder' test_type = 'test-type' test_data = [ { 'tests': { 'Test1.testproc1': { 'expected': 'PASS', 'actual': 'PASS', 'time': 1, } }, 'build_number': '123', 'version': jsonresults.JSON_RESULTS_HIERARCHICAL_VERSION, 'builder_name': builder, 'seconds_since_epoch': 1406123456, 'num_failures_by_type': { 'FAIL': 0, 'SKIP': 0, 'PASS': 1 }, 'chromium_revision': '761b2a4cbc3103ef5e48cc7e77184f57eb50f6d4', }, { 'tests': { 'Test2.testproc2': { 'expected': 'PASS', 'actual': 'FAIL', 'time': 2, } }, 'build_number': '456', 'version': jsonresults.JSON_RESULTS_HIERARCHICAL_VERSION, 'builder_name': builder, 'seconds_since_epoch': 1406654321, 'num_failures_by_type': { 'FAIL': 1, 'SKIP': 0, 'PASS': 0 }, 'chromium_revision': '761b2a4cbc3103ef5e48cc7e77184f57eb50f6d5', }, ] # Upload a file using old master name # Seed results files using the old name. JsonResults.update( master['name'], builder, test_type, test_data[0], None, True) # Update results files using the new name. JsonResults.update(master['url_name'], builder, test_type, test_data[1], master['name'], True) # Verify that the file keyed by url_name contains both sets of results. files = TestFile.get_files( master['url_name'], builder, test_type, None, None, limit=3) self.assertEqual(len(files), 2) for f in files: j = json.loads(f.data) self.assertItemsEqual(j[builder]['chromeRevision'], ['761b2a4cbc3103ef5e48cc7e77184f57eb50f6d4', '761b2a4cbc3103ef5e48cc7e77184f57eb50f6d5']) tb.deactivate()
def test_deprecated_master_name(self): tb = testbed.Testbed() tb.activate() tb.init_datastore_v3_stub() tb.init_blobstore_stub() master = master_config.getMaster('chromium.chromiumos') builder = 'test-builder' test_type = 'test-type' test_data = [ { 'tests': { 'Test1.testproc1': { 'expected': 'PASS', 'actual': 'PASS', 'time': 1, } }, 'build_number': '123', 'version': jsonresults.JSON_RESULTS_HIERARCHICAL_VERSION, 'builder_name': builder, 'blink_revision': '12345', 'seconds_since_epoch': 1406123456, 'num_failures_by_type': { 'FAIL': 0, 'SKIP': 0, 'PASS': 1 }, 'chromium_revision': '761b2a4cbc3103ef5e48cc7e77184f57eb50f6d4', }, { 'tests': { 'Test2.testproc2': { 'expected': 'PASS', 'actual': 'FAIL', 'time': 2, } }, 'build_number': '456', 'version': jsonresults.JSON_RESULTS_HIERARCHICAL_VERSION, 'builder_name': builder, 'blink_revision': '54321', 'seconds_since_epoch': 1406654321, 'num_failures_by_type': { 'FAIL': 1, 'SKIP': 0, 'PASS': 0 }, 'chromium_revision': '761b2a4cbc3103ef5e48cc7e77184f57eb50f6d5', }, ] # Upload a file using old master name # Seed results files using the old name. JsonResults.update( master['name'], builder, test_type, test_data[0], None, True) # Update results files using the new name. JsonResults.update(master['url_name'], builder, test_type, test_data[1], master['name'], True) # Verify that the file keyed by url_name contains both sets of results. files = TestFile.get_files( master['url_name'], builder, test_type, None, None, limit=3) self.assertEqual(len(files), 2) for f in files: j = json.loads(f.data) self.assertItemsEqual(j[builder]['blinkRevision'], ['12345', '54321']) tb.deactivate()