def fetch_buildbot_data(masters=None): # pragma: no cover start_time = datetime.datetime.now() all_masters_data = [] if masters: masters = [master_config.getMaster(m) for m in masters] else: masters = master_config.getAllMasters() for master_data in masters: all_masters_data.append(master_data) url_name = master_data['url_name'] master_url = MASTER_URL_TEMPLATE % url_name builders = fetch_json(master_url) if not builders: msg = 'Aborting fetch. Could not fetch builders from %s' % master_url logging.warning(msg) raise FetchBuildersException(msg) tests_object = master_data.setdefault('tests', {}) for builder in builders['builders'].keys(): build = fetch_json(BUILDS_URL_TEMPLATE % (urllib2.quote(builder), url_name)) if not build: logging.info('Skipping builder %s on master %s due to empty data.', builder, url_name) continue if not build['builds']: logging.info( 'Skipping builder %s on master %s due to empty builds list.', builder, url_name) continue for step in build['builds'][0]['steps']: step_name = step['name'] if 'test' not in step_name: continue if any(name in step_name for name in NON_TEST_STEP_NAMES): continue if re.search('_only|_ignore|_perf$', step_name): continue # This is just triggering and collecting tests on swarming, not the # actual test types. if (step_name.startswith('[trigger]') or step_name.startswith('[collect]') or step_name.startswith('[skipped]')): continue # Get the test type from the test-results uploading step name. match = GTEST_UPLOADER_STEP_REGEX.match(step_name) if match: step_name = match.group(1) # Aggressively skip all steps that dont have test in the first # word (before first space) and remove all platform cruft. This rule # is based on a manual audit of valid and invalid test types populated # in the dashboard in Q4 2015. # Note: Normalization also happens at upload time to ensure known and # actual test types match. # TODO: Remove NON_TEST_STEP_NAMES since this rule should remove all # of them already. step_name = util.normalize_test_type(step_name) if 'test' not in step_name: continue tests_object.setdefault(step_name, {'builders': set()}) tests_object[step_name]['builders'].add(builder) for builders in tests_object.values(): builders['builders'] = sorted(builders['builders']) output_data = {'masters': all_masters_data, 'no_upload_test_types': TEST_STEPS_THAT_DO_NOT_UPLOAD_YET} delta = datetime.datetime.now() - start_time logging.info('Fetched buildbot data in %s seconds.', delta.seconds) return dump_json(output_data)
def test_removes_instrumentation_test_prefix(self): self.assertEqual( 'content_shell_test_apk (with patch)', util.normalize_test_type( 'Instrumentation test content_shell_test_apk (with patch)'))
def test_ignores_with_patch_when_asked_to(self): self.assertEqual( 'base_unittests', util.normalize_test_type( 'base_unittests on Windows XP (with patch)', ignore_with_patch=True))
def test_on_platform_with_parens_with_patch_even_more_noise(self): self.assertEqual( 'base_unittests (with patch)', util.normalize_test_type( 'base_unittests (ATI GPU) on Windows (with patch) on Windows'))
def test_without_patch_is_discarded(self): self.assertEqual( 'base_unittests', util.normalize_test_type( 'base_unittests on ATI GPU on Windows (without patch) on Windows' ))
def test_on_platform(self): self.assertEqual( 'base_unittests', util.normalize_test_type('base_unittests on Windows XP'))
def test_on_platform_with_patch(self): self.assertEqual( 'base_unittests (with patch)', util.normalize_test_type( 'base_unittests on Windows XP (with patch)'))
def test_without_patch_is_discarded(self): self.assertEqual('base_unittests', util.normalize_test_type( 'base_unittests on ATI GPU on Windows (without patch) on Windows'))
def test_already_clean_name(self): self.assertEqual('base_unittests', util.normalize_test_type('base_unittests'))
def test_on_platform_with_parens_with_patch_even_more_noise(self): self.assertEqual('base_unittests (with patch)', util.normalize_test_type( 'base_unittests (ATI GPU) on Windows (with patch) on Windows'))
def test_on_platform_with_patch(self): self.assertEqual('base_unittests (with patch)', util.normalize_test_type( 'base_unittests on Windows XP (with patch)'))
def test_on_platform(self): self.assertEqual('base_unittests', util.normalize_test_type( 'base_unittests on Windows XP'))
def test_already_clean_name(self): self.assertEqual('base_unittests', util.normalize_test_type( 'base_unittests'))
def post(self): # pragma: no cover file_params = self.request.POST.getall(PARAM_FILE) if not file_params: self.response.out.write("FAIL: missing upload file field.") return builder = self.request.get(PARAM_BUILDER) if not builder: self.response.out.write("FAIL: missing builder parameter.") return master_parameter = self.request.get(PARAM_MASTER) master_data = master_config.getMasterByMasterName(master_parameter) if master_data: deprecated_master = master_parameter master = master_data['url_name'] else: deprecated_master = None master = master_parameter test_type = self.request.get(PARAM_TEST_TYPE) test_type = util.normalize_test_type(test_type) logging.debug( "Processing upload request, master: %s, builder: %s, test_type: %s.", master, builder, test_type) # There are two possible types of each file_params in the request: # one file item or a list of file items. # Normalize file_params to a file item list. files = [] logging.debug("test: %s, type:%s", file_params, type(file_params)) for item in file_params: if not isinstance(item, list) and not isinstance(item, tuple): item = [item] files.extend(item) errors = [] final_status_code = 200 for record in files: file_json = JsonResults.load_json(record.value) if record.filename == "incremental_results.json": # FIXME: Ferret out and eliminate remaining incremental_results.json # producers. logging.info(("incremental_results.json received from master: %s, " "builder: %s, test_type: %s."), master, builder, test_type) status_string, status_code = JsonResults.update(master, builder, test_type, file_json, deprecated_master=deprecated_master, is_full_results_format=False) else: try: build_number = int(file_json.get('build_number', 0)) status_string, status_code = TestFile.add_file(master, builder, test_type, build_number, record.filename, record.value) except (ValueError, TypeError): status_code = 403 status_string = ('Could not cast the build_number field in the ' 'json to an integer.') if status_code == 200: logging.info(status_string) else: logging.error(status_string) errors.append(status_string) final_status_code = status_code if status_code == 200 and record.filename == "full_results.json": status_string, status_code = JsonResults.update(master, builder, test_type, file_json, deprecated_master=deprecated_master, is_full_results_format=True) BuilderState.incremental_update(master, builder, test_type, datetime.now()) EventMonUploader.upload(master, builder, build_number, test_type, file_json) if status_code == 200: logging.info(status_string) else: logging.error(status_string) errors.append(status_string) final_status_code = status_code if errors: messages = "FAIL: " + "; ".join(errors) self.response.set_status(final_status_code, messages) self.response.out.write(messages) else: self.response.set_status(200) self.response.out.write("OK")