def fetch_buildbot_data(masters=None): # pragma: no cover
start_time = datetime.datetime.now()
all_masters_data = []
if masters:
masters = [master_config.getMaster(m) for m in masters]
else:
masters = master_config.getAllMasters()
for master_data in masters:
all_masters_data.append(master_data)
url_name = master_data['url_name']
master_url = MASTER_URL_TEMPLATE % url_name
builders = fetch_json(master_url)
if not builders:
msg = 'Aborting fetch. Could not fetch builders from %s' % master_url
logging.warning(msg)
raise FetchBuildersException(msg)
tests_object = master_data.setdefault('tests', {})
for builder in builders['builders'].keys():
build = fetch_json(BUILDS_URL_TEMPLATE %
(urllib2.quote(builder), url_name))
if not build:
logging.info('Skipping builder %s on master %s due to empty data.',
builder, url_name)
continue
if not build['builds']:
logging.info(
'Skipping builder %s on master %s due to empty builds list.',
builder, url_name)
continue
for step in build['builds'][0]['steps']:
step_name = step['name']
if 'test' not in step_name:
continue
if any(name in step_name for name in NON_TEST_STEP_NAMES):
continue
if re.search('_only|_ignore|_perf$', step_name):
continue
# This is just triggering and collecting tests on swarming, not the
# actual test types.
if (step_name.startswith('[trigger]') or
step_name.startswith('[collect]') or
step_name.startswith('[skipped]')):
continue
# Get the test type from the test-results uploading step name.
match = GTEST_UPLOADER_STEP_REGEX.match(step_name)
if match:
step_name = match.group(1)
# Aggressively skip all steps that dont have test in the first
# word (before first space) and remove all platform cruft. This rule
# is based on a manual audit of valid and invalid test types populated
# in the dashboard in Q4 2015.
# Note: Normalization also happens at upload time to ensure known and
# actual test types match.
# TODO: Remove NON_TEST_STEP_NAMES since this rule should remove all
# of them already.
step_name = util.normalize_test_type(step_name)
if 'test' not in step_name:
continue
tests_object.setdefault(step_name, {'builders': set()})
tests_object[step_name]['builders'].add(builder)
for builders in tests_object.values():
builders['builders'] = sorted(builders['builders'])
output_data = {'masters': all_masters_data,
'no_upload_test_types': TEST_STEPS_THAT_DO_NOT_UPLOAD_YET}
delta = datetime.datetime.now() - start_time
logging.info('Fetched buildbot data in %s seconds.', delta.seconds)
return dump_json(output_data)
def test_removes_instrumentation_test_prefix(self):
self.assertEqual(
'content_shell_test_apk (with patch)',
util.normalize_test_type(
'Instrumentation test content_shell_test_apk (with patch)'))
def test_ignores_with_patch_when_asked_to(self):
self.assertEqual(
'base_unittests',
util.normalize_test_type(
'base_unittests on Windows XP (with patch)',
ignore_with_patch=True))
def test_on_platform_with_parens_with_patch_even_more_noise(self):
self.assertEqual(
'base_unittests (with patch)',
util.normalize_test_type(
'base_unittests (ATI GPU) on Windows (with patch) on Windows'))
def test_without_patch_is_discarded(self):
self.assertEqual(
'base_unittests',
util.normalize_test_type(
'base_unittests on ATI GPU on Windows (without patch) on Windows'
))
def test_on_platform(self):
self.assertEqual(
'base_unittests',
util.normalize_test_type('base_unittests on Windows XP'))
def test_on_platform_with_patch(self):
self.assertEqual(
'base_unittests (with patch)',
util.normalize_test_type(
'base_unittests on Windows XP (with patch)'))
def test_without_patch_is_discarded(self):
self.assertEqual('base_unittests',
util.normalize_test_type(
'base_unittests on ATI GPU on Windows (without patch) on Windows'))
def test_already_clean_name(self):
self.assertEqual('base_unittests',
util.normalize_test_type('base_unittests'))
def test_on_platform_with_parens_with_patch_even_more_noise(self):
self.assertEqual('base_unittests (with patch)',
util.normalize_test_type(
'base_unittests (ATI GPU) on Windows (with patch) on Windows'))
def test_on_platform_with_patch(self):
self.assertEqual('base_unittests (with patch)',
util.normalize_test_type(
'base_unittests on Windows XP (with patch)'))
def test_on_platform(self):
self.assertEqual('base_unittests', util.normalize_test_type(
'base_unittests on Windows XP'))
def test_already_clean_name(self):
self.assertEqual('base_unittests', util.normalize_test_type(
'base_unittests'))
def post(self): # pragma: no cover
file_params = self.request.POST.getall(PARAM_FILE)
if not file_params:
self.response.out.write("FAIL: missing upload file field.")
return
builder = self.request.get(PARAM_BUILDER)
if not builder:
self.response.out.write("FAIL: missing builder parameter.")
return
master_parameter = self.request.get(PARAM_MASTER)
master_data = master_config.getMasterByMasterName(master_parameter)
if master_data:
deprecated_master = master_parameter
master = master_data['url_name']
else:
deprecated_master = None
master = master_parameter
test_type = self.request.get(PARAM_TEST_TYPE)
test_type = util.normalize_test_type(test_type)
logging.debug(
"Processing upload request, master: %s, builder: %s, test_type: %s.",
master, builder, test_type)
# There are two possible types of each file_params in the request:
# one file item or a list of file items.
# Normalize file_params to a file item list.
files = []
logging.debug("test: %s, type:%s", file_params, type(file_params))
for item in file_params:
if not isinstance(item, list) and not isinstance(item, tuple):
item = [item]
files.extend(item)
errors = []
final_status_code = 200
for record in files:
file_json = JsonResults.load_json(record.value)
if record.filename == "incremental_results.json":
# FIXME: Ferret out and eliminate remaining incremental_results.json
# producers.
logging.info(("incremental_results.json received from master: %s, "
"builder: %s, test_type: %s."),
master, builder, test_type)
status_string, status_code = JsonResults.update(master, builder,
test_type, file_json, deprecated_master=deprecated_master,
is_full_results_format=False)
else:
try:
build_number = int(file_json.get('build_number', 0))
status_string, status_code = TestFile.add_file(master, builder,
test_type, build_number, record.filename, record.value)
except (ValueError, TypeError):
status_code = 403
status_string = ('Could not cast the build_number field in the '
'json to an integer.')
if status_code == 200:
logging.info(status_string)
else:
logging.error(status_string)
errors.append(status_string)
final_status_code = status_code
if status_code == 200 and record.filename == "full_results.json":
status_string, status_code = JsonResults.update(master, builder,
test_type, file_json, deprecated_master=deprecated_master,
is_full_results_format=True)
BuilderState.incremental_update(master, builder, test_type,
datetime.now())
EventMonUploader.upload(master, builder, build_number, test_type,
file_json)
if status_code == 200:
logging.info(status_string)
else:
logging.error(status_string)
errors.append(status_string)
final_status_code = status_code
if errors:
messages = "FAIL: " + "; ".join(errors)
self.response.set_status(final_status_code, messages)
self.response.out.write(messages)
else:
self.response.set_status(200)
self.response.out.write("OK")
