def test_basic_upload(self):
master = master_config.getMaster('chromium.chromiumos')
builder = 'test-builder'
test_type = 'test-type'
test_data = {
'tests': {
'Test1.testproc1': {
'expected': 'PASS',
'actual': 'FAIL',
'time': 1,
}
},
'build_number': '123',
'version': JSON_RESULTS_HIERARCHICAL_VERSION,
'builder_name': builder,
'blink_revision': '12345',
'seconds_since_epoch': 1406123456,
'num_failures_by_type': {
'FAIL': 0,
'SKIP': 0,
'PASS': 1
},
'chromium_revision': '67890',
}
params = collections.OrderedDict([
(testfilehandler.PARAM_BUILDER, builder),
(testfilehandler.PARAM_MASTER, master['url_name']),
(testfilehandler.PARAM_TEST_TYPE, test_type),
])
upload_files = [('file', 'full_results.json',
json.JSONEncoder().encode(test_data))]
response = self.test_app.post('/testfile/upload',
params=params,
upload_files=upload_files)
self.assertEqual(response.status_int, 200)
# test aggregated results.json got generated
params = collections.OrderedDict([
(testfilehandler.PARAM_BUILDER, builder),
(testfilehandler.PARAM_MASTER, master['url_name']),
(testfilehandler.PARAM_TEST_TYPE, test_type),
(testfilehandler.PARAM_NAME, 'results.json')
])
response = self.test_app.get('/testfile', params=params)
self.assertEqual(response.status_int, 200)
response_json = json.loads(response.normal_body)
self.assertEqual(response_json[builder]['tests']['Test1.testproc1'], {
'results': [[1, 'Q']],
'times': [[1, 1]]
})
# test testlistjson=1
params[testfilehandler.PARAM_TEST_LIST_JSON] = '1'
response = self.test_app.get('/testfile', params=params)
self.assertEqual(response.status_int, 200)
response_json = json.loads(response.normal_body)
self.assertEqual(response_json[builder]['tests']['Test1.testproc1'],
{})
def test_basic_upload(self):
master = master_config.getMaster('chromium.chromiumos')
builder = 'test-builder'
test_type = 'test-type'
test_data = {
'tests': {
'Test1.testproc1': {
'expected': 'PASS',
'actual': 'FAIL',
'time': 1,
}
},
'build_number': '123',
'version': JSON_RESULTS_HIERARCHICAL_VERSION,
'builder_name': builder,
'blink_revision': '12345',
'seconds_since_epoch': 1406123456,
'num_failures_by_type': {
'FAIL': 0,
'SKIP': 0,
'PASS': 1
},
'chromium_revision': '67890',
}
params = collections.OrderedDict([
(testfilehandler.PARAM_BUILDER, builder),
(testfilehandler.PARAM_MASTER, master['url_name']),
(testfilehandler.PARAM_TEST_TYPE, test_type),
])
upload_files = [
('file', 'full_results.json', json.JSONEncoder().encode(test_data))]
response = self.test_app.post(
'/testfile/upload', params=params, upload_files=upload_files)
self.assertEqual(response.status_int, 200)
# test aggregated results.json got generated
params = collections.OrderedDict([
(testfilehandler.PARAM_BUILDER, builder),
(testfilehandler.PARAM_MASTER, master['url_name']),
(testfilehandler.PARAM_TEST_TYPE, test_type),
(testfilehandler.PARAM_NAME, 'results.json')
])
response = self.test_app.get('/testfile', params=params)
self.assertEqual(response.status_int, 200)
response_json = json.loads(response.normal_body)
self.assertEqual(response_json[builder]['tests']['Test1.testproc1'],
{'results': [[1, 'Q']], 'times': [[1, 1]]})
# test testlistjson=1
params[testfilehandler.PARAM_TEST_LIST_JSON] = '1'
response = self.test_app.get('/testfile', params=params)
self.assertEqual(response.status_int, 200)
response_json = json.loads(response.normal_body)
self.assertEqual(response_json[builder]['tests']['Test1.testproc1'], {})
def get(self): # pragma: no cover
key = self.request.get(PARAM_KEY)
master = self.request.get(PARAM_MASTER)
builder = self.request.get(PARAM_BUILDER)
test_type = self.request.get(PARAM_TEST_TYPE)
build_number = self.request.get(PARAM_BUILD_NUMBER, default_value=None)
name = self.request.get(PARAM_NAME)
before = self.request.get(PARAM_BEFORE)
num_files = self.request.get(PARAM_NUM_FILES)
test_list_json = self.request.get(PARAM_TEST_LIST_JSON)
callback_name = self.request.get(PARAM_CALLBACK)
logging.debug(("Getting files, master %s, builder: %s, test_type: %s, "
"build_number: %s, name: %s, before: %s."), master,
builder, test_type, build_number, name, before)
if key:
json, date = self._get_file_content_from_key(key)
elif (num_files or not master or not builder or not test_type
or (not build_number and not JsonResults.is_aggregate_file(name))
or not name):
limit = int(num_files) if num_files else 100
self._get_file_list(master, builder, test_type, build_number, name,
before, limit, callback_name)
return
else:
# FIXME: Stop using the old master name style after all files have been
# updated.
master_data = master_config.getMaster(master)
if not master_data:
master_data = master_config.getMasterByMasterName(master)
if not master_data:
self.response.headers["Access-Control-Allow-Origin"] = "*"
self.response.set_status(404)
return
json, date = self._get_file_content(master_data['url_name'],
builder, test_type,
build_number, name)
if json is None:
json, date = self._get_file_content(master_data['name'],
builder, test_type,
build_number, name)
if json and test_list_json:
json = JsonResults.get_test_list(builder, json)
if json:
json = _replace_jsonp_callback(json, callback_name)
self._serve_json(json, date)
def get(self): # pragma: no cover
key = self.request.get(PARAM_KEY)
master = self.request.get(PARAM_MASTER)
builder = self.request.get(PARAM_BUILDER)
test_type = self.request.get(PARAM_TEST_TYPE)
build_number = self.request.get(PARAM_BUILD_NUMBER, default_value=None)
name = self.request.get(PARAM_NAME)
before = self.request.get(PARAM_BEFORE)
num_files = self.request.get(PARAM_NUM_FILES)
test_list_json = self.request.get(PARAM_TEST_LIST_JSON)
callback_name = self.request.get(PARAM_CALLBACK)
logging.debug(("Getting files, master %s, builder: %s, test_type: %s, "
"build_number: %s, name: %s, before: %s."),
master, builder, test_type, build_number, name, before)
if key:
json, date = self._get_file_content_from_key(key)
elif (num_files or not master or not builder or not test_type
or (not build_number and not JsonResults.is_aggregate_file(name))
or not name):
limit = int(num_files) if num_files else 100
self._get_file_list(master, builder, test_type, build_number, name,
before, limit, callback_name)
return
else:
# FIXME: Stop using the old master name style after all files have been
# updated.
master_data = master_config.getMaster(master)
if not master_data:
master_data = master_config.getMasterByMasterName(master)
if not master_data:
self.response.headers["Access-Control-Allow-Origin"] = "*"
self.response.set_status(404)
return
json, date = self._get_file_content(
master_data['url_name'], builder, test_type, build_number, name)
if json is None:
json, date = self._get_file_content(
master_data['name'], builder, test_type, build_number, name)
if json and test_list_json:
json = JsonResults.get_test_list(builder, json)
if json:
json = _replace_jsonp_callback(json, callback_name)
self._serve_json(json, date)
def test_get_nonexistant_results(self):
master = master_config.getMaster('chromium.chromiumos')
builder = 'test-builder'
test_type = 'test-type'
params = collections.OrderedDict([
(testfilehandler.PARAM_BUILDER, builder),
(testfilehandler.PARAM_MASTER, master['url_name']),
(testfilehandler.PARAM_TEST_TYPE, test_type),
(testfilehandler.PARAM_NAME, 'results.json')
])
self.test_app.get('/testfile', params=params, status=404)
params[testfilehandler.PARAM_TEST_LIST_JSON] = '1'
self.test_app.get('/testfile', params=params, status=404)
def test_get_nonexistant_results(self):
master = master_config.getMaster('chromium.chromiumos')
builder = 'test-builder'
test_type = 'test-type'
params = collections.OrderedDict([
(testfilehandler.PARAM_BUILDER, builder),
(testfilehandler.PARAM_MASTER, master['url_name']),
(testfilehandler.PARAM_TEST_TYPE, test_type),
(testfilehandler.PARAM_NAME, 'results.json')
])
self.test_app.get('/testfile', params=params, status=404)
params[testfilehandler.PARAM_TEST_LIST_JSON] = '1'
self.test_app.get('/testfile', params=params, status=404)
def fetch_buildbot_data(masters=None): # pragma: no cover
start_time = datetime.datetime.now()
all_masters_data = []
if masters:
masters = [master_config.getMaster(m) for m in masters]
else:
masters = master_config.getAllMasters()
for master_data in masters:
all_masters_data.append(master_data)
url_name = master_data['url_name']
master_url = MASTER_URL_TEMPLATE % url_name
builders = fetch_json(master_url)
if not builders:
msg = 'Aborting fetch. Could not fetch builders from %s' % master_url
logging.warning(msg)
raise FetchBuildersException(msg)
tests_object = master_data.setdefault('tests', {})
for builder in builders['builders'].keys():
build = fetch_json(BUILDS_URL_TEMPLATE %
(urllib2.quote(builder), url_name))
if not build:
logging.info('Skipping builder %s on master %s due to empty data.',
builder, url_name)
continue
if not build['builds']:
logging.info(
'Skipping builder %s on master %s due to empty builds list.',
builder, url_name)
continue
for step in build['builds'][0]['steps']:
step_name = step['name']
if 'test' not in step_name:
continue
if any(name in step_name for name in NON_TEST_STEP_NAMES):
continue
if re.search('_only|_ignore|_perf$', step_name):
continue
# This is just triggering and collecting tests on swarming, not the
# actual test types.
if (step_name.startswith('[trigger]') or
step_name.startswith('[collect]') or
step_name.startswith('[skipped]')):
continue
# Get the test type from the test-results uploading step name.
match = GTEST_UPLOADER_STEP_REGEX.match(step_name)
if match:
step_name = match.group(1)
# Aggressively skip all steps that dont have test in the first
# word (before first space) and remove all platform cruft. This rule
# is based on a manual audit of valid and invalid test types populated
# in the dashboard in Q4 2015.
# Note: Normalization also happens at upload time to ensure known and
# actual test types match.
# TODO: Remove NON_TEST_STEP_NAMES since this rule should remove all
# of them already.
step_name = util.normalize_test_type(step_name)
if 'test' not in step_name:
continue
tests_object.setdefault(step_name, {'builders': set()})
tests_object[step_name]['builders'].add(builder)
for builders in tests_object.values():
builders['builders'] = sorted(builders['builders'])
output_data = {'masters': all_masters_data,
'no_upload_test_types': TEST_STEPS_THAT_DO_NOT_UPLOAD_YET}
delta = datetime.datetime.now() - start_time
logging.info('Fetched buildbot data in %s seconds.', delta.seconds)
return dump_json(output_data)
def test_deprecated_master_name(self):
"""Verify that a file uploaded with a deprecated master name
can be downloaded by the corresponding new-style master name.
"""
master = master_config.getMaster('chromium.chromiumos')
builder = 'test-builder'
test_type = 'test-type'
test_data = {
'tests': {
'Test1.testproc1': {
'expected': 'PASS',
'actual': 'PASS',
'time': 1,
}
},
'build_number': '123',
'version': JSON_RESULTS_HIERARCHICAL_VERSION,
'builder_name': builder,
'blink_revision': '12345',
'seconds_since_epoch': 1406123456,
'num_failures_by_type': {
'FAIL': 0,
'SKIP': 0,
'PASS': 1
},
'chromium_revision': '67890',
}
# Upload file using deprecated master name.
params = collections.OrderedDict([
(testfilehandler.PARAM_BUILDER, builder),
(testfilehandler.PARAM_MASTER, master['name']),
(testfilehandler.PARAM_TEST_TYPE, test_type),
])
upload_files = [
('file', 'full_results.json', json.JSONEncoder().encode(test_data))]
response = self.test_app.post(
'/testfile/upload', params=params, upload_files=upload_files)
self.assertEqual(response.status_int, 200)
# Download file using deprecated master name.
params = collections.OrderedDict([
(testfilehandler.PARAM_BUILDER, builder),
(testfilehandler.PARAM_MASTER, master['name']),
(testfilehandler.PARAM_TEST_TYPE, test_type),
(testfilehandler.PARAM_BUILD_NUMBER, '123'),
(testfilehandler.PARAM_NAME, 'full_results.json')
])
response = self.test_app.get('/testfile', params=params)
self.assertEqual(response.status_int, 200)
response_json = json.loads(response.normal_body)
self.assertEqual(response_json['chromium_revision'], '67890')
# Download file using new-style name.
params = collections.OrderedDict([
(testfilehandler.PARAM_BUILDER, builder),
(testfilehandler.PARAM_MASTER, master['url_name']),
(testfilehandler.PARAM_TEST_TYPE, test_type),
(testfilehandler.PARAM_BUILD_NUMBER, '123'),
(testfilehandler.PARAM_NAME, 'full_results.json')
])
response = self.test_app.get('/testfile', params=params)
self.assertEqual(response.status_int, 200)
response_json = json.loads(response.normal_body)
self.assertEqual(response_json['chromium_revision'], '67890')
def fetch_buildbot_data(masters=None): # pragma: no cover
start_time = datetime.datetime.now()
all_masters_data = []
if masters:
masters = [master_config.getMaster(m) for m in masters]
else:
masters = master_config.getAllMasters()
for master_data in masters:
all_masters_data.append(master_data)
url_name = master_data['url_name']
master_url = MASTER_URL_TEMPLATE % url_name
builders = fetch_json(master_url)
if not builders:
msg = 'Aborting fetch. Could not fetch builders from %s' % master_url
logging.warning(msg)
raise FetchBuildersException(msg)
tests_object = master_data.setdefault('tests', {})
for builder in builders['builders'].keys():
build = fetch_json(BUILDS_URL_TEMPLATE %
(urllib2.quote(builder), url_name))
if not build:
logging.info('Skipping builder %s on master %s due to empty data.',
builder, url_name)
continue
if not build['builds']:
logging.info(
'Skipping builder %s on master %s due to empty builds list.',
builder, url_name)
continue
for step in build['builds'][0]['steps']:
step_name = step['name']
if not 'test' in step_name:
continue
if any(name in step_name for name in NON_TEST_STEP_NAMES):
continue
if re.search('_only|_ignore|_perf$', step_name):
continue
if step_name == 'webkit_tests':
step_name = 'layout-tests'
tests_object.setdefault(step_name, {'builders': []})
tests_object[step_name]['builders'].append(builder)
for builders in tests_object.values():
builders['builders'].sort()
output_data = {'masters': all_masters_data,
'no_upload_test_types': TEST_STEPS_THAT_DO_NOT_UPLOAD_YET}
delta = datetime.datetime.now() - start_time
logging.info('Fetched buildbot data in %s seconds.', delta.seconds)
return dump_json(output_data)
def fetch_buildbot_data(masters=None): # pragma: no cover
start_time = datetime.datetime.now()
all_masters_data = []
if masters:
masters = [master_config.getMaster(m) for m in masters]
else:
masters = master_config.getAllMasters()
for master_data in masters:
all_masters_data.append(master_data)
url_name = master_data['url_name']
master_url = MASTER_URL_TEMPLATE % url_name
builders = fetch_json(master_url)
if not builders:
msg = 'Aborting fetch. Could not fetch builders from %s' % master_url
logging.warning(msg)
raise FetchBuildersException(msg)
tests_object = master_data.setdefault('tests', {})
for builder in builders['builders'].keys():
build = fetch_json(BUILDS_URL_TEMPLATE %
(urllib2.quote(builder), url_name))
if not build:
logging.info(
'Skipping builder %s on master %s due to empty data.',
builder, url_name)
continue
if not build['builds']:
logging.info(
'Skipping builder %s on master %s due to empty builds list.',
builder, url_name)
continue
for step in build['builds'][0]['steps']:
step_name = step['name']
if not 'test' in step_name:
continue
if any(name in step_name for name in NON_TEST_STEP_NAMES):
continue
if re.search('_only|_ignore|_perf$', step_name):
continue
if step_name == 'webkit_tests':
step_name = 'layout-tests'
tests_object.setdefault(step_name, {'builders': []})
tests_object[step_name]['builders'].append(builder)
for builders in tests_object.values():
builders['builders'].sort()
output_data = {
'masters': all_masters_data,
'no_upload_test_types': TEST_STEPS_THAT_DO_NOT_UPLOAD_YET
}
delta = datetime.datetime.now() - start_time
logging.info('Fetched buildbot data in %s seconds.', delta.seconds)
return dump_json(output_data)
def test_deprecated_master_name(self):
"""Verify that a file uploaded with a deprecated master name
can be downloaded by the corresponding new-style master name.
"""
master = master_config.getMaster('chromium.chromiumos')
builder = 'test-builder'
test_type = 'test-type'
test_data = {
'tests': {
'Test1.testproc1': {
'expected': 'PASS',
'actual': 'PASS',
'time': 1,
}
},
'build_number': '123',
'version': JSON_RESULTS_HIERARCHICAL_VERSION,
'builder_name': builder,
'blink_revision': '12345',
'seconds_since_epoch': 1406123456,
'num_failures_by_type': {
'FAIL': 0,
'SKIP': 0,
'PASS': 1
},
'chromium_revision': '67890',
}
# Upload file using deprecated master name.
params = collections.OrderedDict([
(testfilehandler.PARAM_BUILDER, builder),
(testfilehandler.PARAM_MASTER, master['name']),
(testfilehandler.PARAM_TEST_TYPE, test_type),
])
upload_files = [('file', 'full_results.json',
json.JSONEncoder().encode(test_data))]
response = self.test_app.post('/testfile/upload',
params=params,
upload_files=upload_files)
self.assertEqual(response.status_int, 200)
# Download file using deprecated master name.
params = collections.OrderedDict([
(testfilehandler.PARAM_BUILDER, builder),
(testfilehandler.PARAM_MASTER, master['name']),
(testfilehandler.PARAM_TEST_TYPE, test_type),
(testfilehandler.PARAM_BUILD_NUMBER, '123'),
(testfilehandler.PARAM_NAME, 'full_results.json')
])
response = self.test_app.get('/testfile', params=params)
self.assertEqual(response.status_int, 200)
response_json = json.loads(response.normal_body)
self.assertEqual(response_json['chromium_revision'], '67890')
# Download file using new-style name.
params = collections.OrderedDict([
(testfilehandler.PARAM_BUILDER, builder),
(testfilehandler.PARAM_MASTER, master['url_name']),
(testfilehandler.PARAM_TEST_TYPE, test_type),
(testfilehandler.PARAM_BUILD_NUMBER, '123'),
(testfilehandler.PARAM_NAME, 'full_results.json')
])
response = self.test_app.get('/testfile', params=params)
self.assertEqual(response.status_int, 200)
response_json = json.loads(response.normal_body)
self.assertEqual(response_json['chromium_revision'], '67890')
def test_deprecated_master_name(self):
tb = testbed.Testbed()
tb.activate()
tb.init_datastore_v3_stub()
tb.init_blobstore_stub()
master = master_config.getMaster('chromium.chromiumos')
builder = 'test-builder'
test_type = 'test-type'
test_data = [
{
'tests': {
'Test1.testproc1': {
'expected': 'PASS',
'actual': 'PASS',
'time': 1,
}
},
'build_number': '123',
'version': jsonresults.JSON_RESULTS_HIERARCHICAL_VERSION,
'builder_name': builder,
'seconds_since_epoch': 1406123456,
'num_failures_by_type': {
'FAIL': 0,
'SKIP': 0,
'PASS': 1
},
'chromium_revision': '761b2a4cbc3103ef5e48cc7e77184f57eb50f6d4',
},
{
'tests': {
'Test2.testproc2': {
'expected': 'PASS',
'actual': 'FAIL',
'time': 2,
}
},
'build_number': '456',
'version': jsonresults.JSON_RESULTS_HIERARCHICAL_VERSION,
'builder_name': builder,
'seconds_since_epoch': 1406654321,
'num_failures_by_type': {
'FAIL': 1,
'SKIP': 0,
'PASS': 0
},
'chromium_revision': '761b2a4cbc3103ef5e48cc7e77184f57eb50f6d5',
},
]
# Upload a file using old master name
# Seed results files using the old name.
JsonResults.update(
master['name'], builder, test_type, test_data[0], None, True)
# Update results files using the new name.
JsonResults.update(master['url_name'], builder, test_type, test_data[1],
master['name'], True)
# Verify that the file keyed by url_name contains both sets of results.
files = TestFile.get_files(
master['url_name'], builder, test_type, None, None, limit=3)
self.assertEqual(len(files), 2)
for f in files:
j = json.loads(f.data)
self.assertItemsEqual(j[builder]['chromeRevision'],
['761b2a4cbc3103ef5e48cc7e77184f57eb50f6d4',
'761b2a4cbc3103ef5e48cc7e77184f57eb50f6d5'])
tb.deactivate()
def test_deprecated_master_name(self):
tb = testbed.Testbed()
tb.activate()
tb.init_datastore_v3_stub()
tb.init_blobstore_stub()
master = master_config.getMaster('chromium.chromiumos')
builder = 'test-builder'
test_type = 'test-type'
test_data = [
{
'tests': {
'Test1.testproc1': {
'expected': 'PASS',
'actual': 'PASS',
'time': 1,
}
},
'build_number': '123',
'version': jsonresults.JSON_RESULTS_HIERARCHICAL_VERSION,
'builder_name': builder,
'blink_revision': '12345',
'seconds_since_epoch': 1406123456,
'num_failures_by_type': {
'FAIL': 0,
'SKIP': 0,
'PASS': 1
},
'chromium_revision': '761b2a4cbc3103ef5e48cc7e77184f57eb50f6d4',
},
{
'tests': {
'Test2.testproc2': {
'expected': 'PASS',
'actual': 'FAIL',
'time': 2,
}
},
'build_number': '456',
'version': jsonresults.JSON_RESULTS_HIERARCHICAL_VERSION,
'builder_name': builder,
'blink_revision': '54321',
'seconds_since_epoch': 1406654321,
'num_failures_by_type': {
'FAIL': 1,
'SKIP': 0,
'PASS': 0
},
'chromium_revision': '761b2a4cbc3103ef5e48cc7e77184f57eb50f6d5',
},
]
# Upload a file using old master name
# Seed results files using the old name.
JsonResults.update(
master['name'], builder, test_type, test_data[0], None, True)
# Update results files using the new name.
JsonResults.update(master['url_name'], builder, test_type, test_data[1],
master['name'], True)
# Verify that the file keyed by url_name contains both sets of results.
files = TestFile.get_files(
master['url_name'], builder, test_type, None, None, limit=3)
self.assertEqual(len(files), 2)
for f in files:
j = json.loads(f.data)
self.assertItemsEqual(j[builder]['blinkRevision'], ['12345', '54321'])
tb.deactivate()
