def test_summarize(self):
r = WPTReport()
r._report = {'results': [
{
'test': '/js/with-statement.html',
'status': 'OK',
'message': None,
'subtests': [
{'status': 'PASS', 'message': None, 'name': 'first'},
{'status': 'FAIL', 'message': 'bad', 'name': 'second'}
]
},
{
'test': '/js/isNaN.html',
'status': 'OK',
'message': None,
'subtests': [
{'status': 'PASS', 'message': None, 'name': 'first'},
{'status': 'FAIL', 'message': 'bad', 'name': 'second'},
{'status': 'PASS', 'message': None, 'name': 'third'}
]
}
]}
self.assertEqual(r.summarize(), {
'/js/with-statement.html': [2, 3],
'/js/isNaN.html': [3, 4]
})
def test_normalize_version(self):
r = WPTReport()
r._report = {'run_info': {
'browser_version': 'Technology Preview (Release 67, 13607.1.9.0.1)'
}}
r.normalize_version()
self.assertEqual(r.run_info['browser_version'], '67 preview')
def test_populate_upload_directory(self):
# This also tests write_summary() and write_result_directory().
revision = '0bdaaf9c1622ca49eb140381af1ece6d8001c934'
r = WPTReport()
r._report = {
'results': [{
'test': '/foo/bar.html',
'status': 'PASS',
'message': None,
'subtests': []
}],
'run_info': {
'revision': revision,
'product': 'firefox',
'browser_version': '59.0',
'os': 'linux'
}
}
r.hashsum = '0123456789'
r.populate_upload_directory(output_dir=self.tmp_dir)
self.assertTrue(
os.path.isfile(
os.path.join(self.tmp_dir, revision,
'firefox-59.0-linux-0123456789-summary.json.gz')))
self.assertTrue(
os.path.isfile(
os.path.join(self.tmp_dir, revision,
'firefox-59.0-linux-0123456789', 'foo',
'bar.html')))
def test_test_run_metadata_optional_fields(self):
r = WPTReport()
r._report = {
'run_info': {
'revision': '0bdaaf9c1622ca49eb140381af1ece6d8001c934',
'product': 'firefox',
'browser_version': '59.0',
'os': 'windows',
'os_version': '10'
},
'time_start': 1529606394218,
'time_end': 1529611429000,
}
self.assertDictEqual(
r.test_run_metadata, {
'browser_name': 'firefox',
'browser_version': '59.0',
'os_name': 'windows',
'os_version': '10',
'revision': '0bdaaf9c16',
'full_revision_hash':
'0bdaaf9c1622ca49eb140381af1ece6d8001c934',
'time_start': '2018-06-21T18:39:54.218000+00:00',
'time_end': '2018-06-21T20:03:49+00:00',
})
def test_summarize_duplicate_results(self):
r = WPTReport()
r._report = {'results': [
{
'test': '/js/with-statement.html',
'status': 'OK',
'message': None,
'subtests': [
{'status': 'PASS', 'message': None, 'name': 'first'},
{'status': 'FAIL', 'message': 'bad', 'name': 'second'}
]
},
{
'test': '/js/with-statement.html',
'status': 'OK',
'message': None,
'subtests': [
{'status': 'PASS', 'message': None, 'name': 'first'},
{'status': 'FAIL', 'message': 'bad', 'name': 'second'},
{'status': 'FAIL', 'message': 'bad', 'name': 'third'},
{'status': 'FAIL', 'message': 'bad', 'name': 'fourth'}
]
}
]}
with self.assertRaises(ConflictingDataError):
r.summarize()
def test_normalize_product_edgechromium(self):
r = WPTReport()
r._report = {
'run_info': {
'product': 'edgechromium',
}
}
self.assertSetEqual(normalize_product(r), {'edge', 'edgechromium'})
self.assertEqual(r.run_info['product'], 'edge')
def test_normalize_product(self):
r = WPTReport()
r._report = {
'run_info': {
'product': 'edge_webdriver',
}
}
self.assertSetEqual(normalize_product(r), {'edge', 'webdriver'})
self.assertEqual(r.run_info['product'], 'edge')
def test_normalize_product_noop(self):
r = WPTReport()
r._report = {
'run_info': {
'product': 'firefox',
}
}
self.assertSetEqual(normalize_product(r), set())
self.assertEqual(r.run_info['product'], 'firefox')
def test_normalize_product_webkitgtk_minibrowser(self):
r = WPTReport()
r._report = {
'run_info': {
'product': 'webkitgtk_minibrowser',
}
}
self.assertSetEqual(normalize_product(r), {'webkitgtk', 'minibrowser'})
self.assertEqual(r.run_info['product'], 'webkitgtk')
def test_test_run_metadata_missing_required_fields(self):
r = WPTReport()
r._report = {
'run_info': {
'product': 'firefox',
'os': 'linux'
}
}
with self.assertRaises(MissingMetadataError):
r.test_run_metadata
def test_product_id_sanitize(self):
r = WPTReport()
r._report = {
'run_info': {
'product': 'chrome!',
'browser_version': '1.2.3 dev-1',
'os': 'linux',
}
}
r.hashsum = 'afa59408e1797c7091d7e89de5561612f7da440d'
self.assertEqual(r.product_id(separator='-', sanitize=True),
'chrome_-1.2.3_dev-1-linux-afa59408e1')
def test_sha_summary_path(self):
r = WPTReport()
r._report = {
'run_info': {
'revision': '0bdaaf9c1622ca49eb140381af1ece6d8001c934',
'product': 'firefox',
'browser_version': '59.0',
'os': 'linux'
}
}
r.hashsum = 'afa59408e1797c7091d7e89de5561612f7da440d'
self.assertEqual(
r.sha_summary_path, '0bdaaf9c1622ca49eb140381af1ece6d8001c934/'
'firefox-59.0-linux-afa59408e1-summary.json.gz')
def test_each_result(self):
expected_results = [
{
'test': '/js/with-statement.html',
'status': 'OK',
'message': None,
'subtests': [
{'status': 'PASS', 'message': None, 'name': 'first'},
{'status': 'FAIL', 'message': 'bad', 'name': 'second'}
]
},
{
'test': '/js/isNaN.html',
'status': 'OK',
'message': None,
'subtests': [
{'status': 'PASS', 'message': None, 'name': 'first'},
{'status': 'FAIL', 'message': 'bad', 'name': 'second'},
{'status': 'PASS', 'message': None, 'name': 'third'}
]
},
{
'test': '/js/do-while-statement.html',
'status': 'OK',
'message': None,
'subtests': [
{'status': 'PASS', 'message': None, 'name': 'first'}
]
},
{
'test': '/js/symbol-unscopables.html',
'status': 'TIMEOUT',
'message': None,
'subtests': []
},
{
'test': '/js/void-statement.html',
'status': 'OK',
'message': None,
'subtests': [
{'status': 'PASS', 'message': None, 'name': 'first'},
{'status': 'FAIL', 'message': 'bad', 'name': 'second'},
{'status': 'FAIL', 'message': 'bad', 'name': 'third'},
{'status': 'FAIL', 'message': 'bad', 'name': 'fourth'}
]
}
]
r = WPTReport()
r._report = {'results': expected_results}
self.assertListEqual(list(r.each_result()), expected_results)
def test_product_id(self):
r = WPTReport()
r._report = {
'run_info': {
'product': 'firefox',
'browser_version': '59.0',
'os': 'linux',
}
}
r.hashsum = 'afa59408e1797c7091d7e89de5561612f7da440d'
self.assertEqual(r.product_id(), 'firefox-59.0-linux-afa59408e1')
r._report['run_info']['os_version'] = '4.4'
self.assertEqual(r.product_id(separator='_'),
'firefox_59.0_linux_4.4_afa59408e1')
def test_prepare_labels_from_browser_channel(self):
r = WPTReport()
r._report = {
'run_info': {
'product': 'firefox',
'browser_channel': 'dev',
}
}
self.assertListEqual(prepare_labels(r, '', 'blade-runner'),
['blade-runner', 'experimental', 'firefox'])
r._report['run_info']['browser_channel'] = 'nightly'
self.assertListEqual(prepare_labels(r, '', 'blade-runner'),
['blade-runner', 'experimental', 'firefox'])
r._report['run_info']['browser_channel'] = 'beta'
self.assertListEqual(prepare_labels(r, '', 'blade-runner'),
['beta', 'blade-runner', 'firefox'])
def test_test_run_metadata(self):
r = WPTReport()
r._report = {
'run_info': {
'revision': '0bdaaf9c1622ca49eb140381af1ece6d8001c934',
'product': 'firefox',
'browser_version': '59.0',
'os': 'linux'
}
}
self.assertDictEqual(r.test_run_metadata, {
'browser_name': 'firefox',
'browser_version': '59.0',
'os_name': 'linux',
'revision': '0bdaaf9c16',
'full_revision_hash': '0bdaaf9c1622ca49eb140381af1ece6d8001c934',
})
def test_summarize_whitespaces(self):
r = WPTReport()
r._report = {
'results': [{
'test': ' /ref/reftest.html',
'status': 'PASS',
'message': None,
'subtests': []
}, {
'test': '/ref/reftest-fail.html\n',
'status': 'FAIL',
'message': None,
'subtests': []
}]
}
self.assertEqual(r.summarize(), {
'/ref/reftest.html': [1, 1],
'/ref/reftest-fail.html': [0, 1]
})
def test_prepare_labels_from_browser_channel(self):
# Chrome Dev
r = WPTReport()
r._report = {
'run_info': {
'product': 'chrome',
'browser_channel': 'dev',
}
}
self.assertSetEqual(prepare_labels(r, '', 'blade-runner'),
{'blade-runner', 'dev', 'experimental', 'chrome'})
# Chrome Canary
r._report['run_info']['browser_channel'] = 'canary'
self.assertSetEqual(prepare_labels(r, '', 'blade-runner'),
{'blade-runner', 'canary', 'nightly', 'chrome'})
# Chrome Nightly
r._report['run_info']['browser_channel'] = 'nightly'
self.assertSetEqual(prepare_labels(r, '', 'blade-runner'),
{'blade-runner', 'nightly', 'chrome'})
# WebKitGTK Nightly
r._report['run_info']['product'] = 'webkitgtk_minibrowser'
self.assertSetEqual(prepare_labels(r, '', 'blade-runner'), {
'blade-runner', 'nightly', 'experimental', 'webkitgtk_minibrowser'
})
# Firefox Nightly
r._report['run_info']['product'] = 'firefox'
self.assertSetEqual(
prepare_labels(r, '', 'blade-runner'),
{'blade-runner', 'nightly', 'experimental', 'firefox'})
# Firefox Beta
r._report['run_info']['browser_channel'] = 'beta'
self.assertSetEqual(prepare_labels(r, '', 'blade-runner'),
{'blade-runner', 'beta', 'firefox'})
def test_normalize_version_missing_version(self):
r = WPTReport()
r._report = {'run_info': {}}
r.normalize_version()
# Do not throw!
self.assertIsNone(r.run_info.get('browser_version'))
