def test_summarize(self): r = WPTReport() r._report = {'results': [ { 'test': '/js/with-statement.html', 'status': 'OK', 'message': None, 'subtests': [ {'status': 'PASS', 'message': None, 'name': 'first'}, {'status': 'FAIL', 'message': 'bad', 'name': 'second'} ] }, { 'test': '/js/isNaN.html', 'status': 'OK', 'message': None, 'subtests': [ {'status': 'PASS', 'message': None, 'name': 'first'}, {'status': 'FAIL', 'message': 'bad', 'name': 'second'}, {'status': 'PASS', 'message': None, 'name': 'third'} ] } ]} self.assertEqual(r.summarize(), { '/js/with-statement.html': [2, 3], '/js/isNaN.html': [3, 4] })
def test_normalize_version(self): r = WPTReport() r._report = {'run_info': { 'browser_version': 'Technology Preview (Release 67, 13607.1.9.0.1)' }} r.normalize_version() self.assertEqual(r.run_info['browser_version'], '67 preview')
def test_populate_upload_directory(self): # This also tests write_summary() and write_result_directory(). revision = '0bdaaf9c1622ca49eb140381af1ece6d8001c934' r = WPTReport() r._report = { 'results': [{ 'test': '/foo/bar.html', 'status': 'PASS', 'message': None, 'subtests': [] }], 'run_info': { 'revision': revision, 'product': 'firefox', 'browser_version': '59.0', 'os': 'linux' } } r.hashsum = '0123456789' r.populate_upload_directory(output_dir=self.tmp_dir) self.assertTrue( os.path.isfile( os.path.join(self.tmp_dir, revision, 'firefox-59.0-linux-0123456789-summary.json.gz'))) self.assertTrue( os.path.isfile( os.path.join(self.tmp_dir, revision, 'firefox-59.0-linux-0123456789', 'foo', 'bar.html')))
def test_test_run_metadata_optional_fields(self): r = WPTReport() r._report = { 'run_info': { 'revision': '0bdaaf9c1622ca49eb140381af1ece6d8001c934', 'product': 'firefox', 'browser_version': '59.0', 'os': 'windows', 'os_version': '10' }, 'time_start': 1529606394218, 'time_end': 1529611429000, } self.assertDictEqual( r.test_run_metadata, { 'browser_name': 'firefox', 'browser_version': '59.0', 'os_name': 'windows', 'os_version': '10', 'revision': '0bdaaf9c16', 'full_revision_hash': '0bdaaf9c1622ca49eb140381af1ece6d8001c934', 'time_start': '2018-06-21T18:39:54.218000+00:00', 'time_end': '2018-06-21T20:03:49+00:00', })
def test_summarize_duplicate_results(self): r = WPTReport() r._report = {'results': [ { 'test': '/js/with-statement.html', 'status': 'OK', 'message': None, 'subtests': [ {'status': 'PASS', 'message': None, 'name': 'first'}, {'status': 'FAIL', 'message': 'bad', 'name': 'second'} ] }, { 'test': '/js/with-statement.html', 'status': 'OK', 'message': None, 'subtests': [ {'status': 'PASS', 'message': None, 'name': 'first'}, {'status': 'FAIL', 'message': 'bad', 'name': 'second'}, {'status': 'FAIL', 'message': 'bad', 'name': 'third'}, {'status': 'FAIL', 'message': 'bad', 'name': 'fourth'} ] } ]} with self.assertRaises(ConflictingDataError): r.summarize()
def test_normalize_product_edgechromium(self): r = WPTReport() r._report = { 'run_info': { 'product': 'edgechromium', } } self.assertSetEqual(normalize_product(r), {'edge', 'edgechromium'}) self.assertEqual(r.run_info['product'], 'edge')
def test_normalize_product(self): r = WPTReport() r._report = { 'run_info': { 'product': 'edge_webdriver', } } self.assertSetEqual(normalize_product(r), {'edge', 'webdriver'}) self.assertEqual(r.run_info['product'], 'edge')
def test_normalize_product_noop(self): r = WPTReport() r._report = { 'run_info': { 'product': 'firefox', } } self.assertSetEqual(normalize_product(r), set()) self.assertEqual(r.run_info['product'], 'firefox')
def test_normalize_product_webkitgtk_minibrowser(self): r = WPTReport() r._report = { 'run_info': { 'product': 'webkitgtk_minibrowser', } } self.assertSetEqual(normalize_product(r), {'webkitgtk', 'minibrowser'}) self.assertEqual(r.run_info['product'], 'webkitgtk')
def test_test_run_metadata_missing_required_fields(self): r = WPTReport() r._report = { 'run_info': { 'product': 'firefox', 'os': 'linux' } } with self.assertRaises(MissingMetadataError): r.test_run_metadata
def test_product_id_sanitize(self): r = WPTReport() r._report = { 'run_info': { 'product': 'chrome!', 'browser_version': '1.2.3 dev-1', 'os': 'linux', } } r.hashsum = 'afa59408e1797c7091d7e89de5561612f7da440d' self.assertEqual(r.product_id(separator='-', sanitize=True), 'chrome_-1.2.3_dev-1-linux-afa59408e1')
def test_sha_summary_path(self): r = WPTReport() r._report = { 'run_info': { 'revision': '0bdaaf9c1622ca49eb140381af1ece6d8001c934', 'product': 'firefox', 'browser_version': '59.0', 'os': 'linux' } } r.hashsum = 'afa59408e1797c7091d7e89de5561612f7da440d' self.assertEqual( r.sha_summary_path, '0bdaaf9c1622ca49eb140381af1ece6d8001c934/' 'firefox-59.0-linux-afa59408e1-summary.json.gz')
def test_each_result(self): expected_results = [ { 'test': '/js/with-statement.html', 'status': 'OK', 'message': None, 'subtests': [ {'status': 'PASS', 'message': None, 'name': 'first'}, {'status': 'FAIL', 'message': 'bad', 'name': 'second'} ] }, { 'test': '/js/isNaN.html', 'status': 'OK', 'message': None, 'subtests': [ {'status': 'PASS', 'message': None, 'name': 'first'}, {'status': 'FAIL', 'message': 'bad', 'name': 'second'}, {'status': 'PASS', 'message': None, 'name': 'third'} ] }, { 'test': '/js/do-while-statement.html', 'status': 'OK', 'message': None, 'subtests': [ {'status': 'PASS', 'message': None, 'name': 'first'} ] }, { 'test': '/js/symbol-unscopables.html', 'status': 'TIMEOUT', 'message': None, 'subtests': [] }, { 'test': '/js/void-statement.html', 'status': 'OK', 'message': None, 'subtests': [ {'status': 'PASS', 'message': None, 'name': 'first'}, {'status': 'FAIL', 'message': 'bad', 'name': 'second'}, {'status': 'FAIL', 'message': 'bad', 'name': 'third'}, {'status': 'FAIL', 'message': 'bad', 'name': 'fourth'} ] } ] r = WPTReport() r._report = {'results': expected_results} self.assertListEqual(list(r.each_result()), expected_results)
def test_product_id(self): r = WPTReport() r._report = { 'run_info': { 'product': 'firefox', 'browser_version': '59.0', 'os': 'linux', } } r.hashsum = 'afa59408e1797c7091d7e89de5561612f7da440d' self.assertEqual(r.product_id(), 'firefox-59.0-linux-afa59408e1') r._report['run_info']['os_version'] = '4.4' self.assertEqual(r.product_id(separator='_'), 'firefox_59.0_linux_4.4_afa59408e1')
def test_prepare_labels_from_browser_channel(self): r = WPTReport() r._report = { 'run_info': { 'product': 'firefox', 'browser_channel': 'dev', } } self.assertListEqual(prepare_labels(r, '', 'blade-runner'), ['blade-runner', 'experimental', 'firefox']) r._report['run_info']['browser_channel'] = 'nightly' self.assertListEqual(prepare_labels(r, '', 'blade-runner'), ['blade-runner', 'experimental', 'firefox']) r._report['run_info']['browser_channel'] = 'beta' self.assertListEqual(prepare_labels(r, '', 'blade-runner'), ['beta', 'blade-runner', 'firefox'])
def test_test_run_metadata(self): r = WPTReport() r._report = { 'run_info': { 'revision': '0bdaaf9c1622ca49eb140381af1ece6d8001c934', 'product': 'firefox', 'browser_version': '59.0', 'os': 'linux' } } self.assertDictEqual(r.test_run_metadata, { 'browser_name': 'firefox', 'browser_version': '59.0', 'os_name': 'linux', 'revision': '0bdaaf9c16', 'full_revision_hash': '0bdaaf9c1622ca49eb140381af1ece6d8001c934', })
def test_summarize_whitespaces(self): r = WPTReport() r._report = { 'results': [{ 'test': ' /ref/reftest.html', 'status': 'PASS', 'message': None, 'subtests': [] }, { 'test': '/ref/reftest-fail.html\n', 'status': 'FAIL', 'message': None, 'subtests': [] }] } self.assertEqual(r.summarize(), { '/ref/reftest.html': [1, 1], '/ref/reftest-fail.html': [0, 1] })
def test_prepare_labels_from_browser_channel(self): # Chrome Dev r = WPTReport() r._report = { 'run_info': { 'product': 'chrome', 'browser_channel': 'dev', } } self.assertSetEqual(prepare_labels(r, '', 'blade-runner'), {'blade-runner', 'dev', 'experimental', 'chrome'}) # Chrome Canary r._report['run_info']['browser_channel'] = 'canary' self.assertSetEqual(prepare_labels(r, '', 'blade-runner'), {'blade-runner', 'canary', 'nightly', 'chrome'}) # Chrome Nightly r._report['run_info']['browser_channel'] = 'nightly' self.assertSetEqual(prepare_labels(r, '', 'blade-runner'), {'blade-runner', 'nightly', 'chrome'}) # WebKitGTK Nightly r._report['run_info']['product'] = 'webkitgtk_minibrowser' self.assertSetEqual(prepare_labels(r, '', 'blade-runner'), { 'blade-runner', 'nightly', 'experimental', 'webkitgtk_minibrowser' }) # Firefox Nightly r._report['run_info']['product'] = 'firefox' self.assertSetEqual( prepare_labels(r, '', 'blade-runner'), {'blade-runner', 'nightly', 'experimental', 'firefox'}) # Firefox Beta r._report['run_info']['browser_channel'] = 'beta' self.assertSetEqual(prepare_labels(r, '', 'blade-runner'), {'blade-runner', 'beta', 'firefox'})
def test_normalize_version_missing_version(self): r = WPTReport() r._report = {'run_info': {}} r.normalize_version() # Do not throw! self.assertIsNone(r.run_info.get('browser_version'))