def test_prepare_labels_from_stable_label(self): r = WPTReport() r.update_metadata(browser_name='firefox') self.assertListEqual( prepare_labels(r, 'stable', 'blade-runner'), ['blade-runner', 'firefox', 'stable'] )
def test_summarize_duplicate_results(self): r = WPTReport() r._report = {'results': [ { 'test': '/js/with-statement.html', 'status': 'OK', 'message': None, 'subtests': [ {'status': 'PASS', 'message': None, 'name': 'first'}, {'status': 'FAIL', 'message': 'bad', 'name': 'second'} ] }, { 'test': '/js/with-statement.html', 'status': 'OK', 'message': None, 'subtests': [ {'status': 'PASS', 'message': None, 'name': 'first'}, {'status': 'FAIL', 'message': 'bad', 'name': 'second'}, {'status': 'FAIL', 'message': 'bad', 'name': 'third'}, {'status': 'FAIL', 'message': 'bad', 'name': 'fourth'} ] } ]} with self.assertRaises(ConflictingDataError): r.summarize()
def test_normalize_version(self): r = WPTReport() r._report = {'run_info': { 'browser_version': 'Technology Preview (Release 67, 13607.1.9.0.1)' }} r.normalize_version() self.assertEqual(r.run_info['browser_version'], '67 preview')
def test_prepare_labels_from_experimental_label(self): r = WPTReport() r.update_metadata(browser_name='firefox') self.assertSetEqual( prepare_labels(r, 'experimental', 'blade-runner'), {'blade-runner', 'experimental', 'firefox'} )
def test_prepare_labels_from_custom_labels(self): r = WPTReport() r.update_metadata(browser_name='firefox') self.assertSetEqual( prepare_labels(r, 'foo,bar', 'blade-runner'), {'bar', 'blade-runner', 'firefox', 'foo', 'stable'} )
def test_summarize(self): r = WPTReport() r._report = {'results': [ { 'test': '/js/with-statement.html', 'status': 'OK', 'message': None, 'subtests': [ {'status': 'PASS', 'message': None, 'name': 'first'}, {'status': 'FAIL', 'message': 'bad', 'name': 'second'} ] }, { 'test': '/js/isNaN.html', 'status': 'OK', 'message': None, 'subtests': [ {'status': 'PASS', 'message': None, 'name': 'first'}, {'status': 'FAIL', 'message': 'bad', 'name': 'second'}, {'status': 'PASS', 'message': None, 'name': 'third'} ] } ]} self.assertEqual(r.summarize(), { '/js/with-statement.html': [2, 3], '/js/isNaN.html': [3, 4] })
def test_test_run_metadata_optional_fields(self): r = WPTReport() r._report = { 'run_info': { 'revision': '0bdaaf9c1622ca49eb140381af1ece6d8001c934', 'product': 'firefox', 'browser_version': '59.0', 'os': 'windows', 'os_version': '10' }, 'time_start': 1529606394218, 'time_end': 1529611429000, } self.assertDictEqual( r.test_run_metadata, { 'browser_name': 'firefox', 'browser_version': '59.0', 'os_name': 'windows', 'os_version': '10', 'revision': '0bdaaf9c16', 'full_revision_hash': '0bdaaf9c1622ca49eb140381af1ece6d8001c934', 'time_start': '2018-06-21T18:39:54.218000+00:00', 'time_end': '2018-06-21T20:03:49+00:00', })
def test_load_json_empty_report(self): tmp_path = os.path.join(self.tmp_dir, 'test.json') with open(tmp_path, 'wt') as f: f.write('{}') r = WPTReport() with open(tmp_path, 'rb') as f: with self.assertRaises(InsufficientDataError): r.load_json(f)
def test_load_json_invalid_json(self): tmp_path = os.path.join(self.tmp_dir, 'test.json') with open(tmp_path, 'wt') as f: f.write('{[') r = WPTReport() with open(tmp_path, 'rb') as f: with self.assertRaises(InvalidJSONError): r.load_json(f)
def test_write_json(self): obj = {'results': [{'test': 'foo'}]} tmp_path = os.path.join(self.tmp_dir, 'test.json') with open(tmp_path, 'wb') as f: WPTReport.write_json(f, obj) with open(tmp_path, 'rt') as f: round_trip = json.load(f) self.assertDictEqual(obj, round_trip)
def test_normalize_product_edgechromium(self): r = WPTReport() r._report = { 'run_info': { 'product': 'edgechromium', } } self.assertSetEqual(normalize_product(r), {'edge', 'edgechromium'}) self.assertEqual(r.run_info['product'], 'edge')
def test_normalize_product(self): r = WPTReport() r._report = { 'run_info': { 'product': 'edge_webdriver', } } self.assertSetEqual(normalize_product(r), {'edge', 'webdriver'}) self.assertEqual(r.run_info['product'], 'edge')
def test_normalize_product_webkitgtk_minibrowser(self): r = WPTReport() r._report = { 'run_info': { 'product': 'webkitgtk_minibrowser', } } self.assertSetEqual(normalize_product(r), {'webkitgtk', 'minibrowser'}) self.assertEqual(r.run_info['product'], 'webkitgtk')
def test_normalize_product_noop(self): r = WPTReport() r._report = { 'run_info': { 'product': 'firefox', } } self.assertSetEqual(normalize_product(r), set()) self.assertEqual(r.run_info['product'], 'firefox')
def test_load_json(self): tmp_path = os.path.join(self.tmp_dir, 'test.json') with open(tmp_path, 'wt') as f: f.write('{"results": [{"test": "foo"}]}') r = WPTReport() with open(tmp_path, 'rb') as f: r.load_json(f) self.assertEqual(len(r.results), 1) # This is the sha1sum of the string written above. self.assertEqual(r.hashsum, 'afa59408e1797c7091d7e89de5561612f7da440d')
def test_test_run_metadata_missing_required_fields(self): r = WPTReport() r._report = { 'run_info': { 'product': 'firefox', 'os': 'linux' } } with self.assertRaises(MissingMetadataError): r.test_run_metadata
def test_product_id_sanitize(self): r = WPTReport() r._report = { 'run_info': { 'product': 'chrome!', 'browser_version': '1.2.3 dev-1', 'os': 'linux', } } r.hashsum = 'afa59408e1797c7091d7e89de5561612f7da440d' self.assertEqual(r.product_id(separator='-', sanitize=True), 'chrome_-1.2.3_dev-1-linux-afa59408e1')
def test_each_result(self): expected_results = [ { 'test': '/js/with-statement.html', 'status': 'OK', 'message': None, 'subtests': [ {'status': 'PASS', 'message': None, 'name': 'first'}, {'status': 'FAIL', 'message': 'bad', 'name': 'second'} ] }, { 'test': '/js/isNaN.html', 'status': 'OK', 'message': None, 'subtests': [ {'status': 'PASS', 'message': None, 'name': 'first'}, {'status': 'FAIL', 'message': 'bad', 'name': 'second'}, {'status': 'PASS', 'message': None, 'name': 'third'} ] }, { 'test': '/js/do-while-statement.html', 'status': 'OK', 'message': None, 'subtests': [ {'status': 'PASS', 'message': None, 'name': 'first'} ] }, { 'test': '/js/symbol-unscopables.html', 'status': 'TIMEOUT', 'message': None, 'subtests': [] }, { 'test': '/js/void-statement.html', 'status': 'OK', 'message': None, 'subtests': [ {'status': 'PASS', 'message': None, 'name': 'first'}, {'status': 'FAIL', 'message': 'bad', 'name': 'second'}, {'status': 'FAIL', 'message': 'bad', 'name': 'third'}, {'status': 'FAIL', 'message': 'bad', 'name': 'fourth'} ] } ] r = WPTReport() r._report = {'results': expected_results} self.assertListEqual(list(r.each_result()), expected_results)
def test_sha_summary_path(self): r = WPTReport() r._report = { 'run_info': { 'revision': '0bdaaf9c1622ca49eb140381af1ece6d8001c934', 'product': 'firefox', 'browser_version': '59.0', 'os': 'linux' } } r.hashsum = 'afa59408e1797c7091d7e89de5561612f7da440d' self.assertEqual( r.sha_summary_path, '0bdaaf9c1622ca49eb140381af1ece6d8001c934/' 'firefox-59.0-linux-afa59408e1-summary.json.gz')
def test_write_gzip_json(self): # This case also covers the Unicode testing of write_json(). obj = {'results': [{ 'test': 'ABC~‾¥≈¤・・•∙·☼★星🌟星★☼·∙•・・¤≈¥‾~XYZ', 'message': None, 'status': 'PASS' }]} tmp_path = os.path.join(self.tmp_dir, 'foo', 'bar.json.gz') WPTReport.write_gzip_json(tmp_path, obj) with open(tmp_path, 'rb') as f: with gzip.GzipFile(fileobj=f, mode='rb') as gf: with io.TextIOWrapper(gf, encoding='utf-8') as tf: round_trip = json.load(tf) self.assertDictEqual(obj, round_trip)
def test_update_metadata(self): r = WPTReport() r.update_metadata(revision='0bdaaf9c1622ca49eb140381af1ece6d8001c934', browser_name='firefox', browser_version='59.0', os_name='linux', os_version='4.4') self.assertDictEqual( r.run_info, { 'revision': '0bdaaf9c1622ca49eb140381af1ece6d8001c934', 'product': 'firefox', 'browser_version': '59.0', 'os': 'linux', 'os_version': '4.4' })
def test_prepare_labels_from_browser_channel(self): r = WPTReport() r._report = { 'run_info': { 'product': 'firefox', 'browser_channel': 'dev', } } self.assertListEqual(prepare_labels(r, '', 'blade-runner'), ['blade-runner', 'experimental', 'firefox']) r._report['run_info']['browser_channel'] = 'nightly' self.assertListEqual(prepare_labels(r, '', 'blade-runner'), ['blade-runner', 'experimental', 'firefox']) r._report['run_info']['browser_channel'] = 'beta' self.assertListEqual(prepare_labels(r, '', 'blade-runner'), ['beta', 'blade-runner', 'firefox'])
def test_test_run_metadata(self): r = WPTReport() r._report = { 'run_info': { 'revision': '0bdaaf9c1622ca49eb140381af1ece6d8001c934', 'product': 'firefox', 'browser_version': '59.0', 'os': 'linux' } } self.assertDictEqual(r.test_run_metadata, { 'browser_name': 'firefox', 'browser_version': '59.0', 'os_name': 'linux', 'revision': '0bdaaf9c16', 'full_revision_hash': '0bdaaf9c1622ca49eb140381af1ece6d8001c934', })
def test_load_json_multiple_chunks_metadata(self): tmp_path = os.path.join(self.tmp_dir, 'test.json') r = WPTReport() # Load a report with no metadata first to test the handling of None. with open(tmp_path, 'wt') as f: f.write('{"results": [{"test": "foo"}]}\n') with open(tmp_path, 'rb') as f: r.load_json(f) with open(tmp_path, 'wt') as f: json.dump( { 'results': [{ 'test1': 'foo' }], 'run_info': { 'product': 'firefox', 'os': 'linux' }, 'time_start': 100, 'time_end': 200, }, f) with open(tmp_path, 'rb') as f: r.load_json(f) with open(tmp_path, 'wt') as f: json.dump( { 'results': [{ 'test2': 'bar' }], 'run_info': { 'product': 'firefox', 'browser_version': '59.0' }, 'time_start': 10, 'time_end': 500, }, f) with open(tmp_path, 'rb') as f: r.load_json(f) self.assertEqual(len(r.results), 3) # run_info should be the union of all run_info. self.assertDictEqual(r.run_info, { 'product': 'firefox', 'browser_version': '59.0', 'os': 'linux' }) # The smallest time_start should be kept. self.assertEqual(r._report['time_start'], 10) # The largest time_end should be kept. self.assertEqual(r._report['time_end'], 500)
def test_summarize_whitespaces(self): r = WPTReport() r._report = { 'results': [{ 'test': ' /ref/reftest.html', 'status': 'PASS', 'message': None, 'subtests': [] }, { 'test': '/ref/reftest-fail.html\n', 'status': 'FAIL', 'message': None, 'subtests': [] }] } self.assertEqual(r.summarize(), { '/ref/reftest.html': [1, 1], '/ref/reftest-fail.html': [0, 1] })
def test_load_gzip_json(self): # This case also covers the Unicode testing of load_json(). obj = { 'results': [{ 'test': 'ABC~‾¥≈¤・・•∙·☼★星🌟星★☼·∙•・・¤≈¥‾~XYZ', 'message': None, 'status': 'PASS' }] } json_s = json.dumps(obj, ensure_ascii=False) tmp_path = os.path.join(self.tmp_dir, 'test.json.gz') with open(tmp_path, 'wb') as f: gzip_file = gzip.GzipFile(fileobj=f, mode='wb') gzip_file.write(json_s.encode('utf-8')) gzip_file.close() r = WPTReport() with open(tmp_path, 'rb') as f: r.load_gzip_json(f) self.assertDictEqual(r._report, obj)
def test_load_json_multiple_chunks_conflicting_data(self): tmp_path = os.path.join(self.tmp_dir, 'test.json') r = WPTReport() with open(tmp_path, 'wt') as f: json.dump({ 'results': [{'test1': 'foo'}], 'run_info': { 'revision': '0bdaaf9c1622ca49eb140381af1ece6d8001c934', 'product': 'firefox', 'browser_version': '59', }, }, f) with open(tmp_path, 'rb') as f: r.load_json(f) with open(tmp_path, 'wt') as f: json.dump({ 'results': [{'test2': 'bar'}], 'run_info': { 'revision': '0bdaaf9c1622ca49eb140381af1ece6d8001c934', 'product': 'chrome', 'browser_version': '70', }, }, f) with open(tmp_path, 'rb') as f: with self.assertRaisesRegex(ConflictingDataError, "product, browser_version"): r.load_json(f) # Fields without conflict should be preserved. self.assertEqual(r.run_info['revision'], '0bdaaf9c1622ca49eb140381af1ece6d8001c934') # Conflicting fields should be set to None. self.assertIsNone(r.run_info['product']) self.assertIsNone(r.run_info['browser_version'])
def test_load_json_multiple_chunks_ignored_conflicting_data(self): tmp_path = os.path.join(self.tmp_dir, 'test.json') r = WPTReport() with open(tmp_path, 'wt') as f: json.dump( { 'results': [{ 'test1': 'foo' }], 'run_info': { 'browser_build_id': '1', 'browser_changeset': 'r1', }, }, f) with open(tmp_path, 'rb') as f: r.load_json(f) with open(tmp_path, 'wt') as f: json.dump( { 'results': [{ 'test2': 'bar' }], 'run_info': { 'browser_build_id': '2', 'browser_changeset': 'r2', }, }, f) with open(tmp_path, 'rb') as f: r.load_json(f) self.assertIsNone(r.run_info['browser_build_id']) self.assertIsNone(r.run_info['browser_changeset'])
def test_load_json_multiple_chunks_conflicting_data(self): tmp_path = os.path.join(self.tmp_dir, 'test.json') r = WPTReport() with open(tmp_path, 'wt') as f: json.dump( { 'results': [{ 'test1': 'foo' }], 'run_info': { 'product': 'firefox' }, }, f) with open(tmp_path, 'rb') as f: r.load_json(f) with open(tmp_path, 'wt') as f: json.dump( { 'results': [{ 'test2': 'bar' }], 'run_info': { 'product': 'chrome' }, }, f) with open(tmp_path, 'rb') as f: with self.assertRaises(ConflictingDataError): r.load_json(f)
def test_populate_upload_directory(self): # This also tests write_summary() and write_result_directory(). revision = '0bdaaf9c1622ca49eb140381af1ece6d8001c934' r = WPTReport() r._report = { 'results': [{ 'test': '/foo/bar.html', 'status': 'PASS', 'message': None, 'subtests': [] }], 'run_info': { 'revision': revision, 'product': 'firefox', 'browser_version': '59.0', 'os': 'linux' } } r.hashsum = '0123456789' r.populate_upload_directory(output_dir=self.tmp_dir) self.assertTrue( os.path.isfile( os.path.join(self.tmp_dir, revision, 'firefox-59.0-linux-0123456789-summary.json.gz'))) self.assertTrue( os.path.isfile( os.path.join(self.tmp_dir, revision, 'firefox-59.0-linux-0123456789', 'foo', 'bar.html')))