def test_json_merge_tests_non_dict_exception(self): result_1 = json.loads(self.sample_json_string) result_2 = json.loads(self.sample_json_string) result_1['tests'] = json.loads(''' { "Benchmark-1": { "Story-1": { "actual": "PASS" } } } ''') result_2['tests'] = json.loads(''' { "Benchmark-1": { "Story-1": { "actual": "FAIL" } } } ''') with self.assertRaises(results_merger.MergeException) as c: results_merger.merge_test_results([result_1, result_2]) self.assertTrue( 'not mergable' in str(c.exception), 'Merge failure message is not in exception. Exception: %s' % c.exception)
def test_json_required_field_check_exception(self): json_string = '{"seconds_since_epoch": 1.0, "version": 3}' result = json.loads(json_string) with self.assertRaises(results_merger.MergeException) as c: results_merger.merge_test_results([result]) self.assertTrue( 'Invalid json test results' in str(c.exception), 'Required key check failure message is not in exception. Exception: %s' % c.exception)
def test_json_version_check_exception(self): json_string = '{"seconds_since_epoch": 1.0, "version": 2}' result = json.loads(json_string) with self.assertRaises(results_merger.MergeException) as c: results_merger.merge_test_results([result]) self.assertTrue( 'Unsupported version' in str(c.exception), 'Version check failure message is not in exception. Exception: %s' % c.exception)
def _merge_json_output(output_json, jsons_to_merge, extra_links, test_cross_device=False): """Merges the contents of one or more results JSONs. Args: output_json: A path to a JSON file to which the merged results should be written. jsons_to_merge: A list of JSON files that should be merged. extra_links: a (key, value) map in which keys are the human-readable strings which describe the data, and value is logdog url that contain the data. """ begin_time = time.time() merged_results = results_merger.merge_test_results(jsons_to_merge, test_cross_device) # Only append the perf results links if present if extra_links: merged_results['links'] = extra_links with open(output_json, 'w') as f: json.dump(merged_results, f) end_time = time.time() print_duration('Merging json test results', begin_time, end_time) return 0
def test_json_merge_interrupted(self): result_1 = json.loads(self.sample_json_string) result_2 = json.loads(self.sample_json_string) result_2['interrupted'] = True merged_results = results_merger.merge_test_results( [result_1, result_2]) self.assertEqual(merged_results['interrupted'], True)
def test_json_merge_seconds_since_epoch(self): result_1 = json.loads(self.sample_json_string) result_2 = json.loads(self.sample_json_string) result_2['seconds_since_epoch'] = 5.0 merged_results = results_merger.merge_test_results( [result_1, result_2]) self.assertEqual(merged_results['seconds_since_epoch'], 5.0)
def test_json_merge_tests_cross_device_times_missing(self): result_1 = json.loads(self.sample_json_string) result_2 = json.loads(self.sample_json_string) result_1['tests'] = json.loads(''' { "Benchmark-1": { "Story-1": { "actual": "PASS", "expected": "PASS" } } } ''') result_2['tests'] = json.loads(''' { "Benchmark-1": { "Story-1": { "actual": "PASS", "expected": "PASS", "time": 20.0, "times": [20.0, 30.0] } } } ''') merged_results = results_merger.merge_test_results( [result_1, result_2], True) self.assertEqual( 2, len(merged_results['tests']['Benchmark-1']['Story-1']['times'])) self.assertEqual( 20.0, merged_results['tests']['Benchmark-1']['Story-1']['time'])
def test_json_merge_tests_cross_device_actual_pass(self): result_1 = json.loads(self.sample_json_string) result_2 = json.loads(self.sample_json_string) result_1['tests'] = json.loads(''' { "Benchmark-1": { "Story-1": { "actual": "PASS", "expected": "PASS", "is_unexpected": false } } } ''') result_2['tests'] = json.loads(''' { "Benchmark-1": { "Story-1": { "actual": "PASS", "expected": "PASS", "is_unexpected": false } } } ''') merged_results = results_merger.merge_test_results( [result_1, result_2], True) self.assertEqual( 'PASS PASS', merged_results['tests']['Benchmark-1']['Story-1']['actual']) self.assertEqual( False, merged_results['tests']['Benchmark-1']['Story-1']['is_unexpected'])
def _merge_json_output(output_json, jsons_to_merge): """Merges the contents of one or more results JSONs. Args: output_json: A path to a JSON file to which the merged results should be written. jsons_to_merge: A list of JSON files that should be merged. """ merged_results = results_merger.merge_test_results(jsons_to_merge) with open(output_json, 'w') as f: json.dump(merged_results, f) return 0
def test_json_merge_tests(self): result_1 = json.loads(self.sample_json_string) result_2 = json.loads(self.sample_json_string) result_3 = json.loads(self.sample_json_string) result_1['tests'] = json.loads(''' { "Benchmark-1": { "Story-1": { "actual": "PASS" }, "Story-2": { "actual": "SKIP" } } } ''') result_2['tests'] = json.loads(''' { "Benchmark-1": { "Story-3": { "actual": "FAIL" } }, "Benchmark-2": { "Story-1": { "actual": "SKIP" } } } ''') result_3['tests'] = json.loads(''' { "Benchmark-2": { "Story-2": { "actual": "PASS" } }, "Benchmark-3": { "Story-1": { "actual": "PASS" } } } ''') merged_results = results_merger.merge_test_results( [result_1, result_2, result_3]) self.assertEqual(len(merged_results['tests']), 3) self.assertEqual(len(merged_results['tests']['Benchmark-1']), 3) self.assertEqual(len(merged_results['tests']['Benchmark-2']), 2) self.assertEqual(len(merged_results['tests']['Benchmark-3']), 1)
def test_json_merge_tests_cross_device_artifacts(self): result_1 = json.loads(self.sample_json_string) result_2 = json.loads(self.sample_json_string) result_1['tests'] = json.loads(''' { "Benchmark-1": { "Story-1": { "actual": "PASS", "expected": "PASS", "artifacts": { "logs.txt": [ "123/1/logs.txt" ] } } } } ''') result_2['tests'] = json.loads(''' { "Benchmark-1": { "Story-1": { "actual": "PASS", "expected": "PASS", "artifacts": { "logs.txt": [ "456/1/logs.txt" ], "trace.html": [ "123/1/trace.html" ] } } } } ''') merged_results = results_merger.merge_test_results( [result_1, result_2], True) self.assertEqual( 2, len(merged_results['tests']['Benchmark-1']['Story-1']['artifacts'] ['logs.txt'])) self.assertEqual( 1, len(merged_results['tests']['Benchmark-1']['Story-1']['artifacts'] ['trace.html']))
def _merge_json_output(output_json, jsons_to_merge, perf_results_link, perf_results_file_name): """Merges the contents of one or more results JSONs. Args: output_json: A path to a JSON file to which the merged results should be written. jsons_to_merge: A list of JSON files that should be merged. perf_results_link: A link for the logdog mapping of benchmarks to logs. """ merged_results = results_merger.merge_test_results(jsons_to_merge) merged_results['links'] = {perf_results_file_name: perf_results_link} with open(output_json, 'w') as f: json.dump(merged_results, f) return 0
def test_json_merge_nums(self): result_1 = json.loads(self.sample_json_string) result_2 = json.loads(self.sample_json_string) result_1['num_failures_by_type'] = json.loads(''' { "PASS": 1, "SKIP": 5 } ''') result_2['num_failures_by_type'] = json.loads(''' { "PASS": 3, "FAIL": 2 } ''') merged_results = results_merger.merge_test_results( [result_1, result_2]) self.assertEqual(merged_results['num_failures_by_type']['PASS'], 4) self.assertEqual(merged_results['num_failures_by_type']['SKIP'], 5) self.assertEqual(merged_results['num_failures_by_type']['FAIL'], 2)
def test_json_merge_tests_cross_device(self): result_1 = json.loads(self.sample_json_string) result_2 = json.loads(self.sample_json_string) result_1['tests'] = json.loads(''' { "Benchmark-1": { "Story-1": { "actual": "PASS PASS", "artifacts": { "logs.txt": [ "123/1/logs.txt", "123/2/logs.txt" ], "trace.html": [ "123/1/trace.html", "123/2/trace.html" ] }, "expected": "PASS", "is_unexpected": false, "shard": 0, "time": 1.0, "times": [ 1.0, 1.1 ] } } } ''') result_2['tests'] = json.loads(''' { "Benchmark-1": { "Story-1": { "actual": "FAIL PASS", "artifacts": { "logs.txt": [ "456/1/logs.txt", "456/2/logs.txt" ], "screenshot.png": [ "456/1/screenshot.png" ] }, "expected": "PASS", "is_unexpected": true, "shard": 1, "time": 1.0, "times": [ 1.0, 1.2 ] } } } ''') merged_results = results_merger.merge_test_results( [result_1, result_2], True) self.assertEqual(len(merged_results['tests']), 1) self.assertEqual(len(merged_results['tests']['Benchmark-1']), 1) self.assertIn( 'FAIL', merged_results['tests']['Benchmark-1']['Story-1'] ['actual'].split()) self.assertIn( 'PASS', merged_results['tests']['Benchmark-1']['Story-1'] ['actual'].split()) self.assertEqual( 4, len(merged_results['tests']['Benchmark-1']['Story-1']['artifacts'] ['logs.txt'])) self.assertEqual( 2, len(merged_results['tests']['Benchmark-1']['Story-1']['artifacts'] ['trace.html'])) self.assertEqual( 1, len(merged_results['tests']['Benchmark-1']['Story-1']['artifacts'] ['screenshot.png'])) self.assertEqual( 4, len(merged_results['tests']['Benchmark-1']['Story-1']['times'])) self.assertNotIn('shard', merged_results['tests']['Benchmark-1']['Story-1']) self.assertEqual( True, merged_results['tests']['Benchmark-1']['Story-1']['is_unexpected'])