def test_json_merge_tests_non_dict_exception(self):
result_1 = json.loads(self.sample_json_string)
result_2 = json.loads(self.sample_json_string)
result_1['tests'] = json.loads('''
{
"Benchmark-1": {
"Story-1": {
"actual": "PASS"
}
}
}
''')
result_2['tests'] = json.loads('''
{
"Benchmark-1": {
"Story-1": {
"actual": "FAIL"
}
}
}
''')
with self.assertRaises(results_merger.MergeException) as c:
results_merger.merge_test_results([result_1, result_2])
self.assertTrue(
'not mergable' in str(c.exception),
'Merge failure message is not in exception. Exception: %s' %
c.exception)
def test_json_required_field_check_exception(self):
json_string = '{"seconds_since_epoch": 1.0, "version": 3}'
result = json.loads(json_string)
with self.assertRaises(results_merger.MergeException) as c:
results_merger.merge_test_results([result])
self.assertTrue(
'Invalid json test results' in str(c.exception),
'Required key check failure message is not in exception. Exception: %s'
% c.exception)
def test_json_version_check_exception(self):
json_string = '{"seconds_since_epoch": 1.0, "version": 2}'
result = json.loads(json_string)
with self.assertRaises(results_merger.MergeException) as c:
results_merger.merge_test_results([result])
self.assertTrue(
'Unsupported version' in str(c.exception),
'Version check failure message is not in exception. Exception: %s'
% c.exception)
def _merge_json_output(output_json,
jsons_to_merge,
extra_links,
test_cross_device=False):
"""Merges the contents of one or more results JSONs.
Args:
output_json: A path to a JSON file to which the merged results should be
written.
jsons_to_merge: A list of JSON files that should be merged.
extra_links: a (key, value) map in which keys are the human-readable strings
which describe the data, and value is logdog url that contain the data.
"""
begin_time = time.time()
merged_results = results_merger.merge_test_results(jsons_to_merge,
test_cross_device)
# Only append the perf results links if present
if extra_links:
merged_results['links'] = extra_links
with open(output_json, 'w') as f:
json.dump(merged_results, f)
end_time = time.time()
print_duration('Merging json test results', begin_time, end_time)
return 0
def test_json_merge_interrupted(self):
result_1 = json.loads(self.sample_json_string)
result_2 = json.loads(self.sample_json_string)
result_2['interrupted'] = True
merged_results = results_merger.merge_test_results(
[result_1, result_2])
self.assertEqual(merged_results['interrupted'], True)
def test_json_merge_seconds_since_epoch(self):
result_1 = json.loads(self.sample_json_string)
result_2 = json.loads(self.sample_json_string)
result_2['seconds_since_epoch'] = 5.0
merged_results = results_merger.merge_test_results(
[result_1, result_2])
self.assertEqual(merged_results['seconds_since_epoch'], 5.0)
def test_json_merge_tests_cross_device_times_missing(self):
result_1 = json.loads(self.sample_json_string)
result_2 = json.loads(self.sample_json_string)
result_1['tests'] = json.loads('''
{
"Benchmark-1": {
"Story-1": {
"actual": "PASS",
"expected": "PASS"
}
}
}
''')
result_2['tests'] = json.loads('''
{
"Benchmark-1": {
"Story-1": {
"actual": "PASS",
"expected": "PASS",
"time": 20.0,
"times": [20.0, 30.0]
}
}
}
''')
merged_results = results_merger.merge_test_results(
[result_1, result_2], True)
self.assertEqual(
2, len(merged_results['tests']['Benchmark-1']['Story-1']['times']))
self.assertEqual(
20.0, merged_results['tests']['Benchmark-1']['Story-1']['time'])
def test_json_merge_tests_cross_device_actual_pass(self):
result_1 = json.loads(self.sample_json_string)
result_2 = json.loads(self.sample_json_string)
result_1['tests'] = json.loads('''
{
"Benchmark-1": {
"Story-1": {
"actual": "PASS",
"expected": "PASS",
"is_unexpected": false
}
}
}
''')
result_2['tests'] = json.loads('''
{
"Benchmark-1": {
"Story-1": {
"actual": "PASS",
"expected": "PASS",
"is_unexpected": false
}
}
}
''')
merged_results = results_merger.merge_test_results(
[result_1, result_2], True)
self.assertEqual(
'PASS PASS',
merged_results['tests']['Benchmark-1']['Story-1']['actual'])
self.assertEqual(
False,
merged_results['tests']['Benchmark-1']['Story-1']['is_unexpected'])
def _merge_json_output(output_json, jsons_to_merge):
"""Merges the contents of one or more results JSONs.
Args:
output_json: A path to a JSON file to which the merged results should be
written.
jsons_to_merge: A list of JSON files that should be merged.
"""
merged_results = results_merger.merge_test_results(jsons_to_merge)
with open(output_json, 'w') as f:
json.dump(merged_results, f)
return 0
def test_json_merge_tests(self):
result_1 = json.loads(self.sample_json_string)
result_2 = json.loads(self.sample_json_string)
result_3 = json.loads(self.sample_json_string)
result_1['tests'] = json.loads('''
{
"Benchmark-1": {
"Story-1": {
"actual": "PASS"
},
"Story-2": {
"actual": "SKIP"
}
}
}
''')
result_2['tests'] = json.loads('''
{
"Benchmark-1": {
"Story-3": {
"actual": "FAIL"
}
},
"Benchmark-2": {
"Story-1": {
"actual": "SKIP"
}
}
}
''')
result_3['tests'] = json.loads('''
{
"Benchmark-2": {
"Story-2": {
"actual": "PASS"
}
},
"Benchmark-3": {
"Story-1": {
"actual": "PASS"
}
}
}
''')
merged_results = results_merger.merge_test_results(
[result_1, result_2, result_3])
self.assertEqual(len(merged_results['tests']), 3)
self.assertEqual(len(merged_results['tests']['Benchmark-1']), 3)
self.assertEqual(len(merged_results['tests']['Benchmark-2']), 2)
self.assertEqual(len(merged_results['tests']['Benchmark-3']), 1)
def test_json_merge_tests_cross_device_artifacts(self):
result_1 = json.loads(self.sample_json_string)
result_2 = json.loads(self.sample_json_string)
result_1['tests'] = json.loads('''
{
"Benchmark-1": {
"Story-1": {
"actual": "PASS",
"expected": "PASS",
"artifacts": {
"logs.txt": [
"123/1/logs.txt"
]
}
}
}
}
''')
result_2['tests'] = json.loads('''
{
"Benchmark-1": {
"Story-1": {
"actual": "PASS",
"expected": "PASS",
"artifacts": {
"logs.txt": [
"456/1/logs.txt"
],
"trace.html": [
"123/1/trace.html"
]
}
}
}
}
''')
merged_results = results_merger.merge_test_results(
[result_1, result_2], True)
self.assertEqual(
2,
len(merged_results['tests']['Benchmark-1']['Story-1']['artifacts']
['logs.txt']))
self.assertEqual(
1,
len(merged_results['tests']['Benchmark-1']['Story-1']['artifacts']
['trace.html']))
def _merge_json_output(output_json, jsons_to_merge, perf_results_link,
perf_results_file_name):
"""Merges the contents of one or more results JSONs.
Args:
output_json: A path to a JSON file to which the merged results should be
written.
jsons_to_merge: A list of JSON files that should be merged.
perf_results_link: A link for the logdog mapping of benchmarks to logs.
"""
merged_results = results_merger.merge_test_results(jsons_to_merge)
merged_results['links'] = {perf_results_file_name: perf_results_link}
with open(output_json, 'w') as f:
json.dump(merged_results, f)
return 0
def test_json_merge_nums(self):
result_1 = json.loads(self.sample_json_string)
result_2 = json.loads(self.sample_json_string)
result_1['num_failures_by_type'] = json.loads('''
{
"PASS": 1,
"SKIP": 5
}
''')
result_2['num_failures_by_type'] = json.loads('''
{
"PASS": 3,
"FAIL": 2
}
''')
merged_results = results_merger.merge_test_results(
[result_1, result_2])
self.assertEqual(merged_results['num_failures_by_type']['PASS'], 4)
self.assertEqual(merged_results['num_failures_by_type']['SKIP'], 5)
self.assertEqual(merged_results['num_failures_by_type']['FAIL'], 2)
def test_json_merge_tests_cross_device(self):
result_1 = json.loads(self.sample_json_string)
result_2 = json.loads(self.sample_json_string)
result_1['tests'] = json.loads('''
{
"Benchmark-1": {
"Story-1": {
"actual": "PASS PASS",
"artifacts": {
"logs.txt": [
"123/1/logs.txt",
"123/2/logs.txt"
],
"trace.html": [
"123/1/trace.html",
"123/2/trace.html"
]
},
"expected": "PASS",
"is_unexpected": false,
"shard": 0,
"time": 1.0,
"times": [
1.0,
1.1
]
}
}
}
''')
result_2['tests'] = json.loads('''
{
"Benchmark-1": {
"Story-1": {
"actual": "FAIL PASS",
"artifacts": {
"logs.txt": [
"456/1/logs.txt",
"456/2/logs.txt"
],
"screenshot.png": [
"456/1/screenshot.png"
]
},
"expected": "PASS",
"is_unexpected": true,
"shard": 1,
"time": 1.0,
"times": [
1.0,
1.2
]
}
}
}
''')
merged_results = results_merger.merge_test_results(
[result_1, result_2], True)
self.assertEqual(len(merged_results['tests']), 1)
self.assertEqual(len(merged_results['tests']['Benchmark-1']), 1)
self.assertIn(
'FAIL', merged_results['tests']['Benchmark-1']['Story-1']
['actual'].split())
self.assertIn(
'PASS', merged_results['tests']['Benchmark-1']['Story-1']
['actual'].split())
self.assertEqual(
4,
len(merged_results['tests']['Benchmark-1']['Story-1']['artifacts']
['logs.txt']))
self.assertEqual(
2,
len(merged_results['tests']['Benchmark-1']['Story-1']['artifacts']
['trace.html']))
self.assertEqual(
1,
len(merged_results['tests']['Benchmark-1']['Story-1']['artifacts']
['screenshot.png']))
self.assertEqual(
4, len(merged_results['tests']['Benchmark-1']['Story-1']['times']))
self.assertNotIn('shard',
merged_results['tests']['Benchmark-1']['Story-1'])
self.assertEqual(
True,
merged_results['tests']['Benchmark-1']['Story-1']['is_unexpected'])
