def test_perf_metrics_for_test(self):
individual_test_timings = []
individual_test_timings.append(
json_results_generator.TestResult('foo/bar/baz.html',
elapsed_time=1.2))
individual_test_timings.append(
json_results_generator.TestResult('foo/bar/ba.html',
elapsed_time=1.4))
individual_test_timings.append(
json_results_generator.TestResult('bar.html', elapsed_time=0.0001))
metrics = json_results_generator.perf_metrics_for_test(
1200, individual_test_timings)
expected_metrics = {
"layout_tests": {
"metrics": {
"Time": ["Total", "Arithmetic"],
},
"tests": {
"foo": {
"metrics": {
"Time": ["Total", "Arithmetic"],
},
"tests": {
"bar": {
"metrics": {
"Time": {
"current": [2600]
},
}
}
}
},
"bar.html": {
"metrics": {
"Time": {
"current": [0]
},
}
}
}
},
"layout_tests_run_time": {
"metrics": {
"Time": {
"current": [1200]
},
}
}
}
self.assertEqual(json.dumps(metrics), json.dumps(expected_metrics))
def AddResults(self, test_results):
# TODO(frankf): Differentiate between fail/crash/timeouts.
conversion_map = [
(test_results.GetPass(), False,
json_results_generator.JSONResultsGeneratorBase.PASS_RESULT),
(test_results.GetFail(), True,
json_results_generator.JSONResultsGeneratorBase.FAIL_RESULT),
(test_results.GetCrash(), True,
json_results_generator.JSONResultsGeneratorBase.FAIL_RESULT),
(test_results.GetTimeout(), True,
json_results_generator.JSONResultsGeneratorBase.FAIL_RESULT),
(test_results.GetUnknown(), True,
json_results_generator.JSONResultsGeneratorBase.NO_DATA_RESULT),
]
for results_list, failed, modifier in conversion_map:
for single_test_result in results_list:
test_result = json_results_generator.TestResult(
test=single_test_result.GetName(),
failed=failed,
elapsed_time=single_test_result.GetDur() / 1000)
# The WebKit TestResult object sets the modifier it based on test name.
# Since we don't use the same test naming convention as WebKit the
# modifier will be wrong, so we need to overwrite it.
test_result.modifier = modifier
self._test_results_map[single_test_result.GetName()] = test_result
def test_test_timings_trie(self):
test_port = test.TestPort(MockHost())
individual_test_timings = []
individual_test_timings.append(json_results_generator.TestResult('foo/bar/baz.html', elapsed_time=1.2))
individual_test_timings.append(json_results_generator.TestResult('bar.html', elapsed_time=0.0001))
trie = json_results_generator.test_timings_trie(test_port, individual_test_timings)
expected_trie = {
'bar.html': 0,
'foo': {
'bar': {
'baz.html': 1200,
}
}
}
self.assertEqual(json.dumps(trie), json.dumps(expected_trie))
def _test_json_generation(self, passed_tests_list, failed_tests_list):
tests_set = set(passed_tests_list) | set(failed_tests_list)
DISABLED_tests = set([t for t in tests_set
if t.startswith('DISABLED_')])
FLAKY_tests = set([t for t in tests_set
if t.startswith('FLAKY_')])
FAILS_tests = set([t for t in tests_set
if t.startswith('FAILS_')])
PASS_tests = tests_set - (DISABLED_tests | FLAKY_tests | FAILS_tests)
failed_tests = set(failed_tests_list) - DISABLED_tests
failed_count_map = dict([(t, 1) for t in failed_tests])
test_timings = {}
i = 0
for test in tests_set:
test_timings[test] = float(self._num_runs * 100 + i)
i += 1
test_results_map = dict()
for test in tests_set:
test_results_map[test] = json_results_generator.TestResult(test,
failed=(test in failed_tests),
elapsed_time=test_timings[test])
host = MockHost()
port = Mock()
port._filesystem = host.filesystem
generator = json_results_generator.JSONResultsGeneratorBase(port,
self.builder_name, self.build_name, self.build_number,
'',
None, # don't fetch past json results archive
test_results_map)
failed_count_map = dict([(t, 1) for t in failed_tests])
# Test incremental json results
incremental_json = generator.get_json()
self._verify_json_results(
tests_set,
test_timings,
failed_count_map,
len(PASS_tests),
len(DISABLED_tests),
len(FLAKY_tests),
len(DISABLED_tests | failed_tests),
incremental_json,
1)
# We don't verify the results here, but at least we make sure the code runs without errors.
generator.generate_json_output()
generator.generate_times_ms_file()
def AddResults(self, test_results):
conversion_map = [
(test_results.ok, False,
json_results_generator.JSONResultsGeneratorBase.PASS_RESULT),
(test_results.failed, True,
json_results_generator.JSONResultsGeneratorBase.FAIL_RESULT),
(test_results.crashed, True, "C"),
(test_results.unknown, True,
json_results_generator.JSONResultsGeneratorBase.NO_DATA_RESULT),
]
for results_list, failed, modifier in conversion_map:
for single_test_result in results_list:
test_result = json_results_generator.TestResult(
test=single_test_result.name,
failed=failed,
elapsed_time=single_test_result.dur / 1000)
# The WebKit TestResult object sets the modifier it based on test name.
# Since we don't use the same test naming convention as WebKit the
# modifier will be wrong, so we need to overwrite it.
test_result.modifier = modifier
self._test_results_map[single_test_result.name] = test_result
