def _write_json_files(self, summarized_full_results, summarized_failing_results, initial_results, running_all_tests):
_log.debug("Writing JSON files in %s.", self._results_directory)
# FIXME: Upload stats.json to the server and delete times_ms.
times_trie = json_results_generator.test_timings_trie(initial_results.results_by_name.values())
times_json_path = self._filesystem.join(self._results_directory, "times_ms.json")
json_results_generator.write_json(self._filesystem, times_trie, times_json_path)
# Save out the times data so we can use it for --fastest in the future.
if running_all_tests:
bot_test_times_path = self._port.bot_test_times_path()
self._filesystem.maybe_make_directory(self._filesystem.dirname(bot_test_times_path))
json_results_generator.write_json(self._filesystem, times_trie, bot_test_times_path)
stats_trie = self._stats_trie(initial_results)
stats_path = self._filesystem.join(self._results_directory, "stats.json")
self._filesystem.write_text_file(stats_path, json.dumps(stats_trie))
full_results_path = self._filesystem.join(self._results_directory, "full_results.json")
json_results_generator.write_json(self._filesystem, summarized_full_results, full_results_path)
full_results_path = self._filesystem.join(self._results_directory, "failing_results.json")
# We write failing_results.json out as jsonp because we need to load it
# from a file url for results.html and Chromium doesn't allow that.
json_results_generator.write_json(self._filesystem, summarized_failing_results, full_results_path, callback="ADD_RESULTS")
if self._options.json_test_results:
json_results_generator.write_json(self._filesystem, summarized_failing_results, self._options.json_test_results)
_log.debug("Finished writing JSON files.")
def _write_json_files(self, summarized_full_results,
summarized_failing_results, initial_results):
_log.debug("Writing JSON files in %s." % self._results_directory)
# FIXME: Upload stats.json to the server and delete times_ms.
times_trie = json_results_generator.test_timings_trie(
self._port, initial_results.results_by_name.values())
times_json_path = self._filesystem.join(self._results_directory,
"times_ms.json")
json_results_generator.write_json(self._filesystem, times_trie,
times_json_path)
stats_trie = self._stats_trie(initial_results)
stats_path = self._filesystem.join(self._results_directory,
"stats.json")
self._filesystem.write_text_file(stats_path, json.dumps(stats_trie))
full_results_path = self._filesystem.join(self._results_directory,
"full_results.json")
json_results_generator.write_json(self._filesystem,
summarized_full_results,
full_results_path)
full_results_path = self._filesystem.join(self._results_directory,
"failing_results.json")
# We write failing_results.json out as jsonp because we need to load it from a file url for results.html and Chromium doesn't allow that.
json_results_generator.write_json(self._filesystem,
summarized_failing_results,
full_results_path,
callback="ADD_RESULTS")
_log.debug("Finished writing JSON files.")
def test_test_timings_trie(self):
test_port = test.TestPort(MockHost())
individual_test_timings = []
individual_test_timings.append(json_results_generator.TestResult("foo/bar/baz.html", elapsed_time=1.2))
individual_test_timings.append(json_results_generator.TestResult("bar.html", elapsed_time=0.0001))
trie = json_results_generator.test_timings_trie(test_port, individual_test_timings)
expected_trie = {"bar.html": 0, "foo": {"bar": {"baz.html": 1200}}}
self.assertEqual(json.dumps(trie), json.dumps(expected_trie))
def _upload_json_files(self, summarized_results, initial_results, results_including_passes=None, start_time=None, end_time=None):
"""Writes the results of the test run as JSON files into the results
dir and upload the files to the appengine server.
Args:
summarized_results: dict of results
initial_results: full summary object
"""
_log.debug("Writing JSON files in %s." % self._results_directory)
# FIXME: Upload stats.json to the server and delete times_ms.
times_trie = json_results_generator.test_timings_trie(self._port, initial_results.results_by_name.values())
times_json_path = self._filesystem.join(self._results_directory, "times_ms.json")
json_results_generator.write_json(self._filesystem, times_trie, times_json_path)
stats_trie = self._stats_trie(initial_results)
stats_path = self._filesystem.join(self._results_directory, "stats.json")
self._filesystem.write_text_file(stats_path, json.dumps(stats_trie))
full_results_path = self._filesystem.join(self._results_directory, "full_results.json")
# We write full_results.json out as jsonp because we need to load it from a file url and Chromium doesn't allow that.
json_results_generator.write_json(self._filesystem, summarized_results, full_results_path, callback="ADD_RESULTS")
results_json_path = self._filesystem.join(self._results_directory, "results_including_passes.json")
if results_including_passes:
json_results_generator.write_json(self._filesystem, results_including_passes, results_json_path)
generator = json_layout_results_generator.JSONLayoutResultsGenerator(
self._port, self._options.builder_name, self._options.build_name,
self._options.build_number, self._results_directory,
self._expectations, initial_results,
self._options.test_results_server,
"layout-tests",
self._options.master_name)
if generator.generate_json_output():
_log.debug("Finished writing JSON file for the test results server.")
else:
_log.debug("Failed to generate JSON file for the test results server.")
return
json_files = ["incremental_results.json", "full_results.json", "times_ms.json"]
generator.upload_json_files(json_files)
if results_including_passes:
self.upload_results(results_json_path, start_time, end_time)
incremental_results_path = self._filesystem.join(self._results_directory, "incremental_results.json")
# Remove these files from the results directory so they don't take up too much space on the buildbot.
# The tools use the version we uploaded to the results server anyway.
self._filesystem.remove(times_json_path)
self._filesystem.remove(incremental_results_path)
if results_including_passes:
self._filesystem.remove(results_json_path)
def _upload_json_files(self, summarized_results, initial_results, results_including_passes=None, start_time=None, end_time=None):
"""Writes the results of the test run as JSON files into the results
dir and upload the files to the appengine server.
Args:
summarized_results: dict of results
initial_results: full summary object
"""
_log.debug("Writing JSON files in %s." % self._results_directory)
# FIXME: Upload stats.json to the server and delete times_ms.
times_trie = json_results_generator.test_timings_trie(self._port, initial_results.results_by_name.values())
times_json_path = self._filesystem.join(self._results_directory, "times_ms.json")
json_results_generator.write_json(self._filesystem, times_trie, times_json_path)
stats_trie = self._stats_trie(initial_results)
stats_path = self._filesystem.join(self._results_directory, "stats.json")
self._filesystem.write_text_file(stats_path, json.dumps(stats_trie))
full_results_path = self._filesystem.join(self._results_directory, "full_results.json")
# We write full_results.json out as jsonp because we need to load it from a file url and Chromium doesn't allow that.
json_results_generator.write_json(self._filesystem, summarized_results, full_results_path, callback="ADD_RESULTS")
results_json_path = self._filesystem.join(self._results_directory, "results_including_passes.json")
if results_including_passes:
json_results_generator.write_json(self._filesystem, results_including_passes, results_json_path)
generator = json_layout_results_generator.JSONLayoutResultsGenerator(
self._port, self._options.builder_name, self._options.build_name,
self._options.build_number, self._results_directory,
self._expectations, initial_results,
self._options.test_results_server,
"layout-tests",
self._options.master_name)
if generator.generate_json_output():
_log.debug("Finished writing JSON file for the test results server.")
else:
_log.debug("Failed to generate JSON file for the test results server.")
return
json_files = ["incremental_results.json", "full_results.json", "times_ms.json"]
generator.upload_json_files(json_files)
if results_including_passes:
self.upload_results(results_json_path, start_time, end_time)
incremental_results_path = self._filesystem.join(self._results_directory, "incremental_results.json")
# Remove these files from the results directory so they don't take up too much space on the buildbot.
# The tools use the version we uploaded to the results server anyway.
self._filesystem.remove(times_json_path)
self._filesystem.remove(incremental_results_path)
if results_including_passes:
self._filesystem.remove(results_json_path)
def _upload_json_files(self, summarized_results, result_summary,
individual_test_timings):
"""Writes the results of the test run as JSON files into the results
dir and upload the files to the appengine server.
Args:
unexpected_results: dict of unexpected results
summarized_results: dict of results
result_summary: full summary object
individual_test_timings: list of test times (used by the flakiness
dashboard).
"""
_log.debug("Writing JSON files in %s." % self._results_directory)
times_trie = json_results_generator.test_timings_trie(
self._port, individual_test_timings)
times_json_path = self._filesystem.join(self._results_directory,
"times_ms.json")
json_results_generator.write_json(self._filesystem, times_trie,
times_json_path)
full_results_path = self._filesystem.join(self._results_directory,
"full_results.json")
# We write full_results.json out as jsonp because we need to load it from a file url and Chromium doesn't allow that.
json_results_generator.write_json(
self._filesystem,
summarized_results,
full_results_path,
callback="ADD_RESULTS")
generator = json_layout_results_generator.JSONLayoutResultsGenerator(
self._port, self._options.builder_name, self._options.build_name,
self._options.build_number, self._results_directory,
BUILDER_BASE_URL, individual_test_timings, self._expectations,
result_summary, self._test_names,
self._options.test_results_server, "layout-tests",
self._options.master_name)
_log.debug("Finished writing JSON files.")
json_files = [
"incremental_results.json", "full_results.json", "times_ms.json"
]
generator.upload_json_files(json_files)
incremental_results_path = self._filesystem.join(
self._results_directory, "incremental_results.json")
# Remove these files from the results directory so they don't take up too much space on the buildbot.
# The tools use the version we uploaded to the results server anyway.
self._filesystem.remove(times_json_path)
self._filesystem.remove(incremental_results_path)
def _upload_json_files(self, summarized_results, result_summary,
individual_test_timings):
"""Writes the results of the test run as JSON files into the results
dir and upload the files to the appengine server.
Args:
unexpected_results: dict of unexpected results
summarized_results: dict of results
result_summary: full summary object
individual_test_timings: list of test times (used by the flakiness
dashboard).
"""
_log.debug("Writing JSON files in %s." % self._results_directory)
times_trie = json_results_generator.test_timings_trie(
self._port, individual_test_timings)
times_json_path = self._filesystem.join(self._results_directory,
"times_ms.json")
json_results_generator.write_json(self._filesystem, times_trie,
times_json_path)
full_results_path = self._filesystem.join(self._results_directory,
"full_results.json")
# We write full_results.json out as jsonp because we need to load it from a file url and Chromium doesn't allow that.
json_results_generator.write_json(self._filesystem,
summarized_results,
full_results_path,
callback="ADD_RESULTS")
generator = json_layout_results_generator.JSONLayoutResultsGenerator(
self._port, self._options.builder_name, self._options.build_name,
self._options.build_number, self._results_directory,
BUILDER_BASE_URL, individual_test_timings, self._expectations,
result_summary, self._test_names,
self._options.test_results_server, "layout-tests",
self._options.master_name)
_log.debug("Finished writing JSON files.")
json_files = [
"incremental_results.json", "full_results.json", "times_ms.json"
]
generator.upload_json_files(json_files)
incremental_results_path = self._filesystem.join(
self._results_directory, "incremental_results.json")
# Remove these files from the results directory so they don't take up too much space on the buildbot.
# The tools use the version we uploaded to the results server anyway.
self._filesystem.remove(times_json_path)
self._filesystem.remove(incremental_results_path)
def test_test_timings_trie(self):
individual_test_timings = []
individual_test_timings.append(json_results_generator.TestResult('foo/bar/baz.html', elapsed_time=1.2))
individual_test_timings.append(json_results_generator.TestResult('bar.html', elapsed_time=0.0001))
trie = json_results_generator.test_timings_trie(individual_test_timings)
expected_trie = {
'bar.html': 0,
'foo': {
'bar': {
'baz.html': 1200,
}
}
}
self.assertEqual(json.dumps(trie), json.dumps(expected_trie))
def _write_json_files(self, summarized_full_results,
summarized_failing_results, initial_results,
running_all_tests):
_log.debug("Writing JSON files in %s.", self._results_directory)
# FIXME: Upload stats.json to the server and delete times_ms.
times_trie = json_results_generator.test_timings_trie(
initial_results.results_by_name.values())
times_json_path = self._filesystem.join(self._results_directory,
"times_ms.json")
json_results_generator.write_json(self._filesystem, times_trie,
times_json_path)
# Save out the times data so we can use it for --fastest in the future.
if running_all_tests:
bot_test_times_path = self._port.bot_test_times_path()
self._filesystem.maybe_make_directory(
self._filesystem.dirname(bot_test_times_path))
json_results_generator.write_json(self._filesystem, times_trie,
bot_test_times_path)
stats_trie = self._stats_trie(initial_results)
stats_path = self._filesystem.join(self._results_directory,
"stats.json")
self._filesystem.write_text_file(stats_path, json.dumps(stats_trie))
full_results_path = self._filesystem.join(self._results_directory,
"full_results.json")
json_results_generator.write_json(self._filesystem,
summarized_full_results,
full_results_path)
full_results_path = self._filesystem.join(self._results_directory,
"failing_results.json")
# We write failing_results.json out as jsonp because we need to load it
# from a file url for results.html and Chromium doesn't allow that.
json_results_generator.write_json(self._filesystem,
summarized_failing_results,
full_results_path,
callback="ADD_RESULTS")
if self._options.json_test_results:
json_results_generator.write_json(self._filesystem,
summarized_failing_results,
self._options.json_test_results)
_log.debug("Finished writing JSON files.")
def test_test_timings_trie(self):
test_port = test.TestPort(MockHost())
individual_test_timings = []
individual_test_timings.append(json_results_generator.TestResult('foo/bar/baz.html', elapsed_time=1.2))
individual_test_timings.append(json_results_generator.TestResult('bar.html', elapsed_time=0.0001))
trie = json_results_generator.test_timings_trie(test_port, individual_test_timings)
expected_trie = {
'bar.html': 0,
'foo': {
'bar': {
'baz.html': 1200,
}
}
}
self.assertEqual(json.dumps(trie), json.dumps(expected_trie))
def test_test_timings_trie(self):
test_port = test.TestPort()
individual_test_timings = []
individual_test_timings.append(json_results_generator.TestResult('/test.checkout/LayoutTests/foo/bar/baz.html', elapsed_time=1.2))
individual_test_timings.append(json_results_generator.TestResult('/test.checkout/LayoutTests/bar.html', elapsed_time=0.0001))
trie = json_results_generator.test_timings_trie(test_port, individual_test_timings)
expected_trie = {
'bar.html': 0,
'foo': {
'bar': {
'baz.html': 1200,
}
}
}
self.assertEqual(simplejson.dumps(trie), simplejson.dumps(expected_trie))
def _write_json_files(self, summarized_full_results, summarized_failing_results, initial_results, running_all_tests):
_log.debug("Writing JSON files in %s.", self._results_directory)
# FIXME: Upload stats.json to the server and delete times_ms.
times_trie = json_results_generator.test_timings_trie(initial_results.results_by_name.values())
times_json_path = self._filesystem.join(self._results_directory, 'times_ms.json')
json_results_generator.write_json(self._filesystem, times_trie, times_json_path)
# Save out the times data so we can use it for --fastest in the future.
if running_all_tests:
bot_test_times_path = self._port.bot_test_times_path()
self._filesystem.maybe_make_directory(self._filesystem.dirname(bot_test_times_path))
json_results_generator.write_json(self._filesystem, times_trie, bot_test_times_path)
stats_trie = self._stats_trie(initial_results)
stats_path = self._filesystem.join(self._results_directory, 'stats.json')
self._filesystem.write_text_file(stats_path, json.dumps(stats_trie))
full_results_path = self._filesystem.join(self._results_directory, 'full_results.json')
json_results_generator.write_json(self._filesystem, summarized_full_results, full_results_path)
full_results_jsonp_path = self._filesystem.join(self._results_directory, 'full_results_jsonp.js')
json_results_generator.write_json(self._filesystem,
summarized_full_results,
full_results_jsonp_path,
callback='ADD_FULL_RESULTS')
full_results_path = self._filesystem.join(self._results_directory, 'failing_results.json')
# We write failing_results.json out as jsonp because we need to load it
# from a file url for results.html and Chromium doesn't allow that.
json_results_generator.write_json(self._filesystem, summarized_failing_results, full_results_path, callback='ADD_RESULTS')
# Write out the JSON files suitable for other tools to process.
# As the output can be quite large (as there are 60k+ tests) we also
# support only outputting the failing results.
if self._options.json_failing_test_results:
# FIXME(tansell): Make sure this includes an *unexpected* results
# (IE Passing when expected to be failing.)
json_results_generator.write_json(self._filesystem, summarized_failing_results, self._options.json_failing_test_results)
if self._options.json_test_results:
json_results_generator.write_json(self._filesystem, summarized_full_results, self._options.json_test_results)
_log.debug('Finished writing JSON files.')
def _write_json_files(self, summarized_full_results, summarized_failing_results, initial_results):
_log.debug("Writing JSON files in %s." % self._results_directory)
# FIXME: Upload stats.json to the server and delete times_ms.
times_trie = json_results_generator.test_timings_trie(initial_results.results_by_name.values())
times_json_path = self._filesystem.join(self._results_directory, "times_ms.json")
json_results_generator.write_json(self._filesystem, times_trie, times_json_path)
stats_trie = self._stats_trie(initial_results)
stats_path = self._filesystem.join(self._results_directory, "stats.json")
self._filesystem.write_text_file(stats_path, json.dumps(stats_trie))
full_results_path = self._filesystem.join(self._results_directory, "full_results.json")
json_results_generator.write_json(self._filesystem, summarized_full_results, full_results_path)
full_results_path = self._filesystem.join(self._results_directory, "failing_results.json")
# We write failing_results.json out as jsonp because we need to load it from a file url for results.html and Chromium doesn't allow that.
json_results_generator.write_json(self._filesystem, summarized_failing_results, full_results_path, callback="ADD_RESULTS")
_log.debug("Finished writing JSON files.")
