def _write_json_files(self, summarized_full_results, summarized_failing_results, initial_results, running_all_tests): _log.debug("Writing JSON files in %s.", self._results_directory) # FIXME: Upload stats.json to the server and delete times_ms. times_trie = json_results_generator.test_timings_trie(initial_results.results_by_name.values()) times_json_path = self._filesystem.join(self._results_directory, "times_ms.json") json_results_generator.write_json(self._filesystem, times_trie, times_json_path) # Save out the times data so we can use it for --fastest in the future. if running_all_tests: bot_test_times_path = self._port.bot_test_times_path() self._filesystem.maybe_make_directory(self._filesystem.dirname(bot_test_times_path)) json_results_generator.write_json(self._filesystem, times_trie, bot_test_times_path) stats_trie = self._stats_trie(initial_results) stats_path = self._filesystem.join(self._results_directory, "stats.json") self._filesystem.write_text_file(stats_path, json.dumps(stats_trie)) full_results_path = self._filesystem.join(self._results_directory, "full_results.json") json_results_generator.write_json(self._filesystem, summarized_full_results, full_results_path) full_results_path = self._filesystem.join(self._results_directory, "failing_results.json") # We write failing_results.json out as jsonp because we need to load it # from a file url for results.html and Chromium doesn't allow that. json_results_generator.write_json(self._filesystem, summarized_failing_results, full_results_path, callback="ADD_RESULTS") if self._options.json_test_results: json_results_generator.write_json(self._filesystem, summarized_failing_results, self._options.json_test_results) _log.debug("Finished writing JSON files.")
def _write_json_files(self, summarized_full_results, summarized_failing_results, initial_results): _log.debug("Writing JSON files in %s." % self._results_directory) # FIXME: Upload stats.json to the server and delete times_ms. times_trie = json_results_generator.test_timings_trie( self._port, initial_results.results_by_name.values()) times_json_path = self._filesystem.join(self._results_directory, "times_ms.json") json_results_generator.write_json(self._filesystem, times_trie, times_json_path) stats_trie = self._stats_trie(initial_results) stats_path = self._filesystem.join(self._results_directory, "stats.json") self._filesystem.write_text_file(stats_path, json.dumps(stats_trie)) full_results_path = self._filesystem.join(self._results_directory, "full_results.json") json_results_generator.write_json(self._filesystem, summarized_full_results, full_results_path) full_results_path = self._filesystem.join(self._results_directory, "failing_results.json") # We write failing_results.json out as jsonp because we need to load it from a file url for results.html and Chromium doesn't allow that. json_results_generator.write_json(self._filesystem, summarized_failing_results, full_results_path, callback="ADD_RESULTS") _log.debug("Finished writing JSON files.")
def test_test_timings_trie(self): test_port = test.TestPort(MockHost()) individual_test_timings = [] individual_test_timings.append(json_results_generator.TestResult("foo/bar/baz.html", elapsed_time=1.2)) individual_test_timings.append(json_results_generator.TestResult("bar.html", elapsed_time=0.0001)) trie = json_results_generator.test_timings_trie(test_port, individual_test_timings) expected_trie = {"bar.html": 0, "foo": {"bar": {"baz.html": 1200}}} self.assertEqual(json.dumps(trie), json.dumps(expected_trie))
def _upload_json_files(self, summarized_results, initial_results, results_including_passes=None, start_time=None, end_time=None): """Writes the results of the test run as JSON files into the results dir and upload the files to the appengine server. Args: summarized_results: dict of results initial_results: full summary object """ _log.debug("Writing JSON files in %s." % self._results_directory) # FIXME: Upload stats.json to the server and delete times_ms. times_trie = json_results_generator.test_timings_trie(self._port, initial_results.results_by_name.values()) times_json_path = self._filesystem.join(self._results_directory, "times_ms.json") json_results_generator.write_json(self._filesystem, times_trie, times_json_path) stats_trie = self._stats_trie(initial_results) stats_path = self._filesystem.join(self._results_directory, "stats.json") self._filesystem.write_text_file(stats_path, json.dumps(stats_trie)) full_results_path = self._filesystem.join(self._results_directory, "full_results.json") # We write full_results.json out as jsonp because we need to load it from a file url and Chromium doesn't allow that. json_results_generator.write_json(self._filesystem, summarized_results, full_results_path, callback="ADD_RESULTS") results_json_path = self._filesystem.join(self._results_directory, "results_including_passes.json") if results_including_passes: json_results_generator.write_json(self._filesystem, results_including_passes, results_json_path) generator = json_layout_results_generator.JSONLayoutResultsGenerator( self._port, self._options.builder_name, self._options.build_name, self._options.build_number, self._results_directory, self._expectations, initial_results, self._options.test_results_server, "layout-tests", self._options.master_name) if generator.generate_json_output(): _log.debug("Finished writing JSON file for the test results server.") else: _log.debug("Failed to generate JSON file for the test results server.") return json_files = ["incremental_results.json", "full_results.json", "times_ms.json"] generator.upload_json_files(json_files) if results_including_passes: self.upload_results(results_json_path, start_time, end_time) incremental_results_path = self._filesystem.join(self._results_directory, "incremental_results.json") # Remove these files from the results directory so they don't take up too much space on the buildbot. # The tools use the version we uploaded to the results server anyway. self._filesystem.remove(times_json_path) self._filesystem.remove(incremental_results_path) if results_including_passes: self._filesystem.remove(results_json_path)
def _upload_json_files(self, summarized_results, result_summary, individual_test_timings): """Writes the results of the test run as JSON files into the results dir and upload the files to the appengine server. Args: unexpected_results: dict of unexpected results summarized_results: dict of results result_summary: full summary object individual_test_timings: list of test times (used by the flakiness dashboard). """ _log.debug("Writing JSON files in %s." % self._results_directory) times_trie = json_results_generator.test_timings_trie( self._port, individual_test_timings) times_json_path = self._filesystem.join(self._results_directory, "times_ms.json") json_results_generator.write_json(self._filesystem, times_trie, times_json_path) full_results_path = self._filesystem.join(self._results_directory, "full_results.json") # We write full_results.json out as jsonp because we need to load it from a file url and Chromium doesn't allow that. json_results_generator.write_json( self._filesystem, summarized_results, full_results_path, callback="ADD_RESULTS") generator = json_layout_results_generator.JSONLayoutResultsGenerator( self._port, self._options.builder_name, self._options.build_name, self._options.build_number, self._results_directory, BUILDER_BASE_URL, individual_test_timings, self._expectations, result_summary, self._test_names, self._options.test_results_server, "layout-tests", self._options.master_name) _log.debug("Finished writing JSON files.") json_files = [ "incremental_results.json", "full_results.json", "times_ms.json" ] generator.upload_json_files(json_files) incremental_results_path = self._filesystem.join( self._results_directory, "incremental_results.json") # Remove these files from the results directory so they don't take up too much space on the buildbot. # The tools use the version we uploaded to the results server anyway. self._filesystem.remove(times_json_path) self._filesystem.remove(incremental_results_path)
def _upload_json_files(self, summarized_results, result_summary, individual_test_timings): """Writes the results of the test run as JSON files into the results dir and upload the files to the appengine server. Args: unexpected_results: dict of unexpected results summarized_results: dict of results result_summary: full summary object individual_test_timings: list of test times (used by the flakiness dashboard). """ _log.debug("Writing JSON files in %s." % self._results_directory) times_trie = json_results_generator.test_timings_trie( self._port, individual_test_timings) times_json_path = self._filesystem.join(self._results_directory, "times_ms.json") json_results_generator.write_json(self._filesystem, times_trie, times_json_path) full_results_path = self._filesystem.join(self._results_directory, "full_results.json") # We write full_results.json out as jsonp because we need to load it from a file url and Chromium doesn't allow that. json_results_generator.write_json(self._filesystem, summarized_results, full_results_path, callback="ADD_RESULTS") generator = json_layout_results_generator.JSONLayoutResultsGenerator( self._port, self._options.builder_name, self._options.build_name, self._options.build_number, self._results_directory, BUILDER_BASE_URL, individual_test_timings, self._expectations, result_summary, self._test_names, self._options.test_results_server, "layout-tests", self._options.master_name) _log.debug("Finished writing JSON files.") json_files = [ "incremental_results.json", "full_results.json", "times_ms.json" ] generator.upload_json_files(json_files) incremental_results_path = self._filesystem.join( self._results_directory, "incremental_results.json") # Remove these files from the results directory so they don't take up too much space on the buildbot. # The tools use the version we uploaded to the results server anyway. self._filesystem.remove(times_json_path) self._filesystem.remove(incremental_results_path)
def test_test_timings_trie(self): individual_test_timings = [] individual_test_timings.append(json_results_generator.TestResult('foo/bar/baz.html', elapsed_time=1.2)) individual_test_timings.append(json_results_generator.TestResult('bar.html', elapsed_time=0.0001)) trie = json_results_generator.test_timings_trie(individual_test_timings) expected_trie = { 'bar.html': 0, 'foo': { 'bar': { 'baz.html': 1200, } } } self.assertEqual(json.dumps(trie), json.dumps(expected_trie))
def _write_json_files(self, summarized_full_results, summarized_failing_results, initial_results, running_all_tests): _log.debug("Writing JSON files in %s.", self._results_directory) # FIXME: Upload stats.json to the server and delete times_ms. times_trie = json_results_generator.test_timings_trie( initial_results.results_by_name.values()) times_json_path = self._filesystem.join(self._results_directory, "times_ms.json") json_results_generator.write_json(self._filesystem, times_trie, times_json_path) # Save out the times data so we can use it for --fastest in the future. if running_all_tests: bot_test_times_path = self._port.bot_test_times_path() self._filesystem.maybe_make_directory( self._filesystem.dirname(bot_test_times_path)) json_results_generator.write_json(self._filesystem, times_trie, bot_test_times_path) stats_trie = self._stats_trie(initial_results) stats_path = self._filesystem.join(self._results_directory, "stats.json") self._filesystem.write_text_file(stats_path, json.dumps(stats_trie)) full_results_path = self._filesystem.join(self._results_directory, "full_results.json") json_results_generator.write_json(self._filesystem, summarized_full_results, full_results_path) full_results_path = self._filesystem.join(self._results_directory, "failing_results.json") # We write failing_results.json out as jsonp because we need to load it # from a file url for results.html and Chromium doesn't allow that. json_results_generator.write_json(self._filesystem, summarized_failing_results, full_results_path, callback="ADD_RESULTS") if self._options.json_test_results: json_results_generator.write_json(self._filesystem, summarized_failing_results, self._options.json_test_results) _log.debug("Finished writing JSON files.")
def test_test_timings_trie(self): test_port = test.TestPort(MockHost()) individual_test_timings = [] individual_test_timings.append(json_results_generator.TestResult('foo/bar/baz.html', elapsed_time=1.2)) individual_test_timings.append(json_results_generator.TestResult('bar.html', elapsed_time=0.0001)) trie = json_results_generator.test_timings_trie(test_port, individual_test_timings) expected_trie = { 'bar.html': 0, 'foo': { 'bar': { 'baz.html': 1200, } } } self.assertEqual(json.dumps(trie), json.dumps(expected_trie))
def test_test_timings_trie(self): test_port = test.TestPort() individual_test_timings = [] individual_test_timings.append(json_results_generator.TestResult('/test.checkout/LayoutTests/foo/bar/baz.html', elapsed_time=1.2)) individual_test_timings.append(json_results_generator.TestResult('/test.checkout/LayoutTests/bar.html', elapsed_time=0.0001)) trie = json_results_generator.test_timings_trie(test_port, individual_test_timings) expected_trie = { 'bar.html': 0, 'foo': { 'bar': { 'baz.html': 1200, } } } self.assertEqual(simplejson.dumps(trie), simplejson.dumps(expected_trie))
def _write_json_files(self, summarized_full_results, summarized_failing_results, initial_results, running_all_tests): _log.debug("Writing JSON files in %s.", self._results_directory) # FIXME: Upload stats.json to the server and delete times_ms. times_trie = json_results_generator.test_timings_trie(initial_results.results_by_name.values()) times_json_path = self._filesystem.join(self._results_directory, 'times_ms.json') json_results_generator.write_json(self._filesystem, times_trie, times_json_path) # Save out the times data so we can use it for --fastest in the future. if running_all_tests: bot_test_times_path = self._port.bot_test_times_path() self._filesystem.maybe_make_directory(self._filesystem.dirname(bot_test_times_path)) json_results_generator.write_json(self._filesystem, times_trie, bot_test_times_path) stats_trie = self._stats_trie(initial_results) stats_path = self._filesystem.join(self._results_directory, 'stats.json') self._filesystem.write_text_file(stats_path, json.dumps(stats_trie)) full_results_path = self._filesystem.join(self._results_directory, 'full_results.json') json_results_generator.write_json(self._filesystem, summarized_full_results, full_results_path) full_results_jsonp_path = self._filesystem.join(self._results_directory, 'full_results_jsonp.js') json_results_generator.write_json(self._filesystem, summarized_full_results, full_results_jsonp_path, callback='ADD_FULL_RESULTS') full_results_path = self._filesystem.join(self._results_directory, 'failing_results.json') # We write failing_results.json out as jsonp because we need to load it # from a file url for results.html and Chromium doesn't allow that. json_results_generator.write_json(self._filesystem, summarized_failing_results, full_results_path, callback='ADD_RESULTS') # Write out the JSON files suitable for other tools to process. # As the output can be quite large (as there are 60k+ tests) we also # support only outputting the failing results. if self._options.json_failing_test_results: # FIXME(tansell): Make sure this includes an *unexpected* results # (IE Passing when expected to be failing.) json_results_generator.write_json(self._filesystem, summarized_failing_results, self._options.json_failing_test_results) if self._options.json_test_results: json_results_generator.write_json(self._filesystem, summarized_full_results, self._options.json_test_results) _log.debug('Finished writing JSON files.')
def _write_json_files(self, summarized_full_results, summarized_failing_results, initial_results): _log.debug("Writing JSON files in %s." % self._results_directory) # FIXME: Upload stats.json to the server and delete times_ms. times_trie = json_results_generator.test_timings_trie(initial_results.results_by_name.values()) times_json_path = self._filesystem.join(self._results_directory, "times_ms.json") json_results_generator.write_json(self._filesystem, times_trie, times_json_path) stats_trie = self._stats_trie(initial_results) stats_path = self._filesystem.join(self._results_directory, "stats.json") self._filesystem.write_text_file(stats_path, json.dumps(stats_trie)) full_results_path = self._filesystem.join(self._results_directory, "full_results.json") json_results_generator.write_json(self._filesystem, summarized_full_results, full_results_path) full_results_path = self._filesystem.join(self._results_directory, "failing_results.json") # We write failing_results.json out as jsonp because we need to load it from a file url for results.html and Chromium doesn't allow that. json_results_generator.write_json(self._filesystem, summarized_failing_results, full_results_path, callback="ADD_RESULTS") _log.debug("Finished writing JSON files.")