Exemple #1
0
    def _write_json_files(self, summarized_full_results, summarized_failing_results, initial_results, running_all_tests):
        _log.debug("Writing JSON files in %s.", self._results_directory)

        # FIXME: Upload stats.json to the server and delete times_ms.
        times_trie = json_results_generator.test_timings_trie(initial_results.results_by_name.values())
        times_json_path = self._filesystem.join(self._results_directory, "times_ms.json")
        json_results_generator.write_json(self._filesystem, times_trie, times_json_path)

        # Save out the times data so we can use it for --fastest in the future.
        if running_all_tests:
            bot_test_times_path = self._port.bot_test_times_path()
            self._filesystem.maybe_make_directory(self._filesystem.dirname(bot_test_times_path))
            json_results_generator.write_json(self._filesystem, times_trie, bot_test_times_path)

        stats_trie = self._stats_trie(initial_results)
        stats_path = self._filesystem.join(self._results_directory, "stats.json")
        self._filesystem.write_text_file(stats_path, json.dumps(stats_trie))

        full_results_path = self._filesystem.join(self._results_directory, "full_results.json")
        json_results_generator.write_json(self._filesystem, summarized_full_results, full_results_path)

        full_results_path = self._filesystem.join(self._results_directory, "failing_results.json")
        # We write failing_results.json out as jsonp because we need to load it
        # from a file url for results.html and Chromium doesn't allow that.
        json_results_generator.write_json(self._filesystem, summarized_failing_results, full_results_path, callback="ADD_RESULTS")

        if self._options.json_test_results:
            json_results_generator.write_json(self._filesystem, summarized_failing_results, self._options.json_test_results)

        _log.debug("Finished writing JSON files.")
Exemple #2
0
    def _upload_json_files(self, summarized_results, result_summary,
                           individual_test_timings):
        """Writes the results of the test run as JSON files into the results
        dir and upload the files to the appengine server.

        Args:
          unexpected_results: dict of unexpected results
          summarized_results: dict of results
          result_summary: full summary object
          individual_test_timings: list of test times (used by the flakiness
            dashboard).
        """
        _log.debug("Writing JSON files in %s." % self._results_directory)

        times_trie = json_results_generator.test_timings_trie(
            self._port, individual_test_timings)
        times_json_path = self._filesystem.join(self._results_directory,
                                                "times_ms.json")
        json_results_generator.write_json(self._filesystem, times_trie,
                                          times_json_path)

        full_results_path = self._filesystem.join(self._results_directory,
                                                  "full_results.json")
        # We write full_results.json out as jsonp because we need to load it from a file url and Chromium doesn't allow that.
        json_results_generator.write_json(
            self._filesystem,
            summarized_results,
            full_results_path,
            callback="ADD_RESULTS")

        generator = json_layout_results_generator.JSONLayoutResultsGenerator(
            self._port, self._options.builder_name, self._options.build_name,
            self._options.build_number, self._results_directory,
            BUILDER_BASE_URL, individual_test_timings, self._expectations,
            result_summary, self._test_names,
            self._options.test_results_server, "layout-tests",
            self._options.master_name)

        _log.debug("Finished writing JSON files.")

        json_files = [
            "incremental_results.json", "full_results.json", "times_ms.json"
        ]

        generator.upload_json_files(json_files)

        incremental_results_path = self._filesystem.join(
            self._results_directory, "incremental_results.json")

        # Remove these files from the results directory so they don't take up too much space on the buildbot.
        # The tools use the version we uploaded to the results server anyway.
        self._filesystem.remove(times_json_path)
        self._filesystem.remove(incremental_results_path)
Exemple #3
0
    def _upload_json_files(self, summarized_results, initial_results):
        """Writes the results of the test run as JSON files into the results
        dir and upload the files to the appengine server.

        Args:
          summarized_results: dict of results
          initial_results: full summary object
        """
        _log.debug("Writing JSON files in %s." % self._results_directory)

        # FIXME: Upload stats.json to the server and delete times_ms.
        times_trie = json_results_generator.test_timings_trie(
            self._port, initial_results.results_by_name.values())
        times_json_path = self._filesystem.join(self._results_directory,
                                                "times_ms.json")
        json_results_generator.write_json(self._filesystem, times_trie,
                                          times_json_path)

        stats_trie = self._stats_trie(initial_results)
        stats_path = self._filesystem.join(self._results_directory,
                                           "stats.json")
        self._filesystem.write_text_file(stats_path, json.dumps(stats_trie))

        full_results_path = self._filesystem.join(self._results_directory,
                                                  "full_results.json")
        # We write full_results.json out as jsonp because we need to load it from a file url and Chromium doesn't allow that.
        json_results_generator.write_json(self._filesystem,
                                          summarized_results,
                                          full_results_path,
                                          callback="ADD_RESULTS")

        generator = json_layout_results_generator.JSONLayoutResultsGenerator(
            self._port, self._options.builder_name, self._options.build_name,
            self._options.build_number, self._results_directory,
            BUILDER_BASE_URL, self._expectations, initial_results,
            self._options.test_results_server, "layout-tests",
            self._options.master_name)

        _log.debug("Finished writing JSON files.")

        json_files = [
            "incremental_results.json", "full_results.json", "times_ms.json"
        ]

        generator.upload_json_files(json_files)

        incremental_results_path = self._filesystem.join(
            self._results_directory, "incremental_results.json")

        # Remove these files from the results directory so they don't take up too much space on the buildbot.
        # The tools use the version we uploaded to the results server anyway.
        self._filesystem.remove(times_json_path)
        self._filesystem.remove(incremental_results_path)
Exemple #4
0
    def _upload_json_files(self, summarized_results, result_summary,
                           individual_test_timings):
        """Writes the results of the test run as JSON files into the results
        dir and upload the files to the appengine server.

        Args:
          unexpected_results: dict of unexpected results
          summarized_results: dict of results
          result_summary: full summary object
          individual_test_timings: list of test times (used by the flakiness
            dashboard).
        """
        _log.debug("Writing JSON files in %s." % self._results_directory)

        times_trie = json_results_generator.test_timings_trie(
            self._port, individual_test_timings)
        times_json_path = self._filesystem.join(self._results_directory,
                                                "times_ms.json")
        json_results_generator.write_json(self._filesystem, times_trie,
                                          times_json_path)

        full_results_path = self._filesystem.join(self._results_directory,
                                                  "full_results.json")
        # We write full_results.json out as jsonp because we need to load it from a file url and Chromium doesn't allow that.
        json_results_generator.write_json(self._filesystem,
                                          summarized_results,
                                          full_results_path,
                                          callback="ADD_RESULTS")

        generator = json_layout_results_generator.JSONLayoutResultsGenerator(
            self._port, self._options.builder_name, self._options.build_name,
            self._options.build_number, self._results_directory,
            BUILDER_BASE_URL, individual_test_timings, self._expectations,
            result_summary, self._test_names,
            self._options.test_results_server, "layout-tests",
            self._options.master_name)

        _log.debug("Finished writing JSON files.")

        json_files = [
            "incremental_results.json", "full_results.json", "times_ms.json"
        ]

        generator.upload_json_files(json_files)

        incremental_results_path = self._filesystem.join(
            self._results_directory, "incremental_results.json")

        # Remove these files from the results directory so they don't take up too much space on the buildbot.
        # The tools use the version we uploaded to the results server anyway.
        self._filesystem.remove(times_json_path)
        self._filesystem.remove(incremental_results_path)
    def _upload_json_files(self, summarized_results, initial_results):
        """Writes the results of the test run as JSON files into the results
        dir and upload the files to the appengine server.

        Args:
          summarized_results: dict of results
          initial_results: full summary object
        """
        _log.debug("Writing JSON files in %s." % self._results_directory)

        # FIXME: Upload stats.json to the server and delete times_ms.
        times_trie = json_results_generator.test_timings_trie(self._port, initial_results.results_by_name.values())
        times_json_path = self._filesystem.join(self._results_directory, "times_ms.json")
        json_results_generator.write_json(self._filesystem, times_trie, times_json_path)

        stats_trie = self._stats_trie(initial_results)
        stats_path = self._filesystem.join(self._results_directory, "stats.json")
        self._filesystem.write_text_file(stats_path, json.dumps(stats_trie))

        full_results_path = self._filesystem.join(self._results_directory, "full_results.json")
        # We write full_results.json out as jsonp because we need to load it from a file url and Chromium doesn't allow that.
        json_results_generator.write_json(
            self._filesystem, summarized_results, full_results_path, callback="ADD_RESULTS"
        )

        generator = json_layout_results_generator.JSONLayoutResultsGenerator(
            self._port,
            self._options.builder_name,
            self._options.build_name,
            self._options.build_number,
            self._results_directory,
            BUILDER_BASE_URL,
            self._expectations,
            initial_results,
            self._options.test_results_server,
            "layout-tests",
            self._options.master_name,
        )

        _log.debug("Finished writing JSON files.")

        json_files = ["incremental_results.json", "full_results.json", "times_ms.json"]

        generator.upload_json_files(json_files)

        incremental_results_path = self._filesystem.join(self._results_directory, "incremental_results.json")

        # Remove these files from the results directory so they don't take up too much space on the buildbot.
        # The tools use the version we uploaded to the results server anyway.
        self._filesystem.remove(times_json_path)
        self._filesystem.remove(incremental_results_path)
Exemple #6
0
    def _write_json_files(self, summarized_full_results, summarized_failing_results, initial_results, running_all_tests):
        _log.debug("Writing JSON files in %s." % self._results_directory)

        # FIXME: Upload stats.json to the server and delete times_ms.
        times_trie = json_results_generator.test_timings_trie(initial_results.results_by_name.values())
        times_json_path = self._filesystem.join(self._results_directory, "times_ms.json")
        json_results_generator.write_json(self._filesystem, times_trie, times_json_path)

        # Save out the times data so we can use it for --fastest in the future.
        if running_all_tests:
            bot_test_times_path = self._port.bot_test_times_path()
            self._filesystem.maybe_make_directory(self._filesystem.dirname(bot_test_times_path))
            json_results_generator.write_json(self._filesystem, times_trie, bot_test_times_path)

        stats_trie = self._stats_trie(initial_results)
        stats_path = self._filesystem.join(self._results_directory, "stats.json")
        self._filesystem.write_text_file(stats_path, json.dumps(stats_trie))

        full_results_path = self._filesystem.join(self._results_directory, "full_results.json")
        json_results_generator.write_json(self._filesystem, summarized_full_results, full_results_path)

        full_results_path = self._filesystem.join(self._results_directory, "failing_results.json")
        # We write failing_results.json out as jsonp because we need to load it from a file url for results.html and Chromium doesn't allow that.
        json_results_generator.write_json(self._filesystem, summarized_failing_results, full_results_path, callback="ADD_RESULTS")

        _log.debug("Finished writing JSON files.")
    def _upload_json_files(self, unexpected_results, summarized_results, result_summary,
                           individual_test_timings):
        """Writes the results of the test run as JSON files into the results
        dir and upload the files to the appengine server.

        There are three different files written into the results dir:
          unexpected_results.json: A short list of any unexpected results.
            This is used by the buildbots to display results.
          expectations.json: This is used by the flakiness dashboard.
          results.json: A full list of the results - used by the flakiness
            dashboard and the aggregate results dashboard.

        Args:
          unexpected_results: dict of unexpected results
          summarized_results: dict of results
          result_summary: full summary object
          individual_test_timings: list of test times (used by the flakiness
            dashboard).
        """
        _log.debug("Writing JSON files in %s." % self._results_directory)

        unexpected_json_path = self._fs.join(self._results_directory, "unexpected_results.json")
        json_results_generator.write_json(self._fs, unexpected_results, unexpected_json_path)

        full_results_path = self._fs.join(self._results_directory, "full_results.json")
        json_results_generator.write_json(self._fs, summarized_results, full_results_path)

        # Write a json file of the test_expectations.txt file for the layout
        # tests dashboard.
        expectations_path = self._fs.join(self._results_directory, "expectations.json")
        expectations_json = \
            self._expectations.get_expectations_json_for_all_platforms()
        self._fs.write_text_file(expectations_path,
                                 u"ADD_EXPECTATIONS(%s);" % expectations_json)

        generator = json_layout_results_generator.JSONLayoutResultsGenerator(
            self._port, self._options.builder_name, self._options.build_name,
            self._options.build_number, self._results_directory,
            BUILDER_BASE_URL, individual_test_timings,
            self._expectations, result_summary, self._test_files_list,
            self._options.test_results_server,
            "layout-tests",
            self._options.master_name)

        _log.debug("Finished writing JSON files.")

        json_files = ["expectations.json", "incremental_results.json", "full_results.json"]

        generator.upload_json_files(json_files)
Exemple #8
0
    def _write_json_files(self, summarized_full_results, summarized_failing_results, initial_results):
        _log.debug("Writing JSON files in %s." % self._results_directory)

        # FIXME: Upload stats.json to the server and delete times_ms.
        times_trie = json_results_generator.test_timings_trie(initial_results.results_by_name.values())
        times_json_path = self._filesystem.join(self._results_directory, "times_ms.json")
        json_results_generator.write_json(self._filesystem, times_trie, times_json_path)

        stats_trie = self._stats_trie(initial_results)
        stats_path = self._filesystem.join(self._results_directory, "stats.json")
        self._filesystem.write_text_file(stats_path, json.dumps(stats_trie))

        full_results_path = self._filesystem.join(self._results_directory, "full_results.json")
        json_results_generator.write_json(self._filesystem, summarized_full_results, full_results_path)

        full_results_path = self._filesystem.join(self._results_directory, "failing_results.json")
        # We write failing_results.json out as jsonp because we need to load it from a file url for results.html and Chromium doesn't allow that.
        json_results_generator.write_json(self._filesystem, summarized_failing_results, full_results_path, callback="ADD_RESULTS")

        _log.debug("Finished writing JSON files.")
Exemple #9
0
    def _write_json_files(self, summarized_full_results,
                          summarized_failing_results, initial_results,
                          running_all_tests):
        _log.debug("Writing JSON files in %s.", self._results_directory)

        # FIXME: Upload stats.json to the server and delete times_ms.
        times_trie = json_results_generator.test_timings_trie(
            initial_results.results_by_name.values())
        times_json_path = self._filesystem.join(self._results_directory,
                                                'times_ms.json')
        json_results_generator.write_json(self._filesystem, times_trie,
                                          times_json_path)

        # Save out the times data so we can use it for --fastest in the future.
        if running_all_tests:
            bot_test_times_path = self._port.bot_test_times_path()
            self._filesystem.maybe_make_directory(
                self._filesystem.dirname(bot_test_times_path))
            json_results_generator.write_json(self._filesystem, times_trie,
                                              bot_test_times_path)

        stats_trie = self._stats_trie(initial_results)
        stats_path = self._filesystem.join(self._results_directory,
                                           'stats.json')
        self._filesystem.write_text_file(stats_path, json.dumps(stats_trie))

        full_results_path = self._filesystem.join(self._results_directory,
                                                  'full_results.json')
        json_results_generator.write_json(self._filesystem,
                                          summarized_full_results,
                                          full_results_path)

        full_results_jsonp_path = self._filesystem.join(
            self._results_directory, 'full_results_jsonp.js')
        json_results_generator.write_json(self._filesystem,
                                          summarized_full_results,
                                          full_results_jsonp_path,
                                          callback='ADD_FULL_RESULTS')
        full_results_path = self._filesystem.join(self._results_directory,
                                                  'failing_results.json')
        # We write failing_results.json out as jsonp because we need to load it
        # from a file url for results.html and Chromium doesn't allow that.
        json_results_generator.write_json(self._filesystem,
                                          summarized_failing_results,
                                          full_results_path,
                                          callback='ADD_RESULTS')

        # Write out the JSON files suitable for other tools to process.
        # As the output can be quite large (as there are 60k+ tests) we also
        # support only outputting the failing results.
        if self._options.json_failing_test_results:
            # FIXME(tansell): Make sure this includes an *unexpected* results
            # (IE Passing when expected to be failing.)
            json_results_generator.write_json(
                self._filesystem, summarized_failing_results,
                self._options.json_failing_test_results)
        if self._options.json_test_results:
            json_results_generator.write_json(self._filesystem,
                                              summarized_full_results,
                                              self._options.json_test_results)

        _log.debug('Finished writing JSON files.')