def GenerateJSONResults(test_results_map, options): """Generates a JSON results file from the given test_results_map, returning the associated generator for use with UploadJSONResults, below. Args: test_results_map: A map of TestResult. options: options for json generation. See GENERATE_JSON_RESULTS_OPTIONS and OptionParser's help messages below for expected options and their details. """ if not test_results_map: logging.warn('No input results map was given.') return # Make sure we have all the required options (set empty string otherwise). for opt in GENERATE_JSON_RESULTS_OPTIONS: if not getattr(options, opt, None): logging.warn('No value is given for option %s', opt) setattr(options, opt, '') try: int(options.build_number) except ValueError: logging.error('options.build_number needs to be a number: %s', options.build_number) return if not os.path.exists(options.results_directory): os.makedirs(options.results_directory) print('Generating json: ' 'builder_name:%s, build_name:%s, build_number:%s, ' 'results_directory:%s, builder_base_url:%s, ' 'webkit_revision:%s, chrome_revision:%s ' 'test_results_server:%s, test_type:%s, master_name:%s' % (options.builder_name, options.build_name, options.build_number, options.results_directory, options.builder_base_url, options.webkit_revision, options.chrome_revision, options.test_results_server, options.test_type, options.master_name)) generator = JSONResultsGenerator( options.builder_name, options.build_name, options.build_number, options.results_directory, options.builder_base_url, test_results_map, svn_revisions=(('blink', options.webkit_revision), ('chromium', options.chrome_revision)), test_results_server=options.test_results_server, test_type=options.test_type, master_name=options.master_name) generator.generate_json_output() generator.generate_times_ms_file() return generator
def GenerateAndUploadJSONResults(test_results_map, options): """Generates a JSON results file from the given test_results_map and upload it to the results server if options.test_results_server is given. Args: test_results_map: A map of TestResult. options: options for json generation. See GENERATE_JSON_RESULTS_OPTIONS and OptionParser's help messages below for expected options and their details. """ if not test_results_map: logging.warn("No input results map was given.") return if not os.path.exists(options.webkit_dir): logging.warn("No options.webkit_dir (--webkit-dir) was given.") return # Make sure we have all the required options (set empty string otherwise). for opt in GENERATE_JSON_RESULTS_OPTIONS: if not getattr(options, opt, None): logging.warn("No value is given for option %s", opt) setattr(options, opt, '') try: int(options.build_number) except ValueError: logging.error("options.build_number needs to be a number: %s", options.build_number) return if not os.path.exists(options.results_directory): os.makedirs(options.results_directory) print("Generating json: " "builder_name:%s, build_name:%s, build_number:%s, " "results_directory:%s, builder_base_url:%s, " "webkit_dir:%s, chrome_dir:%s " "test_results_server:%s, test_type:%s, master_name:%s" % (options.builder_name, options.build_name, options.build_number, options.results_directory, options.builder_base_url, options.webkit_dir, options.chrome_dir, options.test_results_server, options.test_type, options.master_name)) generator = JSONResultsGenerator( options.builder_name, options.build_name, options.build_number, options.results_directory, options.builder_base_url, test_results_map, svn_repositories=(('webkit', options.webkit_dir), ('chrome', options.chrome_dir)), test_results_server=options.test_results_server, test_type=options.test_type, master_name=options.master_name) generator.generate_json_output() generator.generate_times_ms_file() generator.upload_json_files([INCREMENTAL_RESULTS_FILENAME, TIMES_MS_FILENAME])
def GenerateAndUploadJSONResults(test_results_map, options): """Generates a JSON results file from the given test_results_map and upload it to the results server if options.test_results_server is given. Args: test_results_map: A map of TestResult. options: options for json generation. See GENERATE_JSON_RESULTS_OPTIONS and OptionParser's help messages below for expected options and their details. """ if not test_results_map: logging.warn('No input results map was given.') return if not os.path.exists(options.webkit_dir): logging.warn('No options.webkit_dir (--webkit-dir) was given.') return # Make sure we have all the required options (set empty string otherwise). for opt in GENERATE_JSON_RESULTS_OPTIONS: if not getattr(options, opt, None): logging.warn('No value is given for option %s', opt) setattr(options, opt, '') try: int(options.build_number) except ValueError: logging.error('options.build_number needs to be a number: %s', options.build_number) return if not os.path.exists(options.results_directory): os.makedirs(options.results_directory) print('Generating json: ' 'builder_name:%s, build_name:%s, build_number:%s, ' 'results_directory:%s, builder_base_url:%s, ' 'webkit_dir:%s, chrome_dir:%s ' 'test_results_server:%s, test_type:%s, master_name:%s' % (options.builder_name, options.build_name, options.build_number, options.results_directory, options.builder_base_url, options.webkit_dir, options.chrome_dir, options.test_results_server, options.test_type, options.master_name)) generator = JSONResultsGenerator( options.builder_name, options.build_name, options.build_number, options.results_directory, options.builder_base_url, test_results_map, svn_repositories=(('webkit', options.webkit_dir), ('chrome', options.chrome_dir)), test_results_server=options.test_results_server, test_type=options.test_type, master_name=options.master_name) generator.generate_json_output() generator.generate_times_ms_file() generator.upload_json_files([INCREMENTAL_RESULTS_FILENAME, TIMES_MS_FILENAME])
def _generate_and_test_full_results_json(self, passed_tests_list, failed_tests_list): tests_set = set(passed_tests_list) | set(failed_tests_list) get_test_set = lambda ts, label: set( [t for t in ts if t.startswith(label)]) DISABLED_tests = get_test_set(tests_set, 'DISABLED_') FLAKY_tests = get_test_set(tests_set, 'FLAKY_') MAYBE_tests = get_test_set(tests_set, 'MAYBE_') FAILS_tests = get_test_set(tests_set, 'FAILS_') PASS_tests = tests_set - (DISABLED_tests | FLAKY_tests | FAILS_tests | MAYBE_tests) - set(failed_tests_list) failed_tests = set(failed_tests_list) - DISABLED_tests test_timings = {} test_results_map = {} for i, test in enumerate(tests_set): test_name = canonical_name(test) test_timings[test_name] = i test_results_map[test_name] = [ TestResult(test, failed=(test in failed_tests), elapsed_time=test_timings[test_name]) ] # Do not write to an actual file. mock_writer = lambda path, data: True generator = JSONResultsGenerator( self.builder_name, self.build_name, self.build_number, '', None, # don't fetch past json results archive test_results_map, svn_revisions=[('blink', '12345')], file_writer=mock_writer) results_json = generator.get_full_results_json() self._verify_full_json_results(results_json, tests_set, PASS_tests, failed_tests, test_timings) self.assertEqual(results_json.get('blink_revision'), '12345')
def _test_json_generation(self, passed_tests_list, failed_tests_list, expected_test_list=None): tests_set = set(passed_tests_list) | set(failed_tests_list) get_test_set = lambda ts, label: set( [t for t in ts if t.startswith(label)]) DISABLED_tests = get_test_set(tests_set, 'DISABLED_') FLAKY_tests = get_test_set(tests_set, 'FLAKY_') MAYBE_tests = get_test_set(tests_set, 'MAYBE_') FAILS_tests = get_test_set(tests_set, 'FAILS_') PASS_tests = tests_set - (DISABLED_tests | FLAKY_tests | FAILS_tests | MAYBE_tests) failed_tests = set(failed_tests_list) - DISABLED_tests failed_count_map = dict([(t, 1) for t in failed_tests]) test_timings = {} i = 0 test_results_map = dict() for test in tests_set: test_name = canonical_name(test) test_timings[test_name] = float(self._num_runs * 100 + i) i += 1 test_results_map[test_name] = TestResult( test, failed=(test in failed_tests), elapsed_time=test_timings[test_name]) # Do not write to an actual file. mock_writer = lambda path, data: True generator = JSONResultsGenerator( self.builder_name, self.build_name, self.build_number, '', None, # don't fetch past json results archive test_results_map, svn_repositories=[('webkit', '.')], file_writer=mock_writer) failed_count_map = dict([(t, 1) for t in failed_tests]) # Test incremental json results incremental_json = generator.get_json() self._verify_json_results(tests_set, test_timings, failed_count_map, len(PASS_tests), len(DISABLED_tests), len(FLAKY_tests), len(DISABLED_tests | failed_tests), incremental_json, 1, expected_test_list) # We don't verify the results here, but at least we make sure the code # runs without errors. generator.generate_json_output() generator.generate_times_ms_file()
def _generate_and_test_full_results_json(self, passed_tests_list, failed_tests_list): tests_set = set(passed_tests_list) | set(failed_tests_list) get_test_set = lambda ts, label: set([t for t in ts if t.startswith(label)]) DISABLED_tests = get_test_set(tests_set, 'DISABLED_') FLAKY_tests = get_test_set(tests_set, 'FLAKY_') MAYBE_tests = get_test_set(tests_set, 'MAYBE_') FAILS_tests = get_test_set(tests_set, 'FAILS_') PASS_tests = tests_set - (DISABLED_tests | FLAKY_tests | FAILS_tests | MAYBE_tests) - set(failed_tests_list) failed_tests = set(failed_tests_list) - DISABLED_tests test_timings = {} test_results_map = {} for i, test in enumerate(tests_set): test_name = canonical_name(test) test_timings[test_name] = i test_results_map[test_name] = TestResult(test, failed=(test in failed_tests), elapsed_time=test_timings[test_name]) # Do not write to an actual file. mock_writer = lambda path, data: True generator = JSONResultsGenerator( self.builder_name, self.build_name, self.build_number, '', None, # don't fetch past json results archive test_results_map, svn_revisions=[('blink', '12345')], file_writer=mock_writer) results_json = generator.get_full_results_json() self._verify_full_json_results(results_json, tests_set, PASS_tests, failed_tests, test_timings) self.assertEqual(results_json.get('blink_revision'), '12345')
def GenerateJSONResults(test_results_map, options): """Generates a JSON results file from the given test_results_map, returning the associated generator for use with UploadJSONResults, below. Args: test_results_map: A map of TestResult. options: options for json generation. See GENERATE_JSON_RESULTS_OPTIONS and OptionParser's help messages below for expected options and their details. """ if not test_results_map: logging.warn('No input results map was given.') return # Make sure we have all the required options (set empty string otherwise). for opt in GENERATE_JSON_RESULTS_OPTIONS: if not getattr(options, opt, None): logging.warn('No value is given for option %s', opt) setattr(options, opt, '') try: int(options.build_number) except ValueError: logging.error('options.build_number needs to be a number: %s', options.build_number) return if not os.path.exists(options.results_directory): os.makedirs(options.results_directory) print( 'Generating json: ' 'builder_name:%s, build_name:%s, build_number:%s, ' 'results_directory:%s, builder_base_url:%s, ' 'webkit_revision:%s, chrome_revision:%s ' 'test_results_server:%s, test_type:%s, master_name:%s' % (options.builder_name, options.build_name, options.build_number, options.results_directory, options.builder_base_url, options.webkit_revision, options.chrome_revision, options.test_results_server, options.test_type, options.master_name)) generator = JSONResultsGenerator( options.builder_name, options.build_name, options.build_number, options.results_directory, options.builder_base_url, test_results_map, svn_revisions=(('blink', options.webkit_revision), ('chrome', options.chrome_revision)), test_results_server=options.test_results_server, test_type=options.test_type, master_name=options.master_name) generator.generate_json_output() generator.generate_times_ms_file() return generator
def _test_json_generation(self, passed_tests_list, failed_tests_list, expected_test_list=None): tests_set = set(passed_tests_list) | set(failed_tests_list) get_test_set = lambda ts, label: set([t for t in ts if t.startswith(label)]) DISABLED_tests = get_test_set(tests_set, 'DISABLED_') FLAKY_tests = get_test_set(tests_set, 'FLAKY_') MAYBE_tests = get_test_set(tests_set, 'MAYBE_') FAILS_tests = get_test_set(tests_set, 'FAILS_') PASS_tests = tests_set - (DISABLED_tests | FLAKY_tests | FAILS_tests | MAYBE_tests) failed_tests = set(failed_tests_list) - DISABLED_tests failed_count_map = dict([(t, 1) for t in failed_tests]) test_timings = {} i = 0 test_results_map = dict() for test in tests_set: test_name = canonical_name(test) test_timings[test_name] = float(self._num_runs * 100 + i) i += 1 test_results_map[test_name] = TestResult(test, failed=(test in failed_tests), elapsed_time=test_timings[test_name]) # Do not write to an actual file. mock_writer = lambda path, data: True generator = JSONResultsGenerator( self.builder_name, self.build_name, self.build_number, '', None, # don't fetch past json results archive test_results_map, svn_revisions=[('blink', '.')], file_writer=mock_writer) failed_count_map = dict([(t, 1) for t in failed_tests]) # Test incremental json results incremental_json = generator.get_json() self._verify_json_results( tests_set, test_timings, failed_count_map, len(PASS_tests), len(DISABLED_tests), len(FLAKY_tests), len(DISABLED_tests | failed_tests), incremental_json, 1, expected_test_list) # We don't verify the results here, but at least we make sure the code # runs without errors. generator.generate_json_output() generator.generate_times_ms_file()