def GenerateJSONResults(test_results_map, options):
"""Generates a JSON results file from the given test_results_map,
returning the associated generator for use with UploadJSONResults, below.
Args:
test_results_map: A map of TestResult.
options: options for json generation. See GENERATE_JSON_RESULTS_OPTIONS
and OptionParser's help messages below for expected options and their
details.
"""
if not test_results_map:
logging.warn('No input results map was given.')
return
# Make sure we have all the required options (set empty string otherwise).
for opt in GENERATE_JSON_RESULTS_OPTIONS:
if not getattr(options, opt, None):
logging.warn('No value is given for option %s', opt)
setattr(options, opt, '')
try:
int(options.build_number)
except ValueError:
logging.error('options.build_number needs to be a number: %s',
options.build_number)
return
if not os.path.exists(options.results_directory):
os.makedirs(options.results_directory)
print('Generating json: '
'builder_name:%s, build_name:%s, build_number:%s, '
'results_directory:%s, builder_base_url:%s, '
'webkit_revision:%s, chrome_revision:%s '
'test_results_server:%s, test_type:%s, master_name:%s' %
(options.builder_name, options.build_name, options.build_number,
options.results_directory, options.builder_base_url,
options.webkit_revision, options.chrome_revision,
options.test_results_server, options.test_type,
options.master_name))
generator = JSONResultsGenerator(
options.builder_name, options.build_name, options.build_number,
options.results_directory, options.builder_base_url,
test_results_map,
svn_revisions=(('blink', options.webkit_revision),
('chromium', options.chrome_revision)),
test_results_server=options.test_results_server,
test_type=options.test_type,
master_name=options.master_name)
generator.generate_json_output()
generator.generate_times_ms_file()
return generator
def GenerateAndUploadJSONResults(test_results_map, options):
"""Generates a JSON results file from the given test_results_map and
upload it to the results server if options.test_results_server is given.
Args:
test_results_map: A map of TestResult.
options: options for json generation. See GENERATE_JSON_RESULTS_OPTIONS
and OptionParser's help messages below for expected options and their
details.
"""
if not test_results_map:
logging.warn("No input results map was given.")
return
if not os.path.exists(options.webkit_dir):
logging.warn("No options.webkit_dir (--webkit-dir) was given.")
return
# Make sure we have all the required options (set empty string otherwise).
for opt in GENERATE_JSON_RESULTS_OPTIONS:
if not getattr(options, opt, None):
logging.warn("No value is given for option %s", opt)
setattr(options, opt, '')
try:
int(options.build_number)
except ValueError:
logging.error("options.build_number needs to be a number: %s",
options.build_number)
return
if not os.path.exists(options.results_directory):
os.makedirs(options.results_directory)
print("Generating json: "
"builder_name:%s, build_name:%s, build_number:%s, "
"results_directory:%s, builder_base_url:%s, "
"webkit_dir:%s, chrome_dir:%s "
"test_results_server:%s, test_type:%s, master_name:%s" %
(options.builder_name, options.build_name, options.build_number,
options.results_directory, options.builder_base_url,
options.webkit_dir, options.chrome_dir,
options.test_results_server, options.test_type,
options.master_name))
generator = JSONResultsGenerator(
options.builder_name, options.build_name, options.build_number,
options.results_directory, options.builder_base_url,
test_results_map,
svn_repositories=(('webkit', options.webkit_dir),
('chrome', options.chrome_dir)),
test_results_server=options.test_results_server,
test_type=options.test_type,
master_name=options.master_name)
generator.generate_json_output()
generator.generate_times_ms_file()
generator.upload_json_files([INCREMENTAL_RESULTS_FILENAME, TIMES_MS_FILENAME])
def GenerateAndUploadJSONResults(test_results_map, options):
"""Generates a JSON results file from the given test_results_map and
upload it to the results server if options.test_results_server is given.
Args:
test_results_map: A map of TestResult.
options: options for json generation. See GENERATE_JSON_RESULTS_OPTIONS
and OptionParser's help messages below for expected options and their
details.
"""
if not test_results_map:
logging.warn('No input results map was given.')
return
if not os.path.exists(options.webkit_dir):
logging.warn('No options.webkit_dir (--webkit-dir) was given.')
return
# Make sure we have all the required options (set empty string otherwise).
for opt in GENERATE_JSON_RESULTS_OPTIONS:
if not getattr(options, opt, None):
logging.warn('No value is given for option %s', opt)
setattr(options, opt, '')
try:
int(options.build_number)
except ValueError:
logging.error('options.build_number needs to be a number: %s',
options.build_number)
return
if not os.path.exists(options.results_directory):
os.makedirs(options.results_directory)
print('Generating json: '
'builder_name:%s, build_name:%s, build_number:%s, '
'results_directory:%s, builder_base_url:%s, '
'webkit_dir:%s, chrome_dir:%s '
'test_results_server:%s, test_type:%s, master_name:%s' %
(options.builder_name, options.build_name, options.build_number,
options.results_directory, options.builder_base_url,
options.webkit_dir, options.chrome_dir,
options.test_results_server, options.test_type,
options.master_name))
generator = JSONResultsGenerator(
options.builder_name, options.build_name, options.build_number,
options.results_directory, options.builder_base_url,
test_results_map,
svn_repositories=(('webkit', options.webkit_dir),
('chrome', options.chrome_dir)),
test_results_server=options.test_results_server,
test_type=options.test_type,
master_name=options.master_name)
generator.generate_json_output()
generator.generate_times_ms_file()
generator.upload_json_files([INCREMENTAL_RESULTS_FILENAME, TIMES_MS_FILENAME])
def _generate_and_test_full_results_json(self, passed_tests_list,
failed_tests_list):
tests_set = set(passed_tests_list) | set(failed_tests_list)
get_test_set = lambda ts, label: set(
[t for t in ts if t.startswith(label)])
DISABLED_tests = get_test_set(tests_set, 'DISABLED_')
FLAKY_tests = get_test_set(tests_set, 'FLAKY_')
MAYBE_tests = get_test_set(tests_set, 'MAYBE_')
FAILS_tests = get_test_set(tests_set, 'FAILS_')
PASS_tests = tests_set - (DISABLED_tests | FLAKY_tests | FAILS_tests
| MAYBE_tests) - set(failed_tests_list)
failed_tests = set(failed_tests_list) - DISABLED_tests
test_timings = {}
test_results_map = {}
for i, test in enumerate(tests_set):
test_name = canonical_name(test)
test_timings[test_name] = i
test_results_map[test_name] = [
TestResult(test,
failed=(test in failed_tests),
elapsed_time=test_timings[test_name])
]
# Do not write to an actual file.
mock_writer = lambda path, data: True
generator = JSONResultsGenerator(
self.builder_name,
self.build_name,
self.build_number,
'',
None, # don't fetch past json results archive
test_results_map,
svn_revisions=[('blink', '12345')],
file_writer=mock_writer)
results_json = generator.get_full_results_json()
self._verify_full_json_results(results_json, tests_set, PASS_tests,
failed_tests, test_timings)
self.assertEqual(results_json.get('blink_revision'), '12345')
def _test_json_generation(self,
passed_tests_list,
failed_tests_list,
expected_test_list=None):
tests_set = set(passed_tests_list) | set(failed_tests_list)
get_test_set = lambda ts, label: set(
[t for t in ts if t.startswith(label)])
DISABLED_tests = get_test_set(tests_set, 'DISABLED_')
FLAKY_tests = get_test_set(tests_set, 'FLAKY_')
MAYBE_tests = get_test_set(tests_set, 'MAYBE_')
FAILS_tests = get_test_set(tests_set, 'FAILS_')
PASS_tests = tests_set - (DISABLED_tests | FLAKY_tests | FAILS_tests
| MAYBE_tests)
failed_tests = set(failed_tests_list) - DISABLED_tests
failed_count_map = dict([(t, 1) for t in failed_tests])
test_timings = {}
i = 0
test_results_map = dict()
for test in tests_set:
test_name = canonical_name(test)
test_timings[test_name] = float(self._num_runs * 100 + i)
i += 1
test_results_map[test_name] = TestResult(
test,
failed=(test in failed_tests),
elapsed_time=test_timings[test_name])
# Do not write to an actual file.
mock_writer = lambda path, data: True
generator = JSONResultsGenerator(
self.builder_name,
self.build_name,
self.build_number,
'',
None, # don't fetch past json results archive
test_results_map,
svn_repositories=[('webkit', '.')],
file_writer=mock_writer)
failed_count_map = dict([(t, 1) for t in failed_tests])
# Test incremental json results
incremental_json = generator.get_json()
self._verify_json_results(tests_set, test_timings, failed_count_map,
len(PASS_tests), len(DISABLED_tests),
len(FLAKY_tests),
len(DISABLED_tests | failed_tests),
incremental_json, 1, expected_test_list)
# We don't verify the results here, but at least we make sure the code
# runs without errors.
generator.generate_json_output()
generator.generate_times_ms_file()
def _generate_and_test_full_results_json(self, passed_tests_list,
failed_tests_list):
tests_set = set(passed_tests_list) | set(failed_tests_list)
get_test_set = lambda ts, label: set([t for t in ts if t.startswith(label)])
DISABLED_tests = get_test_set(tests_set, 'DISABLED_')
FLAKY_tests = get_test_set(tests_set, 'FLAKY_')
MAYBE_tests = get_test_set(tests_set, 'MAYBE_')
FAILS_tests = get_test_set(tests_set, 'FAILS_')
PASS_tests = tests_set - (DISABLED_tests | FLAKY_tests | FAILS_tests |
MAYBE_tests) - set(failed_tests_list)
failed_tests = set(failed_tests_list) - DISABLED_tests
test_timings = {}
test_results_map = {}
for i, test in enumerate(tests_set):
test_name = canonical_name(test)
test_timings[test_name] = i
test_results_map[test_name] = TestResult(test,
failed=(test in failed_tests),
elapsed_time=test_timings[test_name])
# Do not write to an actual file.
mock_writer = lambda path, data: True
generator = JSONResultsGenerator(
self.builder_name, self.build_name, self.build_number,
'',
None, # don't fetch past json results archive
test_results_map,
svn_revisions=[('blink', '12345')],
file_writer=mock_writer)
results_json = generator.get_full_results_json()
self._verify_full_json_results(results_json, tests_set, PASS_tests,
failed_tests, test_timings)
self.assertEqual(results_json.get('blink_revision'), '12345')
def GenerateJSONResults(test_results_map, options):
"""Generates a JSON results file from the given test_results_map,
returning the associated generator for use with UploadJSONResults, below.
Args:
test_results_map: A map of TestResult.
options: options for json generation. See GENERATE_JSON_RESULTS_OPTIONS
and OptionParser's help messages below for expected options and their
details.
"""
if not test_results_map:
logging.warn('No input results map was given.')
return
# Make sure we have all the required options (set empty string otherwise).
for opt in GENERATE_JSON_RESULTS_OPTIONS:
if not getattr(options, opt, None):
logging.warn('No value is given for option %s', opt)
setattr(options, opt, '')
try:
int(options.build_number)
except ValueError:
logging.error('options.build_number needs to be a number: %s',
options.build_number)
return
if not os.path.exists(options.results_directory):
os.makedirs(options.results_directory)
print(
'Generating json: '
'builder_name:%s, build_name:%s, build_number:%s, '
'results_directory:%s, builder_base_url:%s, '
'webkit_revision:%s, chrome_revision:%s '
'test_results_server:%s, test_type:%s, master_name:%s' %
(options.builder_name, options.build_name, options.build_number,
options.results_directory, options.builder_base_url,
options.webkit_revision, options.chrome_revision,
options.test_results_server, options.test_type, options.master_name))
generator = JSONResultsGenerator(
options.builder_name,
options.build_name,
options.build_number,
options.results_directory,
options.builder_base_url,
test_results_map,
svn_revisions=(('blink', options.webkit_revision),
('chrome', options.chrome_revision)),
test_results_server=options.test_results_server,
test_type=options.test_type,
master_name=options.master_name)
generator.generate_json_output()
generator.generate_times_ms_file()
return generator
def _test_json_generation(self, passed_tests_list, failed_tests_list, expected_test_list=None):
tests_set = set(passed_tests_list) | set(failed_tests_list)
get_test_set = lambda ts, label: set([t for t in ts if t.startswith(label)])
DISABLED_tests = get_test_set(tests_set, 'DISABLED_')
FLAKY_tests = get_test_set(tests_set, 'FLAKY_')
MAYBE_tests = get_test_set(tests_set, 'MAYBE_')
FAILS_tests = get_test_set(tests_set, 'FAILS_')
PASS_tests = tests_set - (DISABLED_tests | FLAKY_tests | FAILS_tests |
MAYBE_tests)
failed_tests = set(failed_tests_list) - DISABLED_tests
failed_count_map = dict([(t, 1) for t in failed_tests])
test_timings = {}
i = 0
test_results_map = dict()
for test in tests_set:
test_name = canonical_name(test)
test_timings[test_name] = float(self._num_runs * 100 + i)
i += 1
test_results_map[test_name] = TestResult(test,
failed=(test in failed_tests),
elapsed_time=test_timings[test_name])
# Do not write to an actual file.
mock_writer = lambda path, data: True
generator = JSONResultsGenerator(
self.builder_name, self.build_name, self.build_number,
'',
None, # don't fetch past json results archive
test_results_map,
svn_revisions=[('blink', '.')],
file_writer=mock_writer)
failed_count_map = dict([(t, 1) for t in failed_tests])
# Test incremental json results
incremental_json = generator.get_json()
self._verify_json_results(
tests_set,
test_timings,
failed_count_map,
len(PASS_tests),
len(DISABLED_tests),
len(FLAKY_tests),
len(DISABLED_tests | failed_tests),
incremental_json,
1,
expected_test_list)
# We don't verify the results here, but at least we make sure the code
# runs without errors.
generator.generate_json_output()
generator.generate_times_ms_file()
