def test_packed_test(self):
upload = Upload(
suite='webkitpy-tests',
commits=[Upload.create_commit(
repository_id='webkit',
id='5',
branch='trunk',
)],
results={
'dir1/sub-dir1/test1': Upload.create_test_result(actual=Upload.Expectations.FAIL),
'dir1/sub-dir1/test2': Upload.create_test_result(actual=Upload.Expectations.TIMEOUT),
'dir1/sub-dir2/test3': {},
'dir1/sub-dir2/test4': {},
'dir2/sub-dir3/test5': {},
'dir2/test6': {},
}
)
generated_dict = self.normalize(json.loads(json.dumps(upload, cls=Upload.Encoder)))
self.assertEqual(generated_dict['test_results']['results'], self.normalize({
'dir1': {
'sub-dir1': {
'test1': {'actual': Upload.Expectations.FAIL},
'test2': {'actual': Upload.Expectations.TIMEOUT},
}, 'sub-dir2': {
'test3': {},
'test4': {},
}
}, 'dir2': {
'sub-dir3': {'test5': {}},
'test6': {},
},
}))
def test_buildbot(self):
upload = Upload(suite='webkitpy-tests',
commits=[
Upload.create_commit(
repository_id='webkit',
id='5',
branch='trunk',
)
],
details=Upload.create_details(options=self.Options(
buildbot_master='webkit.org',
builder_name='Queue-1',
build_number=1,
)))
with self.assertRaises(ValueError):
json.dumps(upload, cls=Upload.Encoder)
upload.details['buildbot-worker'] = 'bot123'
generated_dict = self.normalize(
json.loads(json.dumps(upload, cls=Upload.Encoder)))
self.assertEqual(
generated_dict['test_results']['details'],
self.normalize({
'buildbot-master': 'webkit.org',
'builder-name': 'Queue-1',
'build-number': 1,
'buildbot-worker': 'bot123',
}))
def test_no_suite(self):
upload = Upload(
commits=[Upload.create_commit(
repository_id='webkit',
id='5',
branch='trunk',
)],
)
with self.assertRaises(ValueError):
json.dumps(upload, cls=Upload.Encoder)
def test_encoding(self):
start_time, end_time = time.time() - 3, time.time()
upload = Upload(
suite='webkitpy-tests',
configuration=Upload.create_configuration(
platform='mac',
version='10.13.0',
version_name='High Sierra',
architecture='x86_64',
sdk='17A405',
),
details=Upload.create_details(link='https://webkit.org'),
commits=[Upload.create_commit(
repository_id='webkit',
id='5',
branch='trunk',
)],
run_stats=Upload.create_run_stats(
start_time=start_time,
end_time=end_time,
tests_skipped=0,
),
results={
'webkitpy.test1': {},
'webkitpy.test2': Upload.create_test_result(expected=Upload.Expectations.PASS, actual=Upload.Expectations.FAIL),
},
)
generated_dict = self.normalize(json.loads(json.dumps(upload, cls=Upload.Encoder)))
self.assertEqual(generated_dict['version'], 0)
self.assertEqual(generated_dict['suite'], 'webkitpy-tests')
self.assertEqual(generated_dict['configuration'], self.normalize(dict(
platform='mac',
is_simulator=False,
version='10.13.0',
version_name='High Sierra',
architecture='x86_64',
sdk='17A405',
)))
self.assertEqual(generated_dict['commits'], [dict(
repository_id='webkit',
id='5',
branch='trunk',
)])
self.assertEqual(generated_dict['test_results']['details'], self.normalize(dict(link='https://webkit.org')))
self.assertEqual(generated_dict['test_results']['run_stats'], self.normalize(dict(
start_time=start_time,
end_time=end_time,
tests_skipped=0,
)))
self.assertEqual(generated_dict['test_results']['results'], self.normalize({
'webkitpy.test1': {},
'webkitpy.test2': Upload.create_test_result(expected=Upload.Expectations.PASS, actual=Upload.Expectations.FAIL),
}))
def test_archive_upload(self):
upload = Upload(
suite='webkitpy-tests',
commits=[
Upload.create_commit(
repository_id='webkit',
id='5',
branch='trunk',
)
],
)
with mock.patch('requests.post',
new=lambda url, headers={}, data={}, files={}, verify=
True: self.MockResponse()):
self.assertTrue(
upload.upload_archive('https://results.webkit.org',
archive='content',
log_line_func=lambda _: None))
with mock.patch('requests.post',
new=lambda url, headers={}, data={}, files={}, verify=
True: self.raise_requests_ConnectionError()):
lines = []
self.assertFalse(
upload.upload_archive(
'https://results.webkit.org',
archive='content',
log_line_func=lambda line: lines.append(line)))
self.assertEqual([
' ' * 4 +
'Failed to upload test archive to https://results.webkit.org, results server not online'
], lines)
mock_404 = mock.patch(
'requests.post',
new=lambda url, headers={}, data={}, files={
}, verify=True: self.MockResponse(
status_code=404,
text=json.dumps(dict(description='No such address')),
))
with mock_404:
lines = []
self.assertFalse(
upload.upload_archive(
'https://results.webkit.org',
archive='content',
log_line_func=lambda line: lines.append(line)))
self.assertEqual([
' ' * 4 +
'Error uploading archive to https://results.webkit.org',
' ' * 8 + 'No such address',
], lines)
def test_upload(self):
upload = Upload(
suite='webkitpy-tests',
commits=[Upload.create_commit(
repository_id='webkit',
id='5',
branch='trunk',
)],
)
with mock.patch('requests.post', new=lambda url, data: self.MockResponse()):
self.assertTrue(upload.upload('https://webkit.org/results', log_line_func=lambda _: None))
with mock.patch('requests.post', new=lambda url, data: self.raise_requests_ConnectionError()):
self.assertFalse(upload.upload('https://webkit.org/results', log_line_func=lambda _: None))
mock_404 = mock.patch('requests.post', new=lambda url, data: self.MockResponse(
status_code=404,
text=json.dumps(dict(description='No such address')),
))
with mock_404:
self.assertFalse(upload.upload('https://webkit.org/results', log_line_func=lambda _: None))
def _run_tests(self, names):
# Make sure PYTHONPATH is set up properly.
sys.path = self.finder.additional_paths(sys.path) + sys.path
# We autoinstall everything up so that we can run tests concurrently
# and not have to worry about autoinstalling packages concurrently.
self.printer.write_update("Checking autoinstalled packages ...")
from webkitpy.thirdparty import autoinstall_everything
autoinstall_everything()
start_time = time.time()
if getattr(self._options, 'coverage', False):
_log.warning("Checking code coverage, so running things serially")
self._options.child_processes = 1
import webkitpy.thirdparty.autoinstalled.coverage as coverage
cov = coverage.coverage(omit=[
"/usr/*",
"*/webkitpy/thirdparty/autoinstalled/*",
"*/webkitpy/thirdparty/BeautifulSoup.py",
"*/webkitpy/thirdparty/BeautifulSoup_legacy.py",
])
cov.start()
self.printer.write_update("Checking imports ...")
if not self._check_imports(names):
return False
self.printer.write_update("Finding the individual test methods ...")
loader = _Loader()
parallel_tests, serial_tests = self._test_names(loader, names)
self.printer.write_update("Running the tests ...")
self.printer.num_tests = len(parallel_tests) + len(serial_tests)
start = time.time()
test_runner = Runner(self.printer, loader)
test_runner.run(parallel_tests,
getattr(self._options, 'child_processes', 1))
test_runner.run(serial_tests, 1)
end_time = time.time()
self.printer.print_result(time.time() - start)
if getattr(self._options, 'json', False):
_print_results_as_json(
sys.stdout, itertools.chain(parallel_tests, serial_tests),
test_runner.failures, test_runner.errors)
if getattr(self._options, 'json_file_name', None):
self._options.json_file_name = os.path.abspath(
self._options.json_file_name)
with open(self._options.json_file_name, 'w') as json_file:
_print_results_as_json(
json_file, itertools.chain(parallel_tests, serial_tests),
test_runner.failures, test_runner.errors)
if getattr(self._options, 'coverage', False):
cov.stop()
cov.save()
failed_uploads = 0
if getattr(self._options, 'report_urls', None):
self.printer.meter.writeln('\n')
self.printer.write_update('Preparing upload data ...')
# Empty test results indicate a PASS.
results = {test: {} for test in test_runner.tests_run}
for test, errors in test_runner.errors:
results[test] = Upload.create_test_result(
actual=Upload.Expectations.ERROR, log='/n'.join(errors))
for test, failures in test_runner.failures:
results[test] = Upload.create_test_result(
actual=Upload.Expectations.FAIL, log='/n'.join(failures))
_host.initialize_scm()
upload = Upload(
suite='webkitpy-tests',
configuration=Upload.create_configuration(
platform=_host.platform.os_name,
version=str(_host.platform.os_version),
version_name=_host.platform.os_version_name(),
style=self.upload_style,
sdk=_host.platform.build_version(),
flavor=self._options.result_report_flavor,
),
details=Upload.create_details(options=self._options),
commits=[
Upload.create_commit(
repository_id='webkit',
id=_host.scm().native_revision(_webkit_root),
branch=_host.scm().native_branch(_webkit_root),
)
],
run_stats=Upload.create_run_stats(
start_time=start_time,
end_time=end_time,
tests_skipped=len(test_runner.tests_run) -
len(parallel_tests) - len(serial_tests),
),
results=results,
)
for url in self._options.report_urls:
self.printer.write_update('Uploading to {} ...'.format(url))
failed_uploads = failed_uploads if upload.upload(
url, log_line_func=self.printer.meter.writeln) else (
failed_uploads + 1)
self.printer.meter.writeln('Uploads completed!')
if getattr(self._options, 'coverage', False):
cov.report(show_missing=False)
return not self.printer.num_errors and not self.printer.num_failures and not failed_uploads
def _run_tests(self, names, will_run_lldb_webkit_tests):
# Make sure PYTHONPATH is set up properly.
sys.path = self.finder.additional_paths(sys.path) + sys.path
# We autoinstall everything up so that we can run tests concurrently
# and not have to worry about autoinstalling packages concurrently.
self.printer.write_update("Checking autoinstalled packages ...")
from webkitpy.thirdparty import autoinstall_everything
autoinstall_everything()
start_time = time.time()
config = Config(_host.executive, self.finder.filesystem)
configuration_to_use = self._options.configuration or config.default_configuration(
)
if will_run_lldb_webkit_tests:
self.printer.write_update('Building lldbWebKitTester ...')
build_lldbwebkittester = self.finder.filesystem.join(
_webkit_root, 'Tools', 'Scripts', 'build-lldbwebkittester')
try:
_host.executive.run_and_throw_if_fail(
[
build_lldbwebkittester,
config.flag_for_configuration(configuration_to_use)
],
quiet=(not bool(self._options.verbose)))
except ScriptError as e:
_log.error(e.message_with_output(output_limit=None))
return False
os.environ['LLDB_WEBKIT_TESTER_EXECUTABLE'] = str(
self.finder.filesystem.join(
config.build_directory(configuration_to_use),
'lldbWebKitTester'))
if not self.finder.filesystem.exists(
os.environ['LLDB_WEBKIT_TESTER_EXECUTABLE']):
_log.error('Failed to find lldbWebKitTester.')
return False
if self._options.coverage:
_log.warning("Checking code coverage, so running things serially")
self._options.child_processes = 1
import webkitpy.thirdparty.autoinstalled.coverage as coverage
cov = coverage.coverage(omit=[
"/usr/*", "*/webkitpy/thirdparty/autoinstalled/*",
"*/webkitpy/thirdparty/BeautifulSoup.py"
])
cov.start()
self.printer.write_update("Checking imports ...")
if not self._check_imports(names):
return False
self.printer.write_update("Finding the individual test methods ...")
loader = _Loader()
parallel_tests, serial_tests = self._test_names(loader, names)
self.printer.write_update("Running the tests ...")
self.printer.num_tests = len(parallel_tests) + len(serial_tests)
start = time.time()
test_runner = Runner(self.printer, loader)
test_runner.run(parallel_tests, self._options.child_processes)
test_runner.run(serial_tests, 1)
end_time = time.time()
self.printer.print_result(time.time() - start)
if self._options.json:
_print_results_as_json(
sys.stdout, itertools.chain(parallel_tests, serial_tests),
test_runner.failures, test_runner.errors)
if self._options.json_file_name:
self._options.json_file_name = os.path.abspath(
self._options.json_file_name)
with open(self._options.json_file_name, 'w') as json_file:
_print_results_as_json(
json_file, itertools.chain(parallel_tests, serial_tests),
test_runner.failures, test_runner.errors)
if self._options.coverage:
cov.stop()
cov.save()
failed_uploads = 0
if self._options.report_urls:
self.printer.meter.writeln('\n')
self.printer.write_update('Preparing upload data ...')
# Empty test results indicate a PASS.
results = {test: {} for test in test_runner.tests_run}
for test, errors in test_runner.errors:
results[test] = Upload.create_test_result(
actual=Upload.Expectations.ERROR, log='/n'.join(errors))
for test, failures in test_runner.failures:
results[test] = Upload.create_test_result(
actual=Upload.Expectations.FAIL, log='/n'.join(failures))
_host.initialize_scm()
upload = Upload(
suite='webkitpy-tests',
configuration=Upload.create_configuration(
platform=_host.platform.os_name,
version=str(_host.platform.os_version),
version_name=_host.platform.os_version_name(),
style='asan'
if config.asan else configuration_to_use.lower(),
sdk=_host.platform.build_version(),
flavor=self._options.result_report_flavor,
),
details=Upload.create_details(options=self._options),
commits=[
Upload.create_commit(
repository_id='webkit',
id=_host.scm().native_revision(_webkit_root),
branch=_host.scm().native_branch(_webkit_root),
)
],
run_stats=Upload.create_run_stats(
start_time=start_time,
end_time=end_time,
tests_skipped=len(test_runner.tests_run) -
len(parallel_tests) - len(serial_tests),
),
results=results,
)
for url in self._options.report_urls:
self.printer.write_update('Uploading to {} ...'.format(url))
failed_uploads = failed_uploads if upload.upload(
url, log_line_func=self.printer.meter.writeln) else (
failed_uploads + 1)
self.printer.meter.writeln('Uploads completed!')
if self._options.coverage:
cov.report(show_missing=False)
return not self.printer.num_errors and not self.printer.num_failures and not failed_uploads
