def test_stderr_is_saved(self):
fs = port.unit_test_filesystem()
self.assertTrue(passing_run(filesystem=fs))
self.assertEquals(
fs.read_text_file(
'/tmp/layout-test-results/passes/error-stderr.txt'),
'stuff going to stderr')
def test_host_port_and_filesystem(options, expectations):
filesystem = port.unit_test_filesystem()
host_port_obj = port.get("test", options, filesystem=filesystem, user=mocktool.MockUser())
expectations_path = host_port_obj.path_to_test_expectations_file()
filesystem.write_text_file(expectations_path, expectations)
return (host_port_obj, filesystem)
def make_generator(self, files, tests):
options = mocktool.MockOptions(configuration=None, html_directory="/tmp")
host_port = port.get("test", options, filesystem=port.unit_test_filesystem(files))
generator = rebaseline_chromium_webkit_tests.HtmlGenerator(
host_port, target_port=None, options=options, platforms=["test-mac-leopard"], rebaselining_tests=tests
)
return generator, host_port
def test_last_results(self):
fs = port.unit_test_filesystem()
# We do a logging run here instead of a passing run in order to
# suppress the output from the json generator.
(res, buildbot_output, regular_output, user) = logging_run(['--clobber-old-results'], record_results=True, filesystem=fs)
(res, buildbot_output, regular_output, user) = logging_run(
['--print-last-failures'], filesystem=fs)
self.assertEqual(regular_output.get(), ['\n\n'])
self.assertEqual(buildbot_output.get(), [])
def test_results_directory_relative(self):
# We run a configuration that should fail, to generate output, then
# look for what the output results url was.
fs = port.unit_test_filesystem()
fs.maybe_make_directory('/tmp/cwd')
fs.chdir('/tmp/cwd')
res, out, err, user = logging_run(['--results-directory=foo'],
tests_included=True, filesystem=fs)
self.assertEqual(user.opened_urls, ['/tmp/cwd/foo/results.html'])
def test_results_directory_absolute(self):
# We run a configuration that should fail, to generate output, then
# look for what the output results url was.
fs = port.unit_test_filesystem()
with fs.mkdtemp() as tmpdir:
res, out, err, user = logging_run(['--results-directory=' + str(tmpdir)],
tests_included=True, filesystem=fs)
self.assertEqual(user.opened_urls, [fs.join(tmpdir, 'results.html')])
def test_test_list_with_prefix(self):
fs = port.unit_test_filesystem()
filename = '/tmp/foo.txt'
fs.write_text_file(filename, 'LayoutTests/passes/text.html')
tests_run = get_tests_run(['--test-list=%s' % filename],
tests_included=True,
flatten_batches=True,
filesystem=fs)
self.assertEquals(['passes/text.html'], tests_run)
def test_results_directory_relative(self):
# We run a configuration that should fail, to generate output, then
# look for what the output results url was.
fs = port.unit_test_filesystem()
fs.maybe_make_directory('/tmp/cwd')
fs.chdir('/tmp/cwd')
res, out, err, user = logging_run(['--results-directory=foo'],
tests_included=True,
filesystem=fs)
self.assertEqual(user.opened_urls, ['/tmp/cwd/foo/results.html'])
def test_host_port_and_filesystem(options, expectations):
filesystem = port.unit_test_filesystem()
host_port_obj = port.get('test',
options,
filesystem=filesystem,
user=mocktool.MockUser())
expectations_path = host_port_obj.path_to_test_expectations_file()
filesystem.write_text_file(expectations_path, expectations)
return (host_port_obj, filesystem)
def test_last_results(self):
fs = port.unit_test_filesystem()
# We do a logging run here instead of a passing run in order to
# suppress the output from the json generator.
res, buildbot_output, regular_output, user = logging_run(
['--clobber-old-results'], record_results=True, filesystem=fs)
res, buildbot_output, regular_output, user = logging_run(
['--print-last-failures'], filesystem=fs)
self.assertEqual(regular_output.get(), ['\n\n'])
self.assertEqual(buildbot_output.get(), [])
def test_test_list(self):
fs = port.unit_test_filesystem()
filename = '/tmp/foo.txt'
fs.write_text_file(filename, 'passes/text.html')
tests_run = get_tests_run(['--test-list=%s' % filename], tests_included=True, flatten_batches=True, filesystem=fs)
self.assertEquals(['passes/text.html'], tests_run)
fs.remove(filename)
res, out, err, user = logging_run(['--test-list=%s' % filename],
tests_included=True, filesystem=fs)
self.assertEqual(res, -1)
self.assertFalse(err.empty())
def test_exit_after_n_failures_upload(self):
fs = port.unit_test_filesystem()
res, buildbot_output, regular_output, user = logging_run([
'failures/unexpected/text-image-checksum.html',
'passes/text.html',
'--exit-after-n-failures', '1',
],
tests_included=True,
record_results=True,
filesystem=fs)
self.assertTrue('/tmp/layout-test-results/incremental_results.json' in fs.files)
def test_results_directory_absolute(self):
# We run a configuration that should fail, to generate output, then
# look for what the output results url was.
fs = port.unit_test_filesystem()
with fs.mkdtemp() as tmpdir:
res, out, err, user = logging_run(
['--results-directory=' + str(tmpdir)],
tests_included=True,
filesystem=fs)
self.assertEqual(user.opened_urls,
[fs.join(tmpdir, 'results.html')])
def make_generator(self, files, tests):
options = mocktool.MockOptions(configuration=None,
html_directory='/tmp')
host_port = port.get('test',
options,
filesystem=port.unit_test_filesystem(files))
generator = rebaseline_chromium_webkit_tests.HtmlGenerator(
host_port,
target_port=None,
options=options,
platforms=['test-mac-leopard'],
rebaselining_tests=tests)
return generator, host_port
def test_reset_results(self):
# Test that we update expectations in place. If the expectation
# is missing, update the expected generic location.
fs = port.unit_test_filesystem()
passing_run(['--pixel-tests',
'--reset-results',
'passes/image.html',
'failures/expected/missing_image.html'],
tests_included=True, filesystem=fs)
file_list = fs.written_files.keys()
file_list.remove('/tmp/layout-test-results/tests_run0.txt')
self.assertEqual(len(file_list), 4)
self.assertBaselines(file_list, "/passes/image")
self.assertBaselines(file_list, "/failures/expected/missing_image")
def test_exit_after_n_failures_upload(self):
fs = port.unit_test_filesystem()
res, buildbot_output, regular_output, user = logging_run(
[
'failures/unexpected/text-image-checksum.html',
'passes/text.html',
'--exit-after-n-failures',
'1',
],
tests_included=True,
record_results=True,
filesystem=fs)
self.assertTrue(
'/tmp/layout-test-results/incremental_results.json' in fs.files)
def test_test_list(self):
fs = port.unit_test_filesystem()
filename = '/tmp/foo.txt'
fs.write_text_file(filename, 'passes/text.html')
tests_run = get_tests_run(['--test-list=%s' % filename],
tests_included=True,
flatten_batches=True,
filesystem=fs)
self.assertEquals(['passes/text.html'], tests_run)
fs.remove(filename)
res, out, err, user = logging_run(['--test-list=%s' % filename],
tests_included=True,
filesystem=fs)
self.assertEqual(res, -1)
self.assertFalse(err.empty())
def test_reset_results(self):
# Test that we update expectations in place. If the expectation
# is missing, update the expected generic location.
fs = port.unit_test_filesystem()
passing_run([
'--pixel-tests', '--reset-results', 'passes/image.html',
'failures/expected/missing_image.html'
],
tests_included=True,
filesystem=fs)
file_list = fs.written_files.keys()
file_list.remove('/tmp/layout-test-results/tests_run0.txt')
self.assertEqual(len(file_list), 6)
self.assertBaselines(file_list, "/passes/image")
self.assertBaselines(file_list, "/failures/expected/missing_image")
def test_new_baseline(self):
# Test that we update the platform expectations. If the expectation
# is mssing, then create a new expectation in the platform dir.
fs = port.unit_test_filesystem()
passing_run(['--pixel-tests',
'--new-baseline',
'passes/image.html',
'failures/expected/missing_image.html'],
tests_included=True, filesystem=fs)
file_list = fs.written_files.keys()
file_list.remove('/tmp/layout-test-results/tests_run.txt')
self.assertEqual(len(file_list), 6)
self.assertBaselines(file_list,
"/platform/test-mac/passes/image")
self.assertBaselines(file_list,
"/platform/test-mac/failures/expected/missing_image")
def test_new_baseline(self):
# Test that we update the platform expectations. If the expectation
# is mssing, then create a new expectation in the platform dir.
fs = port.unit_test_filesystem()
passing_run([
'--pixel-tests', '--new-baseline', 'passes/image.html',
'failures/expected/missing_image.html'
],
tests_included=True,
filesystem=fs)
file_list = fs.written_files.keys()
file_list.remove('/tmp/layout-test-results/tests_run0.txt')
self.assertEqual(len(file_list), 6)
self.assertBaselines(file_list,
"/platform/test-mac-leopard/passes/image")
self.assertBaselines(
file_list,
"/platform/test-mac-leopard/failures/expected/missing_image")
def test_stderr_is_saved(self):
fs = port.unit_test_filesystem()
self.assertTrue(passing_run(filesystem=fs))
self.assertEquals(fs.read_text_file('/tmp/layout-test-results/passes/error-stderr.txt'),
'stuff going to stderr')
def test_test_list_with_prefix(self):
fs = port.unit_test_filesystem()
filename = '/tmp/foo.txt'
fs.write_text_file(filename, 'LayoutTests/passes/text.html')
tests_run = get_tests_run(['--test-list=%s' % filename], tests_included=True, flatten_batches=True, filesystem=fs)
self.assertEquals(['passes/text.html'], tests_run)
