def setUp(self):
# Create mock dependencies
self.mock_browser = mock.MagicMock(Browser)
self.mock_page_server = mock.MagicMock(SuitePageServer)
self.mock_result_reporters = [
mock.MagicMock(ConsoleResultReporter),
mock.MagicMock(XUnitResultReporter)
]
self.mock_coverage_reporters = [
mock.MagicMock(HtmlCoverageReporter),
mock.MagicMock(XmlCoverageReporter)
]
# Configure the page server to provide a suite page URL
self._set_suite_urls(['http://127.0.0.1:8080/suite/0'])
# Configure the page server to provide coverage data
self.mock_coverage_data = mock.MagicMock(CoverageData)
self.mock_page_server.all_coverage_data.return_value = self.mock_coverage_data
# Start with no test results
self.mock_browser.get_page_results.return_value = []
self.mock_browser.name.return_value = 'chrome'
# Create a SuiteRunner instance
self.runner = SuiteRunner(
[self.mock_browser],
self.mock_page_server,
self.mock_result_reporters,
self.mock_coverage_reporters
)
def test_result_data(self):
other_browser = mock.MagicMock(Browser)
other_browser.name.return_value = 'firefox'
other_browser.get_page_results.return_value = []
self.runner = SuiteRunner(
[self.mock_browser, other_browser],
self.mock_page_server,
self.mock_result_reporters,
self.mock_coverage_reporters
)
# Add test results for the Chrome browser
self._add_browser_result(
self.mock_browser,
'Adder test', 'it should start at zero',
'pass', ''
)
self._add_browser_result(
self.mock_browser,
'Adder test', 'it should add to the sum',
'pass', ''
)
# Add test results for the Firefox browser
self._add_browser_result(
other_browser,
'Adder test', 'it should start at zero',
'pass', ''
)
self._add_browser_result(
other_browser,
'Adder test', 'it should add to the sum',
'pass', ''
)
# Expect that we get the test results back
# in a `ResultData` object
result_data = self.runner.run()
self.assertEqual(
result_data.browsers(),
['chrome', 'firefox']
)
expected_results = [
{
'test_group': 'Adder test',
'test_name': 'it should start at zero',
'status': 'pass', 'detail': ''
},
{
'test_group': 'Adder test',
'test_name': 'it should add to the sum',
'status': 'pass', 'detail': ''
},
]
self.assertEqual(
result_data.test_results('chrome'),
expected_results
)
self.assertEqual(
result_data.test_results('firefox'),
expected_results
)
class SuiteRunnerTest(TestCase):
def setUp(self):
# Create mock dependencies
self.mock_browser = mock.MagicMock(Browser)
self.mock_page_server = mock.MagicMock(SuitePageServer)
self.mock_result_reporters = [
mock.MagicMock(ConsoleResultReporter),
mock.MagicMock(XUnitResultReporter)
]
self.mock_coverage_reporters = [
mock.MagicMock(HtmlCoverageReporter),
mock.MagicMock(XmlCoverageReporter)
]
# Configure the page server to provide a suite page URL
self._set_suite_urls(['http://127.0.0.1:8080/suite/0'])
# Configure the page server to provide coverage data
self.mock_coverage_data = mock.MagicMock(CoverageData)
self.mock_page_server.all_coverage_data.return_value = self.mock_coverage_data
# Start with no test results
self.mock_browser.get_page_results.return_value = []
self.mock_browser.name.return_value = 'chrome'
# Create a SuiteRunner instance
self.runner = SuiteRunner(
[self.mock_browser],
self.mock_page_server,
self.mock_result_reporters,
self.mock_coverage_reporters
)
def test_page_server_started_and_stopped(self):
# Load all the suite pages
self.runner.run()
# Expect that the page server was started/stopped
self.mock_page_server.start.assert_called_once_with()
self.mock_page_server.stop.assert_called_once_with()
def test_loads_all_suite_urls(self):
# Configure the suite runner to load multiple pages
suite_urls = ['http://127.0.0.1:8080/suite/{}'.format(suite_num)
for suite_num in range(10)]
self._set_suite_urls(suite_urls)
# Load all the suite pages
self.runner.run()
# Expect that the browser was asked to load all the pages
loaded_urls = [args[0] for (args, _)
in self.mock_browser.get_page_results.call_args_list]
self.assertEqual(sorted(suite_urls), sorted(loaded_urls))
def test_result_data(self):
other_browser = mock.MagicMock(Browser)
other_browser.name.return_value = 'firefox'
other_browser.get_page_results.return_value = []
self.runner = SuiteRunner(
[self.mock_browser, other_browser],
self.mock_page_server,
self.mock_result_reporters,
self.mock_coverage_reporters
)
# Add test results for the Chrome browser
self._add_browser_result(
self.mock_browser,
'Adder test', 'it should start at zero',
'pass', ''
)
self._add_browser_result(
self.mock_browser,
'Adder test', 'it should add to the sum',
'pass', ''
)
# Add test results for the Firefox browser
self._add_browser_result(
other_browser,
'Adder test', 'it should start at zero',
'pass', ''
)
self._add_browser_result(
other_browser,
'Adder test', 'it should add to the sum',
'pass', ''
)
# Expect that we get the test results back
# in a `ResultData` object
result_data = self.runner.run()
self.assertEqual(
result_data.browsers(),
['chrome', 'firefox']
)
expected_results = [
{
'test_group': 'Adder test',
'test_name': 'it should start at zero',
'status': 'pass', 'detail': ''
},
{
'test_group': 'Adder test',
'test_name': 'it should add to the sum',
'status': 'pass', 'detail': ''
},
]
self.assertEqual(
result_data.test_results('chrome'),
expected_results
)
self.assertEqual(
result_data.test_results('firefox'),
expected_results
)
def test_multiple_suites(self):
# Configure multiple test suite pages
suite_urls = ['http://127.0.0.1:8080/suite/0',
'http://127.0.0.1:8080/suite/1']
self._set_suite_urls(suite_urls)
# Add test results
self._add_browser_result(
self.mock_browser,
'Adder test', 'it should start at zero',
'pass', ''
)
# Run the tests
result_data = self.runner.run()
# Because of the way we set up the browser mock, each page
# loaded will report the same test result. So we expect
# that we get duplicate results, one for each URL.
self.assertEqual(result_data.browsers(), ['chrome'])
expected_results = [
{
'test_group': 'Adder test',
'test_name': 'it should start at zero',
'status': 'pass', 'detail': ''
},
{
'test_group': 'Adder test',
'test_name': 'it should start at zero',
'status': 'pass', 'detail': ''
},
]
self.assertEqual(
result_data.test_results('chrome'),
expected_results
)
def test_no_results(self):
# Do not add any test results
# Run the tests
result_data = self.runner.run()
# Expect that we pass by default
self.assertTrue(result_data.all_passed())
def test_generate_result_reports(self):
# Run the suite to generate test result reports
result_data = self.runner.run()
# Check that each of our reporters was called
for reporter in self.mock_result_reporters:
reporter.write_report.assert_called_once_with(result_data)
def test_write_coverage_reports(self):
# Get the coverage data by running the test suite
self.runner.run()
# Trigger the runner to write coverage reports to an output file
self.runner.write_coverage_reports()
# Expect that each of the coverage reporters was called
# with the data returned by the suite page server.
for reporter in self.mock_coverage_reporters:
reporter.write_report.assert_called_with(self.mock_coverage_data)
def test_coverage_timeout(self):
# Simulate `all_coverage_data()` timeout
self.mock_page_server.all_coverage_data.side_effect = TimeoutError
# Load all the suite pages
# This should not raise an exception
self.runner.run()
# Expect that when we write the coverage report
# nothing happens
self.runner.write_coverage_reports()
for reporter in self.mock_coverage_reporters:
self.assertEqual(reporter.write_report.call_args_list, list())
def _set_suite_urls(self, url_list):
"""
Configure the suite page server to use each url in `url_list`
as the suite page.
"""
self.mock_page_server.suite_url_list.return_value = url_list
def _add_browser_result(self, mock_browser, group_name, test_name, status, detail):
"""
Configure `mock_browser` (a `Browser mock) to return the test result:
`group_name`: name of the test group (e.g. 'Adder tests')
`test_name`: name of the specific test case
(e.g. 'it should start at zero')
`status`: pass | fail | skip
`detail`: details of the test case (e.g. a stack trace)
"""
# Retrieve the current results dict
results = mock_browser.get_page_results.return_value
# Append the new result
results.append({'test_group': group_name,
'test_name': test_name,
'status': status,
'detail': detail})
def _assert_reports_equal(self, report, expected_report):
"""
Asserts that two console reports are equal, with
some extra debugging logging.
Strips the strings to avoid failures due to starting/ending
newline issues.
"""
assert_long_str_equal(report, expected_report, strip=True)