def test_discover_tests_runtime_error_on_import(self):
# When
with cd(self.tempdir):
suite = Discoverer(Loader()).discover('TestSomething', None)
# Then
self.assertEqual(suite.countTestCases(), 1)
case, = find_test_cases(suite)
self.assertIsInstance(case, ModuleImportError)
self.assertEqual(case._testMethodName, 'test_error')
def test_discover_error_emacs_recovery_file(self):
# When
with cd(self.tempdir):
suite = Discoverer(Loader()).discover('TestSomething', None)
# Then
self.assertEqual(suite.countTestCases(), 1)
case, = find_test_cases(suite)
self.assertIsInstance(case, unittest.TestCase)
self.assertEqual(case._testMethodName, 'test_method')
def test_discover_tests(self):
# When
with cd(self.tempdir):
suite = Discoverer(Loader()).discover(self.tempdir, self.tempdir)
# Then
self.assertEqual(suite.countTestCases(), 1)
case, = find_test_cases(suite)
self.assertEqual(type(case).__name__, 'TestExpected')
self.assertEqual(case._testMethodName, 'test_expected')
def test_discover_tests_with_dot_slash(self):
# Given
start = './fixture/package/test_something.py'
# When
with cd(self.tempdir):
suite = Discoverer(Loader()).discover(start, None)
# Then
self.assertEqual(suite.countTestCases(), 1)
case, = find_test_cases(suite)
self.assertEqual(type(case).__name__, 'TestSomething')
self.assertEqual(case._testMethodName, 'test_method')
def test_case_dunder_str(self):
# Given
test_yaml = textwrap.dedent("""
---
version: '1.0'
config:
host: test.domain
cases:
- name: "Basic"
tests:
- name: "Test root URL"
url: "/"
""")
test_data = yaml.safe_load(test_yaml)
# When
suite = self.loader.load_tests_from_yaml(
test_data, '/path/to/foo.yaml')
# Then
self.assertIsInstance(suite, TestSuite)
self.assertEqual(suite.countTestCases(), 1)
case = next(find_test_cases(suite))
# When
case_str = str(case)
description = case.shortDescription()
# Then
self.assertEqual(case_str, "'Basic:Test root URL' (/path/to/foo.yaml)")
self.assertIsNone(description)
def test_load_yaml_tests_case_max_diff_number(self):
# Given
test_yaml = textwrap.dedent("""
---
version: '1.0'
config:
host: test.domain
cases:
- name: "Basic"
max-diff: 1234
tests:
- name: "Test root URL"
url: "/"
- name: "Test sub URL"
url: "/sub/"
""")
test_data = yaml.safe_load(test_yaml)
# When
suite = self.loader.load_tests_from_yaml(test_data,
'/path/to/foo.yaml')
# Then
self.assertIsInstance(suite, TestSuite)
self.assertEqual(suite.countTestCases(), 2)
cls1, cls2 = [type(case) for case in find_test_cases(suite)]
def test_case_dunder_str(self):
# Given
test_yaml = textwrap.dedent("""
---
version: '1.0'
config:
host: test.domain
cases:
- name: "Basic"
tests:
- name: "Test root URL"
url: "/"
""")
test_data = yaml.safe_load(test_yaml)
# When
suite = self.loader.load_tests_from_yaml(test_data,
'/path/to/foo.yaml')
# Then
self.assertIsInstance(suite, TestSuite)
self.assertEqual(suite.countTestCases(), 1)
case = next(find_test_cases(suite))
# When
case_str = str(case)
description = case.shortDescription()
# Then
self.assertEqual(case_str, "'Basic:Test root URL' (/path/to/foo.yaml)")
self.assertIsNone(description)
def test_load_yaml_tests_case_max_diff_number(self):
# Given
test_yaml = textwrap.dedent("""
---
version: '1.0'
config:
host: test.domain
cases:
- name: "Basic"
max-diff: 1234
tests:
- name: "Test root URL"
url: "/"
- name: "Test sub URL"
url: "/sub/"
""")
test_data = yaml.safe_load(test_yaml)
# When
suite = self.loader.load_tests_from_yaml(
test_data, '/path/to/foo.yaml')
# Then
self.assertIsInstance(suite, TestSuite)
self.assertEqual(suite.countTestCases(), 2)
cls1, cls2 = [type(case) for case in find_test_cases(suite)]
self.assertIs(cls1, cls2)
self.assertEqual(cls1.maxDiff, 1234)
def test_discover_creates_importerror_testcase(self):
with cd(self.tempdir):
suite = Discoverer(Loader()).discover(
self.tempdir, self.tempdir)
self.assertEqual(suite.countTestCases(), 3)
case_names = [
type(case).__name__ for case in find_test_cases(suite)]
self.assertEqual(
case_names, ['ModuleImportError', 'TestSomething',
'TestSomething'])
def test_discover_tests(self):
# When
with cd(self.tempdir):
suite = Discoverer(Loader()).discover(self.tempdir, self.tempdir)
# Then
self.assertEqual(suite.countTestCases(), 1)
case, = find_test_cases(suite)
self.assertEqual(type(case).__name__, 'TestExpected')
self.assertEqual(case._testMethodName, 'test_expected')
def test_discover_error_emacs_recovery_file(self):
# When
with cd(self.tempdir):
suite = Discoverer(Loader()).discover('TestSomething', None)
# Then
self.assertEqual(suite.countTestCases(), 1)
case, = find_test_cases(suite)
self.assertIsInstance(case, unittest.TestCase)
self.assertEqual(case._testMethodName, 'test_method')
def test_discover_tests_runtime_error_on_import(self):
# When
with cd(self.tempdir):
suite = Discoverer(Loader()).discover('TestSomething', None)
# Then
self.assertEqual(suite.countTestCases(), 1)
case, = find_test_cases(suite)
self.assertIsInstance(case, ModuleImportError)
self.assertEqual(case._testMethodName, 'test_error')
def test_discover_no_such_file(self):
# Given
test_filename = os.path.join(self.temp_dir, 'dont_exist.yml')
# When
suite = self.discoverer.discover(test_filename)
# Then
self.assertIsInstance(suite, TestSuite)
self.assertEqual(suite.countTestCases(), 0)
for case in find_test_cases(suite):
self.assertIsInstance(case, unittest.TestCase)
def test_discover_no_such_file(self):
# Given
test_filename = os.path.join(self.temp_dir, 'dont_exist.yml')
# When
suite = self.discoverer.discover(test_filename)
# Then
self.assertIsInstance(suite, TestSuite)
self.assertEqual(suite.countTestCases(), 0)
for case in find_test_cases(suite):
self.assertIsInstance(case, unittest.TestCase)
def test_discover_from_directory(self):
# Given
test_yaml_1 = textwrap.dedent("""
---
version: '1.0'
config:
host: test.domain
cases:
- name: "Basic"
tests:
- name: "Test root URL"
url: "/"
""")
test_yaml_2 = textwrap.dedent("""
---
version: '1.0'
config:
host: test.domain
cases:
- name: "A case"
tests:
- name: "Download authorization failure"
url:
template: "{data}/test"
- name: "Upload authorization failure"
url:
template: "/foo/{upload}/test"
""")
with tempfile.NamedTemporaryFile(delete=False,
prefix='test',
suffix='.yml',
dir=self.temp_dir) as fh:
fh.write(test_yaml_1.encode('utf-8'))
with tempfile.NamedTemporaryFile(delete=False,
prefix='test',
suffix='.yml',
dir=self.temp_dir) as fh:
fh.write(test_yaml_2.encode('utf-8'))
# When
suite = self.discoverer.discover(self.temp_dir)
# Then
self.assertIsInstance(suite, TestSuite)
self.assertEqual(suite.countTestCases(), 3)
for case in find_test_cases(suite):
self.assertIsInstance(case, unittest.TestCase)
def test_discover_tests_with_dot_slash(self):
# Given
start = './fixture/package/test_something.py'
# When
with cd(self.tempdir):
suite = Discoverer(Loader()).discover(start, None)
# Then
self.assertEqual(suite.countTestCases(), 1)
case, = find_test_cases(suite)
self.assertEqual(type(case).__name__, 'TestSomething')
self.assertEqual(case._testMethodName, 'test_method')
def test_discover_from_directory(self):
# Given
test_yaml_1 = textwrap.dedent("""
---
version: '1.0'
config:
host: test.domain
cases:
- name: "Basic"
tests:
- name: "Test root URL"
url: "/"
""")
test_yaml_2 = textwrap.dedent("""
---
version: '1.0'
config:
host: test.domain
cases:
- name: "A case"
tests:
- name: "Download authorization failure"
url:
template: "{data}/test"
- name: "Upload authorization failure"
url:
template: "/foo/{upload}/test"
""")
with tempfile.NamedTemporaryFile(
delete=False, prefix='test', suffix='.yml',
dir=self.temp_dir) as fh:
fh.write(test_yaml_1.encode('utf-8'))
with tempfile.NamedTemporaryFile(
delete=False, prefix='test', suffix='.yml',
dir=self.temp_dir) as fh:
fh.write(test_yaml_2.encode('utf-8'))
# When
suite = self.discoverer.discover(self.temp_dir)
# Then
self.assertIsInstance(suite, TestSuite)
self.assertEqual(suite.countTestCases(), 3)
for case in find_test_cases(suite):
self.assertIsInstance(case, unittest.TestCase)
def test_execute_single_case(self):
# Given
test_yaml = textwrap.dedent("""
---
version: '1.0'
config:
host: test.domain
cases:
- name: "Basic"
tests:
- name: "Test root URL"
url: "/"
assertions:
- name: status_code
expected: 200
""")
responses.add(
responses.GET,
'http://test.domain/',
status=200,
)
test_data = yaml.safe_load(test_yaml)
# When
suite = self.loader.load_tests_from_yaml(
test_data, '/path/to/foo.yaml')
# Then
self.assertIsInstance(suite, TestSuite)
self.assertEqual(suite.countTestCases(), 1)
case = next(find_test_cases(suite))
# Given
result = ResultCollecter()
# When
case(result)
# Then
self.assertTrue(result.wasSuccessful())
def test_execute_single_case(self):
# Given
test_yaml = textwrap.dedent("""
---
version: '1.0'
config:
host: test.domain
cases:
- name: "Basic"
tests:
- name: "Test root URL"
url: "/"
assertions:
- name: status_code
expected: 200
""")
responses.add(
responses.GET,
'http://test.domain/',
status=200,
)
test_data = yaml.safe_load(test_yaml)
# When
suite = self.loader.load_tests_from_yaml(test_data,
'/path/to/foo.yaml')
# Then
self.assertIsInstance(suite, TestSuite)
self.assertEqual(suite.countTestCases(), 1)
case = next(find_test_cases(suite))
# Given
result = ResultCollecter()
# When
case(result)
# Then
self.assertTrue(result.wasSuccessful())
def test_load_invalid_yaml_tests(self):
# Given
test_yaml = textwrap.dedent("""
---
version: '1.0'
config:
host: test.domain
tests:
- name: "Test root URL"
url: "/"
""")
test_data = yaml.safe_load(test_yaml)
# When
suite = self.loader.load_tests_from_yaml(
test_data, '/path/to/foo.yaml')
# Then
self.assertIsInstance(suite, TestSuite)
self.assertEqual(suite.countTestCases(), 1)
case = next(find_test_cases(suite))
self.assertIsInstance(case, unittest.TestCase)
self.assertIsInstance(case, ModuleImportError)
# Given
result = ResultCollecter()
# When/Then
meth = getattr(case, case._testMethodName)
with self.assertRaises(YamlParseError):
meth()
# When
case(result)
# Then
self.assertFalse(result.wasSuccessful())
self.assertEqual(len(result.errors), 1)
def test_load_invalid_yaml_tests(self):
# Given
test_yaml = textwrap.dedent("""
---
version: '1.0'
config:
host: test.domain
tests:
- name: "Test root URL"
url: "/"
""")
test_data = yaml.safe_load(test_yaml)
# When
suite = self.loader.load_tests_from_yaml(test_data,
'/path/to/foo.yaml')
# Then
self.assertIsInstance(suite, TestSuite)
self.assertEqual(suite.countTestCases(), 1)
case = next(find_test_cases(suite))
self.assertIsInstance(case, unittest.TestCase)
self.assertIsInstance(case, ModuleImportError)
# Given
result = ResultCollecter()
# When/Then
meth = getattr(case, case._testMethodName)
with self.assertRaises(YamlParseError):
meth()
# When
case(result)
# Then
self.assertFalse(result.wasSuccessful())
self.assertEqual(len(result.errors), 1)
def _run_tests(self, result, test):
pool = Pool(processes=self.process_count,
initializer=self.initializer,
maxtasksperchild=self.maxtasksperchild)
try:
def callback(collected_result):
self._handle_result(result, collected_result)
error_tests = []
call_results = []
for test_case in find_test_cases(test):
if isinstance(test_case, ModuleImportError):
error_tests.append(test_case)
else:
call_result = pool.apply_async(_run_test_in_process,
args=(test_case, ),
callback=callback)
call_results.append(call_result)
for test_case in error_tests:
collected_result = _run_test_in_process(test_case)
callback(collected_result)
finally:
pool.close()
# In some cases (when processes > CPU_CORE_COUNT), the
# combination of pool.close() and pool.join() does not
# make the Pool terminate, one or more processes remains
# alive and the program hangs.
# To work around this, We wait for all jobs submitted to
# the pool to complete, and then explicitly call
# pool.terminate() before pool.join().
while len(call_results) > 0:
call_results[0].wait(0.25)
call_results = [
call_result for call_result in call_results
if not call_result.ready()
]
pool.terminate()
pool.join()
def filter_test_suite(suite, filter_name):
"""Filter test cases in a test suite by a substring in the full dotted
test name.
Parameters
----------
suite : haas.suite.TestSuite
The test suite containing tests to be filtered.
filter_name : str
The substring of the full dotted name on which to filter. This
should not contain a leading or trailing dot.
"""
filtered_cases = []
for test in find_test_cases(suite):
type_ = type(test)
name = "{0}.{1}.{2}".format(type_.__module__, type_.__name__, test._testMethodName)
filter_internal = ".{0}.".format(filter_name)
if _is_import_error_test(test) or filter_internal in name or name.endswith(filter_internal[:-1]):
filtered_cases.append(test)
return filtered_cases
def test_load_yaml_tests(self):
# Given
test_yaml = textwrap.dedent("""
---
version: '1.0'
config:
host: test.domain
cases:
- name: "Basic"
tests:
- name: "Test root URL"
url: "/"
- name: "A case"
tests:
- name: "Download authorization failure"
url:
template: "{data}/test"
- name: "Upload authorization failure"
url:
template: "/foo/{upload}/test"
""")
test_data = yaml.safe_load(test_yaml)
# When
suite = self.loader.load_tests_from_yaml(
test_data, '/path/to/foo.yaml')
# Then
self.assertIsInstance(suite, TestSuite)
self.assertEqual(suite.countTestCases(), 3)
for case in find_test_cases(suite):
self.assertIsInstance(case, unittest.TestCase)
def _run_tests(self, result, test):
pool = Pool(processes=self.process_count,
initializer=self.initializer,
maxtasksperchild=self.maxtasksperchild)
try:
def callback(collected_result):
self._handle_result(result, collected_result)
error_tests = []
call_results = []
for test_case in find_test_cases(test):
if isinstance(test_case, ModuleImportError):
error_tests.append(test_case)
else:
call_result = pool.apply_async(
_run_test_in_process, args=(test_case,),
callback=callback)
call_results.append(call_result)
for test_case in error_tests:
collected_result = _run_test_in_process(test_case)
callback(collected_result)
finally:
pool.close()
# In some cases (when processes > CPU_CORE_COUNT), the
# combination of pool.close() and pool.join() does not
# make the Pool terminate, one or more processes remains
# alive and the program hangs.
# To work around this, We wait for all jobs submitted to
# the pool to complete, and then explicitly call
# pool.terminate() before pool.join().
while len(call_results) > 0:
call_results[0].wait(0.25)
call_results = [call_result for call_result in call_results
if not call_result.ready()]
pool.terminate()
pool.join()
def test_load_yaml_tests(self):
# Given
test_yaml = textwrap.dedent("""
---
version: '1.0'
config:
host: test.domain
cases:
- name: "Basic"
tests:
- name: "Test root URL"
url: "/"
- name: "A case"
tests:
- name: "Download authorization failure"
url:
template: "{data}/test"
- name: "Upload authorization failure"
url:
template: "/foo/{upload}/test"
""")
test_data = yaml.safe_load(test_yaml)
# When
suite = self.loader.load_tests_from_yaml(test_data,
'/path/to/foo.yaml')
# Then
self.assertIsInstance(suite, TestSuite)
self.assertEqual(suite.countTestCases(), 3)
for case in find_test_cases(suite):
self.assertIsInstance(case, unittest.TestCase)
def filter_test_suite(suite, filter_name):
"""Filter test cases in a test suite by a substring in the full dotted
test name.
Parameters
----------
suite : haas.suite.TestSuite
The test suite containing tests to be filtered.
filter_name : str
The substring of the full dotted name on which to filter. This
should not contain a leading or trailing dot.
"""
filtered_cases = []
for test in find_test_cases(suite):
type_ = type(test)
name = '{0}.{1}.{2}'.format(
type_.__module__, type_.__name__, test._testMethodName)
filter_internal = '.{0}.'.format(filter_name)
if _is_import_error_test(test) or \
filter_internal in name or \
name.endswith(filter_internal[:-1]):
filtered_cases.append(test)
return filtered_cases
def get_test_cases(self, suite):
for test in find_test_cases(suite):
yield test
def test_discover_creates_importerror_testcase(self):
with cd(self.tempdir):
suite = Discoverer(Loader()).discover(self.tempdir, self.tempdir)
self.assertEqual(suite.countTestCases(), 3)
case_names = [type(case).__name__ for case in find_test_cases(suite)]
def get_test_cases(self, suite):
for test in find_test_cases(suite):
yield test