def test_discover_tests_runtime_error_on_import(self): # When with cd(self.tempdir): suite = Discoverer(Loader()).discover('TestSomething', None) # Then self.assertEqual(suite.countTestCases(), 1) case, = find_test_cases(suite) self.assertIsInstance(case, ModuleImportError) self.assertEqual(case._testMethodName, 'test_error')
def test_discover_error_emacs_recovery_file(self): # When with cd(self.tempdir): suite = Discoverer(Loader()).discover('TestSomething', None) # Then self.assertEqual(suite.countTestCases(), 1) case, = find_test_cases(suite) self.assertIsInstance(case, unittest.TestCase) self.assertEqual(case._testMethodName, 'test_method')
def test_discover_tests(self): # When with cd(self.tempdir): suite = Discoverer(Loader()).discover(self.tempdir, self.tempdir) # Then self.assertEqual(suite.countTestCases(), 1) case, = find_test_cases(suite) self.assertEqual(type(case).__name__, 'TestExpected') self.assertEqual(case._testMethodName, 'test_expected')
def test_discover_tests_with_dot_slash(self): # Given start = './fixture/package/test_something.py' # When with cd(self.tempdir): suite = Discoverer(Loader()).discover(start, None) # Then self.assertEqual(suite.countTestCases(), 1) case, = find_test_cases(suite) self.assertEqual(type(case).__name__, 'TestSomething') self.assertEqual(case._testMethodName, 'test_method')
def test_case_dunder_str(self): # Given test_yaml = textwrap.dedent(""" --- version: '1.0' config: host: test.domain cases: - name: "Basic" tests: - name: "Test root URL" url: "/" """) test_data = yaml.safe_load(test_yaml) # When suite = self.loader.load_tests_from_yaml( test_data, '/path/to/foo.yaml') # Then self.assertIsInstance(suite, TestSuite) self.assertEqual(suite.countTestCases(), 1) case = next(find_test_cases(suite)) # When case_str = str(case) description = case.shortDescription() # Then self.assertEqual(case_str, "'Basic:Test root URL' (/path/to/foo.yaml)") self.assertIsNone(description)
def test_load_yaml_tests_case_max_diff_number(self): # Given test_yaml = textwrap.dedent(""" --- version: '1.0' config: host: test.domain cases: - name: "Basic" max-diff: 1234 tests: - name: "Test root URL" url: "/" - name: "Test sub URL" url: "/sub/" """) test_data = yaml.safe_load(test_yaml) # When suite = self.loader.load_tests_from_yaml(test_data, '/path/to/foo.yaml') # Then self.assertIsInstance(suite, TestSuite) self.assertEqual(suite.countTestCases(), 2) cls1, cls2 = [type(case) for case in find_test_cases(suite)]
def test_case_dunder_str(self): # Given test_yaml = textwrap.dedent(""" --- version: '1.0' config: host: test.domain cases: - name: "Basic" tests: - name: "Test root URL" url: "/" """) test_data = yaml.safe_load(test_yaml) # When suite = self.loader.load_tests_from_yaml(test_data, '/path/to/foo.yaml') # Then self.assertIsInstance(suite, TestSuite) self.assertEqual(suite.countTestCases(), 1) case = next(find_test_cases(suite)) # When case_str = str(case) description = case.shortDescription() # Then self.assertEqual(case_str, "'Basic:Test root URL' (/path/to/foo.yaml)") self.assertIsNone(description)
def test_load_yaml_tests_case_max_diff_number(self): # Given test_yaml = textwrap.dedent(""" --- version: '1.0' config: host: test.domain cases: - name: "Basic" max-diff: 1234 tests: - name: "Test root URL" url: "/" - name: "Test sub URL" url: "/sub/" """) test_data = yaml.safe_load(test_yaml) # When suite = self.loader.load_tests_from_yaml( test_data, '/path/to/foo.yaml') # Then self.assertIsInstance(suite, TestSuite) self.assertEqual(suite.countTestCases(), 2) cls1, cls2 = [type(case) for case in find_test_cases(suite)] self.assertIs(cls1, cls2) self.assertEqual(cls1.maxDiff, 1234)
def test_discover_creates_importerror_testcase(self): with cd(self.tempdir): suite = Discoverer(Loader()).discover( self.tempdir, self.tempdir) self.assertEqual(suite.countTestCases(), 3) case_names = [ type(case).__name__ for case in find_test_cases(suite)] self.assertEqual( case_names, ['ModuleImportError', 'TestSomething', 'TestSomething'])
def test_discover_no_such_file(self): # Given test_filename = os.path.join(self.temp_dir, 'dont_exist.yml') # When suite = self.discoverer.discover(test_filename) # Then self.assertIsInstance(suite, TestSuite) self.assertEqual(suite.countTestCases(), 0) for case in find_test_cases(suite): self.assertIsInstance(case, unittest.TestCase)
def test_discover_from_directory(self): # Given test_yaml_1 = textwrap.dedent(""" --- version: '1.0' config: host: test.domain cases: - name: "Basic" tests: - name: "Test root URL" url: "/" """) test_yaml_2 = textwrap.dedent(""" --- version: '1.0' config: host: test.domain cases: - name: "A case" tests: - name: "Download authorization failure" url: template: "{data}/test" - name: "Upload authorization failure" url: template: "/foo/{upload}/test" """) with tempfile.NamedTemporaryFile(delete=False, prefix='test', suffix='.yml', dir=self.temp_dir) as fh: fh.write(test_yaml_1.encode('utf-8')) with tempfile.NamedTemporaryFile(delete=False, prefix='test', suffix='.yml', dir=self.temp_dir) as fh: fh.write(test_yaml_2.encode('utf-8')) # When suite = self.discoverer.discover(self.temp_dir) # Then self.assertIsInstance(suite, TestSuite) self.assertEqual(suite.countTestCases(), 3) for case in find_test_cases(suite): self.assertIsInstance(case, unittest.TestCase)
def test_discover_from_directory(self): # Given test_yaml_1 = textwrap.dedent(""" --- version: '1.0' config: host: test.domain cases: - name: "Basic" tests: - name: "Test root URL" url: "/" """) test_yaml_2 = textwrap.dedent(""" --- version: '1.0' config: host: test.domain cases: - name: "A case" tests: - name: "Download authorization failure" url: template: "{data}/test" - name: "Upload authorization failure" url: template: "/foo/{upload}/test" """) with tempfile.NamedTemporaryFile( delete=False, prefix='test', suffix='.yml', dir=self.temp_dir) as fh: fh.write(test_yaml_1.encode('utf-8')) with tempfile.NamedTemporaryFile( delete=False, prefix='test', suffix='.yml', dir=self.temp_dir) as fh: fh.write(test_yaml_2.encode('utf-8')) # When suite = self.discoverer.discover(self.temp_dir) # Then self.assertIsInstance(suite, TestSuite) self.assertEqual(suite.countTestCases(), 3) for case in find_test_cases(suite): self.assertIsInstance(case, unittest.TestCase)
def test_execute_single_case(self): # Given test_yaml = textwrap.dedent(""" --- version: '1.0' config: host: test.domain cases: - name: "Basic" tests: - name: "Test root URL" url: "/" assertions: - name: status_code expected: 200 """) responses.add( responses.GET, 'http://test.domain/', status=200, ) test_data = yaml.safe_load(test_yaml) # When suite = self.loader.load_tests_from_yaml( test_data, '/path/to/foo.yaml') # Then self.assertIsInstance(suite, TestSuite) self.assertEqual(suite.countTestCases(), 1) case = next(find_test_cases(suite)) # Given result = ResultCollecter() # When case(result) # Then self.assertTrue(result.wasSuccessful())
def test_execute_single_case(self): # Given test_yaml = textwrap.dedent(""" --- version: '1.0' config: host: test.domain cases: - name: "Basic" tests: - name: "Test root URL" url: "/" assertions: - name: status_code expected: 200 """) responses.add( responses.GET, 'http://test.domain/', status=200, ) test_data = yaml.safe_load(test_yaml) # When suite = self.loader.load_tests_from_yaml(test_data, '/path/to/foo.yaml') # Then self.assertIsInstance(suite, TestSuite) self.assertEqual(suite.countTestCases(), 1) case = next(find_test_cases(suite)) # Given result = ResultCollecter() # When case(result) # Then self.assertTrue(result.wasSuccessful())
def test_load_invalid_yaml_tests(self): # Given test_yaml = textwrap.dedent(""" --- version: '1.0' config: host: test.domain tests: - name: "Test root URL" url: "/" """) test_data = yaml.safe_load(test_yaml) # When suite = self.loader.load_tests_from_yaml( test_data, '/path/to/foo.yaml') # Then self.assertIsInstance(suite, TestSuite) self.assertEqual(suite.countTestCases(), 1) case = next(find_test_cases(suite)) self.assertIsInstance(case, unittest.TestCase) self.assertIsInstance(case, ModuleImportError) # Given result = ResultCollecter() # When/Then meth = getattr(case, case._testMethodName) with self.assertRaises(YamlParseError): meth() # When case(result) # Then self.assertFalse(result.wasSuccessful()) self.assertEqual(len(result.errors), 1)
def test_load_invalid_yaml_tests(self): # Given test_yaml = textwrap.dedent(""" --- version: '1.0' config: host: test.domain tests: - name: "Test root URL" url: "/" """) test_data = yaml.safe_load(test_yaml) # When suite = self.loader.load_tests_from_yaml(test_data, '/path/to/foo.yaml') # Then self.assertIsInstance(suite, TestSuite) self.assertEqual(suite.countTestCases(), 1) case = next(find_test_cases(suite)) self.assertIsInstance(case, unittest.TestCase) self.assertIsInstance(case, ModuleImportError) # Given result = ResultCollecter() # When/Then meth = getattr(case, case._testMethodName) with self.assertRaises(YamlParseError): meth() # When case(result) # Then self.assertFalse(result.wasSuccessful()) self.assertEqual(len(result.errors), 1)
def _run_tests(self, result, test): pool = Pool(processes=self.process_count, initializer=self.initializer, maxtasksperchild=self.maxtasksperchild) try: def callback(collected_result): self._handle_result(result, collected_result) error_tests = [] call_results = [] for test_case in find_test_cases(test): if isinstance(test_case, ModuleImportError): error_tests.append(test_case) else: call_result = pool.apply_async(_run_test_in_process, args=(test_case, ), callback=callback) call_results.append(call_result) for test_case in error_tests: collected_result = _run_test_in_process(test_case) callback(collected_result) finally: pool.close() # In some cases (when processes > CPU_CORE_COUNT), the # combination of pool.close() and pool.join() does not # make the Pool terminate, one or more processes remains # alive and the program hangs. # To work around this, We wait for all jobs submitted to # the pool to complete, and then explicitly call # pool.terminate() before pool.join(). while len(call_results) > 0: call_results[0].wait(0.25) call_results = [ call_result for call_result in call_results if not call_result.ready() ] pool.terminate() pool.join()
def filter_test_suite(suite, filter_name): """Filter test cases in a test suite by a substring in the full dotted test name. Parameters ---------- suite : haas.suite.TestSuite The test suite containing tests to be filtered. filter_name : str The substring of the full dotted name on which to filter. This should not contain a leading or trailing dot. """ filtered_cases = [] for test in find_test_cases(suite): type_ = type(test) name = "{0}.{1}.{2}".format(type_.__module__, type_.__name__, test._testMethodName) filter_internal = ".{0}.".format(filter_name) if _is_import_error_test(test) or filter_internal in name or name.endswith(filter_internal[:-1]): filtered_cases.append(test) return filtered_cases
def test_load_yaml_tests(self): # Given test_yaml = textwrap.dedent(""" --- version: '1.0' config: host: test.domain cases: - name: "Basic" tests: - name: "Test root URL" url: "/" - name: "A case" tests: - name: "Download authorization failure" url: template: "{data}/test" - name: "Upload authorization failure" url: template: "/foo/{upload}/test" """) test_data = yaml.safe_load(test_yaml) # When suite = self.loader.load_tests_from_yaml( test_data, '/path/to/foo.yaml') # Then self.assertIsInstance(suite, TestSuite) self.assertEqual(suite.countTestCases(), 3) for case in find_test_cases(suite): self.assertIsInstance(case, unittest.TestCase)
def _run_tests(self, result, test): pool = Pool(processes=self.process_count, initializer=self.initializer, maxtasksperchild=self.maxtasksperchild) try: def callback(collected_result): self._handle_result(result, collected_result) error_tests = [] call_results = [] for test_case in find_test_cases(test): if isinstance(test_case, ModuleImportError): error_tests.append(test_case) else: call_result = pool.apply_async( _run_test_in_process, args=(test_case,), callback=callback) call_results.append(call_result) for test_case in error_tests: collected_result = _run_test_in_process(test_case) callback(collected_result) finally: pool.close() # In some cases (when processes > CPU_CORE_COUNT), the # combination of pool.close() and pool.join() does not # make the Pool terminate, one or more processes remains # alive and the program hangs. # To work around this, We wait for all jobs submitted to # the pool to complete, and then explicitly call # pool.terminate() before pool.join(). while len(call_results) > 0: call_results[0].wait(0.25) call_results = [call_result for call_result in call_results if not call_result.ready()] pool.terminate() pool.join()
def test_load_yaml_tests(self): # Given test_yaml = textwrap.dedent(""" --- version: '1.0' config: host: test.domain cases: - name: "Basic" tests: - name: "Test root URL" url: "/" - name: "A case" tests: - name: "Download authorization failure" url: template: "{data}/test" - name: "Upload authorization failure" url: template: "/foo/{upload}/test" """) test_data = yaml.safe_load(test_yaml) # When suite = self.loader.load_tests_from_yaml(test_data, '/path/to/foo.yaml') # Then self.assertIsInstance(suite, TestSuite) self.assertEqual(suite.countTestCases(), 3) for case in find_test_cases(suite): self.assertIsInstance(case, unittest.TestCase)
def filter_test_suite(suite, filter_name): """Filter test cases in a test suite by a substring in the full dotted test name. Parameters ---------- suite : haas.suite.TestSuite The test suite containing tests to be filtered. filter_name : str The substring of the full dotted name on which to filter. This should not contain a leading or trailing dot. """ filtered_cases = [] for test in find_test_cases(suite): type_ = type(test) name = '{0}.{1}.{2}'.format( type_.__module__, type_.__name__, test._testMethodName) filter_internal = '.{0}.'.format(filter_name) if _is_import_error_test(test) or \ filter_internal in name or \ name.endswith(filter_internal[:-1]): filtered_cases.append(test) return filtered_cases
def get_test_cases(self, suite): for test in find_test_cases(suite): yield test
def test_discover_creates_importerror_testcase(self): with cd(self.tempdir): suite = Discoverer(Loader()).discover(self.tempdir, self.tempdir) self.assertEqual(suite.countTestCases(), 3) case_names = [type(case).__name__ for case in find_test_cases(suite)]