def test_matching_multiple_tags(self): loader = tinctest.TINCTestLoader() test_suite = loader.loadTestsFromName('tinctest.test.test_core.MockTINCTestCaseForLoaderDiscovery.test_lacking_product_version') filtered_test_suite = loader._filter_and_expand(test_suite, TINCDiscoveryQueryHandler(['tags=storage', 'tags=executor'])) self.assertEquals(len(filtered_test_suite._tests), 1) filtered_test_suite = loader._filter_and_expand(test_suite, TINCDiscoveryQueryHandler(['tags=storage', 'tags=optimizer'])) self.assertEquals(len(filtered_test_suite._tests), 0)
def test_matching_maintainer(self): loader = tinctest.TINCTestLoader() test_suite = loader.loadTestsFromName('tinctest.test.test_core.MockTINCTestCaseForLoaderDiscovery.test_lacking_product_version') filtered_test_suite = loader._filter_and_expand(test_suite, TINCDiscoveryQueryHandler(['maintainer=balasr3'])) self.assertEquals(len(filtered_test_suite._tests), 1) filtered_test_suite = loader._filter_and_expand(test_suite, TINCDiscoveryQueryHandler(['maintainer=kumara64'])) self.assertEquals(len(filtered_test_suite._tests), 0)
def test_negative_queries(self): handler = TINCDiscoveryQueryHandler("class=test_smoke*") test_queries = [] test_queries.append("classtest_smoke* and moduletestsmoke*") # Partial match should error out test_queries.append("class=test_smoke* remaining text") # Unsupported operator test_queries.append("class=test* XOR package=storage.*") test_queries.append("class = test* AND module = test* OR packagetest*") # Mixed cases not for operators supported test_queries.append("claSS=test* oR Package=storage.*") test_queries.append("class=test* and module=test* And package=storage") # TBD: Following query currently passes , it should throw an exception instead # test_queries.append("class = test*AND module = test*") # Unclosed quotes in predicate should error out test_queries.append("class='test*") # Unmatched quotes in predicate should error out test_queries.append("class='test*\"") #hanging operators should error out test_queries.append("class=test* and method=test* or") for query in test_queries: try: handler = TINCDiscoveryQueryHandler(query) except TINCDiscoveryException: continue self.fail("Query %s did not throw an exception" %query)
def test_discovery_with_single_predicate(self): query_handler = TINCDiscoveryQueryHandler("class=*SmokeTests*") pwd = os.path.dirname(inspect.getfile(self.__class__)) test_dir = os.path.join(pwd, 'discovery') tinc_test_result = self._discover_and_run_tests(start_dirs = [test_dir], patterns =['test*'], top_level_dir = None, query_handler = query_handler) # Should have run four smoke tests - 2 each from test_smoke_*.py in mockstorage/uao and mockquery/cardinality self.assertEquals(tinc_test_result.testsRun, 4) # Predicate on module query_handler = TINCDiscoveryQueryHandler("module=*_functional_*") pwd = os.path.dirname(inspect.getfile(self.__class__)) test_dir = os.path.join(pwd, 'discovery') tinc_test_result = self._discover_and_run_tests(start_dirs = [test_dir], patterns =['test*'], top_level_dir = None, query_handler = query_handler) # Should have run four functional tests - 2 each from *FunctionalTests in test_functional_*.py in mockstorage/uao and mockquery/cardinality self.assertEquals(tinc_test_result.testsRun, 4) # Predicate on package query_handler = TINCDiscoveryQueryHandler("package=*query.*") pwd = os.path.dirname(inspect.getfile(self.__class__)) test_dir = os.path.join(pwd, 'discovery') tinc_test_result = self._discover_and_run_tests(start_dirs = [test_dir], patterns =['test*'], top_level_dir = None, query_handler = query_handler) # Should have run only tests from the package mockquery/* self.assertEquals(tinc_test_result.testsRun, 7) # predicate on test method query_handler = TINCDiscoveryQueryHandler("method=*functional*") pwd = os.path.dirname(inspect.getfile(self.__class__)) test_dir = os.path.join(pwd, 'discovery') tinc_test_result = self._discover_and_run_tests(start_dirs = [test_dir], patterns =['test*'], top_level_dir = None, query_handler = query_handler) # Should have run only four functional tests (test_functional_*) from *FunctionalTests in the modules test_functional_*.py in # mockquery/cardinality and mockstorage/uao self.assertEquals(tinc_test_result.testsRun, 4) # predicate on metadtata query_handler = TINCDiscoveryQueryHandler("tags = stress") pwd = os.path.dirname(inspect.getfile(self.__class__)) test_dir = os.path.join(pwd, 'discovery') tinc_test_result = self._discover_and_run_tests(start_dirs = [test_dir], patterns =['test*'], top_level_dir = None, query_handler = query_handler) # Should have run the two test methods tagged 'stress' in test_uao.py and test_cardinality.py self.assertEquals(tinc_test_result.testsRun, 2)
def test_single_predicate_not_equals(self): query_handler = TINCDiscoveryQueryHandler("class != *SmokeTests*") pwd = os.path.dirname(inspect.getfile(self.__class__)) test_dir = os.path.join(pwd, 'discovery') tinc_test_result = self._discover_and_run_tests(start_dirs = [test_dir], patterns =['test*'], top_level_dir = None, query_handler = query_handler) # Should have run everything except the tests in test_smoke_*.py self.assertEquals(tinc_test_result.testsRun, 10) # Predicate on module query_handler = TINCDiscoveryQueryHandler("module != *_functional_*") pwd = os.path.dirname(inspect.getfile(self.__class__)) test_dir = os.path.join(pwd, 'discovery') tinc_test_result = self._discover_and_run_tests(start_dirs = [test_dir], patterns =['test*'], top_level_dir = None, query_handler = query_handler) # Should have run everything except the tests in test_functional*.py in mockquery and mockstorage self.assertEquals(tinc_test_result.testsRun, 10) # Predicate on package query_handler = TINCDiscoveryQueryHandler("package != *query.*") pwd = os.path.dirname(inspect.getfile(self.__class__)) test_dir = os.path.join(pwd, 'discovery') tinc_test_result = self._discover_and_run_tests(start_dirs = [test_dir], patterns =['test*'], top_level_dir = None, query_handler = query_handler) # Should have run all the tests from within mockstorage self.assertEquals(tinc_test_result.testsRun, 7) # predicate on test method query_handler = TINCDiscoveryQueryHandler("method != *functional*") pwd = os.path.dirname(inspect.getfile(self.__class__)) test_dir = os.path.join(pwd, 'discovery') tinc_test_result = self._discover_and_run_tests(start_dirs = [test_dir], patterns =['test*'], top_level_dir = None, query_handler = query_handler) # Should have run all tests in all the modules except for the tests in test_functioanl_*.py of the form test_functional* self.assertEquals(tinc_test_result.testsRun, 10) # predicate on metadtata query_handler = TINCDiscoveryQueryHandler("tags != stress") pwd = os.path.dirname(inspect.getfile(self.__class__)) test_dir = os.path.join(pwd, 'discovery') tinc_test_result = self._discover_and_run_tests(start_dirs = [test_dir], patterns =['test*'], top_level_dir = None, query_handler = query_handler) # Should have run all the tests except for the tests tagged 'stress' within test_uao.py and test_cardinality.py self.assertEquals(tinc_test_result.testsRun, 12)
def test_multiple_predicates_within_a_query(self): # Run all smoke tests query_handler = TINCDiscoveryQueryHandler("tags = smoke or method = *smoke* or class = *SmokeTests*") pwd = os.path.dirname(inspect.getfile(self.__class__)) test_dir = os.path.join(pwd, 'discovery') tinc_test_result = self._discover_and_run_tests(start_dirs = [test_dir], patterns =['test*'], top_level_dir = None, query_handler = query_handler) # Should have run all the tests from within test_module*.py and one test tagged 'smoke' in test_uao.py and test_cardinality.py self.assertEquals(tinc_test_result.testsRun, 6) # Run all smoke tests that are not tagged query_handler = TINCDiscoveryQueryHandler("method = *smoke* or class = *SmokeTests*") pwd = os.path.dirname(inspect.getfile(self.__class__)) test_dir = os.path.join(pwd, 'discovery') tinc_test_result = self._discover_and_run_tests(start_dirs = [test_dir], patterns =['test*'], top_level_dir = None, query_handler = query_handler) # Should have run four tests, two each from within test_smoke_*.py in mockstorage and mockquery self.assertEquals(tinc_test_result.testsRun, 4) # Run all functional tests query_handler = TINCDiscoveryQueryHandler("tags = functional or method = *functional* or class = *FunctionalTests*") pwd = os.path.dirname(inspect.getfile(self.__class__)) test_dir = os.path.join(pwd, 'discovery') tinc_test_result = self._discover_and_run_tests(start_dirs = [test_dir], patterns =['test*'], top_level_dir = None, query_handler = query_handler) # Should have run all the functional tests , four from test_functional_*.py and one test each tagged 'functional' # from within test_uao.py and test_cardinality.py self.assertEquals(tinc_test_result.testsRun, 6) # With AND predicates query_handler = TINCDiscoveryQueryHandler("method = *functional* and class = *FunctionalTests*") pwd = os.path.dirname(inspect.getfile(self.__class__)) test_dir = os.path.join(pwd, 'discovery') tinc_test_result = self._discover_and_run_tests(start_dirs = [test_dir], patterns =['test*'], top_level_dir = None, query_handler = query_handler) # Should have run only four tests from within test_functional*.py in mockquery and mockstorage self.assertEquals(tinc_test_result.testsRun, 4) # Run all the tests except for stress tests query_handler = TINCDiscoveryQueryHandler("module = test_* and tags != stress") pwd = os.path.dirname(inspect.getfile(self.__class__)) test_dir = os.path.join(pwd, 'discovery') tinc_test_result = self._discover_and_run_tests(start_dirs = [test_dir], patterns =['test*'], top_level_dir = None, query_handler = query_handler) # Run all the tests from within mockquery and mockstorage except the tests tagged 'stress' within # test_uao.py and test_cardinality.py self.assertEquals(tinc_test_result.testsRun, 12)
def test_sql_test_case_with_discovery_queries(self): tinc_test_loader = tinctest.TINCTestLoader() pwd = os.path.dirname(inspect.getfile(self.__class__)) test_dir = os.path.join(pwd, 'sql_pattern') tinc_test_suite = tinc_test_loader.discover(start_dirs = [test_dir], patterns = ['sql_pattern.py'], top_level_dir = None, query_handler = TINCDiscoveryQueryHandler(['method=test_functional_*'])) test_case = None test_result = None with closing(_WritelnDecorator(StringIO())) as buffer: tinc_test_runner = TINCTestRunner(stream = buffer, descriptions = True, verbosity = 1) test_result = tinc_test_runner.run(tinc_test_suite) self.assertEqual(test_result.testsRun, 6) self.assertEqual(len(test_result.skipped), 6) # Queries using metadata from sql files tinc_test_loader = tinctest.TINCTestLoader() pwd = os.path.dirname(inspect.getfile(self.__class__)) test_dir = os.path.join(pwd, 'sql_pattern') tinc_test_suite = tinc_test_loader.discover(start_dirs = [test_dir], patterns = ['sql_pattern.py'], top_level_dir = None, query_handler = TINCDiscoveryQueryHandler(['method=test_functional_* and tags != long'])) test_case = None test_result = None with closing(_WritelnDecorator(StringIO())) as buffer: tinc_test_runner = TINCTestRunner(stream = buffer, descriptions = True, verbosity = 1) test_result = tinc_test_runner.run(tinc_test_suite) self.assertEqual(test_result.testsRun, 3) self.assertEqual(len(test_result.skipped), 3)
def test_multiple_queries(self): # Run all tests in class *SmokeTests* and starts with test_smoke_* queries = ['class = *SmokeTests*', 'method=test_smoke*'] query_handler = TINCDiscoveryQueryHandler(queries) pwd = os.path.dirname(inspect.getfile(self.__class__)) test_dir = os.path.join(pwd, 'discovery') tinc_test_result = self._discover_and_run_tests(start_dirs = [test_dir], patterns =['test*'], top_level_dir = None, query_handler = query_handler) self.assertEquals(tinc_test_result.testsRun, 4)
def test_queries_with_patterns(self): # patterns should be applied first queries = ['method = test_*'] query_handler = TINCDiscoveryQueryHandler(queries) pwd = os.path.dirname(inspect.getfile(self.__class__)) test_dir = os.path.join(pwd, 'discovery', 'mockquery') tinc_test_result = self._discover_and_run_tests(start_dirs = [test_dir], patterns =['test_functional*'], top_level_dir = None, query_handler = query_handler) # Only run two tests within mockquery since the patterns should be applied first and the queries should be applied # on the tests that match the pattern self.assertEquals(tinc_test_result.testsRun, 2)
def test_apply_queries_with_multiple_predicates(self): queries = [] queries.append("method=test_smoke* OR class = Mock* or class = MockTINC*") queries.append("package=storage.uao.* OR method=test_smoke*") queries.append("package=storage.uao.* and method=*testsmoke* and module!=test_discovery or method=test_smoke_01") #queries.append("package != storage.uao or method=*testsmoke* and module != test_discovery and class != Mock* and tags = smoke and author != alice") handler = TINCDiscoveryQueryHandler(queries) tinc_test_case = MockTINCTestCase('test_smoke_01') self.assertTrue(handler.apply_queries(tinc_test_case)) queries = [] queries.append("method=test_smoke* OR class = Mock* or class = MockTINC*") queries.append("package=storage.uao.* OR method=test_smoke*") queries.append("package=storage.uao.* and method=*testsmoke* and module!=test_discovery or method=test_smoke_01") queries.append("package != storage.uao or method=*testsmoke* and module != test_discovery and class != Mock*") # Introduce false at the end queries.append("package != storage and method != test_smoke_01") handler = TINCDiscoveryQueryHandler(queries) tinc_test_case = MockTINCTestCase('test_smoke_01') self.assertFalse(handler.apply_queries(tinc_test_case))
def test_apply_queries_with_multiple_queries(self): queries = [] queries.append("method=test_smoke*") queries.append("class=Mock*") queries.append("class='MockTINC*'") queries.append("module='test_discovery*'") queries.append("package = tinc*.test*") queries.append("method != test_smoke_02") queries.append("class != NonMock*") queries.append("package != storage.uao.*") queries.append("module != test_regression*") queries.append("author = bob") queries.append("tags = smoke") queries.append("author != alice") queries.append("tags != functional") handler = TINCDiscoveryQueryHandler(queries) tinc_test_case = MockTINCTestCase('test_smoke_01') self.assertTrue(handler.apply_queries(tinc_test_case)) queries = [] queries.append("method=test_smoke*") queries.append("class=Mock*") queries.append("class='MockTINC*'") queries.append("module='test_discovery*'") queries.append("package = storage.uao.*") handler = TINCDiscoveryQueryHandler(queries) tinc_test_case = MockTINCTestCase('test_smoke_01') self.assertFalse(handler.apply_queries(tinc_test_case))
def test_apply_queries_with_multiple_predicates(self): queries = [] queries.append( "method=test_smoke* OR class = Mock* or class = MockTINC*") queries.append("package=storage.uao.* OR method=test_smoke*") queries.append( "package=storage.uao.* and method=*testsmoke* and module!=test_discovery or method=test_smoke_01" ) #queries.append("package != storage.uao or method=*testsmoke* and module != test_discovery and class != Mock* and tags = smoke and author != alice") handler = TINCDiscoveryQueryHandler(queries) tinc_test_case = MockTINCTestCase('test_smoke_01') self.assertTrue(handler.apply_queries(tinc_test_case)) queries = [] queries.append( "method=test_smoke* OR class = Mock* or class = MockTINC*") queries.append("package=storage.uao.* OR method=test_smoke*") queries.append( "package=storage.uao.* and method=*testsmoke* and module!=test_discovery or method=test_smoke_01" ) queries.append( "package != storage.uao or method=*testsmoke* and module != test_discovery and class != Mock*" ) # Introduce false at the end queries.append("package != storage and method != test_smoke_01") handler = TINCDiscoveryQueryHandler(queries) tinc_test_case = MockTINCTestCase('test_smoke_01') self.assertFalse(handler.apply_queries(tinc_test_case))
def test_suite_construction_with_discover_and_tinc_queries(self): tinc_test_loader = tinctest.TINCTestLoader() pwd = os.path.dirname(inspect.getfile(self.__class__)) test_dir = os.path.join(pwd, 'data_provider') query_handler = TINCDiscoveryQueryHandler("tags=tag1") tinc_test_suite = tinc_test_loader.discover( start_dirs=[test_dir], patterns=['test_*.py'], top_level_dir=test_dir, query_handler=query_handler) with closing(_WritelnDecorator(StringIO())) as buffer: tinc_test_result = tinctest.TINCTestResultSet(buffer, True, 1) tinc_test_suite.run(tinc_test_result) # This should have filtered 4 tests and hence run 7 tests self.assertEquals(tinc_test_result.testsRun, 7)
def test_filtered_tests_with_dryrun(self): """ Test that tests filtered out with tinc queries are reported correctly in dryrun """ tinc_test_loader = tinctest.TINCTestLoader() pwd = os.path.dirname(inspect.getfile(self.__class__)) test_dir = os.path.join(pwd, 'mocktests') query_handler = TINCDiscoveryQueryHandler("tags != hawq") tinc_test_suite = tinc_test_loader.discover( start_dirs=[test_dir], patterns=['dryrun_test_sample2.py'], top_level_dir=test_dir, query_handler=query_handler, dryrun=True) with closing(_WritelnDecorator(StringIO())) as buffer: tinc_test_result = tinctest.TINCTestResultSet(buffer, True, 1) tinc_test_suite.run(tinc_test_result) self.assertEquals(tinc_test_result.testsRun, 3) # This variable should be 0 even after the tests are run in the dryrun mode since # we do not expect to run the test code self.assertEquals(tinctest.test.dryrun.tests_run_count, 0) # Two tests should be skipped one because of skip metadata and one because of filtering out self.assertEquals(len(tinc_test_result.skipped), 2) metadata_skip_test_tuple = None queries_skip_test_tuple = None # tinc_test_result.skipped will be a list of tuple containing test instance and the skip message for test_tuple in tinc_test_result.skipped: if test_tuple[0]._testMethodName == 'test_03': metadata_skip_test_tuple = test_tuple elif test_tuple[0]._testMethodName == 'test_01': queries_skip_test_tuple = test_tuple self.assertIsNotNone(metadata_skip_test_tuple) self.assertIsNotNone(queries_skip_test_tuple) self.assertTrue( "Filtering out test as it did not satisy tinc queries check" in queries_skip_test_tuple[1]) self.assertTrue("just skipping" in metadata_skip_test_tuple[1])
def test_positive_queries(self): test_queries = [] test_queries.append("class=test_smoke*") test_queries.append("module=test_smoke*") test_queries.append("method=test_smoke*") test_queries.append("package=storage.uao.*") test_queries.append("class=* and module=* or package=* and class=SmokeTests*") test_queries.append("class=* AND module=* OR package=* AND class=SmokeTests*") test_queries.append("class=* and module=* or package=* and class=SmokeTests*") test_queries.append("""class != * and module != * or package != * and class != SmokeTests* AND class = test""") test_queries.append("class != \"test_smoke\"") test_queries.append("class != 'test_smoke'") test_queries.append("""class = 'test' AND module=\"test\"""") test_queries.append("class = test* and method=test*") test_queries.append("tags = tag1") handler = TINCDiscoveryQueryHandler(test_queries) self.assertEquals(len(handler.query_list), len(test_queries))
def test_all_tests_loaded(self): tinc_test_loader = tinctest.TINCTestLoader() pwd = os.path.dirname(inspect.getfile(self.__class__)) test_dir1 = os.path.join(pwd, 'dryrun/mocktests') tinc_test_suite = tinc_test_loader.discover(start_dirs = [test_dir1], patterns = ['dryrun_test_sample2.py'], top_level_dir = None) self.assertEquals(tinc_test_suite.countTestCases(), 3) flat_test_suite = TINCTestSuite() tinc_test_suite.flatten(flat_test_suite) for test in flat_test_suite._tests: self.assertTrue(test.__class__._all_tests_loaded) test_dir1 = os.path.join(pwd, 'dryrun/mocktests') tinc_test_suite = tinc_test_loader.discover(start_dirs = [test_dir1], patterns = ['dryrun_test_sample2.py'], top_level_dir = None, query_handler = TINCDiscoveryQueryHandler('tags=hawq')) self.assertEquals(tinc_test_suite.countTestCases(), 1) for test in flat_test_suite._tests: self.assertFalse(test.__class__._all_tests_loaded)
def test_apply_queries_with_single_query(self): handler = TINCDiscoveryQueryHandler("method=test_smoke*") tinc_test_case = MockTINCTestCase('test_smoke_01') self.assertTrue(handler.apply_queries(tinc_test_case)) # metadata equals predicate handler = TINCDiscoveryQueryHandler("author=bob") tinc_test_case = MockTINCTestCase('test_smoke_01') self.assertTrue(handler.apply_queries(tinc_test_case)) # metadata equals predicate with dash (typically used for bugs) handler = TINCDiscoveryQueryHandler("tags=bug-1") tinc_test_case = MockTINCTestCase('test_smoke_01') self.assertTrue(handler.apply_queries(tinc_test_case)) #metadata not equals predicate handler = TINCDiscoveryQueryHandler("tags != functional") tinc_test_case = MockTINCTestCase('test_smoke_01') self.assertTrue(handler.apply_queries(tinc_test_case)) # metadata non-match # metadata equals predicate handler = TINCDiscoveryQueryHandler("author=alice") tinc_test_case = MockTINCTestCase('test_smoke_01') self.assertFalse(handler.apply_queries(tinc_test_case)) #metadata not equals predicate handler = TINCDiscoveryQueryHandler("tags != smoke") tinc_test_case = MockTINCTestCase('test_smoke_01') self.assertFalse(handler.apply_queries(tinc_test_case)) # non existing metadata should return False handler = TINCDiscoveryQueryHandler("non_existing_tags = smoke") # not equals on a non existing metadata will currently return True # We will have to decide whether or not to throw an exception for such tests tinc_test_case = MockTINCTestCase('test_smoke_01') self.assertFalse(handler.apply_queries(tinc_test_case))