def test_matching_multiple_tags(self):
loader = tinctest.TINCTestLoader()
test_suite = loader.loadTestsFromName('tinctest.test.test_core.MockTINCTestCaseForLoaderDiscovery.test_lacking_product_version')
filtered_test_suite = loader._filter_and_expand(test_suite, TINCDiscoveryQueryHandler(['tags=storage', 'tags=executor']))
self.assertEquals(len(filtered_test_suite._tests), 1)
filtered_test_suite = loader._filter_and_expand(test_suite, TINCDiscoveryQueryHandler(['tags=storage', 'tags=optimizer']))
self.assertEquals(len(filtered_test_suite._tests), 0)
def test_matching_maintainer(self):
loader = tinctest.TINCTestLoader()
test_suite = loader.loadTestsFromName('tinctest.test.test_core.MockTINCTestCaseForLoaderDiscovery.test_lacking_product_version')
filtered_test_suite = loader._filter_and_expand(test_suite, TINCDiscoveryQueryHandler(['maintainer=balasr3']))
self.assertEquals(len(filtered_test_suite._tests), 1)
filtered_test_suite = loader._filter_and_expand(test_suite, TINCDiscoveryQueryHandler(['maintainer=kumara64']))
self.assertEquals(len(filtered_test_suite._tests), 0)
def test_negative_queries(self):
handler = TINCDiscoveryQueryHandler("class=test_smoke*")
test_queries = []
test_queries.append("classtest_smoke* and moduletestsmoke*")
# Partial match should error out
test_queries.append("class=test_smoke* remaining text")
# Unsupported operator
test_queries.append("class=test* XOR package=storage.*")
test_queries.append("class = test* AND module = test* OR packagetest*")
# Mixed cases not for operators supported
test_queries.append("claSS=test* oR Package=storage.*")
test_queries.append("class=test* and module=test* And package=storage")
# TBD: Following query currently passes , it should throw an exception instead
# test_queries.append("class = test*AND module = test*")
# Unclosed quotes in predicate should error out
test_queries.append("class='test*")
# Unmatched quotes in predicate should error out
test_queries.append("class='test*\"")
#hanging operators should error out
test_queries.append("class=test* and method=test* or")
for query in test_queries:
try:
handler = TINCDiscoveryQueryHandler(query)
except TINCDiscoveryException:
continue
self.fail("Query %s did not throw an exception" %query)
def test_discovery_with_single_predicate(self):
query_handler = TINCDiscoveryQueryHandler("class=*SmokeTests*")
pwd = os.path.dirname(inspect.getfile(self.__class__))
test_dir = os.path.join(pwd, 'discovery')
tinc_test_result = self._discover_and_run_tests(start_dirs = [test_dir], patterns =['test*'],
top_level_dir = None, query_handler = query_handler)
# Should have run four smoke tests - 2 each from test_smoke_*.py in mockstorage/uao and mockquery/cardinality
self.assertEquals(tinc_test_result.testsRun, 4)
# Predicate on module
query_handler = TINCDiscoveryQueryHandler("module=*_functional_*")
pwd = os.path.dirname(inspect.getfile(self.__class__))
test_dir = os.path.join(pwd, 'discovery')
tinc_test_result = self._discover_and_run_tests(start_dirs = [test_dir], patterns =['test*'],
top_level_dir = None, query_handler = query_handler)
# Should have run four functional tests - 2 each from *FunctionalTests in test_functional_*.py in mockstorage/uao and mockquery/cardinality
self.assertEquals(tinc_test_result.testsRun, 4)
# Predicate on package
query_handler = TINCDiscoveryQueryHandler("package=*query.*")
pwd = os.path.dirname(inspect.getfile(self.__class__))
test_dir = os.path.join(pwd, 'discovery')
tinc_test_result = self._discover_and_run_tests(start_dirs = [test_dir], patterns =['test*'],
top_level_dir = None, query_handler = query_handler)
# Should have run only tests from the package mockquery/*
self.assertEquals(tinc_test_result.testsRun, 7)
# predicate on test method
query_handler = TINCDiscoveryQueryHandler("method=*functional*")
pwd = os.path.dirname(inspect.getfile(self.__class__))
test_dir = os.path.join(pwd, 'discovery')
tinc_test_result = self._discover_and_run_tests(start_dirs = [test_dir], patterns =['test*'],
top_level_dir = None, query_handler = query_handler)
# Should have run only four functional tests (test_functional_*) from *FunctionalTests in the modules test_functional_*.py in
# mockquery/cardinality and mockstorage/uao
self.assertEquals(tinc_test_result.testsRun, 4)
# predicate on metadtata
query_handler = TINCDiscoveryQueryHandler("tags = stress")
pwd = os.path.dirname(inspect.getfile(self.__class__))
test_dir = os.path.join(pwd, 'discovery')
tinc_test_result = self._discover_and_run_tests(start_dirs = [test_dir], patterns =['test*'],
top_level_dir = None, query_handler = query_handler)
# Should have run the two test methods tagged 'stress' in test_uao.py and test_cardinality.py
self.assertEquals(tinc_test_result.testsRun, 2)
def test_single_predicate_not_equals(self):
query_handler = TINCDiscoveryQueryHandler("class != *SmokeTests*")
pwd = os.path.dirname(inspect.getfile(self.__class__))
test_dir = os.path.join(pwd, 'discovery')
tinc_test_result = self._discover_and_run_tests(start_dirs = [test_dir], patterns =['test*'],
top_level_dir = None, query_handler = query_handler)
# Should have run everything except the tests in test_smoke_*.py
self.assertEquals(tinc_test_result.testsRun, 10)
# Predicate on module
query_handler = TINCDiscoveryQueryHandler("module != *_functional_*")
pwd = os.path.dirname(inspect.getfile(self.__class__))
test_dir = os.path.join(pwd, 'discovery')
tinc_test_result = self._discover_and_run_tests(start_dirs = [test_dir], patterns =['test*'],
top_level_dir = None, query_handler = query_handler)
# Should have run everything except the tests in test_functional*.py in mockquery and mockstorage
self.assertEquals(tinc_test_result.testsRun, 10)
# Predicate on package
query_handler = TINCDiscoveryQueryHandler("package != *query.*")
pwd = os.path.dirname(inspect.getfile(self.__class__))
test_dir = os.path.join(pwd, 'discovery')
tinc_test_result = self._discover_and_run_tests(start_dirs = [test_dir], patterns =['test*'],
top_level_dir = None, query_handler = query_handler)
# Should have run all the tests from within mockstorage
self.assertEquals(tinc_test_result.testsRun, 7)
# predicate on test method
query_handler = TINCDiscoveryQueryHandler("method != *functional*")
pwd = os.path.dirname(inspect.getfile(self.__class__))
test_dir = os.path.join(pwd, 'discovery')
tinc_test_result = self._discover_and_run_tests(start_dirs = [test_dir], patterns =['test*'],
top_level_dir = None, query_handler = query_handler)
# Should have run all tests in all the modules except for the tests in test_functioanl_*.py of the form test_functional*
self.assertEquals(tinc_test_result.testsRun, 10)
# predicate on metadtata
query_handler = TINCDiscoveryQueryHandler("tags != stress")
pwd = os.path.dirname(inspect.getfile(self.__class__))
test_dir = os.path.join(pwd, 'discovery')
tinc_test_result = self._discover_and_run_tests(start_dirs = [test_dir], patterns =['test*'],
top_level_dir = None, query_handler = query_handler)
# Should have run all the tests except for the tests tagged 'stress' within test_uao.py and test_cardinality.py
self.assertEquals(tinc_test_result.testsRun, 12)
def test_multiple_predicates_within_a_query(self):
# Run all smoke tests
query_handler = TINCDiscoveryQueryHandler("tags = smoke or method = *smoke* or class = *SmokeTests*")
pwd = os.path.dirname(inspect.getfile(self.__class__))
test_dir = os.path.join(pwd, 'discovery')
tinc_test_result = self._discover_and_run_tests(start_dirs = [test_dir], patterns =['test*'],
top_level_dir = None, query_handler = query_handler)
# Should have run all the tests from within test_module*.py and one test tagged 'smoke' in test_uao.py and test_cardinality.py
self.assertEquals(tinc_test_result.testsRun, 6)
# Run all smoke tests that are not tagged
query_handler = TINCDiscoveryQueryHandler("method = *smoke* or class = *SmokeTests*")
pwd = os.path.dirname(inspect.getfile(self.__class__))
test_dir = os.path.join(pwd, 'discovery')
tinc_test_result = self._discover_and_run_tests(start_dirs = [test_dir], patterns =['test*'],
top_level_dir = None, query_handler = query_handler)
# Should have run four tests, two each from within test_smoke_*.py in mockstorage and mockquery
self.assertEquals(tinc_test_result.testsRun, 4)
# Run all functional tests
query_handler = TINCDiscoveryQueryHandler("tags = functional or method = *functional* or class = *FunctionalTests*")
pwd = os.path.dirname(inspect.getfile(self.__class__))
test_dir = os.path.join(pwd, 'discovery')
tinc_test_result = self._discover_and_run_tests(start_dirs = [test_dir], patterns =['test*'],
top_level_dir = None, query_handler = query_handler)
# Should have run all the functional tests , four from test_functional_*.py and one test each tagged 'functional'
# from within test_uao.py and test_cardinality.py
self.assertEquals(tinc_test_result.testsRun, 6)
# With AND predicates
query_handler = TINCDiscoveryQueryHandler("method = *functional* and class = *FunctionalTests*")
pwd = os.path.dirname(inspect.getfile(self.__class__))
test_dir = os.path.join(pwd, 'discovery')
tinc_test_result = self._discover_and_run_tests(start_dirs = [test_dir], patterns =['test*'],
top_level_dir = None, query_handler = query_handler)
# Should have run only four tests from within test_functional*.py in mockquery and mockstorage
self.assertEquals(tinc_test_result.testsRun, 4)
# Run all the tests except for stress tests
query_handler = TINCDiscoveryQueryHandler("module = test_* and tags != stress")
pwd = os.path.dirname(inspect.getfile(self.__class__))
test_dir = os.path.join(pwd, 'discovery')
tinc_test_result = self._discover_and_run_tests(start_dirs = [test_dir], patterns =['test*'],
top_level_dir = None, query_handler = query_handler)
# Run all the tests from within mockquery and mockstorage except the tests tagged 'stress' within
# test_uao.py and test_cardinality.py
self.assertEquals(tinc_test_result.testsRun, 12)
def test_sql_test_case_with_discovery_queries(self):
tinc_test_loader = tinctest.TINCTestLoader()
pwd = os.path.dirname(inspect.getfile(self.__class__))
test_dir = os.path.join(pwd, 'sql_pattern')
tinc_test_suite = tinc_test_loader.discover(start_dirs = [test_dir], patterns = ['sql_pattern.py'],
top_level_dir = None, query_handler = TINCDiscoveryQueryHandler(['method=test_functional_*']))
test_case = None
test_result = None
with closing(_WritelnDecorator(StringIO())) as buffer:
tinc_test_runner = TINCTestRunner(stream = buffer, descriptions = True, verbosity = 1)
test_result = tinc_test_runner.run(tinc_test_suite)
self.assertEqual(test_result.testsRun, 6)
self.assertEqual(len(test_result.skipped), 6)
# Queries using metadata from sql files
tinc_test_loader = tinctest.TINCTestLoader()
pwd = os.path.dirname(inspect.getfile(self.__class__))
test_dir = os.path.join(pwd, 'sql_pattern')
tinc_test_suite = tinc_test_loader.discover(start_dirs = [test_dir], patterns = ['sql_pattern.py'],
top_level_dir = None,
query_handler = TINCDiscoveryQueryHandler(['method=test_functional_* and tags != long']))
test_case = None
test_result = None
with closing(_WritelnDecorator(StringIO())) as buffer:
tinc_test_runner = TINCTestRunner(stream = buffer, descriptions = True, verbosity = 1)
test_result = tinc_test_runner.run(tinc_test_suite)
self.assertEqual(test_result.testsRun, 3)
self.assertEqual(len(test_result.skipped), 3)
def test_multiple_queries(self):
# Run all tests in class *SmokeTests* and starts with test_smoke_*
queries = ['class = *SmokeTests*', 'method=test_smoke*']
query_handler = TINCDiscoveryQueryHandler(queries)
pwd = os.path.dirname(inspect.getfile(self.__class__))
test_dir = os.path.join(pwd, 'discovery')
tinc_test_result = self._discover_and_run_tests(start_dirs = [test_dir], patterns =['test*'],
top_level_dir = None, query_handler = query_handler)
self.assertEquals(tinc_test_result.testsRun, 4)
def test_queries_with_patterns(self):
# patterns should be applied first
queries = ['method = test_*']
query_handler = TINCDiscoveryQueryHandler(queries)
pwd = os.path.dirname(inspect.getfile(self.__class__))
test_dir = os.path.join(pwd, 'discovery', 'mockquery')
tinc_test_result = self._discover_and_run_tests(start_dirs = [test_dir], patterns =['test_functional*'],
top_level_dir = None, query_handler = query_handler)
# Only run two tests within mockquery since the patterns should be applied first and the queries should be applied
# on the tests that match the pattern
self.assertEquals(tinc_test_result.testsRun, 2)
def test_apply_queries_with_multiple_predicates(self):
queries = []
queries.append("method=test_smoke* OR class = Mock* or class = MockTINC*")
queries.append("package=storage.uao.* OR method=test_smoke*")
queries.append("package=storage.uao.* and method=*testsmoke* and module!=test_discovery or method=test_smoke_01")
#queries.append("package != storage.uao or method=*testsmoke* and module != test_discovery and class != Mock* and tags = smoke and author != alice")
handler = TINCDiscoveryQueryHandler(queries)
tinc_test_case = MockTINCTestCase('test_smoke_01')
self.assertTrue(handler.apply_queries(tinc_test_case))
queries = []
queries.append("method=test_smoke* OR class = Mock* or class = MockTINC*")
queries.append("package=storage.uao.* OR method=test_smoke*")
queries.append("package=storage.uao.* and method=*testsmoke* and module!=test_discovery or method=test_smoke_01")
queries.append("package != storage.uao or method=*testsmoke* and module != test_discovery and class != Mock*")
# Introduce false at the end
queries.append("package != storage and method != test_smoke_01")
handler = TINCDiscoveryQueryHandler(queries)
tinc_test_case = MockTINCTestCase('test_smoke_01')
self.assertFalse(handler.apply_queries(tinc_test_case))
def test_apply_queries_with_multiple_queries(self):
queries = []
queries.append("method=test_smoke*")
queries.append("class=Mock*")
queries.append("class='MockTINC*'")
queries.append("module='test_discovery*'")
queries.append("package = tinc*.test*")
queries.append("method != test_smoke_02")
queries.append("class != NonMock*")
queries.append("package != storage.uao.*")
queries.append("module != test_regression*")
queries.append("author = bob")
queries.append("tags = smoke")
queries.append("author != alice")
queries.append("tags != functional")
handler = TINCDiscoveryQueryHandler(queries)
tinc_test_case = MockTINCTestCase('test_smoke_01')
self.assertTrue(handler.apply_queries(tinc_test_case))
queries = []
queries.append("method=test_smoke*")
queries.append("class=Mock*")
queries.append("class='MockTINC*'")
queries.append("module='test_discovery*'")
queries.append("package = storage.uao.*")
handler = TINCDiscoveryQueryHandler(queries)
tinc_test_case = MockTINCTestCase('test_smoke_01')
self.assertFalse(handler.apply_queries(tinc_test_case))
def test_apply_queries_with_multiple_predicates(self):
queries = []
queries.append(
"method=test_smoke* OR class = Mock* or class = MockTINC*")
queries.append("package=storage.uao.* OR method=test_smoke*")
queries.append(
"package=storage.uao.* and method=*testsmoke* and module!=test_discovery or method=test_smoke_01"
)
#queries.append("package != storage.uao or method=*testsmoke* and module != test_discovery and class != Mock* and tags = smoke and author != alice")
handler = TINCDiscoveryQueryHandler(queries)
tinc_test_case = MockTINCTestCase('test_smoke_01')
self.assertTrue(handler.apply_queries(tinc_test_case))
queries = []
queries.append(
"method=test_smoke* OR class = Mock* or class = MockTINC*")
queries.append("package=storage.uao.* OR method=test_smoke*")
queries.append(
"package=storage.uao.* and method=*testsmoke* and module!=test_discovery or method=test_smoke_01"
)
queries.append(
"package != storage.uao or method=*testsmoke* and module != test_discovery and class != Mock*"
)
# Introduce false at the end
queries.append("package != storage and method != test_smoke_01")
handler = TINCDiscoveryQueryHandler(queries)
tinc_test_case = MockTINCTestCase('test_smoke_01')
self.assertFalse(handler.apply_queries(tinc_test_case))
def test_suite_construction_with_discover_and_tinc_queries(self):
tinc_test_loader = tinctest.TINCTestLoader()
pwd = os.path.dirname(inspect.getfile(self.__class__))
test_dir = os.path.join(pwd, 'data_provider')
query_handler = TINCDiscoveryQueryHandler("tags=tag1")
tinc_test_suite = tinc_test_loader.discover(
start_dirs=[test_dir],
patterns=['test_*.py'],
top_level_dir=test_dir,
query_handler=query_handler)
with closing(_WritelnDecorator(StringIO())) as buffer:
tinc_test_result = tinctest.TINCTestResultSet(buffer, True, 1)
tinc_test_suite.run(tinc_test_result)
# This should have filtered 4 tests and hence run 7 tests
self.assertEquals(tinc_test_result.testsRun, 7)
def test_filtered_tests_with_dryrun(self):
"""
Test that tests filtered out with tinc queries are reported correctly in dryrun
"""
tinc_test_loader = tinctest.TINCTestLoader()
pwd = os.path.dirname(inspect.getfile(self.__class__))
test_dir = os.path.join(pwd, 'mocktests')
query_handler = TINCDiscoveryQueryHandler("tags != hawq")
tinc_test_suite = tinc_test_loader.discover(
start_dirs=[test_dir],
patterns=['dryrun_test_sample2.py'],
top_level_dir=test_dir,
query_handler=query_handler,
dryrun=True)
with closing(_WritelnDecorator(StringIO())) as buffer:
tinc_test_result = tinctest.TINCTestResultSet(buffer, True, 1)
tinc_test_suite.run(tinc_test_result)
self.assertEquals(tinc_test_result.testsRun, 3)
# This variable should be 0 even after the tests are run in the dryrun mode since
# we do not expect to run the test code
self.assertEquals(tinctest.test.dryrun.tests_run_count, 0)
# Two tests should be skipped one because of skip metadata and one because of filtering out
self.assertEquals(len(tinc_test_result.skipped), 2)
metadata_skip_test_tuple = None
queries_skip_test_tuple = None
# tinc_test_result.skipped will be a list of tuple containing test instance and the skip message
for test_tuple in tinc_test_result.skipped:
if test_tuple[0]._testMethodName == 'test_03':
metadata_skip_test_tuple = test_tuple
elif test_tuple[0]._testMethodName == 'test_01':
queries_skip_test_tuple = test_tuple
self.assertIsNotNone(metadata_skip_test_tuple)
self.assertIsNotNone(queries_skip_test_tuple)
self.assertTrue(
"Filtering out test as it did not satisy tinc queries check" in
queries_skip_test_tuple[1])
self.assertTrue("just skipping" in metadata_skip_test_tuple[1])
def test_positive_queries(self):
test_queries = []
test_queries.append("class=test_smoke*")
test_queries.append("module=test_smoke*")
test_queries.append("method=test_smoke*")
test_queries.append("package=storage.uao.*")
test_queries.append("class=* and module=* or package=* and class=SmokeTests*")
test_queries.append("class=* AND module=* OR package=* AND class=SmokeTests*")
test_queries.append("class=* and module=* or package=* and class=SmokeTests*")
test_queries.append("""class != * and module != * or package != * and class != SmokeTests*
AND class = test""")
test_queries.append("class != \"test_smoke\"")
test_queries.append("class != 'test_smoke'")
test_queries.append("""class = 'test' AND module=\"test\"""")
test_queries.append("class = test* and method=test*")
test_queries.append("tags = tag1")
handler = TINCDiscoveryQueryHandler(test_queries)
self.assertEquals(len(handler.query_list), len(test_queries))
def test_all_tests_loaded(self):
tinc_test_loader = tinctest.TINCTestLoader()
pwd = os.path.dirname(inspect.getfile(self.__class__))
test_dir1 = os.path.join(pwd, 'dryrun/mocktests')
tinc_test_suite = tinc_test_loader.discover(start_dirs = [test_dir1],
patterns = ['dryrun_test_sample2.py'],
top_level_dir = None)
self.assertEquals(tinc_test_suite.countTestCases(), 3)
flat_test_suite = TINCTestSuite()
tinc_test_suite.flatten(flat_test_suite)
for test in flat_test_suite._tests:
self.assertTrue(test.__class__._all_tests_loaded)
test_dir1 = os.path.join(pwd, 'dryrun/mocktests')
tinc_test_suite = tinc_test_loader.discover(start_dirs = [test_dir1],
patterns = ['dryrun_test_sample2.py'],
top_level_dir = None,
query_handler = TINCDiscoveryQueryHandler('tags=hawq'))
self.assertEquals(tinc_test_suite.countTestCases(), 1)
for test in flat_test_suite._tests:
self.assertFalse(test.__class__._all_tests_loaded)
def test_apply_queries_with_single_query(self):
handler = TINCDiscoveryQueryHandler("method=test_smoke*")
tinc_test_case = MockTINCTestCase('test_smoke_01')
self.assertTrue(handler.apply_queries(tinc_test_case))
# metadata equals predicate
handler = TINCDiscoveryQueryHandler("author=bob")
tinc_test_case = MockTINCTestCase('test_smoke_01')
self.assertTrue(handler.apply_queries(tinc_test_case))
# metadata equals predicate with dash (typically used for bugs)
handler = TINCDiscoveryQueryHandler("tags=bug-1")
tinc_test_case = MockTINCTestCase('test_smoke_01')
self.assertTrue(handler.apply_queries(tinc_test_case))
#metadata not equals predicate
handler = TINCDiscoveryQueryHandler("tags != functional")
tinc_test_case = MockTINCTestCase('test_smoke_01')
self.assertTrue(handler.apply_queries(tinc_test_case))
# metadata non-match
# metadata equals predicate
handler = TINCDiscoveryQueryHandler("author=alice")
tinc_test_case = MockTINCTestCase('test_smoke_01')
self.assertFalse(handler.apply_queries(tinc_test_case))
#metadata not equals predicate
handler = TINCDiscoveryQueryHandler("tags != smoke")
tinc_test_case = MockTINCTestCase('test_smoke_01')
self.assertFalse(handler.apply_queries(tinc_test_case))
# non existing metadata should return False
handler = TINCDiscoveryQueryHandler("non_existing_tags = smoke")
# not equals on a non existing metadata will currently return True
# We will have to decide whether or not to throw an exception for such tests
tinc_test_case = MockTINCTestCase('test_smoke_01')
self.assertFalse(handler.apply_queries(tinc_test_case))
