class GpFilespaceRegressionTests(unittest.TestCase):
def __init__(self, methodName):
self.gpfs = Gpfilespace()
self.gpfs_h = HAWQGpfilespace()
super(GpFilespaceRegressionTests, self).__init__(methodName)
def tearDown(self):
PSQL.run_sql_command('Drop filespace test_fs_a;')
def test_create_filespace(self):
self.gpfs.create_filespace('test_fs_a')
fs_list = PSQL.run_sql_command(
"select fsname from pg_filespace where fsname<>'pg_system';",
flags='-q -t')
self.assertTrue('test_fs_a' in fs_list)
def test_drop_fiespace(self):
self.gpfs.create_filespace('test_fs_b')
self.assertTrue(self.gpfs.drop_filespace('test_fs_b'))
def test_fs_exists(self):
self.gpfs.create_filespace('test_fs_a')
self.assertTrue(self.gpfs.exists('test_fs_a'))
def test_showtempfiles(self):
result = self.gpfs.showtempfiles()
show = False
for line in result.stdout.splitlines():
if 'Current Filespace for TEMPORARY_FILES' in line:
show = True
self.assertTrue(show)
def test_get_filespace_location(self):
result = self.gpfs.get_filespace_location()
self.assertTrue(len(result) > 0)
def test_get_filespace_directory(self):
result = self.gpfs.get_filespace_directory()
self.assertTrue(len(result) > 0)
def test_get_hosts_for_filespace(self):
self.gpfs.create_filespace('test_fs_a')
fs_location = PSQL.run_sql_command(
"select fselocation from pg_filespace_entry where fselocation like '%test_fs_a%' and fsedbid=2;",
flags='-q -t')
result = self.gpfs.get_hosts_for_filespace(fs_location.strip())
self.assertEquals(result[0]['location'], fs_location.strip())
def test_create_filespace_hawq(self):
self.gpfs_h.create_filespace('test_fs_hq')
fs_list = PSQL.run_sql_command(
"select fsname from pg_filespace where fsname<>'pg_system';",
flags='-q -t')
self.assertTrue('test_fs_hq' in fs_list)
class GpFilespaceRegressionTests(unittest.TestCase):
def __init__(self, methodName):
self.gpfs = Gpfilespace()
self.gpfs_h = HAWQGpfilespace()
super(GpFilespaceRegressionTests, self).__init__(methodName)
def tearDown(self):
PSQL.run_sql_command('Drop filespace test_fs_a;')
def test_create_filespace(self):
self.gpfs.create_filespace('test_fs_a')
fs_list = PSQL.run_sql_command("select fsname from pg_filespace where fsname<>'pg_system';", flags = '-q -t')
self.assertTrue('test_fs_a' in fs_list)
def test_drop_fiespace(self):
self.gpfs.create_filespace('test_fs_b')
self.assertTrue(self.gpfs.drop_filespace('test_fs_b'))
def test_fs_exists(self):
self.gpfs.create_filespace('test_fs_a')
self.assertTrue(self.gpfs.exists('test_fs_a'))
def test_showtempfiles(self):
result = self.gpfs.showtempfiles()
show = False
for line in result.stdout.splitlines():
if 'Current Filespace for TEMPORARY_FILES' in line:
show = True
self.assertTrue(show)
def test_get_filespace_location(self):
result = self.gpfs.get_filespace_location()
self.assertTrue(len(result) >0)
def test_get_filespace_directory(self):
result = self.gpfs.get_filespace_directory()
self.assertTrue(len(result) >0)
def test_get_hosts_for_filespace(self):
self.gpfs.create_filespace('test_fs_a')
fs_location = PSQL.run_sql_command("select fselocation from pg_filespace_entry where fselocation like '%test_fs_a%' and fsedbid=2;", flags = '-q -t')
result = self.gpfs.get_hosts_for_filespace(fs_location.strip())
self.assertEquals(result[0]['location'],fs_location.strip())
def test_create_filespace_hawq(self):
self.gpfs_h.create_filespace('test_fs_hq')
fs_list = PSQL.run_sql_command("select fsname from pg_filespace where fsname<>'pg_system';", flags = '-q -t')
self.assertTrue('test_fs_hq' in fs_list)
class PgtwoPhaseTestCase(ScenarioTestCase, MPPTestCase):
''' Testing state of prepared transactions upon crash-recovery'''
def __init__(self, methodName):
self.gpfile = Gpfilespace()
self.filereputil = Filerepe2e_Util()
super(PgtwoPhaseTestCase,self).__init__(methodName)
def setUp(self):
'''Create filespace '''
self.gpfile.create_filespace('filespace_test_a')
super(PgtwoPhaseTestCase,self).setUp()
def tearDown(self):
''' Cleanup up the filespace created , reset skip chekpoint fault'''
self.gpfile.drop_filespace('filespace_test_a')
port = os.getenv('PGPORT')
self.filereputil.inject_fault(f='checkpoint', y='reset', r='primary', o='0', p=port)
super(PgtwoPhaseTestCase,self).tearDown()
def execute_split_sqls(self, skip_state, cluster_state, ddl_type, fault_type, crash_type):
'''
@param skip_state : skip/noskip checkpoint
@param cluster_state : sync/change_tracking/resync
@param ddl_type : create/drop
@fault_type : commit/abort . Uses the same parameter to pass in 'end_prepare_two_phase_sleep'
@crash_type : gpstop_i/gpstop_a/failover_to_primary/failover_to_mirror
@description: Test the state of prepared transactions upon crash-recovery.
Faults are used to suspend the transactions before segments flush commit/abort to xlog.
Different types of crash followed by recovery are performed to evaluate the transaction state
Steps:
0. Check the state of the cluster before proceeding the test execution
1. Run any faults before pre_sqls depending on the clsuter_state
2. Run pre_sqls if any
3. Run any faults required before the trigger_sqls based on the fault_type as well as cluster_state
4. Run trigger_sqls - these are the transactions which will be suspended
5. Crash and recover. Resume suspended faults if needed
6. Run post_sqls to validate whether the transactions at step 4 are commited/ aborted as expected
7. Recover the cluster in case if neeed
8. Validate using gpcheckcat and gpcheckmirrorseg
'''
if fault_type == 'end_prepare_two_phase_sleep':
test_dir = '%s_%s_tests' % ('abort', ddl_type)
else:
test_dir = '%s_%s_tests' % (fault_type, ddl_type)
tinctest.logger.info('fault_type %s test_dir %s ddl_type %s' % (fault_type, test_dir, ddl_type))
test_case_list0 = []
test_case_list0.append('mpp.gpdb.tests.storage.lib.dbstate.DbStateClass.check_system')
self.test_case_scenario.append(test_case_list0)
test_case_list1 = []
test_case_list1.append(('mpp.gpdb.tests.storage.pg_twophase.PgtwoPhaseClass.run_faults_before_pre', [cluster_state]))
self.test_case_scenario.append(test_case_list1)
test_case_list2 = []
test_case_list2.append('mpp.gpdb.tests.storage.pg_twophase.%s.pre_sql.test_presqls.TestPreSQLClass' % test_dir)
self.test_case_scenario.append(test_case_list2)
test_case_list3 = []
test_case_list3.append(('mpp.gpdb.tests.storage.pg_twophase.PgtwoPhaseClass.run_faults_before_trigger', [skip_state, cluster_state, fault_type]))
self.test_case_scenario.append(test_case_list3)
test_case_list4 = []
test_case_list4.append('mpp.gpdb.tests.storage.pg_twophase.%s.trigger_sql.test_triggersqls.TestTriggerSQLClass' % test_dir)
test_case_list4.append(('mpp.gpdb.tests.storage.pg_twophase.PgtwoPhaseClass.run_crash_and_recover', [crash_type, fault_type, test_dir, cluster_state, skip_state]))
self.test_case_scenario.append(test_case_list4)
test_case_list5 = []
test_case_list5.append('mpp.gpdb.tests.storage.pg_twophase.%s.post_sql.test_postsqls.TestPostSQLClass' % test_dir)
self.test_case_scenario.append(test_case_list5)
test_case_list6 = []
test_case_list6.append(('mpp.gpdb.tests.storage.pg_twophase.PgtwoPhaseClass.run_gprecover', [crash_type, cluster_state]))
self.test_case_scenario.append(test_case_list6)
test_case_list7 = []
test_case_list7.append('mpp.gpdb.tests.storage.lib.dbstate.DbStateClass.run_validation')
self.test_case_scenario.append(test_case_list7)
test_case_list8 = []
test_case_list8.append('mpp.gpdb.tests.storage.pg_twophase.PgtwoPhaseClass.cleanup_dangling_processes')
self.test_case_scenario.append(test_case_list8)
class SuspendcheckpointCrashrecoveryTestCase(ScenarioTestCase):
'''
Testing state of prepared transactions upon crash-recovery
@gucs gp_create_table_random_default_distribution=off
'''
def __init__(self, methodName):
self.gpfile = Gpfilespace()
self.filereputil = Filerepe2e_Util()
super(SuspendcheckpointCrashrecoveryTestCase,
self).__init__(methodName)
def setUp(self):
super(SuspendcheckpointCrashrecoveryTestCase, self).setUp()
'''Create filespace '''
self.gpfile.create_filespace('filespace_test_a')
def tearDown(self):
''' Cleanup up the filespace created , reset skip chekpoint fault'''
self.gpfile.drop_filespace('filespace_test_a')
port = os.getenv('PGPORT')
self.filereputil.inject_fault(f='checkpoint',
y='reset',
r='primary',
o='0',
p=port)
super(SuspendcheckpointCrashrecoveryTestCase, self).tearDown()
def test_crash_recovery_04_to_10(self):
'''
@note : Steps are same as Cdbfast and Previous tinc schedule
@param skip_state : skip checkpoint
@param cluster_state : sync/change_tracking/resync
@param ddl_type : create/drop
@fault_type : commit/abort .
@crash_type : gpstop_i/gpstop_a/failover_to_primary
@description: Test the state of prepared transactions upon crash-recovery.
Faults are used to suspend the transactions before segments flush commit/abort to xlog.
Crash followed by recovery are performed to evaluate the transaction state
Steps:
0. Check the state of the cluster before proceeding the test execution
1. Run any fault 'skip checkpoint' before pre_sqls
2. Run pre_sqls if any
3. Run any faults required before the trigger_sqls based on the fault_type as well as cluster_state
4. Run trigger_sqls - these are the transactions which will be suspended
5. Crash and recover.
6. Run post_sqls to validate whether the transactions at step 4 are commited/ aborted as expected
7. Recover and Validate using gpcheckcat and gpcheckmirrorseg
@data_provider data_types_provider
'''
test_num = self.test_data[0][0] + self.test_data[0][1]
tinctest.logger.info(
"\n ===============================================")
tinctest.logger.info("\n Starting New Test: %s " % test_num)
tinctest.logger.info(
"\n ===============================================")
pass_num = self.test_data[1][0]
cluster_state = self.test_data[1][1]
ddl_type = self.test_data[1][2]
test_type = self.test_data[1][3]
aborting_create_needed = self.test_data[1][4]
if test_type == 'abort':
test_dir = '%s_%s_tests' % ('abort', ddl_type)
elif aborting_create_needed == 'True':
test_dir = '%s_%s_%s_tests' % ('abort', ddl_type, 'needed')
else:
test_dir = '%s_%s_tests' % (test_type, ddl_type)
if aborting_create_needed == True and test_type == 'commit':
test_dir = 'abort_create_needed_tests'
elif aborting_create_needed == True and test_type == 'abort':
test_dir = 'abort_abort_create_needed_tests'
tinctest.logger.info("TestDir == %s " % test_dir)
test_case_list0 = []
test_case_list0.append(
'mpp.gpdb.tests.storage.lib.dbstate.DbStateClass.check_system')
self.test_case_scenario.append(test_case_list0)
test_case_list1 = []
test_case_list1.append((
'mpp.gpdb.tests.storage.crashrecovery.SuspendCheckpointCrashRecovery.set_faults_before_executing_pre_sqls',
[cluster_state]))
self.test_case_scenario.append(test_case_list1)
test_case_list2 = []
test_case_list2.append(
'mpp.gpdb.tests.storage.crashrecovery.%s.pre_sql.test_pre_sqls.TestPreSQLClass'
% test_dir)
self.test_case_scenario.append(test_case_list2)
test_case_list3 = []
test_case_list3.append((
'mpp.gpdb.tests.storage.crashrecovery.SuspendCheckpointCrashRecovery.set_faults_before_executing_trigger_sqls',
[
pass_num, cluster_state, test_type, ddl_type,
aborting_create_needed
]))
self.test_case_scenario.append(test_case_list3)
test_case_list4 = []
test_case_list4.append(
'mpp.gpdb.tests.storage.crashrecovery.%s.trigger_sql.test_triggersqls.TestTriggerSQLClass'
% test_dir)
test_case_list4.append((
'mpp.gpdb.tests.storage.crashrecovery.SuspendCheckpointCrashRecovery.run_crash_and_recovery',
[test_dir, cluster_state]))
self.test_case_scenario.append(test_case_list4)
test_case_list5 = []
test_case_list5.append(
'mpp.gpdb.tests.storage.crashrecovery.%s.post_sql.test_postsqls.TestPostSQLClass'
% test_dir)
self.test_case_scenario.append(test_case_list5)
test_case_list6 = []
test_case_list6.append((
'mpp.gpdb.tests.storage.crashrecovery.SuspendCheckpointCrashRecovery.validate_system',
[cluster_state]))
self.test_case_scenario.append(test_case_list6)
test_case_list7 = []
test_case_list7.append((
'mpp.gpdb.tests.storage.crashrecovery.SuspendCheckpointCrashRecovery.backup_output_dir',
[test_dir, test_num]))
self.test_case_scenario.append(test_case_list7)
class SuspendcheckpointCrashrecoveryTestCase(ScenarioTestCase):
"""
Testing state of prepared transactions upon crash-recovery
@gucs gp_create_table_random_default_distribution=off
"""
def __init__(self, methodName):
self.gpfile = Gpfilespace()
self.filereputil = Filerepe2e_Util()
super(SuspendcheckpointCrashrecoveryTestCase, self).__init__(methodName)
def setUp(self):
super(SuspendcheckpointCrashrecoveryTestCase, self).setUp()
"""Create filespace """
self.gpfile.create_filespace("filespace_test_a")
def tearDown(self):
""" Cleanup up the filespace created , reset skip chekpoint fault"""
self.gpfile.drop_filespace("filespace_test_a")
port = os.getenv("PGPORT")
self.filereputil.inject_fault(f="checkpoint", y="reset", r="primary", o="0", p=port)
super(SuspendcheckpointCrashrecoveryTestCase, self).tearDown()
def test_crash_recovery_04_to_10(self):
"""
@note : Steps are same as Cdbfast and Previous tinc schedule
@param skip_state : skip checkpoint
@param cluster_state : sync/change_tracking/resync
@param ddl_type : create/drop
@fault_type : commit/abort .
@crash_type : gpstop_i/gpstop_a/failover_to_primary
@description: Test the state of prepared transactions upon crash-recovery.
Faults are used to suspend the transactions before segments flush commit/abort to xlog.
Crash followed by recovery are performed to evaluate the transaction state
Steps:
0. Check the state of the cluster before proceeding the test execution
1. Run any fault 'skip checkpoint' before pre_sqls
2. Run pre_sqls if any
3. Run any faults required before the trigger_sqls based on the fault_type as well as cluster_state
4. Run trigger_sqls - these are the transactions which will be suspended
5. Crash and recover.
6. Run post_sqls to validate whether the transactions at step 4 are commited/ aborted as expected
7. Recover and Validate using gpcheckcat and gpcheckmirrorseg
@data_provider data_types_provider
"""
test_num = self.test_data[0][0] + self.test_data[0][1]
tinctest.logger.info("\n ===============================================")
tinctest.logger.info("\n Starting New Test: %s " % test_num)
tinctest.logger.info("\n ===============================================")
pass_num = self.test_data[1][0]
cluster_state = self.test_data[1][1]
ddl_type = self.test_data[1][2]
test_type = self.test_data[1][3]
aborting_create_needed = self.test_data[1][4]
if test_type == "abort":
test_dir = "%s_%s_tests" % ("abort", ddl_type)
elif aborting_create_needed == "True":
test_dir = "%s_%s_%s_tests" % ("abort", ddl_type, "needed")
else:
test_dir = "%s_%s_tests" % (test_type, ddl_type)
if aborting_create_needed == True and test_type == "commit":
test_dir = "abort_create_needed_tests"
elif aborting_create_needed == True and test_type == "abort":
test_dir = "abort_abort_create_needed_tests"
tinctest.logger.info("TestDir == %s " % test_dir)
test_case_list0 = []
test_case_list0.append("mpp.gpdb.tests.storage.lib.dbstate.DbStateClass.check_system")
self.test_case_scenario.append(test_case_list0)
test_case_list1 = []
test_case_list1.append(
(
"mpp.gpdb.tests.storage.crashrecovery.SuspendCheckpointCrashRecovery.set_faults_before_executing_pre_sqls",
[cluster_state],
)
)
self.test_case_scenario.append(test_case_list1)
test_case_list2 = []
test_case_list2.append(
"mpp.gpdb.tests.storage.crashrecovery.%s.pre_sql.test_pre_sqls.TestPreSQLClass" % test_dir
)
self.test_case_scenario.append(test_case_list2)
test_case_list3 = []
test_case_list3.append(
(
"mpp.gpdb.tests.storage.crashrecovery.SuspendCheckpointCrashRecovery.set_faults_before_executing_trigger_sqls",
[pass_num, cluster_state, test_type, ddl_type, aborting_create_needed],
)
)
self.test_case_scenario.append(test_case_list3)
test_case_list4 = []
test_case_list4.append(
"mpp.gpdb.tests.storage.crashrecovery.%s.trigger_sql.test_triggersqls.TestTriggerSQLClass" % test_dir
)
test_case_list4.append(
(
"mpp.gpdb.tests.storage.crashrecovery.SuspendCheckpointCrashRecovery.run_crash_and_recovery",
[test_dir, cluster_state],
)
)
self.test_case_scenario.append(test_case_list4)
test_case_list5 = []
test_case_list5.append(
"mpp.gpdb.tests.storage.crashrecovery.%s.post_sql.test_postsqls.TestPostSQLClass" % test_dir
)
self.test_case_scenario.append(test_case_list5)
test_case_list6 = []
test_case_list6.append(
("mpp.gpdb.tests.storage.crashrecovery.SuspendCheckpointCrashRecovery.validate_system", [cluster_state])
)
self.test_case_scenario.append(test_case_list6)
test_case_list7 = []
test_case_list7.append(
(
"mpp.gpdb.tests.storage.crashrecovery.SuspendCheckpointCrashRecovery.backup_output_dir",
[test_dir, test_num],
)
)
self.test_case_scenario.append(test_case_list7)
class SuspendcheckpointCrashrecoveryTestCase(ScenarioTestCase):
'''
Testing state of prepared transactions upon crash-recovery
@gucs gp_create_table_random_default_distribution=off;optimizer_print_missing_stats=off
'''
def __init__(self, methodName):
tinctest.logger.info("\n =====================In __init__ method ==========================")
self.gpfile = Gpfilespace()
self.filereputil = Filerepe2e_Util()
super(SuspendcheckpointCrashrecoveryTestCase,self).__init__(methodName)
def setUp(self):
tinctest.logger.info("\n =====================In setUp method ==========================")
super(SuspendcheckpointCrashrecoveryTestCase, self).setUp()
'''Create filespace '''
self.gpfile.create_filespace('filespace_test_a')
def tearDown(self):
tinctest.logger.info("\n ======================In tesrDoen method =========================")
''' Cleanup up the filespace created , reset skip chekpoint fault'''
self.gpfile.drop_filespace('filespace_test_a')
port = os.getenv('PGPORT')
self.filereputil.inject_fault(f='checkpoint', y='reset', r='primary', o='0', p=port)
super(SuspendcheckpointCrashrecoveryTestCase, self).tearDown()
def test_crash_recovery_31_to_42(self):
'''
@note : Steps are same as Cdbfast and Previous tinc schedule
@param skip_state : skip checkpoint
@param cluster_state : sync/change_tracking/resync
@param ddl_type : create/drop
@fault_type : commit/abort .
@crash_type : gpstop_i/gpstop_a/failover_to_primary
@description: Test the state of prepared transactions upon crash-recovery.
Faults are used to suspend the transactions before segments flush commit/abort to xlog.
Crash followed by recovery are performed to evaluate the transaction state
Steps:
0. Check the state of the cluster before proceeding the test execution
1. Run any fault 'skip checkpoint' before pre_sqls
2. Run pre_sqls if any
3. Run any faults required before the trigger_sqls based on the fault_type as well as cluster_state
4. Run trigger_sqls - these are the transactions which will be suspended
5. Crash and recover.
6. Run post_sqls to validate whether the transactions at step 4 are commited/ aborted as expected
7. Recover and Validate using gpcheckcat and gpcheckmirrorseg
@data_provider data_types_provider
'''
tinctest.logger.info("\n ===============================================")
tinctest.logger.info("\n Starting New Test: %s%s " % (self.test_data[0][0],self.test_data[0][1]))
tinctest.logger.info("\n ===============================================")
pass_num = self.test_data[1][0]
cluster_state = self.test_data[1][1]
ddl_type = self.test_data[1][2]
test_type = self.test_data[1][3]
aborting_create_needed = self.test_data[1][4]
if test_type == 'abort':
test_dir = '%s_%s_tests' % ('abort', ddl_type)
elif aborting_create_needed == 'True':
test_dir = '%s_%s_%s_tests' % ('abort', ddl_type, 'needed')
else:
test_dir = '%s_%s_tests' % (test_type, ddl_type)
if aborting_create_needed == True and test_type == 'commit':
test_dir = 'abort_create_needed_tests'
elif aborting_create_needed == True and test_type == 'abort' :
test_dir = 'abort_abort_create_needed_tests'
tinctest.logger.info("TestDir == %s " % test_dir )
test_case_list0 = []
test_case_list0.append('mpp.gpdb.tests.storage.lib.dbstate.DbStateClass.check_system')
self.test_case_scenario.append(test_case_list0)
test_case_list1 = []
test_case_list1.append(('mpp.gpdb.tests.storage.crashrecovery.SuspendCheckpointCrashRecovery.set_faults_before_executing_pre_sqls', [cluster_state]))
self.test_case_scenario.append(test_case_list1)
test_case_list2 = []
test_case_list2.append('mpp.gpdb.tests.storage.crashrecovery.%s.pre_sql.test_pre_sqls.TestPreSQLClass' % test_dir)
self.test_case_scenario.append(test_case_list2)
test_case_list3 = []
test_case_list3.append(('mpp.gpdb.tests.storage.crashrecovery.SuspendCheckpointCrashRecovery.set_faults_before_executing_trigger_sqls', [pass_num, cluster_state, test_type, ddl_type, aborting_create_needed]))
self.test_case_scenario.append(test_case_list3)
test_case_list4 = []
test_case_list4.append('mpp.gpdb.tests.storage.crashrecovery.%s.trigger_sql.test_triggersqls.TestTriggerSQLClass' % test_dir)
test_case_list4.append(('mpp.gpdb.tests.storage.crashrecovery.SuspendCheckpointCrashRecovery.run_crash_and_recovery_fast', [test_dir, pass_num, cluster_state, test_type, ddl_type, aborting_create_needed]))
self.test_case_scenario.append(test_case_list4)
test_case_list5 = []
test_case_list5.append('mpp.gpdb.tests.storage.crashrecovery.%s.post_sql.test_postsqls.TestPostSQLClass' % test_dir)
self.test_case_scenario.append(test_case_list5)
test_case_list6 = []
test_case_list6.append(('mpp.gpdb.tests.storage.crashrecovery.SuspendCheckpointCrashRecovery.validate_system',[cluster_state]))
self.test_case_scenario.append(test_case_list6)
class SwitchCheckpointTestCase(ScenarioTestCase):
''' Testing state of transactions with faults on master after crash-recovery'''
def __init__(self, methodName):
self.gpfile = Gpfilespace()
super(SwitchCheckpointTestCase, self).__init__(methodName)
def setUp(self):
'''Create filespace '''
self.gpfile.create_filespace('filespace_test_a')
super(SwitchCheckpointTestCase, self).setUp()
def tearDown(self):
''' Cleanup up the filespace created '''
self.gpfile.drop_filespace('filespace_test_a')
super(SwitchCheckpointTestCase, self).tearDown()
def switch_checkpoint(self, cluster_state, fault_type, crash_type):
'''
@param skip_state : skip/noskip checkpoint
@param cluster_state : sync/change_tracking/resync
@param ddl_type : create/drop
@fault_type : commit/abort . Uses the same parameter to pass in 'end_prepare_two_phase_sleep'
@crash_type : gpstop_i/gpstop_a/failover_to_primary/failover_to_mirror
@description: Test the state of transactions after fault on master at diff stages followed by crash-recovery.
Faults are used to suspend the transactions at three stages
1. After prepare has been sent by master and acknowledge by all workers
2. After distributed commit has been flushed to xlog on master
3. After commit prepared has been sent out acknowledged before distributed forget
Steps:
0. Check the state of the cluster before proceeding the test execution
1. Run pre_sqls
2. Run any faults required before the trigger_sqls based on the fault_type as well as cluster_state
3. Run switch_checkpoint loop. switch_xlog, checkpoint in case of 'dtm_broadcast_prepare' fault
4. Crash and recover. Resume suspended faults if needed
5. Run validate_loop
7. Recover the cluster in case if needed
8. Validate using gpcheckcat and gpcheckmirrorseg
'''
test_dir = {
"dtm_broadcast_prepare": "switch_ckpt_a,switch_ckpt_b",
"dtm_broadcast_commit_prepared": "switch_ckpt_a,switch_ckpt_b",
"dtm_xlog_distributed_commit": "switch_ckpt_c"
}
test_case_list0 = []
test_case_list0.append(
'mpp.gpdb.tests.storage.lib.dbstate.DbStateClass.check_system')
self.test_case_scenario.append(test_case_list0)
test_case_list1 = []
if fault_type == 'dtm_broadcast_commit_prepared':
test_case_list1.append(
'mpp.gpdb.tests.storage.pg_twophase.switch_ckpt_a.pre_sql.test_presqls.TestPreSQLClass'
)
test_case_list1.append(
'mpp.gpdb.tests.storage.pg_twophase.switch_ckpt_b.pre_sql.test_presqls.TestPreSQLClass'
)
if fault_type == 'dtm_broadcast_prepare':
test_case_list1.append(
'mpp.gpdb.tests.storage.pg_twophase.switch_ckpt_a.pre_sql.test_presqls.TestPreSQLClass'
)
test_case_list1.append(
'mpp.gpdb.tests.storage.pg_twophase.checkpoint.test_checkpoint.TestCheckpointClass'
)
test_case_list1.append(
'mpp.gpdb.tests.storage.pg_twophase.switch_ckpt_b.pre_sql.test_presqls.TestPreSQLClass'
)
if fault_type == 'dtm_xlog_distributed_commit':
test_case_list1.append(
'mpp.gpdb.tests.storage.pg_twophase.switch_ckpt_c.pre_sql.test_presqls.TestPreSQLClass'
)
self.test_case_scenario.append(test_case_list1)
test_case_list2 = []
test_case_list2.append((
'mpp.gpdb.tests.storage.pg_twophase.PgtwoPhaseClass.switch_ckpt_faults_before_trigger',
[cluster_state, fault_type]))
self.test_case_scenario.append(test_case_list2)
test_case_list3 = []
test_case_list3.append(
'mpp.gpdb.tests.storage.pg_twophase.PgtwoPhaseClass.switch_ckpt_switch_xlog'
)
self.test_case_scenario.append(test_case_list3)
test_case_list4 = []
if fault_type == 'dtm_broadcast_commit_prepared':
test_case_list4.append(
'mpp.gpdb.tests.storage.pg_twophase.switch_ckpt_a.trigger_sql.test_triggersqls.TestTriggerSQLClass'
)
test_case_list4.append(
'mpp.gpdb.tests.storage.pg_twophase.switch_ckpt_b.trigger_sql.test_triggersqls.TestTriggerSQLClass'
)
if fault_type == 'dtm_broadcast_prepare':
test_case_list4.append(
'mpp.gpdb.tests.storage.pg_twophase.switch_ckpt_a.trigger_sql.test_triggersqls.TestTriggerSQLClass'
)
test_case_list4.append(
'mpp.gpdb.tests.storage.pg_twophase.checkpoint.test_checkpoint.TestCheckpointClass'
)
test_case_list4.append(
'mpp.gpdb.tests.storage.pg_twophase.switch_ckpt_b.trigger_sql.test_triggersqls.TestTriggerSQLClass'
)
if fault_type == 'dtm_xlog_distributed_commit':
test_case_list4.append(
'mpp.gpdb.tests.storage.pg_twophase.switch_ckpt_c.trigger_sql.test_triggersqls.TestTriggerSQLClass'
)
test_case_list4.append((
'mpp.gpdb.tests.storage.pg_twophase.PgtwoPhaseClass.switch_ckpt_crash_and_recover',
[crash_type, fault_type, test_dir[fault_type], cluster_state]))
self.test_case_scenario.append(test_case_list4)
test_case_list5 = []
test_case_list5.append((
'mpp.gpdb.tests.storage.pg_twophase.PgtwoPhaseClass.run_gprecover',
[crash_type, cluster_state]))
self.test_case_scenario.append(test_case_list5)
test_case_list6 = []
if fault_type == 'dtm_broadcast_commit_prepared':
test_case_list6.append(
'mpp.gpdb.tests.storage.pg_twophase.switch_ckpt_a.post_sql.test_postsqls.TestPostSQLClass'
)
test_case_list6.append(
'mpp.gpdb.tests.storage.pg_twophase.switch_ckpt_b.post_sql.test_postsqls.TestPostSQLClass'
)
# No tables will be created for crash_type failover_to_mirror
if fault_type == 'dtm_broadcast_prepare' and crash_type not in (
'gpstop_i'):
test_case_list6.append(
'mpp.gpdb.tests.storage.pg_twophase.switch_ckpt_a.post_sql.test_postsqls.TestPostSQLClass'
)
test_case_list6.append(
'mpp.gpdb.tests.storage.pg_twophase.checkpoint.test_checkpoint.TestCheckpointClass'
)
test_case_list6.append(
'mpp.gpdb.tests.storage.pg_twophase.switch_ckpt_b.post_sql.test_postsqls.TestPostSQLClass'
)
if fault_type == 'dtm_xlog_distributed_commit':
test_case_list6.append(
'mpp.gpdb.tests.storage.pg_twophase.switch_ckpt_c.post_sql.test_postsqls.TestPostSQLClass'
)
self.test_case_scenario.append(test_case_list6)
test_case_list7 = []
if fault_type in ('dtm_broadcast_commit_prepared',
'dtm_broadcast_prepare'):
test_case_list7.append(
'mpp.gpdb.tests.storage.pg_twophase.switch_ckpt_b.cleanup_sql.test_cleanup.TestCleanupClass'
)
if fault_type == 'dtm_xlog_distributed_commit':
test_case_list7.append(
'mpp.gpdb.tests.storage.pg_twophase.switch_ckpt_c.cleanup_sql.test_cleanup.TestCleanupClass'
)
self.test_case_scenario.append(test_case_list7)
test_case_list8 = []
test_case_list8.append(
'mpp.gpdb.tests.storage.lib.dbstate.DbStateClass.run_validation')
self.test_case_scenario.append(test_case_list8)
test_case_list9 = []
test_case_list9.append(
'mpp.gpdb.tests.storage.pg_twophase.PgtwoPhaseClass.cleanup_dangling_processes'
)
self.test_case_scenario.append(test_case_list9)
def switch_checkpoint_serial(self,
cluster_state,
fault_type,
crash_type='gpstop_i'):
'''
@description: Tests with more switch_xlogs before trigger sqls
'''
test_case_list0 = []
test_case_list0.append(
'mpp.gpdb.tests.storage.lib.dbstate.DbStateClass.check_system')
self.test_case_scenario.append(test_case_list0)
test_case_list1 = []
test_case_list1.append(
'mpp.gpdb.tests.storage.pg_twophase.switch_ckpt_serial.pre_sql.test_presqls.TestPreSQLClass'
)
self.test_case_scenario.append(test_case_list1)
test_case_list2 = []
test_case_list2.append((
'mpp.gpdb.tests.storage.pg_twophase.PgtwoPhaseClass.switch_ckpt_faults_before_trigger',
[cluster_state, fault_type]))
self.test_case_scenario.append(test_case_list2)
test_case_list3 = []
test_case_list3.append(
'mpp.gpdb.tests.storage.pg_twophase.PgtwoPhaseClass.switch_ckpt_switch_xlog'
)
self.test_case_scenario.append(test_case_list3)
test_dir = 'switch_ckpt_serial'
test_case_list4 = []
test_case_list4.append(
'mpp.gpdb.tests.storage.pg_twophase.switch_ckpt_serial.trigger_sql.test_triggersqls.TestTriggerSQLClass'
)
test_case_list4.append((
'mpp.gpdb.tests.storage.pg_twophase.PgtwoPhaseClass.switch_ckpt_crash_and_recover',
[crash_type, fault_type, test_dir, cluster_state]))
self.test_case_scenario.append(test_case_list4)
test_case_list5 = []
test_case_list5.append((
'mpp.gpdb.tests.storage.pg_twophase.PgtwoPhaseClass.switch_checkpoint_loop',
[fault_type]))
self.test_case_scenario.append(test_case_list5)
test_case_list6 = []
test_case_list6.append((
'mpp.gpdb.tests.storage.pg_twophase.PgtwoPhaseClass.run_gprecover',
[crash_type, cluster_state]))
self.test_case_scenario.append(test_case_list6)
if fault_type not in ('dtm_broadcast_prepare'):
test_case_list7 = []
test_case_list7.append(
'mpp.gpdb.tests.storage.pg_twophase.switch_ckpt_serial.post_sql.test_postsqls.TestPostSQLClass'
)
self.test_case_scenario.append(test_case_list7)
test_case_list8 = []
test_case_list8.append(
'mpp.gpdb.tests.storage.pg_twophase.switch_ckpt_serial.cleanup_sql.test_cleanup.TestCleanupClass'
)
self.test_case_scenario.append(test_case_list8)
test_case_list9 = []
test_case_list9.append(
'mpp.gpdb.tests.storage.lib.dbstate.DbStateClass.run_validation')
self.test_case_scenario.append(test_case_list9)
test_case_list10 = []
test_case_list10.append(
'mpp.gpdb.tests.storage.pg_twophase.PgtwoPhaseClass.cleanup_dangling_processes'
)
self.test_case_scenario.append(test_case_list10)
class PgtwoPhaseTestCase(ScenarioTestCase, MPPTestCase):
''' Testing state of prepared transactions upon crash-recovery'''
def __init__(self, methodName):
self.gpfile = Gpfilespace()
self.filereputil = Filerepe2e_Util()
super(PgtwoPhaseTestCase, self).__init__(methodName)
def setUp(self):
'''Create filespace '''
self.gpfile.create_filespace('filespace_test_a')
super(PgtwoPhaseTestCase, self).setUp()
def tearDown(self):
''' Cleanup up the filespace created , reset skip chekpoint fault'''
self.gpfile.drop_filespace('filespace_test_a')
port = os.getenv('PGPORT')
self.filereputil.inject_fault(f='checkpoint',
y='reset',
r='primary',
o='0',
p=port)
super(PgtwoPhaseTestCase, self).tearDown()
def execute_split_sqls(self, skip_state, cluster_state, ddl_type,
fault_type, crash_type):
'''
@param skip_state : skip/noskip checkpoint
@param cluster_state : sync/change_tracking/resync
@param ddl_type : create/drop
@fault_type : commit/abort . Uses the same parameter to pass in 'end_prepare_two_phase_sleep'
@crash_type : gpstop_i/gpstop_a/failover_to_primary/failover_to_mirror
@description: Test the state of prepared transactions upon crash-recovery.
Faults are used to suspend the transactions before segments flush commit/abort to xlog.
Different types of crash followed by recovery are performed to evaluate the transaction state
Steps:
0. Check the state of the cluster before proceeding the test execution
1. Run any faults before pre_sqls depending on the clsuter_state
2. Run pre_sqls if any
3. Run any faults required before the trigger_sqls based on the fault_type as well as cluster_state
4. Run trigger_sqls - these are the transactions which will be suspended
5. Crash and recover. Resume suspended faults if needed
6. Run post_sqls to validate whether the transactions at step 4 are commited/ aborted as expected
7. Recover the cluster in case if neeed
8. Validate using gpcheckcat and gpcheckmirrorseg
'''
if fault_type == 'end_prepare_two_phase_sleep':
test_dir = '%s_%s_tests' % ('abort', ddl_type)
else:
test_dir = '%s_%s_tests' % (fault_type, ddl_type)
tinctest.logger.info('fault_type %s test_dir %s ddl_type %s' %
(fault_type, test_dir, ddl_type))
test_case_list0 = []
test_case_list0.append(
'mpp.gpdb.tests.storage.lib.dbstate.DbStateClass.check_system')
self.test_case_scenario.append(test_case_list0)
test_case_list1 = []
test_case_list1.append((
'mpp.gpdb.tests.storage.pg_twophase.PgtwoPhaseClass.run_faults_before_pre',
[cluster_state]))
self.test_case_scenario.append(test_case_list1)
test_case_list2 = []
test_case_list2.append(
'mpp.gpdb.tests.storage.pg_twophase.%s.pre_sql.test_presqls.TestPreSQLClass'
% test_dir)
self.test_case_scenario.append(test_case_list2)
test_case_list3 = []
test_case_list3.append((
'mpp.gpdb.tests.storage.pg_twophase.PgtwoPhaseClass.run_faults_before_trigger',
[skip_state, cluster_state, fault_type]))
self.test_case_scenario.append(test_case_list3)
test_case_list4 = []
test_case_list4.append(
'mpp.gpdb.tests.storage.pg_twophase.%s.trigger_sql.test_triggersqls.TestTriggerSQLClass'
% test_dir)
test_case_list4.append((
'mpp.gpdb.tests.storage.pg_twophase.PgtwoPhaseClass.run_crash_and_recover',
[crash_type, fault_type, test_dir, cluster_state, skip_state]))
self.test_case_scenario.append(test_case_list4)
test_case_list5 = []
test_case_list5.append(
'mpp.gpdb.tests.storage.pg_twophase.%s.post_sql.test_postsqls.TestPostSQLClass'
% test_dir)
self.test_case_scenario.append(test_case_list5)
test_case_list6 = []
test_case_list6.append((
'mpp.gpdb.tests.storage.pg_twophase.PgtwoPhaseClass.run_gprecover',
[crash_type, cluster_state]))
self.test_case_scenario.append(test_case_list6)
test_case_list7 = []
test_case_list7.append(
'mpp.gpdb.tests.storage.lib.dbstate.DbStateClass.run_validation')
self.test_case_scenario.append(test_case_list7)
test_case_list8 = []
test_case_list8.append(
'mpp.gpdb.tests.storage.pg_twophase.PgtwoPhaseClass.cleanup_dangling_processes'
)
self.test_case_scenario.append(test_case_list8)
class SwitchCheckpointTestCase(ScenarioTestCase):
''' Testing state of transactions with faults on master after crash-recovery'''
def __init__(self, methodName):
self.gpfile = Gpfilespace()
super(SwitchCheckpointTestCase,self).__init__(methodName)
def setUp(self):
'''Create filespace '''
self.gpfile.create_filespace('filespace_test_a')
super(SwitchCheckpointTestCase,self).setUp()
def tearDown(self):
''' Cleanup up the filespace created '''
self.gpfile.drop_filespace('filespace_test_a')
super(SwitchCheckpointTestCase,self).tearDown()
def switch_checkpoint(self, cluster_state, fault_type, crash_type):
'''
@param skip_state : skip/noskip checkpoint
@param cluster_state : sync/change_tracking/resync
@param ddl_type : create/drop
@fault_type : commit/abort . Uses the same parameter to pass in 'end_prepare_two_phase_sleep'
@crash_type : gpstop_i/gpstop_a/failover_to_primary/failover_to_mirror
@description: Test the state of transactions after fault on master at diff stages followed by crash-recovery.
Faults are used to suspend the transactions at three stages
1. After prepare has been sent by master and acknowledge by all workers
2. After distributed commit has been flushed to xlog on master
3. After commit prepared has been sent out acknowledged before distributed forget
Steps:
0. Check the state of the cluster before proceeding the test execution
1. Run pre_sqls
2. Run any faults required before the trigger_sqls based on the fault_type as well as cluster_state
3. Run switch_checkpoint loop. switch_xlog, checkpoint in case of 'dtm_broadcast_prepare' fault
4. Crash and recover. Resume suspended faults if needed
5. Run validate_loop
7. Recover the cluster in case if needed
8. Validate using gpcheckcat and gpcheckmirrorseg
'''
test_dir = {"dtm_broadcast_prepare":"switch_ckpt_a,switch_ckpt_b", "dtm_broadcast_commit_prepared":"switch_ckpt_a,switch_ckpt_b", "dtm_xlog_distributed_commit":"switch_ckpt_c"}
test_case_list0 = []
test_case_list0.append('mpp.gpdb.tests.storage.lib.dbstate.DbStateClass.check_system')
self.test_case_scenario.append(test_case_list0)
test_case_list1 = []
if fault_type == 'dtm_broadcast_commit_prepared':
test_case_list1.append('mpp.gpdb.tests.storage.pg_twophase.switch_ckpt_a.pre_sql.test_presqls.TestPreSQLClass')
test_case_list1.append('mpp.gpdb.tests.storage.pg_twophase.switch_ckpt_b.pre_sql.test_presqls.TestPreSQLClass')
if fault_type == 'dtm_broadcast_prepare':
test_case_list1.append('mpp.gpdb.tests.storage.pg_twophase.switch_ckpt_a.pre_sql.test_presqls.TestPreSQLClass')
test_case_list1.append('mpp.gpdb.tests.storage.pg_twophase.checkpoint.test_checkpoint.TestCheckpointClass')
test_case_list1.append('mpp.gpdb.tests.storage.pg_twophase.switch_ckpt_b.pre_sql.test_presqls.TestPreSQLClass')
if fault_type == 'dtm_xlog_distributed_commit':
test_case_list1.append('mpp.gpdb.tests.storage.pg_twophase.switch_ckpt_c.pre_sql.test_presqls.TestPreSQLClass')
self.test_case_scenario.append(test_case_list1)
test_case_list2 = []
test_case_list2.append(('mpp.gpdb.tests.storage.pg_twophase.PgtwoPhaseClass.switch_ckpt_faults_before_trigger', [cluster_state, fault_type]))
self.test_case_scenario.append(test_case_list2)
test_case_list3 = []
test_case_list3.append('mpp.gpdb.tests.storage.pg_twophase.PgtwoPhaseClass.switch_ckpt_switch_xlog')
self.test_case_scenario.append(test_case_list3)
test_case_list4 = []
if fault_type == 'dtm_broadcast_commit_prepared':
test_case_list4.append('mpp.gpdb.tests.storage.pg_twophase.switch_ckpt_a.trigger_sql.test_triggersqls.TestTriggerSQLClass')
test_case_list4.append('mpp.gpdb.tests.storage.pg_twophase.switch_ckpt_b.trigger_sql.test_triggersqls.TestTriggerSQLClass')
if fault_type == 'dtm_broadcast_prepare':
test_case_list4.append('mpp.gpdb.tests.storage.pg_twophase.switch_ckpt_a.trigger_sql.test_triggersqls.TestTriggerSQLClass')
test_case_list4.append('mpp.gpdb.tests.storage.pg_twophase.checkpoint.test_checkpoint.TestCheckpointClass')
test_case_list4.append('mpp.gpdb.tests.storage.pg_twophase.switch_ckpt_b.trigger_sql.test_triggersqls.TestTriggerSQLClass')
if fault_type == 'dtm_xlog_distributed_commit':
test_case_list4.append('mpp.gpdb.tests.storage.pg_twophase.switch_ckpt_c.trigger_sql.test_triggersqls.TestTriggerSQLClass')
test_case_list4.append(('mpp.gpdb.tests.storage.pg_twophase.PgtwoPhaseClass.switch_ckpt_crash_and_recover', [crash_type, fault_type, test_dir[fault_type], cluster_state]))
self.test_case_scenario.append(test_case_list4)
test_case_list5 = []
test_case_list5.append(('mpp.gpdb.tests.storage.pg_twophase.PgtwoPhaseClass.run_gprecover', [crash_type, cluster_state]))
self.test_case_scenario.append(test_case_list5)
test_case_list6 = []
if fault_type == 'dtm_broadcast_commit_prepared':
test_case_list6.append('mpp.gpdb.tests.storage.pg_twophase.switch_ckpt_a.post_sql.test_postsqls.TestPostSQLClass')
test_case_list6.append('mpp.gpdb.tests.storage.pg_twophase.switch_ckpt_b.post_sql.test_postsqls.TestPostSQLClass')
# No tables will be created for crash_type failover_to_mirror
if fault_type == 'dtm_broadcast_prepare' and crash_type not in ('gpstop_i'):
test_case_list6.append('mpp.gpdb.tests.storage.pg_twophase.switch_ckpt_a.post_sql.test_postsqls.TestPostSQLClass')
test_case_list6.append('mpp.gpdb.tests.storage.pg_twophase.checkpoint.test_checkpoint.TestCheckpointClass')
test_case_list6.append('mpp.gpdb.tests.storage.pg_twophase.switch_ckpt_b.post_sql.test_postsqls.TestPostSQLClass')
if fault_type == 'dtm_xlog_distributed_commit':
test_case_list6.append('mpp.gpdb.tests.storage.pg_twophase.switch_ckpt_c.post_sql.test_postsqls.TestPostSQLClass')
self.test_case_scenario.append(test_case_list6)
test_case_list7 = []
if fault_type in ('dtm_broadcast_commit_prepared', 'dtm_broadcast_prepare'):
test_case_list7.append('mpp.gpdb.tests.storage.pg_twophase.switch_ckpt_b.cleanup_sql.test_cleanup.TestCleanupClass')
if fault_type == 'dtm_xlog_distributed_commit':
test_case_list7.append('mpp.gpdb.tests.storage.pg_twophase.switch_ckpt_c.cleanup_sql.test_cleanup.TestCleanupClass')
self.test_case_scenario.append(test_case_list7)
test_case_list8 = []
test_case_list8.append('mpp.gpdb.tests.storage.lib.dbstate.DbStateClass.run_validation')
self.test_case_scenario.append(test_case_list8)
test_case_list9 = []
test_case_list9.append('mpp.gpdb.tests.storage.pg_twophase.PgtwoPhaseClass.cleanup_dangling_processes')
self.test_case_scenario.append(test_case_list9)
def switch_checkpoint_serial(self, cluster_state, fault_type, crash_type='gpstop_i'):
'''
@description: Tests with more switch_xlogs before trigger sqls
'''
test_case_list0 = []
test_case_list0.append('mpp.gpdb.tests.storage.lib.dbstate.DbStateClass.check_system')
self.test_case_scenario.append(test_case_list0)
test_case_list1 = []
test_case_list1.append('mpp.gpdb.tests.storage.pg_twophase.switch_ckpt_serial.pre_sql.test_presqls.TestPreSQLClass')
self.test_case_scenario.append(test_case_list1)
test_case_list2 = []
test_case_list2.append(('mpp.gpdb.tests.storage.pg_twophase.PgtwoPhaseClass.switch_ckpt_faults_before_trigger', [cluster_state, fault_type]))
self.test_case_scenario.append(test_case_list2)
test_case_list3 = []
test_case_list3.append('mpp.gpdb.tests.storage.pg_twophase.PgtwoPhaseClass.switch_ckpt_switch_xlog')
self.test_case_scenario.append(test_case_list3)
test_dir = 'switch_ckpt_serial'
test_case_list4 = []
test_case_list4.append('mpp.gpdb.tests.storage.pg_twophase.switch_ckpt_serial.trigger_sql.test_triggersqls.TestTriggerSQLClass')
test_case_list4.append(('mpp.gpdb.tests.storage.pg_twophase.PgtwoPhaseClass.switch_ckpt_crash_and_recover', [crash_type, fault_type, test_dir, cluster_state]))
self.test_case_scenario.append(test_case_list4)
test_case_list5 = []
test_case_list5.append(('mpp.gpdb.tests.storage.pg_twophase.PgtwoPhaseClass.switch_checkpoint_loop', [fault_type]))
self.test_case_scenario.append(test_case_list5)
test_case_list6 = []
test_case_list6.append(('mpp.gpdb.tests.storage.pg_twophase.PgtwoPhaseClass.run_gprecover', [crash_type, cluster_state]))
self.test_case_scenario.append(test_case_list6)
if fault_type not in ('dtm_broadcast_prepare'):
test_case_list7 = []
test_case_list7.append('mpp.gpdb.tests.storage.pg_twophase.switch_ckpt_serial.post_sql.test_postsqls.TestPostSQLClass')
self.test_case_scenario.append(test_case_list7)
test_case_list8 = []
test_case_list8.append('mpp.gpdb.tests.storage.pg_twophase.switch_ckpt_serial.cleanup_sql.test_cleanup.TestCleanupClass')
self.test_case_scenario.append(test_case_list8)
test_case_list9 = []
test_case_list9.append('mpp.gpdb.tests.storage.lib.dbstate.DbStateClass.run_validation')
self.test_case_scenario.append(test_case_list9)
test_case_list10 = []
test_case_list10.append('mpp.gpdb.tests.storage.pg_twophase.PgtwoPhaseClass.cleanup_dangling_processes')
self.test_case_scenario.append(test_case_list10)