예제 #1
0
class CrashRecoveryFilerepTestCase(ScenarioTestCase, MPPTestCase):
    """
    @gucs gp_create_table_random_default_distribution=off
    """
    def __init__(self, methodName):
        self.filerep = FilerepTestCase('preprocess')
        self.path = os.path.join(self.get_source_dir(),
                                 '../filerep_end_to_end/data')
        super(CrashRecoveryFilerepTestCase, self).__init__(methodName)

    def setUp(self):
        '''
        gpfdist port should be dynamically generated, this is a chance
        that it may fail to start gpfdist service as other processes are
        likely to be running on this specific port.   
        '''
        db = Database()
        db.setupDatabase('gptest')

        self.filerep.preprocess()
        self.filerep.setupGpfdist('8088', self.path)
        self.filerep.method_setup()
        super(CrashRecoveryFilerepTestCase, self).setUp()

    def tearDown(self):
        self.filerep.cleanupGpfdist('8088', self.path)
        super(CrashRecoveryFilerepTestCase, self).tearDown()

    def test_crash_recovery_filerep(self):
        # Flow of the function is:
        #  1 inject fault to skip check point
        #  2 inject fault to suspend filerep_transition_to_sync_before_checkpoint
        #  3 inject fault to suspend filerep_transition_to_sync_begin
        #  4 run the test for filerep incr mirror failover
        #  5 do stop start validation, recoverseg
        #  6 wait for insync transition and check the integrity of the cluster

        tinctest.logger.info('Skip Checkpointing using fault injector.')

        test_list_0 = []
        test_list_0.append(
            'mpp.gpdb.tests.storage.crashrecovery.SuspendCheckpointCrashRecovery.run_pre_filerep_faults'
        )
        self.test_case_scenario.append(test_list_0)

        test_list_1 = []
        test_list_1.append(
            'mpp.gpdb.tests.storage.filerep_end_to_end.test_filerep_e2e.FilerepE2EScenarioTestCase.test_incr_mirror'
        )
        self.test_case_scenario.append(test_list_1)

        test_list_2 = []
        test_list_2.append(
            'mpp.gpdb.tests.storage.crashrecovery.SuspendCheckpointCrashRecovery.do_post_run_checks'
        )
        self.test_case_scenario.append(test_list_2)
class CrashRecoveryFilerepTestCase(ScenarioTestCase,MPPTestCase):
    """
    @gucs gp_create_table_random_default_distribution=off
    """

    def __init__(self, methodName):
        self.filerep = FilerepTestCase('preprocess')
        self.path = os.path.join(self.get_source_dir(), '../filerep_end_to_end/data')
        super(CrashRecoveryFilerepTestCase, self).__init__(methodName)

    def setUp(self):
        '''
        gpfdist port should be dynamically generated, this is a chance
        that it may fail to start gpfdist service as other processes are
        likely to be running on this specific port.   
        '''
        db = Database()
        db.setupDatabase('gptest')

        self.filerep.preprocess()
        self.filerep.setupGpfdist('8088', self.path)
        self.filerep.method_setup()
        super(CrashRecoveryFilerepTestCase, self).setUp()

    def tearDown(self):
        self.filerep.cleanupGpfdist('8088', self.path)
        super(CrashRecoveryFilerepTestCase, self).tearDown()

    def test_crash_recovery_filerep(self):
        # Flow of the function is:
        #  1 inject fault to skip check point
        #  2 inject fault to suspend filerep_transition_to_sync_before_checkpoint
        #  3 inject fault to suspend filerep_transition_to_sync_begin
        #  4 run the test for filerep incr mirror failover
        #  5 do stop start validation, recoverseg 
        #  6 wait for insync transition and check the integrity of the cluster
       

        tinctest.logger.info('Skip Checkpointing using fault injector.')
        
        test_list_0 = []
        test_list_0.append('mpp.gpdb.tests.storage.crashrecovery.SuspendCheckpointCrashRecovery.run_pre_filerep_faults')
        self.test_case_scenario.append(test_list_0)
        
        test_list_1 = []
        test_list_1.append('mpp.gpdb.tests.storage.filerep_end_to_end.test_filerep_e2e.FilerepE2EScenarioTestCase.test_incr_mirror')
        self.test_case_scenario.append(test_list_1)

        test_list_2 = []
        test_list_2.append('mpp.gpdb.tests.storage.crashrecovery.SuspendCheckpointCrashRecovery.do_post_run_checks')
        self.test_case_scenario.append(test_list_2)
예제 #3
0
class FilerepE2EScenarioTestCase(ScenarioTestCase, MPPTestCase):
    """
    @gucs gp_create_table_random_default_distribution=off
    """
    def __init__(self, methodName):
        self.filerep = FilerepTestCase('preprocess')
        self.path = local_path("data")
        super(FilerepE2EScenarioTestCase, self).__init__(methodName)

    def setUp(self):
        '''
        gpfdist port should be dynamically generated, this is a chance
        that it may fail to start gpfdist service as other processes are
        likely to be running on this specific port.   
        '''

        super(FilerepE2EScenarioTestCase, self).setUp()
        db = Database()
        db.setupDatabase('gptest')

        self.filerep.preprocess()
        self.filerep.setupGpfdist('8088', self.path)
        self.filerep.method_setup()

    def tearDown(self):
        self.filerep.cleanupGpfdist('8088', self.path)
        super(FilerepE2EScenarioTestCase, self).tearDown()

    @classmethod
    def setUpClass(cls):
        pass

    @classmethod
    def tearDownClass(cls):
        pass

    def test_incr_mirror(self):
        self.do_test('incr', 'mirror')

    def test_full_mirror(self):
        self.do_test('full', 'mirror')

    def test_incr_primary(self):
        self.do_test('incr', 'primary')

    def test_full_primary(self):
        self.do_test('full', 'primary')

    def test_flat_file_resync_on_checksum_mismatch(self):

        filename = 'pg_distributedlog/FFFF'
        # This should cause the md5 check to fail and trigger a resync of the flat files
        self.filerep.create_file_in_datadir(0, 'p', filename)

        try:
            self.filerep.stop_start_validate()
            self.filerep.verify_file_exists(0, 'm', filename)
        finally:
            self.filerep.remove_file_in_datadir(0, 'p', filename)
            self.filerep.remove_file_in_datadir(0, 'm', filename)

    def test_flat_file_resync_on_checksum_match(self):

        filename = 'pg_distributedlog/FFFF'

        self.filerep.create_file_in_datadir(0, 'p', filename)
        self.filerep.create_file_in_datadir(0, 'm', filename)

        try:
            timestamp_before_restart = self.filerep.get_timestamp_of_file_in_datadir(
                0, 'm', filename)
            self.filerep.stop_start_validate()
            timestamp_after_restart = self.filerep.get_timestamp_of_file_in_datadir(
                0, 'm', filename)
            self.assertEqual(
                timestamp_before_restart, timestamp_after_restart,
                'File %s transferred after restart. timestamp before restart (%s), timestamp after restart (%s)'
                %
                (filename, timestamp_before_restart, timestamp_after_restart))
        finally:
            self.filerep.remove_file_in_datadir(0, 'p', filename)
            self.filerep.remove_file_in_datadir(0, 'm', filename)

    def do_test(self, rec_mode, fail_type):
        '''
        @rec_mode: recovery mode, can be full or incremental
        @fail_type, failover type, can be mirror or primary
        '''

        self.do_sync1_tests(fail_type)
        self.do_ck_sync1_tests()
        self.do_ct_tests(fail_type)
        self.do_resync_tests(rec_mode, fail_type)
        self.do_sync2_tests(rec_mode, fail_type)
        self.do_clean_tests()

    def do_sync1_tests(self, fail_type):
        '''
        Run workload while insync transition state, creates workload to be tested by following
        transition phases.
        @fail_type: failover type, can be mirror or primary
        '''

        list_cl = []
        list_cl.append(
            "mpp.gpdb.tests.storage.filerep_end_to_end.FilerepTestCase.clean_data"
        )
        self.test_case_scenario.append(list_cl)

        list_set_sync1 = []
        list_set_sync1.append(
            "mpp.gpdb.tests.storage.filerep_end_to_end.set_sync1.test_set_sync1.SetSync1TestCase"
        )
        self.test_case_scenario.append(list_set_sync1)

        list_sync1 = []
        list_sync1.append(
            "mpp.gpdb.tests.storage.filerep_end_to_end.sync1.test_sync1.Sync1TestCase"
        )
        self.test_case_scenario.append(list_sync1)

        list_serial = []
        list_serial.append(
            "mpp.gpdb.tests.storage.filerep_end_to_end.sync1.vacuum.test_vacuum.VacuumTestCase"
        )
        list_serial.append(
            "mpp.gpdb.tests.storage.filerep_end_to_end.FilerepTestCase.check_mirror_seg"
        )
        list_serial.append((
            "mpp.gpdb.tests.storage.filerep_end_to_end.FilerepTestCase.run_gpstate",
            [fail_type, 'sync1']))
        list_serial.append((
            "mpp.gpdb.tests.storage.filerep_end_to_end.FilerepTestCase.do_gpcheckcat",
            {
                'outputFile': 'sync1_checkcat.out'
            }))
        self.test_case_scenario.append(list_serial, serial=True)

    def do_ck_sync1_tests(self):
        '''
        Run workload in insync but after pushing a checkpoint, in case heap data pages maybe in flight getting flushed to mirror 
        and not on primary.
        '''

        list_cl_and_ck = []
        list_cl_and_ck.append(
            "mpp.gpdb.tests.storage.filerep_end_to_end.FilerepTestCase.clean_data"
        )
        list_cl_and_ck.append(
            "mpp.gpdb.tests.storage.filerep_end_to_end.runcheckpoint.runCheckPointSQL.runCheckPointTestCase"
        )
        self.test_case_scenario.append(list_cl_and_ck, serial=True)

        list_set_checkpoint_sync1 = []
        list_set_checkpoint_sync1.append(
            "mpp.gpdb.tests.storage.filerep_end_to_end.set_ck_sync1.test_set_ck_sync1.SetCkSync1TestCase"
        )
        self.test_case_scenario.append(list_set_checkpoint_sync1)

        list_checkpoint_sync1 = []
        list_checkpoint_sync1.append(
            "mpp.gpdb.tests.storage.filerep_end_to_end.ck_sync1.test_ck_sync1.CkSync1TestCase"
        )
        self.test_case_scenario.append(list_checkpoint_sync1)

        list_serial = []
        list_serial.append(
            "mpp.gpdb.tests.storage.filerep_end_to_end.ck_sync1.vacuum.test_vacuum.VacuumTestCase"
        )
        list_serial.append((
            "mpp.gpdb.tests.storage.filerep_end_to_end.FilerepTestCase.do_gpcheckcat",
            {
                'outputFile': 'ck_sync1_checkcat.out'
            }))
        self.test_case_scenario.append(list_serial, serial=True)

    def do_ct_tests(self, fail_type):
        '''
        Run workload in change tracking
        @fail_type: failover type, can be mirror or primary
        '''

        list = []
        list.append(
            "mpp.gpdb.tests.storage.filerep_end_to_end.FilerepTestCase.clean_data"
        )
        list.append((
            "mpp.gpdb.tests.storage.filerep_end_to_end.FilerepTestCase.method_run_failover",
            [fail_type]))
        if (fail_type == 'mirror'):
            list.append(
                "mpp.gpdb.tests.storage.filerep_end_to_end.FilerepTestCase.trigger_transition"
            )
        list.append(
            "mpp.gpdb.tests.storage.filerep_end_to_end.FilerepTestCase.wait_till_change_tracking_transition"
        )
        self.test_case_scenario.append(list, serial=True)

        list_set_ct = []
        list_set_ct.append(
            "mpp.gpdb.tests.storage.filerep_end_to_end.set_ct.test_set_ct.SetCtTestCase"
        )
        self.test_case_scenario.append(list_set_ct)

        list_ct = []
        list_ct.append(
            "mpp.gpdb.tests.storage.filerep_end_to_end.ct.test_ct.CtTestCase")
        self.test_case_scenario.append(list_ct)

        list_serial = []
        list_serial.append(
            "mpp.gpdb.tests.storage.filerep_end_to_end.ct.vacuum.test_vacuum.VacuumTestCase"
        )
        list_serial.append((
            "mpp.gpdb.tests.storage.filerep_end_to_end.FilerepTestCase.do_gpcheckcat",
            {
                'outputFile': 'ct_checkcat.out'
            }))
        list_serial.append((
            "mpp.gpdb.tests.storage.filerep_end_to_end.FilerepTestCase.run_gpstate",
            [fail_type, 'ct']))
        self.test_case_scenario.append(list_serial, serial=True)

    def do_resync_tests(self, rec_mode, fail_type):
        '''
        Suspend the transition to resync, trigger gprecoverseg, run workload.
        @rec_mode: recovery type, can be full or incremental.
        @fail_type: failover type, can be mirror or primary.
        '''

        list = []
        list.append(
            "mpp.gpdb.tests.storage.filerep_end_to_end.FilerepTestCase.clean_data"
        )
        list.append(
            "mpp.gpdb.tests.storage.filerep_end_to_end.FilerepTestCase.run_method_suspendresync"
        )
        if ((rec_mode == 'incr') and (fail_type == 'primary')):
            list.append(
                "mpp.gpdb.tests.storage.filerep_end_to_end.FilerepTestCase.inject_fault_on_first_primary"
            )
        list.append((
            "mpp.gpdb.tests.storage.filerep_end_to_end.FilerepTestCase.run_gprecoverseg",
            [rec_mode]))
        list.append((
            "mpp.gpdb.tests.storage.filerep_end_to_end.FilerepTestCase.run_gpstate",
            [fail_type, 'resync_' + rec_mode]))
        self.test_case_scenario.append(list, serial=True)

        list_set_resync = []
        list_set_resync.append(
            "mpp.gpdb.tests.storage.filerep_end_to_end.set_resync.test_set_resync.SetResyncTestCase"
        )
        self.test_case_scenario.append(list_set_resync)

        list_resync = []
        list_resync.append(
            "mpp.gpdb.tests.storage.filerep_end_to_end.resync.test_resync.ResyncTestCase"
        )
        self.test_case_scenario.append(list_resync)

        list_serial = []
        list_serial.append(
            "mpp.gpdb.tests.storage.filerep_end_to_end.resync.vacuum.test_vacuum.VacuumTestCase"
        )
        list_serial.append((
            "mpp.gpdb.tests.storage.filerep_end_to_end.FilerepTestCase.do_gpcheckcat",
            {
                'outputFile': 'resync_checkcat.out'
            }))
        self.test_case_scenario.append(list_serial, serial=True)

    def do_sync2_tests(self, rec_mode, fail_type):
        '''
        Resume and reset the fault, bring cluster into insync, run workload, perform checkmirrorseg and check catalog.
        '''

        list_resume_validate = []
        list_resume_validate.append(
            "mpp.gpdb.tests.storage.filerep_end_to_end.FilerepTestCase.clean_data"
        )
        list_resume_validate.append(
            "mpp.gpdb.tests.storage.filerep_end_to_end.FilerepTestCase.method_resume_filerep_resync"
        )
        list_resume_validate.append(
            "mpp.gpdb.tests.storage.filerep_end_to_end.FilerepTestCase.method_reset_fault_injection"
        )
        if ((rec_mode == 'incr') and (fail_type == 'primary')):
            list_resume_validate.append(
                "mpp.gpdb.tests.storage.filerep_end_to_end.FilerepTestCase.inject_fault_on_first_mirror"
            )
            list_resume_validate.append((
                "mpp.gpdb.tests.storage.filerep_end_to_end.FilerepTestCase.stop_start_validate",
                [False]))
        else:
            list_resume_validate.append(
                "mpp.gpdb.tests.storage.filerep_end_to_end.FilerepTestCase.stop_start_validate"
            )
        list_resume_validate.append(
            "mpp.gpdb.tests.storage.filerep_end_to_end.FilerepTestCase.wait_till_insync_transition"
        )
        self.test_case_scenario.append(list_resume_validate, serial=True)

        list_set_sync2 = []
        list_set_sync2.append(
            "mpp.gpdb.tests.storage.filerep_end_to_end.set_sync2.test_set_sync2.SetSync2TestCase"
        )
        self.test_case_scenario.append(list_set_sync2)

        list_sync2 = []
        list_sync2.append(
            "mpp.gpdb.tests.storage.filerep_end_to_end.sync2.test_sync2.Sync2TestCase"
        )
        self.test_case_scenario.append(list_sync2)

        list_serial = []
        list_serial.append(
            "mpp.gpdb.tests.storage.filerep_end_to_end.sync2.vacuum.test_vacuum.VacuumTestCase"
        )
        list_serial.append(
            "mpp.gpdb.tests.storage.filerep_end_to_end.FilerepTestCase.check_mirror_seg"
        )
        list_serial.append((
            "mpp.gpdb.tests.storage.filerep_end_to_end.FilerepTestCase.do_gpcheckcat",
            {
                'outputFile': 'sync2_checkcat.out'
            }))
        self.test_case_scenario.append(list_serial, serial=True)

    def do_clean_tests(self):
        '''
        Drop databases, tablespaces, and filespaces.
        '''

        list_cleanup = []
        list_cleanup.append(
            "mpp.gpdb.tests.storage.filerep_end_to_end.clean.test_clean.CleanTestCase"
        )
        self.test_case_scenario.append(list_cleanup)
예제 #4
0
class FilerepE2EScenarioTestCase(ScenarioTestCase, MPPTestCase):
    """
    @gucs gp_create_table_random_default_distribution=off
    """
    
    def __init__(self, methodName):
        self.filerep = FilerepTestCase('preprocess')
        self.path = local_path("data")
        super(FilerepE2EScenarioTestCase,self).__init__(methodName)


    def setUp(self):
        '''
        gpfdist port should be dynamically generated, this is a chance
        that it may fail to start gpfdist service as other processes are
        likely to be running on this specific port.   
        '''

        super(FilerepE2EScenarioTestCase, self).setUp()
        db = Database()
        db.setupDatabase('gptest')
        
        self.filerep.preprocess()
        self.filerep.setupGpfdist('8088', self.path)
        self.filerep.method_setup()
        
    def tearDown(self):
        self.filerep.cleanupGpfdist('8088', self.path)
        super(FilerepE2EScenarioTestCase, self).tearDown()
         
    @classmethod
    def setUpClass(cls):
        pass

    @classmethod
    def tearDownClass(cls):
        pass

    def test_incr_mirror(self):
        self.do_test('incr','mirror')

    def test_full_mirror(self): 
        self.do_test('full','mirror')

    def test_incr_primary(self):
        self.do_test('incr','primary')

    def test_full_primary(self):
        self.do_test('full', 'primary')

    def test_flat_file_resync_on_checksum_mismatch(self):

        filename = 'pg_distributedlog/FFFF'
        # This should cause the md5 check to fail and trigger a resync of the flat files
        self.filerep.create_file_in_datadir(0, 'p', filename)

        try:
            self.filerep.stop_start_validate()
            self.filerep.verify_file_exists(0, 'm', filename)
        finally:
            self.filerep.remove_file_in_datadir(0, 'p', filename)
            self.filerep.remove_file_in_datadir(0, 'm', filename)

    def test_flat_file_resync_on_checksum_match(self):

        filename = 'pg_distributedlog/FFFF'

        self.filerep.create_file_in_datadir(0, 'p', filename)
        self.filerep.create_file_in_datadir(0, 'm', filename)

        try:
            timestamp_before_restart = self.filerep.get_timestamp_of_file_in_datadir(0, 'm', filename)
            self.filerep.stop_start_validate()
            timestamp_after_restart = self.filerep.get_timestamp_of_file_in_datadir(0, 'm', filename)
            self.assertEqual(timestamp_before_restart, timestamp_after_restart,
                             'File %s transferred after restart. timestamp before restart (%s), timestamp after restart (%s)' %
                              (filename, timestamp_before_restart, timestamp_after_restart))
        finally:
            self.filerep.remove_file_in_datadir(0, 'p', filename)
            self.filerep.remove_file_in_datadir(0, 'm', filename)

    def do_test(self,rec_mode,fail_type):
        '''
        @rec_mode: recovery mode, can be full or incremental
        @fail_type, failover type, can be mirror or primary
        '''

        self.do_sync1_tests(fail_type)
        self.do_ck_sync1_tests()
        self.do_ct_tests(fail_type)
        self.do_resync_tests(rec_mode, fail_type)
        self.do_sync2_tests(rec_mode, fail_type)
        self.do_clean_tests()

    def do_sync1_tests(self, fail_type):
        '''
        Run workload while insync transition state, creates workload to be tested by following
        transition phases.
        @fail_type: failover type, can be mirror or primary
        '''

        list_cl = []
        list_cl.append("mpp.gpdb.tests.storage.filerep_end_to_end.FilerepTestCase.clean_data")
        self.test_case_scenario.append(list_cl) 

        list_set_sync1 = []
        list_set_sync1.append("mpp.gpdb.tests.storage.filerep_end_to_end.set_sync1.test_set_sync1.SetSync1TestCase")
        self.test_case_scenario.append(list_set_sync1)
        
        list_sync1 = []
        list_sync1.append("mpp.gpdb.tests.storage.filerep_end_to_end.sync1.test_sync1.Sync1TestCase")
        self.test_case_scenario.append(list_sync1)
 
        list_serial = []
        list_serial.append("mpp.gpdb.tests.storage.filerep_end_to_end.sync1.vacuum.test_vacuum.VacuumTestCase")
        list_serial.append("mpp.gpdb.tests.storage.filerep_end_to_end.FilerepTestCase.check_mirror_seg")
        list_serial.append(("mpp.gpdb.tests.storage.filerep_end_to_end.FilerepTestCase.run_gpstate", [fail_type, 'sync1']))
        list_serial.append(("mpp.gpdb.tests.storage.filerep_end_to_end.FilerepTestCase.do_gpcheckcat", {'outputFile':'sync1_checkcat.out'}))
        self.test_case_scenario.append(list_serial, serial=True)

    def do_ck_sync1_tests(self):
        '''
        Run workload in insync but after pushing a checkpoint, in case heap data pages maybe in flight getting flushed to mirror 
        and not on primary.
        '''

        list_cl_and_ck = []
        list_cl_and_ck.append("mpp.gpdb.tests.storage.filerep_end_to_end.FilerepTestCase.clean_data")
        list_cl_and_ck.append("mpp.gpdb.tests.storage.filerep_end_to_end.runcheckpoint.runCheckPointSQL.runCheckPointTestCase")
        self.test_case_scenario.append(list_cl_and_ck, serial=True)

        list_set_checkpoint_sync1 = []
        list_set_checkpoint_sync1.append("mpp.gpdb.tests.storage.filerep_end_to_end.set_ck_sync1.test_set_ck_sync1.SetCkSync1TestCase")
        self.test_case_scenario.append(list_set_checkpoint_sync1)

        list_checkpoint_sync1 = []
        list_checkpoint_sync1.append("mpp.gpdb.tests.storage.filerep_end_to_end.ck_sync1.test_ck_sync1.CkSync1TestCase")
        self.test_case_scenario.append(list_checkpoint_sync1)

        list_serial = []
        list_serial.append("mpp.gpdb.tests.storage.filerep_end_to_end.ck_sync1.vacuum.test_vacuum.VacuumTestCase")
        list_serial.append(("mpp.gpdb.tests.storage.filerep_end_to_end.FilerepTestCase.do_gpcheckcat", {'outputFile':'ck_sync1_checkcat.out'}))
        self.test_case_scenario.append(list_serial, serial=True)        

    def do_ct_tests(self, fail_type):
        '''
        Run workload in change tracking
        @fail_type: failover type, can be mirror or primary
        '''

        list = []
        list.append("mpp.gpdb.tests.storage.filerep_end_to_end.FilerepTestCase.clean_data")
        list.append(("mpp.gpdb.tests.storage.filerep_end_to_end.FilerepTestCase.method_run_failover",[fail_type]))
        if (fail_type=='mirror'):
            list.append("mpp.gpdb.tests.storage.filerep_end_to_end.FilerepTestCase.trigger_transition")
        list.append("mpp.gpdb.tests.storage.filerep_end_to_end.FilerepTestCase.wait_till_change_tracking_transition")
        self.test_case_scenario.append(list,serial=True)
        
        list_set_ct = []
        list_set_ct.append("mpp.gpdb.tests.storage.filerep_end_to_end.set_ct.test_set_ct.SetCtTestCase")
        self.test_case_scenario.append(list_set_ct)

        list_ct = []
        list_ct.append("mpp.gpdb.tests.storage.filerep_end_to_end.ct.test_ct.CtTestCase")
        self.test_case_scenario.append(list_ct)

        list_serial = []
        list_serial.append("mpp.gpdb.tests.storage.filerep_end_to_end.ct.vacuum.test_vacuum.VacuumTestCase")
        list_serial.append(("mpp.gpdb.tests.storage.filerep_end_to_end.FilerepTestCase.do_gpcheckcat",{'outputFile':'ct_checkcat.out'}))
        list_serial.append(("mpp.gpdb.tests.storage.filerep_end_to_end.FilerepTestCase.run_gpstate",[fail_type, 'ct']))
        self.test_case_scenario.append(list_serial, serial=True)

    def do_resync_tests(self, rec_mode, fail_type):
        '''
        Suspend the transition to resync, trigger gprecoverseg, run workload.
        @rec_mode: recovery type, can be full or incremental.
        @fail_type: failover type, can be mirror or primary.
        '''

        list = []
        list.append("mpp.gpdb.tests.storage.filerep_end_to_end.FilerepTestCase.clean_data")
        list.append("mpp.gpdb.tests.storage.filerep_end_to_end.FilerepTestCase.run_method_suspendresync")
        if ((rec_mode == 'incr') and (fail_type == 'primary')):
            list.append("mpp.gpdb.tests.storage.filerep_end_to_end.FilerepTestCase.inject_fault_on_first_primary");
        list.append(("mpp.gpdb.tests.storage.filerep_end_to_end.FilerepTestCase.run_gprecoverseg",[rec_mode]))
        list.append(("mpp.gpdb.tests.storage.filerep_end_to_end.FilerepTestCase.run_gpstate",[fail_type,'resync_'+rec_mode]))
        self.test_case_scenario.append(list, serial=True) 

        list_set_resync = []
        list_set_resync.append("mpp.gpdb.tests.storage.filerep_end_to_end.set_resync.test_set_resync.SetResyncTestCase")
        self.test_case_scenario.append(list_set_resync)

        list_resync = []
        list_resync.append("mpp.gpdb.tests.storage.filerep_end_to_end.resync.test_resync.ResyncTestCase")
        self.test_case_scenario.append(list_resync)

        list_serial = []
        list_serial.append("mpp.gpdb.tests.storage.filerep_end_to_end.resync.vacuum.test_vacuum.VacuumTestCase")
        list_serial.append(("mpp.gpdb.tests.storage.filerep_end_to_end.FilerepTestCase.do_gpcheckcat",{'outputFile':'resync_checkcat.out'}))
        self.test_case_scenario.append(list_serial,serial=True) 

    def do_sync2_tests(self, rec_mode, fail_type):
        '''
        Resume and reset the fault, bring cluster into insync, run workload, perform checkmirrorseg and check catalog.
        '''

        list_resume_validate = []
        list_resume_validate.append("mpp.gpdb.tests.storage.filerep_end_to_end.FilerepTestCase.clean_data")
        list_resume_validate.append("mpp.gpdb.tests.storage.filerep_end_to_end.FilerepTestCase.method_resume_filerep_resync")
        list_resume_validate.append("mpp.gpdb.tests.storage.filerep_end_to_end.FilerepTestCase.method_reset_fault_injection")
        if ((rec_mode == 'incr') and (fail_type == 'primary')):
            list_resume_validate.append("mpp.gpdb.tests.storage.filerep_end_to_end.FilerepTestCase.inject_fault_on_first_mirror");
            list_resume_validate.append(("mpp.gpdb.tests.storage.filerep_end_to_end.FilerepTestCase.stop_start_validate", [False]))
        else:
            list_resume_validate.append("mpp.gpdb.tests.storage.filerep_end_to_end.FilerepTestCase.stop_start_validate")
        list_resume_validate.append("mpp.gpdb.tests.storage.filerep_end_to_end.FilerepTestCase.wait_till_insync_transition")
        self.test_case_scenario.append(list_resume_validate,serial=True)

        list_set_sync2 = []
        list_set_sync2.append("mpp.gpdb.tests.storage.filerep_end_to_end.set_sync2.test_set_sync2.SetSync2TestCase")
        self.test_case_scenario.append(list_set_sync2)

        list_sync2 = []
        list_sync2.append("mpp.gpdb.tests.storage.filerep_end_to_end.sync2.test_sync2.Sync2TestCase")
        self.test_case_scenario.append(list_sync2)

        list_serial = []
        list_serial.append("mpp.gpdb.tests.storage.filerep_end_to_end.sync2.vacuum.test_vacuum.VacuumTestCase")
        list_serial.append("mpp.gpdb.tests.storage.filerep_end_to_end.FilerepTestCase.check_mirror_seg")
        list_serial.append(("mpp.gpdb.tests.storage.filerep_end_to_end.FilerepTestCase.do_gpcheckcat",{'outputFile':'sync2_checkcat.out'}))
        self.test_case_scenario.append(list_serial,serial=True)

    def do_clean_tests(self):
        '''
        Drop databases, tablespaces, and filespaces.
        '''

        list_cleanup = []
        list_cleanup.append("mpp.gpdb.tests.storage.filerep_end_to_end.clean.test_clean.CleanTestCase")
        self.test_case_scenario.append(list_cleanup)