def crash_system_after_checkpoint(self, mode):
'''Issue checkpoint followed by gpstop -ai immediately '''
newfault = Fault()
if mode == 'InSync':
#Sleep 10-15 min
ntime = randint(600, 900)
tinctest.logger.info('sleep for %s sec...' %ntime)
sleep(ntime)
else:
#Sleep 10-15 min
ntime = randint(600, 900)
tinctest.logger.info('sleep for %s sec...' %ntime)
sleep(ntime)
#put system in CT
is_ct_mode_enabled = self.do_changetracking()
if not is_ct_mode_enabled:
self.fail("Change Tracking mode not enabled!!")
#Sleep 10-15 min
ntime = randint(600, 900)
tinctest.logger.info('sleep for %s sec...' %ntime)
sleep(ntime)
#issue checkpoint
newfault.issue_checkpoint()
newfault.restart_db()
def setUp(self):
''' Need to rebalance cluster as primary segments are killed during test'''
super(PersistentTables,self).setUp()
tinctest.logger.info('***Rebalancing cluster state***')
fault_recovery = Fault()
if not fault_recovery.rebalance_cluster():
raise ClusterStateException("**FATAL!! Cluster rebalancing failed - segments went down after \
gprecoverseg -ar, even incremental recovery couldn't bring the segments up. \
Cannot proceed with the tests!! ")
def test_changetracking_kill_primary_reboot(self):
sleep(30) #sleep 30sec, so that there will be some inflight Transactions
is_ct_mode_enabled = self.do_changetracking()
if not is_ct_mode_enabled:
self.fail("Change Tracking mode not enabled!!")
sleep(100) #sleep 100sec, so that there will be some inflight Transactions
#Reboot so that concurrent sessions will end
newfault = Fault()
newfault.restart_db()
def test_partition_table_scenario(self):
tinctest.logger.debug('Starting the Partition tables scenario')
newfault = Fault()
if newfault.issue_checkpoint():
tinctest.logger.info('Checkpoint forced..')
else:
tinctest.logger.info('Unable to force checkpoint - Is the server running?')
self.fail('Unable to force checkpoint - Check the server')
newfault.restart_db()
tinctest.logger.debug('Partition tables scenario completed')
def test_changetracking_kill_primary(self):
sleep(30) #sleep 30sec, so that there will be some inflight Transactions
is_ct_mode_enabled = self.do_changetracking()
if not is_ct_mode_enabled:
self.fail("Change Tracking mode not enabled!!")
newfault = Fault()
sleep(100)
tinctest.logger.info('Executing incremental recovery in the background')
newfault._run_sys_cmd('gprecoverseg -a &')
newfault.stop_db('i')
def setUp(self):
''' Need to rebalance cluster as primary segments are killed during test'''
super(PersistentTables, self).setUp()
tinctest.logger.info('***Rebalancing cluster state***')
fault_recovery = Fault()
if not fault_recovery.rebalance_cluster():
raise ClusterStateException(
"**FATAL!! Cluster rebalancing failed - segments went down after \
gprecoverseg -ar, even incremental recovery couldn't bring the segments up. \
Cannot proceed with the tests!! ")
def setUpClass(cls):
super(PersistentTables,cls).setUpClass()
tinctest.logger.info('Setup Database ...')
setdb = Fault()
setdb.create_db()
tinctest.logger.info('Running the initial setup sql files')
setup = InitialSetup()
setup.createSQLFiles()
setup.runSQLFiles()
def setUpClass(cls):
super(PersistentTables, cls).setUpClass()
tinctest.logger.info('Setup Database ...')
setdb = Fault()
setdb.create_db()
tinctest.logger.info('Running the initial setup sql files')
setup = InitialSetup()
setup.createSQLFiles()
setup.runSQLFiles()
def test_failover_to_mirror_during_transactions(self):
tinctest.logger.debug('Fault Injection Tests - starting the failover to mirror scenario')
#Sleep introduced so that the concurrently running generate_sqls gets time
#to actually generate some load
tinctest.logger.info('Sleep introduced of 120 sec')
sleep(120)
newfault = Fault()
newfault.kill_processes_with_role('p')
newfault.restart_db()
tinctest.logger.debug('Fault Injection Tests - failover to mirror completed')
def do_changetracking(self, segment_type='primary'):
newfault = Fault()
newfault.changetracking(segment_type)
rtrycnt = 0
max_rtrycnt = 20
while (newfault.is_changetracking() == False and rtrycnt < max_rtrycnt):
tinctest.logger.info("waiting [%s] for DB to go into change tracking" %rtrycnt)
sleep(10)
rtrycnt = rtrycnt + 1
if rtrycnt == max_rtrycnt:
tinctest.logger.error("Segments not up after incremental recovery!!")
return False
return True
def setUpClass(cls):
super(PersistentTables,cls).setUpClass()
tinctest.logger.info('Setup Database ...')
setdb = Fault()
setdb.create_db()
tinctest.logger.info('Running the initial setup sql files')
setup = InitialSetup()
setup.createSQLFiles()
setup.runSQLFiles()
tinctest.logger.info('Generating the load - sql files to be run concurrently')
sqldatagen = GenerateSqls()
sqldatagen.generate_sqls()
def setUpClass(cls):
super(PersistentTables, cls).setUpClass()
tinctest.logger.info('Setup Database ...')
setdb = Fault()
setdb.create_db()
tinctest.logger.info('Running the initial setup sql files')
setup = InitialSetup()
setup.createSQLFiles()
setup.runSQLFiles()
tinctest.logger.info(
'Generating the load - sql files to be run concurrently')
sqldatagen = GenerateSqls()
sqldatagen.generate_sqls()
def setUpClass(cls):
super(PersistentTables,cls).setUpClass()
tinctest.logger.info('Setup Database ...')
setdb = Fault()
setdb.create_db()
tinctest.logger.info('Running the initial setup sql files')
setup = InitialSetup()
setup.createSQLFiles()
setup.runSQLFiles()
tinctest.logger.info('Running the initial setup sql files - Done')
tinctest.logger.info('Generating the load - sql files to be run concurrently')
sqldatagen = GenerateSqls()
sqldatagen.generate_sqls()
tinctest.logger.info('Generating the load - sql files to be run concurrently - Done')
tinctest.logger.info('Generating the sql files required for Partition table scenario')
sql_generation = GeneratePartitionSqls()
sql_generation.generate_sqls()
tinctest.logger.info('Partition SQL files created.')
def setUpClass(cls):
super(PersistentTables, cls).setUpClass()
tinctest.logger.info('Setup Database ...')
setdb = Fault()
setdb.create_db()
tinctest.logger.info('Running the initial setup sql files')
setup = InitialSetup()
setup.createSQLFiles()
setup.runSQLFiles()
tinctest.logger.info('Running the initial setup sql files - Done')
tinctest.logger.info(
'Generating the load - sql files to be run concurrently')
sqldatagen = GenerateSqls()
sqldatagen.generate_sqls()
tinctest.logger.info(
'Generating the load - sql files to be run concurrently - Done')
tinctest.logger.info(
'Generating the sql files required for Partition table scenario')
sql_generation = GeneratePartitionSqls()
sql_generation.generate_sqls()
tinctest.logger.info('Partition SQL files created.')
def test_drop_db_dir(self):
''' Drop Database dir only '''
newfault = Fault()
#Get segment host, data dir, and DB oid to create fault
(hostname, dirloc, oid) = newfault.get_host_dir_oid()
# Move database dir to take it offline
DirPath = dirloc + '/base/'+ oid
newfault.drop_dir(hostname, DirPath)
#issue checkpoint
newfault.issue_checkpoint()
def post_master_reset_during_transaction(self,kill_all = True):
tinctest.logger.debug('Fault Injection Tests - starting postmaster reset scenario')
#Sleep introduced so that the concurrently running generate_sqls gets time
#to actually generate some load
tinctest.logger.info('Sleep introduced of 120 sec')
sleep(120)
newfault = Fault()
if kill_all:
newfault.kill_all_segment_processes()
else:
newfault.kill_all_segment_processes(False)
result = newfault.restart_db()
while result.rc != 0:
tinctest.logger.debug("gpstop -air failed - cluster restarting!!")
tinctest.logger.debug("executing gpstop -air again...")
result = newfault.restart_db()
tinctest.logger.debug('Fault Injection Tests - postmaster reset scenario completed')
def test_kill_segments_periodically(self):
''' Create system faults by killing segments randomly '''
newfault = Fault()
#1. Kill Mirror Segments
newfault.kill_segment_processes_randomly('m')
#Sleep 10-15 min
ntime = randint(600, 900)
tinctest.logger.info('sleep for %s sec. before start killing primary process' %ntime)
sleep(ntime)
#3. Kill primary Process
newfault.kill_segment_processes_randomly('p')
#Sleep 10-15 min
ntime = randint(600, 900)
tinctest.logger.info('sleep for %s sec. befor reboot' %ntime)
sleep(ntime)
#5. Restart the cluster
newfault.restart_db()
def test_put_system_in_change_tracking_mode(self):
''' Putting the system in Change Tracking mode '''
newfault = Fault()
config = GPDBConfig()
tinctest.logger.info('Putting the system in change tracking mode')
newfault.kill_processes_with_role('p')
def test_rebalance_segment(self):
newfault = Fault()
self.assertTrue(newfault.rebalance_cluster(),
"Segments not rebalanced!!")
def gpstart_db_restricted(self):
''' Start Database with Restricted mode '''
sleep(5)
newfault = Fault()
newfault.start_db('R')
def test_oom_scenario(self):
tinctest.logger.debug('Restarting the db in 40 mins for hitting OOM')
newfault = Fault()
# Sleep for 40 minutes so that the OOM scenario is hit before restarting the system
sleep(2400)
newfault.restart_db()
def test_drop_db(self):
''' Test to Simulate drop_db when database dir is offline '''
newfault = Fault()
sleep(20)
#Get segment host, data dir, and DB oid to create fault
(hostname, dirloc, oid) = newfault.get_host_dir_oid()
# Move database dir to take it offline
fromDir = dirloc + '/base/'+ oid
toDir = dirloc + '/base/../../'
newfault.move_dir(hostname, fromDir, toDir)
newfault.drop_db()
#restore database dir
fromDir = dirloc + '/base/../../'+ oid
toDir = dirloc + '/base/'
newfault.move_dir(hostname, fromDir, toDir)
newfault.drop_db()
#Create db so that validations will work
newfault.create_db()
def test_rebalance_segment(self):
newfault = Fault()
self.assertTrue(newfault.rebalance_cluster(),"Segments not rebalanced!!")
def gpstart_db_restricted(self):
''' Start Database with Restricted mode '''
sleep(5)
newfault = Fault()
newfault.start_db('R')
def gpstart_db(self):
''' Start Database with normal mode '''
sleep(5)
newfault = Fault()
newfault.start_db()
def gpstop_db(self):
''' Stop database with normal mode '''
newfault = Fault()
sleep(5)
newfault.stop_db()
def test_recovery_full_abort(self):
newfault = Fault()
sleep(100)
newfault._run_sys_cmd('gprecoverseg -aF &')
newfault.stop_db('i')
def test_recovery_full_abort(self):
newfault = Fault()
sleep(100)
newfault._run_sys_cmd('gprecoverseg -aF &')
newfault.stop_db('i')
def gpstop_db_immediate(self):
''' Stop database with immediate mode '''
newfault = Fault()
sleep(5)
newfault.stop_db('i')
def test_stop_db(self):
newfault = Fault()
newfault.stop_db('i')
def gprestart_db(self):
''' Restarts the Database '''
sleep(5)
newfault = Fault()
newfault.restart_db()
def gpstop_db_immediate(self):
''' Stop database with immediate mode '''
newfault = Fault()
sleep(5)
newfault.stop_db('i')
def gpstart_db(self):
''' Start Database with normal mode '''
sleep(5)
newfault = Fault()
newfault.start_db()
def gprestart_db(self):
''' Restarts the Database '''
sleep(5)
newfault = Fault()
newfault.restart_db()
def gpstop_db(self):
''' Stop database with normal mode '''
newfault = Fault()
sleep(5)
newfault.stop_db()
def test_skip_checkpoint(self):
tinctest.logger.info('Skipping the checkpoint...')
newfault = Fault()
newfault.skip_checkpoint()
