def test_uao_crash_compaction_before_cleanup_phase_master_with_aocs(self):
setup_file = self.get_sql_files("uao_crash_compaction_before_cleanup_phase_master_with_aocs_setup")[0]
(sql_file1, out_file1,ans_file1) = self.get_sql_files("uao_crash_compaction_before_cleanup_phase_master_with_aocs1")
(sql_file2, out_file2, ans_file2) = self.get_sql_files("uao_crash_compaction_before_cleanup_phase_master_with_aocs2")
if not os.path.exists(os.path.dirname(out_file1)):
os.mkdir(os.path.dirname(out_file1))
set_fault_in_master_panic = 'source %s/greenplum_path.sh;gpfaultinjector -p %s -f compaction_before_cleanup_phase -y panic --seg_dbid 1' % (os.getenv('GPHOME'), os.getenv('PGPORT'))
set_fault_in_master_reset = 'source %s/greenplum_path.sh;gpfaultinjector -p %s -f compaction_before_cleanup_phase -y reset --seg_dbid 1' % (os.getenv('GPHOME'), os.getenv('PGPORT'))
cmd_type = 'fault injector'
PSQL.run_sql_file(setup_file)
gpfaultinjector = Command(cmd_type, set_fault_in_master_panic)
gpfaultinjector.run()
PSQL.run_sql_file(sql_file1, out_file=out_file1)
result1 = Gpdiff.are_files_equal(out_file1, ans_file1, match_sub=[gpdiff_init_file])
PSQL.wait_for_database_up();
gpfaultinjector = Command(cmd_type, set_fault_in_master_reset)
gpfaultinjector.run()
PSQL.run_sql_file(sql_file2, out_file=out_file2)
result2 = Gpdiff.are_files_equal(out_file2, ans_file2, match_sub=[gpdiff_init_file])
self.assertTrue(result1)
self.assertTrue(result2)
def run_faults_before_trigger(self, checkpoint, cluster_state, fault_type):
'''
@param checkpoint : skip/noskip
@param cluster_state : sync/change_tracking/resync
@param fault_type : commit/abort
@param end_prepare_two_phase_sleep : True/False
@description : 1. Suspend resync faults. 2. Issue Checkpoint before the skip checkpoint, so that the bufferpool is cleared. 3. If skip issue 'skip checkpoint'. 4. Suspend transaction_faults based on test_type.
'''
if cluster_state == 'change_tracking':
self.invoke_fault('filerep_consumer', 'fault')
self.filereputil.wait_till_change_tracking_transition()
tinctest.logger.info('Change_tracking transition complete')
if cluster_state == 'resync':
self.invoke_fault('filerep_resync', 'suspend', role='primary')
if checkpoint == 'skip':
self.invoke_fault('filerep_transition_to_sync_before_checkpoint', 'suspend', role='primary', port=self.port, occurence='0')
rc = self.gprecover.incremental()
if not rc:
raise Exception('Gprecvoerseg failed')
tinctest.logger.info('Cluster in resync state')
PSQL.run_sql_command('CHECKPOINT;', dbname='postgres')
if checkpoint == 'skip':
self.invoke_fault('checkpoint', 'skip', role='primary', port= self.port, occurence='0')
self.inject_fault(fault_type)
if cluster_state == 'resync':
self.filereputil.inject_fault(f='filerep_resync', y='resume', r='primary')
PSQL.wait_for_database_up();
def test_uao_crash_vacuum_with_ins_fault(self):
setup_file = self.get_sql_files("uaocs_crash_update_setup")[0]
(sql_file1, out_file1,ans_file1) = self.get_sql_files("uao_crash_vacuum1")
(sql_file2, out_file2, ans_file2) = self.get_sql_files("uao_crash_vacuum2")
if not os.path.exists(os.path.dirname(out_file1)):
os.mkdir(os.path.dirname(out_file1))
set_fault_in_seg_panic = 'source %s/greenplum_path.sh;gpfaultinjector -p %s -f appendonly_insert -t foo -y panic --seg_dbid 2' % (os.getenv('GPHOME'), os.getenv('PGPORT'))
set_fault_in_seg_reset = 'source %s/greenplum_path.sh;gpfaultinjector -p %s -f appendonly_insert -t foo -y reset --seg_dbid 2' % (os.getenv('GPHOME'), os.getenv('PGPORT'))
cmd_type = 'fault injector'
PSQL.run_sql_file(setup_file)
gpfaultinjector = Command(cmd_type, set_fault_in_seg_panic)
gpfaultinjector.run()
PSQL.run_sql_file(sql_file1, out_file=out_file1)
result1 = Gpdiff.are_files_equal(out_file1, ans_file1, match_sub=[gpdiff_init_file])
PSQL.wait_for_database_up();
gpfaultinjector = Command(cmd_type, set_fault_in_seg_reset)
gpfaultinjector.run()
PSQL.run_sql_file(sql_file2, out_file=out_file2)
result2 = Gpdiff.are_files_equal(out_file2, ans_file2, match_sub=[gpdiff_init_file])
self.assertTrue(result1)
self.assertTrue(result2)
def test_uao_crash_vacuum_with_ins_fault(self):
setup_file = self.get_sql_files("uaocs_crash_update_setup")[0]
(sql_file1, out_file1,ans_file1) = self.get_sql_files("uao_crash_vacuum1")
(sql_file2, out_file2, ans_file2) = self.get_sql_files("uao_crash_vacuum2")
if not os.path.exists(os.path.dirname(out_file1)):
os.mkdir(os.path.dirname(out_file1))
set_fault_in_seg_panic = 'source %s/greenplum_path.sh;gpfaultinjector -p %s -f appendonly_insert -t foo -y panic --seg_dbid 2' % (os.getenv('GPHOME'), os.getenv('PGPORT'))
set_fault_in_seg_reset = 'source %s/greenplum_path.sh;gpfaultinjector -p %s -f appendonly_insert -t foo -y reset --seg_dbid 2' % (os.getenv('GPHOME'), os.getenv('PGPORT'))
cmd_type = 'fault injector'
PSQL.run_sql_file(setup_file)
gpfaultinjector = Command(cmd_type, set_fault_in_seg_panic)
gpfaultinjector.run()
PSQL.run_sql_file(sql_file1, out_file=out_file1)
result1 = Gpdiff.are_files_equal(out_file1, ans_file1, match_sub=[gpdiff_init_file])
PSQL.wait_for_database_up();
gpfaultinjector = Command(cmd_type, set_fault_in_seg_reset)
gpfaultinjector.run()
PSQL.run_sql_file(sql_file2, out_file=out_file2)
result2 = Gpdiff.are_files_equal(out_file2, ans_file2, match_sub=[gpdiff_init_file])
self.assertTrue(result1)
self.assertTrue(result2)
def test_uao_crash_compaction_before_cleanup_phase_master_with_aocs(self):
setup_file = self.get_sql_files("uao_crash_compaction_before_cleanup_phase_master_with_aocs_setup")[0]
(sql_file1, out_file1,ans_file1) = self.get_sql_files("uao_crash_compaction_before_cleanup_phase_master_with_aocs1")
(sql_file2, out_file2, ans_file2) = self.get_sql_files("uao_crash_compaction_before_cleanup_phase_master_with_aocs2")
if not os.path.exists(os.path.dirname(out_file1)):
os.mkdir(os.path.dirname(out_file1))
set_fault_in_master_panic = 'source %s/greenplum_path.sh;gpfaultinjector -p %s -f compaction_before_cleanup_phase -y panic --seg_dbid 1' % (os.getenv('GPHOME'), os.getenv('PGPORT'))
set_fault_in_master_reset = 'source %s/greenplum_path.sh;gpfaultinjector -p %s -f compaction_before_cleanup_phase -y reset --seg_dbid 1' % (os.getenv('GPHOME'), os.getenv('PGPORT'))
cmd_type = 'fault injector'
PSQL.run_sql_file(setup_file)
gpfaultinjector = Command(cmd_type, set_fault_in_master_panic)
gpfaultinjector.run()
PSQL.run_sql_file(sql_file1, out_file=out_file1)
result1 = Gpdiff.are_files_equal(out_file1, ans_file1, match_sub=[gpdiff_init_file])
PSQL.wait_for_database_up();
gpfaultinjector = Command(cmd_type, set_fault_in_master_reset)
gpfaultinjector.run()
PSQL.run_sql_file(sql_file2, out_file=out_file2)
result2 = Gpdiff.are_files_equal(out_file2, ans_file2, match_sub=[gpdiff_init_file])
self.assertTrue(result1)
self.assertTrue(result2)
def run_faults_before_trigger(self, checkpoint, cluster_state, fault_type):
'''
@param checkpoint : skip/noskip
@param cluster_state : sync/change_tracking/resync
@param fault_type : commit/abort
@param end_prepare_two_phase_sleep : True/False
@description : 1. Suspend resync faults. 2. Issue Checkpoint before the skip checkpoint, so that the bufferpool is cleared. 3. If skip issue 'skip checkpoint'. 4. Suspend transaction_faults based on test_type.
'''
if cluster_state == 'change_tracking':
self.invoke_fault('filerep_consumer', 'fault')
self.filereputil.wait_till_change_tracking_transition()
tinctest.logger.info('Change_tracking transition complete')
if cluster_state == 'resync':
self.invoke_fault('filerep_resync', 'suspend', role='primary')
if checkpoint == 'skip':
self.invoke_fault('filerep_transition_to_sync_before_checkpoint', 'suspend', role='primary', port=self.port, occurence='0')
rc = self.gprecover.incremental()
if not rc:
raise Exception('Gprecvoerseg failed')
tinctest.logger.info('Cluster in resync state')
PSQL.run_sql_command('CHECKPOINT;', dbname='postgres')
if checkpoint == 'skip':
self.invoke_fault('checkpoint', 'skip', role='primary', port= self.port, occurence='0')
self.inject_fault(fault_type)
# Can't do it after filerep_resync resume as gets stuck due to
# filerep_transition_to_sync_before_checkpoint suspend above for
# MirroedLock
PSQL.wait_for_database_up()
if cluster_state == 'resync':
self.filereputil.inject_fault(f='filerep_resync', y='resume', r='primary')
def test_uao_crash_compaction_vacuum(self):
setup_file = self.get_sql_files("uao_crash_compaction_vacuum_setup")[0]
(sql_file1, out_file1, ans_file1) = self.get_sql_files("uao_crash_compaction_vacuum1")
(sql_file2, out_file2, ans_file2) = self.get_sql_files("uao_crash_compaction_vacuum2")
if not os.path.exists(os.path.dirname(out_file1)):
os.mkdir(os.path.dirname(out_file1))
set_fault_in_seg_panic = 'source %s/greenplum_path.sh;gpfaultinjector -p %s -f compaction_before_segmentfile_drop -y panic --seg_dbid 2' % (os.getenv('GPHOME'), os.getenv('PGPORT'))
set_fault_in_seg_reset = 'source %s/greenplum_path.sh;gpfaultinjector -p %s -f compaction_before_segmentfile_drop -y reset --seg_dbid 2' % (os.getenv('GPHOME'), os.getenv('PGPORT'))
cmd_type = 'fault injector'
PSQL.run_sql_file(setup_file)
gpfaultinjector = Command(cmd_type, set_fault_in_seg_panic)
gpfaultinjector.run()
PSQL.run_sql_file(sql_file1, out_file=out_file1)
result1 = Gpdiff.are_files_equal(out_file1, ans_file1)
PSQL.wait_for_database_up();
gpfaultinjector = Command(cmd_type, set_fault_in_seg_reset)
gpfaultinjector.run()
PSQL.run_sql_file(sql_file2, out_file=out_file2)
result2 = Gpdiff.are_files_equal(out_file2, ans_file2)
self.assertTrue(result2)
def crash_and_recover(self,
crash_type,
fault_type,
checkpoint='noskip',
cluster_state='sync'):
'''
@param crash_type : gpstop_i/gpstop_a/failover_to_primary/failover_to_mirror
@note: when skip checkpoint is enabled, gpstop -a returns a non-rc return code and fails in the library. To workaround, using a local function
'''
if crash_type == 'gpstop_i':
rc = self.gpstop.run_gpstop_cmd(immediate=True)
if not rc:
raise Exception('Failed to stop the cluster')
tinctest.logger.info('Stopped cluster immediately')
self.start_db()
elif crash_type == 'gpstop_a':
self.resume_faults(fault_type, cluster_state)
if checkpoint == 'skip':
self.stop_db()
else:
rc = self.gpstop.run_gpstop_cmd()
if not rc:
raise Exception('Failed to stop the cluster')
tinctest.logger.info('Smart stop completed')
self.start_db()
elif crash_type == 'failover_to_primary':
self.invoke_fault('filerep_consumer', 'fault')
self.resume_faults(fault_type, cluster_state)
(rc, num) = self.filereputil.wait_till_change_tracking_transition()
tinctest.logger.info('Value of rc and num_down %s, %s, %s' %
(rc, num, fault_type))
elif crash_type == 'failover_to_mirror':
self.invoke_fault('postmaster', 'panic', role='primary')
if fault_type in ('dtm_broadcast_prepare',
'dtm_broadcast_commit_prepared',
'dtm_xlog_distributed_commit'):
self.resume_faults(fault_type, cluster_state)
PSQL.wait_for_database_up()
(rc, num) = self.filereputil.wait_till_change_tracking_transition()
tinctest.logger.info('Value of rc and num_down %s, %s' % (rc, num))
if fault_type == 'abort':
self.filereputil.inject_fault(
f='transaction_abort_after_distributed_prepared',
y='reset',
p=self.port,
o='0',
seg_id='1')
if cluster_state == 'resync':
if not self.gprecover.wait_till_insync_transition():
raise Exception('Segments not in sync')
def run_post_sqls(self, fault_name ='', trans_state=''):
PSQL.wait_for_database_up();
if (trans_state == 'failover_to_primary' or trans_state == ''):
post_sql = "failover_sql/subt_create_table_ao_post_commit"
else:
post_sql = "failover_sql/subt_create_table_ao_post_abort"
sql_file = post_sql+".sql"
ans_file = post_sql+".ans"
out_file = post_sql+".out"
PSQL.run_sql_file(sql_file = local_path(sql_file), out_file = local_path(out_file))
diff_res = Gpdiff.are_files_equal(local_path(out_file), local_path(ans_file))
if not diff_res:
self.fail("[STLRTest]Gpdiff failed for : %s %s" %(fault_name, trans_state))
def run_sequence(self, sql, fault, fault_type, segid, should_panic=True):
(ok,out) = self.util.inject_fault(f=fault, y=fault_type, seg_id=segid);
if not ok:
raise Exception("Fault dtm_broadcast_commit_prepared injection failed")
psql = PSQL(sql_cmd=sql);
tinctest.logger.debug("Executing:" + sql)
psql.run()
results = psql.get_results()
tinctest.logger.debug(results.stderr)
self.check_no_dangling_prepared_transaction()
if "PANIC" not in results.stderr and should_panic:
raise Exception("Fault %s type %s (on segid: %d) did not cause the master reset" % (fault, fault_type, segid))
if "PANIC" in results.stderr and not should_panic:
raise Exception("Fault %s type %s (on segid: %d) caused a PANIC. dtx two phase retry failed" % (fault, fault_type, segid))
PSQL.wait_for_database_up()
def run_post_sqls(self, fault_name='', trans_state=''):
PSQL.wait_for_database_up()
if (trans_state == 'failover_to_primary' or trans_state == ''):
post_sql = "failover_sql/subt_create_table_ao_post_commit"
else:
post_sql = "failover_sql/subt_create_table_ao_post_abort"
sql_file = post_sql + ".sql"
ans_file = post_sql + ".ans"
out_file = post_sql + ".out"
PSQL.run_sql_file(sql_file=local_path(sql_file),
out_file=local_path(out_file))
diff_res = Gpdiff.are_files_equal(local_path(out_file),
local_path(ans_file))
if not diff_res:
self.fail("[STLRTest]Gpdiff failed for : %s %s" %
(fault_name, trans_state))
def crash_and_recover(self, crash_type, fault_type, checkpoint='noskip', cluster_state='sync'):
'''
@param crash_type : gpstop_i/gpstop_a/failover_to_primary/failover_to_mirror
@note: when skip checkpoint is enabled, gpstop -a returns a non-rc return code and fails in the library. To workaround, using a local function
'''
if crash_type == 'gpstop_i' :
rc = self.gpstop.run_gpstop_cmd(immediate = True)
if not rc:
raise Exception('Failed to stop the cluster')
tinctest.logger.info('Stopped cluster immediately')
self.start_db()
elif crash_type == 'gpstop_a':
self.resume_faults(fault_type, cluster_state)
if checkpoint == 'skip' :
self.stop_db()
else:
rc = self.gpstop.run_gpstop_cmd()
if not rc:
raise Exception('Failed to stop the cluster')
tinctest.logger.info('Smart stop completed')
self.start_db()
elif crash_type == 'failover_to_primary':
self.invoke_fault('filerep_consumer', 'fault')
self.resume_faults(fault_type, cluster_state)
(rc, num) =self.filereputil.wait_till_change_tracking_transition()
tinctest.logger.info('Value of rc and num_down %s, %s, %s' % (rc, num, fault_type))
elif crash_type == 'failover_to_mirror':
self.invoke_fault('postmaster', 'panic', role='primary')
if fault_type in ('dtm_broadcast_prepare', 'dtm_broadcast_commit_prepared', 'dtm_xlog_distributed_commit') :
self.resume_faults(fault_type, cluster_state)
PSQL.wait_for_database_up()
(rc, num) = self.filereputil.wait_till_change_tracking_transition()
tinctest.logger.info('Value of rc and num_down %s, %s' % (rc, num))
if fault_type == 'abort' :
self.filereputil.inject_fault(f='transaction_abort_after_distributed_prepared', y='reset',p=self.port , o='0', seg_id='1')
if cluster_state == 'resync':
if not self.gprecover.wait_till_insync_transition():
raise Exception('Segments not in sync')
def test_uaocs_crash_update_in_tran(self):
setup_file = self.get_sql_files("uaocs_crash_update_setup")[0]
(sql_file1, out_file1,ans_file1) = self.get_sql_files("uaocs_crash_update_intran1")
(sql_file2, out_file2, ans_file2) = self.get_sql_files("uaocs_crash_update_intran2")
if not os.path.exists(os.path.dirname(out_file1)):
os.mkdir(os.path.dirname(out_file1))
# We set the fault in appendonly_update and appendonly_insert
# because planner will go through appendonly_update and ORCA
# will do appendonly_delete and appendonly_insert
set_fault_in_seg_panic_update = 'source %s/greenplum_path.sh;gpfaultinjector -p %s -f appendonly_update -t foo -y panic --seg_dbid 2' % (os.getenv('GPHOME'), os.getenv('PGPORT'))
set_fault_in_seg_reset_update = 'source %s/greenplum_path.sh;gpfaultinjector -p %s -f appendonly_update -t foo -y reset --seg_dbid 2' % (os.getenv('GPHOME'), os.getenv('PGPORT'))
set_fault_in_seg_panic_insert = 'source %s/greenplum_path.sh;gpfaultinjector -p %s -f appendonly_insert -t foo -y panic --seg_dbid 2' % (os.getenv('GPHOME'), os.getenv('PGPORT'))
set_fault_in_seg_reset_insert = 'source %s/greenplum_path.sh;gpfaultinjector -p %s -f appendonly_insert -t foo -y reset --seg_dbid 2' % (os.getenv('GPHOME'), os.getenv('PGPORT'))
cmd_type = 'fault injector'
PSQL.run_sql_file(setup_file)
gpfaultinjector = Command(cmd_type, set_fault_in_seg_panic_update)
gpfaultinjector.run()
gpfaultinjector = Command(cmd_type, set_fault_in_seg_panic_insert)
gpfaultinjector.run()
PSQL.run_sql_file(sql_file1, out_file=out_file1)
result1 = Gpdiff.are_files_equal(out_file1, ans_file1, match_sub=[gpdiff_init_file])
PSQL.wait_for_database_up();
gpfaultinjector = Command(cmd_type, set_fault_in_seg_reset_update)
gpfaultinjector.run()
gpfaultinjector = Command(cmd_type, set_fault_in_seg_reset_insert)
gpfaultinjector.run()
PSQL.run_sql_file(sql_file2, out_file=out_file2)
result2 = Gpdiff.are_files_equal(out_file2, ans_file2, match_sub=[gpdiff_init_file])
self.assertTrue(result1)
self.assertTrue(result2)
def test_uaocs_crash_update_in_tran(self):
setup_file = self.get_sql_files("uaocs_crash_update_setup")[0]
(sql_file1, out_file1,ans_file1) = self.get_sql_files("uaocs_crash_update_intran1")
(sql_file2, out_file2, ans_file2) = self.get_sql_files("uaocs_crash_update_intran2")
if not os.path.exists(os.path.dirname(out_file1)):
os.mkdir(os.path.dirname(out_file1))
# We set the fault in appendonly_update and appendonly_insert
# because planner will go through appendonly_update and ORCA
# will do appendonly_delete and appendonly_insert
set_fault_in_seg_panic_update = 'source %s/greenplum_path.sh;gpfaultinjector -p %s -f appendonly_update -t foo -y panic --seg_dbid 2' % (os.getenv('GPHOME'), os.getenv('PGPORT'))
set_fault_in_seg_reset_update = 'source %s/greenplum_path.sh;gpfaultinjector -p %s -f appendonly_update -t foo -y reset --seg_dbid 2' % (os.getenv('GPHOME'), os.getenv('PGPORT'))
set_fault_in_seg_panic_insert = 'source %s/greenplum_path.sh;gpfaultinjector -p %s -f appendonly_insert -t foo -y panic --seg_dbid 2' % (os.getenv('GPHOME'), os.getenv('PGPORT'))
set_fault_in_seg_reset_insert = 'source %s/greenplum_path.sh;gpfaultinjector -p %s -f appendonly_insert -t foo -y reset --seg_dbid 2' % (os.getenv('GPHOME'), os.getenv('PGPORT'))
cmd_type = 'fault injector'
PSQL.run_sql_file(setup_file)
gpfaultinjector = Command(cmd_type, set_fault_in_seg_panic_update)
gpfaultinjector.run()
gpfaultinjector = Command(cmd_type, set_fault_in_seg_panic_insert)
gpfaultinjector.run()
PSQL.run_sql_file(sql_file1, out_file=out_file1)
result1 = Gpdiff.are_files_equal(out_file1, ans_file1, match_sub=[gpdiff_init_file])
PSQL.wait_for_database_up();
gpfaultinjector = Command(cmd_type, set_fault_in_seg_reset_update)
gpfaultinjector.run()
gpfaultinjector = Command(cmd_type, set_fault_in_seg_reset_insert)
gpfaultinjector.run()
PSQL.run_sql_file(sql_file2, out_file=out_file2)
result2 = Gpdiff.are_files_equal(out_file2, ans_file2, match_sub=[gpdiff_init_file])
self.assertTrue(result1)
self.assertTrue(result2)
def run_trigger_sql(self, wait_for_db=True):
''' Run a sql statement to trigger postmaster reset '''
PSQL.run_sql_file(local_path('test_ddl.sql'))
if wait_for_db:
PSQL.wait_for_database_up()
def check_connection(self):
'''
Wait till the system is up, as master/segments may take some time
to come back after FI crash.
'''
PSQL.wait_for_database_up()
def test_tli_mismatch(self):
"""
Test to verify if TLI mismatch issue during Pass 3 of xlog record
(checkpoint) replay occurs or not. A set of checkpoints one after
the other when replayed on the standby and then if the
standby is promoted, it should go through the prmotion just fine.
The flow of this test is as follows.
1. Initiate the Standby using the Master (primary) postmaster
paramerters.
2. Perform explicit checkpoints and wait so that they get replicated.
3. Then promote the standby, wait and then try to access it.
4. If we can successfully access it, its a Pass otherwise a Fail.
"""
PSQL.run_sql_command('DROP table if exists foo')
PSQL.run_sql_command('create table foo (a int)')
PSQL.run_sql_command('insert into foo '
'select * from generate_series(1, 1000)')
# Initial setup, forcibly remove standby and install new one
pgutil = GpUtility()
pgutil.remove_standby()
logger.info('\nCreate a standby...')
res = self.standby.create()
self.assertEqual(res, 0)
res = self.standby.start()
self.assertTrue(res.wasSuccessful())
# Wait for the walreceiver to start
num_walsender = self.wait_for_walsender()
self.assertEqual(num_walsender, 1)
logger.info('Standby activated...')
logger.info('Give the standby some time to catchup...')
time.sleep(3)
logger.info('Create checkpoints and let them get replicated...')
PSQL.run_sql_command('checkpoint')
PSQL.run_sql_command('checkpoint')
PSQL.run_sql_command('checkpoint')
PSQL.run_sql_command('checkpoint')
PSQL.run_sql_command('checkpoint')
PSQL.run_sql_command('checkpoint')
PSQL.run_sql_command('checkpoint')
PSQL.run_sql_command('checkpoint')
PSQL.run_sql_command('checkpoint')
PSQL.run_sql_command('checkpoint')
PSQL.run_sql_command('checkpoint')
PSQL.run_sql_command('checkpoint')
PSQL.run_sql_command('checkpoint')
PSQL.run_sql_command('checkpoint')
PSQL.run_sql_command('checkpoint')
PSQL.run_sql_command('checkpoint')
PSQL.run_sql_command('checkpoint')
PSQL.run_sql_command('checkpoint')
PSQL.run_sql_command('checkpoint')
PSQL.run_sql_command('checkpoint')
PSQL.run_sql_command('checkpoint')
PSQL.run_sql_command('checkpoint')
time.sleep(2)
# Promote standby
# We don't kill/stop the primary, as it's convenient for
# next testing
logger.info('Promote the standby immediatly')
self.standby.promote()
logger.info(
'Wait for the standby to be ready to accept connections ...')
PSQL.wait_for_database_up(port=self.standby.port)
# Verify the result replicated to the standby.
logger.info('Verify if table foo exists...')
proc = self.run_sql('select count(*) from foo', str(self.standby.port))
# The table should exist
(stdout, stderr) = proc.communicate()
logger.info("stdout: %s" % stdout)
logger.info("stderr: %s" % stderr)
search = "1000"
self.assertTrue(stdout.find(search) >= 0)
logger.info('Pass')
def check_connection(self):
'''
Wait till the system is up, as master/segments may take some time
to come back after FI crash.
'''
PSQL.wait_for_database_up()
def test_tli_mismatch(self):
"""
Test to verify if TLI mismatch issue during Pass 3 of xlog record
(checkpoint) replay occurs or not. A set of checkpoints one after
the other when replayed on the standby and then if the
standby is promoted, it should go through the prmotion just fine.
The flow of this test is as follows.
1. Initiate the Standby using the Master (primary) postmaster
paramerters.
2. Perform explicit checkpoints and wait so that they get replicated.
3. Then promote the standby, wait and then try to access it.
4. If we can successfully access it, its a Pass otherwise a Fail.
"""
PSQL.run_sql_command('DROP table if exists foo')
PSQL.run_sql_command('create table foo (a int)')
PSQL.run_sql_command('insert into foo '
'select * from generate_series(1, 1000)')
# Initial setup, forcibly remove standby and install new one
pgutil = GpUtility()
pgutil.remove_standby()
logger.info ('\nCreate a standby...')
res = self.standby.create()
self.assertEqual(res, 0)
res = self.standby.start()
self.assertTrue(res.wasSuccessful())
# Wait for the walreceiver to start
num_walsender = self.wait_for_walsender()
self.assertEqual(num_walsender, 1)
logger.info('Standby activated...')
logger.info('Give the standby some time to catchup...')
time.sleep(3)
logger.info('Create checkpoints and let them get replicated...')
PSQL.run_sql_command('checkpoint')
PSQL.run_sql_command('checkpoint')
PSQL.run_sql_command('checkpoint')
PSQL.run_sql_command('checkpoint')
PSQL.run_sql_command('checkpoint')
PSQL.run_sql_command('checkpoint')
PSQL.run_sql_command('checkpoint')
PSQL.run_sql_command('checkpoint')
PSQL.run_sql_command('checkpoint')
PSQL.run_sql_command('checkpoint')
PSQL.run_sql_command('checkpoint')
PSQL.run_sql_command('checkpoint')
PSQL.run_sql_command('checkpoint')
PSQL.run_sql_command('checkpoint')
PSQL.run_sql_command('checkpoint')
PSQL.run_sql_command('checkpoint')
PSQL.run_sql_command('checkpoint')
PSQL.run_sql_command('checkpoint')
PSQL.run_sql_command('checkpoint')
PSQL.run_sql_command('checkpoint')
PSQL.run_sql_command('checkpoint')
PSQL.run_sql_command('checkpoint')
time.sleep(2)
# Promote standby
# We don't kill/stop the primary, as it's convenient for
# next testing
logger.info('Promote the standby immediatly')
self.standby.promote()
logger.info('Wait for the standby to be ready to accept connections ...')
PSQL.wait_for_database_up(port=self.standby.port)
# Verify the result replicated to the standby.
logger.info('Verify if table foo exists...')
proc = self.run_sql('select count(*) from foo',
str(self.standby.port))
# The table should exist
(stdout, stderr) = proc.communicate()
logger.info("stdout: %s" % stdout)
logger.info("stderr: %s" % stderr)
search = "1000"
self.assertTrue(stdout.find(search) >= 0)
logger.info('Pass')
