def test_option_d(self):
"""
check the -d option of gpaddmirrors
"""
gprecover = GpRecover()
self._setup_gpaddmirrors()
self._cleanup_segment_data_dir(self.host_file, self.mirror_data_dir)
del os.environ['MASTER_DATA_DIRECTORY']
Command('run gpaddmirrors -i -d', 'gpaddmirrors -a -i %s -d %s' % (self.mirror_config_file, self.mdd)).run(validateAfter=True)
os.environ['MASTER_DATA_DIRECTORY']=self.mdd
gprecover.wait_till_insync_transition()
self.verify_config_file_with_gp_config()
self.check_mirror_seg()
def test_wrong_dataparam(self):
"""
Test if conflicting -d option is reported.
@tags sanity
"""
cmd = Command('gpactivatestandby',
'gpactivatestandby -a -d invalid_directory')
cmd.run()
results = cmd.get_results()
expected = 'Current setting of MASTER_DATA_DIRECTORY not same as -d parameter'
self.assertIn(expected, results.stdout)
self.assertEqual(results.rc, 2)
def check_db_is_running(self):
"""
Check whether DB is running
TODO: Need to put in gpdbSystem
"""
cmd = Command(name='pg_ctl call',cmdStr="%s/bin/pg_ctl -D %s status" % (self.gphome, self.mdd))
cmd.run(validateAfter=False)
result = cmd.get_results()
if result.rc == 0:
if result.stdout.splitlines()[0].find("server is running")>0:
tinctest.logger.info('Server is Running')
return True
return False
def _fill(self, filename, host):
cmd_prefix = "ssh " + host + " \""
cmd_postfix = "\""
location = '/data'
if not os.path.isdir('%s/diskfill/' % location):
os.makedirs('%s/diskfill/' % location)
cmd_str = cmd_prefix + "dd if=/dev/zero bs=16384K count=2000 of=" + location + "/diskfill/" + filename + cmd_postfix
cmd = Command(name='Fill Disk', cmdStr=cmd_str)
tinctest.logger.info(" %s" % cmd)
cmd.run(validateAfter=False)
result = cmd.get_results()
if result.rc != 0:
tinctest.logger.error('disk fill not working. Its already full')
def get_walsender_pid(self):
''' get wal sender pid '''
pid_cmd = "ps -ef|grep \'%s\' |grep -v grep" % ('wal sender')
cmd = Command('Get the pid of the wal sender process', cmdStr=pid_cmd)
tinctest.logger.info('%s' % cmd)
cmd.run(validateAfter=False)
result = cmd.get_results()
while (result.rc != 0):
cmd.run(validateAfter=False)
result = cmd.get_results()
tinctest.logger.info(result)
pid = result.stdout.splitlines()[0].split()[1].strip()
return pid
def test_uao_crash_compaction_before_cleanup_phase_master_with_ao(self):
setup_file = self.get_sql_files(
"uao_crash_compaction_before_cleanup_phase_master_with_aocs_setup"
)[0]
(sql_file1, out_file1, ans_file1) = self.get_sql_files(
"uao_crash_compaction_before_cleanup_phase_master_with_ao1")
(sql_file2, out_file2, ans_file2) = self.get_sql_files(
"uao_crash_compaction_before_cleanup_phase_master_with_ao2")
if not os.path.exists(os.path.dirname(out_file1)):
os.mkdir(os.path.dirname(out_file1))
set_fault_in_master_panic = 'source %s/greenplum_path.sh;gpfaultinjector -p %s -f compaction_before_cleanup_phase -y panic --seg_dbid 1' % (
os.getenv('GPHOME'), os.getenv('PGPORT'))
set_fault_in_master_reset = 'source %s/greenplum_path.sh;gpfaultinjector -p %s -f compaction_before_cleanup_phase -y reset --seg_dbid 1' % (
os.getenv('GPHOME'), os.getenv('PGPORT'))
cmd_type = 'fault injector'
PSQL.run_sql_file(setup_file)
gpfaultinjector = Command(cmd_type, set_fault_in_master_panic)
gpfaultinjector.run()
PSQL.run_sql_file(sql_file1, out_file=out_file1)
result1 = Gpdiff.are_files_equal(out_file1,
ans_file1,
match_sub=[gpdiff_init_file])
PSQL.wait_for_database_up()
gpfaultinjector = Command(cmd_type, set_fault_in_master_reset)
gpfaultinjector.run()
PSQL.run_sql_file(sql_file2, out_file=out_file2)
result2 = Gpdiff.are_files_equal(out_file2,
ans_file2,
match_sub=[gpdiff_init_file])
self.assertTrue(result1)
self.assertTrue(result2)
def test_valid_cmd_non_default_options_passes(self, mock_workerpool):
mock_workerpool.return_value = Mock()
cmd1 = Command('testcmd', 'testcmdstr')
cmd1.set_results(CommandResult(0, b'', b'', True, False))
mock_workerpool.return_value.getCompletedItems = Mock(return_value=[cmd1, cmd1])
sys.argv = ['recovery_base', '-l', '/tmp/logdir',
'-c {}'.format(self.confinfo),
'-b 10', '-f', '-v', '--era', '1234_2021']
stderr_buf, ex = self.run_recovery_base_get_stderr()
self._asserts_for_passing_tests(stderr_buf, ex, enable_verbose_count=1)
self.assertEqual([call('test_file.py', ANY, ANY, logdir='/tmp/logdir')],
self.mock_setup_tool_logging.call_args_list)
self._assert_workerpool_calls(mock_workerpool)
def test00_pg_hba_conf_file(self):
os.environ[
self.GP_COMMAND_FAULT_POINT] = 'gpexpand tar segment template'
cmd = Command(name='run gpexpand',
cmdStr='gpexpand -D %s -i %s' %
(self.TEST_DB, self.EXPANSION_INPUT_FILE))
with self.assertRaisesRegexp(ExecutionError, 'Fault Injection'):
cmd.run(validateAfter=True)
#Read from the pg_hba.conf file and ensure that
#The address of the new hosts is present.
cmd = Command(name='get the temp pg_hba.conf file',
cmdStr="ls %s" %
os.path.join(os.path.dirname(self.MASTER_DATA_DIRECTORY),
'gpexpand*', 'pg_hba.conf'))
cmd.run(validateAfter=True)
results = cmd.get_results()
temp_pg_hba_conf = results.stdout.strip()
actual_values = set()
expected_values = set(
[self.primary_host_address, self.mirror_host_address])
with open(temp_pg_hba_conf) as f:
for line in f:
if line.strip() == '# %s' % self.primary_host_name or\
line.strip() == '# %s' % self.mirror_host_name:
address = f.next().strip().split()[3]
address = address[:address.rfind('/')]
actual_values.add(address)
self.assertEqual(actual_values, expected_values)
GpStart(name='start the database in master only mode',
masterOnly=True).run(validateAfter=True)
Command(name='rollback the expansion',
cmdStr='gpexpand -r -D %s' %
self.TEST_DB).run(validateAfter=True)
GpStart(name='start the database').run(validateAfter=True)
def _reset_VLIM_SLIM_REDZONEPERCENT():
# Reset GUCs for VLIM (gp_vmem_protect_limit), SLIM (gp_vmem_limit_per_query) and RQT activation percent (runaway_detector_activation_percent)
tinctest.logger.info(
'Resetting GUCs for VLIM gp_vmem_protect_limit, SLIM gp_vmem_limit_per_query, and RQT activation percent runaway_detector_activation_percent'
)
Command(
'Run gpconfig to reset GUC gp_vmem_protect_limit',
'source $GPHOME/greenplum_path.sh;gpconfig -c gp_vmem_protect_limit -v 8192'
).run(validateAfter=True)
Command(
'Run gpconfig to reset GUC gp_vmem_limit_per_query',
'source $GPHOME/greenplum_path.sh;gpconfig -r gp_vmem_limit_per_query --skipvalidation'
).run(validateAfter=True)
Command(
'Run gpconfig to reset GUC runaway_detector_activation_percent',
'source $GPHOME/greenplum_path.sh;gpconfig -r runaway_detector_activation_percent --skipvalidation'
).run(validateAfter=True)
# Restart DB
Command('Restart database for GUCs to take effect',
'source $GPHOME/greenplum_path.sh && gpstop -ar').run(
validateAfter=True)
def impl(context):
cmd = """ps ux | grep "/bin/postgres \-D %s " | grep -v grep""" % (
context.remote_mirror_datadir)
cmd = Command(name='user command',
cmdStr=cmd,
ctxt=REMOTE,
remoteHost=context.remote_mirror_segdbname)
cmd.run(validateAfter=True)
res = cmd.get_results()
if not res.stdout.strip():
raise Exception(
'Mirror segment "%s" not active on "%s"' %
(context.remote_mirror_datadir, context.remote_mirror_segdbname))
def test_create_local(self):
"""
Initialize a temporary GP cluster with a standby locally running.
"""
init_file = local_path('gp_init_config')
port = self.standby_port
filespaces = 'pg_system:{0}/gpseg-standby'.format(self.standby_dir)
cmd_str = 'gpinitsystem -ac {0} -P {1} -F {2}'.format(
init_file, port, filespaces)
cmd = Command('gpinitsystem', cmd_str)
cmd.run()
self.assertIn(cmd.get_results().rc, (0, 1))
def get_content(self, filename, hostname="localhost"):
"""
Get the content of a text file
@param filename: Absolute location of the file
@param hostname: hostname
"""
if hostname in (socket.gethostname(), 'localhost'):
cmdLine = "cat %s" % filename
cmd = Command(name=' Search content of a file', cmdStr=cmdLine)
else:
cmdLine = '"cat %s"' % filename
cmd = Command(name=' Search content of a file',
cmdStr=cmdLine,
ctxt=REMOTE,
remoteHost=hostname)
cmd.run(validateAfter=False)
result = cmd.get_results()
(ok, out) = (result.rc, result.stdout)
if not ok:
return out
else:
raise FileUtilError("Error getting content for %s" % filename)
def gpstartCheck(self):
"""
Checks if the cluster is brought up correctly and all segments are in sync
"""
bashCmd = 'source ' + (self.gphome) + '/greenplum_path.sh;' + (
self.gphome
) + '/bin/pg_ctl status -D $MASTER_DATA_DIRECTORY | grep \'pg_ctl: server is running\''
dbStart = Command(name='gpstartCheck ', cmdStr=bashCmd)
dbStart.run()
rc = dbStart.get_results().rc
if rc != 0:
return False
return True
def initstand_by_with_default(self):
master_host = self.get_masterhost()
gp_cmd = "/bin/bash -c 'gpinitstandby -s %s'" % (master_host)
cmd = Command(name='Running the command', cmdStr=gp_cmd)
tinctest.logger.info('%s' % cmd)
cmd.run(validateAfter=False)
sleep(2)
result = cmd.get_results()
lines = result.stdout.splitlines()
for line in lines:
if 'Data directory already exists' in line:
return True
return False
def create_dir_on_standby(self, standby, location):
fs_cmd = "gpssh -h %s -e 'rm -rf %s; mkdir -p %s' " % (
standby, location, location)
cmd = Command(
name='Make dierctory on standby before running the command',
cmdStr=fs_cmd)
tinctest.logger.info('%s' % cmd)
cmd.run(validateAfter=True)
result = cmd.get_results()
if result.rc != 0:
raise WalReplException('Unable to create directory on standby')
else:
return True
def test_shared_mem_is_cleaned(self):
"""
[feature]: Check if the shared memory is cleaned
"""
newfault = Fault()
seginfo = newfault.get_seginfo(preferred_role='p',content=0)
cmd = Command('check for shared memory', cmdStr="ipcs -a", ctxt=REMOTE, remoteHost=seginfo.getSegmentHostName())
cmd.run(validateAfter=True)
result = cmd.get_results().stdout.split('\n')
for r in result:
if r and r.split()[-1] == '0':
raise Exception('Shared memory not cleaned up for %s' % r)
def test_backup_restore(self):
# Create standby if not present
Command('createdb bkdb', 'dropdb bkdb; createdb bkdb').run()
self.create_standby()
# Run workload
self.run_workload('sql')
# Create backup
self.assertTrue(self.run_backup())
# Activate standby
gpac = GpactivateStandby()
gpac.activate()
# Restore from new master
self.assertTrue(self.run_restore())
def get_timestamp_of_file_in_datadir(self, content, role, filename):
dbid = self.config.get_dbid(content=content, seg_role=role)
host, datadir = self.config.get_host_and_datadir_of_segment(dbid=dbid)
file_path = os.path.join(datadir, filename)
cmd = Command(
'check timestamp',
""" python -c "import os; print os.stat('%s').st_mtime" """ %
file_path,
ctxt=REMOTE,
remoteHost=host)
cmd.run(validateAfter=True)
res = cmd.get_results().stdout.strip()
return res
def test_gpstart_logDir(self):
tinctest.logger.info("Running test_gpstart_logDir")
self.logdir=''.join([self.basedir,'/logs'])
cmd = Command(name='Remove gpstop<nnnn>.log', cmdStr='rm -f %s/gpstop*' % (self.logdir))
tinctest.logger.info("Removing gpstop<nnnn>.log : %s" % cmd)
cmd.run(validateAfter=True)
result = cmd.get_results()
if result.rc != 0 or result.stderr:
raise gpstopException("Not able to delete existing gpstop<nnnn>.log")
lcmd=' '.join(['ls',self.logdir, '| wc -l'])
res=False
if self.is_not_running_gpdb():
res=self.gp.run_gpstart_cmd(logdir=self.logdir)
if res is not True:
raise GPstopError("Error : run_gpstart_cmd(logdir) failed \n")
cmd = Command(name='count of gpstart<nnnn>.log', cmdStr=' %s ' % (lcmd))
tinctest.logger.info("Count gpstart<nnnn>.log : %s" % cmd)
cmd.run(validateAfter=True)
result = cmd.get_results()
if result.rc != 0 or result.stderr:
raise gpstopException("Not able to get count of gpstart<nnnn>.log")
assert int(result.stdout) > 0
def failback_to_original_master(self):
# Check if master is running.
bashCmd = (
self.gphome
) + '/bin/pg_ctl status -D $MASTER_DATA_DIRECTORY | grep \'pg_ctl: server is running\''
cmd = Command(name='Running cmd %s' % bashCmd,
cmdStr="source %s/greenplum_path.sh; %s" %
(self.gphome, bashCmd))
try:
cmd.run()
except Exception, e:
tinctest.logger.error("Error running command %s\n" % e)
return
def startGpfdist(self):
if self.host in ('127.0.0.1',socket.gethostbyname(socket.gethostname()),socket.gethostname(),'localhost'):
cmdStr = 'nohup gpfdist -d %s -p %s > /dev/null 2> /dev/null < ' \
'/dev/null & echo \\$! > %s' % (self.dir, self.port,
self.pid_file)
else:
cmdStr = 'gpssh -h %s -e "source %s; nohup gpfdist -d %s -p %s > /dev/null 2> /dev/null < ' \
'/dev/null & echo \\$! > %s"' % (self.host, self.source_file, self.dir, self.port,
self.pid_file)
cmd = Command(self.name, cmdStr, self.ctxt, self.host)
cmd.run()
return self.check_gpfdist_process()
def run_using_workerpool(self, option=''):
if not (set(option.split()) <= set(
['-F', '-r', '--persistent-check', ' '])):
raise GpRecoversegException('Not a valid option with gprecoverseg')
rcvr_cmd = 'gprecoverseg -a %s' % option
cmd = Command(name='Run gprecoverseg',
cmdStr='source %s/greenplum_path.sh;%s' %
(self.gphome, rcvr_cmd))
tinctest.logger.info("Running gprecoverseg : %s" % cmd)
pool = WorkerPool()
pool.addCommand(cmd)
def test_accept(self):
"""
Replication connection should be accepted when pg_hba says so.
@tags sanity
"""
Command('gpinitstandby -ra', 'gpinitstandby -ra').run()
self.enable_replication()
with WalClient('replication=true') as client:
self.assertEqual(client.status(), pqwrap.CONNECTION_OK)
with WalClient('replication=true host=localhost') as client:
self.assertEqual(client.status(), pqwrap.CONNECTION_OK)
def _generate_gpaddmirrors_input_files(self, port_offset=1000):
with open(self.datadir_config_file, 'w') as f:
for i in range(0, self.number_of_segments_per_host):
f.write(self.mirror_data_dir + '\n')
if port_offset != 1000:
cmdStr = 'gpaddmirrors -p %s -o %s -m %s -d %s' % (
port_offset, self.mirror_config_file, self.datadir_config_file,
self.mdd)
else:
cmdStr = 'gpaddmirrors -o %s -m %s -d %s' % (
self.mirror_config_file, self.datadir_config_file, self.mdd)
Command('generate the sample_mirror_config file',
cmdStr).run(validateAfter=True)
def config_primaries_for_replication(gpArray, hba_hostnames):
logger.info("Starting to modify pg_hba.conf on primary segments to allow replication connections")
try:
for segmentPair in gpArray.getSegmentList():
# We cannot update the pg_hba.conf which uses ssh for hosts that are unreachable.
if segmentPair.primaryDB.unreachable or segmentPair.mirrorDB.unreachable:
continue
# Start with an empty string so that the later .join prepends a newline to the first entry
entries = ['']
# Add the samehost replication entry to support single-host development
entries.append('host replication {username} samehost trust'.format(username=unix.getUserName()))
if hba_hostnames:
mirror_hostname, _, _ = socket.gethostbyaddr(segmentPair.mirrorDB.getSegmentHostName())
entries.append("host all {username} {hostname} trust".format(username=unix.getUserName(), hostname=mirror_hostname))
entries.append("host replication {username} {hostname} trust".format(username=unix.getUserName(), hostname=mirror_hostname))
primary_hostname, _, _ = socket.gethostbyaddr(segmentPair.primaryDB.getSegmentHostName())
if mirror_hostname != primary_hostname:
entries.append("host replication {username} {hostname} trust".format(username=unix.getUserName(), hostname=primary_hostname))
else:
mirror_ips = gp.IfAddrs.list_addrs(segmentPair.mirrorDB.getSegmentHostName())
for ip in mirror_ips:
cidr_suffix = '/128' if ':' in ip else '/32'
cidr = ip + cidr_suffix
hba_line_entry = "host all {username} {cidr} trust".format(username=unix.getUserName(), cidr=cidr)
entries.append(hba_line_entry)
mirror_hostname = segmentPair.mirrorDB.getSegmentHostName()
segment_pair_ips = gp.IfAddrs.list_addrs(mirror_hostname)
primary_hostname = segmentPair.primaryDB.getSegmentHostName()
if mirror_hostname != primary_hostname:
segment_pair_ips.extend(gp.IfAddrs.list_addrs(primary_hostname))
for ip in segment_pair_ips:
cidr_suffix = '/128' if ':' in ip else '/32'
cidr = ip + cidr_suffix
hba_line_entry = "host replication {username} {cidr} trust".format(username=unix.getUserName(), cidr=cidr)
entries.append(hba_line_entry)
cmdStr = ". {gphome}/greenplum_path.sh; echo '{entries}' >> {datadir}/pg_hba.conf; pg_ctl -D {datadir} reload".format(
gphome=os.environ["GPHOME"],
entries="\n".join(entries),
datadir=segmentPair.primaryDB.datadir)
logger.debug(cmdStr)
cmd = Command(name="append to pg_hba.conf", cmdStr=cmdStr, ctxt=base.REMOTE, remoteHost=segmentPair.primaryDB.hostname)
cmd.run(validateAfter=True)
except Exception as e:
logger.error("Failed while modifying pg_hba.conf on primary segments to allow replication connections: %s" % str(e))
raise
else:
logger.info("Successfully modified pg_hba.conf on primary segments to allow replication connections")
def update_pg_hba_conf_on_segments(gparr,
standby_host,
is_hba_hostnames=False,
unreachable_hosts=[]):
"""
Updates the pg_hba.conf on all of the segments
present in the array
"""
logger.debug('Updating pg_hba.conf file on segments...')
standby_pg_hba_info = get_standby_pg_hba_info(standby_host,
is_hba_hostnames)
json_standby_pg_hba_info = json.dumps(standby_pg_hba_info)
host_to_seg_map = defaultdict(list)
for seg in gparr.getDbList():
if not seg.isSegmentCoordinator() and not seg.isSegmentStandby():
host_to_seg_map[seg.getSegmentHostName()].append(
seg.getSegmentDataDirectory())
pool = WorkerPool(numWorkers=DEFAULT_BATCH_SIZE)
try:
for host, data_dirs_list in list(host_to_seg_map.items()):
if host in unreachable_hosts:
logger.warning(
"Manual update of the pg_hba_conf files for all segments on unreachable host %s will be required."
% host)
continue
json_data_dirs_list = json.dumps(data_dirs_list)
cmdStr = "$GPHOME/lib/python/gppylib/operations/initstandby.py -p '%s' -d '%s'" % (
json_standby_pg_hba_info, json_data_dirs_list)
cmd = Command('Update the pg_hba.conf on remote hosts',
cmdStr=cmdStr,
ctxt=REMOTE,
remoteHost=host)
pool.addCommand(cmd)
pool.join()
for item in pool.getCompletedItems():
result = item.get_results()
if result.rc != 0:
logger.error('Unable to update pg_hba.conf %s' %
str(result.stderr))
logger.error(
'Please check the segment log file for more details')
finally:
pool.haltWork()
pool.joinWorkers()
pool = None
def test_uaocs_crash_update_in_tran(self):
setup_file = self.get_sql_files("uaocs_crash_update_setup")[0]
(sql_file1, out_file1,ans_file1) = self.get_sql_files("uaocs_crash_update_intran1")
(sql_file2, out_file2, ans_file2) = self.get_sql_files("uaocs_crash_update_intran2")
if not os.path.exists(os.path.dirname(out_file1)):
os.mkdir(os.path.dirname(out_file1))
# We set the fault in appendonly_update and appendonly_insert
# because planner will go through appendonly_update and ORCA
# will do appendonly_delete and appendonly_insert
set_fault_in_seg_panic_update = 'source %s/greenplum_path.sh;gpfaultinjector -p %s -f appendonly_update -t foo -y panic --seg_dbid 2' % (os.getenv('GPHOME'), os.getenv('PGPORT'))
set_fault_in_seg_reset_update = 'source %s/greenplum_path.sh;gpfaultinjector -p %s -f appendonly_update -t foo -y reset --seg_dbid 2' % (os.getenv('GPHOME'), os.getenv('PGPORT'))
set_fault_in_seg_panic_insert = 'source %s/greenplum_path.sh;gpfaultinjector -p %s -f appendonly_insert -t foo -y panic --seg_dbid 2' % (os.getenv('GPHOME'), os.getenv('PGPORT'))
set_fault_in_seg_reset_insert = 'source %s/greenplum_path.sh;gpfaultinjector -p %s -f appendonly_insert -t foo -y reset --seg_dbid 2' % (os.getenv('GPHOME'), os.getenv('PGPORT'))
cmd_type = 'fault injector'
PSQL.run_sql_file(setup_file)
gpfaultinjector = Command(cmd_type, set_fault_in_seg_panic_update)
gpfaultinjector.run()
gpfaultinjector = Command(cmd_type, set_fault_in_seg_panic_insert)
gpfaultinjector.run()
PSQL.run_sql_file(sql_file1, out_file=out_file1)
result1 = Gpdiff.are_files_equal(out_file1, ans_file1, match_sub=[gpdiff_init_file])
PSQL.wait_for_database_up();
gpfaultinjector = Command(cmd_type, set_fault_in_seg_reset_update)
gpfaultinjector.run()
gpfaultinjector = Command(cmd_type, set_fault_in_seg_reset_insert)
gpfaultinjector.run()
PSQL.run_sql_file(sql_file2, out_file=out_file2)
result2 = Gpdiff.are_files_equal(out_file2, ans_file2, match_sub=[gpdiff_init_file])
self.assertTrue(result1)
self.assertTrue(result2)
def test_start(self):
"""
Verify if gpstart stops the standby and gpstart starts it
correctly.
"""
# set up standby
res = self.standby.create()
self.assertEqual(res, 0)
res = self.standby.start()
self.assertTrue(res.wasSuccessful())
# stop the whole cluster including standby
cmd = Command("gpstop", "gpstop -a")
cmd.run()
self.assertEqual(cmd.get_results().rc, 0, str(cmd))
# check if standby is shut down
cmd = Command("pg_ctl status",
"pg_ctl status -D {0}".format(self.standby.datadir))
cmd.run()
# pg_ctl status returns 1 if it's not running
self.assertEqual(cmd.get_results().rc, 1, str(cmd))
# bring up the cluster. gpstart should bring up the standby, too.
cmd = Command("gpstart", "gpstart -a")
cmd.run()
self.assertTrue(cmd.get_results().rc in (0, 1), str(cmd))
# wait for the standby to connect to the primary
num_walsender = self.wait_for_walsender()
self.assertEqual(num_walsender, 1)
# make sure pg_ctl also agrees it's running
cmd = Command("pg_ctl status",
"pg_ctl status -D {0}".format(self.standby.datadir))
cmd.run()
self.assertEqual(cmd.get_results().rc, 0, str(cmd))
def test_reindex_pg_class(self):
tinctest.logger.info("create checkpoint")
results = {'rc': 0, 'stdout': '', 'stderr': ''}
PSQL.run_sql_command("checkpoint", results=results)
assert results['rc'] == 0, results['stderr']
tinctest.logger.info("inject fault to skip checkpoints")
cmd = Command(
"skip checkpoint on primaries",
"gpfaultinjector -f checkpoint -m async -y skip -o 0"
" -H ALL -r primary")
cmd.run(validateAfter=True)
tinctest.logger.info(cmd.get_results().printResult())
cmd = Command(
"skip checkpoint on master",
"gpfaultinjector -f checkpoint -m async -y skip -o 0 -s 1")
cmd.run(validateAfter=True)
tinctest.logger.info(cmd.get_results().printResult())
tinctest.logger.info("reindex pg_class indexes")
assert PSQL.run_sql_file(local_path('reindex_pg_class.sql'))
tinctest.logger.info("shutdown immediate")
cmd = Command("shutdown immediate", "gpstop -ai")
cmd.run(validateAfter=True)
tinctest.logger.info(cmd.get_results().printResult())
tinctest.logger.info("trigger recovery")
cmd = Command("restart the cluster", "gpstart -a")
cmd.run(validateAfter=True)
tinctest.logger.info(cmd.get_results().printResult())
tinctest.logger.info("validate recovery succeeded")
results = {'rc': 0, 'stdout': '', 'stderr': ''}
PSQL.run_sql_command("DROP TABLE reindex_pg_class_test",
results=results)
assert results['rc'] == 0, results['stderr']
def run_shell_command(cmdstr, cmdname = 'shell command', results={'rc':0, 'stdout':'', 'stderr':''}, verbose=False):
cmd = Command(cmdname, cmdstr)
cmd.run()
result = cmd.get_results()
results['rc'] = result.rc
results['stdout'] = result.stdout
results['stderr'] = result.stderr
if verbose:
print "command output: %s" % results['stdout']
if results['rc'] != 0:
if verbose:
print "command error: %s" % results['stderr']
return results
