class SwitchClass(MPPTestCase):
DESTDIR = 'base'
def __init__(self, methodName):
self.standby = Standby(self.DESTDIR, 5433)
super(SwitchClass, self).__init__(methodName)
def run_pg_basebackup(self):
shutil.rmtree(self.DESTDIR, True)
self.standby.create()
def start_standby(self):
res = self.standby.start()
self.assertTrue(res.wasSuccessful())
self.assertTrue(self.check_walreceiver())
def check_walreceiver(self):
grep_cmd = 'ps -ef|grep %s|grep -v grep' % ('wal receiver')
cmd = Command(name='Check if no wal receiver process', cmdStr=grep_cmd)
tinctest.logger.info('%s' % cmd)
cmd.run(validateAfter=False)
result = cmd.get_results()
if result.rc != 0:
return True
return False
class SwitchClass(MPPTestCase):
DESTDIR = 'base'
def __init__(self,methodName):
self.standby = Standby(self.DESTDIR, 5433)
super(SwitchClass,self).__init__(methodName)
def run_pg_basebackup(self):
shutil.rmtree(self.DESTDIR, True)
self.standby.create()
def start_standby(self):
res = self.standby.start()
self.assertTrue(res.wasSuccessful())
self.assertTrue(self.check_walreceiver())
def check_walreceiver(self):
grep_cmd = 'ps -ef|grep %s|grep -v grep' % ('wal receiver')
cmd = Command(name='Check if no wal receiver process', cmdStr=grep_cmd)
tinctest.logger.info ('%s' % cmd)
cmd.run(validateAfter=False)
result = cmd.get_results()
if result.rc !=0:
return True
return False
sys.stdout.write('''
***********************************************************
*** Demo for Filespace support
***********************************************************
''')
pline('Initialize standby info in catalog and take online backup')
standby.create()
subprocess.call(['ls', 'standby'])
subprocess.call(['psql', '-c', 'select * from gp_segment_configuration'])
pline('Start standby', 'pg_ctl start -D standby')
standby.create_recovery_conf()
subprocess.call(['cat', 'standby/recovery.conf'])
standby.start()
with open('filespace_setup.sql') as f:
sql = f.read()
pline('Run some SQL on primary while standby is running', sql)
subprocess.call(['psql', '-f', 'filespace_setup.sql'])
pline('Promote standby', 'pg_ctl promote -D standby\n'
'select pg_activate_standby()')
standby.promote()
sql = 'SELECT * FROM fspc_table'
pline('Connect to port 5433', 'psql -p 5433\n{0}'.format(sql))
subprocess.call(['psql', '-p', '5433', '-c', sql])
sql = 'SELECT * FROM gp_segment_configuration'
def test_icg(self):
# 1. Verify if the system is UP and there is no WAL Receiver running
# 2. Perform basebackup and deploy it into some dest. directory
# 3. Copy recover.conf into the dest. directory to be used by Standby
# 4. Initiate the Standby using the Master (primary) postmaster
# paramerters
# 5. Once the WAL receiver waits for the next record to arrive, perform
# installCheck-good
# 0. Stop standby if it's running
standby = Standby('base', 5433)
standby.stop()
# 1. Verify if the system is UP and there is no WAL sender running
self.assertEqual(self.count_wal_sender(), 0)
logger.info('No active WAL Receiver found')
# Set environmental variable GPSRC for now for make installcheck purpose
source_file = sys.modules[self.__class__.__module__].__file__
source_dir = os.path.dirname(source_file)
os.environ['GPSRC'] = os.path.join(source_dir, '../../../../')
# 2. Perform basebackup and deploy it into some dest.
# (currenttly hardcoded 'base') directory
shutil.rmtree('base', True)
logger.info('Performing and deploying base backup ...')
standby.create()
# 3.Copy recover.conf into the dest. directory to be used by StandBy
logger.info('Deploying recovery.conf...')
# 4. Initiate the StandBy using the Master (primary) postmaster
# paramerters
logger.info('Initiating Standby...')
res = standby.start()
self.assertTrue(res.wasSuccessful())
num_walsender = 0
for i in polling(10, 0.5):
num_walsender = self.count_wal_sender()
if num_walsender > 0:
break
self.assertEqual(num_walsender, 1)
logger.info('Activated WAL Receiver...')
# 6. Run installcheck-good
self.assertTrue(os.environ["GPSRC"])
installCheckGoodPath = os.path.join(os.environ.get('GPSRC'),
'test',
'regress')
self.assertTrue(os.path.exists(installCheckGoodPath))
os.chdir(installCheckGoodPath)
regression_diffs = os.path.join(installCheckGoodPath, 'regression.diffs')
if os.path.exists(regression_diffs):
os.remove(regression_diffs)
logger.info('Perform InstallCheck-Good...')
subprocess.check_call('make installcheck-good', shell=True)
# Verify installcheck result by checking if regression.diff is present.
self.assertTrue(not os.path.exists(regression_diffs))
sys.stdout.write('''
***********************************************************
*** Demo for Filespace support
***********************************************************
''')
pline('Initialize standby info in catalog and take online backup')
standby.create()
subprocess.call(['ls', 'standby'])
subprocess.call(['psql', '-c', 'select * from gp_segment_configuration'])
pline('Start standby', 'pg_ctl start -D standby')
standby.create_recovery_conf()
subprocess.call(['cat', 'standby/recovery.conf'])
standby.start()
with open('filespace_setup.sql') as f:
sql = f.read()
pline('Run some SQL on primary while standby is running', sql)
subprocess.call(['psql', '-f', 'filespace_setup.sql'])
pline('Promote standby', 'pg_ctl promote -D standby\n'
'select pg_activate_standby()')
standby.promote()
sql = 'SELECT * FROM fspc_table'
pline('Connect to port 5433', 'psql -p 5433\n{0}'.format(sql))
subprocess.call(['psql', '-p', '5433', '-c', sql])
sql = 'SELECT * FROM gp_segment_configuration'
class OOMClass(object):
'''Class for methods required for OOM testcase'''
standby_port = '5433'
standby_dirname = 'newstandby'
def __init__(self):
self.gpinit = GpinitStandby()
self.mdd = os.environ.get('MASTER_DATA_DIRECTORY')
self.config = GPDBConfig()
self.pgutil = GpUtility()
self.verify = StandbyVerify()
self.host = socket.gethostname()
self.standby_loc = os.path.join(os.path.split(self.mdd)[0],
self.standby_dirname)
self.standby = Standby(self.standby_loc, self.standby_port)
def create_standby(self):
self.pgutil.clean_dir(self.host,self.standby_loc)
self.gpinit.run(option = '-P %s -s %s -F pg_system:%s' % (self.standby_port, self.host, self.standby_loc))
def setup_oom(self):
# Build it before testing.
thisdir = os.path.dirname(__file__)
builddir = os.path.join(thisdir, 'lib')
subprocess.check_call(['make', '-C', builddir, 'install'])
#Copy oom_malloc.so and wrapper.sh to all the segment nodes
for host in config.get_hosts() :
if host.strip() == self.host :
continue
cmd = "gpssh -h %s -e 'mkdir -p %s'; scp %s/oom_malloc.so %s:%s/; scp %s/wrapper.sh %s:%s/" % (host.strip(), builddir, builddir, host.strip(), builddir, builddir, host.strip(), builddir)
self.pgutil.run(cmd)
def touch_malloc(self):
# Touch file oom_malloc in standby directory
cmd = 'touch %s/oom_malloc' % self.standby_loc
self.pgutil.run(cmd)
def startdb(self):
(rc, result) = self.pgutil.run('gpstart -a --wrapper %s' % (local_path('lib/wrapper.sh')))
if rc != 0 and 'Could not start standby master' in result :
return False
return True
def restartdb(self):
# Remove file oom_malloc from standby
cmd = 'rm %s/oom_malloc' % self.standby_loc
self.pgutil.run(cmd)
(rc, result) = self.pgutil.run('gpstop -ar')
if rc == 0 and (self.verify.check_pg_stat_replication()):
return True
return False
def psql_and_oom(self):
#Touch oom_malloc in standby_dir and issue PSQL : Check if processes are gone
self.touch_malloc()
PSQL.run_sql_command('Drop table if exists wal_oomt1;Create table wal_oomt1(a1 int, a2 text) with(appendonly=true);')
sleep(2)
if not (self.verify.check_standby_processes()):
return True
return False
def start_standby(self):
# Remove oom_malloc and start standby : Check if all processes are back
cmd = 'rm %s/oom_malloc' % self.standby_loc
self.pgutil.run(cmd)
res = self.standby.start()
sleep(2)
if (self.verify.check_standby_processes()) :
return True
return False
def test_smart_shutdown(self):
# 1. Verify if the system is UP and there is no WAL Receiver running
# 2. Perform basebackup and deploy it into some dest. directory
# 3. Copy recover.conf into the dest. directory to be used by Standby
# 4. Initiate the Standby using the Master (primary) postmaster
# paramerters
# 5. Perform some transaction to generate xlog. Then do a smart shutdown
# 6. Once the primary DB is down, find the last checkpoint from pg_control
# on primary. Check the last modified xlog seg from the standby and find
# if the last checkpoint from primary exists
# 7. It should be present there!
# 0. Stop standby if it's running
PSQL.run_sql_command('DROP table if exists foo')
standby = Standby('base', 5433)
standby.stop()
# 1. Verify if the system is UP and there is no WAL sender running
self.assertEqual(self.count_walsender(), 0)
logger.info('No active WAL Receiver found')
# 2. Perform basebackup and deploy it into some dest.
# (currenttly hardcoded 'base') directory
shutil.rmtree('base', True)
logger.info('Performing and deploying base backup ...')
standby.create()
# 3.Copy recover.conf into the dest. directory to be used by StandBy
logger.info('Deploying recovery.conf...')
# 4. Initiate the StandBy using the Master (primary) postmaster
# paramerters
logger.info('Initiating Standby...')
res = standby.start()
self.assertTrue(res.wasSuccessful())
num_walsender = 0
for i in polling(10, 0.5):
num_walsender = self.count_walsender()
if num_walsender > 0:
break
self.assertEqual(num_walsender, 1)
logger.info('Activated WAL Receiver...')
# 5. Perform some transaction to generate xlog. Then do a smart shutdown
logger.info('Perform some transaction to generate some XLOG')
PSQL.run_sql_command('Create table foo (a int)')
logger.info('Now perform smart shutdown (gpstop -a)')
cmd = Command(name="gpstop smart",
cmdStr="source %s/greenplum_path.sh;\
gpstop -a" % os.environ["GPHOME"])
cmd.run(validateAfter=True)
# 6. Once the primary DB is down, find the last checkpoint from pg_control
# on primary. Check the last modified xlog seg from the standby and find
# if the last checkpoint from primary exists
logger.info(
'Read the pg_control from primary, find the last checkpoint & see if it made to standby'
)
standby_xlog_path = os.path.join('base', 'pg_xlog')
cmd = Command(
name='pg_controldata ' + os.environ.get('MASTER_DATA_DIRECTORY'),
cmdStr='pg_controldata ' + os.environ.get('MASTER_DATA_DIRECTORY'))
cmd.run(validateAfter=True)
primary_last_ckpt_lsn = self.last_ckpt_lsn((cmd.get_results()).stdout)
logger.info("Primary last checkpoint LSN = " + primary_last_ckpt_lsn)
standby_last_mod_xlog = self.last_mod_file(standby_xlog_path)
logger.info("Last mod standby XLOG = " + standby_last_mod_xlog)
cmd = Command(name='xlogdump standby last modifiled xlog',
cmdStr="xlogdump " + standby_last_mod_xlog)
cmd.run(validateAfter=True)
logger.info('See if we find the shutdown LSN in the XLOG seg file')
lines = (cmd.get_results()).stdout.splitlines()
flag = False
for line in range(0, len(lines) - 1):
if ((lines[line]).find(primary_last_ckpt_lsn) > -1):
self.assertTrue((lines[line]).find("checkpoint") > -1)
self.assertTrue((lines[line]).find("shutdown") > -1)
flag = True
break
self.assertTrue(flag)
logger.info('PASS')
# Re-start the database
logger.info('Now restart the DB (gpstart -a)')
cmd = Command(name="gpstop smart",
cmdStr="source %s/greenplum_path.sh;\
gpstart -a" % os.environ["GPHOME"])
cmd.run(validateAfter=True)
# Cleanup. Currently we dont have a clean way of WAL rcv dying
logger.info(
'Kill the standby processes as clean standby killing is not supported'
)
cmd = Command(
name="kill standby",
cmdStr=
"kill -9 `ps -ef | grep 5433 | grep -v grep | awk '{print $2}'`")
cmd.run(validateAfter=True)
class OOMClass(object):
'''Class for methods required for OOM testcase'''
standby_port = '5433'
standby_dirname = 'newstandby'
def __init__(self):
self.gpinit = GpinitStandby()
self.mdd = os.environ.get('MASTER_DATA_DIRECTORY')
self.config = GPDBConfig()
self.pgutil = GpUtility()
self.verify = StandbyVerify()
self.host = socket.gethostname()
self.standby_loc = os.path.join(
os.path.split(self.mdd)[0], self.standby_dirname)
self.standby = Standby(self.standby_loc, self.standby_port)
def create_standby(self):
self.pgutil.clean_dir(self.host, self.standby_loc)
self.gpinit.run(option='-P %s -s %s -F pg_system:%s' %
(self.standby_port, self.host, self.standby_loc))
def setup_oom(self):
# Build it before testing.
thisdir = os.path.dirname(__file__)
builddir = os.path.join(thisdir, 'lib')
subprocess.check_call(['make', '-C', builddir, 'install'])
#Copy oom_malloc.so and wrapper.sh to all the segment nodes
for host in config.get_hosts():
if host.strip() == self.host:
continue
cmd = "gpssh -h %s -e 'mkdir -p %s; scp %s/oom_malloc.so %s:%s/;scp %s/wrapper.sh %s:%s/'" % (
host.strip(), builddir, builddir, host.strip(), builddir,
builddir, host.strip(), builddir)
self.pgutil.run(cmd)
def touch_malloc(self):
# Touch file oom_malloc in standby directory
cmd = 'touch %s/oom_malloc' % self.standby_loc
self.pgutil.run(cmd)
def startdb(self):
(rc, result) = self.pgutil.run('gpstart -a --wrapper %s' %
(local_path('lib/wrapper.sh')))
if rc != 0 and 'Could not start standby master' in result:
return False
return True
def restartdb(self):
# Remove file oom_malloc from standby
cmd = 'rm %s/oom_malloc' % self.standby_loc
self.pgutil.run(cmd)
(rc, result) = self.pgutil.run('gpstop -ar')
if rc == 0 and (self.verify.check_pg_stat_replication()):
return True
return False
def psql_and_oom(self):
#Touch oom_malloc in standby_dir and issue PSQL : Check if processes are gone
self.touch_malloc()
PSQL.run_sql_command(
'Drop table if exists wal_oomt1;Create table wal_oomt1(a1 int, a2 text) with(appendonly=true);'
)
sleep(2)
if not (self.verify.check_standby_processes()):
return True
return False
def start_standby(self):
# Remove oom_malloc and start standby : Check if all processes are back
cmd = 'rm %s/oom_malloc' % self.standby_loc
self.pgutil.run(cmd)
res = self.standby.start()
sleep(2)
if (self.verify.check_standby_processes()):
return True
return False
def test_icg(self):
# 1. Verify if the system is UP and there is no WAL Receiver running
# 2. Perform basebackup and deploy it into some dest. directory
# 3. Copy recover.conf into the dest. directory to be used by Standby
# 4. Initiate the Standby using the Master (primary) postmaster
# paramerters
# 5. Once the WAL receiver waits for the next record to arrive, perform
# installCheck-good
# 0. Stop standby if it's running
standby = Standby('base', 5433)
standby.stop()
# 1. Verify if the system is UP and there is no WAL sender running
self.assertEqual(self.count_wal_sender(), 0)
logger.info('No active WAL Receiver found')
# Set environmental variable GPSRC for now for make installcheck purpose
source_file = sys.modules[self.__class__.__module__].__file__
source_dir = os.path.dirname(source_file)
os.environ['GPSRC'] = os.path.join(source_dir, '../../../../')
# 2. Perform basebackup and deploy it into some dest.
# (currenttly hardcoded 'base') directory
shutil.rmtree('base', True)
logger.info('Performing and deploying base backup ...')
standby.create()
# 3.Copy recover.conf into the dest. directory to be used by StandBy
logger.info('Deploying recovery.conf...')
# 4. Initiate the StandBy using the Master (primary) postmaster
# paramerters
logger.info('Initiating Standby...')
res = standby.start()
self.assertTrue(res.wasSuccessful())
num_walsender = 0
for i in polling(10, 0.5):
num_walsender = self.count_wal_sender()
if num_walsender > 0:
break
self.assertEqual(num_walsender, 1)
logger.info('Activated WAL Receiver...')
# 6. Run installcheck-good
self.assertTrue(os.environ["GPSRC"])
installCheckGoodPath = os.path.join(os.environ.get('GPSRC'), 'test',
'regress')
self.assertTrue(os.path.exists(installCheckGoodPath))
os.chdir(installCheckGoodPath)
regression_diffs = os.path.join(installCheckGoodPath,
'regression.diffs')
if os.path.exists(regression_diffs):
os.remove(regression_diffs)
logger.info('Perform InstallCheck-Good...')
subprocess.check_call('make installcheck-good', shell=True)
# Verify installcheck result by checking if regression.diff is present.
self.assertTrue(not os.path.exists(regression_diffs))
def test_smart_shutdown(self):
# 1. Verify if the system is UP and there is no WAL Receiver running
# 2. Perform basebackup and deploy it into some dest. directory
# 3. Copy recover.conf into the dest. directory to be used by Standby
# 4. Initiate the Standby using the Master (primary) postmaster
# paramerters
# 5. Perform some transaction to generate xlog. Then do a smart shutdown
# 6. Once the primary DB is down, find the last checkpoint from pg_control
# on primary. Check the last modified xlog seg from the standby and find
# if the last checkpoint from primary exists
# 7. It should be present there!
# 0. Stop standby if it's running
PSQL.run_sql_command('DROP table if exists foo')
standby = Standby('base', 5433)
standby.stop()
# 1. Verify if the system is UP and there is no WAL sender running
self.assertEqual(self.count_walsender(), 0)
logger.info('No active WAL Receiver found')
# 2. Perform basebackup and deploy it into some dest.
# (currenttly hardcoded 'base') directory
shutil.rmtree('base', True)
logger.info('Performing and deploying base backup ...')
standby.create()
# 3.Copy recover.conf into the dest. directory to be used by StandBy
logger.info('Deploying recovery.conf...')
# 4. Initiate the StandBy using the Master (primary) postmaster
# paramerters
logger.info('Initiating Standby...')
res = standby.start()
self.assertTrue(res.wasSuccessful())
num_walsender = 0
for i in polling(10, 0.5):
num_walsender = self.count_walsender()
if num_walsender > 0:
break
self.assertEqual(num_walsender, 1)
logger.info('Activated WAL Receiver...')
# 5. Perform some transaction to generate xlog. Then do a smart shutdown
logger.info('Perform some transaction to generate some XLOG')
PSQL.run_sql_command('Create table foo (a int)')
logger.info('Now perform smart shutdown (gpstop -a)')
cmd = Command(name="gpstop smart",
cmdStr="source %s/greenplum_path.sh;\
gpstop -a" % os.environ["GPHOME"])
cmd.run(validateAfter=True)
# 6. Once the primary DB is down, find the last checkpoint from pg_control
# on primary. Check the last modified xlog seg from the standby and find
# if the last checkpoint from primary exists
logger.info('Read the pg_control from primary, find the last checkpoint & see if it made to standby')
standby_xlog_path = os.path.join('base','pg_xlog')
cmd= Command(name = 'pg_controldata ' + os.environ.get('MASTER_DATA_DIRECTORY'),
cmdStr = 'pg_controldata ' + os.environ.get('MASTER_DATA_DIRECTORY'))
cmd.run(validateAfter=True);
primary_last_ckpt_lsn = self.last_ckpt_lsn((cmd.get_results()).stdout)
logger.info ("Primary last checkpoint LSN = " + primary_last_ckpt_lsn)
standby_last_mod_xlog = self.last_mod_file(standby_xlog_path)
logger.info ("Last mod standby XLOG = " + standby_last_mod_xlog )
cmd= Command(name = 'xlogdump standby last modifiled xlog',
cmdStr="xlogdump " + standby_last_mod_xlog)
cmd.run(validateAfter=True);
logger.info('See if we find the shutdown LSN in the XLOG seg file')
lines = (cmd.get_results()).stdout.splitlines()
flag = False
for line in range(0, len(lines) - 1):
if ((lines[line]).find(primary_last_ckpt_lsn) > -1):
self.assertTrue((lines[line]).find("checkpoint") > -1)
self.assertTrue((lines[line]).find("shutdown") > -1)
flag = True
break
self.assertTrue(flag)
logger.info('PASS')
# Re-start the database
logger.info('Now restart the DB (gpstart -a)')
cmd = Command(name="gpstop smart",
cmdStr="source %s/greenplum_path.sh;\
gpstart -a" % os.environ["GPHOME"])
cmd.run(validateAfter=True)
# Cleanup. Currently we dont have a clean way of WAL rcv dying
logger.info('Kill the standby processes as clean standby killing is not supported')
cmd = Command(name="kill standby",
cmdStr="kill -9 `ps -ef | grep 5433 | grep -v grep | awk '{print $2}'`")
cmd.run(validateAfter=True)
