class SwitchXlogTestCase(ScenarioTestCase):
''' Initiate standby with old pg_basebackup and new xlog'''
def __init__(self, methodName):
self.gp = GpinitStandby()
super(SwitchXlogTestCase,self).__init__(methodName)
def setUp(self):
#Remove standby if present
self.gp.run(option='-r')
def tearDown(self):
#Remove standby
self.gp.run(option='-r')
@unittest.skipIf(os.uname()[0] == 'Darwin', "Skipping this test on OSX")
def test_switch_xlog_after_basebackup(self):
test_case_list0 = []
test_case_list0.append('mpp.gpdb.tests.storage.walrepl.basebackup.SwitchClass.run_pg_basebackup')
self.test_case_scenario.append(test_case_list0)
test_case_list1 = []
test_case_list1.append('mpp.gpdb.tests.storage.walrepl.basebackup.switch.runsql.RunWorkload')
self.test_case_scenario.append(test_case_list1)
test_case_list2 = []
test_case_list2.append('mpp.gpdb.tests.storage.walrepl.basebackup.SwitchClass.start_standby')
self.test_case_scenario.append(test_case_list2)
class LoadTestCase(ScenarioTestCase):
''' Skip Checkpoint, Run workload, then init standby'''
def __init__(self, methodName):
self.gp = GpinitStandby()
super(LoadTestCase,self).__init__(methodName)
def setUp(self):
#Remove standby if present
self.gp.run(option='-r')
@unittest.skipIf(os.uname()[0] == 'Darwin', "Skipping this test on OSX")
def Dtest_load_with_skip_checkpoint(self):
test_case_list0 = []
test_case_list0.append('mpp.gpdb.tests.storage.walrepl.load.LoadClass.skip_checkpoint')
self.test_case_scenario.append(test_case_list0)
test_case_list1 = []
test_case_list1.append('mpp.gpdb.tests.storage.walrepl.load.sql.runsql.RunWorkload')
self.test_case_scenario.append(test_case_list1)
test_case_list2 = []
test_case_list2.append('mpp.gpdb.tests.storage.walrepl.load.LoadClass.init_standby')
self.test_case_scenario.append(test_case_list2)
test_case_list3 = []
test_case_list3.append('mpp.gpdb.tests.storage.walrepl.load.LoadClass.cleanup')
self.test_case_scenario.append(test_case_list3)
class WalRecTestCase(MPPTestCase):
''' Testcases for SIGTERM on wal receiver'''
def __init__(self, methodName):
self.gp = GpinitStandby()
self.mdd = os.environ.get('MASTER_DATA_DIRECTORY')
self.pgutil = GpUtility()
super(WalRecTestCase,self).__init__(methodName)
def setUp(self):
#Remove standby if present
self.gp.run(option='-r')
def invoke_sigterm_and_verify(self):
''' Invoke sigterm on wal receiver and verify that a new process is spawned after '''
gpact_stdby = GpactivateStandby()
standby_host = gpact_stdby.get_current_standby()
standby_port = gpact_stdby.get_standby_port()
wal_rec_pid_1 = self.pgutil.get_pid_by_keyword(host=standby_host, pgport=standby_port, keyword='wal receiver process', option='')
sig_cmd = "gpssh -h %s -e 'kill -15 %s'" % (standby_host, wal_rec_pid_1)
cmd = Command('Issue SIGTERM to wam receiver process', cmdStr=sig_cmd)
tinctest.logger.info ('%s' % cmd)
cmd.run(validateAfter=True)
result = cmd.get_results()
if result.rc != 0:
return False
wal_rec_pid_2 = self.pgutil.get_pid_by_keyword(host=standby_host, pgport=standby_port, keyword='wal receiver process', option='')
if wal_rec_pid_1 == wal_rec_pid_2:
return False
return True
@unittest.skipIf(not config.is_multinode(), "Test applies only to a multinode cluster")
def test_sigterm_on_walreceiver(self):
self.pgutil.install_standby()
self.assertTrue(self.invoke_sigterm_and_verify())
class BackupRestoreTestCase(BkupRestore):
''' gpcrondump from the primary master, gpdbrestore from the standby master'''
def __init__(self, methodName):
super(BackupRestoreTestCase,self).__init__(methodName)
def setUp(self):
self.gp = GpinitStandby()
self.gp.run(option='-r')
def tearDown(self):
self.failback()
@unittest.skipIf(not config.is_multinode(), "Test applies only to a multinode cluster")
def test_backup_restore(self):
# Create standby if not present
Command('createdb bkdb','dropdb bkdb; createdb bkdb').run()
self.create_standby()
# Run workload
self.run_workload('sql')
# Create backup
self.assertTrue(self.run_backup())
# Activate standby
gpac = GpactivateStandby()
gpac.activate()
# Restore from new master
self.assertTrue(self.run_restore())
class LoadClass(MPPTestCase):
def __init__(self,methodName):
self.gp =GpinitStandby()
super(LoadClass,self).__init__(methodName)
def run_skip(self, type):
tinctest.logger.info("skipping checkpoint")
cmd_str = 'gpfaultinjector -p %s -m async -H ALL -r primary -f checkpoint -y %s -o 0' % (os.getenv('PGPORT'), type)
cmd = Command('skip_chkpoint', cmd_str)
cmd.run(validateAfter =False)
def skip_checkpoint(self):
''' Routine to inject fault that skips checkpointing '''
self.run_skip('reset')
self.run_skip('suspend')
def init_standby(self):
pg = GpUtility()
pg.install_standby()
def cleanup(self):
# Remove standby and reset the checkpoint fault
self.gp.run(option='-r')
self.run_skip('reset')
def check_stdby_stop(self):
gpstdby = GpinitStandby()
stdby_host = gpstdby.get_standbyhost()
activate_stdby = GpactivateStandby()
stdby_port = activate_stdby.get_standby_port()
master_pid = self.pgutil.get_pid_by_keyword(host=stdby_host, pgport=stdby_port, keyword="master", option = "bin")
if int(master_pid) != -1:
raise Exception("standby should stop but failed!")
def start_stdby(self):
gpstdby = GpinitStandby()
stdby_host = gpstdby.get_standbyhost()
stdby_dbid = self.get_standby_dbid()
activate_stdby = GpactivateStandby()
stdby_mdd = activate_stdby.get_standby_dd()
stdby_port = activate_stdby.get_standby_port()
cmd="pg_ctl -D %s -o '-p %s --gp_dbid=%s --gp_num_contents_in_cluster=2 --silent-mode=true -i -M master --gp_contentid=-1 -x 0 -E' start &"%(stdby_mdd, stdby_port, stdby_dbid)
self.run_remote(stdby_host,cmd,stdby_port,stdby_mdd)
def __init__(self):
self.gpinit = GpinitStandby()
self.mdd = os.environ.get('MASTER_DATA_DIRECTORY')
self.config = GPDBConfig()
self.pgutil = GpUtility()
self.verify = StandbyVerify()
self.host = socket.gethostname()
self.standby_loc = os.path.join(
os.path.split(self.mdd)[0], self.standby_dirname)
self.standby = Standby(self.standby_loc, self.standby_port)
def setUp(self):
super(WalReplKillProcessScenarioTestCase, self).setUp()
pgutil.check_and_start_gpdb()
# We should forcibly recreate standby, as it might has been promoted.
pgutil.remove_standby()
pgutil.install_standby()
gpact_stdby = GpactivateStandby()
gpinit_stdb = GpinitStandby()
WalReplKillProcessTestCase.stdby_port = gpact_stdby.get_standby_port()
WalReplKillProcessTestCase.stdby_host = gpinit_stdb.get_standbyhost()
def setUp(self):
super(WalReplKillProcessScenarioTestCase, self).setUp()
pgutil.check_and_start_gpdb()
# We should forcibly recreate standby, as it might has been promoted.
pgutil.remove_standby()
pgutil.install_standby()
gpact_stdby = GpactivateStandby()
gpinit_stdb = GpinitStandby()
WalReplKillProcessTestCase.stdby_port = gpact_stdby.get_standby_port()
WalReplKillProcessTestCase.stdby_host = gpinit_stdb.get_standbyhost()
def setUp(self):
pgutil.check_and_start_gpdb()
# We should forcibly recreate standby, as it might has been promoted.
# here we need to install locally, otherwise can not run remote sql
pgutil.remove_standby()
pgutil.install_standby(new_stdby_host=socket.gethostname())
gpact_stdby = GpactivateStandby()
gpinit_stdb = GpinitStandby()
WalReplKillProcessTestCase.stdby_port = gpact_stdby.get_standby_port()
WalReplKillProcessTestCase.stdby_host = gpinit_stdb.get_standbyhost()
self.standby_dir = gpact_stdby.get_standby_dd()
def check_stdby_stop(self):
gpstdby = GpinitStandby()
stdby_host = gpstdby.get_standbyhost()
activate_stdby = GpactivateStandby()
stdby_port = activate_stdby.get_standby_port()
master_pid = self.pgutil.get_pid_by_keyword(host=stdby_host,
pgport=stdby_port,
keyword="master",
option="bin")
if int(master_pid) != -1:
raise Exception("standby should stop but failed!")
def setUp(self):
pgutil.check_and_start_gpdb()
# We should forcibly recreate standby, as it might has been promoted.
# here we need to install locally, otherwise can not run remote sql
pgutil.remove_standby()
pgutil.install_standby(new_stdby_host=socket.gethostname())
gpact_stdby = GpactivateStandby()
gpinit_stdb = GpinitStandby()
WalReplKillProcessTestCase.stdby_port = gpact_stdby.get_standby_port()
WalReplKillProcessTestCase.stdby_host = gpinit_stdb.get_standbyhost()
self.standby_dir = gpact_stdby.get_standby_dd()
def test_gpactivatestandby_on_new_host(self):
gputil.install_standby()
initstdby = GpinitStandby()
gpact_stdby = GpactivateStandby()
self.mdd = gpact_stdby.get_standby_dd()
self.host = initstdby.get_standbyhost()
self.port = gpact_stdby.get_standby_port()
self.standby_pid = gpact_stdby.get_standby_pid(self.host, self.port, self.mdd)
PSQL.run_sql_file(local_path('create_tables.sql'), dbname = self.db_name)
self.assertTrue(gpact_stdby.activate())
self.assertTrue(gpact_stdby.verify_gpactivatestandby(self.standby_pid, self.host, self.port, self.mdd))
gputil.failback_to_original_master(self.origin_mdd,self.host,self.mdd,self.port)
def __init__(self, methodName):
self.gpact = GpactivateStandby()
self.gpinit = GpinitStandby()
self.runmixin = StandbyRunMixin()
self.runmixin.createdb(dbname=self.dbname)
self.bkup_timestamp = ""
self.gphome = os.environ.get('GPHOME')
self.mdd = os.environ.get('MASTER_DATA_DIRECTORY')
self.pgport = os.environ.get('PGPORT')
self.host = socket.gethostname()
self.standby_loc = os.path.split(self.mdd)[0] + self.standby_dirname
super(BkupRestore, self).__init__(methodName)
def test_gpactivatestandby_on_new_host(self):
gputil.install_standby()
initstdby = GpinitStandby()
gpact_stdby = GpactivateStandby()
self.mdd = gpact_stdby.get_standby_dd()
self.host = initstdby.get_standbyhost()
self.port = gpact_stdby.get_standby_port()
self.standby_pid = gpact_stdby.get_standby_pid(self.host, self.port, self.mdd)
PSQL.run_sql_file(local_path('create_tables.sql'), dbname = self.db_name)
self.assertTrue(gpact_stdby.activate())
self.assertTrue(gpact_stdby.verify_gpactivatestandby(self.standby_pid, self.host, self.port, self.mdd))
gputil.failback_to_original_master(self.origin_mdd,self.host,self.mdd,self.port)
def kill_walreceiver(self):
gpstdby = GpinitStandby()
stdby_host = gpstdby.get_standbyhost()
activate_stdby = GpactivateStandby()
stdby_port = activate_stdby.get_standby_port()
pid_list = []
walreceiver_pid = self.pgutil.get_pid_by_keyword(host=stdby_host, pgport=stdby_port, keyword="wal receiver process")
if int(walreceiver_pid) == -1:
tinctest.logger.error("error: wal receiver process does not exist!")
return False
else:
pid_list.append(walreceiver_pid)
self.killProcess_byPid(pid_toKill=pid_list, host=stdby_host)
def kill_walreceiver(self):
gpstdby = GpinitStandby()
stdby_host = gpstdby.get_standbyhost()
activate_stdby = GpactivateStandby()
stdby_port = activate_stdby.get_standby_port()
pid_list = []
walreceiver_pid = self.pgutil.get_pid_by_keyword(host=stdby_host, pgport=stdby_port, keyword="wal receiver process")
if int(walreceiver_pid) == -1:
tinctest.logger.error("error: wal receiver process does not exist!")
return False
else:
pid_list.append(walreceiver_pid)
self.killProcess_byPid(pid_toKill=pid_list, host=stdby_host)
def start_stdby(self):
gpstdby = GpinitStandby()
stdby_host = gpstdby.get_standbyhost()
stdby_dbid = self.get_standby_dbid()
activate_stdby = GpactivateStandby()
stdby_mdd = activate_stdby.get_standby_dd()
stdby_port = activate_stdby.get_standby_port()
cmd = "pg_ctl -D %s -o '-p %s -b %s -z 2 --silent-mode=true -i -M master -C -1 -x 0 -E' start &" % (
stdby_mdd,
stdby_port,
stdby_dbid,
)
self.run_remote(stdby_host, cmd, stdby_port, stdby_mdd)
def __init__(self):
self.gpinit = GpinitStandby()
self.pgutil = GpUtility()
self.runmixin = StandbyRunMixin()
self.runmixin.createdb(dbname='walrepl')
self.gphome = os.environ.get('GPHOME')
self.pgport = os.environ.get('PGPORT')
self.mdd = os.environ.get('MASTER_DATA_DIRECTORY')
self.config = GPDBConfig()
self.host = socket.gethostname()
dburl = dbconn.DbURL()
gparray = GpArray.initFromCatalog(dburl, utility=True)
self.numcontent = gparray.getNumSegmentContents()
self.orig_master = gparray.master
def test_promote_incomplete_stdby(self):
'''
remove the standby base dir, try promote and check if fail
'''
gpactivate_stdby = GpactivateStandby()
gpinit_stdby = GpinitStandby()
stdby_mdd = gpactivate_stdby.get_standby_dd()
stdby_host = gpinit_stdby.get_standbyhost()
stdby_port = gpactivate_stdby.get_standby_port()
destDir = os.path.join(stdby_mdd, 'base')
self.pgutil.clean_dir(stdby_host,destDir)
promote_cmd = "pg_ctl promote -D %s"%stdby_mdd
(rc, output) = gpactivate_stdby.run_remote(stdby_host,promote_cmd ,stdby_port,stdby_mdd)
self.assertEqual(rc, 0)
pid = self.pgutil.get_pid_by_keyword(host=stdby_host, pgport=stdby_port, keyword='master', option='bin')
self.assertTrue(int(pid) == -1, 'incomplete standby data directory promote succeeds.')
def test_gpactivatestandby_new_host_with_filespace(self):
#from mpp.lib.gpfilespace import Gpfilespace
#gpfile = Gpfilespace()
#gpfile.create_filespace('fs_walrepl_a')
#PSQL.run_sql_file(local_path('filespace.sql'), dbname= self.db_name)
gputil.install_standby()
initstdby = GpinitStandby()
gpact_stdby = GpactivateStandby()
self.mdd = gpact_stdby.get_standby_dd()
self.host = initstdby.get_standbyhost()
self.port = gpact_stdby.get_standby_port()
self.standby_pid = gpact_stdby.get_standby_pid(self.host, self.port, self.mdd)
PSQL.run_sql_file(local_path('create_tables.sql'), dbname = self.db_name)
self.assertTrue(gpact_stdby.activate())
self.assertTrue(gpact_stdby.verify_gpactivatestandby(self.standby_pid, self.host, self.port, self.mdd))
gputil.failback_to_original_master(self.origin_mdd,self.host,self.mdd,self.port)
def test_gpactivatestandby_new_host_with_filespace(self):
from mpp.lib.gpfilespace import Gpfilespace
gpfile = Gpfilespace()
gpfile.create_filespace('fs_walrepl_a')
PSQL.run_sql_file(local_path('filespace.sql'), dbname= self.db_name)
gputil.install_standby()
initstdby = GpinitStandby()
gpact_stdby = GpactivateStandby()
self.mdd = gpact_stdby.get_standby_dd()
self.host = initstdby.get_standbyhost()
self.port = gpact_stdby.get_standby_port()
self.standby_pid = gpact_stdby.get_standby_pid(self.host, self.port, self.mdd)
PSQL.run_sql_file(local_path('create_tables.sql'), dbname = self.db_name)
self.assertTrue(gpact_stdby.activate())
self.assertTrue(gpact_stdby.verify_gpactivatestandby(self.standby_pid, self.host, self.port, self.mdd))
gputil.failback_to_original_master(self.origin_mdd,self.host,self.mdd,self.port)
def setUp(self):
# For each test case we create a fresh standby and start it.
self.db_name = self.__class__.db_name
self.createdb(self.db_name)
gputil.remove_standby()
gputil.install_standby(new_stdby_host=socket.gethostname())
self.gpinit_stdby = GpinitStandby()
self.activatestdby = GpactivateStandby()
class OODTestCase(ScenarioTestCase):
''' Standby OOD when transactions in progress, standby need to be restaerted'''
def __init__(self, methodName):
self.gp = GpinitStandby()
super(OODTestCase, self).__init__(methodName)
def setUp(self):
#Remove standby if present
self.gp.run(option='-r')
def tearDown(self):
#Remove standby
self.gp.run(option='-r')
@unittest.skipIf(os.uname()[0] == 'Darwin', "Skipping this test on OSX")
def Dtest_ood_on_standby(self):
test_case_list0 = []
test_case_list0.append(
'mpp.gpdb.tests.storage.walrepl.ood.OODClass.initiate_standby')
self.test_case_scenario.append(test_case_list0)
test_case_list1 = []
test_case_list1.append(
'mpp.gpdb.tests.storage.walrepl.ood.OODClass.filldisk')
self.test_case_scenario.append(test_case_list1)
test_case_list2 = []
test_case_list2.append(
'mpp.gpdb.tests.storage.walrepl.ood.sql.runsql.RunWorkload')
self.test_case_scenario.append(test_case_list2)
test_case_list3 = []
test_case_list3.append(
'mpp.gpdb.tests.storage.walrepl.ood.OODClass.check_standby')
self.test_case_scenario.append(test_case_list3)
test_case_list4 = []
test_case_list4.append(
'mpp.gpdb.tests.storage.walrepl.ood.OODClass.restart_standby')
self.test_case_scenario.append(test_case_list4)
test_case_list5 = []
test_case_list5.append(
'mpp.gpdb.tests.storage.walrepl.ood.OODClass.cleanup')
self.test_case_scenario.append(test_case_list5)
def __init__(self, methodName):
self.gp = GpinitStandby()
self.host = socket.gethostname()
self.mdd = os.environ.get('MASTER_DATA_DIRECTORY')
self.standby = self.gp.get_standbyhostnode()
self.standby_loc = os.path.join(os.path.split(self.mdd)[0],
self.standby_dirname)
self.primary_pid = 0
super(GpinitStandsbyTestCase,self).__init__(methodName)
def __init__(self):
self.gpinit = GpinitStandby()
self.mdd = os.environ.get("MASTER_DATA_DIRECTORY")
self.config = GPDBConfig()
self.pgutil = GpUtility()
self.verify = StandbyVerify()
self.host = socket.gethostname()
self.standby_loc = os.path.join(os.path.split(self.mdd)[0], self.standby_dirname)
self.standby = Standby(self.standby_loc, self.standby_port)
class OODTestCase(ScenarioTestCase):
''' Standby OOD when transactions in progress, standby need to be restaerted'''
def __init__(self, methodName):
self.gp = GpinitStandby()
super(OODTestCase,self).__init__(methodName)
def setUp(self):
#Remove standby if present
self.gp.run(option='-r')
def tearDown(self):
#Remove standby
self.gp.run(option='-r')
@unittest.skipIf(os.uname()[0] == 'Darwin', "Skipping this test on OSX")
def Dtest_ood_on_standby(self):
test_case_list0 = []
test_case_list0.append('mpp.gpdb.tests.storage.walrepl.ood.OODClass.initiate_standby')
self.test_case_scenario.append(test_case_list0)
test_case_list1 = []
test_case_list1.append('mpp.gpdb.tests.storage.walrepl.ood.OODClass.filldisk')
self.test_case_scenario.append(test_case_list1)
test_case_list2 = []
test_case_list2.append('mpp.gpdb.tests.storage.walrepl.ood.sql.runsql.RunWorkload')
self.test_case_scenario.append(test_case_list2)
test_case_list3 = []
test_case_list3.append('mpp.gpdb.tests.storage.walrepl.ood.OODClass.check_standby')
self.test_case_scenario.append(test_case_list3)
test_case_list4 = []
test_case_list4.append('mpp.gpdb.tests.storage.walrepl.ood.OODClass.restart_standby')
self.test_case_scenario.append(test_case_list4)
test_case_list5 = []
test_case_list5.append('mpp.gpdb.tests.storage.walrepl.ood.OODClass.cleanup')
self.test_case_scenario.append(test_case_list5)
class OOMGpstartTestCase(MPPTestCase):
''' Simulating oom on standby master , gpstart fails to start standby'''
# Setup OOM env
oom.setup_oom()
def __init__(self, methodName):
self.gpinitstandby = GpinitStandby()
self.gputil = GpUtility()
super(OOMGpstartTestCase, self).__init__(methodName)
def setUp(self):
self.gpinitstandby.run(option='-r')
def tearDown(self):
self.gpinitstandby.run(option='-r')
def test_oom_with_gpstart(self):
# Create standby
oom.create_standby()
# Stop cluster
self.gputil.run('gpstop -a')
# Run gpstart after creaitng the flag file- Presence of this flag file will return NULL for malloc simulating a no memory scenario
oom.touch_malloc()
self.assertFalse(
oom.startdb(),
'Standby start is expected to fail with no memory error')
# Restart after removing the flag file
self.assertTrue(oom.restartdb(),
'Expected a successful start of standby')
@unittest.skipIf(not config.is_multinode(),
"Test applies only to a multinode cluster")
def test_oom_with_psql_con(self):
# Create standby
oom.create_standby()
# Stop cluster
self.gputil.run('gpstop -a')
# Run gpstart with wrapper
oom.startdb()
# OOM with psql- after adding the flag file to the standby directory to simulate OOM
self.assertTrue(oom.psql_and_oom())
# Start standby after free up memory
self.assertTrue(oom.start_standby())
class WalRecTestCase(MPPTestCase):
''' Testcases for SIGTERM on wal receiver'''
def __init__(self, methodName):
self.gp = GpinitStandby()
self.mdd = os.environ.get('MASTER_DATA_DIRECTORY')
self.pgutil = GpUtility()
super(WalRecTestCase, self).__init__(methodName)
def setUp(self):
#Remove standby if present
self.gp.run(option='-r')
def invoke_sigterm_and_verify(self):
''' Invoke sigterm on wal receiver and verify that a new process is spawned after '''
gpact_stdby = GpactivateStandby()
standby_host = gpact_stdby.get_current_standby()
standby_port = gpact_stdby.get_standby_port()
wal_rec_pid_1 = self.pgutil.get_pid_by_keyword(
host=standby_host,
pgport=standby_port,
keyword='wal receiver process',
option='')
sig_cmd = "gpssh -h %s -e 'kill -15 %s'" % (standby_host,
wal_rec_pid_1)
cmd = Command('Issue SIGTERM to wam receiver process', cmdStr=sig_cmd)
tinctest.logger.info('%s' % cmd)
cmd.run(validateAfter=True)
result = cmd.get_results()
if result.rc != 0:
return False
wal_rec_pid_2 = self.pgutil.get_pid_by_keyword(
host=standby_host,
pgport=standby_port,
keyword='wal receiver process',
option='')
if wal_rec_pid_1 == wal_rec_pid_2:
return False
return True
@unittest.skipIf(not config.is_multinode(),
"Test applies only to a multinode cluster")
def test_sigterm_on_walreceiver(self):
self.pgutil.install_standby()
self.assertTrue(self.invoke_sigterm_and_verify())
def __init__(self,methodName):
self.gpact = GpactivateStandby()
self.gpinit = GpinitStandby()
self.runmixin = StandbyRunMixin()
self.runmixin.createdb(dbname=self.dbname)
self.bkup_timestamp = ""
self.gphome = os.environ.get('GPHOME')
self.mdd = os.environ.get('MASTER_DATA_DIRECTORY')
self.pgport = os.environ.get('PGPORT')
self.host = socket.gethostname()
self.standby_loc = os.path.split(self.mdd)[0]+self.standby_dirname
super(BkupRestore,self).__init__(methodName)
def test_gpactivatestandby_on_same_host(self):
''' Doesn't work due to STO-374'''
gputil.install_standby(new_stdby_host='localhost')
initstdby = GpinitStandby()
gpact_stdby = GpactivateStandby()
self.mdd = gpact_stdby.get_standby_dd()
self.port = gpact_stdby.get_standby_port()
self.standby_pid = gpact_stdby.get_standby_pid('localhost', self.port, self.mdd)
PSQL.run_sql_file(local_path('create_tables.sql'), dbname = self.db_name)
self.assertTrue(gpact_stdby.activate())
self.assertTrue(gpact_stdby.verify_gpactivatestandby(self.standby_pid, 'localhost', self.port, self.mdd))
gputil.failback_to_original_master(self.origin_mdd,socket.gethostname(),self.mdd,self.port)
class OOMGpstartTestCase(MPPTestCase):
''' Simulating oom on standby master , gpstart fails to start standby'''
# Setup OOM env
oom.setup_oom()
def __init__(self, methodName):
self.gpinitstandby= GpinitStandby()
self.gputil = GpUtility()
super(OOMGpstartTestCase,self).__init__(methodName)
def setUp(self):
self.gpinitstandby.run(option='-r')
def tearDown(self):
self.gpinitstandby.run(option='-r')
def test_oom_with_gpstart(self):
# Create standby
oom.create_standby()
# Stop cluster
self.gputil.run('gpstop -a')
# Run gpstart after creaitng the flag file- Presence of this flag file will return NULL for malloc simulating a no memory scenario
oom.touch_malloc()
self.assertFalse(oom.startdb(), 'Standby start is expected to fail with no memory error')
# Restart after removing the flag file
self.assertTrue(oom.restartdb(), 'Expected a successful start of standby')
@unittest.skipIf(not config.is_multinode(), "Test applies only to a multinode cluster")
def test_oom_with_psql_con(self):
# Create standby
oom.create_standby()
# Stop cluster
self.gputil.run('gpstop -a')
# Run gpstart with wrapper
oom.startdb()
# OOM with psql- after adding the flag file to the standby directory to simulate OOM
self.assertTrue(oom.psql_and_oom())
# Start standby after free up memory
self.assertTrue(oom.start_standby())
def test_promote_incomplete_stdby(self):
'''
remove the standby base dir, try promote and check if fail
'''
gpactivate_stdby = GpactivateStandby()
gpinit_stdby = GpinitStandby()
stdby_mdd = gpactivate_stdby.get_standby_dd()
stdby_host = gpinit_stdby.get_standbyhost()
stdby_port = gpactivate_stdby.get_standby_port()
destDir = os.path.join(stdby_mdd, 'base')
self.pgutil.clean_dir(stdby_host, destDir)
promote_cmd = "pg_ctl promote -D %s" % stdby_mdd
(rc, output) = gpactivate_stdby.run_remote(stdby_host, promote_cmd,
stdby_port, stdby_mdd)
self.assertEqual(rc, 0)
pid = self.pgutil.get_pid_by_keyword(host=stdby_host,
pgport=stdby_port,
keyword='master',
option='bin')
self.assertTrue(
int(pid) == -1,
'incomplete standby data directory promote succeeds.')
def __init__(self):
self.gpinit = GpinitStandby()
self.pgutil = GpUtility()
self.runmixin = StandbyRunMixin()
self.runmixin.createdb(dbname='walrepl')
self.gphome = os.environ.get('GPHOME')
self.pgport = os.environ.get('PGPORT')
self.mdd = os.environ.get('MASTER_DATA_DIRECTORY')
self.config = GPDBConfig()
self.host = socket.gethostname()
dburl = dbconn.DbURL()
gparray = GpArray.initFromCatalog(dburl, utility=True)
self.numcontent = gparray.getNumSegmentContents()
self.orig_master = gparray.master
class CancelBkndTestCase(MPPTestCase):
''' Testcases for pg_cancel_backed and pg_terminate_backed on wal sender'''
def __init__(self, methodName):
self.gp = GpinitStandby()
self.mdd = os.environ.get('MASTER_DATA_DIRECTORY')
self.standby = self.gp.get_standbyhostnode()
super(CancelBkndTestCase,self).__init__(methodName)
def setUp(self):
#Remove standby if present
self.gp.run(option='-r')
def get_walsender_pid(self):
''' get wal sender pid '''
pid_cmd = "ps -ef|grep \'%s\' |grep -v grep" % ('wal sender')
cmd = Command('Get the pid of the wal sender process', cmdStr = pid_cmd)
tinctest.logger.info ('%s' % cmd)
cmd.run(validateAfter=False)
result = cmd.get_results()
while(result.rc !=0):
cmd.run(validateAfter=False)
result = cmd.get_results()
tinctest.logger.info(result)
pid = result.stdout.splitlines()[0].split()[1].strip()
return pid
def cancel_query(self,query):
res = 't'
count = 0
while(res.strip() != 'f'):
res = PSQL.run_sql_command(query, dbname='postgres', flags = '-q -t')
if res.strip() == 'f':
return True
else:
sleep(2)
count+=1
if count > 120 :
return False
def terminate_backend(self):
wal_pid_1 = self.get_walsender_pid()
psql_command = "select pg_terminate_backend('%s')" % wal_pid_1
return(self.cancel_query(psql_command))
@unittest.skipIf(not config.is_multinode(), "Test applies only to a multinode cluster")
def test_terminate_bknd_wal_sender(self):
self.gp.create_dir_on_standby(self.standby, os.path.split(self.mdd)[0])
self.assertTrue(self.gp.run(option = '-s %s' % self.standby))
self.assertTrue(self.terminate_backend())
class CancelBkndTestCase(MPPTestCase):
''' Testcases for pg_cancel_backed and pg_terminate_backed on wal sender'''
def __init__(self, methodName):
self.gp = GpinitStandby()
self.mdd = os.environ.get('MASTER_DATA_DIRECTORY')
self.standby = self.gp.get_standbyhostnode()
super(CancelBkndTestCase,self).__init__(methodName)
def setUp(self):
#Remove standby if present
self.gp.run(option='-r')
def get_walsender_pid(self):
''' get wal sender pid '''
pid_cmd = "ps -ef|grep \'%s\' |grep -v grep" % ('wal sender')
cmd = Command('Get the pid of the wal sender process', cmdStr = pid_cmd)
tinctest.logger.info ('%s' % cmd)
cmd.run(validateAfter=False)
result = cmd.get_results()
while(result.rc !=0):
cmd.run(validateAfter=False)
result = cmd.get_results()
tinctest.logger.info(result)
pid = result.stdout.splitlines()[0].split()[1].strip()
return pid
def cancel_query(self,query):
res = 't'
count = 0
while(res.strip() != 'f'):
res = PSQL.run_sql_command(query, dbname='postgres', flags = '-q -t')
if res.strip() == 'f':
return True
else:
sleep(2)
count+=1
if count > 120 :
return False
def terminate_backend(self):
wal_pid_1 = self.get_walsender_pid()
psql_command = "select pg_terminate_backend('%s')" % wal_pid_1
return(self.cancel_query(psql_command))
@unittest.skipIf(not config.is_multinode(), "Test applies only to a multinode cluster")
def test_terminate_bknd_wal_sender(self):
self.gp.create_dir_on_standby(self.standby, os.path.split(self.mdd)[0])
self.assertTrue(self.gp.run(option = '-s %s' % self.standby))
self.assertTrue(self.terminate_backend())
def __init__(self, methodName):
self.gp = GpinitStandby()
super(SwitchXlogTestCase, self).__init__(methodName)
def setUp(self):
self.gp = GpinitStandby()
self.gp.run(option='-r')
class GpactivateStandby(object):
'''Class for gpactivatestandby operations '''
standby_port = '5656'
db_name = 'walrepl'
def __init__(self):
self.gpinit = GpinitStandby()
self.pgutil = GpUtility()
self.runmixin = StandbyRunMixin()
self.runmixin.createdb(dbname='walrepl')
self.gphome = os.environ.get('GPHOME')
self.pgport = os.environ.get('PGPORT')
self.mdd = os.environ.get('MASTER_DATA_DIRECTORY')
self.config = GPDBConfig()
self.host = socket.gethostname()
dburl = dbconn.DbURL()
gparray = GpArray.initFromCatalog(dburl, utility=True)
self.numcontent = gparray.getNumSegmentContents()
self.orig_master = gparray.master
def run_remote(self, standbyhost, rmt_cmd, pgport='', standbydd=''):
'''Runs remote command and returns rc, result '''
export_cmd = "source %s/greenplum_path.sh;export PGPORT=%s;export MASTER_DATA_DIRECTORY=%s" % (
self.gphome, pgport, standbydd)
remote_cmd = "gpssh -h %s -e '%s; %s'" % (standbyhost, export_cmd,
rmt_cmd)
cmd = Command(name='Running Remote command', cmdStr='%s' % remote_cmd)
tinctest.logger.info(" %s" % cmd)
cmd.run(validateAfter=False)
result = cmd.get_results()
return result.rc, result.stdout
def activate(self, option=''):
''' Stop the master and activate current standby to master'''
standby_host = self.get_current_standby()
standby_port = self.get_standby_port()
standby_loc = self.get_standby_dd()
self.run_remote(self.host,
'gpstop -aim',
pgport=self.pgport,
standbydd=self.mdd)
gpactivate_cmd = 'gpactivatestandby -a -d %s %s' % (standby_loc,
option)
(rc, result) = self.run_remote(standby_host,
gpactivate_cmd,
pgport=standby_port,
standbydd=standby_loc)
tinctest.logger.info(
'Result without force option to activate standby %s' % result)
if (rc != 0) and result.find('Force activation required') != -1:
tinctest.logger.info(
'activating standby failed, try force activation...')
gpactivate_cmd = 'gpactivatestandby -a -f -d %s %s' % (standby_loc,
option)
(rc, result) = self.run_remote(standby_host,
gpactivate_cmd,
pgport=standby_port,
standbydd=standby_loc)
if (rc != 0):
tinctest.logger.error('Force activating standby failed!')
return False
tinctest.logger.info('standby acvitated, host value %s' % standby_host)
return True
def remove_standby(self):
return self.gpinit.run(option='-r')
def failback_to_original_master(self):
# Check if master is running.
bashCmd = (
self.gphome
) + '/bin/pg_ctl status -D $MASTER_DATA_DIRECTORY | grep \'pg_ctl: server is running\''
cmd = Command(name='Running cmd %s' % bashCmd,
cmdStr="source %s/greenplum_path.sh; %s" %
(self.gphome, bashCmd))
try:
cmd.run()
except Exception, e:
tinctest.logger.error("Error running command %s\n" % e)
return
result = cmd.get_results()
out = result.stdout
if not out:
tinctest.logger.info('Start the old master again ...')
master = gp.MasterStart("Starting orig Master",
self.orig_master.datadir,
self.orig_master.port,
self.orig_master.dbid, 0, self.numcontent,
None, None, None)
master.run(validateAfter=True)
result = master.get_results()
tinctest.logger.info('orig Master started result : %s' %
result.stdout)
if result.rc != 0:
raise WalReplException(
'Unable to start original master process')
Command('gpinitstandby -ra', 'gpinitstandby -ra').run()
# failing back to old master, it takes a little bit to prepare the cluster ready for connection
if os.path.exists(local_path('drop_filespace.sql')):
PSQL.run_sql_file(local_path('drop_filespace.sql'),
dbname=self.db_name)
def __init__(self, methodName):
self.gp = GpinitStandby()
self.mdd = os.environ.get('MASTER_DATA_DIRECTORY')
self.standby = self.gp.get_standbyhostnode()
super(CancelBkndTestCase,self).__init__(methodName)
def __init__(self, methodName):
self.gp = GpinitStandby()
super(LoadClass, self).__init__(methodName)
class GpinitStandsbyTestCase(MPPTestCase):
''' Testcases for gpinitstandby'''
db_name = 'walrepl'
standby_port = '5433'
standby_dirname = 'newstandby'
def __init__(self, methodName):
self.gp = GpinitStandby()
self.host = socket.gethostname()
self.mdd = os.environ.get('MASTER_DATA_DIRECTORY')
self.standby = self.gp.get_standbyhostnode()
self.standby_loc = os.path.join(os.path.split(self.mdd)[0],
self.standby_dirname)
self.primary_pid = 0
super(GpinitStandsbyTestCase,self).__init__(methodName)
def setUp(self):
#Remove standby if present
self.primary_pid = self.gp.get_primary_pid()
self.gp.run(option='-r')
def tearDown(self):
# Cleanup Filespaces
walrepl.cleanupFilespaces(dbname=self.db_name)
def create_directory(self,location):
pgutil.clean_dir(self.standby,os.path.split(location)[0])
pgutil.create_dir(self.standby,os.path.split(location)[0])
def create_unmodifiable_file(self, filename):
filepath = os.path.join(self.mdd, filename)
with open(filepath, 'w') as fp:
pass
os.chmod(filepath, 000)
def remove_unmodifiable_file(self, filename):
filepath = os.path.join(self.mdd, filename)
os.chmod(filepath, 777)
os.remove(filepath)
def touch_file(self, filename):
file_dir = os.path.split(filename)[0]
if not os.path.exists(file_dir):
os.makedirs(file_dir)
with open(filename, 'w') as fp:
pass
def test_gpinitstandby_exclude_dirs(self):
"""
Test pg_basebackup exclusions when copying filespaces from
the master to the standby during gpinitstandby
"""
os.makedirs(self.mdd + '/db_dumps')
self.touch_file(self.mdd + '/db_dumps/testfile')
self.touch_file(self.mdd + '/gpperfmon/logs/test.log')
self.touch_file(self.mdd + '/gpperfmon/data/testfile')
self.touch_file(self.mdd + '/pg_log/testfile')
self.gp.run(option = '-P %s -s %s -F pg_system:%s' % (self.standby_port, self.host, self.standby_loc))
shutil.rmtree(self.mdd + '/db_dumps')
os.remove(self.mdd + '/gpperfmon/logs/test.log')
os.remove(self.mdd + '/gpperfmon/data/testfile')
os.remove(self.mdd + '/pg_log/testfile')
self.assertFalse(os.path.exists(self.standby_loc + '/db_dumps/testfile'))
self.assertFalse(os.path.exists(self.standby_loc + '/gpperfmon/logs/test.log'))
self.assertFalse(os.path.exists(self.standby_loc + '/gpperfmon/data/testfile'))
self.assertFalse(os.path.exists(self.standby_loc + '/pg_log/testfile'))
self.assertTrue(self.gp.run(option = '-r'))
@unittest.skipIf(not config.is_multinode(), "Test applies only to a multinode cluster")
def test_gpinitstanby_to_new_host(self):
self.create_directory(self.mdd)
self.assertTrue(self.gp.run(option = '-s %s' % self.standby))
self.assertTrue(self.gp.verify_gpinitstandby(self.primary_pid))
def test_gpinitstandby_to_same_host_new_port_and_new_mdd(self):
pgutil.clean_dir(self.host,self.standby_loc)
self.assertTrue(self.gp.run(option = '-P %s -s %s -F pg_system:%s' % (self.standby_port, self.host, self.standby_loc)))
self.assertTrue(self.gp.verify_gpinitstandby(self.primary_pid))
@unittest.skipIf(not config.is_multinode(), "Test applies only to a multinode cluster")
def test_gpinitstandby_new_host_new_port(self):
self.create_directory(self.mdd)
self.assertTrue(self.gp.run(option = '-P %s -s %s' % (self.standby_port, self.standby)))
self.assertTrue(self.gp.verify_gpinitstandby(self.primary_pid))
@unittest.skipIf(not config.is_multinode(), "Test applies only to a multinode cluster")
def test_gpinitstandby_new_host_new_mdd(self):
self.create_directory(self.standby_loc)
self.assertTrue(self.gp.run(option = '-F pg_system:%s -s %s' % (self.standby_loc, self.standby)))
self.assertTrue(self.gp.verify_gpinitstandby(self.primary_pid))
def test_gpinitstandby_to_same_with_filespaces(self):
from mpp.lib.gpfilespace import Gpfilespace
gpfile = Gpfilespace()
gpfile.create_filespace('fs_walrepl_a')
PSQL.run_sql_file(local_path('filespace.sql'), dbname = self.db_name)
filespace_loc = self.gp.get_filespace_location()
filespace_loc = os.path.join(os.path.split(filespace_loc)[0], 'newstandby')
filespaces = "pg_system:%s,fs_walrepl_a:%s" %(self.standby_loc , filespace_loc)
self.assertTrue(self.gp.run(option = '-F %s -s %s -P %s' % (filespaces, self.host, self.standby_port)))
self.assertTrue(self.gp.verify_gpinitstandby(self.primary_pid))
@unittest.skipIf(not config.is_multinode(), "Test applies only to a multinode cluster")
def test_gpinitstandby_new_host_with_filespace(self):
from mpp.lib.gpfilespace import Gpfilespace
gpfile = Gpfilespace()
gpfile.create_filespace('fs_walrepl_a')
PSQL.run_sql_file(local_path('filespace.sql'), dbname = self.db_name)
filespace_loc = self.gp.get_filespace_location()
self.create_directory(filespace_loc)
filespaces = "pg_system:%s,fs_walrepl_a:%s" % (self.mdd, filespace_loc)
self.assertTrue(self.gp.run(option = '-F %s -s %s -P %s' % (filespaces, self.standby, self.standby_port)))
self.assertTrue(self.gp.verify_gpinitstandby(self.primary_pid))
def test_gpinitstandby_remove_from_same_host(self):
#self.gp.create_dir_on_standby(self.host, self.standby_loc)
pgutil.clean_dir(self.host,self.standby_loc)
self.gp.run(option = '-P %s -s %s -F pg_system:%s' % (self.standby_port, self.host, self.standby_loc))
self.assertTrue(self.gp.run(option = '-r'))
@unittest.skipIf(not config.is_multinode(), "Test applies only to a multinode cluster")
def test_gpinitstandby_remove_from_new_host(self):
self.create_directory(self.mdd)
self.gp.run(option = '-s %s' % self.standby)
self.assertTrue(self.gp.run(option = '-r'))
def test_gpinitstandby_n_with_no_standby_stop(self):
pgutil.clean_dir(self.host,self.standby_loc)
self.gp.run(option = '-P %s -s %s -F pg_system:%s' % (self.standby_port, self.host, self.standby_loc))
(rc,out) = pgutil.run(command='gpinitstandby -n')
self.assertTrue('Standy master is already up and running' in out)
def test_gpinitstandby_n_with_standby_stop(self):
pgutil.clean_dir(self.host,self.standby_loc)
self.gp.run(option = '-P %s -s %s -F pg_system:%s' % (self.standby_port, self.host, self.standby_loc))
self.standby = Standby(self.standby_loc,
self.standby_port)
self.standby.stop()
(rc,out) = pgutil.run(command='gpinitstandby -n')
self.assertTrue('Successfully started standby master' in out)
def test_gpinitstandby_with_no_default_path(self):
self.assertTrue(self.gp.initstand_by_with_default())
@unittest.skipIf(not config.is_multinode(), "Test applies only to a multinode cluster")
def test_gpinitstandby_prompt_for_filespace(self):
from mpp.lib.gpfilespace import Gpfilespace
gpfile = Gpfilespace()
gpfile.create_filespace('fs_walrepl_a')
PSQL.run_sql_file(local_path('filespace.sql'), dbname = self.db_name)
filespace_loc = self.gp.get_filespace_location()
self.create_directory(filespace_loc)
self.assertTrue(self.gp.init_with_prompt(filespace_loc))
def test_gpinistandby_with_M(self):
'''
@product_version gpdb: [4.3.1.0-MAIN]
'''
pgutil.clean_dir(self.host,self.standby_loc)
self.assertTrue(self.gp.run(option = '-P %s -s %s -F pg_system:%s -M smart' % (self.standby_port, self.host, self.standby_loc)))
self.assertTrue(self.gp.run(option = '-r'))
def test_gpinitstandby_to_same_host_new_port_and_new_mdd_with_error(self):
'''
@product_version gpdb: [4.3.6.2 - 4.3.99.0)
'''
pgutil.clean_dir(self.host,self.standby_loc)
try:
self.create_unmodifiable_file('foo')
(rc, out) = pgutil.run(command='gpinitstandby -a -P %s -s %s -F pg_system:%s' % (self.standby_port, self.host, self.standby_loc))
self.assertTrue('ERROR: could not open file "./foo": Permission denied' in out)
self.assertFalse('server closed the connection unexpectedly' in out)
finally:
self.remove_unmodifiable_file('foo')
pgutil.clean_dir(self.host,self.standby_loc)
def __init__(self, methodName):
self.gp = GpinitStandby()
self.mdd = os.environ.get('MASTER_DATA_DIRECTORY')
self.standby = self.gp.get_standbyhostnode()
super(CancelBkndTestCase, self).__init__(methodName)
def __init__(self, methodName):
self.gpinitstandby = GpinitStandby()
self.gputil = GpUtility()
super(OOMGpstartTestCase, self).__init__(methodName)
class BkupRestore(MPPTestCase):
dbname = 'bkdb'
standby_port = '5433'
standby_dirname = '_newstandby'
def __init__(self, methodName):
self.gpact = GpactivateStandby()
self.gpinit = GpinitStandby()
self.runmixin = StandbyRunMixin()
self.runmixin.createdb(dbname=self.dbname)
self.bkup_timestamp = ""
self.gphome = os.environ.get('GPHOME')
self.mdd = os.environ.get('MASTER_DATA_DIRECTORY')
self.pgport = os.environ.get('PGPORT')
self.host = socket.gethostname()
self.standby_loc = os.path.split(self.mdd)[0] + self.standby_dirname
super(BkupRestore, self).__init__(methodName)
def createdb(self):
PSQL.run_sql_command('Drop database %s;Create database %s;' %
(self.dbname, self.dbname),
dbname='postgres')
def run_backup(self):
#Cleanup db_dumps folder before running backup
cleanup_cmd = "gpssh -h %s -e 'rm -rf /tmp/db_dumps'" % (self.host)
cmd = Command('cleanup', cleanup_cmd)
cmd.run(validateAfter=False)
gpc_cmd = 'gpcrondump -a -x %s -u /tmp' % self.dbname
cmd = Command(name='Run gpcrondump', cmdStr=gpc_cmd)
cmd.run(validateAfter=True)
result = cmd.get_results()
if result.rc != 0:
return False
else:
self.bkup_timestamp = self.get_timestamp(result.stdout)
return True
def validate_timestamp(self, ts):
try:
int_ts = int(ts)
except Exception as e:
raise Exception('Timestamp is not valid %s' % ts)
if len(ts) != 14:
raise Exception('Timestamp is invalid %s' % ts)
def get_timestamp(self, result):
for line in result.splitlines():
if 'Timestamp key = ' in line:
log_msg, delim, timestamp = line.partition('=')
ts = timestamp.strip()
self.validate_timestamp(ts)
return ts
raise Exception('Timestamp key not found')
def run_restore(self):
gpr_cmd = 'gpdbrestore -t %s -a -u /tmp' % self.bkup_timestamp
(rc, result) = self.run_remote(self.host,
gpr_cmd,
pgport=self.standby_port,
standbydd=self.standby_loc)
if rc != 0:
return False
return True
def create_standby(self, local=True):
''' Create a standby '''
gputil.create_dir(self.host, self.standby_loc)
gputil.clean_dir(self.host, self.standby_loc)
self.gpinit.run(option='-P %s -s %s -F pg_system:%s' %
(self.standby_port, self.host, self.standby_loc))
def run_remote(self, standbyhost, rmt_cmd, pgport='', standbydd=''):
'''Runs remote command and returns rc, result '''
export_cmd = "source %s/greenplum_path.sh;export PGPORT=%s;export MASTER_DATA_DIRECTORY=%s" % (
self.gphome, pgport, standbydd)
remote_cmd = "gpssh -h %s -e '%s;%s' " % (standbyhost, export_cmd,
rmt_cmd)
cmd = Command(name='Running Remote command', cmdStr='%s' % remote_cmd)
tinctest.logger.info(" %s" % cmd)
cmd.run(validateAfter=False)
result = cmd.get_results()
return result.rc, result.stdout
def run_workload(self, dir, verify=False):
tinctest.logger.info("Running workload ...")
load_path = local_path(dir) + os.sep
for file in os.listdir(load_path):
if file.endswith(".sql"):
out_file = file.replace(".sql", ".out")
PSQL.run_sql_file(sql_file=load_path + file,
dbname=self.dbname,
port=self.pgport,
out_file=load_path + out_file)
if verify == True:
self.validate_sql_files(load_path)
def validate_sql_files(self, load_path):
for file in os.listdir(load_path):
if file.endswith(".out"):
out_file = file
ans_file = file.replace('.out', '.ans')
if os.path.exists(load_path + ans_file):
assert Gpdiff.are_files_equal(load_path + out_file,
load_path + ans_file)
else:
raise Exception("No .ans file exists for %s " % out_file)
def failback(self):
gputil.failback_to_original_master(self.mdd, self.host,
self.standby_loc, self.standby_port)
class gpactivatestandby(mpp.gpdb.tests.storage.walrepl.run.StandbyRunMixin, mpp.gpdb.tests.storage.walrepl.lib.PreprocessFileMixin,
SQLTestCase):
"""
@gucs gp_create_table_random_default_distribution=off
"""
sql_dir = 'sql'
ans_dir = 'ans'
out_dir = 'output'
standby_dir = ''
standby_host = ''
standby_port = ''
@unittest.skipIf(config.is_multinode(), "Test applies only to a singlenode cluster")
def setUp(self):
# For each test case we create a fresh standby and start it.
self.db_name = self.__class__.db_name
self.createdb(self.db_name)
gputil.remove_standby()
gputil.install_standby(new_stdby_host=socket.gethostname())
self.gpinit_stdby = GpinitStandby()
self.activatestdby = GpactivateStandby()
def tearDown(self):
# Clean up filespaces to be tidy. Although we want to preserve
# it for the investigation in case of test failure, without cleaning
# it up it will prevent next tests from running correctly.
gputil.failback_to_original_master(origin_mdd,self.standby_host,self.standby_dir,self.standby_port)
# remove standby
cmd = Command('gpinitstandby', 'gpinitstandby -ar')
cmd.run()
def get_gp_dbid(self,standby_dd=''):
# We use gppylib from the installation
from gppylib.gp_dbid import GpDbidFile
return GpDbidFile(standby_dd, True).dbid
def run_test(self):
"""
Override of SQLTestCase. Initialize standby, run some sql,
then activate it, and check if the data is streamed correctly.
"""
sql_file = self.sql_file
ans_file = self.ans_file
nsender = self.wait_for_walsender()
self.assertEqual(nsender, 1, 'replication has not begun')
# setup script is run on primary while standby is running.
# .in file will be substitute with runtime information, if any.
setup_file = sql_file.replace('.sql', '_setup.sql')
if os.path.exists(setup_file + '.in'):
self.preprocess_file(setup_file + '.in')
self.assertTrue(PSQL.run_sql_file(setup_file, dbname=self.db_name))
self.standby_dir = self.activatestdby.get_standby_dd()
self.standby_port = self.activatestdby.get_standby_port()
self.standby_host = self.gpinit_stdby.get_standbyhost()
self.activatestdby.activate()
datadir = os.path.abspath(self.standby_datadir)
with walrepl.NewEnv(MASTER_DATA_DIRECTORY=self.standby_dir,
PGPORT=self.standby_port) as env:
result = super(gpactivatestandby, self).run_test()
sql = 'SHOW gp_dbid'
result = PSQL.run_sql_command(sql, flags='-A -t')
self.assertEqual(result.strip(), '1')
self.assertEqual(self.get_gp_dbid(self.standby_dir), 1, 'gp_dbid should show 1')
if 'cleanup_filespace' in self._metadata:
mpp.gpdb.tests.storage.walrepl.lib.cleanupFilespaces(dbname=self.db_name)
return result
def __init__(self, methodName):
self.gp = GpinitStandby()
super(LoadTestCase,self).__init__(methodName)
class OOMClass(object):
'''Class for methods required for OOM testcase'''
standby_port = '5433'
standby_dirname = 'newstandby'
def __init__(self):
self.gpinit = GpinitStandby()
self.mdd = os.environ.get('MASTER_DATA_DIRECTORY')
self.config = GPDBConfig()
self.pgutil = GpUtility()
self.verify = StandbyVerify()
self.host = socket.gethostname()
self.standby_loc = os.path.join(os.path.split(self.mdd)[0],
self.standby_dirname)
self.standby = Standby(self.standby_loc, self.standby_port)
def create_standby(self):
self.pgutil.clean_dir(self.host,self.standby_loc)
self.gpinit.run(option = '-P %s -s %s -F pg_system:%s' % (self.standby_port, self.host, self.standby_loc))
def setup_oom(self):
# Build it before testing.
thisdir = os.path.dirname(__file__)
builddir = os.path.join(thisdir, 'lib')
subprocess.check_call(['make', '-C', builddir, 'install'])
#Copy oom_malloc.so and wrapper.sh to all the segment nodes
for host in config.get_hosts() :
if host.strip() == self.host :
continue
cmd = "gpssh -h %s -e 'mkdir -p %s'; scp %s/oom_malloc.so %s:%s/; scp %s/wrapper.sh %s:%s/" % (host.strip(), builddir, builddir, host.strip(), builddir, builddir, host.strip(), builddir)
self.pgutil.run(cmd)
def touch_malloc(self):
# Touch file oom_malloc in standby directory
cmd = 'touch %s/oom_malloc' % self.standby_loc
self.pgutil.run(cmd)
def startdb(self):
(rc, result) = self.pgutil.run('gpstart -a --wrapper %s' % (local_path('lib/wrapper.sh')))
if rc != 0 and 'Could not start standby master' in result :
return False
return True
def restartdb(self):
# Remove file oom_malloc from standby
cmd = 'rm %s/oom_malloc' % self.standby_loc
self.pgutil.run(cmd)
(rc, result) = self.pgutil.run('gpstop -ar')
if rc == 0 and (self.verify.check_pg_stat_replication()):
return True
return False
def psql_and_oom(self):
#Touch oom_malloc in standby_dir and issue PSQL : Check if processes are gone
self.touch_malloc()
PSQL.run_sql_command('Drop table if exists wal_oomt1;Create table wal_oomt1(a1 int, a2 text) with(appendonly=true);')
sleep(2)
if not (self.verify.check_standby_processes()):
return True
return False
def start_standby(self):
# Remove oom_malloc and start standby : Check if all processes are back
cmd = 'rm %s/oom_malloc' % self.standby_loc
self.pgutil.run(cmd)
res = self.standby.start()
sleep(2)
if (self.verify.check_standby_processes()) :
return True
return False
def setUp(self):
self.gp = GpinitStandby()
self.gp.run(option='-r')
def __init__(self, methodName):
self.gp = GpinitStandby()
super(OODTestCase, self).__init__(methodName)
def setUp(self):
# Remove standby if present
GpinitStandby().run(option='-r')
super(smart_shutdown, self).setUp()
class OOMClass(object):
'''Class for methods required for OOM testcase'''
standby_port = '5433'
standby_dirname = 'newstandby'
def __init__(self):
self.gpinit = GpinitStandby()
self.mdd = os.environ.get('MASTER_DATA_DIRECTORY')
self.config = GPDBConfig()
self.pgutil = GpUtility()
self.verify = StandbyVerify()
self.host = socket.gethostname()
self.standby_loc = os.path.join(
os.path.split(self.mdd)[0], self.standby_dirname)
self.standby = Standby(self.standby_loc, self.standby_port)
def create_standby(self):
self.pgutil.clean_dir(self.host, self.standby_loc)
self.gpinit.run(option='-P %s -s %s -F pg_system:%s' %
(self.standby_port, self.host, self.standby_loc))
def setup_oom(self):
# Build it before testing.
thisdir = os.path.dirname(__file__)
builddir = os.path.join(thisdir, 'lib')
subprocess.check_call(['make', '-C', builddir, 'install'])
#Copy oom_malloc.so and wrapper.sh to all the segment nodes
for host in config.get_hosts():
if host.strip() == self.host:
continue
cmd = "gpssh -h %s -e 'mkdir -p %s; scp %s/oom_malloc.so %s:%s/;scp %s/wrapper.sh %s:%s/'" % (
host.strip(), builddir, builddir, host.strip(), builddir,
builddir, host.strip(), builddir)
self.pgutil.run(cmd)
def touch_malloc(self):
# Touch file oom_malloc in standby directory
cmd = 'touch %s/oom_malloc' % self.standby_loc
self.pgutil.run(cmd)
def startdb(self):
(rc, result) = self.pgutil.run('gpstart -a --wrapper %s' %
(local_path('lib/wrapper.sh')))
if rc != 0 and 'Could not start standby master' in result:
return False
return True
def restartdb(self):
# Remove file oom_malloc from standby
cmd = 'rm %s/oom_malloc' % self.standby_loc
self.pgutil.run(cmd)
(rc, result) = self.pgutil.run('gpstop -ar')
if rc == 0 and (self.verify.check_pg_stat_replication()):
return True
return False
def psql_and_oom(self):
#Touch oom_malloc in standby_dir and issue PSQL : Check if processes are gone
self.touch_malloc()
PSQL.run_sql_command(
'Drop table if exists wal_oomt1;Create table wal_oomt1(a1 int, a2 text) with(appendonly=true);'
)
sleep(2)
if not (self.verify.check_standby_processes()):
return True
return False
def start_standby(self):
# Remove oom_malloc and start standby : Check if all processes are back
cmd = 'rm %s/oom_malloc' % self.standby_loc
self.pgutil.run(cmd)
res = self.standby.start()
sleep(2)
if (self.verify.check_standby_processes()):
return True
return False
def __init__(self, methodName):
self.gp = GpinitStandby()
self.mdd = os.environ.get('MASTER_DATA_DIRECTORY')
self.pgutil = GpUtility()
super(WalRecTestCase, self).__init__(methodName)
def __init__(self,methodName):
self.gp =GpinitStandby()
super(LoadClass,self).__init__(methodName)
def __init__(self, methodName):
self.gpinitstandby= GpinitStandby()
self.gputil = GpUtility()
super(OOMGpstartTestCase,self).__init__(methodName)
