class SwitchClass(MPPTestCase): DESTDIR = 'base' def __init__(self, methodName): self.standby = Standby(self.DESTDIR, 5433) super(SwitchClass, self).__init__(methodName) def run_pg_basebackup(self): shutil.rmtree(self.DESTDIR, True) self.standby.create() def start_standby(self): res = self.standby.start() self.assertTrue(res.wasSuccessful()) self.assertTrue(self.check_walreceiver()) def check_walreceiver(self): grep_cmd = 'ps -ef|grep %s|grep -v grep' % ('wal receiver') cmd = Command(name='Check if no wal receiver process', cmdStr=grep_cmd) tinctest.logger.info('%s' % cmd) cmd.run(validateAfter=False) result = cmd.get_results() if result.rc != 0: return True return False
class SwitchClass(MPPTestCase): DESTDIR = 'base' def __init__(self,methodName): self.standby = Standby(self.DESTDIR, 5433) super(SwitchClass,self).__init__(methodName) def run_pg_basebackup(self): shutil.rmtree(self.DESTDIR, True) self.standby.create() def start_standby(self): res = self.standby.start() self.assertTrue(res.wasSuccessful()) self.assertTrue(self.check_walreceiver()) def check_walreceiver(self): grep_cmd = 'ps -ef|grep %s|grep -v grep' % ('wal receiver') cmd = Command(name='Check if no wal receiver process', cmdStr=grep_cmd) tinctest.logger.info ('%s' % cmd) cmd.run(validateAfter=False) result = cmd.get_results() if result.rc !=0: return True return False
def __init__(self): self.gpinit = GpinitStandby() self.mdd = os.environ.get('MASTER_DATA_DIRECTORY') self.config = GPDBConfig() self.pgutil = GpUtility() self.verify = StandbyVerify() self.host = socket.gethostname() self.standby_loc = os.path.join( os.path.split(self.mdd)[0], self.standby_dirname) self.standby = Standby(self.standby_loc, self.standby_port)
def setUp(self): """ Override """ # Removing standby as these tests will fail if there is a standby enabled # Ignore any failures run_shell_command(cmdstr='gpinitstandby -ra', cmdname='remove standby') self.createdb(self.db_name) self.standby = Standby(self.standby_datadir, self.standby_port)
def __init__(self): self.gpinit = GpinitStandby() self.mdd = os.environ.get("MASTER_DATA_DIRECTORY") self.config = GPDBConfig() self.pgutil = GpUtility() self.verify = StandbyVerify() self.host = socket.gethostname() self.standby_loc = os.path.join(os.path.split(self.mdd)[0], self.standby_dirname) self.standby = Standby(self.standby_loc, self.standby_port)
class basebackup_filespace(StandbyRunMixin, MPPTestCase, PreprocessFileMixin): def setUp(self): """ Override """ # Removing standby as these tests will fail if there is a standby enabled # Ignore any failures run_shell_command(cmdstr='gpinitstandby -ra', cmdname='remove standby') self.createdb(self.db_name) self.standby = Standby(self.standby_datadir, self.standby_port) def tearDown(self): """ Override """ try: dburl = dbconn.DbURL() self.standby.remove_catalog_standby(dburl) except Exception: pass cleanupFilespaces(dbname=self.db_name) def test_filespace(self): """ pg_basebackup should work with user filespace. @tags sanity """ # Add standby entry in catalog before regisering filespace. fsbase = os.path.join(self.fsprefix, 'fs') shutil.rmtree(fsbase, True) os.makedirs(fsbase) shutil.rmtree(self.standby_datadir, True) dburl = dbconn.DbURL() gparray = GpArray.initFromCatalog(dburl, utility=True) if gparray.standbyMaster: self.standby.remove_catalog_standby(dburl) self.standby.add_catalog_standby(dburl, gparray) #self.preprocess_file(local_path('filespace.sql.in')) sql_file = local_path('filespace.sql') from mpp.lib.gpfilespace import Gpfilespace gpfile = Gpfilespace() gpfile.create_filespace('fs_walrepl_a') result = PSQL.run_sql_file(sql_file, dbname=self.db_name) self.assertTrue(result) subprocess.check_call(['pg_basebackup', '-D', self.standby_datadir]) #fsdir = os.path.join(self.fsprefix, 'fs', 'gpdb1') fsdir = os.path.join(os.path.split(self.standby_datadir)[0], 'fs_walrepl_a','mirror','pg_system') self.assertTrue(os.path.isdir(fsdir), '{0} does not dir'.format(fsdir))
class basebackup_filespace(StandbyRunMixin, MPPTestCase, PreprocessFileMixin): def setUp(self): """ Override """ # Removing standby as these tests will fail if there is a standby enabled # Ignore any failures run_shell_command(cmdstr='gpinitstandby -ra', cmdname='remove standby') self.createdb(self.db_name) self.standby = Standby(self.standby_datadir, self.standby_port) def tearDown(self): """ Override """ try: dburl = dbconn.DbURL() self.standby.remove_catalog_standby(dburl) except Exception: pass cleanupFilespaces(dbname=self.db_name) def test_filespace(self): """ pg_basebackup should work with user filespace. @tags sanity """ # Add standby entry in catalog before regisering filespace. fsbase = os.path.join(self.fsprefix, 'fs') shutil.rmtree(fsbase, True) os.makedirs(fsbase) shutil.rmtree(self.standby_datadir, True) dburl = dbconn.DbURL() gparray = GpArray.initFromCatalog(dburl, utility=True) if gparray.standbyMaster: self.standby.remove_catalog_standby(dburl) self.standby.add_catalog_standby(dburl, gparray) #self.preprocess_file(local_path('filespace.sql.in')) sql_file = local_path('filespace.sql') from mpp.lib.gpfilespace import Gpfilespace gpfile = Gpfilespace() gpfile.create_filespace('fs_walrepl_a') result = PSQL.run_sql_file(sql_file, dbname=self.db_name) self.assertTrue(result) subprocess.check_call(['pg_basebackup', '-D', self.standby_datadir]) #fsdir = os.path.join(self.fsprefix, 'fs', 'gpdb1') fsdir = os.path.join( os.path.split(self.standby_datadir)[0], 'fs_walrepl_a', 'mirror', 'pg_system') self.assertTrue(os.path.isdir(fsdir), '{0} does not dir'.format(fsdir))
import subprocess import sys from mpp.gpdb.tests.storage.walrepl.lib.standby import Standby def pline(msg, cmd=None): sys.stdout.write('----' + msg + '\n') if cmd: sys.stdout.write(cmd + '\n') sys.stdin.readline() if __name__ == '__main__': mpp.gpdb.tests.storage.walrepl.lib.cleanupFilespaces(dbname='haradh1') standby = Standby('standby', '5433') standby.stop() standby.stop() shutil.rmtree('standby', True) sys.stdout.write(''' *********************************************************** *** Demo for Filespace support *********************************************************** ''') pline('Initialize standby info in catalog and take online backup') standby.create() subprocess.call(['ls', 'standby']) subprocess.call(['psql', '-c', 'select * from gp_segment_configuration'])
def test_smart_shutdown(self): # 1. Verify if the system is UP and there is no WAL Receiver running # 2. Perform basebackup and deploy it into some dest. directory # 3. Copy recover.conf into the dest. directory to be used by Standby # 4. Initiate the Standby using the Master (primary) postmaster # paramerters # 5. Perform some transaction to generate xlog. Then do a smart shutdown # 6. Once the primary DB is down, find the last checkpoint from pg_control # on primary. Check the last modified xlog seg from the standby and find # if the last checkpoint from primary exists # 7. It should be present there! # 0. Stop standby if it's running PSQL.run_sql_command('DROP table if exists foo') standby = Standby('base', 5433) standby.stop() # 1. Verify if the system is UP and there is no WAL sender running self.assertEqual(self.count_walsender(), 0) logger.info('No active WAL Receiver found') # 2. Perform basebackup and deploy it into some dest. # (currenttly hardcoded 'base') directory shutil.rmtree('base', True) logger.info('Performing and deploying base backup ...') standby.create() # 3.Copy recover.conf into the dest. directory to be used by StandBy logger.info('Deploying recovery.conf...') # 4. Initiate the StandBy using the Master (primary) postmaster # paramerters logger.info('Initiating Standby...') res = standby.start() self.assertTrue(res.wasSuccessful()) num_walsender = 0 for i in polling(10, 0.5): num_walsender = self.count_walsender() if num_walsender > 0: break self.assertEqual(num_walsender, 1) logger.info('Activated WAL Receiver...') # 5. Perform some transaction to generate xlog. Then do a smart shutdown logger.info('Perform some transaction to generate some XLOG') PSQL.run_sql_command('Create table foo (a int)') logger.info('Now perform smart shutdown (gpstop -a)') cmd = Command(name="gpstop smart", cmdStr="source %s/greenplum_path.sh;\ gpstop -a" % os.environ["GPHOME"]) cmd.run(validateAfter=True) # 6. Once the primary DB is down, find the last checkpoint from pg_control # on primary. Check the last modified xlog seg from the standby and find # if the last checkpoint from primary exists logger.info('Read the pg_control from primary, find the last checkpoint & see if it made to standby') standby_xlog_path = os.path.join('base','pg_xlog') cmd= Command(name = 'pg_controldata ' + os.environ.get('MASTER_DATA_DIRECTORY'), cmdStr = 'pg_controldata ' + os.environ.get('MASTER_DATA_DIRECTORY')) cmd.run(validateAfter=True); primary_last_ckpt_lsn = self.last_ckpt_lsn((cmd.get_results()).stdout) logger.info ("Primary last checkpoint LSN = " + primary_last_ckpt_lsn) standby_last_mod_xlog = self.last_mod_file(standby_xlog_path) logger.info ("Last mod standby XLOG = " + standby_last_mod_xlog ) cmd= Command(name = 'xlogdump standby last modifiled xlog', cmdStr="xlogdump " + standby_last_mod_xlog) cmd.run(validateAfter=True); logger.info('See if we find the shutdown LSN in the XLOG seg file') lines = (cmd.get_results()).stdout.splitlines() flag = False for line in range(0, len(lines) - 1): if ((lines[line]).find(primary_last_ckpt_lsn) > -1): self.assertTrue((lines[line]).find("checkpoint") > -1) self.assertTrue((lines[line]).find("shutdown") > -1) flag = True break self.assertTrue(flag) logger.info('PASS') # Re-start the database logger.info('Now restart the DB (gpstart -a)') cmd = Command(name="gpstop smart", cmdStr="source %s/greenplum_path.sh;\ gpstart -a" % os.environ["GPHOME"]) cmd.run(validateAfter=True) # Cleanup. Currently we dont have a clean way of WAL rcv dying logger.info('Kill the standby processes as clean standby killing is not supported') cmd = Command(name="kill standby", cmdStr="kill -9 `ps -ef | grep 5433 | grep -v grep | awk '{print $2}'`") cmd.run(validateAfter=True)
def __init__(self,methodName): self.standby = Standby(self.DESTDIR, 5433) super(SwitchClass,self).__init__(methodName)
import shutil import subprocess import sys from mpp.gpdb.tests.storage.walrepl.lib.standby import Standby def pline(msg, cmd=None): sys.stdout.write('----' + msg + '\n') if cmd: sys.stdout.write(cmd + '\n') sys.stdin.readline() if __name__ == '__main__': mpp.gpdb.tests.storage.walrepl.lib.cleanupFilespaces(dbname='haradh1') standby = Standby('standby', '5433') standby.stop() standby.stop() shutil.rmtree('standby', True) sys.stdout.write(''' *********************************************************** *** Demo for Filespace support *********************************************************** ''') pline('Initialize standby info in catalog and take online backup') standby.create() subprocess.call(['ls', 'standby']) subprocess.call(['psql', '-c', 'select * from gp_segment_configuration'])
def test_icg(self): # 1. Verify if the system is UP and there is no WAL Receiver running # 2. Perform basebackup and deploy it into some dest. directory # 3. Copy recover.conf into the dest. directory to be used by Standby # 4. Initiate the Standby using the Master (primary) postmaster # paramerters # 5. Once the WAL receiver waits for the next record to arrive, perform # installCheck-good # 0. Stop standby if it's running standby = Standby('base', 5433) standby.stop() # 1. Verify if the system is UP and there is no WAL sender running self.assertEqual(self.count_wal_sender(), 0) logger.info('No active WAL Receiver found') # Set environmental variable GPSRC for now for make installcheck purpose source_file = sys.modules[self.__class__.__module__].__file__ source_dir = os.path.dirname(source_file) os.environ['GPSRC'] = os.path.join(source_dir, '../../../../') # 2. Perform basebackup and deploy it into some dest. # (currenttly hardcoded 'base') directory shutil.rmtree('base', True) logger.info('Performing and deploying base backup ...') standby.create() # 3.Copy recover.conf into the dest. directory to be used by StandBy logger.info('Deploying recovery.conf...') # 4. Initiate the StandBy using the Master (primary) postmaster # paramerters logger.info('Initiating Standby...') res = standby.start() self.assertTrue(res.wasSuccessful()) num_walsender = 0 for i in polling(10, 0.5): num_walsender = self.count_wal_sender() if num_walsender > 0: break self.assertEqual(num_walsender, 1) logger.info('Activated WAL Receiver...') # 6. Run installcheck-good self.assertTrue(os.environ["GPSRC"]) installCheckGoodPath = os.path.join(os.environ.get('GPSRC'), 'test', 'regress') self.assertTrue(os.path.exists(installCheckGoodPath)) os.chdir(installCheckGoodPath) regression_diffs = os.path.join(installCheckGoodPath, 'regression.diffs') if os.path.exists(regression_diffs): os.remove(regression_diffs) logger.info('Perform InstallCheck-Good...') subprocess.check_call('make installcheck-good', shell=True) # Verify installcheck result by checking if regression.diff is present. self.assertTrue(not os.path.exists(regression_diffs))
class OOMClass(object): '''Class for methods required for OOM testcase''' standby_port = '5433' standby_dirname = 'newstandby' def __init__(self): self.gpinit = GpinitStandby() self.mdd = os.environ.get('MASTER_DATA_DIRECTORY') self.config = GPDBConfig() self.pgutil = GpUtility() self.verify = StandbyVerify() self.host = socket.gethostname() self.standby_loc = os.path.join(os.path.split(self.mdd)[0], self.standby_dirname) self.standby = Standby(self.standby_loc, self.standby_port) def create_standby(self): self.pgutil.clean_dir(self.host,self.standby_loc) self.gpinit.run(option = '-P %s -s %s -F pg_system:%s' % (self.standby_port, self.host, self.standby_loc)) def setup_oom(self): # Build it before testing. thisdir = os.path.dirname(__file__) builddir = os.path.join(thisdir, 'lib') subprocess.check_call(['make', '-C', builddir, 'install']) #Copy oom_malloc.so and wrapper.sh to all the segment nodes for host in config.get_hosts() : if host.strip() == self.host : continue cmd = "gpssh -h %s -e 'mkdir -p %s'; scp %s/oom_malloc.so %s:%s/; scp %s/wrapper.sh %s:%s/" % (host.strip(), builddir, builddir, host.strip(), builddir, builddir, host.strip(), builddir) self.pgutil.run(cmd) def touch_malloc(self): # Touch file oom_malloc in standby directory cmd = 'touch %s/oom_malloc' % self.standby_loc self.pgutil.run(cmd) def startdb(self): (rc, result) = self.pgutil.run('gpstart -a --wrapper %s' % (local_path('lib/wrapper.sh'))) if rc != 0 and 'Could not start standby master' in result : return False return True def restartdb(self): # Remove file oom_malloc from standby cmd = 'rm %s/oom_malloc' % self.standby_loc self.pgutil.run(cmd) (rc, result) = self.pgutil.run('gpstop -ar') if rc == 0 and (self.verify.check_pg_stat_replication()): return True return False def psql_and_oom(self): #Touch oom_malloc in standby_dir and issue PSQL : Check if processes are gone self.touch_malloc() PSQL.run_sql_command('Drop table if exists wal_oomt1;Create table wal_oomt1(a1 int, a2 text) with(appendonly=true);') sleep(2) if not (self.verify.check_standby_processes()): return True return False def start_standby(self): # Remove oom_malloc and start standby : Check if all processes are back cmd = 'rm %s/oom_malloc' % self.standby_loc self.pgutil.run(cmd) res = self.standby.start() sleep(2) if (self.verify.check_standby_processes()) : return True return False
def test_smart_shutdown(self): # 1. Verify if the system is UP and there is no WAL Receiver running # 2. Perform basebackup and deploy it into some dest. directory # 3. Copy recover.conf into the dest. directory to be used by Standby # 4. Initiate the Standby using the Master (primary) postmaster # paramerters # 5. Perform some transaction to generate xlog. Then do a smart shutdown # 6. Once the primary DB is down, find the last checkpoint from pg_control # on primary. Check the last modified xlog seg from the standby and find # if the last checkpoint from primary exists # 7. It should be present there! # 0. Stop standby if it's running PSQL.run_sql_command('DROP table if exists foo') standby = Standby('base', 5433) standby.stop() # 1. Verify if the system is UP and there is no WAL sender running self.assertEqual(self.count_walsender(), 0) logger.info('No active WAL Receiver found') # 2. Perform basebackup and deploy it into some dest. # (currenttly hardcoded 'base') directory shutil.rmtree('base', True) logger.info('Performing and deploying base backup ...') standby.create() # 3.Copy recover.conf into the dest. directory to be used by StandBy logger.info('Deploying recovery.conf...') # 4. Initiate the StandBy using the Master (primary) postmaster # paramerters logger.info('Initiating Standby...') res = standby.start() self.assertTrue(res.wasSuccessful()) num_walsender = 0 for i in polling(10, 0.5): num_walsender = self.count_walsender() if num_walsender > 0: break self.assertEqual(num_walsender, 1) logger.info('Activated WAL Receiver...') # 5. Perform some transaction to generate xlog. Then do a smart shutdown logger.info('Perform some transaction to generate some XLOG') PSQL.run_sql_command('Create table foo (a int)') logger.info('Now perform smart shutdown (gpstop -a)') cmd = Command(name="gpstop smart", cmdStr="source %s/greenplum_path.sh;\ gpstop -a" % os.environ["GPHOME"]) cmd.run(validateAfter=True) # 6. Once the primary DB is down, find the last checkpoint from pg_control # on primary. Check the last modified xlog seg from the standby and find # if the last checkpoint from primary exists logger.info( 'Read the pg_control from primary, find the last checkpoint & see if it made to standby' ) standby_xlog_path = os.path.join('base', 'pg_xlog') cmd = Command( name='pg_controldata ' + os.environ.get('MASTER_DATA_DIRECTORY'), cmdStr='pg_controldata ' + os.environ.get('MASTER_DATA_DIRECTORY')) cmd.run(validateAfter=True) primary_last_ckpt_lsn = self.last_ckpt_lsn((cmd.get_results()).stdout) logger.info("Primary last checkpoint LSN = " + primary_last_ckpt_lsn) standby_last_mod_xlog = self.last_mod_file(standby_xlog_path) logger.info("Last mod standby XLOG = " + standby_last_mod_xlog) cmd = Command(name='xlogdump standby last modifiled xlog', cmdStr="xlogdump " + standby_last_mod_xlog) cmd.run(validateAfter=True) logger.info('See if we find the shutdown LSN in the XLOG seg file') lines = (cmd.get_results()).stdout.splitlines() flag = False for line in range(0, len(lines) - 1): if ((lines[line]).find(primary_last_ckpt_lsn) > -1): self.assertTrue((lines[line]).find("checkpoint") > -1) self.assertTrue((lines[line]).find("shutdown") > -1) flag = True break self.assertTrue(flag) logger.info('PASS') # Re-start the database logger.info('Now restart the DB (gpstart -a)') cmd = Command(name="gpstop smart", cmdStr="source %s/greenplum_path.sh;\ gpstart -a" % os.environ["GPHOME"]) cmd.run(validateAfter=True) # Cleanup. Currently we dont have a clean way of WAL rcv dying logger.info( 'Kill the standby processes as clean standby killing is not supported' ) cmd = Command( name="kill standby", cmdStr= "kill -9 `ps -ef | grep 5433 | grep -v grep | awk '{print $2}'`") cmd.run(validateAfter=True)
def __init__(self, methodName): self.standby = Standby(self.DESTDIR, 5433) super(SwitchClass, self).__init__(methodName)
class OOMClass(object): '''Class for methods required for OOM testcase''' standby_port = '5433' standby_dirname = 'newstandby' def __init__(self): self.gpinit = GpinitStandby() self.mdd = os.environ.get('MASTER_DATA_DIRECTORY') self.config = GPDBConfig() self.pgutil = GpUtility() self.verify = StandbyVerify() self.host = socket.gethostname() self.standby_loc = os.path.join( os.path.split(self.mdd)[0], self.standby_dirname) self.standby = Standby(self.standby_loc, self.standby_port) def create_standby(self): self.pgutil.clean_dir(self.host, self.standby_loc) self.gpinit.run(option='-P %s -s %s -F pg_system:%s' % (self.standby_port, self.host, self.standby_loc)) def setup_oom(self): # Build it before testing. thisdir = os.path.dirname(__file__) builddir = os.path.join(thisdir, 'lib') subprocess.check_call(['make', '-C', builddir, 'install']) #Copy oom_malloc.so and wrapper.sh to all the segment nodes for host in config.get_hosts(): if host.strip() == self.host: continue cmd = "gpssh -h %s -e 'mkdir -p %s; scp %s/oom_malloc.so %s:%s/;scp %s/wrapper.sh %s:%s/'" % ( host.strip(), builddir, builddir, host.strip(), builddir, builddir, host.strip(), builddir) self.pgutil.run(cmd) def touch_malloc(self): # Touch file oom_malloc in standby directory cmd = 'touch %s/oom_malloc' % self.standby_loc self.pgutil.run(cmd) def startdb(self): (rc, result) = self.pgutil.run('gpstart -a --wrapper %s' % (local_path('lib/wrapper.sh'))) if rc != 0 and 'Could not start standby master' in result: return False return True def restartdb(self): # Remove file oom_malloc from standby cmd = 'rm %s/oom_malloc' % self.standby_loc self.pgutil.run(cmd) (rc, result) = self.pgutil.run('gpstop -ar') if rc == 0 and (self.verify.check_pg_stat_replication()): return True return False def psql_and_oom(self): #Touch oom_malloc in standby_dir and issue PSQL : Check if processes are gone self.touch_malloc() PSQL.run_sql_command( 'Drop table if exists wal_oomt1;Create table wal_oomt1(a1 int, a2 text) with(appendonly=true);' ) sleep(2) if not (self.verify.check_standby_processes()): return True return False def start_standby(self): # Remove oom_malloc and start standby : Check if all processes are back cmd = 'rm %s/oom_malloc' % self.standby_loc self.pgutil.run(cmd) res = self.standby.start() sleep(2) if (self.verify.check_standby_processes()): return True return False