class SwitchClass(MPPTestCase): DESTDIR = 'base' def __init__(self, methodName): self.standby = Standby(self.DESTDIR, 5433) super(SwitchClass, self).__init__(methodName) def run_pg_basebackup(self): shutil.rmtree(self.DESTDIR, True) self.standby.create() def start_standby(self): res = self.standby.start() self.assertTrue(res.wasSuccessful()) self.assertTrue(self.check_walreceiver()) def check_walreceiver(self): grep_cmd = 'ps -ef|grep %s|grep -v grep' % ('wal receiver') cmd = Command(name='Check if no wal receiver process', cmdStr=grep_cmd) tinctest.logger.info('%s' % cmd) cmd.run(validateAfter=False) result = cmd.get_results() if result.rc != 0: return True return False
class SwitchClass(MPPTestCase): DESTDIR = 'base' def __init__(self,methodName): self.standby = Standby(self.DESTDIR, 5433) super(SwitchClass,self).__init__(methodName) def run_pg_basebackup(self): shutil.rmtree(self.DESTDIR, True) self.standby.create() def start_standby(self): res = self.standby.start() self.assertTrue(res.wasSuccessful()) self.assertTrue(self.check_walreceiver()) def check_walreceiver(self): grep_cmd = 'ps -ef|grep %s|grep -v grep' % ('wal receiver') cmd = Command(name='Check if no wal receiver process', cmdStr=grep_cmd) tinctest.logger.info ('%s' % cmd) cmd.run(validateAfter=False) result = cmd.get_results() if result.rc !=0: return True return False
sys.stdout.write(''' *********************************************************** *** Demo for Filespace support *********************************************************** ''') pline('Initialize standby info in catalog and take online backup') standby.create() subprocess.call(['ls', 'standby']) subprocess.call(['psql', '-c', 'select * from gp_segment_configuration']) pline('Start standby', 'pg_ctl start -D standby') standby.create_recovery_conf() subprocess.call(['cat', 'standby/recovery.conf']) standby.start() with open('filespace_setup.sql') as f: sql = f.read() pline('Run some SQL on primary while standby is running', sql) subprocess.call(['psql', '-f', 'filespace_setup.sql']) pline('Promote standby', 'pg_ctl promote -D standby\n' 'select pg_activate_standby()') standby.promote() sql = 'SELECT * FROM fspc_table' pline('Connect to port 5433', 'psql -p 5433\n{0}'.format(sql)) subprocess.call(['psql', '-p', '5433', '-c', sql]) sql = 'SELECT * FROM gp_segment_configuration'
def test_icg(self): # 1. Verify if the system is UP and there is no WAL Receiver running # 2. Perform basebackup and deploy it into some dest. directory # 3. Copy recover.conf into the dest. directory to be used by Standby # 4. Initiate the Standby using the Master (primary) postmaster # paramerters # 5. Once the WAL receiver waits for the next record to arrive, perform # installCheck-good # 0. Stop standby if it's running standby = Standby('base', 5433) standby.stop() # 1. Verify if the system is UP and there is no WAL sender running self.assertEqual(self.count_wal_sender(), 0) logger.info('No active WAL Receiver found') # Set environmental variable GPSRC for now for make installcheck purpose source_file = sys.modules[self.__class__.__module__].__file__ source_dir = os.path.dirname(source_file) os.environ['GPSRC'] = os.path.join(source_dir, '../../../../') # 2. Perform basebackup and deploy it into some dest. # (currenttly hardcoded 'base') directory shutil.rmtree('base', True) logger.info('Performing and deploying base backup ...') standby.create() # 3.Copy recover.conf into the dest. directory to be used by StandBy logger.info('Deploying recovery.conf...') # 4. Initiate the StandBy using the Master (primary) postmaster # paramerters logger.info('Initiating Standby...') res = standby.start() self.assertTrue(res.wasSuccessful()) num_walsender = 0 for i in polling(10, 0.5): num_walsender = self.count_wal_sender() if num_walsender > 0: break self.assertEqual(num_walsender, 1) logger.info('Activated WAL Receiver...') # 6. Run installcheck-good self.assertTrue(os.environ["GPSRC"]) installCheckGoodPath = os.path.join(os.environ.get('GPSRC'), 'test', 'regress') self.assertTrue(os.path.exists(installCheckGoodPath)) os.chdir(installCheckGoodPath) regression_diffs = os.path.join(installCheckGoodPath, 'regression.diffs') if os.path.exists(regression_diffs): os.remove(regression_diffs) logger.info('Perform InstallCheck-Good...') subprocess.check_call('make installcheck-good', shell=True) # Verify installcheck result by checking if regression.diff is present. self.assertTrue(not os.path.exists(regression_diffs))
class OOMClass(object): '''Class for methods required for OOM testcase''' standby_port = '5433' standby_dirname = 'newstandby' def __init__(self): self.gpinit = GpinitStandby() self.mdd = os.environ.get('MASTER_DATA_DIRECTORY') self.config = GPDBConfig() self.pgutil = GpUtility() self.verify = StandbyVerify() self.host = socket.gethostname() self.standby_loc = os.path.join(os.path.split(self.mdd)[0], self.standby_dirname) self.standby = Standby(self.standby_loc, self.standby_port) def create_standby(self): self.pgutil.clean_dir(self.host,self.standby_loc) self.gpinit.run(option = '-P %s -s %s -F pg_system:%s' % (self.standby_port, self.host, self.standby_loc)) def setup_oom(self): # Build it before testing. thisdir = os.path.dirname(__file__) builddir = os.path.join(thisdir, 'lib') subprocess.check_call(['make', '-C', builddir, 'install']) #Copy oom_malloc.so and wrapper.sh to all the segment nodes for host in config.get_hosts() : if host.strip() == self.host : continue cmd = "gpssh -h %s -e 'mkdir -p %s'; scp %s/oom_malloc.so %s:%s/; scp %s/wrapper.sh %s:%s/" % (host.strip(), builddir, builddir, host.strip(), builddir, builddir, host.strip(), builddir) self.pgutil.run(cmd) def touch_malloc(self): # Touch file oom_malloc in standby directory cmd = 'touch %s/oom_malloc' % self.standby_loc self.pgutil.run(cmd) def startdb(self): (rc, result) = self.pgutil.run('gpstart -a --wrapper %s' % (local_path('lib/wrapper.sh'))) if rc != 0 and 'Could not start standby master' in result : return False return True def restartdb(self): # Remove file oom_malloc from standby cmd = 'rm %s/oom_malloc' % self.standby_loc self.pgutil.run(cmd) (rc, result) = self.pgutil.run('gpstop -ar') if rc == 0 and (self.verify.check_pg_stat_replication()): return True return False def psql_and_oom(self): #Touch oom_malloc in standby_dir and issue PSQL : Check if processes are gone self.touch_malloc() PSQL.run_sql_command('Drop table if exists wal_oomt1;Create table wal_oomt1(a1 int, a2 text) with(appendonly=true);') sleep(2) if not (self.verify.check_standby_processes()): return True return False def start_standby(self): # Remove oom_malloc and start standby : Check if all processes are back cmd = 'rm %s/oom_malloc' % self.standby_loc self.pgutil.run(cmd) res = self.standby.start() sleep(2) if (self.verify.check_standby_processes()) : return True return False
def test_smart_shutdown(self): # 1. Verify if the system is UP and there is no WAL Receiver running # 2. Perform basebackup and deploy it into some dest. directory # 3. Copy recover.conf into the dest. directory to be used by Standby # 4. Initiate the Standby using the Master (primary) postmaster # paramerters # 5. Perform some transaction to generate xlog. Then do a smart shutdown # 6. Once the primary DB is down, find the last checkpoint from pg_control # on primary. Check the last modified xlog seg from the standby and find # if the last checkpoint from primary exists # 7. It should be present there! # 0. Stop standby if it's running PSQL.run_sql_command('DROP table if exists foo') standby = Standby('base', 5433) standby.stop() # 1. Verify if the system is UP and there is no WAL sender running self.assertEqual(self.count_walsender(), 0) logger.info('No active WAL Receiver found') # 2. Perform basebackup and deploy it into some dest. # (currenttly hardcoded 'base') directory shutil.rmtree('base', True) logger.info('Performing and deploying base backup ...') standby.create() # 3.Copy recover.conf into the dest. directory to be used by StandBy logger.info('Deploying recovery.conf...') # 4. Initiate the StandBy using the Master (primary) postmaster # paramerters logger.info('Initiating Standby...') res = standby.start() self.assertTrue(res.wasSuccessful()) num_walsender = 0 for i in polling(10, 0.5): num_walsender = self.count_walsender() if num_walsender > 0: break self.assertEqual(num_walsender, 1) logger.info('Activated WAL Receiver...') # 5. Perform some transaction to generate xlog. Then do a smart shutdown logger.info('Perform some transaction to generate some XLOG') PSQL.run_sql_command('Create table foo (a int)') logger.info('Now perform smart shutdown (gpstop -a)') cmd = Command(name="gpstop smart", cmdStr="source %s/greenplum_path.sh;\ gpstop -a" % os.environ["GPHOME"]) cmd.run(validateAfter=True) # 6. Once the primary DB is down, find the last checkpoint from pg_control # on primary. Check the last modified xlog seg from the standby and find # if the last checkpoint from primary exists logger.info( 'Read the pg_control from primary, find the last checkpoint & see if it made to standby' ) standby_xlog_path = os.path.join('base', 'pg_xlog') cmd = Command( name='pg_controldata ' + os.environ.get('MASTER_DATA_DIRECTORY'), cmdStr='pg_controldata ' + os.environ.get('MASTER_DATA_DIRECTORY')) cmd.run(validateAfter=True) primary_last_ckpt_lsn = self.last_ckpt_lsn((cmd.get_results()).stdout) logger.info("Primary last checkpoint LSN = " + primary_last_ckpt_lsn) standby_last_mod_xlog = self.last_mod_file(standby_xlog_path) logger.info("Last mod standby XLOG = " + standby_last_mod_xlog) cmd = Command(name='xlogdump standby last modifiled xlog', cmdStr="xlogdump " + standby_last_mod_xlog) cmd.run(validateAfter=True) logger.info('See if we find the shutdown LSN in the XLOG seg file') lines = (cmd.get_results()).stdout.splitlines() flag = False for line in range(0, len(lines) - 1): if ((lines[line]).find(primary_last_ckpt_lsn) > -1): self.assertTrue((lines[line]).find("checkpoint") > -1) self.assertTrue((lines[line]).find("shutdown") > -1) flag = True break self.assertTrue(flag) logger.info('PASS') # Re-start the database logger.info('Now restart the DB (gpstart -a)') cmd = Command(name="gpstop smart", cmdStr="source %s/greenplum_path.sh;\ gpstart -a" % os.environ["GPHOME"]) cmd.run(validateAfter=True) # Cleanup. Currently we dont have a clean way of WAL rcv dying logger.info( 'Kill the standby processes as clean standby killing is not supported' ) cmd = Command( name="kill standby", cmdStr= "kill -9 `ps -ef | grep 5433 | grep -v grep | awk '{print $2}'`") cmd.run(validateAfter=True)
class OOMClass(object): '''Class for methods required for OOM testcase''' standby_port = '5433' standby_dirname = 'newstandby' def __init__(self): self.gpinit = GpinitStandby() self.mdd = os.environ.get('MASTER_DATA_DIRECTORY') self.config = GPDBConfig() self.pgutil = GpUtility() self.verify = StandbyVerify() self.host = socket.gethostname() self.standby_loc = os.path.join( os.path.split(self.mdd)[0], self.standby_dirname) self.standby = Standby(self.standby_loc, self.standby_port) def create_standby(self): self.pgutil.clean_dir(self.host, self.standby_loc) self.gpinit.run(option='-P %s -s %s -F pg_system:%s' % (self.standby_port, self.host, self.standby_loc)) def setup_oom(self): # Build it before testing. thisdir = os.path.dirname(__file__) builddir = os.path.join(thisdir, 'lib') subprocess.check_call(['make', '-C', builddir, 'install']) #Copy oom_malloc.so and wrapper.sh to all the segment nodes for host in config.get_hosts(): if host.strip() == self.host: continue cmd = "gpssh -h %s -e 'mkdir -p %s; scp %s/oom_malloc.so %s:%s/;scp %s/wrapper.sh %s:%s/'" % ( host.strip(), builddir, builddir, host.strip(), builddir, builddir, host.strip(), builddir) self.pgutil.run(cmd) def touch_malloc(self): # Touch file oom_malloc in standby directory cmd = 'touch %s/oom_malloc' % self.standby_loc self.pgutil.run(cmd) def startdb(self): (rc, result) = self.pgutil.run('gpstart -a --wrapper %s' % (local_path('lib/wrapper.sh'))) if rc != 0 and 'Could not start standby master' in result: return False return True def restartdb(self): # Remove file oom_malloc from standby cmd = 'rm %s/oom_malloc' % self.standby_loc self.pgutil.run(cmd) (rc, result) = self.pgutil.run('gpstop -ar') if rc == 0 and (self.verify.check_pg_stat_replication()): return True return False def psql_and_oom(self): #Touch oom_malloc in standby_dir and issue PSQL : Check if processes are gone self.touch_malloc() PSQL.run_sql_command( 'Drop table if exists wal_oomt1;Create table wal_oomt1(a1 int, a2 text) with(appendonly=true);' ) sleep(2) if not (self.verify.check_standby_processes()): return True return False def start_standby(self): # Remove oom_malloc and start standby : Check if all processes are back cmd = 'rm %s/oom_malloc' % self.standby_loc self.pgutil.run(cmd) res = self.standby.start() sleep(2) if (self.verify.check_standby_processes()): return True return False
def test_smart_shutdown(self): # 1. Verify if the system is UP and there is no WAL Receiver running # 2. Perform basebackup and deploy it into some dest. directory # 3. Copy recover.conf into the dest. directory to be used by Standby # 4. Initiate the Standby using the Master (primary) postmaster # paramerters # 5. Perform some transaction to generate xlog. Then do a smart shutdown # 6. Once the primary DB is down, find the last checkpoint from pg_control # on primary. Check the last modified xlog seg from the standby and find # if the last checkpoint from primary exists # 7. It should be present there! # 0. Stop standby if it's running PSQL.run_sql_command('DROP table if exists foo') standby = Standby('base', 5433) standby.stop() # 1. Verify if the system is UP and there is no WAL sender running self.assertEqual(self.count_walsender(), 0) logger.info('No active WAL Receiver found') # 2. Perform basebackup and deploy it into some dest. # (currenttly hardcoded 'base') directory shutil.rmtree('base', True) logger.info('Performing and deploying base backup ...') standby.create() # 3.Copy recover.conf into the dest. directory to be used by StandBy logger.info('Deploying recovery.conf...') # 4. Initiate the StandBy using the Master (primary) postmaster # paramerters logger.info('Initiating Standby...') res = standby.start() self.assertTrue(res.wasSuccessful()) num_walsender = 0 for i in polling(10, 0.5): num_walsender = self.count_walsender() if num_walsender > 0: break self.assertEqual(num_walsender, 1) logger.info('Activated WAL Receiver...') # 5. Perform some transaction to generate xlog. Then do a smart shutdown logger.info('Perform some transaction to generate some XLOG') PSQL.run_sql_command('Create table foo (a int)') logger.info('Now perform smart shutdown (gpstop -a)') cmd = Command(name="gpstop smart", cmdStr="source %s/greenplum_path.sh;\ gpstop -a" % os.environ["GPHOME"]) cmd.run(validateAfter=True) # 6. Once the primary DB is down, find the last checkpoint from pg_control # on primary. Check the last modified xlog seg from the standby and find # if the last checkpoint from primary exists logger.info('Read the pg_control from primary, find the last checkpoint & see if it made to standby') standby_xlog_path = os.path.join('base','pg_xlog') cmd= Command(name = 'pg_controldata ' + os.environ.get('MASTER_DATA_DIRECTORY'), cmdStr = 'pg_controldata ' + os.environ.get('MASTER_DATA_DIRECTORY')) cmd.run(validateAfter=True); primary_last_ckpt_lsn = self.last_ckpt_lsn((cmd.get_results()).stdout) logger.info ("Primary last checkpoint LSN = " + primary_last_ckpt_lsn) standby_last_mod_xlog = self.last_mod_file(standby_xlog_path) logger.info ("Last mod standby XLOG = " + standby_last_mod_xlog ) cmd= Command(name = 'xlogdump standby last modifiled xlog', cmdStr="xlogdump " + standby_last_mod_xlog) cmd.run(validateAfter=True); logger.info('See if we find the shutdown LSN in the XLOG seg file') lines = (cmd.get_results()).stdout.splitlines() flag = False for line in range(0, len(lines) - 1): if ((lines[line]).find(primary_last_ckpt_lsn) > -1): self.assertTrue((lines[line]).find("checkpoint") > -1) self.assertTrue((lines[line]).find("shutdown") > -1) flag = True break self.assertTrue(flag) logger.info('PASS') # Re-start the database logger.info('Now restart the DB (gpstart -a)') cmd = Command(name="gpstop smart", cmdStr="source %s/greenplum_path.sh;\ gpstart -a" % os.environ["GPHOME"]) cmd.run(validateAfter=True) # Cleanup. Currently we dont have a clean way of WAL rcv dying logger.info('Kill the standby processes as clean standby killing is not supported') cmd = Command(name="kill standby", cmdStr="kill -9 `ps -ef | grep 5433 | grep -v grep | awk '{print $2}'`") cmd.run(validateAfter=True)