class GpstopTestCase(MPPTestCase): '''testcase for gpstart''' origin_mdd = os.environ.get('MASTER_DATA_DIRECTORY') def __init__(self,methodName): self.gputil = GpUtility() super(GpstopTestCase,self).__init__(methodName) def setUp(self): self.gputil.check_and_start_gpdb() stdby_presence = self.gputil.check_standby_presence() if stdby_presence: self.gputil.remove_standby() self.gputil.install_standby() def tearDown(self): self.gputil.remove_standby() self.gputil.run('gpstart -a') self.gputil.run('gprecoverseg -a') def test_gpstop_from_master(self): self.assertTrue(self.gputil.gpstop_and_verify()) self.gputil.run('gpstart -a') def test_gpstop_master_only(self): self.assertTrue(self.gputil.gpstop_and_verify(option = '-m')) self.gputil.run('gpstart -a') def test_gpstop_fast(self): #run transactions, and stop fast, check if transaction aborted, and the cluster was stopped self.assertTrue(self.gputil.gpstop_and_verify(option = '-M fast')) self.gputil.run('gpstart -a') def test_gpstop_immediate(self): self.assertTrue(self.gputil.gpstop_and_verify(option = '-M immediate')) self.gputil.run('gpstart -a') def test_gpstop_smart(self): self.assertTrue(self.gputil.gpstop_and_verify(option = '-M smart')) self.gputil.run('gpstart -a') def test_gpdb_restart(self): self.assertTrue(self.gputil.gpstop_and_verify('-r')) def test_gpdb_reload(self): self.assertTrue(self.gputil.gpstop_and_verify('-u')) def test_gpstop_except_stdby(self): self.assertTrue(self.gputil.gpstop_and_verify('-y')) self.gputil.run('gpstart -y') def test_gpstop_after_failover(self): tinctest.logger.info("test gpstop from new master after failover") activatestdby = GpactivateStandby() standby_host = activatestdby.get_current_standby() standby_port = activatestdby.get_standby_port() standby_mdd = activatestdby.get_standby_dd() activatestdby.activate() (rc,stdout)=activatestdby.run_remote(standby_host, rmt_cmd='gpstop -a -M fast', pgport=standby_port,standbydd=standby_mdd) self.assertEqual(0,rc) activatestdby.run_remote(standby_host, rmt_cmd='gpstart -a', pgport=standby_port, standbydd=standby_mdd) self.gputil.failback_to_original_master(self.origin_mdd, standby_host, standby_mdd, standby_port)
class GpstartTestCase(MPPTestCase): ''' testcase for gpstart gpstart may return status code 1 as well as 0 in the success case. The difference is whether it produces WARNING or not, but here we don't care. ''' origin_mdd = os.environ.get('MASTER_DATA_DIRECTORY') def __init__(self,methodName): self.gputil = GpUtility() self.stdby = StandbyVerify() super(GpstartTestCase,self).__init__(methodName) def setUp(self): self.gputil.check_and_start_gpdb() stdby_presence = self.gputil.check_standby_presence() # We should forcibly recreate standby, as it might has been promoted. if stdby_presence: self.gputil.remove_standby() self.gputil.install_standby() def tearDown(self): self.gputil.remove_standby() """ Gpstart test cases in recovery mode """ def test_gpstart_from_master(self): """ tag """ self.gputil.check_and_stop_gpdb() (rc, stdout) = self.gputil.run('gpstart -a ') self.assertIn(rc, (0, 1)) self.assertTrue(self.gputil.gpstart_and_verify()) sleep(2) self.assertTrue(self.stdby.check_gp_segment_config(),'standby master not cofigured') self.assertTrue(self.stdby.check_pg_stat_replication(),'standby not in replication status') self.assertTrue(self.stdby.check_standby_processes(), 'standby processes not running') (rc, output) = self.gputil.run(command = 'ps -ef|grep "wal sender "|grep -v grep') self.assertIsNotNone(output) def test_gpstart_master_only(self): """ tag """ self.gputil.check_and_stop_gpdb() (rc, stdout) = self.gputil.run('export GPSTART_INTERNAL_MASTER_ONLY=1; ' 'gpstart -a -m ') self.assertIn(rc, (0, 1)) self.assertTrue(self.gputil.gpstart_and_verify()) (rc,output) = self.gputil.run('PGDATABASE=template1 ' "PGOPTIONS='-c gp_session_role=utility' " 'psql') self.assertEqual(rc, 0) (rc, output) = self.gputil.run('psql template1') # should fail due to master only mode self.assertEqual(rc, 2) self.gputil.run('gpstop -a -m') self.gputil.run('gpstart -a') def test_gpstart_restricted_mode_master(self): """Test -R option with standby.""" self.gputil.check_and_stop_gpdb() (rc, stdout) = self.gputil.run('gpstart -a -R') self.assertIn(rc, (0, 1)) self.assertTrue(self.gputil.gpstart_and_verify()) (rc,output) = self.gputil.run(command = 'psql template1') self.assertIn(rc, (0, 1)) self.gputil.run('gpstop -ar') def test_gpstart_master_w_timeout(self): """Test -t option with standby.""" self.gputil.check_and_stop_gpdb() (rc, output) = self.gputil.run('gpstart -a -t 30') self.assertIn(rc, (0, 1)) self.assertTrue(self.gputil.gpstart_and_verify()) self.gputil.run('gpstop -ar') def test_gpstart_no_standby(self): """Test -y with standby configured.""" self.gputil.check_and_stop_gpdb() (rc, stdout) = self.gputil.run('gpstart -a -y') self.assertIn(rc, (0, 1)) self.assertTrue(self.gputil.gpstart_and_verify()) self.assertFalse(self.stdby.check_standby_processes(), 'gpstart without standby failed, standby was running') self.gputil.run('gpstop -ar') def test_gpstart_wo_standby(self): """Test -y without standby configured.""" self.gputil.remove_standby() self.gputil.check_and_stop_gpdb() (rc, stdout) = self.gputil.run('gpstart -a -y') self.assertIn(rc, (0, 1)) self.assertTrue(self.gputil.gpstart_and_verify()) self.assertFalse(self.stdby.check_standby_processes(), 'standby processes presented') self.gputil.run('gpstop -ar') """ Gpstart, test case in failover mode """ def test_gpstart_master_only_after_failover(self): """ for test purpose, failing back to old master should remove standby from primary after activate standby """ tinctest.logger.info("start master only with -m option after failover") activatestdby = GpactivateStandby() standby_host = activatestdby.get_current_standby() standby_mdd = activatestdby.get_standby_dd() standby_port = activatestdby.get_standby_port() activatestdby.activate() self.stdby._run_remote_command(standby_host,command = 'gpstop -a') stdout = self.stdby._run_remote_command(standby_host,command = 'export GPSTART_INTERNAL_MASTER_ONLY=1; gpstart -a -m') self.assertNotRegexpMatches(stdout,"ERROR","Start master only after failover failed") self.assertTrue(self.gputil.gpstart_and_verify(master_dd = standby_mdd, host = standby_host)) self.stdby._run_remote_command(standby_host,command = 'gpstop -a -m') self.gputil.run(command = 'gpstop -ar') self.gputil.failback_to_original_master(self.origin_mdd, standby_host, standby_mdd, standby_port) def test_gpstart_master_after_failover(self): """ failover, start from new master, then recover the cluster back to have the old master active. """ tinctest.logger.info("failover, and run gpstart master test") self.gputil.check_and_start_gpdb() activatestdby = GpactivateStandby() standby_host = activatestdby.get_current_standby() standby_mdd = activatestdby.get_standby_dd() standby_port = activatestdby.get_standby_port() activatestdby.activate() self.stdby._run_remote_command(standby_host, command = 'gpstop -a') stdout = self.stdby._run_remote_command(standby_host,command = 'gpstart -a') self.assertNotRegexpMatches(stdout,"FATAL","ERROR") self.assertTrue(self.gputil.gpstart_and_verify(master_dd = standby_mdd, host = standby_host)) self.gputil.failback_to_original_master(self.origin_mdd, standby_host, standby_mdd, standby_port) def test_gpstart_original_master_after_promote(self): """ failover, start from new master, then recover the cluster back to have the old master active. """ tinctest.logger.info("activate and run gpstart for original master") activatestdby = GpactivateStandby() standby_host = activatestdby.get_current_standby() standby_mdd = activatestdby.get_standby_dd() standby_port = activatestdby.get_standby_port() activatestdby.activate() (rc, stdout) = self.gputil.run('gpstart -a -v') self.gputil.run('pg_controldata %s' % self.origin_mdd) self.stdby._run_remote_command(standby_host, command = 'pg_controldata %s' % standby_mdd) self.assertNotEqual(rc, 0) # This below error message comes from gpstart product code (if its modified change it here as well.) self.assertRegexpMatches(stdout,"Standby activated, this node no more can act as master.") self.gputil.failback_to_original_master(self.origin_mdd, standby_host, standby_mdd, standby_port)
class GpstateTestCase(MPPTestCase): '''testcase for gpstart''' origin_mdd = os.environ.get('MASTER_DATA_DIRECTORY') def __init__(self,methodName): self.gputil = GpUtility() self.mirrorConfig = [] self.master_port = os.environ.get('PGPORT') self.masterdd = os.environ.get('MASTER_DATA_DIRECTORY') self.activatestdby = "" super(GpstateTestCase,self).__init__(methodName) def setUp(self): self.gputil.check_and_start_gpdb() stdby_presence = self.gputil.check_standby_presence() if stdby_presence: self.gputil.remove_standby() self.gputil.install_standby() get_mirror_sql = '''select port, hostname, fselocation from gp_segment_configuration, pg_filespace_entry where dbid = fsedbid and content != -1 and preferred_role=\'m\' ;''' segments=self.gputil.run_SQLQuery(get_mirror_sql, dbname='template1') for seg in segments: port = seg[0] host = seg[1] dir = seg[2] self.mirrorConfig.append(port) self.mirrorConfig.append(host) self.mirrorConfig.append(dir) self.activatestdby = GpactivateStandby() def tearDown(self): del self.mirrorConfig[:] self.gputil.remove_standby() def test_gpstate_disp_recovery(self): ''' run gpstate with -f option''' standby_host = self.activatestdby.get_current_standby() standby_port = self.activatestdby.get_standby_port() standby_dir = self.activatestdby.get_standby_dd() standby_pid = self.gputil.get_pid_by_keyword(host=standby_host, user=os.environ.get('USER'), pgport=standby_port,keyword='master',option='bin') (rc, stdout) = self.gputil.run('gpstate -f') self.assertEqual(rc, 0) context = stdout.split('\n') for line in context: if "=" not in line: continue items = line.split('=') if "Standby address" in line: stdby_addr = items[1].strip() self.assertEqual(stdby_addr, standby_host) elif "Standby data directory" in line: stdby_dir = items[1].strip() self.assertEqual(stdby_dir, standby_dir) elif "Standby port" in line: stdby_port = int(items[1].strip()) self.assertEqual(stdby_port, int(standby_port)) elif "Standby PID" in line: pid = items[1].strip() self.assertEqual(pid, standby_pid) def test_gpstate_disp_failover(self): '''test if the master configuration detail changed after activating standby''' standby_host = self.activatestdby.get_current_standby() standby_port = self.activatestdby.get_standby_port() standby_dir = self.activatestdby.get_standby_dd() standby_pid = self.gputil.get_pid_by_keyword(host=standby_host, user=os.environ.get('USER'), pgport=standby_port,keyword='master',option='bin') self.activatestdby.activate() (rc,stdout)=self.activatestdby.run_remote(standby_host, rmt_cmd='gpstate -s', pgport=standby_port, standbydd=standby_dir) self.assertIn(rc, (0,1)) context = stdout.split('\n') for line in context: if "=" not in line: continue items = line.strip().split('=') if "Master host" in line: master_host = items[1].strip() self.assertEqual(master_host, standby_host) elif "Master postgres process ID" in line: master_pid = items[1].strip() self.assertEqual(master_pid, standby_pid) elif "Master data directory" in line: master_dir = items[1].strip() self.assertEqual(master_dir, standby_dir) elif "Master port" in line: master_port = int(items[1].strip()) self.assertEqual(master_port, int(standby_port)) self.gputil.failback_to_original_master(self.origin_mdd,standby_host,standby_dir,standby_port) def test_gpstate_active_segment_failover(self): ''' test if gpstate show correct # of up and down nodes after failover''' count_up_seg = '''select count(*) from gp_segment_configuration where content != -1 and status = \'u\';''' count_down_seg = '''select count(*) from gp_segment_configuration where content != -1 and status = \'d\';''' number_up_segment = PSQL.run_sql_command(count_up_seg, flags = '-q -t', dbname='template1') number_down_segment = PSQL.run_sql_command(count_down_seg, flags = '-q -t', dbname='template1') standby_host = self.activatestdby.get_current_standby() standby_port = self.activatestdby.get_standby_port() standby_dir = self.activatestdby.get_standby_dd() self.activatestdby.activate() (rc,stdout)=self.activatestdby.run_remote(standby_host, rmt_cmd='gpstate -Q', pgport=standby_port, standbydd=standby_dir) self.assertIn(rc, (0,1)) context = stdout.split('\n') for line in context: if "=" not in line: continue items = line.strip().split('=') if "up segments" in line: self.assertEqual(number_up_segment.strip(),items[1].strip()) elif "down segments" in line: self.assertEqual(number_down_segment.strip(),items[1].strip()) self.gputil.failback_to_original_master(self.origin_mdd,standby_host,standby_dir,standby_port) def test_gpstate_disp_mirror_failover(self): ''' check if new master is able to get correct mirror configuration with gpstate -m''' inside_block = False keywords = ("Mirror","Datadir","Port") standby_host = self.activatestdby.get_current_standby() standby_port = self.activatestdby.get_standby_port() standby_dir = self.activatestdby.get_standby_dd() self.activatestdby.activate() (rc,stdout)=self.activatestdby.run_remote(standby_host, rmt_cmd='gpstate -m', pgport=standby_port, standbydd=standby_dir) self.assertEqual(rc, 0) for line in stdout: if inside_block: line_split = line.split('') line_split = [elem for elem in line_split if elem != ''] mirror_host = line_split[2] mirror_dir = line_split[3] mirror_port = line_split[4] self.assertTrue(mirror_host in self.mirrorConfig) self.assertTrue(mirror_dir in self.mirrorConfig) self.assertTrue(mirror_port in self.mirrorConfig) elif not all (s in line for s in keywords): continue else: inside_block = True self.gputil.failback_to_original_master(self.origin_mdd,standby_host,standby_dir,standby_port)
class GpstopTestCase(MPPTestCase): '''testcase for gpstart''' origin_mdd = os.environ.get('MASTER_DATA_DIRECTORY') def __init__(self,methodName): self.gputil = GpUtility() super(GpstopTestCase,self).__init__(methodName) def setUp(self): self.gputil.check_and_start_gpdb() stdby_presence = self.gputil.check_standby_presence() if stdby_presence: self.gputil.remove_standby() self.gputil.install_standby() def tearDown(self): self.gputil.remove_standby() self.gputil.run('gpstart -a') def test_gpstop_from_master(self): self.assertTrue(self.gputil.gpstop_and_verify()) self.gputil.run('gpstart -a') def test_gpstop_master_only(self): self.assertTrue(self.gputil.gpstop_and_verify(option = '-m')) self.gputil.run('gpstart -a') def test_gpstop_fast(self): #run transactions, and stop fast, check if transaction aborted, and the cluster was stopped self.assertTrue(self.gputil.gpstop_and_verify(option = '-M fast')) self.gputil.run('gpstart -a') def test_gpstop_immediate(self): self.assertTrue(self.gputil.gpstop_and_verify(option = '-M immediate')) self.gputil.run('gpstart -a') def test_gpstop_smart(self): self.assertTrue(self.gputil.gpstop_and_verify(option = '-M smart')) self.gputil.run('gpstart -a') def test_gpdb_restart(self): self.assertTrue(self.gputil.gpstop_and_verify('-r')) def test_gpdb_reload(self): self.assertTrue(self.gputil.gpstop_and_verify('-u')) def test_gpstop_except_stdby(self): self.assertTrue(self.gputil.gpstop_and_verify('-y')) self.gputil.run('gpstart -y') def test_gpstop_after_failover(self): tinctest.logger.info("test gpstop from new master after failover") activatestdby = GpactivateStandby() standby_host = activatestdby.get_current_standby() standby_port = activatestdby.get_standby_port() standby_mdd = activatestdby.get_standby_dd() activatestdby.activate() (rc,stdout)=activatestdby.run_remote(standby_host, rmt_cmd='gpstop -a -M fast', pgport=standby_port,standbydd=standby_mdd) self.assertEqual(0,rc) activatestdby.run_remote(standby_host, rmt_cmd='gpstart -a', pgport=standby_port, standbydd=standby_mdd) self.gputil.failback_to_original_master(self.origin_mdd, standby_host, standby_mdd, standby_port)