class WalRecTestCase(MPPTestCase):
''' Testcases for SIGTERM on wal receiver'''
def __init__(self, methodName):
self.gp = GpinitStandby()
self.mdd = os.environ.get('MASTER_DATA_DIRECTORY')
self.pgutil = GpUtility()
super(WalRecTestCase,self).__init__(methodName)
def setUp(self):
#Remove standby if present
self.gp.run(option='-r')
def invoke_sigterm_and_verify(self):
''' Invoke sigterm on wal receiver and verify that a new process is spawned after '''
gpact_stdby = GpactivateStandby()
standby_host = gpact_stdby.get_current_standby()
standby_port = gpact_stdby.get_standby_port()
wal_rec_pid_1 = self.pgutil.get_pid_by_keyword(host=standby_host, pgport=standby_port, keyword='wal receiver process', option='')
sig_cmd = "gpssh -h %s -e 'kill -15 %s'" % (standby_host, wal_rec_pid_1)
cmd = Command('Issue SIGTERM to wam receiver process', cmdStr=sig_cmd)
tinctest.logger.info ('%s' % cmd)
cmd.run(validateAfter=True)
result = cmd.get_results()
if result.rc != 0:
return False
wal_rec_pid_2 = self.pgutil.get_pid_by_keyword(host=standby_host, pgport=standby_port, keyword='wal receiver process', option='')
if wal_rec_pid_1 == wal_rec_pid_2:
return False
return True
@unittest.skipIf(not config.is_multinode(), "Test applies only to a multinode cluster")
def test_sigterm_on_walreceiver(self):
self.pgutil.install_standby()
self.assertTrue(self.invoke_sigterm_and_verify())
class WalRecTestCase(MPPTestCase):
''' Testcases for SIGTERM on wal receiver'''
def __init__(self, methodName):
self.gp = GpinitStandby()
self.mdd = os.environ.get('MASTER_DATA_DIRECTORY')
self.pgutil = GpUtility()
super(WalRecTestCase,self).__init__(methodName)
def setUp(self):
#Remove standby if present
self.gp.run(option='-r')
def invoke_sigterm_and_verify(self):
''' Invoke sigterm on wal receiver and verify that a new process is spawned after '''
gpact_stdby = GpactivateStandby()
standby_host = gpact_stdby.get_current_standby()
standby_port = gpact_stdby.get_standby_port()
wal_rec_pid_1 = self.pgutil.get_pid_by_keyword(host=standby_host, pgport=standby_port, keyword='wal receiver process', option='')
sig_cmd = "gpssh -h %s -e 'kill -15 %s'" % (standby_host, wal_rec_pid_1)
cmd = Command('Issue SIGTERM to wam receiver process', cmdStr=sig_cmd)
tinctest.logger.info ('%s' % cmd)
cmd.run(validateAfter=True)
result = cmd.get_results()
if result.rc != 0:
return False
wal_rec_pid_2 = self.pgutil.get_pid_by_keyword(host=standby_host, pgport=standby_port, keyword='wal receiver process', option='')
if wal_rec_pid_1 == wal_rec_pid_2:
return False
return True
@unittest.skipIf(not config.is_multinode(), "Test applies only to a multinode cluster")
def test_sigterm_on_walreceiver(self):
self.pgutil.install_standby()
self.assertTrue(self.invoke_sigterm_and_verify())
def test_run_five(self):
for i in xrange(5):
with NewEnv(PGPORT=self.port,
MASTER_DATA_DIRECTORY=self.mdd):
pguti = GpUtility()
if i == 0:
pguti.install_standby(socket.gethostname(), self.mdd)
# starting from second time, init standby from new master, standby_dir will be like master_newstandby_newstandby...
else:
pguti.install_standby(socket.gethostname(), os.path.join(self.mdd,'newstandby'))
gpact = GpactivateStandby()
self.mdd = gpact.get_standby_dd()
self.port = gpact.get_standby_port()
gpact.activate()
tinctest.logger.info("self.mdd is %s, self.port is %s"%(self.mdd, self.port))
def __init__(self):
self.stdby = StandbyVerify()
self.runmixin = StandbyRunMixin()
self.runmixin.createdb(dbname='walrepl')
self.mdd = os.environ.get('MASTER_DATA_DIRECTORY')
self.config = GPDBConfig()
self.pgutil = GpUtility()
self.host = socket.gethostname()
def __init__(self,methodName):
self.gphome = os.environ.get('GPHOME')
self.pgport = os.environ.get('PGPORT')
self.pgdatabase = os.environ.get('PGDATABASE')
self.stdby_host = 'localhost'
self.master_dd = os.environ.get('MASTER_DATA_DIRECTORY')
self.pgutil = GpUtility()
self.stdby = StandbyVerify()
super(WalReplKillProcessTestCase,self).__init__(methodName)
def __init__(self):
self.gpinit = GpinitStandby()
self.mdd = os.environ.get("MASTER_DATA_DIRECTORY")
self.config = GPDBConfig()
self.pgutil = GpUtility()
self.verify = StandbyVerify()
self.host = socket.gethostname()
self.standby_loc = os.path.join(os.path.split(self.mdd)[0], self.standby_dirname)
self.standby = Standby(self.standby_loc, self.standby_port)
class OOMGpstartTestCase(MPPTestCase):
''' Simulating oom on standby master , gpstart fails to start standby'''
# Setup OOM env
oom.setup_oom()
def __init__(self, methodName):
self.gpinitstandby= GpinitStandby()
self.gputil = GpUtility()
super(OOMGpstartTestCase,self).__init__(methodName)
def setUp(self):
self.gpinitstandby.run(option='-r')
def tearDown(self):
self.gpinitstandby.run(option='-r')
def test_oom_with_gpstart(self):
# Create standby
oom.create_standby()
# Stop cluster
self.gputil.run('gpstop -a')
# Run gpstart after creaitng the flag file- Presence of this flag file will return NULL for malloc simulating a no memory scenario
oom.touch_malloc()
self.assertFalse(oom.startdb(), 'Standby start is expected to fail with no memory error')
# Restart after removing the flag file
self.assertTrue(oom.restartdb(), 'Expected a successful start of standby')
@unittest.skipIf(not config.is_multinode(), "Test applies only to a multinode cluster")
def test_oom_with_psql_con(self):
# Create standby
oom.create_standby()
# Stop cluster
self.gputil.run('gpstop -a')
# Run gpstart with wrapper
oom.startdb()
# OOM with psql- after adding the flag file to the standby directory to simulate OOM
self.assertTrue(oom.psql_and_oom())
# Start standby after free up memory
self.assertTrue(oom.start_standby())
from mpp.models import MPPTestCase
from gppylib.commands.base import Command
from mpp.gpdb.tests.storage.walrepl.gpactivatestandby import GpactivateStandby
from mpp.gpdb.tests.storage.walrepl.gpinitstandby import GpinitStandby
from mpp.gpdb.tests.storage.walrepl.lib import NewEnv
from mpp.gpdb.tests.storage.walrepl.lib.pg_util import GpUtility
from mpp.lib.PSQL import PSQL
from tinctest.lib import local_path
import tinctest
import os
import shutil
import socket
from time import sleep
pgutil = GpUtility()
pgutil.check_and_start_gpdb()
origin_mdd = os.environ.get('MASTER_DATA_DIRECTORY')
class TakeFive(MPPTestCase):
port = os.environ.get('PGPORT')
mdd = os.environ.get('MASTER_DATA_DIRECTORY')
def __init__(self, methodName):
super(TakeFive, self).__init__(methodName)
def setUp(self):
Command('gpinitstandby -ra', 'gpinitstandby -ra').run()
def tearDown(self):
limitations under the License.
"""
import unittest2 as unittest
from tinctest import logger
from gppylib.commands.base import Command
from mpp.gpdb.tests.storage.walrepl import lib as walrepl
from mpp.gpdb.tests.storage.walrepl.walreceiver import GPLibPQTestCase
from mpp.gpdb.tests.storage.walrepl.lib.pqwrap import *
from mpp.gpdb.tests.storage.walrepl.lib.pg_util import GpUtility
from mpp.lib.config import GPDBConfig
import select
gputil = GpUtility()
gputil.check_and_start_gpdb()
config = GPDBConfig()
class case(GPLibPQTestCase):
"""Basic test cases to see gplibpq and walsender are talking correctly."""
def setUp(self):
# cleanup
cmd = Command('gpinitstandby', 'gpinitstandby -ar')
# don't care the result in case standby is not configured
cmd.run()
@unittest.skipIf(config.is_multinode(), "Test applies only to a singlenode cluster")
def test_connect(self):
"""
def __init__(self,methodName):
self.gputil = GpUtility()
super(GpstopTestCase,self).__init__(methodName)
class GpstopTestCase(MPPTestCase):
'''testcase for gpstart'''
origin_mdd = os.environ.get('MASTER_DATA_DIRECTORY')
def __init__(self,methodName):
self.gputil = GpUtility()
super(GpstopTestCase,self).__init__(methodName)
def setUp(self):
self.gputil.check_and_start_gpdb()
stdby_presence = self.gputil.check_standby_presence()
if stdby_presence:
self.gputil.remove_standby()
self.gputil.install_standby()
def tearDown(self):
self.gputil.remove_standby()
self.gputil.run('gpstart -a')
self.gputil.run('gprecoverseg -a')
def test_gpstop_from_master(self):
self.assertTrue(self.gputil.gpstop_and_verify())
self.gputil.run('gpstart -a')
def test_gpstop_master_only(self):
self.assertTrue(self.gputil.gpstop_and_verify(option = '-m'))
self.gputil.run('gpstart -a')
def test_gpstop_fast(self):
#run transactions, and stop fast, check if transaction aborted, and the cluster was stopped
self.assertTrue(self.gputil.gpstop_and_verify(option = '-M fast'))
self.gputil.run('gpstart -a')
def test_gpstop_immediate(self):
self.assertTrue(self.gputil.gpstop_and_verify(option = '-M immediate'))
self.gputil.run('gpstart -a')
def test_gpstop_smart(self):
self.assertTrue(self.gputil.gpstop_and_verify(option = '-M smart'))
self.gputil.run('gpstart -a')
def test_gpdb_restart(self):
self.assertTrue(self.gputil.gpstop_and_verify('-r'))
def test_gpdb_reload(self):
self.assertTrue(self.gputil.gpstop_and_verify('-u'))
def test_gpstop_except_stdby(self):
self.assertTrue(self.gputil.gpstop_and_verify('-y'))
self.gputil.run('gpstart -y')
def test_gpstop_after_failover(self):
tinctest.logger.info("test gpstop from new master after failover")
activatestdby = GpactivateStandby()
standby_host = activatestdby.get_current_standby()
standby_port = activatestdby.get_standby_port()
standby_mdd = activatestdby.get_standby_dd()
activatestdby.activate()
(rc,stdout)=activatestdby.run_remote(standby_host, rmt_cmd='gpstop -a -M fast',
pgport=standby_port,standbydd=standby_mdd)
self.assertEqual(0,rc)
activatestdby.run_remote(standby_host, rmt_cmd='gpstart -a',
pgport=standby_port,
standbydd=standby_mdd)
self.gputil.failback_to_original_master(self.origin_mdd, standby_host, standby_mdd, standby_port)
def test_tli_mismatch(self):
"""
Test to verify if TLI mismatch issue during Pass 3 of xlog record
(checkpoint) replay occurs or not. A set of checkpoints one after
the other when replayed on the standby and then if the
standby is promoted, it should go through the prmotion just fine.
The flow of this test is as follows.
1. Initiate the Standby using the Master (primary) postmaster
paramerters.
2. Perform explicit checkpoints and wait so that they get replicated.
3. Then promote the standby, wait and then try to access it.
4. If we can successfuly access it, its a Pass otherwise a Fail.
"""
PSQL.run_sql_command('DROP table if exists foo')
PSQL.run_sql_command('create table foo (a int)')
PSQL.run_sql_command('insert into foo '
'select * from generate_series(1, 1000)')
# Initial setup, forcibly remove standby and install new one
pgutil = GpUtility()
pgutil.remove_standby()
logger.info ('\nCreate a standby...')
res = self.standby.create()
self.assertEqual(res, 0)
res = self.standby.start()
self.assertTrue(res.wasSuccessful())
# Wait for the walreceiver to start
num_walsender = self.wait_for_walsender()
self.assertEqual(num_walsender, 1)
logger.info('Standby activated...')
logger.info('Give the standby some time to catchup...')
time.sleep(3)
logger.info('Create checkpoints and let them get replicated...')
PSQL.run_sql_command('checkpoint')
PSQL.run_sql_command('checkpoint')
PSQL.run_sql_command('checkpoint')
PSQL.run_sql_command('checkpoint')
PSQL.run_sql_command('checkpoint')
PSQL.run_sql_command('checkpoint')
PSQL.run_sql_command('checkpoint')
PSQL.run_sql_command('checkpoint')
PSQL.run_sql_command('checkpoint')
PSQL.run_sql_command('checkpoint')
PSQL.run_sql_command('checkpoint')
PSQL.run_sql_command('checkpoint')
PSQL.run_sql_command('checkpoint')
PSQL.run_sql_command('checkpoint')
PSQL.run_sql_command('checkpoint')
PSQL.run_sql_command('checkpoint')
PSQL.run_sql_command('checkpoint')
PSQL.run_sql_command('checkpoint')
PSQL.run_sql_command('checkpoint')
PSQL.run_sql_command('checkpoint')
PSQL.run_sql_command('checkpoint')
PSQL.run_sql_command('checkpoint')
time.sleep(2)
# Promote standby
# We don't kill/stop the primary, as it's convenient for
# next testing
logger.info('Promote the standby immediatly')
self.standby.promote()
logger.info('Wait for the standby to be ready to accept connections ...')
time.sleep(3)
# Verify the result replicated to the standby.
logger.info('Verify if table foo exists...')
proc = self.run_sql('select count(*) from foo',
str(self.standby.port))
# The table should exist
stdout = proc.communicate()[0]
logger.info(stdout)
search = "1000"
self.assertTrue(stdout.find(search) >= 0)
logger.info('Pass')
def __init__(self, methodName):
self.gpinitstandby= GpinitStandby()
self.gputil = GpUtility()
super(OOMGpstartTestCase,self).__init__(methodName)
class PromoteTestCase(MPPTestCase):
'''testcase for gpstart'''
def __init__(self,methodName):
self.pgutil = GpUtility()
super(PromoteTestCase,self).__init__(methodName)
def setUp(self):
self.pgutil.check_and_start_gpdb()
# We should forcibly recreate standby, as it might has been promoted.
self.pgutil.remove_standby()
self.pgutil.install_standby()
def tearDown(self):
self.pgutil.remove_standby()
def test_promote_incomplete_stdby(self):
'''
remove the standby base dir, try promote and check if fail
'''
gpactivate_stdby = GpactivateStandby()
gpinit_stdby = GpinitStandby()
stdby_mdd = gpactivate_stdby.get_standby_dd()
stdby_host = gpinit_stdby.get_standbyhost()
stdby_port = gpactivate_stdby.get_standby_port()
destDir = os.path.join(stdby_mdd, 'base')
self.pgutil.clean_dir(stdby_host,destDir)
promote_cmd = "pg_ctl promote -D %s"%stdby_mdd
(rc, output) = gpactivate_stdby.run_remote(stdby_host,promote_cmd ,stdby_port,stdby_mdd)
self.assertEqual(rc, 0)
pid = self.pgutil.get_pid_by_keyword(host=stdby_host, pgport=stdby_port, keyword='master', option='bin')
self.assertTrue(int(pid) == -1, 'incomplete standby data directory promote succeeds.')
class OOMClass(object):
'''Class for methods required for OOM testcase'''
standby_port = '5433'
standby_dirname = 'newstandby'
def __init__(self):
self.gpinit = GpinitStandby()
self.mdd = os.environ.get('MASTER_DATA_DIRECTORY')
self.config = GPDBConfig()
self.pgutil = GpUtility()
self.verify = StandbyVerify()
self.host = socket.gethostname()
self.standby_loc = os.path.join(os.path.split(self.mdd)[0],
self.standby_dirname)
self.standby = Standby(self.standby_loc, self.standby_port)
def create_standby(self):
self.pgutil.clean_dir(self.host,self.standby_loc)
self.gpinit.run(option = '-P %s -s %s -F pg_system:%s' % (self.standby_port, self.host, self.standby_loc))
def setup_oom(self):
# Build it before testing.
thisdir = os.path.dirname(__file__)
builddir = os.path.join(thisdir, 'lib')
subprocess.check_call(['make', '-C', builddir, 'install'])
#Copy oom_malloc.so and wrapper.sh to all the segment nodes
for host in config.get_hosts() :
if host.strip() == self.host :
continue
cmd = "gpssh -h %s -e 'mkdir -p %s'; scp %s/oom_malloc.so %s:%s/; scp %s/wrapper.sh %s:%s/" % (host.strip(), builddir, builddir, host.strip(), builddir, builddir, host.strip(), builddir)
self.pgutil.run(cmd)
def touch_malloc(self):
# Touch file oom_malloc in standby directory
cmd = 'touch %s/oom_malloc' % self.standby_loc
self.pgutil.run(cmd)
def startdb(self):
(rc, result) = self.pgutil.run('gpstart -a --wrapper %s' % (local_path('lib/wrapper.sh')))
if rc != 0 and 'Could not start standby master' in result :
return False
return True
def restartdb(self):
# Remove file oom_malloc from standby
cmd = 'rm %s/oom_malloc' % self.standby_loc
self.pgutil.run(cmd)
(rc, result) = self.pgutil.run('gpstop -ar')
if rc == 0 and (self.verify.check_pg_stat_replication()):
return True
return False
def psql_and_oom(self):
#Touch oom_malloc in standby_dir and issue PSQL : Check if processes are gone
self.touch_malloc()
PSQL.run_sql_command('Drop table if exists wal_oomt1;Create table wal_oomt1(a1 int, a2 text) with(appendonly=true);')
sleep(2)
if not (self.verify.check_standby_processes()):
return True
return False
def start_standby(self):
# Remove oom_malloc and start standby : Check if all processes are back
cmd = 'rm %s/oom_malloc' % self.standby_loc
self.pgutil.run(cmd)
res = self.standby.start()
sleep(2)
if (self.verify.check_standby_processes()) :
return True
return False
class WalReplKillProcessTestCase(TINCTestCase):
# this is not hard code, will be updated
stdby_host = 'localhost'
stdby_port = '5432'
def __init__(self,methodName):
self.gphome = os.environ.get('GPHOME')
self.pgport = os.environ.get('PGPORT')
self.pgdatabase = os.environ.get('PGDATABASE')
self.stdby_host = 'localhost'
self.master_dd = os.environ.get('MASTER_DATA_DIRECTORY')
self.pgutil = GpUtility()
self.stdby = StandbyVerify()
super(WalReplKillProcessTestCase,self).__init__(methodName)
def killProcess_byPid(self, signal=9, pid_toKill=[], host="localhost"):
pid_list = ""
for pid in pid_toKill:
pid_list = pid_list + " " + str(pid)
kill_cmd = "%s/bin/gpssh -h %s -e 'kill -%s %s'" % (os.environ.get('GPHOME'), host, signal, pid_list)
(rc, result) = self.pgutil.run(kill_cmd)
if rc == 0:
tinctest.logger.info("Process killed, %s" % result)
return True
else:
tinctest.logger.error("Killing process error, Status Code non zero, cmd: %s\n"%kill_cmd)
return False
def kill_walstartup(self):
gpstdby = GpinitStandby()
stdby_host = gpstdby.get_standbyhost()
activate_stdby = GpactivateStandby()
stdby_port = activate_stdby.get_standby_port()
pid_list = []
startup_pid = self.pgutil.get_pid_by_keyword(host=stdby_host, pgport=stdby_port, keyword="startup process")
if int(startup_pid) == -1:
tinctest.logger.error("error:startup process does not exist!")
return False
else:
pid_list.append(startup_pid)
self.killProcess_byPid(pid_toKill=pid_list, host=stdby_host)
def kill_walreceiver(self):
gpstdby = GpinitStandby()
stdby_host = gpstdby.get_standbyhost()
activate_stdby = GpactivateStandby()
stdby_port = activate_stdby.get_standby_port()
pid_list = []
walreceiver_pid = self.pgutil.get_pid_by_keyword(host=stdby_host, pgport=stdby_port, keyword="wal receiver process")
if int(walreceiver_pid) == -1:
tinctest.logger.error("error: wal receiver process does not exist!")
return False
else:
pid_list.append(walreceiver_pid)
self.killProcess_byPid(pid_toKill=pid_list, host=stdby_host)
def kill_walsender_check_postmaster_reset(self):
pid_list = []
walsender_old_pid=self.pgutil.get_pid_by_keyword(pgport=self.pgport,keyword="wal sender process")
if int(walsender_old_pid) == -1:
tinctest.logger.error("error: process wal sender does not exist on host")
return False
else:
pid_list.append(walsender_old_pid)
self.killProcess_byPid(pid_toKill=pid_list)
sleep(2)
walsender_new_pid=self.pgutil.get_pid_by_keyword(pgport=self.pgport,keyword="wal sender process")
if walsender_old_pid == walsender_new_pid:
raise Exception("Killing walsender failed to force postmaster reset")
else:
return True
def kill_transc_backend_check_reset(self):
dict_process = { 'stats collector process': -1, 'writer process': -1,
'checkpointer process': -1,'seqserver process': -1,
'ftsprobe process': -1,'sweeper process': -1,'wal sender process': -1}
for process in dict_process:
pid = self.pgutil.get_pid_by_keyword(pgport=self.pgport,keyword=process)
dict_process[process] = pid
self.kill_transc_backend()
for process in dict_process:
pid = self.pgutil.get_pid_by_keyword(pgport=self.pgport,keyword=process)
delay = 1
while dict_process.get(process) == pid and delay < 5:
pid = self.pgutil.get_pid_by_keyword(pgport=self.pgport,keyword=process)
sleep(1)
delay = delay +1
if delay == 5:
tinctest.logger.error("Killing transaction backend process failed to force postmaster reset: %s"%process)
raise Exception("Killing transaction backend process failed to force postmaster reset child process")
def kill_transc_backend(self):
pid_list = []
sql = "SELECT procpid FROM pg_stat_activity WHERE datname='{0}' AND current_query like 'INSERT INTO%'".format(self.pgdatabase)
tinctest.logger.info("running sql command to get transaction backend process: --- %s"%sql)
procid = PSQL.run_sql_command(sql, flags = '-q -t', dbname= self.pgdatabase)
count = 1
while not procid.strip() and count < 5:
sleep(1)
count += 1
procid = PSQL.run_sql_command(sql, flags = '-q -t', dbname= self.pgdatabase)
if procid.strip():
tinctest.logger.info("got procid to kill: %s " % procid)
pid_list.append(procid)
self.killProcess_byPid(pid_toKill = pid_list)
else:
tinctest.logger.error("There is no active backend process")
def check_stdby_stop(self):
gpstdby = GpinitStandby()
stdby_host = gpstdby.get_standbyhost()
activate_stdby = GpactivateStandby()
stdby_port = activate_stdby.get_standby_port()
master_pid = self.pgutil.get_pid_by_keyword(host=stdby_host, pgport=stdby_port, keyword="master", option = "bin")
if int(master_pid) != -1:
raise Exception("standby should stop but failed!")
def start_stdby(self):
gpstdby = GpinitStandby()
stdby_host = gpstdby.get_standbyhost()
stdby_dbid = self.get_standby_dbid()
activate_stdby = GpactivateStandby()
stdby_mdd = activate_stdby.get_standby_dd()
stdby_port = activate_stdby.get_standby_port()
cmd="pg_ctl -D %s -o '-p %s --gp_dbid=%s --gp_num_contents_in_cluster=2 --silent-mode=true -i -M master --gp_contentid=-1 -x 0 -E' start &"%(stdby_mdd, stdby_port, stdby_dbid)
self.run_remote(stdby_host,cmd,stdby_port,stdby_mdd)
def run_remote(self, standbyhost, rmt_cmd, pgport = '', standbydd = ''):
'''Runs remote command and returns rc, result '''
export_cmd = "source %s/greenplum_path.sh;export PGPORT=%s;export MASTER_DATA_DIRECTORY=%s" % (self.gphome, pgport, standbydd)
remote_cmd = "gpssh -h %s -e \"%s; %s\"" % (standbyhost, export_cmd, rmt_cmd)
cmd = Command(name='Running Remote command', cmdStr='%s' % remote_cmd)
tinctest.logger.info(" %s" % cmd)
cmd.run(validateAfter=False)
result = cmd.get_results()
return result.rc,result.stdout
def check_mirror_seg(self):
dbstate = DbStateClass('run_validation')
dbstate.check_mirrorintegrity(master=True)
def get_standby_dbid(self):
std_sql = "select dbid from gp_segment_configuration where content='-1' and role='m';"
standby_dbid = PSQL.run_sql_command(std_sql, flags = '-q -t', dbname= 'template1')
return standby_dbid.strip()
def run_transaction_backend(self):
tinctest.logger.info("local path for backend.sql is %s"%local_path(''))
for file in os.listdir(local_path('')):
if fnmatch.fnmatch(file,'backend.sql'):
PSQL.run_sql_file(local_path(file))
def get_down_segment(self):
query = 'select * from gp_segment_configuration where mode <> \'s\' and status <>\'u\''
result = PSQL.run_sql_command(query, flags = '-q -t', dbname='template1')
return result.strip()
def check_gpdb_status(self):
down_segments = self.get_down_segment()
self.assertEqual(down_segments,'')
def gpstart_helper(self):
'''helper method to run in scenario test'''
(rc, result) = self.pgutil.run('gpstart -a')
self.assertIn(rc,(0,1))
def gpstop_helper(self):
'''helper method to run in scenario test'''
cmd = Command('run gpstop', cmdStr = 'gpstop -a')
cmd.run(validateAfter=True)
def gpinitstandby_helper(self):
'''helper method to create a new standby'''
self.pgutil.install_standby()
def removestandby_helper(self):
''' helper method to remove standby'''
self.pgutil.remove_standby()
def verify_standby_sync(self):
if (self.stdby.check_gp_segment_config()) and (self.stdby.check_pg_stat_replication()) and (self.stdby.check_standby_processes()):
return True
else:
raise Exception('standby and master out of sync!')
def kill_standby_postmaster(self):
pid_list = []
delay = 0
postmaster_pid = self.pgutil.get_pid_by_keyword(host=WalReplKillProcessTestCase.stdby_host, pgport=WalReplKillProcessTestCase.stdby_port, keyword="master", option="bin")
while int(postmaster_pid) == -1 and delay < 20:
sleep(1)
delay = delay + 1
postmaster_pid = self.pgutil.get_pid_by_keyword(host=WalReplKillProcessTestCase.stdby_host, pgport=WalReplKillProcessTestCase.stdby_port, keyword="master", option="bin")
if int(postmaster_pid) == -1 or delay == 20:
tinctest.logger.error("error: standby postmaster process does not exist!")
return False
else:
pid_list.append(postmaster_pid)
return self.killProcess_byPid(pid_toKill=pid_list, host=WalReplKillProcessTestCase.stdby_host)
def initial_setup(self):
keyword = 'rh55-qavm65'
config = GPDBConfig()
(seg_host,seg_port) = config.get_hostandport_of_segment(psegmentNumber = 0, pRole = 'p')
cur_path = local_path('')
dir1 = os.path.join(cur_path, 'dml', 'sql','insert_from_external.sql.in')
dir2 = os.path.join(cur_path, 'dml', 'sql','insert_from_external.sql')
dir3 = os.path.join(cur_path, 'dml', 'expected','insert_from_external.ans.in')
dir4 = os.path.join(cur_path, 'dml', 'expected','insert_from_external.ans')
f1 = open(dir1,'r')
f2 = open(dir2,'w')
f3 = open(dir3,'r')
f4 = open(dir4,'w')
for line in f1:
f2.write(line.replace(keyword,seg_host))
f1.close()
f2.close()
for line in f3:
f4.write(line.replace(keyword,seg_host))
f3.close()
f4.close()
dir5 = os.path.join(cur_path, 'dml', 'sql','insert_with_gpload.sql.in')
dir6 = os.path.join(cur_path, 'dml', 'sql','insert_with_gpload.sql')
yaml_path = local_path('dml/sql/config/gpl.yaml')
f5 = open(dir5,'r')
f6 = open(dir6,'w')
for line in f5:
f6.write(line.replace('gpl.yaml',yaml_path))
f5.close()
f6.close()
dir7 = os.path.join(cur_path,'dml','sql','config','gpl.yaml.in')
dir8 = os.path.join(cur_path,'dml','sql','config','gpl.yaml')
f7 = open(dir7,'r')
f8 = open(dir8,'w')
for line in f7:
if 'DATABASE' in line:
f8.write(line.replace('tangp3',os.environ.get('PGDATABASE')))
elif 'USER' in line:
f8.write(line.replace('tangp3',os.environ.get('USER')))
elif 'HOST' in line:
f8.write(line.replace('rh55-qavm61',socket.gethostname()))
elif 'PORT' in line and '5432' in line:
f8.write(line.replace('5432',os.environ.get('PGPORT')))
elif 'mydata' in line:
f8.write(line.replace('mydata',local_path('dml/sql/gpload/mydata')))
else:
f8.write(line)
f7.close()
f8.close()
dir9 = os.path.join(cur_path,'dml','expected','insert_with_gpload.ans.in')
dir10 = os.path.join(cur_path,'dml','expected','insert_with_gpload.ans')
f9 = open(dir9,'r')
f10 = open(dir10,'w')
for line in f9:
f10.write(line.replace('gpl.yaml',yaml_path))
f9.close()
f10.close()
dir11 = os.path.join(cur_path,'dml','sql','select_from_copy_table.sql.in')
dir12 = os.path.join(cur_path,'dml','sql','select_from_copy_table.sql')
f11 = open(dir11,'r')
f12 = open(dir12,'w')
for line in f11:
if 'tenk.data' in line:
f12.write(line.replace('tenk.data',local_path('dml/sql/_data/tenk.data')))
else:
f12.write(line)
f11.close()
f12.close()
dir13 = os.path.join(cur_path,'dml','expected','select_from_copy_table.ans.in')
dir14 = os.path.join(cur_path,'dml','expected','select_from_copy_table.ans')
f13 = open(dir13,'r')
f14 = open(dir14,'w')
for line in f13:
if 'tenk.data' in line:
f14.write(line.replace('tenk.data',local_path('dml/sql/_data/tenk.data')))
else:
f14.write(line)
f13.close()
f14.close()
external_table = local_path('dml/sql/_data/quote.csv')
clean_file = 'rm -rf /tmp/quote.csv'
rmt_cmd = "gpssh -h %s -e '%s' " % (seg_host, clean_file)
cmd = Command(name='Running a remote command', cmdStr = rmt_cmd)
cmd.run(validateAfter=False)
command = 'scp %s %s:/tmp'%(external_table,seg_host)
cmd = Command(name='run %s'%command, cmdStr = '%s' % command)
try:
cmd.run(validateAfter=True)
except Exception, e:
tinctest.logger.error("Error running command %s\n" % e)
def __init__(self,methodName):
self.gputil = GpUtility()
self.stdby = StandbyVerify()
super(GpstartTestCase,self).__init__(methodName)
class GpstartTestCase(MPPTestCase):
'''
testcase for gpstart
gpstart may return status code 1 as well as 0 in the success case. The
difference is whether it produces WARNING or not, but here we don't care.
'''
origin_mdd = os.environ.get('MASTER_DATA_DIRECTORY')
def __init__(self,methodName):
self.gputil = GpUtility()
self.stdby = StandbyVerify()
super(GpstartTestCase,self).__init__(methodName)
def setUp(self):
self.gputil.check_and_start_gpdb()
stdby_presence = self.gputil.check_standby_presence()
# We should forcibly recreate standby, as it might has been promoted.
if stdby_presence:
self.gputil.remove_standby()
self.gputil.install_standby()
def tearDown(self):
self.gputil.remove_standby()
"""
Gpstart test cases in recovery mode
"""
def test_gpstart_from_master(self):
"""
tag
"""
self.gputil.check_and_stop_gpdb()
(rc, stdout) = self.gputil.run('gpstart -a ')
self.assertIn(rc, (0, 1))
self.assertTrue(self.gputil.gpstart_and_verify())
sleep(2)
self.assertTrue(self.stdby.check_gp_segment_config(),'standby master not cofigured')
self.assertTrue(self.stdby.check_pg_stat_replication(),'standby not in replication status')
self.assertTrue(self.stdby.check_standby_processes(), 'standby processes not running')
(rc, output) = self.gputil.run(command = 'ps -ef|grep "wal sender "|grep -v grep')
self.assertIsNotNone(output)
def test_gpstart_master_only(self):
"""
tag
"""
self.gputil.check_and_stop_gpdb()
(rc, stdout) = self.gputil.run('export GPSTART_INTERNAL_MASTER_ONLY=1; '
'gpstart -a -m ')
self.assertIn(rc, (0, 1))
self.assertTrue(self.gputil.gpstart_and_verify())
(rc,output) = self.gputil.run('PGDATABASE=template1 '
"PGOPTIONS='-c gp_session_role=utility' "
'psql')
self.assertEqual(rc, 0)
(rc, output) = self.gputil.run('psql template1')
# should fail due to master only mode
self.assertEqual(rc, 2)
self.gputil.run('gpstop -a -m')
self.gputil.run('gpstart -a')
def test_gpstart_restricted_mode_master(self):
"""Test -R option with standby."""
self.gputil.check_and_stop_gpdb()
(rc, stdout) = self.gputil.run('gpstart -a -R')
self.assertIn(rc, (0, 1))
self.assertTrue(self.gputil.gpstart_and_verify())
(rc,output) = self.gputil.run(command = 'psql template1')
self.assertIn(rc, (0, 1))
self.gputil.run('gpstop -ar')
def test_gpstart_master_w_timeout(self):
"""Test -t option with standby."""
self.gputil.check_and_stop_gpdb()
(rc, output) = self.gputil.run('gpstart -a -t 30')
self.assertIn(rc, (0, 1))
self.assertTrue(self.gputil.gpstart_and_verify())
self.gputil.run('gpstop -ar')
def test_gpstart_no_standby(self):
"""Test -y with standby configured."""
self.gputil.check_and_stop_gpdb()
(rc, stdout) = self.gputil.run('gpstart -a -y')
self.assertIn(rc, (0, 1))
self.assertTrue(self.gputil.gpstart_and_verify())
self.assertFalse(self.stdby.check_standby_processes(),
'gpstart without standby failed, standby was running')
self.gputil.run('gpstop -ar')
def test_gpstart_wo_standby(self):
"""Test -y without standby configured."""
self.gputil.remove_standby()
self.gputil.check_and_stop_gpdb()
(rc, stdout) = self.gputil.run('gpstart -a -y')
self.assertIn(rc, (0, 1))
self.assertTrue(self.gputil.gpstart_and_verify())
self.assertFalse(self.stdby.check_standby_processes(), 'standby processes presented')
self.gputil.run('gpstop -ar')
"""
Gpstart, test case in failover mode
"""
def test_gpstart_master_only_after_failover(self):
"""
for test purpose, failing back to old master should
remove standby from primary after activate standby
"""
tinctest.logger.info("start master only with -m option after failover")
activatestdby = GpactivateStandby()
standby_host = activatestdby.get_current_standby()
standby_mdd = activatestdby.get_standby_dd()
standby_port = activatestdby.get_standby_port()
activatestdby.activate()
self.stdby._run_remote_command(standby_host,command = 'gpstop -a')
stdout = self.stdby._run_remote_command(standby_host,command = 'export GPSTART_INTERNAL_MASTER_ONLY=1; gpstart -a -m')
self.assertNotRegexpMatches(stdout,"ERROR","Start master only after failover failed")
self.assertTrue(self.gputil.gpstart_and_verify(master_dd = standby_mdd, host = standby_host))
self.stdby._run_remote_command(standby_host,command = 'gpstop -a -m')
self.gputil.run(command = 'gpstop -ar')
self.gputil.failback_to_original_master(self.origin_mdd, standby_host, standby_mdd, standby_port)
def test_gpstart_master_after_failover(self):
"""
failover, start from new master, then recover the cluster back to
have the old master active.
"""
tinctest.logger.info("failover, and run gpstart master test")
self.gputil.check_and_start_gpdb()
activatestdby = GpactivateStandby()
standby_host = activatestdby.get_current_standby()
standby_mdd = activatestdby.get_standby_dd()
standby_port = activatestdby.get_standby_port()
activatestdby.activate()
self.stdby._run_remote_command(standby_host, command = 'gpstop -a')
stdout = self.stdby._run_remote_command(standby_host,command = 'gpstart -a')
self.assertNotRegexpMatches(stdout,"FATAL","ERROR")
self.assertTrue(self.gputil.gpstart_and_verify(master_dd = standby_mdd, host = standby_host))
self.gputil.failback_to_original_master(self.origin_mdd, standby_host, standby_mdd, standby_port)
def test_gpstart_original_master_after_promote(self):
"""
failover, start from new master, then recover the cluster back to
have the old master active.
"""
tinctest.logger.info("activate and run gpstart for original master")
activatestdby = GpactivateStandby()
standby_host = activatestdby.get_current_standby()
standby_mdd = activatestdby.get_standby_dd()
standby_port = activatestdby.get_standby_port()
activatestdby.activate()
(rc, stdout) = self.gputil.run('gpstart -a -v')
self.gputil.run('pg_controldata %s' % self.origin_mdd)
self.stdby._run_remote_command(standby_host, command = 'pg_controldata %s' % standby_mdd)
self.assertNotEqual(rc, 0)
# This below error message comes from gpstart product code (if its modified change it here as well.)
self.assertRegexpMatches(stdout,"Standby activated, this node no more can act as master.")
self.gputil.failback_to_original_master(self.origin_mdd, standby_host, standby_mdd, standby_port)
def __init__(self, methodName):
self.gp = GpinitStandby()
self.mdd = os.environ.get('MASTER_DATA_DIRECTORY')
self.pgutil = GpUtility()
super(WalRecTestCase,self).__init__(methodName)
class GpinitStandby(object):
'''Class for gpinitstandby operations
Disclaimer: Some of these may repeat with the mpp/lib version'''
def __init__(self):
self.stdby = StandbyVerify()
self.runmixin = StandbyRunMixin()
self.runmixin.createdb(dbname='walrepl')
self.mdd = os.environ.get('MASTER_DATA_DIRECTORY')
self.config = GPDBConfig()
self.pgutil = GpUtility()
self.host = socket.gethostname()
def run(self, option = ''):
'''Runs gpinitstandby and returns True if successfull '''
gpinitstandby_cmd = 'gpinitstandby -a %s' % option
cmd = Command(name='Running Gpinitstandby', cmdStr="%s" % gpinitstandby_cmd)
tinctest.logger.info(" %s" % cmd)
cmd.run(validateAfter=False)
result = cmd.get_results()
if result.rc != 0:
return False
return True
def verify_gpinitstandby(self, primary_pid):
'''Verify the presence of standby in recovery mode '''
if (self.stdby.check_gp_segment_config()) and (self.stdby.check_pg_stat_replication()) and (self.stdby.check_standby_processes())and self.compare_primary_pid(primary_pid) :
return True
return False
def get_masterhost(self):
std_sql = "select hostname from gp_segment_configuration where content=-1 and role='p';"
master_host = PSQL.run_sql_command(std_sql, flags = '-q -t', dbname= 'postgres')
return master_host.strip()
def get_standbyhost(self):
std_sql = "select hostname from gp_segment_configuration where content='-1' and role='m';"
standby_host = PSQL.run_sql_command(std_sql, flags = '-q -t', dbname= 'postgres')
return standby_host.strip()
def get_filespace_location(self):
fs_sql = "select fselocation from pg_filespace_entry where fselocation like '%fs_walrepl_a%' and fsedbid=1;"
filespace_loc = PSQL.run_sql_command(fs_sql, flags = '-q -t', dbname= 'postgres')
return filespace_loc.strip()
def get_standbyhostnode(self):
'''
Function used to obtain the hostname of one of the segment node inorder to use it as the standby master node"
@return : returns the hostname of the segment node which can be used as the standby master node
'''
hostlist = self.config.get_hosts()
standby = ''
for host in hostlist:
if host.strip() != self.host:
standby = host.strip()
if len(standby) > 0 :
return standby
else:
tinctest.logger.error('No segment host other than master available to have remote standby')
def get_primary_pid(self):
pid = self.pgutil.get_pid_by_keyword(pgport=os.environ.get('PGPORT'), keyword=self.mdd)
if int(pid) == -1:
raise WalReplException('Unable to get pid of primary master process')
else:
return int(pid)
def compare_primary_pid(self, initial_pid):
final_pid = self.get_primary_pid()
if initial_pid == final_pid :
return True
return False
def create_dir_on_standby(self, standby, location):
fs_cmd = "gpssh -h %s -e 'rm -rf %s; mkdir -p %s' " % (standby, location, location)
cmd = Command(name='Make dierctory on standby before running the command', cmdStr = fs_cmd)
tinctest.logger.info('%s' % cmd)
cmd.run(validateAfter=True)
result = cmd.get_results()
if result.rc != 0:
raise WalReplException('Unable to create directory on standby')
else:
return True
def initstand_by_with_default(self):
master_host = self.get_masterhost()
gp_cmd = "/bin/bash -c 'gpinitstandby -s %s'" % (master_host)
logfile = open(local_path('install.log'),'w')
child = pexpect.spawn(gp_cmd, timeout=400)
child.logfile = logfile
sleep(2)
check = child.expect(['.* Enter standby filespace location for filespace pg_system .*', ' '])
if check != 0:
child.close()
l_file = open(local_path('install.log'),'r')
lines = l_file.readlines()
for line in lines:
if 'default: NA' in line:
return True
return False
def init_with_prompt(self,filespace_loc):
standby = self.get_standbyhostnode()
gp_cmd = "/bin/bash -c 'gpinitstandby -s %s -a'" % (standby)
logfile = open(local_path('install2.log'),'w')
child = pexpect.spawn(gp_cmd, timeout=400)
child.logfile = logfile
sleep(5)
check = child.expect(['.* Enter standby filespace location for filespace.*', ' '])
child.sendline(filespace_loc)
sleep(10)
check = child.expect(['.*Successfully created standby master.*'])
if check != 0:
tinctest.logger.error('gpinitstandy failed')
return False
child.close()
return True
def init_standby(self):
pg = GpUtility()
pg.install_standby()
def __init__(self,methodName):
self.pgutil = GpUtility()
super(PromoteTestCase,self).__init__(methodName)
