class PromoteTestCase(MPPTestCase):
'''testcase for gpstart'''
def __init__(self,methodName):
self.pgutil = GpUtility()
super(PromoteTestCase,self).__init__(methodName)
def setUp(self):
self.pgutil.check_and_start_gpdb()
# We should forcibly recreate standby, as it might has been promoted.
self.pgutil.remove_standby()
self.pgutil.install_standby()
def tearDown(self):
self.pgutil.remove_standby()
def test_promote_incomplete_stdby(self):
'''
remove the standby base dir, try promote and check if fail
'''
gpactivate_stdby = GpactivateStandby()
gpinit_stdby = GpinitStandby()
stdby_mdd = gpactivate_stdby.get_standby_dd()
stdby_host = gpinit_stdby.get_standbyhost()
stdby_port = gpactivate_stdby.get_standby_port()
destDir = os.path.join(stdby_mdd, 'base')
self.pgutil.clean_dir(stdby_host,destDir)
promote_cmd = "pg_ctl promote -D %s"%stdby_mdd
(rc, output) = gpactivate_stdby.run_remote(stdby_host,promote_cmd ,stdby_port,stdby_mdd)
self.assertEqual(rc, 0)
pid = self.pgutil.get_pid_by_keyword(host=stdby_host, pgport=stdby_port, keyword='master', option='bin')
self.assertTrue(int(pid) == -1, 'incomplete standby data directory promote succeeds.')
class PromoteTestCase(MPPTestCase):
'''testcase for gpstart'''
def __init__(self, methodName):
self.pgutil = GpUtility()
super(PromoteTestCase, self).__init__(methodName)
def setUp(self):
self.pgutil.check_and_start_gpdb()
# We should forcibly recreate standby, as it might has been promoted.
self.pgutil.remove_standby()
self.pgutil.install_standby()
def tearDown(self):
self.pgutil.remove_standby()
def test_promote_incomplete_stdby(self):
'''
remove the standby base dir, try promote and check if fail
'''
gpactivate_stdby = GpactivateStandby()
gpinit_stdby = GpinitStandby()
stdby_mdd = gpactivate_stdby.get_standby_dd()
stdby_host = gpinit_stdby.get_standbyhost()
stdby_port = gpactivate_stdby.get_standby_port()
destDir = os.path.join(stdby_mdd, 'base')
self.pgutil.clean_dir(stdby_host, destDir)
promote_cmd = "pg_ctl promote -D %s" % stdby_mdd
(rc, output) = gpactivate_stdby.run_remote(stdby_host, promote_cmd,
stdby_port, stdby_mdd)
self.assertEqual(rc, 0)
pid = self.pgutil.get_pid_by_keyword(host=stdby_host,
pgport=stdby_port,
keyword='master',
option='bin')
self.assertTrue(
int(pid) == -1,
'incomplete standby data directory promote succeeds.')
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
from mpp.models import MPPTestCase
from mpp.lib.config import GPDBConfig
from mpp.gpdb.tests.storage.walrepl import lib as walrepl
from mpp.gpdb.tests.storage.walrepl.lib.pqwrap import *
from mpp.gpdb.tests.storage.walrepl.lib.pg_util import GpUtility
import subprocess
import os
gputil = GpUtility()
gputil.check_and_start_gpdb()
config = GPDBConfig()
class GPLibPQTestCase(MPPTestCase):
"""This test case is to use gplibpq library, which is SQL function
interface for walreceiver's libpq part.
List of functions
Schema | Name | Result data type | Argument data types
--------+-----------------+------------------+----------------------
public | test_connect | boolean | text
public | test_disconnect | boolean |
public | test_receive | boolean |
public | test_scenario1 | boolean | text
public | test_send | boolean |
"""
from gppylib.commands.base import Command
from mpp.gpdb.tests.storage.walrepl.gpactivatestandby import GpactivateStandby
from mpp.gpdb.tests.storage.walrepl.gpinitstandby import GpinitStandby
from mpp.gpdb.tests.storage.walrepl.lib import NewEnv
from mpp.gpdb.tests.storage.walrepl.lib.pg_util import GpUtility
from mpp.lib.PSQL import PSQL
from tinctest.lib import local_path
import tinctest
import os
import shutil
import socket
from time import sleep
pgutil = GpUtility()
pgutil.check_and_start_gpdb()
origin_mdd = os.environ.get('MASTER_DATA_DIRECTORY')
class TakeFive(MPPTestCase):
port = os.environ.get('PGPORT')
mdd = os.environ.get('MASTER_DATA_DIRECTORY')
def __init__(self, methodName):
super(TakeFive, self).__init__(methodName)
def setUp(self):
Command('gpinitstandby -ra', 'gpinitstandby -ra').run()
def tearDown(self):
from gppylib.commands.base import Command
from mpp.gpdb.tests.storage.walrepl.gpactivatestandby import GpactivateStandby
from mpp.gpdb.tests.storage.walrepl.gpinitstandby import GpinitStandby
from mpp.gpdb.tests.storage.walrepl.lib import NewEnv
from mpp.gpdb.tests.storage.walrepl.lib.pg_util import GpUtility
from mpp.lib.PSQL import PSQL
from tinctest.lib import local_path
import tinctest
import os
import shutil
import socket
from time import sleep
pgutil = GpUtility()
pgutil.check_and_start_gpdb()
origin_mdd = os.environ.get('MASTER_DATA_DIRECTORY')
class TakeFive(MPPTestCase):
port = os.environ.get('PGPORT')
mdd = os.environ.get('MASTER_DATA_DIRECTORY')
def __init__(self, methodName):
super(TakeFive, self).__init__(methodName)
def setUp(self):
Command('gpinitstandby -ra', 'gpinitstandby -ra').run()
def tearDown(self):
# failback_to_original_master responsible for removing standby, but
"""
import unittest2 as unittest
from tinctest import logger
from gppylib.commands.base import Command
from mpp.gpdb.tests.storage.walrepl import lib as walrepl
from mpp.gpdb.tests.storage.walrepl.walreceiver import GPLibPQTestCase
from mpp.gpdb.tests.storage.walrepl.lib.pqwrap import *
from mpp.gpdb.tests.storage.walrepl.lib.pg_util import GpUtility
from mpp.lib.config import GPDBConfig
import select
gputil = GpUtility()
gputil.check_and_start_gpdb()
config = GPDBConfig()
class case(GPLibPQTestCase):
"""Basic test cases to see gplibpq and walsender are talking correctly."""
def setUp(self):
# cleanup
cmd = Command('gpinitstandby', 'gpinitstandby -ar')
# don't care the result in case standby is not configured
cmd.run()
@unittest.skipIf(config.is_multinode(), "Test applies only to a singlenode cluster")
def test_connect(self):
"""
@tags sanity
class GpstopTestCase(MPPTestCase):
'''testcase for gpstart'''
origin_mdd = os.environ.get('MASTER_DATA_DIRECTORY')
def __init__(self,methodName):
self.gputil = GpUtility()
super(GpstopTestCase,self).__init__(methodName)
def setUp(self):
self.gputil.check_and_start_gpdb()
stdby_presence = self.gputil.check_standby_presence()
if stdby_presence:
self.gputil.remove_standby()
self.gputil.install_standby()
def tearDown(self):
self.gputil.remove_standby()
self.gputil.run('gpstart -a')
self.gputil.run('gprecoverseg -a')
def test_gpstop_from_master(self):
self.assertTrue(self.gputil.gpstop_and_verify())
self.gputil.run('gpstart -a')
def test_gpstop_master_only(self):
self.assertTrue(self.gputil.gpstop_and_verify(option = '-m'))
self.gputil.run('gpstart -a')
def test_gpstop_fast(self):
#run transactions, and stop fast, check if transaction aborted, and the cluster was stopped
self.assertTrue(self.gputil.gpstop_and_verify(option = '-M fast'))
self.gputil.run('gpstart -a')
def test_gpstop_immediate(self):
self.assertTrue(self.gputil.gpstop_and_verify(option = '-M immediate'))
self.gputil.run('gpstart -a')
def test_gpstop_smart(self):
self.assertTrue(self.gputil.gpstop_and_verify(option = '-M smart'))
self.gputil.run('gpstart -a')
def test_gpdb_restart(self):
self.assertTrue(self.gputil.gpstop_and_verify('-r'))
def test_gpdb_reload(self):
self.assertTrue(self.gputil.gpstop_and_verify('-u'))
def test_gpstop_except_stdby(self):
self.assertTrue(self.gputil.gpstop_and_verify('-y'))
self.gputil.run('gpstart -y')
def test_gpstop_after_failover(self):
tinctest.logger.info("test gpstop from new master after failover")
activatestdby = GpactivateStandby()
standby_host = activatestdby.get_current_standby()
standby_port = activatestdby.get_standby_port()
standby_mdd = activatestdby.get_standby_dd()
activatestdby.activate()
(rc,stdout)=activatestdby.run_remote(standby_host, rmt_cmd='gpstop -a -M fast',
pgport=standby_port,standbydd=standby_mdd)
self.assertEqual(0,rc)
activatestdby.run_remote(standby_host, rmt_cmd='gpstart -a',
pgport=standby_port,
standbydd=standby_mdd)
self.gputil.failback_to_original_master(self.origin_mdd, standby_host, standby_mdd, standby_port)
class GpstateTestCase(MPPTestCase):
'''testcase for gpstart'''
origin_mdd = os.environ.get('MASTER_DATA_DIRECTORY')
def __init__(self,methodName):
self.gputil = GpUtility()
self.mirrorConfig = []
self.master_port = os.environ.get('PGPORT')
self.masterdd = os.environ.get('MASTER_DATA_DIRECTORY')
self.activatestdby = ""
super(GpstateTestCase,self).__init__(methodName)
def setUp(self):
self.gputil.check_and_start_gpdb()
stdby_presence = self.gputil.check_standby_presence()
if stdby_presence:
self.gputil.remove_standby()
self.gputil.install_standby()
get_mirror_sql = '''select port, hostname, fselocation
from gp_segment_configuration, pg_filespace_entry
where dbid = fsedbid and content != -1 and preferred_role=\'m\' ;'''
segments=self.gputil.run_SQLQuery(get_mirror_sql, dbname='template1')
for seg in segments:
port = seg[0]
host = seg[1]
dir = seg[2]
self.mirrorConfig.append(port)
self.mirrorConfig.append(host)
self.mirrorConfig.append(dir)
self.activatestdby = GpactivateStandby()
def tearDown(self):
del self.mirrorConfig[:]
self.gputil.remove_standby()
def test_gpstate_disp_recovery(self):
''' run gpstate with -f option'''
standby_host = self.activatestdby.get_current_standby()
standby_port = self.activatestdby.get_standby_port()
standby_dir = self.activatestdby.get_standby_dd()
standby_pid = self.gputil.get_pid_by_keyword(host=standby_host, user=os.environ.get('USER'),
pgport=standby_port,keyword='master',option='bin')
(rc, stdout) = self.gputil.run('gpstate -f')
self.assertEqual(rc, 0)
context = stdout.split('\n')
for line in context:
if "=" not in line:
continue
items = line.split('=')
if "Standby address" in line:
stdby_addr = items[1].strip()
self.assertEqual(stdby_addr, standby_host)
elif "Standby data directory" in line:
stdby_dir = items[1].strip()
self.assertEqual(stdby_dir, standby_dir)
elif "Standby port" in line:
stdby_port = int(items[1].strip())
self.assertEqual(stdby_port, int(standby_port))
elif "Standby PID" in line:
pid = items[1].strip()
self.assertEqual(pid, standby_pid)
def test_gpstate_disp_failover(self):
'''test if the master configuration detail changed after activating standby'''
standby_host = self.activatestdby.get_current_standby()
standby_port = self.activatestdby.get_standby_port()
standby_dir = self.activatestdby.get_standby_dd()
standby_pid = self.gputil.get_pid_by_keyword(host=standby_host, user=os.environ.get('USER'),
pgport=standby_port,keyword='master',option='bin')
self.activatestdby.activate()
(rc,stdout)=self.activatestdby.run_remote(standby_host, rmt_cmd='gpstate -s',
pgport=standby_port,
standbydd=standby_dir)
self.assertIn(rc, (0,1))
context = stdout.split('\n')
for line in context:
if "=" not in line:
continue
items = line.strip().split('=')
if "Master host" in line:
master_host = items[1].strip()
self.assertEqual(master_host, standby_host)
elif "Master postgres process ID" in line:
master_pid = items[1].strip()
self.assertEqual(master_pid, standby_pid)
elif "Master data directory" in line:
master_dir = items[1].strip()
self.assertEqual(master_dir, standby_dir)
elif "Master port" in line:
master_port = int(items[1].strip())
self.assertEqual(master_port, int(standby_port))
self.gputil.failback_to_original_master(self.origin_mdd,standby_host,standby_dir,standby_port)
def test_gpstate_active_segment_failover(self):
''' test if gpstate show correct # of up and down nodes after failover'''
count_up_seg = '''select count(*) from gp_segment_configuration
where content != -1 and status = \'u\';'''
count_down_seg = '''select count(*) from gp_segment_configuration
where content != -1 and status = \'d\';'''
number_up_segment = PSQL.run_sql_command(count_up_seg, flags = '-q -t', dbname='template1')
number_down_segment = PSQL.run_sql_command(count_down_seg, flags = '-q -t', dbname='template1')
standby_host = self.activatestdby.get_current_standby()
standby_port = self.activatestdby.get_standby_port()
standby_dir = self.activatestdby.get_standby_dd()
self.activatestdby.activate()
(rc,stdout)=self.activatestdby.run_remote(standby_host, rmt_cmd='gpstate -Q',
pgport=standby_port,
standbydd=standby_dir)
self.assertIn(rc, (0,1))
context = stdout.split('\n')
for line in context:
if "=" not in line:
continue
items = line.strip().split('=')
if "up segments" in line:
self.assertEqual(number_up_segment.strip(),items[1].strip())
elif "down segments" in line:
self.assertEqual(number_down_segment.strip(),items[1].strip())
self.gputil.failback_to_original_master(self.origin_mdd,standby_host,standby_dir,standby_port)
def test_gpstate_disp_mirror_failover(self):
''' check if new master is able to get correct mirror configuration with gpstate -m'''
inside_block = False
keywords = ("Mirror","Datadir","Port")
standby_host = self.activatestdby.get_current_standby()
standby_port = self.activatestdby.get_standby_port()
standby_dir = self.activatestdby.get_standby_dd()
self.activatestdby.activate()
(rc,stdout)=self.activatestdby.run_remote(standby_host, rmt_cmd='gpstate -m',
pgport=standby_port,
standbydd=standby_dir)
self.assertEqual(rc, 0)
for line in stdout:
if inside_block:
line_split = line.split('')
line_split = [elem for elem in line_split if elem != '']
mirror_host = line_split[2]
mirror_dir = line_split[3]
mirror_port = line_split[4]
self.assertTrue(mirror_host in self.mirrorConfig)
self.assertTrue(mirror_dir in self.mirrorConfig)
self.assertTrue(mirror_port in self.mirrorConfig)
elif not all (s in line for s in keywords):
continue
else:
inside_block = True
self.gputil.failback_to_original_master(self.origin_mdd,standby_host,standby_dir,standby_port)
class GpstartTestCase(MPPTestCase):
'''
testcase for gpstart
gpstart may return status code 1 as well as 0 in the success case. The
difference is whether it produces WARNING or not, but here we don't care.
'''
origin_mdd = os.environ.get('MASTER_DATA_DIRECTORY')
def __init__(self,methodName):
self.gputil = GpUtility()
self.stdby = StandbyVerify()
super(GpstartTestCase,self).__init__(methodName)
def setUp(self):
self.gputil.check_and_start_gpdb()
stdby_presence = self.gputil.check_standby_presence()
# We should forcibly recreate standby, as it might has been promoted.
if stdby_presence:
self.gputil.remove_standby()
self.gputil.install_standby()
def tearDown(self):
self.gputil.remove_standby()
"""
Gpstart test cases in recovery mode
"""
def test_gpstart_from_master(self):
"""
tag
"""
self.gputil.check_and_stop_gpdb()
(rc, stdout) = self.gputil.run('gpstart -a ')
self.assertIn(rc, (0, 1))
self.assertTrue(self.gputil.gpstart_and_verify())
sleep(2)
self.assertTrue(self.stdby.check_gp_segment_config(),'standby master not cofigured')
self.assertTrue(self.stdby.check_pg_stat_replication(),'standby not in replication status')
self.assertTrue(self.stdby.check_standby_processes(), 'standby processes not running')
(rc, output) = self.gputil.run(command = 'ps -ef|grep "wal sender "|grep -v grep')
self.assertIsNotNone(output)
def test_gpstart_master_only(self):
"""
tag
"""
self.gputil.check_and_stop_gpdb()
(rc, stdout) = self.gputil.run('export GPSTART_INTERNAL_MASTER_ONLY=1; '
'gpstart -a -m ')
self.assertIn(rc, (0, 1))
self.assertTrue(self.gputil.gpstart_and_verify())
(rc,output) = self.gputil.run('PGDATABASE=template1 '
"PGOPTIONS='-c gp_session_role=utility' "
'psql')
self.assertEqual(rc, 0)
(rc, output) = self.gputil.run('psql template1')
# should fail due to master only mode
self.assertEqual(rc, 2)
self.gputil.run('gpstop -a -m')
self.gputil.run('gpstart -a')
def test_gpstart_restricted_mode_master(self):
"""Test -R option with standby."""
self.gputil.check_and_stop_gpdb()
(rc, stdout) = self.gputil.run('gpstart -a -R')
self.assertIn(rc, (0, 1))
self.assertTrue(self.gputil.gpstart_and_verify())
(rc,output) = self.gputil.run(command = 'psql template1')
self.assertIn(rc, (0, 1))
self.gputil.run('gpstop -ar')
def test_gpstart_master_w_timeout(self):
"""Test -t option with standby."""
self.gputil.check_and_stop_gpdb()
(rc, output) = self.gputil.run('gpstart -a -t 30')
self.assertIn(rc, (0, 1))
self.assertTrue(self.gputil.gpstart_and_verify())
self.gputil.run('gpstop -ar')
def test_gpstart_no_standby(self):
"""Test -y with standby configured."""
self.gputil.check_and_stop_gpdb()
(rc, stdout) = self.gputil.run('gpstart -a -y')
self.assertIn(rc, (0, 1))
self.assertTrue(self.gputil.gpstart_and_verify())
self.assertFalse(self.stdby.check_standby_processes(),
'gpstart without standby failed, standby was running')
self.gputil.run('gpstop -ar')
def test_gpstart_wo_standby(self):
"""Test -y without standby configured."""
self.gputil.remove_standby()
self.gputil.check_and_stop_gpdb()
(rc, stdout) = self.gputil.run('gpstart -a -y')
self.assertIn(rc, (0, 1))
self.assertTrue(self.gputil.gpstart_and_verify())
self.assertFalse(self.stdby.check_standby_processes(), 'standby processes presented')
self.gputil.run('gpstop -ar')
"""
Gpstart, test case in failover mode
"""
def test_gpstart_master_only_after_failover(self):
"""
for test purpose, failing back to old master should
remove standby from primary after activate standby
"""
tinctest.logger.info("start master only with -m option after failover")
activatestdby = GpactivateStandby()
standby_host = activatestdby.get_current_standby()
standby_mdd = activatestdby.get_standby_dd()
standby_port = activatestdby.get_standby_port()
activatestdby.activate()
self.stdby._run_remote_command(standby_host,command = 'gpstop -a')
stdout = self.stdby._run_remote_command(standby_host,command = 'export GPSTART_INTERNAL_MASTER_ONLY=1; gpstart -a -m')
self.assertNotRegexpMatches(stdout,"ERROR","Start master only after failover failed")
self.assertTrue(self.gputil.gpstart_and_verify(master_dd = standby_mdd, host = standby_host))
self.stdby._run_remote_command(standby_host,command = 'gpstop -a -m')
self.gputil.run(command = 'gpstop -ar')
self.gputil.failback_to_original_master(self.origin_mdd, standby_host, standby_mdd, standby_port)
def test_gpstart_master_after_failover(self):
"""
failover, start from new master, then recover the cluster back to
have the old master active.
"""
tinctest.logger.info("failover, and run gpstart master test")
self.gputil.check_and_start_gpdb()
activatestdby = GpactivateStandby()
standby_host = activatestdby.get_current_standby()
standby_mdd = activatestdby.get_standby_dd()
standby_port = activatestdby.get_standby_port()
activatestdby.activate()
self.stdby._run_remote_command(standby_host, command = 'gpstop -a')
stdout = self.stdby._run_remote_command(standby_host,command = 'gpstart -a')
self.assertNotRegexpMatches(stdout,"FATAL","ERROR")
self.assertTrue(self.gputil.gpstart_and_verify(master_dd = standby_mdd, host = standby_host))
self.gputil.failback_to_original_master(self.origin_mdd, standby_host, standby_mdd, standby_port)
def test_gpstart_original_master_after_promote(self):
"""
failover, start from new master, then recover the cluster back to
have the old master active.
"""
tinctest.logger.info("activate and run gpstart for original master")
activatestdby = GpactivateStandby()
standby_host = activatestdby.get_current_standby()
standby_mdd = activatestdby.get_standby_dd()
standby_port = activatestdby.get_standby_port()
activatestdby.activate()
(rc, stdout) = self.gputil.run('gpstart -a -v')
self.gputil.run('pg_controldata %s' % self.origin_mdd)
self.stdby._run_remote_command(standby_host, command = 'pg_controldata %s' % standby_mdd)
self.assertNotEqual(rc, 0)
# This below error message comes from gpstart product code (if its modified change it here as well.)
self.assertRegexpMatches(stdout,"Standby activated, this node no more can act as master.")
self.gputil.failback_to_original_master(self.origin_mdd, standby_host, standby_mdd, standby_port)
class GpstopTestCase(MPPTestCase):
'''testcase for gpstart'''
origin_mdd = os.environ.get('MASTER_DATA_DIRECTORY')
def __init__(self,methodName):
self.gputil = GpUtility()
super(GpstopTestCase,self).__init__(methodName)
def setUp(self):
self.gputil.check_and_start_gpdb()
stdby_presence = self.gputil.check_standby_presence()
if stdby_presence:
self.gputil.remove_standby()
self.gputil.install_standby()
def tearDown(self):
self.gputil.remove_standby()
self.gputil.run('gpstart -a')
def test_gpstop_from_master(self):
self.assertTrue(self.gputil.gpstop_and_verify())
self.gputil.run('gpstart -a')
def test_gpstop_master_only(self):
self.assertTrue(self.gputil.gpstop_and_verify(option = '-m'))
self.gputil.run('gpstart -a')
def test_gpstop_fast(self):
#run transactions, and stop fast, check if transaction aborted, and the cluster was stopped
self.assertTrue(self.gputil.gpstop_and_verify(option = '-M fast'))
self.gputil.run('gpstart -a')
def test_gpstop_immediate(self):
self.assertTrue(self.gputil.gpstop_and_verify(option = '-M immediate'))
self.gputil.run('gpstart -a')
def test_gpstop_smart(self):
self.assertTrue(self.gputil.gpstop_and_verify(option = '-M smart'))
self.gputil.run('gpstart -a')
def test_gpdb_restart(self):
self.assertTrue(self.gputil.gpstop_and_verify('-r'))
def test_gpdb_reload(self):
self.assertTrue(self.gputil.gpstop_and_verify('-u'))
def test_gpstop_except_stdby(self):
self.assertTrue(self.gputil.gpstop_and_verify('-y'))
self.gputil.run('gpstart -y')
def test_gpstop_after_failover(self):
tinctest.logger.info("test gpstop from new master after failover")
activatestdby = GpactivateStandby()
standby_host = activatestdby.get_current_standby()
standby_port = activatestdby.get_standby_port()
standby_mdd = activatestdby.get_standby_dd()
activatestdby.activate()
(rc,stdout)=activatestdby.run_remote(standby_host, rmt_cmd='gpstop -a -M fast',
pgport=standby_port,standbydd=standby_mdd)
self.assertEqual(0,rc)
activatestdby.run_remote(standby_host, rmt_cmd='gpstart -a',
pgport=standby_port,
standbydd=standby_mdd)
self.gputil.failback_to_original_master(self.origin_mdd, standby_host, standby_mdd, standby_port)
