def dropPL(self, plname=None):
"""
drop a given procedural language on gpdb
@param plname: the name of the procedural language to be dropped
@return: True if the given procedural language is dropped on gpdb or it does not exist, False if it is not correctly dropped
"""
if plname is None:
plname = ""
else:
plname = plname.lower()
if self.isSupported(plname):
if self.isInstalled(plname):
sql = "DROP LANGUAGE %s;" % (plname)
cmd = PSQL(sql_cmd = sql, flags = '-q -t', dbname=os.environ.get('PGDATABASE'))
tinctest.logger.info("Running command - %s" %cmd)
cmd.run(validateAfter = False)
result = cmd.get_results()
ok = result.rc
out = result.stdout.strip()
if not ok:
return True
else:
print out
return False
else:
return True
else:
raise Exception("Unsupported procedural language %s" % (plname))
def isSupported(self, plname=None):
"""
check if a given procedural language is supported by gpdb
@param plname: the name of the procedural language to be checked
@return: True if the given procedural language is supported by gpdb, False if it is not supported
"""
if plname is None:
plname = ""
else:
plname = plname.lower()
sql = "SELECT COUNT(*) FROM (SELECT tmplname AS lanname FROM pg_pltemplate UNION SELECT lanname AS lanname FROM pg_language) t WHERE lanname='%s';" % (plname)
cmd = PSQL(sql_cmd = sql, flags = '-q -t', dbname=os.environ.get('PGDATABASE'))
tinctest.logger.info("Running command - %s" %cmd)
cmd.run(validateAfter = False)
result = cmd.get_results()
ok = result.rc
out = result.stdout.strip()
if not ok:
ans = int( out[0].rstrip().lstrip() )
if ans == 0:
return False
elif ans == 1:
return True
else:
raise Exception("Error when retrieving information about procedural languages from catalog")
else:
raise Exception("Error when retrieving information about procedural languages from catalog")
def test_43_alter_table_with_oid(self):
'''MPP-13870: Alter table Set Without Oids fails in case of inheritance'''
sql_file = local_path('alter_table_with_oid.sql')
out_file = local_path('alter_table_with_oid.out')
ans_file = local_path('alter_table_with_oid.ans')
PSQL.run_sql_file(sql_file = sql_file, out_file = out_file)
self.assertTrue(Gpdiff.are_files_equal(out_file, ans_file))
def doTest(self, sql_filename):
'''Run the file, compare oids in out file '''
sql_file = local_path(sql_filename)
out_file = local_path(sql_filename.split('.sql')[0] + '.out')
PSQL.run_sql_file(sql_file = sql_file, out_file = out_file)
isOk = self.compare_oids(out_file)
self.assertTrue(isOk)
def test_with_fault_injection(self):
"""
add new mirrors run workload to verify if cluster functioning correctly, and
inject the mirror to bring cluster into change tracking, then recoverseg
"""
filerepUtil = Filerepe2e_Util()
gprecover = GpRecover()
self._setup_gpaddmirrors()
self._cleanup_segment_data_dir(self.host_file, self.mirror_data_dir)
res = {'rc': 0, 'stdout' : '', 'stderr': ''}
run_shell_command("gpaddmirrors -a -i %s -d %s --verbose" % (self.mirror_config_file, self.mdd), 'run gpaddmirrros with fault injection', res)
gprecover.wait_till_insync_transition()
self.assertEqual(0, res['rc'])
self.run_simple_ddl_dml()
# after adding new mirrors, check the intergrity between primary and mirror
self.check_mirror_seg()
out_file = local_path('inject_fault_into_ct')
filerepUtil.inject_fault(f='filerep_consumer', m='async', y='fault', r='mirror', H='ALL', outfile=out_file)
# trigger the transtion to change tracking
PSQL.run_sql_command('drop table if exists foo;', dbname = 'template1')
filerepUtil.wait_till_change_tracking_transition()
gprecover.incremental()
gprecover.wait_till_insync_transition()
out_file=local_path('reset_fault')
filerepUtil.inject_fault(f='filerep_consumer', m='async', y='reset', r='mirror', H='ALL', outfile=out_file)
def test_with_concurrent_workload(self):
"""
add new mirrors while concurrent workload in progress, check that mirrors added
and current workload won't get affected, in the end, run checkmirrorseg.
Note that: adding mirrors while running workload has checkmirrorseg issue with MPP-24311
"""
gprecover = GpRecover()
self._setup_gpaddmirrors()
self._cleanup_segment_data_dir(self.host_file, self.mirror_data_dir)
sql_setup_file = local_path('sql/ao_heap_table_setup.sql')
sql_file = local_path('sql/ao_heap_table.sql')
pg_stat_activity = 'SELECT * FROM pg_stat_activity;'
PSQL.run_sql_file(sql_setup_file)
subprocess.Popen(["psql", "-f", sql_file])
time.sleep(15)
subprocess.Popen(["gpaddmirrors", "-ai", self.mirror_config_file, "-d", self.mdd])
time.sleep(15)
result = PSQL.run_sql_command(pg_stat_activity, flags='-q -t', dbname='template1')
result = result.strip()
rows = result.split('\n')
self.assertTrue(len(rows) > 1)
while len(rows) > 1:
result = PSQL.run_sql_command(pg_stat_activity, flags='-q -t', dbname='template1')
result = result.strip()
rows = result.split('\n')
time.sleep(3)
gprecover.wait_till_insync_transition()
self.verify_config_file_with_gp_config()
def doQuery(self, sqlfile, default=''):
sql_file = local_path(sqlfile)
filename_prefix = sqlfile.split('.sql')[0]
out_file = local_path(filename_prefix + '.out')
ans_file = local_path(filename_prefix + '.ans')
PSQL.run_sql_file(sql_file = sql_file, out_file = out_file)
self.assertTrue(Gpdiff.are_files_equal(out_file, ans_file))
def verify_config_file_with_gp_config(self):
"""
compares the gp_segment_configuration and pg_filespace_entry with input file mirror_data_dir, double check
if the cluster is configured as intended
"""
with open(self.mirror_config_file, 'r') as f:
next(f)
for line in f:
line = line.strip()
mirror_seg_infor = line.split('=')[1]
cols = mirror_seg_infor.split(':')
content_id = cols[0]
adress = cols[1]
port = cols[2]
mir_replication_port = cols[3]
query_on_configuration = '''select * from gp_segment_configuration where content=\'%s\' and address=\'%s\' and port=\'%s\'
and replication_port=\'%s\'''' % (content_id, adress, port, mir_replication_port)
config_info = PSQL.run_sql_command(query_on_configuration, flags='-q -t', dbname='template1')
config_info = config_info.strip()
# as intended, the entry should be existing in the cluster
self.assertNotEqual(0, len(config_info))
query_on_fselocation = ''' select fselocation from gp_segment_configuration, pg_filespace_entry where dbid=fsedbid
and preferred_role=\'m\' and content=\'%s\''''%content_id
fs_locations = PSQL.run_sql_command(query_on_fselocation, flags='-q -t', dbname='template1')
size = len(cols)
for fs_index in range(5, size):
fs_location = cols[fs_index]
self.assertIn(os.path.dirname(fs_location), fs_locations)
def setUpClass(self):
super(MapreduceMPPTestCase, self).setUpClass()
gppkg = Gppkg()
gppkg.gppkg_install(product_version, 'plperl')
setup_command = "create language plperl;"
PSQL.run_sql_command(setup_command, dbname = os.environ.get('PGDATABASE'))
"compile functions.c and build functions.so"
makeLog = local_path('testBuildSOLog.out')
cmdMake = 'cd '+local_path('c_functions') + ' && make clean && make'
res = {'rc': 0, 'stdout' : '', 'stderr': ''}
run_shell_command(cmdMake, 'compile functions.c', res)
file = open(makeLog, 'w')
file.write(res['stdout'])
file.close()
if res['rc']:
raise Exception('a problem occurred while creating the so files ')
so_dir = local_path('c_functions')
sharedObj = local_path('c_functions/functions.so')
# if not os.path.isfile(sharedObj):
#raise gptest.GPTestError('so files does not exist')
# For multinode cluster, need to copy shared object tabfunc_gppc_demo.so to all primary segments
if gpdbconfig.is_multinode():
res = {'rc':0, 'stderr':'', 'stdout':''}
hosts = gpdbconfig.get_hosts(segments=True)
scp_cmd = 'gpscp -h ' +' -h '.join(map(str,hosts)) +' '+ sharedObj + ' =:%s' % so_dir
run_shell_command(scp_cmd)
if res['rc']:
raise Exception('Could not copy shared object to primary segment')
def test_scalar_consolidation_NEG_paramDiffType(self):
"""
scalar Consolidation NEG two parameters with different type
"""
self.runFunctionTest("scalar_consolidation","NEG_paramDiffType")
filename = local_path("c_functions/scalar_consolidation/NEG_paramDiffType_cleanup.sql")
PSQL.run_sql_file(filename)
def test_streaming(self):
"""
Run sendtest, let database emit WAL.
sendtest should receive a new WAL records. After all, we kill
the walsender process, otherwise the test doesn't finish.
@tags sanity
"""
PSQL.run_sql_command('DROP TABLE IF EXISTS foo')
with WalClient("replication=true") as client:
(sysid, tli, xpos) = client.identify_system()
xpos_ptr = XLogRecPtr.from_string(xpos)
client.start_replication(xpos_ptr)
# Can't use PSQL here as it is blocked due to Sync Rep
subprocess.Popen(['psql', '-c', 'CREATE TABLE foo(a int, b int)'],
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
while True:
msg = client.receive(1000)
if isinstance(msg, WalMessageData):
header = msg.header
# sync replication needs a reply otherwise backend blocks
client.reply(header.walEnd, header.walEnd, header.walEnd)
# success, should get some 'w' message
break
elif isinstance(msg, WalMessageNoData):
# could be timeout
client.reply(xpos_ptr, xpos_ptr, xpos_ptr)
else:
raise StandardError(msg.errmsg)
def run_test(self):
"""
Override of SQLTestCase. Initialize standby, run some sql,
then activate it, and check if the data is streamed correctly.
"""
sql_file = self.sql_file
ans_file = self.ans_file
nsender = self.wait_for_walsender()
self.assertEqual(nsender, 1, 'replication has not begun')
# setup script is run on primary while standby is running.
# .in file will be substitute with runtime information, if any.
setup_file = sql_file.replace('.sql', '_setup.sql')
if os.path.exists(setup_file + '.in'):
self.preprocess_file(setup_file + '.in')
self.assertTrue(PSQL.run_sql_file(setup_file, dbname=self.db_name))
self.standby_dir = self.activatestdby.get_standby_dd()
self.standby_port = self.activatestdby.get_standby_port()
self.standby_host = self.gpinit_stdby.get_standbyhost()
self.activatestdby.activate()
datadir = os.path.abspath(self.standby_datadir)
with walrepl.NewEnv(MASTER_DATA_DIRECTORY=self.standby_dir,
PGPORT=self.standby_port) as env:
result = super(gpactivatestandby, self).run_test()
sql = 'SHOW gp_dbid'
result = PSQL.run_sql_command(sql, flags='-A -t')
self.assertEqual(result.strip(), '1')
self.assertEqual(self.get_gp_dbid(self.standby_dir), 1, 'gp_dbid should show 1')
if 'cleanup_filespace' in self._metadata:
mpp.gpdb.tests.storage.walrepl.lib.cleanupFilespaces(dbname=self.db_name)
return result
def backend_terminate(self):
"""
Get the backend process pid by issuing a query to table pg_stat_activity, then
execute pg_terminate_backend(pid)
"""
MAX_TRY = 5
counter = 0
get_backend_pid = 'SELECT procpid FROM pg_stat_activity WHERE current_query like \'create temp table t as select%\';'
result = PSQL.run_sql_command(get_backend_pid, flags = '-q -t', dbname = self.db_name)
tinctest.logger.info('result from getting backend procepid is %s'%result)
procpid = result.strip()
while not procpid and counter < MAX_TRY:
result = PSQL.run_sql_command(get_backend_pid, flags = '-q -t', dbname = self.db_name)
procpid = result.strip()
counter += 1
if counter == MAX_TRY:
raise Exception('unable to select out the backend process pid')
kill_backend = 'SELECT pg_terminate_backend(%s);'%procpid
result = PSQL.run_sql_command(kill_backend, flags = '-q -t', dbname = self.db_name)
tinctest.logger.info('result from pg_terminate_backend is %s'%result)
# check if the process was terminated already
result = PSQL.run_sql_command(get_backend_pid, flags = '-q -t', dbname = self.db_name)
procpid_after_terminate = result.strip()
counter = 0
while procpid_after_terminate == procpid and counter < MAX_TRY:
result = PSQL.run_sql_command(get_backend_pid, flags = '-q -t', dbname = self.db_name)
procpid_after_terminate = result.strip()
counter += 1
time.sleep(1)
if counter == MAX_TRY:
raise Exception('Running pg_terminated_backend failed!')
def run_explain_indexjoin_on(self, query, explain_file, explain_file_out, planid_stmt, planid):
syscmd = "rm -f " + explain_file
os.popen(syscmd, 'r')
sqlcontent = [
"--start_ignore",
"set optimizer=on;",
"set optimizer_enable_indexjoin=on;",
"set optimizer_enable_bitmapscan=on;",
"select disable_xform('CXformInnerJoin2NLJoin');" if not planid_stmt else "",
"select disable_xform('CXformInnerJoin2HashJoin');",
planid_stmt,
"set client_min_messages='log';",
"--end_ignore",
"EXPLAIN" ]
for x in query:
sqlcontent.append(x)
self._add_tofile(explain_file, sqlcontent, 0)
PSQL.run_sql_file(explain_file, dbname = self.db_name, out_file = explain_file_out)
if planid != 0:
new_file = explain_file_out+str(planid)
syscmd_cp = "cp %s %s" % (explain_file_out, new_file)
os.popen(syscmd_cp, 'r')
if (self._look_for_string(explain_file_out, "Planner produced plan :1") == True):
return False, "Unexpected fallback with force indexapply on |"
elif (self._look_for_string(explain_file_out, "Index Cond", check_join_cond = True) == False) and self.negativetest == 'False':
return False, "Index Scan in join not being used with force indexapply | " + explain_file_out
elif (self._look_for_string(explain_file_out, "Index Cond", check_join_cond = True) == True) and self.negativetest == 'True':
return False, "Index Scan in join is being used when it shouldn't | "
else:
return True, None
def test_run_sql_file_wth_username(self):
sql_file = os.path.join(os.path.dirname(inspect.getfile(self.__class__)),'test.sql')
username = getpass.getuser()
self.assertTrue(PSQL.run_sql_file(sql_file = sql_file, username = username))
#Invalid username
self.assertFalse(PSQL.run_sql_file(sql_file = sql_file, username = '******'))
def template0_wrap_around(self):
"""
Raise next xid so that age(template0) suffers a wrap around and
becomes negative. Create a new database off template0, which
also suffers wrap around. Reset the new db's age. Sanity
must succeed on the new db.
"""
self._raise_template0_age(self.WRAP_LIMIT, self.gparray.master)
PSQL(sql_cmd="CREATE DATABASE newdb TEMPLATE template0").run(
validateAfter=True)
sql = "SELECT age(datfrozenxid) FROM pg_database WHERE datname='newdb'"
dburl = dbconn.DbURL()
with dbconn.connect(dburl, utility=True) as conn:
age_newdb = int(dbconn.execSQLForSingleton(conn, sql))
# Xid wrap-around should cause template0 and newdb's age to be negative.
self.assertTrue(age_newdb < 0)
# All xids in newdb are frozen at this point. Therefore, we
# can reset its age so that it is not negative.
self._reset_age("newdb")
with dbconn.connect(dburl, utility=True) as conn:
age_newdb = int(dbconn.execSQLForSingleton(conn, sql))
self.assertTrue(age_newdb > 0)
# Verify that normal operations can be performed on newdb post recovery
# from wraparound.
self._basic_sanity_check("clean", {"dbname":"newdb"})
logger.info("Sanity succeeded on newdb, dropping it.")
PSQL.drop_database(dbname="newdb")
def test_gprecoverseg_rebalance(self):
self.gprec.wait_till_insync_transition()
if(self.failover('primary')):
PSQL.run_sql_file(local_path('mirror_failover_trigger.sql'))
self.gprec.incremental()
if (self.gprec.wait_till_insync_transition()):
self.assertTrue(self.gprec.rebalance())
def test_20_use_udf_gp_aovisimap_hidden_info_uaocs_del(self):
tablename ='uaocs_table_test14'
tinctest.logger.info("-------------------------------")
tinctest.logger.info('test_20 Verify the hidden tup_count using UDF gp_aovisimap_hidden_info(oid) for uaocs relation after delete ')
tinctest.logger.info("-------------------------------\n")
out_file = os.path.join(self.outpath,'create_tab_gp_persistent_relation_node_uaocs_table_upd_14.out')
sql_file = os.path.join(self.sqlpath,'create_tab_gp_persistent_relation_node_uaocs_table_upd_14.sql')
ans_file= os.path.join(self.anspath,'create_tab_gp_persistent_relation_node_uaocs_table_upd_14.ans')
#create uaocs table and insert 10 rows
sql_out=PSQL.run_sql_file(sql_file = sql_file,out_file=out_file)
assert Gpdiff.are_files_equal(out_file, ans_file)
#get relid for newly created table
relid = self.get_relid(file_name=tablename )
#get utility mode connection info
utilitymodeinfo=self.get_utilitymode_conn_uaocs_info( relid=relid)
u_port=utilitymodeinfo[0]
u_host=utilitymodeinfo[1]
assert(0 == int(self.get_hidden_tup_cnt(relid=relid,host=u_host,port= u_port)))
# delete 1 row
sql_cmd3="delete from %s where i = (select min(i) from %s );" % (tablename, tablename)
PSQL.run_sql_command_utility_mode(sql_cmd= sql_cmd3,host=u_host, port=u_port,flags='-q -t')
assert(1 == int(self.get_hidden_tup_cnt(relid=relid,host=u_host,port= u_port)))
self.vacuum_full(tablename=tablename)
assert(0 == int(self.get_hidden_tup_cnt(relid=relid,host=u_host,port= u_port)))
def test_01(self):
"SPI: plpgsql"
sql_file = local_path("query01.sql")
out_file = local_path("query01.out")
ans_file = local_path("query01.ans")
PSQL.run_sql_file(sql_file=sql_file, out_file=out_file)
self.assertTrue(Gpdiff.are_files_equal(out_file, ans_file))
def test_xlogcleanup(self):
"""
Test for verifying if xlog seg created while basebackup
dumps out data does not get cleaned
"""
shutil.rmtree('base', True)
PSQL.run_sql_command('DROP table if exists foo')
# Inject fault at post checkpoint create (basebackup)
logger.info ('Injecting base_backup_post_create_checkpoint fault ...')
result = self.suspend_at(
'base_backup_post_create_checkpoint')
logger.info(result.stdout)
self.assertEqual(result.rc, 0, result.stdout)
# Now execute basebackup. It will be blocked due to the
# injected fault.
logger.info ('Perform basebackup with xlog & recovery.conf...')
pg_basebackup = subprocess.Popen(['pg_basebackup', '-x', '-R', '-D', 'base']
, stdout = subprocess.PIPE
, stderr = subprocess.PIPE)
# Give basebackup a moment to reach the fault &
# trigger it
logger.info('Check if suspend fault is hit ...')
triggered = self.wait_triggered(
'base_backup_post_create_checkpoint')
self.assertTrue(triggered, 'Fault was not triggered')
# Perform operations that causes xlog seg generation
logger.info ('Performing xlog seg generation ...')
count = 0
while (count < 10):
PSQL.run_sql_command('select pg_switch_xlog(); select pg_switch_xlog(); checkpoint;')
count = count + 1
# Resume basebackup
result = self.resume('base_backup_post_create_checkpoint')
logger.info(result.stdout)
self.assertEqual(result.rc, 0, result.stdout)
# Wait until basebackup end
logger.info('Waiting for basebackup to end ...')
sql = "SELECT count(*) FROM pg_stat_replication"
with dbconn.connect(dbconn.DbURL(), utility=True) as conn:
while (True):
curs = dbconn.execSQL(conn, sql)
results = curs.fetchall()
if (int(results[0][0]) == 0):
break;
# Verify if basebackup completed successfully
# See if recovery.conf exists (Yes - Pass)
self.assertTrue(os.path.exists(os.path.join('base','recovery.conf')))
logger.info ('Found recovery.conf in the backup directory.')
logger.info ('Pass')
def run_sql_file(self, sql_file, out_file = None, out_dir = None, optimizer=None):
"""
Given a sql file and an ans file, this adds the specified gucs (self.gucs) to the sql file , runs the sql
against the test case databse (self.db_name) and verifies the output with the ans file.
If an 'init_file' exists in the same location as the sql_file, this will be used
while doing gpdiff.
"""
result = True
self.test_artifacts.append(sql_file)
if not out_file:
out_file = os.path.join(self.get_out_dir(), os.path.basename(sql_file).replace('.sql','.out'))
self.test_artifacts.append(out_file)
tinctest.logger.info('running the sql testcase file:')
tinctest.logger.info(sql_file)
if (sql_file.find('hybrid_part_tbl_drop_col')>=0):
default_db = getpass.getuser()
dbase = os.getenv('PGDATABASE',default_db)
datasetobj = DataSetDatabase(database_name = dbase)
tinctest.logger.info('--running dataset reload')
datasetobj.reload_dataset()
PSQL.run_sql_file(sql_file, dbname = self.db_name, out_file = out_file)
return out_file
def test_18_gp_persistent_relation_node_uaocs_table_eof_upd(self):
tablename ='uaocs_table_test14'
tinctest.logger.info("-------------------------------")
tinctest.logger.info('test_18 Verify the eof mark in pg_aoseg and gp_persistant_rel_node table for uaocs relation after update ')
tinctest.logger.info("-------------------------------\n")
out_file = os.path.join(self.outpath,'create_tab_gp_persistent_relation_node_uaocs_table_upd_14.out')
sql_file = os.path.join(self.sqlpath,'create_tab_gp_persistent_relation_node_uaocs_table_upd_14.sql')
ans_file= os.path.join(self.anspath,'create_tab_gp_persistent_relation_node_uaocs_table_upd_14.ans')
#create uaocs table and insert 10 rows
sql_out=PSQL.run_sql_file(sql_file = sql_file,out_file=out_file)
assert Gpdiff.are_files_equal(out_file, ans_file)
#get relid for newly created table
relid = self.get_relid(file_name=tablename )
#get utility mode connection info
utilitymodeinfo=self.get_utilitymode_conn_uaocs_info( relid=relid)
u_port=utilitymodeinfo[0]
u_host=utilitymodeinfo[1]
assert (self.is_same_eof_uaocs_on_segment(relid=relid,host=u_host,port= u_port))
# delete 1 row
sql_cmd3="delete from %s where i = (select min(i) from %s );" % (tablename, tablename)
PSQL.run_sql_command_utility_mode(sql_cmd= sql_cmd3,host=u_host, port=u_port,flags='-q -t')
self.vacuum_full(tablename=tablename)
assert (self.is_same_eof_uaocs_on_segment(relid=relid,host=u_host,port= u_port))
def test_21_use_udf_gp_aovisimap_hidden_info_uao_upd_vacuum(self):
tablename ='uao_table_test11'
tinctest.logger.info("-------------------------------")
tinctest.logger.info('test_21 Verify the hidden tup_count using UDF gp_aovisimap_hidden_info(oid) for uao relation after update_vacuum')
tinctest.logger.info("-------------------------------\n")
out_file = os.path.join(self.outpath,'create_tab_tupcount_in_pg_aoseg_uaotable_upd_11.out')
sql_file = os.path.join(self.sqlpath,'create_tab_tupcount_in_pg_aoseg_uaotable_upd_11.sql')
ans_file= os.path.join(self.anspath,'create_tab_tupcount_in_pg_aoseg_uaotable_upd_11.ans')
#create uao table and insert 10 rows
sql_out=PSQL.run_sql_file(sql_file = sql_file,out_file=out_file)
assert Gpdiff.are_files_equal(out_file, ans_file)
#get relid for newly created table
relid = self.get_relid(file_name=tablename )
#get relid for newly created table
relid = self.get_relid(file_name=tablename )
#get utility mode connection info
utilitymodeinfo=self.get_utilitymode_conn_info( relid=relid)
u_port=utilitymodeinfo[0]
u_host=utilitymodeinfo[1]
assert(0 == int(self.get_hidden_tup_cnt(relid=relid,host=u_host,port= u_port)))
# update rows
sql_cmd3="update %s set j = 'test11' ;" % ( tablename)
PSQL.run_sql_command_utility_mode(sql_cmd= sql_cmd3,host=u_host, port=u_port,flags='-q -t')
assert(int(self.get_hidden_tup_cnt(relid=relid,host=u_host,port= u_port)) > 0)
self.vacuum_full(tablename=tablename)
assert(0 == int(self.get_hidden_tup_cnt(relid=relid,host=u_host,port= u_port)))
def test_09_call_udf_gp_aovisimap_forupdate(self):
tablename='uao_visimap_test09'
tinctest.logger.info("-------------------------------")
tinctest.logger.info('test_09 Verify the usage of UDF gp_aovisimap in utility mode for update tuple')
tinctest.logger.info("-------------------------------\n")
out_file = os.path.join(self.outpath,'create_tab_gp_aovisimap_upd_09.out')
sql_file = os.path.join(self.sqlpath,'create_tab_gp_aovisimap_upd_09.sql')
ans_file= os.path.join(self.anspath,'create_tab_gp_aovisimap_upd_09.ans')
#create uao table and insert 10 rows
sql_out=PSQL.run_sql_file(sql_file = sql_file,out_file=out_file)
assert Gpdiff.are_files_equal(out_file, ans_file)
#get relid for newly created table
relid = self.get_relid(file_name=tablename)
utilitymodeinfo=self.get_utilitymode_conn_info( relid=relid)
u_port=utilitymodeinfo[0]
u_host=utilitymodeinfo[1]
#login to segment in utility mode and execute the gp_aovisimap(relid) UDF
before_tablerowcnt=self.get_rowcnt_table_on_segment(tablename=tablename, host=u_host,port=u_port)
before_visimaprowcnt=self.get_visimap_cnt_on_segment(relid=relid,host=u_host,port=u_port)
assert(int(before_visimaprowcnt) == 0)
sql_cmd="update %s set j = j || '_9';" % (tablename)
PSQL.run_sql_command_utility_mode(sql_cmd=sql_cmd,host=u_host, port=u_port,flags='-q -t')
after_visimaprowcnt=self.get_visimap_cnt_on_segment(relid=relid,host=u_host,port=u_port)
assert(int(after_visimaprowcnt) > 0)
def template0_stop_limit(self):
"""
Raise next xid so that age(template0) grows beyond stop limit.
Create a new database off template0, let GPDB stop accepting
commands. Recover GPDB using the documented proceudure.
Ensure that the new database is sane.
"""
dburl = dbconn.DbURL()
with dbconn.connect(dburl, utility=True) as conn:
sql = "SHOW xid_stop_limit"
slimit_guc = int(dbconn.execSQLForSingleton(conn, sql))
new_limit = xid_sum(slimit_guc, -(10**6))
# Raise nextXid so that template0 age would cross stop limit.
self._raise_template0_age(self.STOP_LIMIT, self.gparray.master)
# newdb's age crosses stop limit and GPDB stops accepting commands.
PSQL(sql_cmd="CREATE DATABASE newdb TEMPLATE template0").run(
validateAfter=True)
self._basic_sanity_check("error")
# Reduce xid_stop_limit as per the standard procedure.
self._reduce_stop_limit_guc(self.gparray.master, new_limit)
# Vacuum freezing newdb should be suffice to recover.
PSQL(sql_cmd="VACUUM FREEZE",
dbname="newdb",
out_file="vacuum_newdb_stop_master.out").run(validateAfter=True)
self._basic_sanity_check("clean")
PSQL.drop_database(dbname="newdb")
self._restore_stop_limit_guc(self.gparray.master.datadir)
def test_reindex_pg_class(self):
tinctest.logger.info("create checkpoint")
results = {'rc':0, 'stdout':'', 'stderr':''}
PSQL.run_sql_command("checkpoint", results=results)
assert results['rc'] == 0, results['stderr']
tinctest.logger.info("inject fault to skip checkpoints")
cmd = Command("skip checkpoint on primaries",
"gpfaultinjector -f checkpoint -m async -y skip -o 0"
" -H ALL -r primary")
cmd.run(validateAfter=True)
tinctest.logger.info(cmd.get_results().printResult())
cmd = Command("skip checkpoint on master",
"gpfaultinjector -f checkpoint -m async -y skip -o 0 -s 1")
cmd.run(validateAfter=True)
tinctest.logger.info(cmd.get_results().printResult())
tinctest.logger.info("reindex pg_class indexes")
assert PSQL.run_sql_file(local_path('reindex_pg_class.sql'))
tinctest.logger.info("shutdown immediate")
cmd = Command("shutdown immediate", "gpstop -ai")
cmd.run(validateAfter=True)
tinctest.logger.info(cmd.get_results().printResult())
tinctest.logger.info("trigger recovery")
cmd = Command("restart the cluster", "gpstart -a")
cmd.run(validateAfter=True)
tinctest.logger.info(cmd.get_results().printResult())
tinctest.logger.info("validate recovery succeeded")
results = {'rc':0, 'stdout':'', 'stderr':''}
PSQL.run_sql_command("DROP TABLE reindex_pg_class_test", results=results)
assert results['rc'] == 0, results['stderr']
def test_run_sql_command_wth_username(self):
sql_cmd = 'SELECT 1'
username = getpass.getuser()
self.assertTrue(PSQL.run_sql_command(sql_cmd = sql_cmd, username = username))
#Invalid username
self.assertFalse(PSQL.run_sql_command(sql_cmd = sql_cmd, username = '******'))
def template0_wrap_around_on_segment(self, primary):
"""
Same as template0_wrap_around, but on segment.
"""
logger.info("template0_wrap_around_on_segment: dbid(%d) %s:%d'" %
(primary.dbid, primary.hostname, primary.port))
self._raise_template0_age(self.WRAP_LIMIT, primary)
PSQL(sql_cmd="CREATE DATABASE newdb TEMPLATE template0").run(
validateAfter=True)
sql = "SELECT age(datfrozenxid) FROM pg_database WHERE datname='newdb'"
# Verify that age of newdb on the segment is negative.
dburl = dbconn.DbURL(hostname=primary.hostname, port=primary.port)
with dbconn.connect(dburl, utility=True) as conn:
age_newdb = int(dbconn.execSQLForSingleton(conn, sql))
self.assertTrue(age_newdb < 0)
# Reset newdb age so as to recover from wrap around.
self._reset_age("newdb", primary)
# Verify that normal operations can be performed on newdb whose age was
# reset to a correct value.
self._basic_sanity_check("clean", {"dbname":"newdb"})
# Verify that age of newdb on the segment is valid.
with dbconn.connect(dburl, utility=True) as conn:
age_newdb = int(dbconn.execSQLForSingleton(conn, sql))
self.assertTrue(age_newdb > 0)
PSQL.drop_database(dbname="newdb")
def test_run_sql_command_catalog_update(self):
sql_cmd = 'show gp_session_role;'
out_file = os.path.join(os.path.dirname(inspect.getfile(self.__class__)),'test_catalog_update.out')
self.assertFalse(os.path.exists(out_file))
try:
PSQL.run_sql_command_catalog_update(sql_cmd = sql_cmd, out_file = out_file)
self.assertTrue(os.path.exists(out_file))
with open(out_file, 'r') as f:
output = f.read()
self.assertIsNotNone(re.search('utility', output))
finally:
os.remove(out_file)
self.assertFalse(os.path.exists(out_file))
sql_cmd = 'show allow_system_table_mods;'
out_file = os.path.join(os.path.dirname(inspect.getfile(self.__class__)),'test_catalog_update.out')
self.assertFalse(os.path.exists(out_file))
try:
PSQL.run_sql_command_catalog_update(sql_cmd = sql_cmd, out_file = out_file)
self.assertTrue(os.path.exists(out_file))
with open(out_file, 'r') as f:
output = f.read()
self.assertIsNotNone(re.search('DML', output))
finally:
os.remove(out_file)
self.assertFalse(os.path.exists(out_file))
def test_08_call_udf_gp_aovisimap_fordelete(self):
tinctest.logger.info("-------------------------------")
tinctest.logger.info('test_08 Verify the usage of UDF gp_aovisimap in utility mode for deleted tuple')
tinctest.logger.info("-------------------------------\n")
out_file = os.path.join(self.outpath,'create_tab_gp_aovisimap_del_08.out')
sql_file = os.path.join(self.sqlpath,'create_tab_gp_aovisimap_del_08.sql')
ans_file= os.path.join(self.anspath,'create_tab_gp_aovisimap_del_08.ans')
tablename='uao_visimap_test08'
#create uao table and insert 10 rows
sql_out=PSQL.run_sql_file(sql_file = sql_file,out_file=out_file)
assert Gpdiff.are_files_equal(out_file, ans_file)
#get relid for newly created table
relid = self.get_relid(file_name=tablename )
#get the segment_id where we'll log in utility mode and then get the hostname and port for this segment
utilitymodeinfo=self.get_utilitymode_conn_info( relid=relid)
u_port=utilitymodeinfo[0]
u_host=utilitymodeinfo[1]
before_tablerowcnt=self.get_rowcnt_table_on_segment(tablename=tablename, host=u_host,port=u_port)
before_visimaprowcnt=self.get_visimap_cnt_on_segment(relid=relid,host=u_host,port=u_port)
assert(int(before_visimaprowcnt) == 0)
sql_cmd="delete from uao_visimap_test08 ;"
PSQL.run_sql_command_utility_mode(sql_cmd=sql_cmd,host=u_host, port=u_port,flags='-q -t')
after_tablerowcnt=self.get_rowcnt_table_on_segment(tablename=tablename, host=u_host,port=u_port)
after_visimaprowcnt=self.get_visimap_cnt_on_segment(relid=relid,host=u_host,port=u_port)
assert(int(after_tablerowcnt) == 0)
def __init__(self):
results = {'rc': -1, 'stdout': '', 'stderr': ''}
PSQL.run_sql_command("create extension if not exists gp_inject_fault",
results=results)
assert results['rc'] == 0
def get_relid(self,file_name=None): sql_cmd="SELECT oid FROM pg_class WHERE relname='%s';\n" % file_name relid= PSQL.run_sql_command(sql_cmd=sql_cmd, flags='-q -t') return relid;
def get_standbyhost(self):
std_sql = "select hostname from gp_segment_configuration where content='-1' and role='m';"
standby_host = PSQL.run_sql_command(std_sql,
flags='-q -t',
dbname='postgres')
return standby_host.strip()
def test_mpp23395(self):
"""
@description Test MPP-20964, uncleaned lock table by pg_terminate_backend
@product_version gpdb: [4.3.3.1-],[4.2.8.5-4.2.99.99]
"""
self.util = Filerepe2e_Util()
(ok,out) = self.util.inject_fault(f='dtm_broadcast_commit_prepared', y='reset', seg_id=1);
if not ok:
raise Exception("Failed to reset the fault dtm_broadcast_commit_prepared")
# setup
PSQL.run_sql_command("""
DROP TABLE IF EXISTS mpp23395;
""")
# Scenario 1: FAULT during Create Table on master
sql = '''
CREATE TABLE mpp23395(a int);
'''
self.run_sequence(sql, 'dtm_broadcast_commit_prepared', 'fatal', 1);
# Scenario 2: FAULT during Drop Table on master, COMMIT case
sql = '''
DROP TABLE mpp23395;
'''
self.run_sequence(sql, 'dtm_broadcast_commit_prepared', 'fatal', 1);
(ok,out) = self.util.inject_fault(f='dtm_broadcast_commit_prepared', y='reset', seg_id=1);
if not ok:
raise Exception("Failed to reset the fault dtm_broadcast_commit_prepared")
# Scenario 3: FAULT during Create Table on segment, COMMIT case
sql = '''
SET dtx_phase2_retry_count = 1;
SET debug_dtm_action_target = "protocol";
SET debug_dtm_action_protocol = "commit_prepared";
SET debug_dtm_action_segment = 0;
SET debug_dtm_action = "fail_begin_command";
CREATE TABLE mpp23395(a int);
'''
self.run_sequence(sql, 'twophase_transaction_commit_prepared', 'error', 2);
# Scenario 4: FAULT during Drop Table on segment, COMMIT case
sql = '''
SET dtx_phase2_retry_count = 1;
SET debug_dtm_action_target = "protocol";
SET debug_dtm_action_protocol = "commit_prepared";
SET debug_dtm_action_segment = 0;
SET debug_dtm_action = "fail_begin_command";
DROP TABLE mpp23395;
'''
self.run_sequence(sql, 'twophase_transaction_commit_prepared', 'error', 2);
# Scenario 5: FAULT during Create Table on master, ABORT case
(ok,out) = self.util.inject_fault(f='transaction_abort_after_distributed_prepared', y='error', seg_id=1);
if not ok:
raise Exception("Failed to set the error fault for transaction_abort_after_distributed_prepared")
sql = '''
CREATE TABLE mpp23395(a int);
'''
self.run_sequence(sql, 'dtm_broadcast_abort_prepared', 'fatal', 1);
(ok,out) = self.util.inject_fault(f='transaction_abort_after_distributed_prepared', y='reset', seg_id=1);
if not ok:
raise Exception("Failed to reset the fault transaction_abort_after_distributed_prepared")
PSQL.run_sql_command("""
CREATE TABLE mpp23395(a int);
""")
# Scenario 6: FAULT during Drop Table on master, ABORT case
(ok,out) = self.util.inject_fault(f='transaction_abort_after_distributed_prepared', y='error', seg_id=1);
if not ok:
raise Exception("Failed to set the error fault for transaction_abort_after_distributed_prepared")
sql = '''
DROP TABLE mpp23395;
'''
self.run_sequence(sql, 'dtm_broadcast_abort_prepared', 'fatal', 1);
(ok,out) = self.util.inject_fault(f='transaction_abort_after_distributed_prepared', y='reset', seg_id=1);
if not ok:
raise Exception("Failed to reset the fault transaction_abort_after_distributed_prepared")
PSQL.run_sql_command("""
DROP TABLE mpp23395;
""")
# Scenario 7: FAULT during Create Table on segment, COMMIT case, succeeds on second retry
sql = '''
DROP TABLE IF EXISTS mpp23395;
SET debug_dtm_action_target = "protocol";
SET debug_dtm_action_protocol = "commit_prepared";
SET debug_dtm_action_segment = 0;
SET debug_dtm_action = "fail_begin_command";
CREATE TABLE mpp23395(a int);
'''
self.run_sequence(sql, 'finish_prepared_after_record_commit_prepared', 'error', 2, False);
# Scenario 8: QE panics after writing prepare xlog record. This should
# cause master to broadcast abort but QEs handle the abort in
# DTX_CONTEXT_LOCAL_ONLY context.
sql = '''
DROP TABLE IF EXISTS mpp23395;
CREATE TABLE mpp23395(a int);
INSERT INTO mpp23395 VALUES(1), (2), (3);
SET debug_abort_after_segment_prepared = true;
DELETE FROM mpp23395;
'''
# No prepared transactions should remain lingering
PSQL.run_sql_command(sql)
self.check_no_dangling_prepared_transaction()
dbstate = DbStateClass('run_validation')
dbstate.check_catalog()
def trigger_transition(self):
PSQL.run_sql_file(local_path('mirrors.sql'))
def test_block_while_catchup_within_range(self):
"""
This test verifies if a backend gets blocked in case
the WAL sender is still in catchup mode.
"""
with WalClient("replication=true") as client:
(sysid, tli, xpos) = client.identify_system()
# Set the guc to > 1 so that we can verify the test
# using less amount of xlog
self.set_guc('repl_catchup_within_range', '3')
# Generate enough xlog in WAL sender startup phase. None of the sql statements
# should get blocked. If blocked we have some issue.
# Checkpointing causes full page writes on updates/inserts. Hence helps
# xlog generation.
i = 0
logger.info(
'Running a bunch of SQLs to generate enough xlog to maintain catchup phase...'
)
while (i < 10):
PSQL.run_sql_command(
'DROP TABLE IF EXISTS foo; CREATE TABLE foo(a int, b int); CHECKPOINT;'
)
i = i + 1
xpos_ptr = XLogRecPtr.from_string(xpos)
client.start_replication(xpos_ptr)
while True:
msg = client.receive(1000)
if isinstance(msg, WalMessageData):
header = msg.header
# walsender must be still in catchup phase as a lot xlog needs to be sent
sql_catchup = "SELECT count(*) FROM pg_stat_replication where state = 'catchup'"
sql_table_present = "SELECT count(*) from pg_class where relname = 'foo'"
sql_bkd_count = (
"SELECT count(*) from pg_stat_activity where waiting ='t' and waiting_reason = 'replication'"
)
with dbconn.connect(dbconn.DbURL(), utility=True) as conn:
curs = dbconn.execSQL(conn, sql_catchup)
results = curs.fetchall()
self.assertEqual(int(results[0][0]), 1,
"No Catchup WAL sender found")
logger.info(
'WAL sender is alive and now is in catchup phase...'
)
logger.info('In catchup phase, create table...')
subprocess.Popen([
'psql', '-c',
'DROP TABLE IF EXISTS raghav; create table raghav (a int);'
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
with dbconn.connect(dbconn.DbURL(), utility=True) as conn:
# verify if WAL sender is still in catchup phase
curs = dbconn.execSQL(conn, sql_catchup)
results = curs.fetchall()
self.assertEqual(
int(results[0][0]), 1,
"WAL sender catchup phase over before verification"
)
logger.info(
'WAL sender is alive, still in catchup phase ..')
while (i < 5):
with dbconn.connect(dbconn.DbURL(),
utility=True) as conn:
# verify if the previous backend is blocked
curs = dbconn.execSQL(conn, sql_bkd_count)
results = curs.fetchall()
if (int(results[0][0]) == 1):
break
if (i == 4):
self.assertTrue(
0, "Previous backend not blocked ...")
i = i + 1
logger.info('But, create table is blocked...')
with dbconn.connect(dbconn.DbURL(), utility=True) as conn:
# verify if WAL sender is still in catchup phase
curs = dbconn.execSQL(conn, sql_catchup)
results = curs.fetchall()
self.assertEqual(
int(results[0][0]), 1,
"WAL sender catchup phase over before verification"
)
logger.info(
'WAL sender is alive, in catchup phase and backend is blocked...'
)
# sync replication needs a reply otherwise backend blocks
client.reply(header.walEnd, header.walEnd, header.walEnd)
# success, should get some 'w' message
logger.info(
"Pass - Backends block if WAL sender is alive and the catchup is within-range"
)
break
elif isinstance(msg, WalMessageNoData):
# could be timeout
client.reply(xpos_ptr, xpos_ptr, xpos_ptr)
else:
raise StandardError(msg.errmsg)
logger.info("Pass")
self.set_guc('repl_catchup_within_range', '1')
def get_segment_cnt(self, relid=0,host=None,port=None):
sql_cmd="select count(*) from gp_toolkit.__gp_aocsseg(%s) group by column_num having count(*) > 1 limit 1" % (relid)
segcnt=PSQL.run_sql_command_utility_mode(sql_cmd=sql_cmd,host=host, port=port,flags='-q -t')
if (len(segcnt.strip()) == 0):
segcnt='0'
return segcnt
def get_segcount_state(self,state):
sql_cmd = "select count(*) from gp_segment_configuration where status = '%s'" % (state)
result=PSQL.run_sql_command(sql_cmd= sql_cmd,flags='-q -t')
tinctest.logger.info('Number of segments in %s State == %d' % (state,(int(result))))
return int(result)
def _reset_age(self, dbname, segdb=None):
"""
Resets datfrozenxid and relfrozenxid's in pg_class of the
specified dbname to a value close to the current xid. This is
a recommended way of resetting age of dbname or a database
that is created off template0.
@param segdb: identifies the segment on which to operate. It is an
instance of GpDB class.
Note that the database dbname must have all tuples frozen (xmin=2).
This holds true of template0 and of a database created off template0,
only if there are no modifications done to the database.
"""
if segdb is None:
segdb = self.gparray.master
dburl = dbconn.DbURL(hostname=segdb.hostname, port=segdb.port)
dburl_dbname = dbconn.DbURL(hostname=segdb.hostname,
port=segdb.port,
dbname=dbname)
with dbconn.connect(dburl, utility=True,
allowSystemTableMods="dml") as conn:
sql = "SELECT get_next_xid()"
next_xid = int(dbconn.execSQLForSingleton(conn, sql))
sql = "UPDATE pg_database SET datfrozenxid='%d'::xid WHERE datname='%s'"
dbconn.execSQL(conn, sql % (next_xid, dbname))
conn.commit()
if dbname == "template0":
self._set_allowconn_template0(True)
with dbconn.connect(dburl_dbname,
utility=True,
allowSystemTableMods="dml") as conn:
sql = ("UPDATE pg_class SET relfrozenxid='%d'::xid WHERE "
"int8in(xidout(relfrozenxid)) > 0")
dbconn.execSQL(conn, sql % next_xid)
conn.commit()
PSQL(sql_cmd="VACUUM FREEZE pg_class",
dbname=dbname,
PGOPTIONS="-c 'gp_session_role=utility'",
host=segdb.hostname,
port=segdb.port,
out_file="vacuum_%s.out" % dbname).run(validateAfter=True)
with dbconn.connect(dburl_dbname,
utility=True,
allowSystemTableMods="dml") as conn:
dbconn.execSQL(conn, "DELETE FROM pg_stat_last_operation")
conn.commit()
PSQL(sql_cmd="VACUUM FREEZE pg_stat_last_operation",
dbname=dbname,
PGOPTIONS="-c 'gp_session_role=utility'",
host=segdb.hostname,
port=segdb.port,
out_file="vacuum_%s.out" % dbname).run(validateAfter=True)
if dbname == "template0":
self._set_allowconn_template0(False)
with dbconn.connect(dburl, utility=True) as conn:
sql = "SELECT age(datfrozenxid) FROM pg_database WHERE datname='%s'"
age_dbname = dbconn.execSQLForSingleton(conn, sql % dbname)
age_dbname = int(age_dbname)
logger.info("Age of %s reset to %d" % (dbname, age_dbname))
# We are OK as long as dbname age is less than xid_warn_limit. The
# 10000 is just a number assumed to be less than xid_warn_limit.
self.assertTrue(
age_dbname > 0 and age_dbname < 10000,
"age(%s) = %d, next xid = %d" % (dbname, age_dbname, next_xid))
def inject_fault(self,
y=None,
f=None,
r='mirror',
seg_id=None,
H='ALL',
m='async',
sleeptime=0,
o=0,
table='',
database=''):
'''
PURPOSE :
Inject the fault using gpfaultinjector
@param
y : suspend/resume/reset/panic/fault
f : Name of the faulti
rest_of_them : same as in gpfaultinjector help
'''
if (not y) or (not f):
raise Exception("Need a value for type and name to continue")
if (not os.getenv('MASTER_DATA_DIRECTORY')):
raise Exception(
'MASTER_DATA_DIRECTORY environment variable is not set.')
fault_cmd = (
"select gp_inject_fault('{fault}', '{type}', '{ddl}', '{database}',"
" '{table}', {occurrences}, {sleeptime}, {dbid})")
if seg_id is None:
fault_cmd = fault_cmd + " from gp_segment_configuration where role = '{role}'"
if r == 'mirror':
fault_sql = fault_cmd.format(fault=f,
type=y,
ddl='',
database=database,
table=table,
occurrences=o,
sleeptime=sleeptime,
dbid='dbid',
role='m')
elif r == 'primary':
fault_sql = fault_cmd.format(fault=f,
type=y,
ddl='',
database=database,
table=table,
occurrences=o,
sleeptime=sleeptime,
dbid='dbid',
role='p')
else:
assert False
else:
fault_sql = fault_cmd.format(fault=f,
type=y,
ddl='',
database=database,
table=table,
occurrences=o,
sleeptime=sleeptime,
dbid=seg_id)
results = {'rc': -1, 'stdout': '', 'stderr': ''}
PSQL.run_sql_command(fault_sql, results=results)
if results['rc'] != 0 and y != 'status':
ok = False
else:
ok = True
if not ok and y != 'status':
raise Exception("Cmd %s Failed to inject fault %s to %s" %
(fault_sql, f, y))
else:
tinctest.logger.info('Injected fault %s ' % fault_sql)
return (ok, results['stdout'] + results['stderr'])
def test_unblock_while_catchup_out_of_range(self):
"""
This test verifies if a backend gets blocked in case
the WAL sender is still in catchup mode.
"""
with WalClient("replication=true") as client:
(sysid, tli, xpos) = client.identify_system()
sql_startup = "SELECT count(*) FROM pg_stat_replication where state = 'startup'"
with dbconn.connect(dbconn.DbURL(), utility=True) as conn:
curs = dbconn.execSQL(conn, sql_startup)
results = curs.fetchall()
self.assertEqual(int(results[0][0]), 1,
"No WAL sender in startup phase found")
logger.info('WAL sender is alive and now is in startup phase...')
# Generate enough xlog in WAL sender startup phase. None of the sql statements
# should get blocked. If blocked we have some issue.
# Checkpointing causes full page writes on updates/inserts. Hence helps
# xlog generation.
i = 0
logger.info(
'Running a bunch of SQLs to generate enough xlog to maintain catchup phase...'
)
while (i < 10):
PSQL.run_sql_command(
'DROP TABLE IF EXISTS foo; CREATE TABLE foo(a int, b int); CHECKPOINT;'
)
i = i + 1
logger.info(
'Pass - Database does not block if WAL sender is alive and in startup phase'
)
logger.info(
'Creating some xlog seg files to simulate catchup out-of-range..'
)
i = 0
while (i < 3):
PSQL.run_sql_command(
'select pg_switch_xlog();select pg_switch_xlog();checkpoint;'
)
i = i + 1
xpos_ptr = XLogRecPtr.from_string(xpos)
client.start_replication(xpos_ptr)
while True:
msg = client.receive(1000)
if isinstance(msg, WalMessageData):
header = msg.header
# walsender must be still in catchup phase as a lot xlog needs to be sent
sql_catchup = "SELECT count(*) FROM pg_stat_replication where state = 'catchup'"
sql_table_present = "SELECT count(*) from pg_class where relname = 'foo'"
sql_bkd_count = (
"SELECT count(*) from pg_stat_activity where waiting ='t' and waiting_reason = 'replication'"
)
with dbconn.connect(dbconn.DbURL(), utility=True) as conn:
curs = dbconn.execSQL(conn, sql_catchup)
results = curs.fetchall()
self.assertEqual(int(results[0][0]), 1,
"No Catchup WAL sender found")
logger.info(
'WAL sender is alive and now is in catchup phase...'
)
logger.info('In catchup phase, run some sql...')
PSQL.run_sql_command(
'DROP TABLE IF EXISTS foo; CREATE TABLE foo(a int, b int);',
dbname='postgres')
while (i < 5):
with dbconn.connect(dbconn.DbURL(),
utility=True) as conn:
# verify if the previous backend is blocked
curs = dbconn.execSQL(conn, sql_bkd_count)
results = curs.fetchall()
if (int(results[0][0]) > 0):
self.assertTrue(
0, "Previous backend was blocked ...")
i = i + 1
logger.info('Create table is NOT blocked...')
with dbconn.connect(dbconn.DbURL(), utility=True) as conn:
# verify if WAL sender is still in catchup phase
curs = dbconn.execSQL(conn, sql_catchup)
results = curs.fetchall()
self.assertEqual(
int(results[0][0]), 1,
"WAL sender catchup phase over before verification"
)
logger.info(
'WAL sender is alive and still in catchup phase...')
with dbconn.connect(dbconn.DbURL(dbname='postgres'),
utility=True) as conn:
# verify if WAL sender is still in catchup phase
curs = dbconn.execSQL(conn, sql_table_present)
results = curs.fetchall()
self.assertEqual(int(results[0][0]), 1,
"Table foo not found")
# sync replication needs a reply otherwise backend blocks
client.reply(header.walEnd, header.walEnd, header.walEnd)
# success, should get some 'w' message
logger.info(
"Pass - Database does not block if WAL sender is alive and "
"the catchup is out-of-range")
break
elif isinstance(msg, WalMessageNoData):
# could be timeout
client.reply(xpos_ptr, xpos_ptr, xpos_ptr)
else:
raise StandardError(msg.errmsg)
class GpactivateStandby(object):
'''Class for gpactivatestandby operations '''
standby_port = '5656'
db_name = 'walrepl'
def __init__(self):
self.gpinit = GpinitStandby()
self.pgutil = GpUtility()
self.runmixin = StandbyRunMixin()
self.runmixin.createdb(dbname='walrepl')
self.gphome = os.environ.get('GPHOME')
self.pgport = os.environ.get('PGPORT')
self.mdd = os.environ.get('MASTER_DATA_DIRECTORY')
self.config = GPDBConfig()
self.host = socket.gethostname()
dburl = dbconn.DbURL()
gparray = GpArray.initFromCatalog(dburl, utility=True)
self.numcontent = gparray.getNumSegmentContents()
self.orig_master = gparray.master
def run_remote(self, standbyhost, rmt_cmd, pgport='', standbydd=''):
'''Runs remote command and returns rc, result '''
export_cmd = "source %s/greenplum_path.sh;export PGPORT=%s;export MASTER_DATA_DIRECTORY=%s" % (
self.gphome, pgport, standbydd)
remote_cmd = "gpssh -h %s -e '%s; %s'" % (standbyhost, export_cmd,
rmt_cmd)
cmd = Command(name='Running Remote command', cmdStr='%s' % remote_cmd)
tinctest.logger.info(" %s" % cmd)
cmd.run(validateAfter=False)
result = cmd.get_results()
return result.rc, result.stdout
def activate(self, option=''):
''' Stop the master and activate current standby to master'''
standby_host = self.get_current_standby()
standby_port = self.get_standby_port()
standby_loc = self.get_standby_dd()
self.run_remote(self.host,
'gpstop -aim',
pgport=self.pgport,
standbydd=self.mdd)
gpactivate_cmd = 'gpactivatestandby -a -d %s %s' % (standby_loc,
option)
(rc, result) = self.run_remote(standby_host,
gpactivate_cmd,
pgport=standby_port,
standbydd=standby_loc)
tinctest.logger.info(
'Result without force option to activate standby %s' % result)
if (rc != 0) and result.find('Force activation required') != -1:
tinctest.logger.info(
'activating standby failed, try force activation...')
gpactivate_cmd = 'gpactivatestandby -a -f -d %s %s' % (standby_loc,
option)
(rc, result) = self.run_remote(standby_host,
gpactivate_cmd,
pgport=standby_port,
standbydd=standby_loc)
if (rc != 0):
tinctest.logger.error('Force activating standby failed!')
return False
tinctest.logger.info('standby acvitated, host value %s' % standby_host)
return True
def remove_standby(self):
return self.gpinit.run(option='-r')
def failback_to_original_master(self):
# Check if master is running.
bashCmd = (
self.gphome
) + '/bin/pg_ctl status -D $MASTER_DATA_DIRECTORY | grep \'pg_ctl: server is running\''
cmd = Command(name='Running cmd %s' % bashCmd,
cmdStr="source %s/greenplum_path.sh; %s" %
(self.gphome, bashCmd))
try:
cmd.run()
except Exception, e:
tinctest.logger.error("Error running command %s\n" % e)
return
result = cmd.get_results()
out = result.stdout
if not out:
tinctest.logger.info('Start the old master again ...')
master = gp.MasterStart("Starting orig Master",
self.orig_master.datadir,
self.orig_master.port,
self.orig_master.dbid, 0, self.numcontent,
None, None, None)
master.run(validateAfter=True)
result = master.get_results()
tinctest.logger.info('orig Master started result : %s' %
result.stdout)
if result.rc != 0:
raise WalReplException(
'Unable to start original master process')
Command('gpinitstandby -ra', 'gpinitstandby -ra').run()
# failing back to old master, it takes a little bit to prepare the cluster ready for connection
if os.path.exists(local_path('drop_filespace.sql')):
PSQL.run_sql_file(local_path('drop_filespace.sql'),
dbname=self.db_name)
def test_syncrep(self):
# 1. Initiate the Standby
# 2. Once the WAL receiver starts, signal it to suspend post xlog flush
# but before sending the ack.
# 3. Now execute a transaction and commit it. The backend is expected
# be blocked.
# 4. Resume the WALReceiver and see the transaction passed and its
# results are visible.
# cleanup
PSQL.run_sql_command('DROP table if exists foo')
# 1. create standby and start
res = self.standby.create()
self.assertEqual(res, 0)
res = self.standby.start()
self.assertTrue(res.wasSuccessful())
# wait for the walreceiver to start
num_walsender = self.wait_for_walsender()
self.assertEqual(num_walsender, 1)
# 2. Once the WAL receiver starts, signal it to suspend post xlog flush
# but before sending the ack.
proc = subprocess.Popen(['ps', '-ef'], stdout=subprocess.PIPE)
stdout = proc.communicate()[0]
search = "wal receiver process"
for line in stdout.split('\n'):
if (line.find(search) > 0):
split_line = re.split(r'\s+', line.strip())
break
self.assertTrue(len(split_line) > 0)
wal_rcv_pid = int(split_line[1])
logger.info('Suspending WAL Receiver(' + str(wal_rcv_pid) + ')...')
self.generate_trigger_file('wait_before_send_ack')
os.kill(wal_rcv_pid, signal.SIGUSR2)
# 3. Now execute a transaction and commit it. The backend is expected
# be blocked.
logger.info('Create table foo...')
# we use subprocess since we expect it'll be blocked.
proc = subprocess.Popen(['psql', '-c', 'create table foo (a int)'],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
readable = self.wait_stdout(proc, 5.0)
self.assertFalse(readable, 'psql did not block')
# 4. Resume the WALReceiver and see the transaction passed and its
# results are visible.
logger.info('Resume the WAL Receiver...')
self.generate_trigger_file('resume')
os.kill(wal_rcv_pid, signal.SIGUSR2)
readable = self.wait_stdout(proc, 5.0)
self.assertTrue(readable, 'psql still blocks')
proc.communicate()
logger.info('No blocked backend found!')
logger.info('Verifying if table exists ? ...')
PSQL(sql_cmd='select * from foo').run(validateAfter=True)
logger.info('Pass')
def create_ao_table(self):
'''Creating a table in Append-Only mode'''
tinctest.logger.info('\nCreating the appendonly table')
out = PSQL.run_sql_file(local_path('create.sql'))
return out
def query_select_count(self, sqlcmd):
(num) = PSQL.run_sql_command(sqlcmd)
num = num.split('\n')[3].strip()
return num
def get_filespace_location(self):
fs_sql = "select fselocation from pg_filespace_entry where fselocation like '%fs_walrepl_a%' and fsedbid=1;"
filespace_loc = PSQL.run_sql_command(fs_sql,
flags='-q -t',
dbname='postgres')
return filespace_loc.strip()
def test_fail_back(self):
"""
This test verifies that the fail-back mode is not allowed.
Fail-back means original master acting as the new standby.
"""
# Verify if the database is up. Run some sql.
PSQL.run_sql_command('DROP table if exists foo')
Command('remove standby', 'gpinitstandby -ra').run()
self.assertEqual(self.standby.create(), 0)
res = self.standby.start()
self.assertTrue(res.wasSuccessful())
# Wait for the walreceiver to start
num_walsender = self.wait_for_walsender()
self.assertEqual(num_walsender, 1)
logger.info('Activated WAL Receiver...')
# Promote the standby & shutdown the old Master
# Generate a recovery.conf file for the old Master so
# to make him the new standby that connects to the new
# master (originally standby)
logger.info('Promoting the standby...')
self.standby.promote()
dburl = dbconn.DbURL()
gparray = GpArray.initFromCatalog(dburl, utility=True)
numcontent = gparray.getNumSegmentContents()
orig_master = gparray.master
self.standby.remove_catalog_standby(dburl)
if (os.path.exists(os.path.join(orig_master.datadir, 'wal_rcv.pid'))):
os.remove(os.path.join(orig_master.datadir, 'wal_rcv.pid'))
logger.info('Stop the original master...')
cmd = Command("gpstop", "gpstop -aim")
cmd.run()
self.assertEqual(cmd.get_results().rc, 0, str(cmd))
logger.info(
'Generate recovery.conf for original master to make a new standby...'
)
master_recv_conf = open(
os.path.join(orig_master.datadir, 'recovery.conf'), 'w')
standby_recv_done = open(
os.path.join(self.standby.datadir, 'recovery.done'))
for line in standby_recv_done:
master_recv_conf.write(
line.replace("port=" + str(os.environ.get('PGPORT')),
"port=" + str(self.standby.port)))
master_recv_conf.close()
standby_recv_done.close()
logger.info(
'Start the old master again (to act as the new standby)...')
master = gp.MasterStart("Starting orig Master in standby mode",
orig_master.datadir, orig_master.port,
orig_master.dbid, 0, numcontent, None, None,
None)
# -w option would wait forever.
master.cmdStr = master.cmdStr.replace(' -w', '')
master.run(validateAfter=True)
self.assertTrue((master.get_results()).wasSuccessful())
# Have to do this to give the new standby some time to be active
subprocess.check_call("psql -c 'create database foo' -p " +
str(self.standby.port),
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
subprocess.check_call("psql -c 'drop database foo' -p " +
str(self.standby.port),
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
time.sleep(3)
# The new standby can re-start but should not be able to connect to the new
# master (originally standby). Thats the test
self.assertTrue(
os.path.exists(os.path.join(orig_master.datadir, 'wal_rcv.pid')))
logger.info(
'The WAL receiver pid file exists which means the new standby started\n'
'but still could not connect to the new Master (originally standby) and hence the\n'
'pid file was not cleared')
# Remove the recovery.conf file from the new standby directory
# as its no more needed
os.remove(os.path.join(orig_master.datadir, 'recovery.conf'))
logger.info('Stop the original master again...')
rc = subprocess.Popen('pg_ctl stop -D ' + orig_master.datadir +
' -m immediate',
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
# Perform gpstart to get the original master (& cluster) back again
cmd = Command("gpstart", "gpstart -a")
cmd.run()
self.assertTrue(cmd.get_results().rc in (0, 1), str(cmd))
logger.info('Pass')
def gpstop_and_verify(self, option = ''):
"""
stop, and verify all the cluster completely shutdown
"""
sql = 'select hostname, port, content, preferred_role from gp_segment_configuration;'
segments = self.run_SQLQuery(sql,dbname= 'template1')
if option == '-u':
std_sql = 'drop user if exists gptest; create user gptest LOGIN PASSWORD \'testpassword\'; '
PSQL.run_sql_command(std_sql, flags = '-q -t', dbname= 'template1')
new_entry = 'local all gptest password \n'
self.add_entry(line = new_entry)
self.run('gpstop -a %s'%option)
return self.gpreload_verify('gptest', 'testpassword')
(rc, output) = self.run('gpstop -a %s'%option)
if rc != 0:
return False
if option == '-r':
no_sync_up_segment_sql = 'select content, preferred_role from gp_segment_configuration where mode <> \'s\' or status <> \'u\';'
segments_with_issue = self.run_SQLQuery(no_sync_up_segment_sql,dbname= 'template1')
if segments_with_issue:
return False
elif option == '-y':
for segment in segments:
seg_content = segment[2]
seg_preferred_role = segment[3]
seg_host = segment[0]
seg_port = segment[1]
if seg_content == -1 and seg_preferred_role == 'm':
pid = self.get_pid_by_keyword(host=seg_host, pgport=seg_port, keyword='bin')
if pid < 0:
tinctest.logger.error("standby host should not be shutdown.")
else:
pid = self.get_pid_by_keyword(host=seg_host, pgport=seg_port, keyword='bin')
if pid > 0:
tinctest.logger.error("%s segment on host %s was not properly shutdown"%(seg_preferred_role, seg_host))
return False
elif option == '-m':
for segment in segments:
seg_content = segment[2]
seg_preferred_role = segment[3]
seg_host = segment[0]
seg_port = segment[1]
if seg_content == -1 and seg_preferred_role == 'p':
pid = self.get_pid_by_keyword(host=seg_host, pgport=seg_port, keyword='bin')
if pid > 0:
tinctest.logger.error("master should shutdown but still is running")
return False
else:
pid = self.get_pid_by_keyword(host=seg_host, pgport=seg_port, keyword='bin')
if pid < 0:
tinctest.logger.error("%s segment on host %s should not be shutdown"%(seg_preferred_role, seg_host))
return False
else:
for segment in segments:
seg_content = segment[2]
seg_preferred_role = segment[3]
seg_host = segment[0]
seg_port = segment[1]
pid = self.get_pid_by_keyword(host=seg_host, pgport=seg_port, keyword='bin')
if pid > 0:
tinctest.logger.error("%s segment on host %s was not properly shutdown"%(seg_preferred_role, seg_host))
return False
return True
def resume_faults(self, fault_type, cluster_state='sync'):
'''
@param fault_type : commit/abort/end_prepare_two_phase_sleep/dtm_broadcast_prepare/dtm_broadcast_commit_prepared/dtm_xlog_distributed_commit
@description : Resume the suspended faults
'''
tinctest.logger.info('coming to resume faults with xact %s' %
fault_type)
if fault_type == 'abort':
self.filereputil.inject_fault(
f='twophase_transaction_abort_prepared',
y='resume',
r='primary',
p=self.port,
o='0')
if cluster_state != 'resync':
self.filereputil.inject_fault(
f='transaction_abort_after_distributed_prepared',
y='reset',
p=self.port,
o='0',
seg_id='1')
elif fault_type == 'commit':
self.filereputil.inject_fault(
f='twophase_transaction_commit_prepared',
y='resume',
r='primary',
p=self.port,
o='0')
elif fault_type == 'dtm_broadcast_prepare':
self.filereputil.inject_fault(f='dtm_broadcast_prepare',
y='resume',
seg_id='1',
p=self.port,
o='0')
elif fault_type == 'dtm_broadcast_commit_prepared':
tinctest.logger.info('coming to if dtm_broadcast_commit_prepared')
self.filereputil.inject_fault(f='dtm_broadcast_commit_prepared',
y='resume',
seg_id='1',
p=self.port,
o='0')
elif fault_type == 'dtm_xlog_distributed_commit':
self.filereputil.inject_fault(f='dtm_xlog_distributed_commit',
y='resume',
seg_id='1',
p=self.port,
o='0')
else:
tinctest.logger.info('No faults to resume')
tinctest.logger.info('Resumed the suspended transaction fault')
#Wait till all the trigger_sqls are complete before returning
sql_count = PSQL.run_sql_command(
'select count(*) from pg_stat_activity;',
flags='-q -t',
dbname='postgres')
while (sql_count.strip() != '1'):
sleep(5)
sql_count = PSQL.run_sql_command(
'select count(*) from pg_stat_activity;',
flags='-q -t',
dbname='postgres')
tinctest.logger.info('stat_activity count %s ' % sql_count)
return
def createdb(self):
PSQL.run_sql_command('Drop database %s;Create database %s;' %
(self.dbname, self.dbname),
dbname='postgres')
def run_sql(self, filename, out_file):
''' Run the provided sql and validate it '''
out_file = local_path(filename.replace(".sql", ".out"))
PSQL.run_sql_file(filename, out_file=out_file)
def get_masterhost(self):
std_sql = "select hostname from gp_segment_configuration where content=-1 and role='p';"
master_host = PSQL.run_sql_command(std_sql,
flags='-q -t',
dbname='postgres')
return master_host.strip()
def test_checkpoint(self):
tinctest.logger.info('Issue Checkpoint')
PSQL.run_sql_file(local_path('checkpoint.sql'), dbname=Steps.dbname)
def run_fts_test_ddl_dml_ct(self):
PSQL.run_sql_file(local_path('fts_test_ddl_dml_ct.sql'))
def run_trigger_sql(self, wait_for_db=True):
''' Run a sql statement to trigger postmaster reset '''
PSQL.run_sql_file(local_path('test_ddl.sql'))
if wait_for_db:
PSQL.wait_for_database_up()
def run_test(self):
sql_file = self.sql_file
ans_file = self.ans_file
nonnegflag = 0
source_file = sys.modules[self.__class__.__module__].__file__
out_directory = self.get_out_dir()
out_file = os.path.join(
out_directory,
os.path.basename(sql_file).replace('.sql', '.out'))
guc_sql_file = os.path.join(
out_directory,
os.path.basename(sql_file).replace('.sql', '_gucs.sql'))
# Create the sql file with GUCS
cmd = "cp " + sql_file + " " + guc_sql_file
shellcmd = "Creating %s" % os.path.basename(guc_sql_file)
run_shell_command(cmd, shellcmd)
# Add the GUCS to the new sql file
self._add_linetofile(
guc_sql_file,
'SET optimizer=on;SET optimizer_log=on;select disable_xform(\'CXformDynamicGet2DynamicTableScan\');\n'
)
# Deal with the positive test cases
if self.negtest == 'False':
nonnegflag = 1
#Create the file containing the plan
explain_file_sql = os.path.join(
out_directory,
os.path.basename(sql_file).replace('.sql', '_explain.sql'))
explain_file_out = os.path.join(
out_directory,
os.path.basename(sql_file).replace('.sql', '_explain.out'))
cmd = "cp " + guc_sql_file + " " + explain_file_sql
shellcmd = "Creating %s" % explain_file_sql
run_shell_command(cmd, shellcmd)
# Remove metadata from explain SQL file
cmd = "sed -i '/^\-\- \@.*/d' " + explain_file_sql
run_shell_command(
cmd, "Remove the metadata info for easier post-processing")
sedcmd = "sed -i -e 's/^SELECT \(.*\) ORDER\(.*\)\;/EXPLAIN SELECT \\1\;/' %s" % explain_file_sql
run_shell_command(sedcmd, "Adding explain SQL")
# Get explain output
PSQL.run_sql_file(explain_file_sql,
dbname=self.db_name,
out_file=explain_file_out)
# Validation part#1: check if plan has dynamic index scan
syscmd = "grep \"Dynamic Index Scan\" " + explain_file_out + " | wc -l "
p = os.popen(syscmd, 'r')
numindexscans = p.readline().strip()
# Run the actual SQL file
PSQL.run_sql_file(guc_sql_file, dbname=self.db_name, out_file=out_file)
result = True
if nonnegflag == 1 and int(numindexscans) <= 0:
result_str = "FAIL: Dynamic Index Scan not used"
self.resultobj = result_str
result = False
else:
self.resultobj = "No need to check query plan. Test case validated with gpdiff"
# Validation part#2: check for wrong results
return Gpdiff.are_files_equal(out_file, ans_file)
if result == False:
extra_diff_file = os.path.join(
out_directory,
os.path.basename(sql_file).replace('.sql',
'_extra_failure.diff'))
cmd = "echo \"" + result_str + "\" > " + extra_diff_file
run_shell_command(cmd, "Adding failure to extra diff file")
cmd = "echo \"Check %s for the explain plan with optimizer ON.\" >> %s" % (
explain_file_out, extra_diff_file)
run_shell_command(cmd, "Adding failure to extra diff file")
return result
def _raise_template0_age(self, limit, segdb):
"""
Increase age of template0 beyond the specified limit on the specified
segment. When a new database is created off template0, the limit will
be exceeded. Assumption: template0 age =~ 0 or at least not already
crossing any of the xid limits. Because this function can only raise
the age, cannot decrease it.
@param limit: one of WARN_LIMIT, STOP_LIMIT and WRAP_LIMIT.
@param segdb: an instance of GpDB class representing the segment on
which the limit will be exceeded.
"""
dburl = dbconn.DbURL(hostname=segdb.hostname, port=segdb.port)
databases = []
with dbconn.connect(dburl, utility=True) as conn:
sql = "SELECT datname FROM pg_database WHERE datallowconn='t'"
for row in dbconn.execSQL(conn, sql):
databases.append(row[0])
sql = "SHOW xid_stop_limit"
stop_limit_guc = int(dbconn.execSQLForSingleton(conn, sql))
sql = "SHOW xid_warn_limit"
warn_limit_guc = int(dbconn.execSQLForSingleton(conn, sql))
sql = ("SELECT datfrozenxid, age(datfrozenxid) FROM pg_database "
"WHERE datname='template0'")
row = dbconn.execSQL(conn, sql).fetchone()
datfxid, age = int(row[0]), int(row[1])
sql = "SELECT get_next_xid()"
current_xid = int(dbconn.execSQLForSingleton(conn, sql))
# Estimate of XIDs consumed by vacuum freeze operaiton on all databases.
vacuum_xids = len(databases) * 500
logger.info("Estimated xids for vacuume freeze: %d" % vacuum_xids)
if limit == self.WARN_LIMIT:
target_age = (2**31) - stop_limit_guc - warn_limit_guc
target_xid = xid_sum(datfxid, target_age)
keep_raising = lambda x: x < target_age
elif limit == self.STOP_LIMIT:
target_age = (2**31) - stop_limit_guc
target_xid = xid_sum(datfxid, target_age)
keep_raising = lambda x: x < target_age
elif limit == self.WRAP_LIMIT:
target_xid = xid_sum(datfxid, 2**31)
keep_raising = lambda x: x > 0
logger.info("Target xid = %d, limit = %d" % (target_xid, limit))
self.assertEqual(
preceding_xid(target_xid, current_xid), current_xid,
"Target xid (%d) precedes current xid (%d)" %
(target_xid, current_xid))
while keep_raising(age):
with dbconn.connect(dburl, utility=True) as conn:
sql = "SELECT get_stop_limit()"
stop_limit = int(dbconn.execSQLForSingleton(conn, sql))
# GPDB may stop accepting connections if we spoof nextXid beyond
# max_xid.
max_xid = xid_sum(stop_limit, -vacuum_xids)
new_xid = preceding_xid(target_xid, max_xid)
logger.info(
"Spoofing next xid to %d, current stop limit = %d" %
(new_xid, stop_limit))
sql = "SELECT spoof_next_xid('%d'::xid)"
dbconn.execSQL(conn, sql % new_xid)
conn.commit()
sql = ("SELECT age(datfrozenxid) FROM pg_database "
"WHERE datname='template0'")
age = int(dbconn.execSQLForSingleton(conn, sql))
logger.info("template0 age raised to %d" % age)
# The vacuum freeze of all databases advances stop_limit further,
# necessary for iterating the while loop. And template0 becomes the
# oldest database aka the only culprit to violate the specified
# limit.
PSQL(sql_cmd='VACUUM FREEZE',
dbname='postgres',
out_file='vacuum_postgres.out').run(validateAfter=True)
for datname in databases:
logger.info('vacuum freeze %s' % datname)
PSQL(sql_cmd='VACUUM FREEZE',
dbname=datname,
out_file='vacuum_%s.out' %
datname).run(validateAfter=True)
def test_master_panic_after_phase1(self):
"""PANIC master after recording distributed commit.
Trigger PANIC in master after completing phase 1 of 2PC,
right after recording distributed commit in xlog but before
broadcasting COMMIT PREPARED to segments. Master's recovery
cycle should correctly broadcast COMMIT PREPARED because
master should find distributed commit record in its xlog
during recovery. Verify that the transaction is committed
after recovery.
JIRA: MPP-19044
"""
tinctest.logger.info("running test: test_crash_master_after_phase1")
gparray = GpArray.initFromCatalog(dbconn.DbURL(), utility=True)
assert len(gparray.getHostList()) == 1, "cannot run on multi-node"
host = gparray.getHostList()[0]
# Must have at least one in-sync and up segment.
primaries = [
p for p in gparray.get_list_of_primary_segments_on_host(host)
if p.getSegmentMode() == "s" and p.getSegmentStatus() == "u"
]
assert len(primaries) > 0, "in-sync and up primary not found"
primary = primaries[0]
tinctest.logger.info("chose primary: %s" % primary.datadir)
# Inject suspend fault after recording distributed commit on master.
cmd = Command("Suspend master post distributed commit",
self.faultcmd % "suspend")
cmd.run(validateAfter=True)
tinctest.logger.info(cmd.get_results().printResult())
# Trigger the fault.
cmd = Command("run DDL",
"psql -f %s" % local_path('sql/ao_create.sql'))
self.proc = cmd.runNoWait()
tinctest.logger.info("runNoWait: %s, pid: %d" %
(cmd.cmdStr, self.proc.pid))
commitBlocked = self.filereputil.check_fault_status(
fault_name='dtm_xlog_distributed_commit',
status="triggered",
seg_id='1',
num_times_hit=1)
# Shutdown of primary (and mirror) should happen only after
# the commit is blocked due to suspend fault.
assert commitBlocked, "timeout waiting for commit to be blocked"
tinctest.logger.info("commit is blocked due to suspend fault")
# At this point, segments have already recorded the
# transaction as prepared by writing PREPARE record in xlog.
# Crash one primary (and its mirror).
mirror = None
mirrors = [
m for m in gparray.get_list_of_mirror_segments_on_host(host)
if m.getSegmentMode() == "s" and m.getSegmentStatus() == "u"
and primary.getSegmentContentId() == m.getSegmentContentId()
]
if len(mirrors) > 0:
mirror = mirrors[0]
tinctest.logger.info("chose mirror: %s" % mirror.datadir)
# Pause FTS probes to avoid a failover while we bring down
# segments. Note that we bring down both primary and its
# mirror, thereby causing double failure. This prevents
# FTS from making changes to segment configuration, even
# if FTS probes are unpaused. It is necessary to unpause
# FTS probes to prevent gang creation from being blocked.
PSQL.run_sql_command_utility_mode("SET gp_fts_probe_pause = on")
tinctest.logger.info("FTS probes paused")
cmdstr = 'pg_ctl -D %s stop -m immediate' % mirror.datadir
tinctest.logger.info("bringing down primary: %s" % cmdstr)
cmd = Command("Shutdown a primary segment", cmdstr)
cmd.run(validateAfter=True)
cmdstr = 'pg_ctl -D %s stop -m immediate' % primary.datadir
tinctest.logger.info("bringing down primary: %s" % cmdstr)
cmd = Command("Shutdown a primary segment", cmdstr)
cmd.run(validateAfter=True)
if mirror is not None:
PSQL.run_sql_command_utility_mode("SET gp_fts_probe_pause = off")
tinctest.logger.info("FTS probes unpaused")
# Resume master. Master should PANIC and go through crash recovery.
cmd = Command("resume master", self.faultcmd % "resume")
cmd.run(validateAfter=True)
tinctest.logger.info(cmd.get_results().printResult())
(rc, out, err) = self.proc.communicate2()
self.proc = None
tinctest.logger.info("runNoWait rc: %d, output: %s, err: %s" %
(rc, out, err))
# Fail if QD did not PANIC.
assert (out.find("commit succeeded") == -1
and err.find("commit succeeded") == -1
and err.find("PANIC") != -1)
# Wait for recovery to complete, timeout after ~ 5 mins.
attempts = 1
recoveryComplete = False
while attempts < 600 and not recoveryComplete:
recoveryComplete = "aaa150" in PSQL.run_sql_command_utility_mode(
"select 'aaa' || (100+50)")
time.sleep(0.5)
attempts = attempts + 1
assert recoveryComplete, "timeout waiting for master to recover"
cmdstr = "gpstop -ar"
cmd = Command("restart", cmdstr)
tinctest.logger.info("restarting the cluster with '%s'" % cmdstr)
cmd.run(validateAfter=True)
tinctest.logger.info("restart complete")
# Verify table got created (commit was successful).
assert PSQL.run_sql_file(local_path('sql/ao_select.sql'))
gpverify = GpdbVerify()
(errorCode, hasError, gpcheckcat_output,
repairScript) = gpverify.gpcheckcat()
assert errorCode == 0, ("gpcheckcat failed: %s" % gpcheckcat_output[0])
# No need to restart GPDB again in tearDown()
self.skipRestart = True
def test_uao_gpload_update(self):
def create_yaml_file():
database = os.environ.get("PGDATABASE", os.environ["USER"])
user = os.environ.get("PGUSER", os.environ["USER"])
port = os.environ.get("PGPORT", "5432")
load_port = os.environ.get("TINC_UAO_LOAD_PORT", "8082")
load_dir = os.path.join(
os.path.dirname(
sys.modules[self.__class__.__module__].__file__), "data")
yaml_filename = os.path.join(
os.path.dirname(
sys.modules[self.__class__.__module__].__file__),
"output/gpload_update.yaml")
yaml_file = open(yaml_filename, "w")
yaml = """VERSION: 1.0.0.1
DATABASE: %s
USER: %s
HOST: localhost
PORT: %s
GPLOAD:
INPUT:
- SOURCE:
LOCAL_HOSTNAME:
- localhost
PORT: %s
FILE:
- %s/*.txt
- COLUMNS:
- id: int
- name: text
- sponsor: text
- FORMAT: text
- DELIMITER: ';'
- ESCAPE: 'OFF'
- ERROR_LIMIT: 25
- LOG_ERRORS: True
OUTPUT:
- TABLE: customer
- MODE: MERGE
- MATCH_COLUMNS:
- id
- UPDATE_COLUMNS:
- name
- sponsor
- MAPPING:
id: id
name: name
sponsor: sponsor
SQL:
""" % (database, user, port, load_port, load_dir)
yaml_file.write(yaml)
return yaml_filename
(setup_file,
setup_out_file) = self.get_sql_files("uao_gpload_update_setup")[0:2]
(sql_file, out_file,
ans_file) = self.get_sql_files("uao_gpload_update")
yaml_filename = create_yaml_file()
PSQL.run_sql_file(setup_file, out_file=setup_out_file)
gphome = os.environ["GPHOME"]
output_file = os.path.join(
os.path.dirname(sys.modules[self.__class__.__module__].__file__),
"output/gpload.out")
load_process = subprocess.Popen(
["%s/bin/gpload" % gphome, "-f", yaml_filename],
stderr=subprocess.STDOUT,
stdout=open(output_file, "w"))
load_process.wait()
PSQL.run_sql_file(sql_file, out_file=out_file)
result = Gpdiff.are_files_equal(out_file, ans_file)
self.assertTrue(result)
def get_dbid(self):
sql_cmd = "select min(dbid) dbid from gp_segment_configuration where role = 'p' and status = 'u' and content > -1"
dbid=PSQL.run_sql_command(sql_cmd= sql_cmd,flags='-q -t')
tinctest.logger.info('Segments %s chosen for fault injection' % (dbid))
return dbid
