def test_with_concurrent_workload(self):
"""
add new mirrors while concurrent workload in progress, check that mirrors added
and current workload won't get affected, in the end, run checkmirrorseg.
Note that: adding mirrors while running workload has checkmirrorseg issue with MPP-24311
"""
gprecover = GpRecover()
self._setup_gpaddmirrors()
self._cleanup_segment_data_dir(self.host_file, self.mirror_data_dir)
sql_setup_file = local_path('sql/ao_heap_table_setup.sql')
sql_file = local_path('sql/ao_heap_table.sql')
pg_stat_activity = 'SELECT * FROM pg_stat_activity;'
PSQL.run_sql_file(sql_setup_file)
subprocess.Popen(["psql", "-f", sql_file])
time.sleep(15)
subprocess.Popen(["gpaddmirrors", "-ai", self.mirror_config_file, "-d", self.mdd])
time.sleep(15)
result = PSQL.run_sql_command(pg_stat_activity, flags='-q -t', dbname='template1')
result = result.strip()
rows = result.split('\n')
self.assertTrue(len(rows) > 1)
while len(rows) > 1:
result = PSQL.run_sql_command(pg_stat_activity, flags='-q -t', dbname='template1')
result = result.strip()
rows = result.split('\n')
time.sleep(3)
gprecover.wait_till_insync_transition()
self.verify_config_file_with_gp_config()
def run_faults_before_trigger(self, checkpoint, cluster_state, fault_type):
'''
@param checkpoint : skip/noskip
@param cluster_state : sync/change_tracking/resync
@param fault_type : commit/abort
@param end_prepare_two_phase_sleep : True/False
@description : 1. Suspend resync faults. 2. Issue Checkpoint before the skip checkpoint, so that the bufferpool is cleared. 3. If skip issue 'skip checkpoint'. 4. Suspend transaction_faults based on test_type.
'''
if cluster_state == 'change_tracking':
self.invoke_fault('filerep_consumer', 'fault')
self.filereputil.wait_till_change_tracking_transition()
tinctest.logger.info('Change_tracking transition complete')
if cluster_state == 'resync':
self.invoke_fault('filerep_resync', 'suspend', role='primary')
if checkpoint == 'skip':
self.invoke_fault('filerep_transition_to_sync_before_checkpoint', 'suspend', role='primary', port=self.port, occurence='0')
rc = self.gprecover.incremental()
if not rc:
raise Exception('Gprecvoerseg failed')
tinctest.logger.info('Cluster in resync state')
PSQL.run_sql_command('CHECKPOINT;', dbname='postgres')
if checkpoint == 'skip':
self.invoke_fault('checkpoint', 'skip', role='primary', port= self.port, occurence='0')
self.inject_fault(fault_type)
if cluster_state == 'resync':
self.filereputil.inject_fault(f='filerep_resync', y='resume', r='primary')
PSQL.wait_for_database_up();
def test_with_fault_injection(self):
"""
add new mirrors run workload to verify if cluster functioning correctly, and
inject the mirror to bring cluster into change tracking, then recoverseg
"""
filerepUtil = Filerepe2e_Util()
gprecover = GpRecover()
self._setup_gpaddmirrors()
self._cleanup_segment_data_dir(self.host_file, self.mirror_data_dir)
res = {'rc': 0, 'stdout' : '', 'stderr': ''}
run_shell_command("gpaddmirrors -a -i %s -d %s --verbose" % (self.mirror_config_file, self.mdd), 'run gpaddmirrros with fault injection', res)
gprecover.wait_till_insync_transition()
self.assertEqual(0, res['rc'])
self.run_simple_ddl_dml()
# after adding new mirrors, check the intergrity between primary and mirror
self.check_mirror_seg()
out_file = local_path('inject_fault_into_ct')
filerepUtil.inject_fault(f='filerep_consumer', m='async', y='fault', r='mirror', H='ALL', outfile=out_file)
# trigger the transtion to change tracking
PSQL.run_sql_command('drop table if exists foo;', dbname = 'template1')
filerepUtil.wait_till_change_tracking_transition()
gprecover.incremental()
gprecover.wait_till_insync_transition()
out_file=local_path('reset_fault')
filerepUtil.inject_fault(f='filerep_consumer', m='async', y='reset', r='mirror', H='ALL', outfile=out_file)
def test_restart_on_failure(self):
"""
@gather_logs_on_failure True
@restart_on_failure True
"""
PSQL.run_sql_command("select * from some_table_that_does_not_exist_to_generate_log_errors")
self.assertTrue(False)
def setUpClass(self):
super(MapreduceMPPTestCase, self).setUpClass()
gppkg = Gppkg()
gppkg.gppkg_install(product_version, 'plperl')
setup_command = "create language plperl;"
PSQL.run_sql_command(setup_command, dbname = os.environ.get('PGDATABASE'))
"compile functions.c and build functions.so"
makeLog = local_path('testBuildSOLog.out')
cmdMake = 'cd '+local_path('c_functions') + ' && make clean && make'
res = {'rc': 0, 'stdout' : '', 'stderr': ''}
run_shell_command(cmdMake, 'compile functions.c', res)
file = open(makeLog, 'w')
file.write(res['stdout'])
file.close()
if res['rc']:
raise Exception('a problem occurred while creating the so files ')
so_dir = local_path('c_functions')
sharedObj = local_path('c_functions/functions.so')
# if not os.path.isfile(sharedObj):
#raise gptest.GPTestError('so files does not exist')
# For multinode cluster, need to copy shared object tabfunc_gppc_demo.so to all primary segments
if gpdbconfig.is_multinode():
res = {'rc':0, 'stderr':'', 'stdout':''}
hosts = gpdbconfig.get_hosts(segments=True)
scp_cmd = 'gpscp -h ' +' -h '.join(map(str,hosts)) +' '+ sharedObj + ' =:%s' % so_dir
run_shell_command(scp_cmd)
if res['rc']:
raise Exception('Could not copy shared object to primary segment')
def test_reindex_pg_class(self):
tinctest.logger.info("create checkpoint")
results = {'rc':0, 'stdout':'', 'stderr':''}
PSQL.run_sql_command("checkpoint", results=results)
assert results['rc'] == 0, results['stderr']
tinctest.logger.info("inject fault to skip checkpoints")
cmd = Command("skip checkpoint on primaries",
"gpfaultinjector -f checkpoint -m async -y skip -o 0"
" -H ALL -r primary")
cmd.run(validateAfter=True)
tinctest.logger.info(cmd.get_results().printResult())
cmd = Command("skip checkpoint on master",
"gpfaultinjector -f checkpoint -m async -y skip -o 0 -s 1")
cmd.run(validateAfter=True)
tinctest.logger.info(cmd.get_results().printResult())
tinctest.logger.info("reindex pg_class indexes")
assert PSQL.run_sql_file(local_path('reindex_pg_class.sql'))
tinctest.logger.info("shutdown immediate")
cmd = Command("shutdown immediate", "gpstop -ai")
cmd.run(validateAfter=True)
tinctest.logger.info(cmd.get_results().printResult())
tinctest.logger.info("trigger recovery")
cmd = Command("restart the cluster", "gpstart -a")
cmd.run(validateAfter=True)
tinctest.logger.info(cmd.get_results().printResult())
tinctest.logger.info("validate recovery succeeded")
results = {'rc':0, 'stdout':'', 'stderr':''}
PSQL.run_sql_command("DROP TABLE reindex_pg_class_test", results=results)
assert results['rc'] == 0, results['stderr']
def test_run_sql_command_wth_username(self):
sql_cmd = 'SELECT 1'
username = getpass.getuser()
self.assertTrue(PSQL.run_sql_command(sql_cmd = sql_cmd, username = username))
#Invalid username
self.assertFalse(PSQL.run_sql_command(sql_cmd = sql_cmd, username = '******'))
def test_11_incremental_after_vacuum(self):
tinctest.logger.info("Test 38: Negative: Full backup, Vacumm, incremental backup -should be an empty dirty_list")
self.cleanup_backup_files()
self.run_workload("backup_dir", 'bkdb38')
self.run_full_backup(dbname = 'bkdb38', option = '--ddboost', location=self.BACKUPDIR)
PSQL.run_sql_command('Vacuum;',dbname='bkdb38')
self.run_incr_backup("dirty_dir_empty", dbname = 'bkdb38', option = '--ddboost', location=self.BACKUPDIR)
def test_no_corruption(self):
"""
Test that gpcheckcat does not report any errors and it does
not generate the verify file if the gpcheckcat test succeeds.
We choose missing_extraneous test for this purpose.
"""
dbname = 'test_no_corruption'
PSQL.run_sql_command('DROP DATABASE IF EXISTS %s' % dbname)
stdout = PSQL.run_sql_command('CREATE DATABASE %s' % dbname)
if not stdout.endswith('CREATE DATABASE\n'):
self.fail('failed to create database: %s' % stdout)
sql_file = local_path('sql/create_tables.sql')
if not PSQL.run_sql_file(sql_file, dbname=dbname,
output_to_file=False):
self.fail('failed to create tables')
res = {'rc': 0, 'stdout' : '', 'stderr': ''}
run_shell_command(
"cd %s && $GPHOME/bin/lib/gpcheckcat -p %s -R missing_extraneous %s" %
(self.gpcheckcat_test_dir, self.master_port, dbname),
results=res)
self.assertEqual(0, res['rc'])
for f in os.listdir(self.gpcheckcat_test_dir):
if fnmatch.fnmatch(f, 'gpcheckcat.verify.%s.*' % dbname):
self.fail('found verify file when not expecting it')
def test_gather_logs(self):
"""
@gather_logs_on_failure True
@restart_on_failure True
"""
PSQL.run_sql_command("select pg_sleep(5)")
self.assertTrue(False)
def test_MPP24237(self):
cmd_cleanup = "psql -Atc \"select datname from pg_database where datname != \'template0\'\" | while read a; do echo \"check for ${a}\";psql -Atc \"select \'drop schema if exists \' || nspname || \' cascade;\' from (select nspname from pg_namespace where nspname like \'pg_temp%\' union select nspname from gp_dist_random(\'pg_namespace\') where nspname like \'pg_temp%\' except select \'pg_temp_\' || sess_id::varchar from pg_stat_activity) as foo\" ${a}; done"
res = {'rc':0, 'stderr':'', 'stdout':''}
run_shell_command(cmd_cleanup, 'do_clean', res)
if res['rc'] > 0:
raise Exception("Failed to do cleanup %s" %res[stderr])
PSQL.run_sql_file(local_path('pre_script.sql'), out_file=local_path('pre_script.out'))
self.assertTrue(Gpdiff.are_files_equal(local_path('pre_script.out'), local_path('pre_script.ans')))
cmd = "select count(*) from pg_tables where schemaname like 'pg_temp%';"
out = PSQL.run_sql_command(cmd, flags ='-q -t')
if int(out) != 0:
tinctest.logger.info("temp tables found")
tinctest.logger.info(PSQL.run_sql_command("select * from pg_tables where schemaname like 'pg_temp%';"))
self.fail("temp tables were found")
PSQL.run_sql_file(local_path('clean_script.sql'))
PSQL.run_sql_file(local_path('clean_script.sql'))
run_shell_command(cmd_cleanup, 'do_clean', res)
if res['rc'] > 0:
raise Exception("Failed to do cleanup %s" %res[stderr])
def backend_terminate(self):
"""
Get the backend process pid by issuing a query to table pg_stat_activity, then
execute pg_terminate_backend(pid)
"""
MAX_TRY = 5
counter = 0
get_backend_pid = 'SELECT procpid FROM pg_stat_activity WHERE current_query like \'create temp table t as select%\';'
result = PSQL.run_sql_command(get_backend_pid, flags = '-q -t', dbname = self.db_name)
tinctest.logger.info('result from getting backend procepid is %s'%result)
procpid = result.strip()
while not procpid and counter < MAX_TRY:
result = PSQL.run_sql_command(get_backend_pid, flags = '-q -t', dbname = self.db_name)
procpid = result.strip()
counter += 1
if counter == MAX_TRY:
raise Exception('unable to select out the backend process pid')
kill_backend = 'SELECT pg_terminate_backend(%s);'%procpid
result = PSQL.run_sql_command(kill_backend, flags = '-q -t', dbname = self.db_name)
tinctest.logger.info('result from pg_terminate_backend is %s'%result)
# check if the process was terminated already
result = PSQL.run_sql_command(get_backend_pid, flags = '-q -t', dbname = self.db_name)
procpid_after_terminate = result.strip()
counter = 0
while procpid_after_terminate == procpid and counter < MAX_TRY:
result = PSQL.run_sql_command(get_backend_pid, flags = '-q -t', dbname = self.db_name)
procpid_after_terminate = result.strip()
counter += 1
time.sleep(1)
if counter == MAX_TRY:
raise Exception('Running pg_terminated_backend failed!')
def resume_faults(self, fault_type, cluster_state='sync'):
'''
@param fault_type : commit/abort/end_prepare_two_phase_sleep/dtm_broadcast_prepare/dtm_broadcast_commit_prepared/dtm_xlog_distributed_commit
@description : Resume the suspended faults
'''
tinctest.logger.info('coming to resume faults with xact %s' % fault_type)
if fault_type == 'abort':
self.filereputil.inject_fault(f='twophase_transaction_abort_prepared', y='resume', r='primary', p=self.port , o='0')
if cluster_state !='resync':
self.filereputil.inject_fault(f='transaction_abort_after_distributed_prepared', y='reset', p=self.port , o='0', seg_id='1')
elif fault_type == 'commit':
self.filereputil.inject_fault(f='twophase_transaction_commit_prepared', y='resume', r='primary', p=self.port , o='0')
elif fault_type == 'dtm_broadcast_prepare':
self.filereputil.inject_fault(f='dtm_broadcast_prepare', y='resume', seg_id = '1', p=self.port, o='0')
elif fault_type == 'dtm_broadcast_commit_prepared':
tinctest.logger.info('coming to if dtm_broadcast_commit_prepared')
self.filereputil.inject_fault(f='dtm_broadcast_commit_prepared', y='resume', seg_id = '1', p=self.port, o='0')
elif fault_type == 'dtm_xlog_distributed_commit':
self.filereputil.inject_fault(f='dtm_xlog_distributed_commit', y='resume', seg_id = '1', p=self.port, o='0')
else:
tinctest.logger.info('No faults to resume')
tinctest.logger.info('Resumed the suspended transaction fault')
#Wait till all the trigger_sqls are complete before returning
sql_count = PSQL.run_sql_command('select count(*) from pg_stat_activity;', flags ='-q -t', dbname='postgres')
while(sql_count.strip() != '1'):
sleep(5)
sql_count = PSQL.run_sql_command('select count(*) from pg_stat_activity;', flags ='-q -t', dbname='postgres')
tinctest.logger.info('stat_activity count %s ' % sql_count)
return
def create_filespace(self, filespace):
'''
@param filespace: Filespace Name
'''
if self.exists(filespace) is True:
tinctest.logger.info('Filespace %s exists' % filespace)
return
file1 = local_path(filespace)
f1 = open(file1+".fs.config","w")
f1.write('filespace:%s\n' % filespace)
f1.write('fsysname:hdfs\n')
fsrep = PSQL.run_sql_command("select fsrep from pg_filespace where fsname='dfs_system';", flags = '-t -q', dbname='postgres')
f1.write('fsreplica:%s\n' % fsrep.strip())
dfs_loc_cmd = "SELECT substring(fselocation from length('hdfs:// ') for (position('/' in substring(fselocation from length('hdfs:// ')))-1)::int) FROM pg_filespace pgfs, pg_filespace_entry pgfse WHERE pgfs.fsname = 'dfs_system' AND fsedbid = 2 AND pgfse.fsefsoid=pgfs.oid ;"
dfs_loc = PSQL.run_sql_command(dfs_loc_cmd,flags = '-t -q', dbname='postgres')
for record in self.config.record:
if record.content == -1:
fileloc = '%s/hdfs_%s' % (os.path.split(record.datadir)[0], filespace)
f1.write("%s:%s:%s/%s\n" % (record.hostname, record.dbid, fileloc, os.path.split(record.datadir)[1]))
cmd = "gpssh -h %s -e 'rm -rf %s; mkdir -p %s'" % (record.hostname, fileloc, fileloc)
run_shell_command(cmd)
else:
f1.write("%s:%s:[%s/%s/%s]\n" % (record.hostname, record.dbid, dfs_loc.strip(), filespace, os.path.split(record.datadir)[1]))
f1.close()
filespace_cmd = '%s/bin/gpfilespace -c %s' % (self.gphome, f1.name)
cmd = Command(name='Gpfilespace command', cmdStr="%s" % (filespace_cmd))
tinctest.logger.info(" %s" % cmd)
cmd.run(validateAfter=True)
result = cmd.get_results()
if result.rc != 0:
raise GPfilespaceException('"gpfilespace creation filespace FAILED". Output = %s ' % resutl.stdout)
def test_11_tupcount_in_pg_aoseg_uaotable_upd(self):
tablename = "uao_table_test11"
tinctest.logger.info("-------------------------------")
tinctest.logger.info("test_11 Verify the tupcount update to pg_aoseg wile updating uao table ")
tinctest.logger.info("-------------------------------\n")
out_file = os.path.join(self.outpath, "create_tab_tupcount_in_pg_aoseg_uaotable_upd_11.out")
sql_file = os.path.join(self.sqlpath, "create_tab_tupcount_in_pg_aoseg_uaotable_upd_11.sql")
ans_file = os.path.join(self.anspath, "create_tab_tupcount_in_pg_aoseg_uaotable_upd_11.ans")
# create uao table and insert 10 rows
sql_out = PSQL.run_sql_file(sql_file=sql_file, out_file=out_file)
assert Gpdiff.are_files_equal(out_file, ans_file)
# get relid for newly created table
relid = self.get_relid(file_name=tablename)
# get the count from pg_aoseg before the update is performed
pgaoseg_tupcount = self.get_tupcount_pgaoseg(relid=relid)
tab_rowcount_bfrupd = self.get_rowcnt_table(tablename=tablename)
sql_cmd3 = "update %s set j=j||'test11' where i = 1;" % tablename
PSQL.run_sql_command(sql_cmd=sql_cmd3, flags="-q -t")
# get the count from pg_aoseg after the update is performed
pgaoseg_tupcount_aftupd = self.get_tupcount_pgaoseg(relid=relid)
tab_rowcount_aftupd = self.get_rowcnt_table(tablename=tablename)
assert int(pgaoseg_tupcount_aftupd) == (int(tab_rowcount_aftupd)) + 1
assert int(tab_rowcount_bfrupd) == (int(tab_rowcount_aftupd))
assert int(pgaoseg_tupcount_aftupd) == (int(pgaoseg_tupcount)) + 1
# vacuum
self.vacuum_full(tablename=tablename)
pgaoseg_tupcount_aftvacuum = self.get_tupcount_pgaoseg(relid=relid)
tab_rowcount_aftvacuum = self.get_rowcnt_table(tablename=tablename)
assert int(pgaoseg_tupcount_aftvacuum) == (int(tab_rowcount_aftvacuum))
def test_ddboost_31_restore_without_compression(self):
"""
MPP-25620: gpdbrestore is failing for ddboost if compression program is
not specified.
"""
PSQL.run_sql_command('''DROP TABLE IF EXISTS t1 CASCADE;
CREATE TABLE t1 (id int);
INSERT INTO t1 select generate_series(1, 10);
CREATE UNIQUE INDEX t1_idx on t1(id);
ALTER TABLE t1 ADD CONSTRAINT verify CHECK (id > 0);
''', dbname = 'gptest');
self.run_gpcrondump(dbname='gptest', option = '-x gptest -z --ddboost')
cmd = 'gpdbrestore -t '+ self.full_backup_timestamp + ' -T public.t1 --ddboost -a'
rc, stdout = self.run_command(cmd)
self.assertEquals(0, rc)
self.assertNotIn("Error executing query", stdout)
self.assertNotIn("Type 2 could not be be found", stdout)
status_file = self.get_restore_status_file_on_master()
with open(status_file, 'r') as fd:
lines = fd.readlines()
self.assertNotIn('psql finished abnormally with return code 1', lines)
def test_ddboost_30_restore_should_not_restore_sequence(self):
"""
MPP-25744, restoring single table should not restore any sequence with ddboost
"""
tinctest.logger.info("Restoring a single table should not restore sequence from other tables")
if not self.check_db_exists('gptest'):
self.create_database('gptest')
PSQL.run_sql_command('''DROP SCHEMA IF EXISTS mfec_test CASCADE;
CREATE SCHEMA mfec_test;
CREATE TABLE mfec_test.kantest_restore (id int,name varchar(10));
INSERT INTO mfec_test.kantest_restore values (10,'kan');
CREATE TABLE mfec_test.test_case_restore (id serial,name varchar(10),adds varchar(10));
INSERT INTO mfec_test.test_case_restore (name ,adds) values ('peter','LA');
''', dbname = 'gptest');
self.run_gpcrondump(dbname='gptest', option = '-s mfec_test --ddboost')
PSQL.run_sql_command('''DROP TABLE mfec_test.kantest_restore;
CREATE TABLE mfec_test.kantest_restore (id int,name varchar(10));
INSERT INTO mfec_test.test_case_restore (name ,adds) values ('peter','LA');
''', dbname = 'gptest');
cmd = 'gpdbrestore -t '+ self.full_backup_timestamp + ' -T mfec_test.kantest_restore --ddboost -a'
self.run_command(cmd)
sql = 'select last_value from mfec_test.test_case_restore_id_seq;'
last_val = self.run_SQLQuery(sql, 'gptest')[0][0]
# sequence number should not be restored back to 1
self.assertEquals(last_val, 2)
def test_streaming(self):
"""
Run sendtest, let database emit WAL.
sendtest should receive a new WAL records. After all, we kill
the walsender process, otherwise the test doesn't finish.
@tags sanity
"""
PSQL.run_sql_command('DROP TABLE IF EXISTS foo')
with WalClient("replication=true") as client:
(sysid, tli, xpos) = client.identify_system()
xpos_ptr = XLogRecPtr.from_string(xpos)
client.start_replication(xpos_ptr)
# Can't use PSQL here as it is blocked due to Sync Rep
subprocess.Popen(['psql', '-c', 'CREATE TABLE foo(a int, b int)'],
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
while True:
msg = client.receive(1000)
if isinstance(msg, WalMessageData):
header = msg.header
# sync replication needs a reply otherwise backend blocks
client.reply(header.walEnd, header.walEnd, header.walEnd)
# success, should get some 'w' message
break
elif isinstance(msg, WalMessageNoData):
# could be timeout
client.reply(xpos_ptr, xpos_ptr, xpos_ptr)
else:
raise StandardError(msg.errmsg)
def test_xlogcleanup(self):
"""
Test for verifying if xlog seg created while basebackup
dumps out data does not get cleaned
"""
shutil.rmtree('base', True)
PSQL.run_sql_command('DROP table if exists foo')
# Inject fault at post checkpoint create (basebackup)
logger.info ('Injecting base_backup_post_create_checkpoint fault ...')
result = self.suspend_at(
'base_backup_post_create_checkpoint')
logger.info(result.stdout)
self.assertEqual(result.rc, 0, result.stdout)
# Now execute basebackup. It will be blocked due to the
# injected fault.
logger.info ('Perform basebackup with xlog & recovery.conf...')
pg_basebackup = subprocess.Popen(['pg_basebackup', '-x', '-R', '-D', 'base']
, stdout = subprocess.PIPE
, stderr = subprocess.PIPE)
# Give basebackup a moment to reach the fault &
# trigger it
logger.info('Check if suspend fault is hit ...')
triggered = self.wait_triggered(
'base_backup_post_create_checkpoint')
self.assertTrue(triggered, 'Fault was not triggered')
# Perform operations that causes xlog seg generation
logger.info ('Performing xlog seg generation ...')
count = 0
while (count < 10):
PSQL.run_sql_command('select pg_switch_xlog(); select pg_switch_xlog(); checkpoint;')
count = count + 1
# Resume basebackup
result = self.resume('base_backup_post_create_checkpoint')
logger.info(result.stdout)
self.assertEqual(result.rc, 0, result.stdout)
# Wait until basebackup end
logger.info('Waiting for basebackup to end ...')
sql = "SELECT count(*) FROM pg_stat_replication"
with dbconn.connect(dbconn.DbURL(), utility=True) as conn:
while (True):
curs = dbconn.execSQL(conn, sql)
results = curs.fetchall()
if (int(results[0][0]) == 0):
break;
# Verify if basebackup completed successfully
# See if recovery.conf exists (Yes - Pass)
self.assertTrue(os.path.exists(os.path.join('base','recovery.conf')))
logger.info ('Found recovery.conf in the backup directory.')
logger.info ('Pass')
def verify_config_file_with_gp_config(self):
"""
compares the gp_segment_configuration and pg_filespace_entry with input file mirror_data_dir, double check
if the cluster is configured as intended
"""
with open(self.mirror_config_file, 'r') as f:
next(f)
for line in f:
line = line.strip()
mirror_seg_infor = line.split('=')[1]
cols = mirror_seg_infor.split(':')
content_id = cols[0]
adress = cols[1]
port = cols[2]
mir_replication_port = cols[3]
query_on_configuration = '''select * from gp_segment_configuration where content=\'%s\' and address=\'%s\' and port=\'%s\'
and replication_port=\'%s\'''' % (content_id, adress, port, mir_replication_port)
config_info = PSQL.run_sql_command(query_on_configuration, flags='-q -t', dbname='template1')
config_info = config_info.strip()
# as intended, the entry should be existing in the cluster
self.assertNotEqual(0, len(config_info))
query_on_fselocation = ''' select fselocation from gp_segment_configuration, pg_filespace_entry where dbid=fsedbid
and preferred_role=\'m\' and content=\'%s\''''%content_id
fs_locations = PSQL.run_sql_command(query_on_fselocation, flags='-q -t', dbname='template1')
size = len(cols)
for fs_index in range(5, size):
fs_location = cols[fs_index]
self.assertIn(os.path.dirname(fs_location), fs_locations)
def psql_and_oom(self):
#Touch oom_malloc in standby_dir and issue PSQL : Check if processes are gone
self.touch_malloc()
PSQL.run_sql_command('Drop table if exists wal_oomt1;Create table wal_oomt1(a1 int, a2 text) with(appendonly=true);')
sleep(2)
if not (self.verify.check_standby_processes()):
return True
return False
def test_run_sql_command_with_pgoptions(self):
sql_cmd = 'show client_min_messages'
output = PSQL.run_sql_command(sql_cmd = sql_cmd, PGOPTIONS = "-c client_min_messages=log")
self.assertIsNotNone(output)
self.assertIsNotNone(re.search('log', output))
#Invalid pgoptions
self.assertFalse(PSQL.run_sql_command(sql_cmd = sql_cmd, PGOPTIONS="test"))
def test_run_sql_command_with_out_file(self):
sql_cmd = 'SELECT 1'
out_file = os.path.join(os.path.dirname(inspect.getfile(self.__class__)),'test.out')
self.assertFalse(os.path.exists(out_file))
PSQL.run_sql_command(sql_cmd = sql_cmd, out_file = out_file)
self.assertTrue(os.path.exists(out_file))
os.remove(out_file)
self.assertFalse(os.path.exists(out_file))
def test_Noperm(self):
"no permission to write output file"
PSQL.run_sql_command('create table mytest (a text);', dbname='template1')
os.system("mkdir ./noperm")
os.system("chmod 000 ./noperm")
cmd = "gpmapreduce -f neg_nopermtowrite.yml"
self.run_gpmapreduce_cmd(gpmaprcmd=cmd, expected_ret = 1)
os.system("chmod 777 ./noperm")
os.system("rm -rf ./noperm")
def get_host_and_db_path(self, dbname, contentid=0):
''' Get the host and database path for the content'''
config = GPDBConfig()
db_oid = PSQL.run_sql_command("select oid from pg_database where datname='%s'" % dbname, flags='-q -t', dbname='postgres')
dbid = PSQL.run_sql_command("select dbid from gp_segment_configuration where content=%s and role='p'" % contentid, flags='-q -t', dbname='postgres')
(host, address) = config.get_host_and_datadir_of_segment(dbid= dbid.strip())
db_path = os.path.join(address, 'base', db_oid.strip())
return (host.strip(), db_path)
def gpcheckmirrorseg(self, options=None, master=False):
"""
gpcheckmirrorseg utility
@param options
@param master
@return True or False and Fix File
"""
# Cleanup a day old out files before proceeding with a new checkmirrorseg
self.cleanup_day_old_out_files()
cur_time = time.strftime("%Y%m%d%H%M%S")
if master:
integrity_outfile = self.LOGPATH + "/checkmastermirrorsegoutput_" + cur_time
else:
integrity_outfile = self.LOGPATH + "/checkmirrorsegoutput_" + cur_time
fix_outfile = self.LOGPATH + "/fixfile_" + cur_time
if options is None:
if master:
version = PSQL.run_sql_command('select version();', flags = '-q -t', dbname= 'postgres')
if '4.3' not in version :
PSQL.run_sql_command("CHECKPOINT; CHECKPOINT; CHECKPOINT; CHECKPOINT; select pg_sleep(60);")
options = " -mastermirror=true -exclude recovery.conf -ignore 'gp_restore' -ignore 'gp_cdatabase' -ignore 'gp_dump' -ignore '_data' -ignore 'wet_execute.tbl' -ignore 'pg_xlog' -ignore 'pg_changetracking' -ignore 'pg_verify' -ignore 'backup_label.old' -dirignore 'pg_xlog' -ignore 'pg_subtrans' -dirignore 'gpperfmon/data' "
else:
options = " -ignore '_data' -ignore 'wet_execute.tbl' -ignore 'gp_dump' -ignore 'core' -ignore pg_changetracking -ignore 'pg_xlog' -ignore 'pg_verify' -parallel=true"
command = "gpcheckmirrorseg.pl -connect '-p %d -d %s' -fixfile %s" \
% (self.pgport, self.dbname, fix_outfile)
command += options
command += "> %s 2>&1" % (integrity_outfile)
tinctest.logger.info("Running gpcheckmirroseg.pl ...")
rc = run_shell_command(command)
if not rc:
tinctest.logger.info('Checkmirrorintegrity shows difference... We will check first if its extra_m before failing')
if not os.path.exists(integrity_outfile):
raise Exception('Integrity output file not present')
ifile = open(integrity_outfile, 'r')
lines = ifile.readlines()
checkmirror = False
if lines[-1].find('no differences found')>=0:
checkmirror = True
else:
# Issue with mirror cleanup from Storage when we abort transaction.
# For now, ignore the extra_m and rewrites
checkmirror = self.ignore_extra_m(fix_outfile)
if master:
if checkmirror:
tinctest.logger.info("-- MasterMirror Integrity passed")
else:
tinctest.logger.error("-- MasterMirror Integrity failed")
else:
if checkmirror:
tinctest.logger.info("-- PrimaryMirror Integrity check passed")
else:
tinctest.logger.info("-- PrimaryMirror Integrity check failed")
return (checkmirror, fix_outfile)
def test_mpp20964(self):
"""
@description Test MPP-20964, uncleaned lock table by pg_terminate_backend
@created 2013-08-21 00:00:00
@modified 2013-08-21 00:00:00
@tags mpp-20964
@product_version gpdb: [4.2.6.3- main]
"""
# setup
PSQL.run_sql_command(
"""
DROP TABLE IF EXISTS foo;
CREATE TABLE foo AS SELECT i a, i b FROM generate_series(1, 100)i;
"""
)
# Let the backend run executor in QE and hang for a while.
sql = """
SELECT * FROM (
SELECT count(CASE WHEN pg_sleep(1) IS NULL THEN 1 END)
FROM foo
)s;
"""
# Run it backend. We'll kill him while he's running.
PSQL(sql_cmd=sql, background=True).run(validateAfter=False)
# Let him reach to QE, just in case.
time.sleep(1)
# Now I'm killing him. Note we kill both QE and QD. QD runs
# proc_exit and tries to get result from running QE. Since
# QE also died due to the termination, QD tries to issue an ERROR,
# turning it to FATAL, calling proc_exit again. He loses a chance
# to release locks.
sql = """
SELECT pg_terminate_backend(procpid)
FROM(
SELECT (pg_stat_get_activity(NULL)).*
FROM gp_dist_random('gp_id')
WHERE gp_segment_id = 0
)s
WHERE sess_id <> current_setting('gp_session_id')
UNION
SELECT pg_terminate_backend(procpid)
FROM pg_stat_activity
WHERE sess_id <> current_setting('gp_session_id')
"""
PSQL.run_sql_command(sql)
# Finally check the pg_locks view to check if corrupted records
# are seen. If proclock was left without valid PGPROC, it would show
# transaction = 0. For some reason, one time might pass the test.
# Check five times.
for i in range(5):
self.check_lock_corruption()
def createdb(self, dbname):
need = False
with DbConn() as conn:
res = conn.execute("SELECT * FROM pg_database " "WHERE datname = '{0}'".format(dbname))
if len(res) == 0:
need = True
# Do it outside of the block above, as it's accessing template1.
if need:
PSQL.run_sql_command('CREATE DATABASE "{0}"'.format(dbname))
def drop_roles(cls):
'''
Cleans up roles that might have been created in the tests.
'''
sql = "select rolname from pg_authid where rolname like 'ext_%'"
lines = PSQL.run_sql_command(sql, flags='-q -t -A')
rolnames = [n.strip() for n in lines.strip().split('\n')]
if len(rolnames) > 0:
sql = "drop owned by {0}; drop role {0}".format(', '.join(rolnames))
PSQL.run_sql_command(sql, dbname='wet')
def test_run_sql_command_with_port(self):
sql_cmd = 'SELECT 1'
if not os.environ['PGPORT']:
port = 5432
else:
port = int(os.environ['PGPORT'])
self.assertTrue(PSQL.run_sql_command(sql_cmd = sql_cmd, port = port))
#Invalid port
self.assertFalse(PSQL.run_sql_command(sql_cmd = sql_cmd, port = 12345))
def installExtension(extdir):
'''
Install C UDF to all hosts
'''
query_result = PSQL.run_sql_command(
'select distinct hostname from gp_segment_configuration',
flags='-t -A')
hosts = []
for host in query_result.strip().split('\n'):
assert (len(host.strip()) > 0)
hosts.append('-h ' + host)
assert (len(hosts) > 0)
hosts = ' '.join(hosts)
make = Command('make', 'make -C {extdir}'.format(extdir=extdir))
make.run(validateAfter=True)
gpscp = Command(
'gpscp',
'gpscp {hosts} {extdir}/*.so =:$GPHOME/lib/postgresql/'.format(
extdir=extdir, hosts=hosts))
gpscp.run(validateAfter=True)
install_file = os.path.join(extdir, 'install.sql')
res = PSQL.run_sql_file(install_file, out_file=install_file + '.out')
assert (res == True)
def check_fault_status(self,fault_name = None, status = None, max_cycle=10):
'''
Check whether a fault is triggered. Poll till the fault is triggered
@param name : Fault name
@param status : Status to be checked - triggered/completed
'''
self.util = Filerepe2e_Util()
tinctest.logger.info("[STLRTest] Running check_fault_status %s", status)
if (not fault_name) or (not status) :
raise Exception("[STLRTest]Need a value for fault_name and status to continue")
poll =0
while(poll < max_cycle):
(ok1,out1) = self.util.inject_fault(f=fault_name, m = 'async', y = 'status', r = 'primary', H ='ALL')
poll +=1
for line in out1.splitlines():
if line.find(fault_name) > 0 and line.find(status) > 0 :
tinctest.logger.info("[STLRTest]Fault %s is %s " % (fault_name,status))
poll = 0
tinctest.logger.info("[STLRTest] Running check_fault_status %s TRUE", status)
return True
tinctest.logger.info("[STLRTest] printing gp segment configuration")
(gp_seg_conf) = PSQL.run_sql_command("select * from gp_segment_configuration order by dbid")
tinctest.logger.info(gp_seg_conf)
#sleep a while before start polling again
sleep(10)
tinctest.logger.info("[STLRTest] Running check_fault_status %s FALSE", status)
return False
def test_mirror_spread(self):
"""
Mirror spreading will place each mirror on a different host within the Greenplum Database array
"""
gprecover = GpRecover()
if self.number_of_segments_per_host > len(self.hosts):
self.skipTest(
'skipping test since the number of host is less than number of segments per hosts'
)
self._setup_gpaddmirrors()
self._cleanup_segment_data_dir(self.host_file, self.mirror_data_dir)
res = {'rc': 0, 'stdout': '', 'stderr': ''}
run_shell_command(
"gpaddmirrors -a -i %s -s -d %s --verbose" %
(self.mirror_config_file, self.mdd),
'run gpaddmirrros with mirror spreading', res)
self.assertEqual(0, res['rc'])
check_mirror_spreading = '''SELECT A.hostname, B.hostname
FROM gp_segment_configuration A, gp_segment_configuration B
WHERE A.preferred_role = \'p\' AND B.preferred_role = \'m\' AND A.content = B.content AND A.hostname <> B.hostname;'''
result = PSQL.run_sql_command(check_mirror_spreading,
flags='-q -t',
dbname='template1')
result = result.strip()
self.assertNotEqual(0, len(result))
rows = result.split('\n')
self.assertEqual(self.number_of_segments, len(rows))
gprecover.wait_till_insync_transition()
self.verify_config_file_with_gp_config()
def check_integrityresults(self):
"""
Check gpverify results from the last token
@return: True or False, -1 is still running
"""
sql = "select vertoken from gp_verification_history order by 1 desc limit 1"
out = PSQL.run_sql_command(sql, flags='-q -t', dbname='postgres')
last_token = out.strip()
if not last_token:
return 0 # No records of gpverify
cmd = Command(name='gpverify',
cmdStr="gpverify --results --token %s" % (last_token))
cmd.run(validateAfter=False)
result = cmd.get_results()
state = result.stdout[len(result.stdout) - 2]
if state.find(self.GPVERIFY_STATE["success"]) > 0:
tinctest.logger.info("gpverify at %s: Successful" % last_token)
return True
elif state.find(self.GPVERIFY_STATE["running"]) > 0:
tinctest.logger.info("gpverify at %s: Running" % last_token)
return -1
elif state.find(self.GPVERIFY_STATE["fail"]) > 0:
tinctest.logger.info("gpverify at %s: Failed" % last_token)
return False
else:
tinctest.logger.info("gpverify has not start")
return 0
def _do_gpinitstandby(self):
"""
Initializes a standby host in the second host in the list self.hosts.
setUpModule would have taken care of installing the binaries on all the hosts.
Hence all we have to do here is a gpinitstandby
"""
standby_host = self.hosts[1]
tinctest.logger.info("Initializing standby master on host: %s" %
standby_host)
# Create master directory on the standby host
res = {'rc': 0, 'stdout': '', 'stderr': ''}
run_shell_command(
"gpssh -h %s -e 'rm -rf %s; mkdir -p %s'" %
(standby_host, self.testcase_master_dir, self.testcase_master_dir),
'create master dir on standby host', res)
if res['rc'] > 0:
raise GpExpandTestCaseException(
"Failed to create segment directories")
# Do gpinitstandby
cmd = GpInitStandby(standby_host,
mdd=os.path.join(self.testcase_master_dir,
'gpseg-1'))
result = cmd.run(validate=True)
if result.rc > 0:
tinctest.logger.error(
"gpinitstandby failed with an error code: %s" % result)
raise GPExpandTestCaseException(
"gpinitstandby failed with an error code. Failing the test module"
)
gpsc = PSQL.run_sql_command("SELECT * FROM gp_segment_configuration")
tinctest.logger.info("Segment configuration: %s" % gpsc)
def getGUCvalue(self, name):
if(len(name) < 1):
return -1
cmd = "show " + name
out = PSQL.run_sql_command(cmd)
result = out.split('\n')
return result[3].strip()
def resume_faults(self,fault_name,trans_state):
''''
@summary : Resume the fault and check status
'''
self.util = Filerepe2e_Util()
tinctest.logger.info("[STLRTest] Running resume_faults")
if not trans_state == 'failover_to_mirror' :
tinctest.logger.info("[STLRTest] fault for %s resume" % fault_name)
(ok1,out1) = self.util.inject_fault(f=fault_name, m = 'async', y = 'resume', r = 'primary', H ='ALL')
if not ok1:
raise Exception("[STLRTest]Fault resume failed")
tinctest.logger.info("[STLRTest]Done fault for %s resume" % fault_name)
if trans_state == 'postmaster_reset':
(ok1,out1) = self.util.inject_fault(f=fault_name, m = 'async', y = 'resume', r = 'mirror', H ='ALL')
if not ok1:
tinctest.logger.info("[STLRTest]Failed fault for %s resume on mirror" % fault_name)
if trans_state == 'failover_to_primary' :
self.check_fault_status(fault_name,'completed')
tinctest.logger.info("[STLRTest] printing gp segment configuration")
(gp_seg_conf) = PSQL.run_sql_command("select * from gp_segment_configuration order by dbid")
tinctest.logger.info(gp_seg_conf)
def stop_start_validate(self, expect_down_segments=False):
"""
Do gpstop -i, gpstart and see if all segments come back up fine
"""
self.gpstart = GpStart()
self.gpstop = GpStop()
tinctest.logger.info("[STLRTest] Running stop_start_validate")
tinctest.logger.info("[STLRTest]Shutting down the cluster")
ok = self.gpstop.run_gpstop_cmd(immediate = 'i')
if not expect_down_segments:
if not ok:
raise Exception('[STLRTest]Problem while shutting down the cluster')
tinctest.logger.info("[STLRTest]Successfully shutdown the cluster.")
tinctest.logger.info("[STLRTest]Restarting the cluster.")
ok = self.gpstart.run_gpstart_cmd()
tinctest.logger.info("[STLRTest] printing gp segment configuration")
(gp_seg_conf) = PSQL.run_sql_command("select * from gp_segment_configuration order by dbid")
tinctest.logger.info(gp_seg_conf)
if not ok:
raise Exception('[STLRTest]Failed to bring the cluster back up')
tinctest.logger.info("[STLRTest]Successfully restarted the cluster.")
if not self.anydownsegments():
raise Exception("[STLRTest]segments were marked down")
else:
return (True, "All segments are up")
def test_option_port_offset(self):
"""
primary port + offset = mirror database port
primary port + (2 * offset) = mirror replication port
primary port + (3 * offset) = primary replication port
"""
gprecover = GpRecover()
port_offset = 500
self._setup_gpaddmirrors(port_offset = port_offset)
self._cleanup_segment_data_dir(self.host_file, self.mirror_data_dir)
res = {'rc': 0, 'stdout' : '', 'stderr': ''}
run_shell_command("gpaddmirrors -a -i %s -d %s --verbose" % (self.mirror_config_file, self.mdd), 'run gpaddmirrros with non default port_offset', res)
self.assertEqual(0, res['rc'])
query_ports = 'SELECT port, replication_port FROM gp_segment_configuration WHERE content = 0 ORDER BY preferred_role DESC;'
result = PSQL.run_sql_command(query_ports, flags='-q -t', dbname='template1')
ports = result.strip().split('\n')
primary_ports = ports[0]
mirror_ports = ports[1]
primary_ports = primary_ports.split('|')
primary_ports = [port.strip() for port in primary_ports]
primary_db_port = int(primary_ports[0])
primary_replic_port = int(primary_ports[1])
mirror_ports = mirror_ports.split('|')
mirror_ports = [port.strip() for port in mirror_ports]
mirror_db_port = int(mirror_ports[0])
mirror_replic_port = int(mirror_ports[1])
self.assertEqual(primary_db_port + port_offset, mirror_db_port)
self.assertEqual(primary_db_port + 2*port_offset, mirror_replic_port)
self.assertEqual(primary_db_port + 3*port_offset, primary_replic_port)
gprecover.wait_till_insync_transition()
self.verify_config_file_with_gp_config()
self.check_mirror_seg()
def test_mpp24119(self):
"""
@description postgres process crashed when running cursor with hold
@product_version gpdb: [4.2.8.1-4.2.99.99], [4.3.3.0-]
"""
start_time = time.time()
conn = pygresql.pg.connect()
conn.query("drop table if exists mpp24119")
conn.query("create table mpp24119(i int, j int)")
conn.query(
"insert into mpp24119 select i , i from generate_series(1, 10) i")
conn.query(
"begin; declare c cursor with hold for select * from mpp24119;")
conn.query("commit")
# The crash happens when exec_execute_message is triggered after a tx with cursor
# with hold is committed. Since there was no way to trigger this, we send a protocol
# message directly to the socket on which the connection is established
sockno = conn.fileno()
msg = pack('!sbi', 'c', 0, 0)
l = len(msg) + 4
res = os.write(sockno, pack('!c', 'E'))
res = os.write(sockno, pack('!i', l))
res = os.write(sockno, msg)
format_start_time = time.strftime("%Y-%m-%d %H:%M:%S",
time.localtime(start_time))
output = PSQL.run_sql_command("SELECT logmessage FROM gp_toolkit.gp_log_system WHERE " + \
"logtime >= '%s' and logdatabase is null and logseverity = 'PANIC'" %(format_start_time), flags= '-q -t')
self.assertFalse(output.strip())
conn.close()
def kill_transc_backend(self):
pid_list = []
sql = "SELECT procpid FROM pg_stat_activity WHERE datname='{0}' AND current_query like 'INSERT INTO%'".format(self.pgdatabase)
tinctest.logger.info("running sql command to get transaction backend process: --- %s"%sql)
procid = PSQL.run_sql_command(sql, flags = '-q -t', dbname= self.pgdatabase)
count = 1
while not procid.strip() and count < 5:
sleep(1)
count += 1
procid = PSQL.run_sql_command(sql, flags = '-q -t', dbname= self.pgdatabase)
if procid.strip():
tinctest.logger.info("got procid to kill: %s " % procid)
pid_list.append(procid)
self.killProcess_byPid(pid_toKill = pid_list)
else:
tinctest.logger.error("There is no active backend process")
def log_segment_state(self):
sql_cmd = "select * from gp_segment_configuration order by dbid"
result = PSQL.run_sql_command(sql_cmd=sql_cmd)
tinctest.logger.info('==========================')
tinctest.logger.info('State of Segments ')
tinctest.logger.info(result)
tinctest.logger.info('==========================')
def do_gpcheckcat(self, dbname=None, alldb=False, online=False, outputFile='checkcat.out', outdir=None):
self.dbstate = DbStateClass('run_validation')
tinctest.logger.info("[STLRTest] Running do_gpcheckcat")
self.dbstate.check_catalog()
(gp_seg_conf) = PSQL.run_sql_command("select * from gp_segment_configuration order by dbid")
tinctest.logger.info(gp_seg_conf)
return True
def has_rows(self, sql, utility=False):
if not utility:
o = PSQL.run_sql_command(sql)
else:
o = PSQL.run_sql_command_utility_mode(sql, port=self.utility_port)
return (o.find("rows)") >= 0
and o.find("(0 rows)") < 0) or o.find("(1 row)") >= 0
def NoRun_test_readserial_concurr_upd_savepoint(self):
test_case_list0 = []
test_case_list0.append('mpp.gpdb.tests.storage.uao.uaocs_mvcc.readserial.readserial_concurr_upd_savepoint.runsql.RunWorkload')
self.test_case_scenario.append(test_case_list0)
sql_cmd="select count(*) from sto_uaocs_mvcc_status t1 , sto_uaocs_mvcc_status t2 where t1.workload='readserial_concurrentupdate_savepoint' and t2.workload='readserial_concurrentupdate_savepoint' and t1.script != t2.script and t1.starttime < t2.starttime and t2.starttime < t1.endtime"
sql_out=PSQL.run_sql_command(sql_cmd = sql_cmd)
self.assertIsNotNone(re.search('1 row', sql_out))
def NoRun_test_readcommit_concurr_alter_vacuum(self):
test_case_list1 = []
test_case_list1.append('mpp.gpdb.tests.storage.uao.uaocs_mvcc.readcommit.readcommit_concurr_alter_vacuum.runsql.RunWorkload')
self.test_case_scenario.append(test_case_list1)
sql_cmd="select count(*) from sto_uaocs_mvcc_status t1 , sto_uaocs_mvcc_status t2 where t1.workload='readcommit_concurr_alter_vacuum' and t2.workload='readcommit_concurr_alter_vacuum' and t1.script != t2.script and t1.starttime < t2.starttime and t2.starttime < t1.endtime"
sql_out=PSQL.run_sql_command(sql_cmd = sql_cmd)
self.assertIsNotNone(re.search('1 row', sql_out))
def create_filespace(self, filespace):
'''
@param filespace: Filespace Name
'''
if self.exists(filespace) is True:
tinctest.logger.info('Filespace %s exists' % filespace)
return
file1 = local_path(filespace)
f1 = open(file1 + ".fs.config", "w")
f1.write('filespace:%s\n' % filespace)
f1.write('fsysname:hdfs\n')
fsrep = PSQL.run_sql_command(
"select fsrep from pg_filespace where fsname='dfs_system';",
flags='-t -q',
dbname='postgres')
f1.write('fsreplica:%s\n' % fsrep.strip())
dfs_loc_cmd = "SELECT substring(fselocation from length('hdfs:// ') for (position('/' in substring(fselocation from length('hdfs:// ')))-1)::int) FROM pg_filespace pgfs, pg_filespace_entry pgfse WHERE pgfs.fsname = 'dfs_system' AND fsedbid = 2 AND pgfse.fsefsoid=pgfs.oid ;"
dfs_loc = PSQL.run_sql_command(dfs_loc_cmd,
flags='-t -q',
dbname='postgres')
for record in self.config.record:
if record.content == -1:
fileloc = '%s/hdfs_%s' % (os.path.split(
record.datadir)[0], filespace)
f1.write("%s:%s:%s/%s\n" %
(record.hostname, record.dbid, fileloc,
os.path.split(record.datadir)[1]))
cmd = "gpssh -h %s -e 'rm -rf %s; mkdir -p %s'" % (
record.hostname, fileloc, fileloc)
run_shell_command(cmd)
else:
f1.write("%s:%s:[%s/%s/%s]\n" %
(record.hostname, record.dbid, dfs_loc.strip(),
filespace, os.path.split(record.datadir)[1]))
f1.close()
filespace_cmd = '%s/bin/gpfilespace -c %s' % (self.gphome, f1.name)
cmd = Command(name='Gpfilespace command',
cmdStr="%s" % (filespace_cmd))
tinctest.logger.info(" %s" % cmd)
cmd.run(validateAfter=True)
result = cmd.get_results()
if result.rc != 0:
raise GPfilespaceException(
'"gpfilespace creation filespace FAILED". Output = %s ' %
resutl.stdout)
def get_all_data_segments_hostname(self, role='m'):
'''Gets all the hosts information based on the input role'''
tinctest.logger.info('Getting all the hosts information..')
get_hosts_cmd = "SELECT DISTINCT(hostname) FROM gp_segment_configuration WHERE role = '%s' AND content != -1" % (
role)
tinctest.logger.info(get_hosts_cmd)
hosts = PSQL.run_sql_command(get_hosts_cmd).split('\n')[3:-3]
return hosts
def get_default_fs_loc(self, role='m', content=0):
fs_sql = '''select fselocation from pg_filespace_entry
where fsefsoid = 3052 and fsedbid = (select dbid from gp_segment_configuration
where role = \'%s\' and content = %s);'''%(role,content)
result = PSQL.run_sql_command(fs_sql, flags = '-q -t', dbname= 'template1')
result = result.strip()
filespace_loc = result.split('\n')
return filespace_loc[0]
def run_faults_before_trigger(self, checkpoint, cluster_state, fault_type):
'''
@param checkpoint : skip/noskip
@param cluster_state : sync/change_tracking/resync
@param fault_type : commit/abort
@param end_prepare_two_phase_sleep : True/False
@description : 1. Suspend resync faults. 2. Issue Checkpoint before the skip checkpoint, so that the bufferpool is cleared. 3. If skip issue 'skip checkpoint'. 4. Suspend transaction_faults based on test_type.
'''
if cluster_state == 'change_tracking':
self.invoke_fault('filerep_consumer', 'fault')
self.filereputil.wait_till_change_tracking_transition()
tinctest.logger.info('Change_tracking transition complete')
if cluster_state == 'resync':
self.invoke_fault('filerep_resync', 'suspend', role='primary')
if checkpoint == 'skip':
self.invoke_fault(
'filerep_transition_to_sync_before_checkpoint',
'suspend',
role='primary',
port=self.port,
occurence='0')
rc = self.gprecover.incremental()
if not rc:
raise Exception('Gprecvoerseg failed')
tinctest.logger.info('Cluster in resync state')
PSQL.run_sql_command('CHECKPOINT;', dbname='postgres')
if checkpoint == 'skip':
self.invoke_fault('checkpoint',
'skip',
role='primary',
port=self.port,
occurence='0')
self.inject_fault(fault_type)
# Can't do it after filerep_resync resume as gets stuck due to
# filerep_transition_to_sync_before_checkpoint suspend above for
# MirroedLock
PSQL.wait_for_database_up()
if cluster_state == 'resync':
self.filereputil.inject_fault(f='filerep_resync',
y='resume',
r='primary')
def is_dbdown(self):
'''return true if system is the system is down'''
tinctest.logger.info("checking if segments are up and in sync")
cmd_str = "select count(*) from gp_segment_configuration where mode = 'c'"
res = PSQL.run_sql_command(cmd_str, dbname = 'template1', flags ='-q -t')
if int(res.strip()) > 0:
return True
return False
def get_count_segments(self):
'''Returns number of segments '''
out = PSQL.run_sql_command(
"select count(*) from gp_segment_configuration where content != -1 and role = 'p' and status = 'u'",
flags='-q -t',
dbname='template1')
for line in out:
return line.strip()
def get_ans_suffix(self):
primary_segments_count = PSQL.run_sql_command(
"select 'primary_segment' from gp_segment_configuration where content >= 0 and \
role = 'p'").count('primary_segment') - 1
if primary_segments_count > 1:
return "%sseg" % primary_segments_count
else:
return None
def wait_till_all_sqls_done(self, count=1):
''' 500 here is just an arbitrarily long time "if-we-exceed-this-then-oh-crap-lets-error-out" value '''
for i in range(1,500):
psql_count = PSQL.run_sql_command("select count(*) from pg_stat_activity where current_query <> '<IDLE>'", flags='-q -t', dbname='postgres')
if int(psql_count.strip()) <= count :
return
sleep(1)
raise Exception('SQLs expected to complete but are still running')
def is_balanced_segments(self):
'''Returns True if primary and mirror are balanced'''
gpseg_sql = "select count(*) from gp_segment_configuration where role != preferred_role;"
balance_segments = PSQL.run_sql_command(gpseg_sql, flags='-t -q')
if balance_segments.strip() != '0':
return False
return True
def get_standby_port(self):
stdby_port_sql = 'select port from gp_segment_configuration where content = -1 and preferred_role=\'m\' ;'
result = PSQL.run_sql_command(stdby_port_sql, flags = '-q -t', dbname= 'template1')
standby_port = result.strip()
if len(standby_port) > 0:
return standby_port
else:
tinctest.logger.error('Not able to get the standby port, check if standby configured!')
def is_not_insync_segments(self):
'''Returns True if no down or change_tracking segments '''
gpseg_sql = "select count(*) from gp_segment_configuration where mode <>'s' or status <> 'u';"
not_insync_segments = PSQL.run_sql_command(gpseg_sql, flags='-t -q')
if not_insync_segments.strip() != '0':
return False
return True
def get_standby_dd(self):
stdby_dir_sql = 'select fselocation from gp_segment_configuration, pg_filespace_entry where dbid = fsedbid and fsefsoid=3052 and content = -1 and preferred_role=\'m\' ;'
result = PSQL.run_sql_command(stdby_dir_sql, flags = '-q -t', dbname= 'template1')
standby_loc = result.strip()
if len(standby_loc) > 0:
return standby_loc
else:
tinctest.logger.error('Not able to get the standby dir, check if standby configured!')
def is_debug(self):
version_str = PSQL.run_sql_command('select version();',
flags='-q -t',
dbname='postgres')
print version_str
if 'assert' in version_str:
print ' coming in if'
return True
def issue_checkpoint(self):
''' Forces the checkpoint to occur'''
cmd_str = "CHECKPOINT;"
result = PSQL.run_sql_command(cmd_str).split('\n')
if len(result) > 1:
return True
else:
return False
