def isSupported(self, plname=None):
"""
check if a given procedural language is supported by gpdb
@param plname: the name of the procedural language to be checked
@return: True if the given procedural language is supported by gpdb, False if it is not supported
"""
if plname is None:
plname = ""
else:
plname = plname.lower()
sql = "SELECT COUNT(*) FROM (SELECT tmplname AS lanname FROM pg_pltemplate UNION SELECT lanname AS lanname FROM pg_language) t WHERE lanname='%s';" % (plname)
cmd = PSQL(sql_cmd = sql, flags = '-q -t', dbname=os.environ.get('PGDATABASE'))
tinctest.logger.info("Running command - %s" %cmd)
cmd.run(validateAfter = False)
result = cmd.get_results()
ok = result.rc
out = result.stdout.strip()
if not ok:
ans = int( out[0].rstrip().lstrip() )
if ans == 0:
return False
elif ans == 1:
return True
else:
raise Exception("Error when retrieving information about procedural languages from catalog")
else:
raise Exception("Error when retrieving information about procedural languages from catalog")
def do_PLPERL_initialize(self):
""" Language PL/PERL upgrade to 9.1: initialize test data """
gpuserRole = GpUserRole(HOST, USER, DBNAME)
gpuserRole.createUser('pltestuser', 'NOSUPERUSER')
gpuserRole.createUser('plsuperuser', 'SUPERUSER')
pg_hba_path = os.path.join(os.environ.get('MASTER_DATA_DIRECTORY'),
'pg_hba.conf')
print 'pg_hba_path', pg_hba_path
pghba_file = PgHba.PgHba(pg_hba_path)
new_ent = PgHba.Entry(entry_type='local',
database=DBNAME,
user='******',
authmethod='trust')
pghba_file.add_entry(new_ent)
new_ent = PgHba.Entry(entry_type='local',
database=DBNAME,
user='******',
authmethod='trust')
pghba_file.add_entry(new_ent)
pghba_file.write()
grantcmd = 'CREATE SCHEMA pltest; GRANT ALL ON SCHEMA pltest TO pltestuser;'
cmd = PSQL(sql_cmd=grantcmd, dbname=DBNAME)
tinctest.logger.info("Running command - %s" % cmd)
cmd.run(validateAfter=False)
result = cmd.get_results()
ok = result.rc
out = result.stdout
if ok:
raise Exception(
'Grant all on schema pltest to pltestuser failed: %s' % out)
def dropPL(self, plname=None):
"""
drop a given procedural language on gpdb
@param plname: the name of the procedural language to be dropped
@return: True if the given procedural language is dropped on gpdb or it does not exist, False if it is not correctly dropped
"""
if plname is None:
plname = ""
else:
plname = plname.lower()
if self.isSupported(plname):
if self.isInstalled(plname):
sql = "DROP LANGUAGE %s;" % (plname)
cmd = PSQL(sql_cmd = sql, flags = '-q -t', dbname=os.environ.get('PGDATABASE'))
tinctest.logger.info("Running command - %s" %cmd)
cmd.run(validateAfter = False)
result = cmd.get_results()
ok = result.rc
out = result.stdout.strip()
if not ok:
return True
else:
print out
return False
else:
return True
else:
raise Exception("Unsupported procedural language %s" % (plname))
def dropPL(self, plname=None):
"""
drop a given procedural language on gpdb
@param plname: the name of the procedural language to be dropped
@return: True if the given procedural language is dropped on gpdb or it does not exist, False if it is not correctly dropped
"""
if plname is None:
plname = ""
else:
plname = plname.lower()
if self.isSupported(plname):
if self.isInstalled(plname):
sql = "DROP LANGUAGE %s;" % (plname)
cmd = PSQL(sql_cmd=sql,
flags='-q -t',
dbname=os.environ.get('PGDATABASE'))
tinctest.logger.info("Running command - %s" % cmd)
cmd.run(validateAfter=False)
result = cmd.get_results()
ok = result.rc
out = result.stdout.strip()
if not ok:
return True
else:
print out
return False
else:
return True
else:
raise Exception("Unsupported procedural language %s" % (plname))
def do_PLPERL_initialize(self):
""" Language PL/PERL upgrade to 9.1: initialize test data """
gpuserRole = GpUserRole(HOST, USER, DBNAME)
gpuserRole.createUser('pltestuser','NOSUPERUSER')
gpuserRole.createUser('plsuperuser','SUPERUSER')
pg_hba_path = os.path.join(os.environ.get('MASTER_DATA_DIRECTORY'), 'pg_hba.conf')
print 'pg_hba_path', pg_hba_path
pghba_file = PgHba.PgHba(pg_hba_path)
new_ent = PgHba.Entry(entry_type='local',
database = DBNAME,
user = '******',
authmethod = 'trust')
pghba_file.add_entry(new_ent)
new_ent = PgHba.Entry(entry_type='local',
database = DBNAME,
user = '******',
authmethod = 'trust')
pghba_file.add_entry(new_ent)
pghba_file.write()
grantcmd = 'CREATE SCHEMA pltest; GRANT ALL ON SCHEMA pltest TO pltestuser;'
cmd = PSQL(sql_cmd = grantcmd, dbname=DBNAME)
tinctest.logger.info("Running command - %s" %cmd)
cmd.run(validateAfter = False)
result = cmd.get_results()
ok = result.rc
out = result.stdout
if ok:
raise Exception('Grant all on schema pltest to pltestuser failed: %s'%out )
def do_PLPERL_initialize(self):
""" Language PL/PERL upgrade to 9.1: initialize test data """
self.doTest(None, "plperl91/test000_initialize", default='-e')
""" Initialize: generate data tbctest.lineitem.tbl, and add users to pg_hba.conf """
fname = os.environ.get(
'TINCREPOHOME') + '/mpp/lib/datagen/datasets/lineitem.csv'
copycmd = 'copy pltest.lineitem from \'' + fname + '\' DELIMITER \'|\';'
cmd = PSQL(sql_cmd=copycmd, dbname=DBNAME)
tinctest.logger.info("Running command - %s" % cmd)
cmd.run(validateAfter=False)
result = cmd.get_results()
ok = result.rc
out = result.stdout
if ok:
raise Exception('Copy statement failed: %s' % out)
gpuserRole = GpUserRole(HOST, USER, DBNAME)
gpuserRole.createUser('pltestuser', 'NOSUPERUSER')
gpuserRole.createUser('plsuperuser', 'SUPERUSER')
pg_hba_path = os.path.join(os.environ.get('MASTER_DATA_DIRECTORY'),
'pg_hba.conf')
print 'pg_hba_path', pg_hba_path
pghba_file = PgHba.PgHba(pg_hba_path)
new_ent = PgHba.Entry(entry_type='local',
database=DBNAME,
user='******',
authmethod='trust')
pghba_file.add_entry(new_ent)
new_ent = PgHba.Entry(entry_type='local',
database=DBNAME,
user='******',
authmethod='trust')
pghba_file.add_entry(new_ent)
pghba_file.write()
grantcmd = 'GRANT ALL ON SCHEMA pltest TO pltestuser;'
cmd = PSQL(sql_cmd=grantcmd, dbname=DBNAME)
tinctest.logger.info("Running command - %s" % cmd)
cmd.run(validateAfter=False)
result = cmd.get_results()
ok = result.rc
out = result.stdout
if ok:
raise Exception(
'Grant all on schema pltest to pltestuser failed: %s' % out)
def GetGpdbVersion(self):
"""
@summary: Returns the version and build number of the Greenplum Database
@return: (version, build)
"""
sql_cmd = 'select version();'
cmd = PSQL(sql_cmd=sql_cmd,
flags='-q -t',
dbname=os.environ.get('PGDATABASE'))
tinctest.logger.info("Running command - %s" % cmd)
cmd.run(validateAfter=False)
result = cmd.get_results()
ok = not result.rc
out = result.stdout.strip()
assert ok, 'Running select version(); returned non-zero code.'
assert len(out) > 0, 'select version() did not return any rows'
# Assumption is that version is enclosed in parenthesis: (Greenplum Database 4.2.1.0 build 1)
version_st = out.find('(') + 1
version_end = out.find(')')
version_str = out[version_st:version_end]
gpdb_version_prefix = 'Greenplum Database '
build_prefix = 'build '
assert version_str.find(
gpdb_version_prefix
) == 0, "'%s' not found in output of select version();" % gpdb_version_prefix
assert version_str.find(build_prefix) > (
version_str.find(gpdb_version_prefix) + len(gpdb_version_prefix)
), "'%s' not found after '%s' in output of select version();" % (
build_prefix, gpdb_version_prefix)
build_st = version_str.find('build')
build_end = build_st + len(build_prefix)
# version is in between 'Greenplum Database ' and 'build ': Greenplum Database 4.2.1.0 build 1
version = version_str[len(gpdb_version_prefix):build_st].strip()
# build number is after 'build ': Greenplum Database 4.2.1.0 build 1
build = version_str[build_end:].strip()
# If "(with assert checking)" is presented, then it is a debug build
# Added by Hai Huang
if (hasExpectedStr(out, '(with assert checking)')):
version += "_debug"
self.is_debug = True
# Strips the STRING portion from the version string
if re.match(".*_.+$", version) and version.find("_debug") == -1:
version = re.sub(r'_.+$', r'', version)
return (version, build)
def check_lock_corruption(self):
"""
Check if pg_locks has records with transaction = 0, which is corrupted.
"""
sql = "SELECT count(*) FROM pg_locks WHERE transaction = 0"
# Use -A and -t, suppress -a, to get only the number.
psql = PSQL(sql_cmd=sql, flags='-A -t')
psql.run()
results = psql.get_results()
# Should be zero.
self.assertEqual(results.stdout.strip(), '0')
def check_no_dangling_prepared_transaction(self):
"""
Check if pg_prepared_xacts reports any records.
"""
while True:
sql = "SELECT count(*) FROM pg_prepared_xacts"
# Use -A and -t, suppress -a, to get only the number.
psql = PSQL(sql_cmd=sql, flags='-A -t')
psql.run()
results = psql.get_results()
if psql.get_results().rc == 0:
break
if (results.stdout.strip() != '0'):
PSQL.run_sql_command("""
SELECT * FROM gp_dist_random('pg_prepared_xacts');
""")
# Should be zero.
self.assertEqual(results.stdout.strip(), '0')
def check_no_dangling_prepared_transaction(self):
"""
Check if pg_prepared_xacts reports any records.
"""
while True:
sql = "SELECT count(*) FROM pg_prepared_xacts"
# Use -A and -t, suppress -a, to get only the number.
psql = PSQL(sql_cmd=sql, flags='-A -t')
psql.run()
results = psql.get_results()
if psql.get_results().rc == 0:
break
if (results.stdout.strip() != '0'):
PSQL.run_sql_command("""
SELECT * FROM gp_dist_random('pg_prepared_xacts');
""")
# Should be zero.
self.assertEqual(results.stdout.strip(), '0')
def check_lock_corruption(self):
"""
Check if pg_locks has records with transaction = 0, which is corrupted.
"""
sql = "SELECT count(*) FROM pg_locks WHERE transaction = 0"
# Use -A and -t, suppress -a, to get only the number.
psql = PSQL(sql_cmd=sql, flags="-A -t")
psql.run()
results = psql.get_results()
# Should be zero.
self.assertEqual(results.stdout.strip(), "0")
def do_PLPERL_initialize(self):
""" Language PL/PERL upgrade to 9.1: initialize test data """
self.doTest(None, "plperl91/test000_initialize", default='-e')
""" Initialize: generate data tbctest.lineitem.tbl, and add users to pg_hba.conf """
fname = os.environ.get('TINCREPOHOME') + '/mpp/lib/datagen/datasets/lineitem.csv'
copycmd = 'copy pltest.lineitem from \'' + fname + '\' DELIMITER \'|\';'
cmd = PSQL(sql_cmd = copycmd, dbname=DBNAME)
tinctest.logger.info("Running command - %s" %cmd)
cmd.run(validateAfter = False)
result = cmd.get_results()
ok = result.rc
out = result.stdout
if ok:
raise Exception('Copy statement failed: %s'%out )
gpuserRole = GpUserRole(HOST, USER, DBNAME)
gpuserRole.createUser('pltestuser','NOSUPERUSER')
gpuserRole.createUser('plsuperuser','SUPERUSER')
pg_hba_path = os.path.join(os.environ.get('MASTER_DATA_DIRECTORY'), 'pg_hba.conf')
print 'pg_hba_path', pg_hba_path
pghba_file = PgHba.PgHba(pg_hba_path)
new_ent = PgHba.Entry(entry_type='local',
database = DBNAME,
user = '******',
authmethod = 'trust')
pghba_file.add_entry(new_ent)
new_ent = PgHba.Entry(entry_type='local',
database = DBNAME,
user = '******',
authmethod = 'trust')
pghba_file.add_entry(new_ent)
pghba_file.write()
grantcmd = 'GRANT ALL ON SCHEMA pltest TO pltestuser;'
cmd = PSQL(sql_cmd = grantcmd, dbname=DBNAME)
tinctest.logger.info("Running command - %s" %cmd)
cmd.run(validateAfter = False)
result = cmd.get_results()
ok = result.rc
out = result.stdout
if ok:
raise Exception('Grant all on schema pltest to pltestuser failed: %s'%out )
def check_lock_corruption(self):
"""
Check if pg_locks has records with NULL pid or mppsessionid = 0.
These must be corrupted lock entries.
"""
sql = "SELECT count(*) FROM pg_locks WHERE pid IS NULL OR mppsessionid = 0"
# Use -A and -t, suppress -a, to get only the number.
psql = PSQL(sql_cmd=sql, flags='-A -t')
psql.run()
results = psql.get_results()
# Should be zero.
self.assertEqual(results.stdout.strip(), '0')
def run_sequence(self, sql, fault, fault_type, segid):
(ok,out) = self.util.inject_fault(f=fault, y=fault_type, seg_id=segid);
if not ok:
raise Exception("Fault dtm_broadcast_commit_prepared injection failed")
psql = PSQL(sql_cmd=sql);
tinctest.logger.debug("Executing:" + sql)
psql.run()
results = psql.get_results()
tinctest.logger.debug(results.stderr)
self.check_no_dangling_prepared_transaction()
if "PANIC" not in results.stderr:
raise Exception("Fault %s type %s (on segid: %d) did not cause the master reset" % (fault, fault_type, segid))
def GetGpdbVersion(self):
"""
@summary: Returns the version and build number of the Greenplum Database
@return: (version, build)
"""
sql_cmd = 'select version();'
cmd = PSQL(sql_cmd = sql_cmd, flags = '-q -t', dbname=os.environ.get('PGDATABASE'))
tinctest.logger.info("Running command - %s" %cmd)
cmd.run(validateAfter = False)
result = cmd.get_results()
ok = not result.rc
out = result.stdout.strip()
assert ok, 'Running select version(); returned non-zero code.'
assert len(out) > 0, 'select version() did not return any rows'
# Assumption is that version is enclosed in parenthesis: (Greenplum Database 4.2.1.0 build 1)
version_st = out.find('(') + 1
version_end = out.find(')')
version_str = out[version_st:version_end]
gpdb_version_prefix = 'Greenplum Database '
build_prefix = 'build '
assert version_str.find(gpdb_version_prefix) == 0, "'%s' not found in output of select version();" % gpdb_version_prefix
assert version_str.find(build_prefix) > (version_str.find(gpdb_version_prefix) + len(gpdb_version_prefix)), "'%s' not found after '%s' in output of select version();" % (build_prefix, gpdb_version_prefix)
build_st = version_str.find('build')
build_end = build_st + len(build_prefix)
# version is in between 'Greenplum Database ' and 'build ': Greenplum Database 4.2.1.0 build 1
version = version_str[len(gpdb_version_prefix):build_st].strip()
# build number is after 'build ': Greenplum Database 4.2.1.0 build 1
build = version_str[build_end:].strip()
# If "(with assert checking)" is presented, then it is a debug build
# Added by Hai Huang
if (hasExpectedStr(out, '(with assert checking)')):
version += "_debug"
self.is_debug = True
# Strips the STRING portion from the version string
if re.match(".*_.+$", version) and version.find("_debug") == -1:
version = re.sub(r'_.+$', r'', version)
return (version, build)
def run(self):
"""Run psql and see if stderr contains "ERROR" string.
If it was an error, re-try with given interval (1 sec by default)
and repeat as many as retry parameter (5 by default). If retry
count exceeds, return False. Otherwise, True.
"""
retry = self.retry
for i in range(self.retry):
cmd = PSQL(**self.psql_args)
logger.debug("Running command: %s" % cmd)
cmd.run(validateAfter=False)
result = cmd.get_results()
if "ERROR" not in result.stderr:
return True
time.sleep(self.interval)
return False
def run_sequence(self, sql, fault, fault_type, segid):
(ok, out) = self.util.inject_fault(f=fault, y=fault_type, seg_id=segid)
if not ok:
raise Exception(
"Fault dtm_broadcast_commit_prepared injection failed")
psql = PSQL(sql_cmd=sql)
tinctest.logger.debug("Executing:" + sql)
psql.run()
results = psql.get_results()
tinctest.logger.debug(results.stderr)
self.check_no_dangling_prepared_transaction()
if "PANIC" not in results.stderr:
raise Exception(
"Fault %s type %s (on segid: %d) did not cause the master reset"
% (fault, fault_type, segid))
def run(self):
"""Run psql and see if stderr contains "ERROR" string.
If it was an error, re-try with given interval (1 sec by default)
and repeat as many as retry parameter (5 by default). If retry
count exceeds, return False. Otherwise, True.
"""
retry = self.retry
for i in range(self.retry):
cmd = PSQL(**self.psql_args)
logger.debug("Running command: %s" % cmd)
cmd.run(validateAfter=False)
result = cmd.get_results()
if "ERROR" not in result.stderr:
return True
time.sleep(self.interval)
return False
def isInstalled(self, plname=None):
"""
check if a given procedural language is installed on gpdb
@param plname: the name of the procedural language to be checked
@return: True if the given procedural language is installed on gpdb, False if it is not installed
"""
if plname is None:
plname = ""
else:
plname = plname.lower()
if self.isSupported(plname):
sql = "SELECT COUNT(*) FROM pg_language WHERE lanname='%s';" % (
plname)
cmd = PSQL(sql_cmd=sql,
flags='-q -t',
dbname=os.environ.get('PGDATABASE'))
tinctest.logger.info("Running command - %s" % cmd)
cmd.run(validateAfter=False)
result = cmd.get_results()
ok = result.rc
out = result.stdout.strip()
print 'ok is', ok
if not ok:
print 'out is', out
ans = int(out[0].rstrip().lstrip())
print 'ans is', ans
if ans == 0:
return False
elif ans == 1:
return True
else:
raise Exception(
"Error when retrieving information about procedural languages from catalog"
)
else:
raise Exception(
"Error when retrieving information about procedural languages from catalog"
)
else:
raise Exception("Unsupported procedural language %s" % (plname))
def run_sequence(self, sql, fault, fault_type, segid, should_panic=True):
(ok,out) = self.util.inject_fault(f=fault, y=fault_type, seg_id=segid);
if not ok:
raise Exception("Fault dtm_broadcast_commit_prepared injection failed")
psql = PSQL(sql_cmd=sql);
tinctest.logger.debug("Executing:" + sql)
psql.run()
results = psql.get_results()
tinctest.logger.debug(results.stderr)
self.check_no_dangling_prepared_transaction()
if "PANIC" not in results.stderr and should_panic:
raise Exception("Fault %s type %s (on segid: %d) did not cause the master reset" % (fault, fault_type, segid))
if "PANIC" in results.stderr and not should_panic:
raise Exception("Fault %s type %s (on segid: %d) caused a PANIC. dtx two phase retry failed" % (fault, fault_type, segid))
PSQL.wait_for_database_up()
def run_SQLCommand(self, sql_cmd = None, dbname = None, username = None, password = None,
PGOPTIONS = None, host = None, port = None):
cmd = PSQL(None, sql_cmd = sql_cmd, dbname = dbname, username = username, password = password, PGOPTIONS = PGOPTIONS, host = host, port = port)
cmd.run(validateAfter = False)
result = cmd.get_results()
return result.rc, result.stdout
def run_SQLCommand(self, sql_cmd = None, dbname = None, username = None, password = None,
PGOPTIONS = None, host = None, port = None):
cmd = PSQL(None, sql_cmd = sql_cmd, dbname = dbname, username = username, password = password, PGOPTIONS = PGOPTIONS, host = host, port = port)
cmd.run(validateAfter = False)
result = cmd.get_results()
return result.rc, result.stdout