def doTest(self, sql_filename):
'''Run the file, compare oids in out file '''
sql_file = local_path(sql_filename)
out_file = local_path(sql_filename.split('.sql')[0] + '.out')
PSQL.run_sql_file(sql_file = sql_file, out_file = out_file)
isOk = self.compare_oids(out_file)
self.assertTrue(isOk)
def setUpClass(cls):
super(GppcTestCase, cls).setUpClass()
"""
compile gppc_test.c and install the gppc_test.so
"""
gpccutil.pre_process()
cmd = 'gpssh --version'
res = {'rc':0, 'stderr':'', 'stdout':''}
run_shell_command (cmd, 'check product version', res)
gppkg = Gppkg()
product_version = res['stdout']
result = gppkg.gppkg_install(product_version, 'libgppc')
#makeLog = loal_path('test00MakeLog.out')
if result:
cmdMakeInstall = 'cd '+local_path('data')+' && make clean && make CPPFLAGS=-D_GNU_SOURCE && make install'
res = {'rc':0, 'stderr':'', 'stdout':''}
run_shell_command (cmdMakeInstall, 'compile gppc_test.c', res)
# Current make file works for linux, but not for Solaris or OSX.
# If compilation fails or installation fails, force system quit: os._exit(1)
if res['rc']:
os._exit(1) # This will exit the test including the next test suites
sharedObj = local_path('data/gppc_test.so')
if not os.path.isfile(sharedObj):
os._exit(1)
# For multinode cluster, need to copy shared object tabfunc_gppc_demo.so to all primary segments
res = {'rc':0, 'stderr':'', 'stdout':''}
hosts = gpdbconfig.get_hosts(segments=True)
scp_cmd = 'gpscp -h ' +' -h '.join(map(str,hosts)) +' '+ sharedObj + ' =:%s' % LIBDIR
run_shell_command(scp_cmd, 'scp share object to all segment', res)
tinctest.logger.info('scp shared object result %s' % res['stdout'])
if res['rc']:
raise Excpetion('Could not copy shared object to primary segment')
def test_with_fault_injection(self):
"""
add new mirrors run workload to verify if cluster functioning correctly, and
inject the mirror to bring cluster into change tracking, then recoverseg
"""
filerepUtil = Filerepe2e_Util()
gprecover = GpRecover()
self._setup_gpaddmirrors()
self._cleanup_segment_data_dir(self.host_file, self.mirror_data_dir)
res = {'rc': 0, 'stdout' : '', 'stderr': ''}
run_shell_command("gpaddmirrors -a -i %s -d %s --verbose" % (self.mirror_config_file, self.mdd), 'run gpaddmirrros with fault injection', res)
gprecover.wait_till_insync_transition()
self.assertEqual(0, res['rc'])
self.run_simple_ddl_dml()
# after adding new mirrors, check the intergrity between primary and mirror
self.check_mirror_seg()
out_file = local_path('inject_fault_into_ct')
filerepUtil.inject_fault(f='filerep_consumer', m='async', y='fault', r='mirror', H='ALL', outfile=out_file)
# trigger the transtion to change tracking
PSQL.run_sql_command('drop table if exists foo;', dbname = 'template1')
filerepUtil.wait_till_change_tracking_transition()
gprecover.incremental()
gprecover.wait_till_insync_transition()
out_file=local_path('reset_fault')
filerepUtil.inject_fault(f='filerep_consumer', m='async', y='reset', r='mirror', H='ALL', outfile=out_file)
def setUpClass(cls):
super(EnhancedTableFunctionTest, cls).setUpClass()
tinctest.logger.info("*** Running the pre-requisite sql files drop.sql and setup.sql")
PSQL.run_sql_file(local_path('sqls/setup/drop.sql'))
PSQL.run_sql_file(local_path('sqls/setup/create.sql'))
tinctest.logger.info("*** Starting the Enhaced table test")
def install_kerberos_conf(self):
"""
Update the kerberos configuration files according the env
and copy in appropriate locations
"""
transforms = {
"%DOMAIN%" : self.kdc_domain,
"%HOSTNAME%" : self.kdc_host
}
input_file_path = local_path(self.krb_template_conf + "/" + self.KRB_CONF_TEMPLATE)
output_file_path = local_path(self.krb_template_conf + "/" + self.KRB_CONF_TEMPLATE[:-2])
with open(input_file_path, 'r') as input:
with open(output_file_path, 'w') as output:
for line in input.readlines():
for key,value in transforms.iteritems():
line = re.sub(key,value,line)
output.write(line)
cmd_str = "sudo cp %s %s" %(output_file_path, self.KRB_CONF)
if not run_shell_command(cmd_str,"Copying krb5.conf"):
raise KerberosUtilException("Couldn't copy krb5.conf")
cmd_str = "sudo cp %s %s" %(local_path(self.krb_template_conf + "/" + self.KDC_CONF_TEMPLATE), self.KDC_CONF)
if not run_shell_command(cmd_str,"Copying kdc.conf"):
raise KerberosUtilException("Couldn't copy kdc.conf")
cmd_str = "sudo cp %s %s" %(local_path(self.krb_template_conf + "/" + self.KADMIN_ACL_CONF_TEMPLATE), self.KADMIN_ACL_CONF)
if not run_shell_command(cmd_str,"Copying kadm5.acl"):
raise KerberosUtilException("Couldn't copy kadm5.acl")
def test_01(self):
"SPI: plpgsql"
sql_file = local_path("query01.sql")
out_file = local_path("query01.out")
ans_file = local_path("query01.ans")
PSQL.run_sql_file(sql_file=sql_file, out_file=out_file)
self.assertTrue(Gpdiff.are_files_equal(out_file, ans_file))
def test_PLPERL91010_super_untrust(self):
""" Language PL/PERL upgrade to 9.1:File system operations are allowed for untrusted PL/PERL """
if self.checkAPPHOMEandLIB("plperl"):
print 'installation'
gp_procedural_languages().installPL('plperlu')
tmpfilename = local_path('plperl91/plsuperuser.tmp')
tmpfile = ''
for i in tmpfilename:
if i == '/':
tmpfile = tmpfile + '\/'
else:
tmpfile = tmpfile + i
tmpfilename = tmpfile
localpath = local_path('')
if sys.platform == 'sunos5':
cmd = 'sed \'s/TMPFILENAME/%s/g\' %s/plperl91/test010_super_untrust.sql > %s/plperl91/test010_super_untrust.sql.tmp && mv %s/plperl91/test010_super_untrust.sql.tmp %s/plperl91/test010_super_untrust.sql' % ( tmpfilename, localpath, localpath, localpath, localpath)
elif sys.platform == 'darwin':
cmd = 'sed -i \'\' \'s/TMPFILENAME/%s/g\' %s/plperl91/test010_super_untrust.sql' % ( tmpfilename, localpath )
else:
cmd = 'sed -i \'s/TMPFILENAME/%s/g\' %s/plperl91/test010_super_untrust.sql' % ( tmpfilename, localpath )
os.system( cmd )
self.doPLPERLUbyuser("plperl91/test010_super_untrust", 'plsuperuser')
checkcmd = 'cat ' + tmpfilename
res = {'rc': 0, 'stdout' : '', 'stderr': ''}
run_shell_command(checkcmd, 'run command %s'%checkcmd, res)
if res['rc']:
raise Exception("Unable to open created file")
def test_outof_shmm_exit_slots(self):
"""
The issue of MPP-19973 is that a shmem exit callback to reset
a temporary namespace is not removed when the temporary namespace is
reset.
In situations, where a temporary namespace is multiple times reset
because of an exception in a subtransaction, the callbacks
use up all shmem_exit slots.
"""
sql_setup_file = local_path('mpp19973_setup.sql')
PSQL.run_sql_file(sql_file=sql_setup_file)
# Test case setup verification
self.assertTrue(table_exists("foo"))
self.assertTrue(function_exists("testfn"))
sql_file = local_path('mpp19973.sql')
out_file = local_path('mpp19973.out')
PSQL.run_sql_file(sql_file=sql_file,
out_file=out_file, output_to_file=True)
# There will be different error messages in the output, but
# we should not run out of shmem exit slots.
self.assertNotRegexpMatches(open(out_file).read(),
"out of on_shmem_exit slots",
"Database should not run out of shmem_exit slots")
def test_43_alter_table_with_oid(self):
'''MPP-13870: Alter table Set Without Oids fails in case of inheritance'''
sql_file = local_path('alter_table_with_oid.sql')
out_file = local_path('alter_table_with_oid.out')
ans_file = local_path('alter_table_with_oid.ans')
PSQL.run_sql_file(sql_file = sql_file, out_file = out_file)
self.assertTrue(Gpdiff.are_files_equal(out_file, ans_file))
def setUpClass(cls):
super(uao_visimap, cls).setUpClass()
sqlpath = local_path("sql")
outpath = local_path("output")
anspath = local_path("expected")
if not os.path.exists(outpath):
os.mkdir(outpath)
def doQuery(self, sqlfile, default=''):
sql_file = local_path(sqlfile)
filename_prefix = sqlfile.split('.sql')[0]
out_file = local_path(filename_prefix + '.out')
ans_file = local_path(filename_prefix + '.ans')
PSQL.run_sql_file(sql_file = sql_file, out_file = out_file)
self.assertTrue(Gpdiff.are_files_equal(out_file, ans_file))
def setUpClass(cls):
super(uao_visimap, cls).setUpClass()
sqlpath=local_path('sql')
outpath=local_path('output')
anspath=local_path('expected')
if not os.path.exists(outpath):
os.mkdir(outpath)
def test_gp_interconnect_fc_ard_142(self):
if (self.cluster_platform.lower().find('red hat enterprise linux server') < 0):
self.skipTest('Test only applies to RHEL platform.')
try:
out = self.checkGUC(self.gp_interconnect_min_retries_before_timeout)
self.assertTrue(len(out) > 4)
out = self.checkGUC(self.gp_interconnect_transmit_timeout)
self.assertTrue(len(out) > 4)
out = self.checkGUC(self.gp_interconnect_fc_method)
self.assertTrue(len(out) > 4)
except:
self.skipTest("GUC " + self.gp_interconnect_min_retries_before_timeout + " or " + self.gp_interconnect_transmit_timeout + " or " + self.gp_interconnect_fc_method + " not defined")
result = runShellCommand('gpssh ' + self.hoststr + ' \"export PATH=$PATH:/sbin; \
sudo insmod ickm.ko ict_type=0x101 seq_array=2 drop_times=80\"')
self.assertTrue(result)
sql_file = local_path(self.common_sql + str(self._testMethodName) + '.sql');
self.assertTrue(PSQL.run_sql_file(local_path(sql_file)))
out_file = sql_file.replace(".sql",".out")
test_ret = "Failed to send packet (seq 2) to" in open(out_file).read() and "retries in 40 seconds" in open(out_file).read()
ret_log = runShellCommand(self.log_str + self._testMethodName + '.log' )
result = runShellCommand('gpssh ' + self.hoststr + ' \"export PATH=$PATH:/sbin;sudo rmmod ickm.ko \"')
self.assertTrue(result)
self.assertTrue(ret_log)
self.assertTrue(test_ret)
def DONTtestMapReduceError007(self):
"MapReduce: MPP-11061: mapreduce crash c_working.yaml"
os.system("cd %s/mpp11061; make" % (local_path('')))
mapr.replaceTemplate(local_path("mpp11061/c_working.yaml.in"), local_path)('')
p = subprocess.Popen(["gpfdist","-d",local_path(''),"-p","8090"])
self.doTest("mpp11061/c_working.yaml")
os.kill(p.pid,signal.SIGKILL)
def replace_new_dirPath(self, orig_filename='recovery.conf', new_filename='recovery_new.conf'):
"""
@summary: Modifies the template config file with new filespace location
@param orig_filename: name of the template config file
@param new_filename: name of the new config file
@return: None
"""
new_file = open(local_path(new_filename),'w')
old_file = open(local_path(orig_filename))
# Finds the gp prefix string from the host entry of the segments
(host, fileLoc) = self.get_segment_host_fileLoc()
slashIndex = fileLoc.rfind('/')
# Extract the gp prefix
gp_prefix = fileLoc[(slashIndex+1):]
lineNo = 0
for line in old_file:
if lineNo == 1 :
# If you checkout the template out file from the command gprecoverseg -o, you'll see that,
# it has failed_host:port:data_dir <SPACE> recovery_host:port:data_dir.
# we intend to change only the data_dir of the recovery_host and not that of the failed host
# so we first divide the lines & modify only the second part
# and then concatenate them again
prefix_index = line.find(gp_prefix)
first_part_of_line = line[:(prefix_index + len(gp_prefix))]
remaining_part_of_line = line[(prefix_index + len(gp_prefix)):]
new_gp_prefix = gp_prefix+'new'
remaining_part_of_line = re.sub(gp_prefix,new_gp_prefix,remaining_part_of_line)
line = first_part_of_line + remaining_part_of_line
lineNo = lineNo + 1
new_file.write(line)
new_file.close()
old_file.close()
def __init__(self, methodName):
self.config = GPDBConfig()
self.mdd = os.environ.get('MASTER_DATA_DIRECTORY')
self.seg_prefix = os.path.basename(self.mdd).split('-')[0]
self.master_host = self.config.get_masterhost()
self.gpinitconfig_template = local_path('configs/gpinitconfig_template')
self.datadir_config_file = local_path('configs/datadir_config_file')
self.mirror_config_file = local_path('configs/mirror_config_file')
self.gpinitconfig_file = local_path('configs/gpinitconfig')
self.host_file = local_path('configs/hosts')
self.hosts = self.config.get_hosts(segments = True)
self.port_base = '40000'
self.master_port = os.environ.get('PGPORT', '5432')
self.primary_data_dir = self.config.get_host_and_datadir_of_segment(dbid = 2)[1]
# initially set the mirror data dir same to primary's
self.mirror_data_dir = os.path.join(os.path.dirname(os.path.dirname(self.primary_data_dir)), 'mirror')
self.gpinitsystem = True
self.number_of_segments = self.config.get_countprimarysegments()
self.number_of_segments_per_host = self.number_of_segments / len(self.hosts)
self.standby_enabled = False
self.number_of_parallelism = 4
self.fs_location = []
super(GPAddmirrorsTestCase, self).__init__(methodName)
def verify_mpp21545(self):
"""
After the pg_terminate_backend(), check if new log file has been generated, if so,
append all new content into a string, plus new content from previous log file(if log
msg cross two logfiles).
Check if new content contains 'PANIC', OR 'Unexpected internal error',or 'Stack trace',
"""
files = [os.path.join(self.PG_LOG_DIR, fname) for fname in os.listdir(self.PG_LOG_DIR)]
latest_logfile = max(files, key=os.path.getmtime)
with open (local_path('log.txt')) as fin:
logfile = fin.read().replace('\n', '')
with open (local_path('linenum.txt')) as fin:
linenum = int(fin.read().replace('\n', ''))
new_logfile_content = ''
previous_logfile_new_content = ''
if latest_logfile != logfile:
with open(latest_logfile) as fin:
new_logfile_content = fin.read().replace('\n', '')
with open(logfile) as fin:
for i, line in enumerate(fin):
if i>=linenum:
previous_logfile_new_content += line
else:
continue
new_log_content = previous_logfile_new_content+' '+new_logfile_content
self.assertNotRegexpMatches(new_log_content,
"PANIC",
"pg_terminate_backend() should not cause PANIC !")
self.assertNotRegexpMatches(new_log_content,
"Unexpected internal error",
"pg_terminate_backend() should not cause Unexpected error !")
self.assertNotRegexpMatches(new_log_content,
"Stack trace",
"pg_terminate_backend() should not cause Stack trace !")
def do_test(self, timeout=0, sqlfile=None, host=None, port=None, username=None, password=None, flags='-a', usetemplate=False):
"""
@summary: Run a PostGIS test case
@param timeout: Number of seconds to run sql file before timing out
@param sqlfile: The path to sql file (relative to TEST.py directory)
@param host: The GPDB master host name to use to connect to database
@param port: The GPDB port used to make connections to the database
@param username: The database username to use to connect to the database
@param password: The password for the database user used to connect to database
"""
if sqlfile is None:
testcase = inspect.stack()[1][3].split('test_')[1]
#file = mkpath(testcase +".sql")
file = local_path(testcase +".sql")
else:
#file = mkpath(sqlfile)
file = local_path(sqlfile)
# run psql on file, and check result
#psql.runfile(file,timeout=timeout,host=host,port=port,username=username,password=password,flag=flags)
#self.checkResult(ifile=file, optionalFlags=" -B")
out_file = local_path(testcase + ".out")
ans_file = local_path(testcase +".ans")
PSQL.run_sql_file(sql_file = file, out_file = out_file)
self.assertTrue(Gpdiff.are_files_equal(out_file, ans_file))
def test_run_sql_file(self):
test_case = MockSQLTestCase('test_query03')
if os.path.exists(test_case.get_out_dir()):
shutil.rmtree(test_case.get_out_dir())
# Default mode
test_case.run_sql_file(local_path('query03.sql'))
self.assertTrue(os.path.exists(os.path.join(test_case.get_out_dir(), 'query03.sql')))
self.assertTrue(os.path.exists(os.path.join(test_case.get_out_dir(), 'query03.out')))
self.assertFalse(self._check_str_in_file('SET optimizer',
os.path.join(test_case.get_out_dir(), 'query03.sql')))
# Optimizer on mode
test_case.run_sql_file(local_path('query03.sql'), optimizer=True)
self.assertTrue(os.path.exists(os.path.join(test_case.get_out_dir(), 'query03_orca.sql')))
self.assertTrue(os.path.exists(os.path.join(test_case.get_out_dir(), 'query03_orca.out')))
self.assertTrue(self._check_str_in_file('SET optimizer=on;',
os.path.join(test_case.get_out_dir(), 'query03_orca.sql')))
# Optimizer off mode
test_case.run_sql_file(local_path('query03.sql'), optimizer=False)
self.assertTrue(os.path.exists(os.path.join(test_case.get_out_dir(), 'query03_planner.sql')))
self.assertTrue(os.path.exists(os.path.join(test_case.get_out_dir(), 'query03_planner.out')))
self.assertTrue(self._check_str_in_file('SET optimizer=off;',
os.path.join(test_case.get_out_dir(), 'query03_planner.sql')))
def test_verify_setup_teardown(self):
test_loader = tinctest.TINCTestLoader()
test_suite = test_loader.loadTestsFromTestCase(MockSQLTestCase)
# As explained above, we want MockSQLTestCase to run if and only if
# it's being invoked by our unit tests. So, it's skipped if discovered
# directly by unit2. Here, bearing in mind that SQLTestCaseTests is itself
# triggered by unit2, we override MockSQLTestCase's skip decorator to allow
# this explicit construction of MockSQLTestCase to proceed.
for test_case in test_suite._tests:
test_case.__class__.__unittest_skip__ = False
if os.path.exists(local_path("output/")):
shutil.rmtree(local_path("output/"))
test_result = unittest.TestResult()
test_suite.run(test_result)
self.assertEqual(test_result.testsRun, 4)
self.assertEqual(len(test_result.errors), 0)
self.assertEqual(len(test_result.skipped), 0)
self.assertEqual(len(test_result.failures), 1)
# Verify if setup and teardown sqls were executed
self.assertTrue(os.path.exists(os.path.join(test_case.get_out_dir(), 'setup.out')))
self.assertTrue(os.path.exists(os.path.join(test_case.get_out_dir(), 'setup', 'setup1.out')))
self.assertTrue(os.path.exists(os.path.join(test_case.get_out_dir(), 'teardown.out')))
self.assertTrue(os.path.exists(os.path.join(test_case.get_out_dir(), 'teardown', 'teardown1.out')))
self.assertTrue(os.path.exists(os.path.join(test_case.get_out_dir(), 'query03_setup.out')))
self.assertTrue(os.path.exists(os.path.join(test_case.get_out_dir(), 'query03_teardown.out')))
def test_with_concurrent_workload(self):
"""
add new mirrors while concurrent workload in progress, check that mirrors added
and current workload won't get affected, in the end, run checkmirrorseg.
Note that: adding mirrors while running workload has checkmirrorseg issue with MPP-24311
"""
gprecover = GpRecover()
self._setup_gpaddmirrors()
self._cleanup_segment_data_dir(self.host_file, self.mirror_data_dir)
sql_setup_file = local_path('sql/ao_heap_table_setup.sql')
sql_file = local_path('sql/ao_heap_table.sql')
pg_stat_activity = 'SELECT * FROM pg_stat_activity;'
PSQL.run_sql_file(sql_setup_file)
subprocess.Popen(["psql", "-f", sql_file])
time.sleep(15)
subprocess.Popen(["gpaddmirrors", "-ai", self.mirror_config_file, "-d", self.mdd])
time.sleep(15)
result = PSQL.run_sql_command(pg_stat_activity, flags='-q -t', dbname='template1')
result = result.strip()
rows = result.split('\n')
self.assertTrue(len(rows) > 1)
while len(rows) > 1:
result = PSQL.run_sql_command(pg_stat_activity, flags='-q -t', dbname='template1')
result = result.strip()
rows = result.split('\n')
time.sleep(3)
gprecover.wait_till_insync_transition()
self.verify_config_file_with_gp_config()
def pre_process(self, product_version):
"""
Just process all the sql and ans files to replace the path with new environment
"""
for sql_file in os.listdir(local_path('sql')):
if sql_file.endswith('.t'):
f_input = open(sql_file, 'r')
f_output = open(sql_file.split('.t')[0], 'w')
for line in f_input:
if line.find('%MYD%') >= 0:
f_output.write(line.replace('%MYD%', local_path('')))
f_input.close()
f_output.close()
for ans_file in os.listdir(local_path('expected')):
if ans_file.endswith('.t'):
f_input = open(ans_file, 'r')
f_output = open(ans_file.split('.t')[0], 'w')
for line in f_input:
if line.find('%MYD%') >= 0:
f_output.write(line.replace('%MYD%', local_path('')))
f_input.close()
f_output.close()
if ans_file.endswith('4.2') and product_version.startswith('4.2'):
shutil.move(os.path.join(local_path('expected'), ans_file), os.path.join(local_path('expected'), ans_file.split('.4.2')[0]))
def setUpClass(self):
super(MapreduceMPPTestCase, self).setUpClass()
gppkg = Gppkg()
gppkg.gppkg_install(product_version, 'plperl')
setup_command = "create language plperl;"
PSQL.run_sql_command(setup_command, dbname = os.environ.get('PGDATABASE'))
"compile functions.c and build functions.so"
makeLog = local_path('testBuildSOLog.out')
cmdMake = 'cd '+local_path('c_functions') + ' && make clean && make'
res = {'rc': 0, 'stdout' : '', 'stderr': ''}
run_shell_command(cmdMake, 'compile functions.c', res)
file = open(makeLog, 'w')
file.write(res['stdout'])
file.close()
if res['rc']:
raise Exception('a problem occurred while creating the so files ')
so_dir = local_path('c_functions')
sharedObj = local_path('c_functions/functions.so')
# if not os.path.isfile(sharedObj):
#raise gptest.GPTestError('so files does not exist')
# For multinode cluster, need to copy shared object tabfunc_gppc_demo.so to all primary segments
if gpdbconfig.is_multinode():
res = {'rc':0, 'stderr':'', 'stdout':''}
hosts = gpdbconfig.get_hosts(segments=True)
scp_cmd = 'gpscp -h ' +' -h '.join(map(str,hosts)) +' '+ sharedObj + ' =:%s' % so_dir
run_shell_command(scp_cmd)
if res['rc']:
raise Exception('Could not copy shared object to primary segment')
def check_duplicate_entry(self):
ans_file = local_path('check_duplicate_entry.ans')
out_file=local_path('check_duplicate_entry.out')
sql_file=local_path('check_duplicate_entry.sql')
PSQL.run_sql_file(sql_file=sql_file,
out_file=out_file)
assert Gpdiff.are_files_equal(out_file,
ans_file)
def test_question_mark_help_option(self):
"""
check the question mark option -? of gpaddmirrors
"""
help_doc = local_path('data/help_doc')
help_output = local_path('data/help_output')
Command('output the help information', 'gpaddmirrors -? > %s' % help_output).run(validateAfter=True)
self.assertTrue(Gpdiff.are_files_equal(help_output, help_doc))
def setUp(self):
seg_dir = os.getcwd()
test_config = local_path("test_gpinit_config_nomirror.template")
hostfile = local_path("test_hostfile")
gpdb_dir = os.getenv("GPHOME", "/usr/local/greenplum-db")
self.mdd = seg_dir + "/master/gpseg-1"
self.create_template(seg_dir, test_config, hostfile)
self.gpdbinit = gpinitsystem(gpdb_dir, test_config.strip(".template"), seg_dir, False)
def setUpClass(cls):
#Create tables that will be used for tests below
super(MVCC_UAOCS_TestCase, cls).setUpClass()
sql_file=local_path('setup/create_tab_foruaocsmvcc.sql')
out_file=local_path('setup/create_tab_foruaocsmvcc.out')
print "sql file for creating tables : "+ (sql_file)
print "output file for sqls : "+ (out_file)
assert PSQL.run_sql_file(sql_file = sql_file,out_file = out_file,flags='-q')
def pre_process(self, sql_files=[], ans_files=[]):
for sql_file in sql_files:
file = local_path(sql_file)
if os.path.isfile(file):
self.replace(file, ' modifies sql data', '')
for ans_file in ans_files:
file = local_path(ans_file)
if os.path.isfile(file):
self.replace(file, ' modifies sql data', '')
def test_ao_toast_empty(self):
PSQL.run_sql_file(local_path("workload_ao_toast_empty.sql"), out_file=local_path("workload_ao_toast_empty.out"))
self.run_validation()
PSQL.run_sql_command("INSERT INTO test_table select i, i*2, i*5, CURRENT_TIMESTAMP from generate_series(1, 20)i;"
"select count(*) from test_table;", flags ='-q -t', out_file=local_path("update_ao_toast_empty.out"))
PSQL.run_sql_command("delete from test_table;"
"select count(*) from test_table;", flags ='-q -t', out_file=local_path("delete_ao_toast_empty.out"))
PSQL.run_sql_command("VACUUM;", flags ='-q -t', out_file=local_path("vac_ao_toast_empty.out"))
self.run_validation()
def test_MapReduceError004(self):
"MapReduce: Test Error Handling 4: MPP-5550"
f1 = open(local_path('mpp5550.yml.in'), 'r')
f2 = open(local_path('mpp5550.yml'), 'w')
for line in f1:
if '@db_user@' in line:
line = line.replace('@db_user@', os.environ.get('PGUSER', getpass.getuser()))
f2.write(line)
f1.close()
f2.close()
self.doTest("mpp5550.yml")
def test_gpactivatestandby_new_host_with_filespace(self):
#from mpp.lib.gpfilespace import Gpfilespace
#gpfile = Gpfilespace()
#gpfile.create_filespace('fs_walrepl_a')
#PSQL.run_sql_file(local_path('filespace.sql'), dbname= self.db_name)
gputil.install_standby()
initstdby = GpinitStandby()
gpact_stdby = GpactivateStandby()
self.mdd = gpact_stdby.get_standby_dd()
self.host = initstdby.get_standbyhost()
self.port = gpact_stdby.get_standby_port()
self.standby_pid = gpact_stdby.get_standby_pid(self.host, self.port, self.mdd)
PSQL.run_sql_file(local_path('create_tables.sql'), dbname = self.db_name)
self.assertTrue(gpact_stdby.activate())
self.assertTrue(gpact_stdby.verify_gpactivatestandby(self.standby_pid, self.host, self.port, self.mdd))
gputil.failback_to_original_master(self.origin_mdd,self.host,self.mdd,self.port)
def test_05_incremental_restore_select_tablefile_with_option_s(self):
tinctest.logger.info("Test25: Test to restore with selected tables in a file with option s , drop and recreate db manualy before restore")
self.run_workload("backup_dir", 'bkdb9')
self.run_full_backup(dbname = 'bkdb9', option = '--ddboost', location=self.BACKUPDIR)
self.run_workload("dirty_dir_1", 'bkdb9')
self.get_data_to_file('bkdb9', 'backup1') #Create a copy of all the tables in database before the last backup
self.run_incr_backup("dirty_dir_1", dbname = 'bkdb9', option = '--ddboost', location=self.BACKUPDIR)
self.drop_create_database('bkdb9')
table_file = local_path('%s/table_file1' % ('restore_incr_T_with_e'))
self.cleanup()
self.run_restore('bkdb9', option = '--table-file=%s --ddboost -e -s bkdb9' % table_file, location=self.BACKUPDIR)
self.validate_restore("restore_incr_T_with_e", 'bkdb9')
def create_filespace(self, filespace):
'''
@param filespace: Filespace Name
'''
if self.exists(filespace) is True:
tinctest.logger.info('Filespace %s exists' % filespace)
return
file1 = local_path(filespace)
f1 = open(file1 + ".fs.config", "w")
f1.write('filespace:%s\n' % filespace)
f1.write('fsysname:hdfs\n')
fsrep = PSQL.run_sql_command(
"select fsrep from pg_filespace where fsname='dfs_system';",
flags='-t -q',
dbname='postgres')
f1.write('fsreplica:%s\n' % fsrep.strip())
dfs_loc_cmd = "SELECT substring(fselocation from length('hdfs:// ') for (position('/' in substring(fselocation from length('hdfs:// ')))-1)::int) FROM pg_filespace pgfs, pg_filespace_entry pgfse WHERE pgfs.fsname = 'dfs_system' AND fsedbid = 2 AND pgfse.fsefsoid=pgfs.oid ;"
dfs_loc = PSQL.run_sql_command(dfs_loc_cmd,
flags='-t -q',
dbname='postgres')
for record in self.config.record:
if record.content == -1:
fileloc = '%s/hdfs_%s' % (os.path.split(
record.datadir)[0], filespace)
f1.write("%s:%s:%s/%s\n" %
(record.hostname, record.dbid, fileloc,
os.path.split(record.datadir)[1]))
cmd = "gpssh -h %s -e 'rm -rf %s; mkdir -p %s'" % (
record.hostname, fileloc, fileloc)
run_shell_command(cmd)
else:
f1.write("%s:%s:[%s/%s/%s]\n" %
(record.hostname, record.dbid, dfs_loc.strip(),
filespace, os.path.split(record.datadir)[1]))
f1.close()
filespace_cmd = '%s/bin/gpfilespace -c %s' % (self.gphome, f1.name)
cmd = Command(name='Gpfilespace command',
cmdStr="%s" % (filespace_cmd))
tinctest.logger.info(" %s" % cmd)
cmd.run(validateAfter=True)
result = cmd.get_results()
if result.rc != 0:
raise GPfilespaceException(
'"gpfilespace creation filespace FAILED". Output = %s ' %
resutl.stdout)
def setUp(self):
super(XidlimitsTests, self).setUp()
Command('re-build regress.so', 'make -C %s xidhelper.so' %
local_path('.')).run(validateAfter=True)
SQLTemplateTestCase.perform_transformation_on_sqlfile(
local_path('load_xidhelper.sql'),
local_path('load_xidhelper.sql.t'),
{'@source@': local_path('xidhelper.so')})
PSQL.run_sql_file(sql_file=local_path('load_xidhelper.sql.t'),
out_file=local_path('load_xidhelper.out.t'))
self.gparray = GpArray.initFromCatalog(dbconn.DbURL(), utility=True)
def test_gpactivatestandby_on_same_host(self):
''' Doesn't work due to STO-374'''
gputil.install_standby(new_stdby_host='localhost')
initstdby = GpinitStandby()
gpact_stdby = GpactivateStandby()
self.mdd = gpact_stdby.get_standby_dd()
self.port = gpact_stdby.get_standby_port()
self.standby_pid = gpact_stdby.get_standby_pid('localhost', self.port,
self.mdd)
PSQL.run_sql_file(local_path('create_tables.sql'), dbname=self.db_name)
self.assertTrue(gpact_stdby.activate())
self.assertTrue(
gpact_stdby.verify_gpactivatestandby(self.standby_pid, 'localhost',
self.port, self.mdd))
gputil.failback_to_original_master(self.origin_mdd,
socket.gethostname(), self.mdd,
self.port)
def create_new_loc_config(self,
hosts,
orig_filename='recovery.conf',
new_filename='recovery_new.conf'):
"""
@summary: Runs recovery with -o option and creates a new config out of template
having new filespace location
@param orig_filename: name of the template config file
@param new_filename: name of the new config file
@return: None
"""
gpcmd = 'source %s/greenplum_path.sh; gprecoverseg -p %s -o %s' % (
GPHOME, hosts, local_path(orig_filename))
self._run_sys_cmd(gpcmd)
self.replace_new_dirPath(orig_filename, new_filename)
def __init__(self,
hadoop_artifact_url,
hadoop_install_dir,
hadoop_data_dir,
template_conf_dir,
hostname='localhost',
secure_hadoop=False):
HadoopUtil.__init__(self, hadoop_artifact_url, hadoop_install_dir,
hadoop_data_dir, hostname)
self.rpmutil = RPMUtil()
self.hostname = hostname
self.hadoop_artifact_url = hadoop_artifact_url
self.hadoop_install_dir = hadoop_install_dir
self.hadoop_binary_loc = ''
self.hadoop_data_dir = hadoop_data_dir
self.template_conf_dir = local_path(template_conf_dir)
self.secure_hadoop = secure_hadoop
# Constants
# under the hadoop template configuration directory
# both the below directories should be present
self.SECURE_DIR_NAME = "conf.secure" # secure configuration files location
self.NON_SECURE_DIR_NAME = "conf.pseudo" # non-secure configuration files location
self.DEPENDENCY_PKGS = [
"fuse-", # eg. fuse-2.8.3-4.el6.x86_64
"fuse-libs", # eg. fuse-libs-2.8.3-4.el6.x86_6
"nc-" # eg. 1.84-22.el6.x86_64"
]
self.PKGS_TO_REMOVE = "^hadoop-*|^bigtop-*|^zookeeper-*|^parquet-*"
self.HADOOP_ENVS = {
"HADOOP_HOME": "/usr/lib/hadoop/",
"HADOOP_COMMON_HOME": "/usr/lib/hadoop/",
"HADOOP_HDFS_HOME": "/usr/lib/hadoop-hdfs/",
"HADOOP_MAPRED_HOME": "/usr/lib/hadoop-mapreduce/",
"YARN_HOME": "/usr/lib/hadoop-yarn/",
"HADOOP_TMP_DIR": "%s/hadoop-hdfs/cache/" % self.hadoop_data_dir,
"MAPRED_TMP_DIR":
"%s/hadoop-mapreduce/cache/" % self.hadoop_data_dir,
"YARN_TMP_DIR": "%s/hadoop-yarn/cache/" % self.hadoop_data_dir,
"HADOOP_CONF_DIR": "/etc/hadoop/conf",
"HADOOP_LOG_DIR":
"%s/hadoop-logs/hadoop-hdfs" % self.hadoop_data_dir,
"MAPRED_LOG_DIR":
"%s/hadoop-logs/hadoop-mapreduce" % self.hadoop_data_dir,
"YARN_LOG_DIR": "%s/hadoop-logs/hadoop-yarn" % self.hadoop_data_dir
}
def download_pkg(self, product_version, gppkg):
"""
Download gppkg from artifacts server.
"""
target_dir = local_path('download/')
if not os.path.exists(target_dir):
os.makedirs(target_dir)
(rc, download_link, package_name) = self.get_download_url_from_build_prod(product_version, gppkg)
if rc != 0:
return (-1, None, None)
wget_cmd = 'wget --html-extension %s -O %s`basename %s`' % (download_link, target_dir, download_link)
logger.debug('Download link: %s' % wget_cmd)
res = {'rc':0, 'stderr':'', 'stdout':''}
run_shell_command (wget_cmd, 'run wget', res)
if res['rc'] > 0:
raise Exception("Gppkg download failed")
return (0, target_dir, package_name)
def run_recovery_with_config(self, filename='recovery_new.conf'):
"""
@summary: Runs incremental recovery using config file.
@param filename: name of the modified recoverseg config file
@return: Boolean value representing the status of recovery process
"""
rcvr_cmd = 'gprecoverseg -a -i %s' % local_path(filename)
cmd = Command(name='Run gprecoverseg',
cmdStr='source %s/greenplum_path.sh;%s' %
(GPHOME, rcvr_cmd))
tinctest.logger.info("Running gprecoverseg : %s" % cmd)
cmd.run(validateAfter=True)
result = cmd.get_results()
if result.rc != 0 or result.stderr:
return False
return True
def test_pg_aocsseg_corruption(self):
self.create_appendonly_tables(row=False)
config = GPDBConfig()
host, port = config.get_hostandport_of_segment()
self.transform_sql_file(
os.path.join(self.sql_dir, 'corrupt_pg_aocsseg.sql.t'), 'co1')
out_file = os.path.join(self.output_dir, 'corrupt_pg_aocsseg.out')
ans_file = os.path.join(self.ans_dir, 'corrupt_pg_aocsseg.ans')
sql_file = os.path.join(self.sql_dir, 'corrupt_pg_aocsseg.sql')
PSQL.run_sql_file_utility_mode(sql_file,
out_file=out_file,
host=host,
port=port,
dbname=os.environ['PGDATABASE'])
if not Gpdiff.are_files_equal(
out_file, ans_file, match_sub=[local_path('sql/init_file')]):
raise Exception(
'Corruption test of pg_aocsseg failed for appendonly tables !')
def create_filespace(self, fsname = 'filespace_a'):
if self.filespace_exists(fsname) is False:
config = self.get_config_with_fs()
file_config = local_path('%s_config' % fsname)
f1 = open(file_config , "w")
f1.write('filespace:%s\n' % fsname)
for record in config:
if record[1] == 'p':
fileloc = '%s/%s/primary' % (os.path.split(record[3])[0], fsname)
else:
fileloc = '%s/%s/mirror' % (os.path.split(record[3])[0], fsname)
cmd = "gpssh -h %s -e 'rm -rf %s; mkdir -p %s'" % (record[2], fileloc, fileloc)
self.run_command(cmd)
f1.write("%s:%s:%s/%s\n" % (record[2], record[0], fileloc, os.path.split(record[3])[1]))
f1.close()
fs_cmd = '%s/bin/gpfilespace -c %s' % (self.gphome, file_config)
self.run_gpcommand(fs_cmd)
def infer_metadata(self):
intended_docstring = ""
sql_file = local_path(self.gucCheck_sql + str(self._testMethodName) + '_1.sql')
with open(sql_file, 'r') as f:
for line in f:
line = line.strip()
if line.find('--') != 0:
break
intended_docstring += line[2:].strip()
intended_docstring += "\n"
line = line[2:].strip()
if line.find('@') != 0:
continue
line = line[1:]
(key, value) = line.split(' ', 1)
self._metadata[key] = value
self.gpdb_version = self._metadata.get('gpdb_version', None)
def do_PLJAVA_setup(self):
"""Language: PL/Java Setup"""
if self.checkAPPHOMEandLIB("pljava", "JAVA_HOME"):
sql_file = local_path("pljava/setup.sql")
PSQL.run_sql_file(sql_file=sql_file)
javahome = os.environ.get("JAVA_HOME")
ldpath = "LD_LIBRARY_PATH=%s/jre/lib/amd64/server:$LD_LIBRARY_PATH\nexport LD_LIBRARY_PATH" % javahome
if platform.machine() == "i686" or platform.machine() == "i386":
ldpath = "LD_LIBRARY_PATH=%s/i386/server/libjvm.so:$LD_LIBRARY_PATH\nexport LD_LIBRARY_PATH" % javahome
Command(name='add ldpath into greenplum_path.sh',
cmdStr="echo '%s' >> $GPHOME/greenplum_path.sh" %
ldpath).run()
self.gpstop.run_gpstop_cmd(restart=True)
pljava_install = os.path.join(
GPHOME, "share/postgresql/pljava/install.sql")
PSQL.run_sql_file(sql_file=pljava_install, dbname='gptest')
else:
self.skipTest('skipped')
def run_sql_under_KM_dropMore(self,suffix):
result = runShellCommand('gpssh' + self.hoststr + ' \"sudo dmesg -c\"')
self.assertTrue(result)
runShellCommand('gpssh ' + self.hoststr + ' \"export PATH=$PATH:/sbin; \
sudo rmmod ickm.ko\"')
result = runShellCommand('gpssh' + self.hoststr + ' \"export PATH=$PATH:/sbin;sudo insmod ickm.ko ict_type=0x101 seq_array=2 drop_times=80\"')
self.assertTrue(result)
sql_file = local_path(self.gucCheck_sql + str(self._testMethodName) + suffix + '.sql')
self.assertTrue(PSQL.run_sql_file(sql_file))
result = runShellCommand('gpssh' + self.hoststr + ' \"export PATH=$PATH:/sbin;sudo rmmod ickm.ko \"')
self.assertTrue(result)
out_file = sql_file.replace(".sql",".out")
return out_file
def test_start_gpdb_with_high_transaction_id(self):
"""
@description GPDB hang after high transaction id
@created 2013-04-18 00:00:00
@modified 2013-04-18 00:00:00
@tags transaction MPP-17302 MPP-17323 MPP-17325 MPP-18462 MPP-18463 schedule_transaction
@note This requires that both primary and mirror to reset xlog.
Repro step from Hitoshi:
gpstop -a
pg_resetxlog -x 100000000 /data/haradh1/gpdata/d/gpseg0
dd if=/dev/zero of=/data/haradh1/gpdata/d/gpseg0/pg_clog/0017 oflag=append conv=notrunc bs=1048576 count=1
cp /data/haradh1/gpdata/d/gpseg0/pg_clog/0017 /data/haradh1/gpdata/d/gpseg0/pg_distributed/02FA
gpstart -a
"""
# @note: need a class to get GPDB configuration, need to get primary/mirror segment location
sqlcmd = "select fselocation from gp_segment_configuration, pg_filespace_entry where dbid=fsedbid and content=0"
with dbconn.connect(dbconn.DbURL()) as conn:
segments = dbconn.execSQL(conn, sqlcmd)
# @note: Issue with self.run_gpstop, hard-coded remoteHost to mdw
# @note: upgrade model uses a series of gpstop and gpstart command, need helper classes
cmd = GpStop("gpstop")
cmd.run(validateAfter=True)
for segment in segments:
cmd = Command(name="reset xlog",
cmdStr="pg_resetxlog -x 100000000 %s" % segment[0])
cmd.run()
xlogfile = local_path('xlog_file')
# @todo: able to copy the xlogfile remotely
shutil.copyfile(xlogfile, "%s/pg_clog/0017" % segment[0])
shutil.copyfile(xlogfile, "%s/pg_distributedlog/02FA" % segment[0])
# @note: workaround the issue with tinc and 4.1 gppylib
cmd = Command(name='run gpstop',
cmdStr='source %s/greenplum_path.sh;\
gpstart -a' % os.environ['GPHOME'])
cmd.run(validateAfter=True)
def test_file_with_gucs(self):
test_case = MockSQLTestCase('test_test_file_with_gucs')
result = test_case.run_test()
self.assertTrue(
os.path.exists(
os.path.join(test_case.get_out_dir(),
'test_file_with_gucs.sql')))
self._check_gucs_exist_in_file(
os.path.join(test_case.get_out_dir(), 'test_file_with_gucs.sql'),
test_case.gucs)
self.assertTrue(
os.path.exists(
os.path.join(test_case.get_out_dir(),
'test_file_with_gucs.out')))
self.assertTrue(
Gpdiff.are_files_equal(
os.path.join(test_case.get_out_dir(),
'test_file_with_gucs.sql'),
local_path('gucs/test_file_with_gucs.sql')))
def test_ddboost_23_full_exclude_schema_file(self):
'''exclude multiple tables '''
tinctest.logger.info("Test-ddboost-20: Test for exclude multiple tables with --ddboost")
self.run_workload("backup_dir_simple_db", 'bkdbbb')
exclude_schema_file = local_path('%s/exclude_schema_file' % ('filter_file'))
self.run_full_backup(dbname = 'bkdbbb', option = "--exclude-schema-file %s --ddboost" % exclude_schema_file)
self.get_data_to_file('bkdbbb', 'backup1') #Create a copy of all the tables in database before the last backup
#truncate other table in the databse before restore it from dump file
cmdTrunc = 'psql -d bkdbbb -c "truncate table schema_ao.ao_table1;"'
self.run_command(cmdTrunc)
#restore from backup
self.cleanup()
self.run_restore('bkdbbb', option = ' -s bkdbbb --ddboost')
self.compare_table_data('bkdbbb')
self.drop_database('bkdbbb')
def init_with_prompt(self, filespace_loc):
standby = self.get_standbyhostnode()
gp_cmd = "/bin/bash -c 'gpinitstandby -s %s -a'" % (standby)
logfile = open(local_path('install2.log'), 'w')
child = pexpect.spawn(gp_cmd, timeout=400)
child.logfile = logfile
sleep(5)
check = child.expect(
['.* Enter standby filespace location for filespace.*', ' '])
child.sendline(filespace_loc)
sleep(10)
check = child.expect(['.*Successfully created standby master.*'])
if check != 0:
tinctest.logger.error('gpinitstandy failed')
return False
child.close()
return True
def doTest(self, num=None, filename='query', default='-a', match_sub=[]):
# get file path to queryXX.sql
if num == None:
sql_file = local_path('%s.sql' % (filename))
out_file = local_path('%s.out' % (filename))
ans_file = local_path('%s.ans' % (filename))
else:
sql_file = local_path('%s%02d.sql' % (filename, num))
out_file = local_path('%s%02d.out' % (filename, num))
ans_file = local_path('%s%02d.ans' % (filename, num))
# run psql on fname, and check result
PSQL.run_sql_file(sql_file = sql_file, out_file = out_file, flags=default)
self.assertTrue(Gpdiff.are_files_equal(out_file=out_file, ans_file=ans_file, match_sub=match_sub))
def validate_test_CatalogCheck(self, action,storage):
file_name =action+'_'+storage
out_file = self.base_dir+ "/sql/"+file_name+'.out'
ans_file = self.base_dir+ "/expected/"+file_name+'.ans'
tinctest.logger.info( 'out-file == %s \n' % out_file)
tinctest.logger.info( 'ans-file == %s \n' % ans_file)
# Validate Ans file
self.validate_sql(ans_file,out_file)
if storage == 'multisegfiles':
''' check if multi_segfile_tab file has multiple segfiles per column '''
tablename='multi_segfile_tab'
relid = self.get_relid(file_name=tablename )
utilitymodeinfo=self.get_utilitymode_conn_info( relid=relid)
u_port=utilitymodeinfo[0]
u_host=utilitymodeinfo[1]
assert(1 < int(self.get_segment_cnt(relid=relid,host=u_host,port= u_port)))
# Check Correctness of the catalog
self.dbstate = DbStateClass('run_validation')
outfile = local_path("gpcheckcat_"+datetime.datetime.fromtimestamp(time.time()).strftime('%Y%m%d%H%M%S')+".out")
self.dbstate.check_catalog(outputFile=outfile)
def create_runaway_udf(dbname=None):
build_udf()
SQLTemplateTestCase.perform_transformation_on_sqlfile(
local_path('./udfs/runaway_test.sql.in'),
local_path('./udfs/runaway_test.sql'),
{'@source@': local_path('./udfs/runaway_test.so')})
tinctest.logger.info(
'Creating Runaway Query Termination testing UDFs by running sql file %s'
% local_path('./udfs/runaway_test.sql'))
PSQL.run_sql_file(sql_file=local_path('./udfs/runaway_test.sql'),
out_file=local_path('./udfs/runaway_test.out'),
dbname=dbname)
def search_log(cmd, key, segdir, logname, start=""):
"""
Search using gplogfilter. Returns True if match has no '0 lines' -
meaning some match found
"""
logfilename = local_path(logname)
GPHOME = os.getenv("GPHOME")
if len(start) > 0:
start = "-b %s" % start
cmd_str = ("gpssh %s -e \"source %s/greenplum_path.sh; "
"gplogfilter %s -m %s %s\" > %s 2>&1" %
(cmd, GPHOME, start, key, segdir, logfilename))
cmd = Command("search_log", cmdStr=cmd_str)
cmd.run(validateAfter=False)
f = open(logfilename, "r")
for line in f.readlines():
line = line.strip()
if line.find("match:") > 0:
if line.find("0 lines") < 0:
return True
return False
def run_crash_and_recovery_fast(self,
test_dir,
pass_num,
cluster_state,
test_type,
ddl_type,
aborting_create_needed=False):
if pass_num == 0:
self.wait_till_all_sqls_done()
else:
mydir = local_path(test_dir) + '/trigger_sql/sql/'
tinctest.logger.info('mydir = %s ' % mydir)
trigger_count = len(glob.glob1(mydir, "*trigger.sql"))
tinctest.logger.info('*** Count of trigger : %s *** ' %
(trigger_count))
if test_dir == "abort_create_tests":
''' vacuum full sql don't hit the suspend fault.'''
trigger_count = trigger_count - 1
if test_dir == "abort_create_needed_tests":
''' Not all SQLs hit the fault for this case, hence wait for them to complete and then others to hit the fault'''
self.wait_till_all_sqls_done(8 + 1)
trigger_count = 8
if test_dir == "abort_abort_create_needed_tests":
''' Not all SQLs hit the fault for this case, hence wait for them to complete and then others to hit the fault'''
self.wait_till_all_sqls_done(6 + 1)
trigger_count = 6
fault_type = self.get_faults_before_executing_trigger_sqls(
pass_num,
cluster_state,
test_type,
ddl_type,
aborting_create_needed=False)
fault_hit = self.fileutil.check_fault_status(
fault_name=fault_type,
status="triggered",
num_times_hit=trigger_count)
if not fault_hit:
raise Exception('Fault not hit expected number of times')
self.stop_start_validate(cluster_state)
def test_reindex_pg_class(self):
tinctest.logger.info("create checkpoint")
results = {'rc': 0, 'stdout': '', 'stderr': ''}
PSQL.run_sql_command("checkpoint", results=results, dbname=self.dbname)
assert results['rc'] == 0, results['stderr']
tinctest.logger.info("inject fault to skip checkpoints")
cmd = Command(
"skip checkpoint on primaries",
"gpfaultinjector -f checkpoint -m async -y skip -o 0"
" -H ALL -r primary")
cmd.run(validateAfter=True)
tinctest.logger.info(cmd.get_results().printResult())
cmd = Command(
"skip checkpoint on master",
"gpfaultinjector -f checkpoint -m async -y skip -o 0 -s 1")
cmd.run(validateAfter=True)
tinctest.logger.info(cmd.get_results().printResult())
tinctest.logger.info("reindex pg_class indexes")
assert PSQL.run_sql_file(local_path('reindex_pg_class.sql'),
dbname=self.dbname)
tinctest.logger.info("shutdown immediate")
cmd = Command("shutdown immediate", "gpstop -ai")
cmd.run(validateAfter=True)
tinctest.logger.info(cmd.get_results().printResult())
tinctest.logger.info("trigger recovery")
cmd = Command("restart the cluster", "gpstart -a")
cmd.run(validateAfter=True)
tinctest.logger.info(cmd.get_results().printResult())
tinctest.logger.info("validate recovery succeeded")
results = {'rc': 0, 'stdout': '', 'stderr': ''}
PSQL.run_sql_command("DROP TABLE reindex_pg_class_test",
results=results,
dbname=self.dbname)
assert results['rc'] == 0, results['stderr']
def run_gpstate(self, type, phase):
tinctest.logger.info("running gpstate")
gphome = os.environ.get('GPHOME')
masterdd = os.environ.get('MASTER_DATA_DIRECTORY')
res = {'rc':0, 'stdout':'', 'stderr':''}
cmd = "gpstate --printSampleExternalTableSql -q -d %s" % (masterdd)
ok = run_shell_command(cmd, results=res)
if res['rc']>0:
raise Exception("Failed to query gpstate --printSampleExternalTableSql");
out = PSQL.run_sql_command(res['stdout'])
if out.find('CREATE EXTERNAL TABLE') == -1 :
raise Exception("Failed to create the external table gpstate_segment_status")
gpstate_outfile = local_path("gpstate_out")
cmd = "gpstate -s -a > %s 2>&1" % (gpstate_outfile)
ok = run_shell_command(cmd)
self.check_gpstate(type, phase)
return ok
def read_config(self, param):
"""Function to read configuration file and get parameters needed for test"""
configParser = ConfigParser.RawConfigParser()
configFilePath = local_path(
"../gpdb/tests/utilities/commandcenter/configs/config.txt")
configParser.read(configFilePath)
if (param == 'metadata'):
tinctest.logger.info('Fetching metdata')
instance_name = configParser.get('valid', 'instance_name')
machine = configParser.get('metadata', 'machine')
pgport = configParser.get('metadata', 'pgport')
port = configParser.get('metadata', 'port')
mdd = configParser.get('metadata', 'mdd')
source = configParser.get('metadata', 'source')
cc_source = configParser.get('metadata', 'cc_source')
return (machine, pgport, port, mdd, source, instance_name,
cc_source)
else:
tinctest.logger.info('Fetching username and password')
username = configParser.get(param, 'username')
password = configParser.get(param, 'password')
return (username, password)
def test_ic_fc_kill_recv_process(self):
if (self.cluster_platform.lower().find('red hat enterprise linux server') < 0):
self.skipTest('Test only applies to RHEL platform.')
cmd_str = "psql -a -f " + local_path(self.faultTolerance_sql + str(self._testMethodName) + '.sql') + " &> output_killrecv &"
out = runShellCommand(cmd_str)
# sleep 1 sec to ensure that some query related to interconnect is running when killing receiver process
sleep(1)
(hostname, processID) = self.getHostProcess("receiver")
if(processID == ""):
self.skipTest("No sender process found, skip this case")
self.killHostProcess(hostname, processID)
# sleep 30 sec to ensure that sql return some reuslts
sleep(50)
# Check the output file to see whether the expected error info is returned.
out = self.checkOutputPattern("output", "server closed the connection unexpectedly")
self.assertTrue(out)
def login(self, param):
""" Function to login to the CC to execute other tests """
tinctest.logger.info("-------------------------------")
tinctest.logger.info('Login operation')
tinctest.logger.info("-------------------------------")
(machine, pgport, port, mdd, source, instance_name,
cc_source) = self.read_config('metadata')
host = machine + ":" + port
h = httplib.HTTPConnection(host)
url = "http://" + machine + ":" + port + "/"
tinctest.logger.info('Fetching username and password from config file')
(username, password) = self.read_config(param)
tinctest.logger.info('In function login')
tinctest.logger.debug('Establising connection')
data = {'username': username, 'password': password}
encoded_data = urllib.urlencode(data)
tinctest.logger.debug('Opening url' + url)
request = urllib2.Request(url + "logon", None)
rs = urllib2.urlopen(request, encoded_data)
h.request('POST', "/logon", encoded_data)
login_msg = rs.read()
# write the xml returned to a file for further processing
login_status_xml = local_path(
"../gpdb/tests/utilities/commandcenter/data/login_status")
f = open(login_status_xml, 'w')
f.write(login_msg)
f.close()
# Call the get_sessionid function to get the session_id
session_id = self.get_sessionid(login_status_xml)
return session_id
def setUpClass(cls):
super(EtablefuncGppcTestCase, cls).setUpClass()
"""
compile tablefunc_gppc_demo.c and install the tablefunc_gppc_demo.so
"""
gppkg = Gppkg()
gpccutil.pre_process(product_version)
result = gppkg.gppkg_install(product_version, 'libgppc')
#makeLog = loal_path('test00MakeLog.out')
if result:
cmdMakeInstall = 'cd ' + local_path(
'data'
) + ' && make clean && make CPPFLAGS=-D_GNU_SOURCE && make install'
res = {'rc': 0, 'stderr': '', 'stdout': ''}
run_shell_command(cmdMakeInstall, 'compile tablefunc_gppc_demo.c',
res)
# Current make file works for linux, but not for Solaris or OSX.
# If compilation fails or installation fails, force system quit: os._exit(1)
if res['rc']:
os._exit(
1
) # This will exit the test including the next test suites
sharedObj = '%s/tabfunc_gppc_demo.so' % (LIBDIR)
if not os.path.isfile(sharedObj):
os._exit(1)
# For multinode cluster, need to copy shared object tabfunc_gppc_demo.so to all primary segments
if gpdbconfig.is_multinode():
res = {'rc': 0, 'stderr': '', 'stdout': ''}
hosts = gpdbconfig.get_hosts(segments=True)
scp_cmd = 'gpscp -h ' + ' -h '.join(map(
str, hosts)) + ' ' + sharedObj + ' =:%s' % LIBDIR
run_shell_command(scp_cmd)
if res['rc']:
raise Excpetion(
'Could not copy shared object to primary segment')
def setUpClass(cls):
hadoop_type = "cdh"
# Url of the CDH distribution
hadoop_artifact_url = os.getenv("HADOOP_ARTIFACT_URL")
if not hadoop_artifact_url:
raise HadoopIntegrationException(
"HADOOP_ARTIFACT_URL is not set!!")
# hadoop installation directory; where CDH tar ball is untared
hadoop_install_dir = os.getenv("HADOOP_INSTALL_PATH", "./installation")
# path of the hadoop data directories and log directories
hadoop_data_dir = os.getenv("HADOOP_DATA_DIR", "/data")
template_conf_dir = local_path("./configs")
secure_hadoop = os.getenv("ENABLE_SECURITY", True)
# provide list of nodes if cluster is multinode - would be used in future, when multinode is enabled
node_list = os.getenv("NODES_LIST", [])
gphdfs_connector = os.getenv("GPHDFS_CONNECTOR", "cdh4.1-gnet-1.2.0.0")
cls.integration = HadoopIntegration(hadoop_type, gphdfs_connector,
hadoop_artifact_url,
hadoop_install_dir,
hadoop_data_dir, template_conf_dir,
secure_hadoop, node_list)
cls.integration.integrate()
cls.test_failures = False
super(CDHIntegrationTest, cls).setUpClass()
