def loadXML(self, case_name):
xml_dir = self.get_xml_dir()
sql_output_dir = os.path.join(self.get_source_dir(), 'output/sql')
TINCSystem.make_dirs(sql_output_dir, ignore_exists_error=True)
# copy init_file from xml dir
init_file_src = os.path.join(xml_dir, 'init_file')
init_file_dst = os.path.join(sql_output_dir, 'init_file')
shutil.copyfile(init_file_src, init_file_dst)
filename = case_name + '.xml'
xml_file = os.path.join(xml_dir, filename)
doc = minidom.parse(xml_file)
case_name = os.path.splitext(filename)[0]
# extract sql and put them as files
sql_elem = doc.getElementsByTagName('sql')
assert(len(sql_elem) == 1)
sql_elem = sql_elem[0]
self.write_file(sql_elem, sql_output_dir, case_name, 'pre')
self.write_file(sql_elem, sql_output_dir, case_name, 'trigger')
self.write_file(sql_elem, sql_output_dir, case_name, 'post')
ans_elem = doc.getElementsByTagName('ans')
assert(len(ans_elem) == 1)
ans_elem = ans_elem[0]
self.write_file(ans_elem, sql_output_dir, case_name, 'pre')
self.write_file(ans_elem, sql_output_dir, case_name, 'trigger')
self.write_file(ans_elem, sql_output_dir, case_name, 'post')
def test_make_dirs(self):
test_dir = os.path.join(os.path.dirname(__file__), 'test_mkdirs')
if os.path.exists(test_dir):
shutil.rmtree(test_dir)
self.assertFalse(os.path.exists(test_dir))
TINCSystem.make_dirs(test_dir)
self.assertTrue(os.path.exists(test_dir))
def test_make_dirs(self):
test_dir = os.path.join(os.path.dirname(__file__), 'test_mkdirs')
if os.path.exists(test_dir):
shutil.rmtree(test_dir)
self.assertFalse(os.path.exists(test_dir))
TINCSystem.make_dirs(test_dir)
self.assertTrue(os.path.exists(test_dir))
def setUpClass(cls):
"""
setUpClass of MPPTestCase does the following:
-> Create out directory for the class if it does not exist.
This is thread safe in case an MPPTestCase is used concurrently
within a ScenarioTestCase or ConcurrencyTestCase
-> Configures the database specified at the class level variable 'db_name'
"""
tinctest.logger.trace_in()
#QAINF-760 - we need to treat db_name in the class level doc string as a class level variable
#rather than an instance level variable
ds = cls.__doc__
if ds:
lines = ds.splitlines()
for line in lines:
line = line.strip()
if line.find('@db_name') != 0:
continue
line = line[1:]
if len(line.split()) <= 1:
break
(key, cls.db_name) = line.split(' ', 1)
break
super(MPPTestCase, cls).setUpClass()
if not os.path.exists(cls.get_out_dir()):
TINCSystem.make_dirs(cls.get_out_dir(), ignore_exists_error = True)
if cls.db_name:
tinctest.logger.debug("Configure database %s from MPPTestCase setUpClass." % cls.db_name)
cls.configure_database(cls.db_name)
tinctest.logger.trace_out()
def loadXML(self, case_name):
xml_dir = self.get_xml_dir()
sql_output_dir = os.path.join(self.get_source_dir(), 'output/sql')
TINCSystem.make_dirs(sql_output_dir, ignore_exists_error=True)
# copy init_file from xml dir
init_file_src = os.path.join(xml_dir, 'init_file')
init_file_dst = os.path.join(sql_output_dir, 'init_file')
shutil.copyfile(init_file_src, init_file_dst)
filename = case_name + '.xml'
xml_file = os.path.join(xml_dir, filename)
doc = minidom.parse(xml_file)
case_name = os.path.splitext(filename)[0]
# extract sql and put them as files
sql_elem = doc.getElementsByTagName('sql')
assert (len(sql_elem) == 1)
sql_elem = sql_elem[0]
self.write_file(sql_elem, sql_output_dir, case_name, 'pre')
self.write_file(sql_elem, sql_output_dir, case_name, 'trigger')
self.write_file(sql_elem, sql_output_dir, case_name, 'post')
ans_elem = doc.getElementsByTagName('ans')
assert (len(ans_elem) == 1)
ans_elem = ans_elem[0]
self.write_file(ans_elem, sql_output_dir, case_name, 'pre')
self.write_file(ans_elem, sql_output_dir, case_name, 'trigger')
self.write_file(ans_elem, sql_output_dir, case_name, 'post')
def setUpClass(cls):
"""
Some unit tests here call's test_case.run() method which does not set up the out dir required (out_dir set up gets
handled in MPPTestCase.setUpClass which will be invoked only when running through a test_suite.run().
Therefore, we setup the out dir required here in the test itself.
"""
TINCSystem.make_dirs(local_path('output'), ignore_exists_error=True)
def setUpClass(cls):
"""
Some unit tests here call's test_case.run() method which does not set up the out dir required (out_dir set up gets
handled in MPPTestCase.setUpClass which will be invoked only when running through a test_suite.run().
Therefore, we setup the out dir required here in the test itself.
"""
TINCSystem.make_dirs(local_path('output'), ignore_exists_error=True)
def _do_gpinitsystem(self):
# Takes care of initializing a cluster according to the test configuration
TINCSystem.make_dirs(self.testcase_out_dir, force_create=True)
TINCSystem.make_dirs(self.testcase_master_dir, force_create=True)
segment_host_file = '/tmp/segment_hosts'
with open(segment_host_file, 'w') as f:
for host in self.hosts:
f.write(host)
f.write('\n')
# Create primary dirs
res = {'rc': 0, 'stdout': '', 'stderr': ''}
run_shell_command(
"gpssh -f %s -e 'rm -rf %s; mkdir -p %s'" %
(segment_host_file, self.testcase_primary_dir,
self.testcase_primary_dir), 'create segment dirs', res)
if res['rc'] > 0:
raise GpExpandTestCaseException(
"Failed to create segment directories")
# Create mirror dirs
res = {'rc': 0, 'stdout': '', 'stderr': ''}
run_shell_command(
"gpssh -f %s -e 'rm -rf %s; mkdir -p %s'" %
(segment_host_file, self.testcase_mirror_dir,
self.testcase_mirror_dir), 'create segment dirs', res)
if res['rc'] > 0:
raise GpExpandTestCaseException(
"Failed to create segment directories")
# Create filespace dirs
res = {'rc': 0, 'stdout': '', 'stderr': ''}
run_shell_command(
"gpssh -f %s -e 'rm -rf %s; mkdir -p %s'" %
(segment_host_file, self.testcase_filespace_dir,
self.testcase_filespace_dir), 'create segment dirs', res)
if res['rc'] > 0:
raise GpExpandTestCaseException(
"Failed to create segment directories")
# Generate the config files to initialize the cluster
self._generate_gpinit_config_files()
self.assertTrue(os.path.exists(self.testcase_init_file))
self.assertTrue(os.path.exists(self.testcase_hosts_file))
# TODO: Do some clean up before init
# Init the system
# Still have to figure out if it is a good idea to init for every test or not
# Pro: easier to develop self contained tests Con: time taken for every test
result = GpInitSystem(self.testcase_init_file).run(validate=False)
# initsystem returns 1 for warnings and 2 for errors
if result.rc > 1:
tinctest.logger.error("Failed initializing the cluster: %s" %
result)
raise GPExpandTestCaseException(
"Failed initializing the cluster. Look into gpAdminLogs for more information"
)
def get_file_names(self, suffix):
stem = self.method_name.replace('test_', '')
sql_file_root = os.path.join(self.get_sql_dir(), stem + '.sql')
sql_file = sql_file_root + '.' + suffix
ans_file = sql_file.replace('.sql.', '.ans.')
out_file_name = os.path.basename(sql_file).replace('.sql.', '.out.')
out_file = os.path.join(self.get_out_dir(), out_file_name)
TINCSystem.make_dirs(self.get_out_dir(), ignore_exists_error=True)
return (sql_file, ans_file, out_file)
def test_make_dirs_existing_ignore(self):
test_dir = os.path.join(os.path.dirname(__file__), 'test_mkdirs')
if os.path.exists(test_dir):
shutil.rmtree(test_dir)
self.assertFalse(os.path.exists(test_dir))
TINCSystem.make_dirs(test_dir)
self.assertTrue(os.path.exists(test_dir))
# This should not fail and ignore existing error
TINCSystem.make_dirs(test_dir, ignore_exists_error=True)
def test_make_dirs_existing_ignore(self):
test_dir = os.path.join(os.path.dirname(__file__), 'test_mkdirs')
if os.path.exists(test_dir):
shutil.rmtree(test_dir)
self.assertFalse(os.path.exists(test_dir))
TINCSystem.make_dirs(test_dir)
self.assertTrue(os.path.exists(test_dir))
# This should not fail and ignore existing error
TINCSystem.make_dirs(test_dir, ignore_exists_error = True)
def test_make_dirs_existing(self):
test_dir = os.path.join(os.path.dirname(__file__), 'test_mkdirs')
if os.path.exists(test_dir):
shutil.rmtree(test_dir)
self.assertFalse(os.path.exists(test_dir))
TINCSystem.make_dirs(test_dir)
self.assertTrue(os.path.exists(test_dir))
# This should fail
with self.assertRaises(OSError) as cm:
TINCSystem.make_dirs(test_dir)
def test_make_dirs_existing(self):
test_dir = os.path.join(os.path.dirname(__file__), 'test_mkdirs')
if os.path.exists(test_dir):
shutil.rmtree(test_dir)
self.assertFalse(os.path.exists(test_dir))
TINCSystem.make_dirs(test_dir)
self.assertTrue(os.path.exists(test_dir))
# This should fail
with self.assertRaises(OSError) as cm:
TINCSystem.make_dirs(test_dir)
def test_force_create_make_dirs(self):
test_dir = os.path.join(os.path.dirname(__file__), 'test_force')
if os.path.exists(test_dir):
shutil.rmtree(test_dir)
self.assertFalse(os.path.exists(test_dir))
TINCSystem.make_dirs(test_dir)
self.assertTrue(os.path.exists(test_dir))
test_file = os.path.join(test_dir, 'testfile')
with open(test_file, 'w') as f:
f.write('test')
self.assertTrue(os.path.exists(test_file))
# Force create
TINCSystem.make_dirs(test_dir, force_create = True)
self.assertFalse(os.path.exists(test_file))
def test_force_create_make_dirs(self):
test_dir = os.path.join(os.path.dirname(__file__), 'test_force')
if os.path.exists(test_dir):
shutil.rmtree(test_dir)
self.assertFalse(os.path.exists(test_dir))
TINCSystem.make_dirs(test_dir)
self.assertTrue(os.path.exists(test_dir))
test_file = os.path.join(test_dir, 'testfile')
with open(test_file, 'w') as f:
f.write('test')
self.assertTrue(os.path.exists(test_file))
# Force create
TINCSystem.make_dirs(test_dir, force_create=True)
self.assertFalse(os.path.exists(test_file))
def split_tbl(cls):
"""
Split data files using split command for the testing
where gpfdist serves multiple files.
"""
source_dir = cls.get_source_dir()
data_dir = os.path.join(source_dir, 'data')
for fn in os.listdir(data_dir):
if not fn.endswith('.tbl'):
continue
split_dir = os.path.join(data_dir, fn + '-dir')
if os.path.exists(split_dir):
continue
TINCSystem.make_dirs(split_dir, ignore_exists_error=True)
os.system('cd {0}; split ../{1}'.format(split_dir, fn))
def split_tbl(cls):
"""
Split data files using split command for the testing
where gpfdist serves multiple files.
"""
source_dir = cls.get_source_dir()
data_dir = os.path.join(source_dir, 'data')
for fn in os.listdir(data_dir):
if not fn.endswith('.tbl'):
continue
split_dir = os.path.join(data_dir, fn + '-dir')
if os.path.exists(split_dir):
continue
TINCSystem.make_dirs(split_dir, ignore_exists_error=True)
os.system('cd {0}; split ../{1}'.format(split_dir, fn))
def _do_gpinitsystem(self):
# Takes care of initializing a cluster according to the test configuration
TINCSystem.make_dirs(self.testcase_out_dir, force_create=True)
TINCSystem.make_dirs(self.testcase_master_dir, force_create=True)
segment_host_file = '/tmp/segment_hosts'
with open(segment_host_file, 'w') as f:
for host in self.hosts:
f.write(host)
f.write('\n')
# Create primary dirs
res = {'rc': 0, 'stdout' : '', 'stderr': ''}
run_shell_command("gpssh -f %s -e 'rm -rf %s; mkdir -p %s'" %(segment_host_file, self.testcase_primary_dir, self.testcase_primary_dir), 'create segment dirs', res)
if res['rc'] > 0:
raise GpExpandTestCaseException("Failed to create segment directories")
# Create mirror dirs
res = {'rc': 0, 'stdout' : '', 'stderr': ''}
run_shell_command("gpssh -f %s -e 'rm -rf %s; mkdir -p %s'" %(segment_host_file, self.testcase_mirror_dir, self.testcase_mirror_dir), 'create segment dirs', res)
if res['rc'] > 0:
raise GpExpandTestCaseException("Failed to create segment directories")
# Create filespace dirs
res = {'rc': 0, 'stdout' : '', 'stderr': ''}
run_shell_command("gpssh -f %s -e 'rm -rf %s; mkdir -p %s'" %(segment_host_file, self.testcase_filespace_dir, self.testcase_filespace_dir), 'create segment dirs', res)
if res['rc'] > 0:
raise GpExpandTestCaseException("Failed to create segment directories")
# Generate the config files to initialize the cluster
self._generate_gpinit_config_files()
self.assertTrue(os.path.exists(self.testcase_init_file))
self.assertTrue(os.path.exists(self.testcase_hosts_file))
# TODO: Do some clean up before init
# Init the system
# Still have to figure out if it is a good idea to init for every test or not
# Pro: easier to develop self contained tests Con: time taken for every test
result = GpInitSystem(self.testcase_init_file).run(validate=False)
# initsystem returns 1 for warnings and 2 for errors
if result.rc > 1:
tinctest.logger.error("Failed initializing the cluster: %s" %result)
raise GPExpandTestCaseException("Failed initializing the cluster. Look into gpAdminLogs for more information")
def setUpClass(cls):
"""
setUpClass of MPPTestCase does the following:
-> Create out directory for the class if it does not exist.
This is thread safe in case an MPPTestCase is used concurrently
within a ScenarioTestCase or ConcurrencyTestCase
-> Configures the database specified at the class level variable 'db_name'
"""
tinctest.logger.trace_in()
#QAINF-760 - we need to treat db_name in the class level doc string as a class level variable
#rather than an instance level variable
ds = cls.__doc__
if ds:
lines = ds.splitlines()
for line in lines:
line = line.strip()
if line.find('@db_name') != 0:
continue
line = line[1:]
if len(line.split()) <= 1:
break
(key, cls.db_name) = line.split(' ', 1)
break
super(MPPTestCase, cls).setUpClass()
if not os.path.exists(cls.get_out_dir()):
TINCSystem.make_dirs(cls.get_out_dir(), ignore_exists_error=True)
if cls.db_name:
tinctest.logger.debug(
"Configure database %s from MPPTestCase setUpClass." %
cls.db_name)
cls.configure_database(cls.db_name)
tinctest.logger.trace_out()
def test_make_dirs_relative_path(self):
test_dir = 'test/output'
with self.assertRaises(TINCSystemException) as cm:
TINCSystem.make_dirs(test_dir)
def setUpClass(cls):
TINCSystem.make_dirs(cls.get_data_dir(), ignore_exists_error=True)
super(ErrorLogConcurrencyTests, cls).setUpClass()
def test_make_dirs_relative_path(self):
test_dir = 'test/output'
with self.assertRaises(TINCSystemException) as cm:
TINCSystem.make_dirs(test_dir)
def setUpClass(cls):
TINCSystem.make_dirs(cls.get_data_dir(), ignore_exists_error=True)
super(ErrTblTestCase, cls).setUpClass()
def setUpClass(cls):
TINCSystem.make_dirs(cls.get_data_dir(), ignore_exists_error=True)
super(ErrorLogConcurrencyTests, cls).setUpClass()
def setUpClass(cls):
super(TableDistrubtionPolicySnapshot, cls).setUpClass()
TINCSystem.make_dirs(cls.get_ans_dir(), force_create=True)
def setUpClass(cls):
super(TableDistrubtionPolicySnapshot, cls).setUpClass()
TINCSystem.make_dirs(cls.get_ans_dir(), force_create=True)
def setUpClass(cls):
TINCSystem.make_dirs(cls.get_data_dir(), ignore_exists_error=True)
super(ErrTblTestCase, cls).setUpClass()
