def test_make_dirs(self): test_dir = os.path.join(os.path.dirname(__file__), 'test_mkdirs') if os.path.exists(test_dir): shutil.rmtree(test_dir) self.assertFalse(os.path.exists(test_dir)) TINCSystem.make_dirs(test_dir) self.assertTrue(os.path.exists(test_dir))
def generate_config_file(self, greenplum_path_file, pgport, cc_install_dir, port, mdd): configFile_template = local_path( "../gpdb/tests/utilities/commandcenter/configs/config_template.txt" ) configFile = local_path( "../gpdb/tests/utilities/commandcenter/configs/config.txt") cc_source = cc_install_dir + "/gpcc_path.sh" transforms = { '%pgport%': pgport, '%source%': greenplum_path_file, '%instance_name%': cc_install_dir, '%port%': port, '%mdd%': mdd, '%cc_source%': cc_source } cmd_str = 'chmod 777 %s; chmod 777 %s;' % (configFile_template, configFile) tinctest.logger.info(cmd_str) cmd = Command(name='chmod', cmdStr=cmd_str, ctxt=REMOTE, remoteHost='localhost') cmd.run() TINCSystem.substitute_strings_in_file(configFile_template, configFile, transforms)
def setUpClass(cls): """ Some unit tests here call's test_case.run() method which does not set up the out dir required (out_dir set up gets handled in MPPTestCase.setUpClass which will be invoked only when running through a test_suite.run(). Therefore, we setup the out dir required here in the test itself. """ TINCSystem.make_dirs(local_path('output'), ignore_exists_error=True)
def loadXML(self, case_name): xml_dir = self.get_xml_dir() sql_output_dir = os.path.join(self.get_source_dir(), 'output/sql') TINCSystem.make_dirs(sql_output_dir, ignore_exists_error=True) # copy init_file from xml dir init_file_src = os.path.join(xml_dir, 'init_file') init_file_dst = os.path.join(sql_output_dir, 'init_file') shutil.copyfile(init_file_src, init_file_dst) filename = case_name + '.xml' xml_file = os.path.join(xml_dir, filename) doc = minidom.parse(xml_file) case_name = os.path.splitext(filename)[0] # extract sql and put them as files sql_elem = doc.getElementsByTagName('sql') assert(len(sql_elem) == 1) sql_elem = sql_elem[0] self.write_file(sql_elem, sql_output_dir, case_name, 'pre') self.write_file(sql_elem, sql_output_dir, case_name, 'trigger') self.write_file(sql_elem, sql_output_dir, case_name, 'post') ans_elem = doc.getElementsByTagName('ans') assert(len(ans_elem) == 1) ans_elem = ans_elem[0] self.write_file(ans_elem, sql_output_dir, case_name, 'pre') self.write_file(ans_elem, sql_output_dir, case_name, 'trigger') self.write_file(ans_elem, sql_output_dir, case_name, 'post')
def _generate_gpinit_config_files(self): transforms = { '%SEG_PREFIX%': self.seg_prefix, '%PORT_BASE%': self.port_base, '%MASTER_HOST%': self.master_host, # First item in self.hosts '%HOSTFILE%': self.host_file, '%MASTER_PORT%': self.master_port, '%MASTER_DATA_DIR%': os.path.dirname(self.mdd), '%DATA_DIR%': (os.path.dirname(self.primary_data_dir) + ' ') * self.number_of_segments_per_host } # First generate host file based on number_of_hosts with open(self.host_file, 'w') as f: for host in self.hosts: f.write(host + '\n') TINCSystem.substitute_strings_in_file(self.gpinitconfig_template, self.gpinitconfig_file, transforms)
def setUpClass(cls): """ setUpClass of MPPTestCase does the following: -> Create out directory for the class if it does not exist. This is thread safe in case an MPPTestCase is used concurrently within a ScenarioTestCase or ConcurrencyTestCase -> Configures the database specified at the class level variable 'db_name' """ tinctest.logger.trace_in() #QAINF-760 - we need to treat db_name in the class level doc string as a class level variable #rather than an instance level variable ds = cls.__doc__ if ds: lines = ds.splitlines() for line in lines: line = line.strip() if line.find('@db_name') != 0: continue line = line[1:] if len(line.split()) <= 1: break (key, cls.db_name) = line.split(' ', 1) break super(MPPTestCase, cls).setUpClass() if not os.path.exists(cls.get_out_dir()): TINCSystem.make_dirs(cls.get_out_dir(), ignore_exists_error = True) if cls.db_name: tinctest.logger.debug("Configure database %s from MPPTestCase setUpClass." % cls.db_name) cls.configure_database(cls.db_name) tinctest.logger.trace_out()
def _generate_gpinit_config_files(self): transforms = {'%PORT_BASE%': self.port_base, '%MASTER_HOST%': self.hosts[0], # First item in self.hosts '%HOSTFILE%': self.testcase_hosts_file, '%MASTER_PORT%': self.master_port, '%MASTER_DATA_DIR%': self.testcase_master_dir, '%DATA_DIR%': (self.testcase_primary_dir + ' ') * self.number_of_segments } if self.mirror_enabled: transforms['%MIRROR_DIR%'] = (self.testcase_mirror_dir + ' ') * self.number_of_segments transforms['%MIRROR_PORT_BASE%'] = self.mirror_port_base transforms['%REP_PORT_BASE%'] = self.rep_port_base transforms['%MIRROR_REP_PORT_BASE%'] = self.mirror_rep_port_base # First generate host file based on number_of_hosts with open(self.testcase_hosts_file, 'w') as f: for host in self.hosts[1:self.number_of_hosts+1]: f.write(host + '\n') if self.mirror_enabled: TINCSystem.substitute_strings_in_file(self.gpinitconfig_mirror_template, self.testcase_init_file, transforms) else: TINCSystem.substitute_strings_in_file(self.gpinitconfig_template, self.testcase_init_file, transforms)
def loadXML(self, case_name): xml_dir = self.get_xml_dir() sql_output_dir = os.path.join(self.get_source_dir(), 'output/sql') TINCSystem.make_dirs(sql_output_dir, ignore_exists_error=True) # copy init_file from xml dir init_file_src = os.path.join(xml_dir, 'init_file') init_file_dst = os.path.join(sql_output_dir, 'init_file') shutil.copyfile(init_file_src, init_file_dst) filename = case_name + '.xml' xml_file = os.path.join(xml_dir, filename) doc = minidom.parse(xml_file) case_name = os.path.splitext(filename)[0] # extract sql and put them as files sql_elem = doc.getElementsByTagName('sql') assert (len(sql_elem) == 1) sql_elem = sql_elem[0] self.write_file(sql_elem, sql_output_dir, case_name, 'pre') self.write_file(sql_elem, sql_output_dir, case_name, 'trigger') self.write_file(sql_elem, sql_output_dir, case_name, 'post') ans_elem = doc.getElementsByTagName('ans') assert (len(ans_elem) == 1) ans_elem = ans_elem[0] self.write_file(ans_elem, sql_output_dir, case_name, 'pre') self.write_file(ans_elem, sql_output_dir, case_name, 'trigger') self.write_file(ans_elem, sql_output_dir, case_name, 'post')
def _do_gpinitsystem(self): # Takes care of initializing a cluster according to the test configuration TINCSystem.make_dirs(self.testcase_out_dir, force_create=True) TINCSystem.make_dirs(self.testcase_master_dir, force_create=True) segment_host_file = '/tmp/segment_hosts' with open(segment_host_file, 'w') as f: for host in self.hosts: f.write(host) f.write('\n') # Create primary dirs res = {'rc': 0, 'stdout': '', 'stderr': ''} run_shell_command( "gpssh -f %s -e 'rm -rf %s; mkdir -p %s'" % (segment_host_file, self.testcase_primary_dir, self.testcase_primary_dir), 'create segment dirs', res) if res['rc'] > 0: raise GpExpandTestCaseException( "Failed to create segment directories") # Create mirror dirs res = {'rc': 0, 'stdout': '', 'stderr': ''} run_shell_command( "gpssh -f %s -e 'rm -rf %s; mkdir -p %s'" % (segment_host_file, self.testcase_mirror_dir, self.testcase_mirror_dir), 'create segment dirs', res) if res['rc'] > 0: raise GpExpandTestCaseException( "Failed to create segment directories") # Create filespace dirs res = {'rc': 0, 'stdout': '', 'stderr': ''} run_shell_command( "gpssh -f %s -e 'rm -rf %s; mkdir -p %s'" % (segment_host_file, self.testcase_filespace_dir, self.testcase_filespace_dir), 'create segment dirs', res) if res['rc'] > 0: raise GpExpandTestCaseException( "Failed to create segment directories") # Generate the config files to initialize the cluster self._generate_gpinit_config_files() self.assertTrue(os.path.exists(self.testcase_init_file)) self.assertTrue(os.path.exists(self.testcase_hosts_file)) # TODO: Do some clean up before init # Init the system # Still have to figure out if it is a good idea to init for every test or not # Pro: easier to develop self contained tests Con: time taken for every test result = GpInitSystem(self.testcase_init_file).run(validate=False) # initsystem returns 1 for warnings and 2 for errors if result.rc > 1: tinctest.logger.error("Failed initializing the cluster: %s" % result) raise GPExpandTestCaseException( "Failed initializing the cluster. Look into gpAdminLogs for more information" )
def get_file_names(self, suffix): stem = self.method_name.replace('test_', '') sql_file_root = os.path.join(self.get_sql_dir(), stem + '.sql') sql_file = sql_file_root + '.' + suffix ans_file = sql_file.replace('.sql.', '.ans.') out_file_name = os.path.basename(sql_file).replace('.sql.', '.out.') out_file = os.path.join(self.get_out_dir(), out_file_name) TINCSystem.make_dirs(self.get_out_dir(), ignore_exists_error=True) return (sql_file, ans_file, out_file)
def test_make_dirs_existing_ignore(self): test_dir = os.path.join(os.path.dirname(__file__), 'test_mkdirs') if os.path.exists(test_dir): shutil.rmtree(test_dir) self.assertFalse(os.path.exists(test_dir)) TINCSystem.make_dirs(test_dir) self.assertTrue(os.path.exists(test_dir)) # This should not fail and ignore existing error TINCSystem.make_dirs(test_dir, ignore_exists_error=True)
def test_make_dirs_existing_ignore(self): test_dir = os.path.join(os.path.dirname(__file__), 'test_mkdirs') if os.path.exists(test_dir): shutil.rmtree(test_dir) self.assertFalse(os.path.exists(test_dir)) TINCSystem.make_dirs(test_dir) self.assertTrue(os.path.exists(test_dir)) # This should not fail and ignore existing error TINCSystem.make_dirs(test_dir, ignore_exists_error = True)
def test_make_dirs_existing(self): test_dir = os.path.join(os.path.dirname(__file__), 'test_mkdirs') if os.path.exists(test_dir): shutil.rmtree(test_dir) self.assertFalse(os.path.exists(test_dir)) TINCSystem.make_dirs(test_dir) self.assertTrue(os.path.exists(test_dir)) # This should fail with self.assertRaises(OSError) as cm: TINCSystem.make_dirs(test_dir)
def test_force_create_make_dirs(self): test_dir = os.path.join(os.path.dirname(__file__), 'test_force') if os.path.exists(test_dir): shutil.rmtree(test_dir) self.assertFalse(os.path.exists(test_dir)) TINCSystem.make_dirs(test_dir) self.assertTrue(os.path.exists(test_dir)) test_file = os.path.join(test_dir, 'testfile') with open(test_file, 'w') as f: f.write('test') self.assertTrue(os.path.exists(test_file)) # Force create TINCSystem.make_dirs(test_dir, force_create = True) self.assertFalse(os.path.exists(test_file))
def test_force_create_make_dirs(self): test_dir = os.path.join(os.path.dirname(__file__), 'test_force') if os.path.exists(test_dir): shutil.rmtree(test_dir) self.assertFalse(os.path.exists(test_dir)) TINCSystem.make_dirs(test_dir) self.assertTrue(os.path.exists(test_dir)) test_file = os.path.join(test_dir, 'testfile') with open(test_file, 'w') as f: f.write('test') self.assertTrue(os.path.exists(test_file)) # Force create TINCSystem.make_dirs(test_dir, force_create=True) self.assertFalse(os.path.exists(test_file))
def split_tbl(cls): """ Split data files using split command for the testing where gpfdist serves multiple files. """ source_dir = cls.get_source_dir() data_dir = os.path.join(source_dir, 'data') for fn in os.listdir(data_dir): if not fn.endswith('.tbl'): continue split_dir = os.path.join(data_dir, fn + '-dir') if os.path.exists(split_dir): continue TINCSystem.make_dirs(split_dir, ignore_exists_error=True) os.system('cd {0}; split ../{1}'.format(split_dir, fn))
def _generate_gpinit_config_files(self): transforms = {'%SEG_PREFIX%': self.seg_prefix, '%PORT_BASE%': self.port_base, '%MASTER_HOST%': self.master_host, # First item in self.hosts '%HOSTFILE%': self.host_file, '%MASTER_PORT%': self.master_port, '%MASTER_DATA_DIR%': os.path.dirname(self.mdd), '%DATA_DIR%': (os.path.dirname(self.primary_data_dir) + ' ') * self.number_of_segments_per_host } # First generate host file based on number_of_hosts with open(self.host_file, 'w') as f: for host in self.hosts: f.write(host + '\n') TINCSystem.substitute_strings_in_file(self.gpinitconfig_template, self.gpinitconfig_file, transforms)
def _do_gpinitsystem(self): # Takes care of initializing a cluster according to the test configuration TINCSystem.make_dirs(self.testcase_out_dir, force_create=True) TINCSystem.make_dirs(self.testcase_master_dir, force_create=True) segment_host_file = '/tmp/segment_hosts' with open(segment_host_file, 'w') as f: for host in self.hosts: f.write(host) f.write('\n') # Create primary dirs res = {'rc': 0, 'stdout' : '', 'stderr': ''} run_shell_command("gpssh -f %s -e 'rm -rf %s; mkdir -p %s'" %(segment_host_file, self.testcase_primary_dir, self.testcase_primary_dir), 'create segment dirs', res) if res['rc'] > 0: raise GpExpandTestCaseException("Failed to create segment directories") # Create mirror dirs res = {'rc': 0, 'stdout' : '', 'stderr': ''} run_shell_command("gpssh -f %s -e 'rm -rf %s; mkdir -p %s'" %(segment_host_file, self.testcase_mirror_dir, self.testcase_mirror_dir), 'create segment dirs', res) if res['rc'] > 0: raise GpExpandTestCaseException("Failed to create segment directories") # Create filespace dirs res = {'rc': 0, 'stdout' : '', 'stderr': ''} run_shell_command("gpssh -f %s -e 'rm -rf %s; mkdir -p %s'" %(segment_host_file, self.testcase_filespace_dir, self.testcase_filespace_dir), 'create segment dirs', res) if res['rc'] > 0: raise GpExpandTestCaseException("Failed to create segment directories") # Generate the config files to initialize the cluster self._generate_gpinit_config_files() self.assertTrue(os.path.exists(self.testcase_init_file)) self.assertTrue(os.path.exists(self.testcase_hosts_file)) # TODO: Do some clean up before init # Init the system # Still have to figure out if it is a good idea to init for every test or not # Pro: easier to develop self contained tests Con: time taken for every test result = GpInitSystem(self.testcase_init_file).run(validate=False) # initsystem returns 1 for warnings and 2 for errors if result.rc > 1: tinctest.logger.error("Failed initializing the cluster: %s" %result) raise GPExpandTestCaseException("Failed initializing the cluster. Look into gpAdminLogs for more information")
def generate_config_file(self,greenplum_path_file,pgport,cc_install_dir,port,mdd): configFile_template = local_path("../gpdb/tests/utilities/commandcenter/configs/config_template.txt") configFile = local_path("../gpdb/tests/utilities/commandcenter/configs/config.txt") cc_source = cc_install_dir + "/gpcc_path.sh" transforms = {'%pgport%': pgport, '%source%': greenplum_path_file, '%instance_name%': cc_install_dir, '%port%': port, '%mdd%': mdd, '%cc_source%': cc_source } cmd_str = 'chmod 777 %s; chmod 777 %s;'% (configFile_template ,configFile) tinctest.logger.info(cmd_str) cmd = Command (name = 'chmod', cmdStr = cmd_str, ctxt = REMOTE, remoteHost = 'localhost') cmd.run() TINCSystem.substitute_strings_in_file(configFile_template, configFile, transforms)
def setUpClass(cls): """ setUpClass of MPPTestCase does the following: -> Create out directory for the class if it does not exist. This is thread safe in case an MPPTestCase is used concurrently within a ScenarioTestCase or ConcurrencyTestCase -> Configures the database specified at the class level variable 'db_name' """ tinctest.logger.trace_in() #QAINF-760 - we need to treat db_name in the class level doc string as a class level variable #rather than an instance level variable ds = cls.__doc__ if ds: lines = ds.splitlines() for line in lines: line = line.strip() if line.find('@db_name') != 0: continue line = line[1:] if len(line.split()) <= 1: break (key, cls.db_name) = line.split(' ', 1) break super(MPPTestCase, cls).setUpClass() if not os.path.exists(cls.get_out_dir()): TINCSystem.make_dirs(cls.get_out_dir(), ignore_exists_error=True) if cls.db_name: tinctest.logger.debug( "Configure database %s from MPPTestCase setUpClass." % cls.db_name) cls.configure_database(cls.db_name) tinctest.logger.trace_out()
def setUpClass(cls): TINCSystem.make_dirs(cls.get_data_dir(), ignore_exists_error=True) super(ErrTblTestCase, cls).setUpClass()
def setUpClass(cls): TINCSystem.make_dirs(cls.get_data_dir(), ignore_exists_error=True) super(ErrorLogConcurrencyTests, cls).setUpClass()
def test_make_dirs_relative_path(self): test_dir = 'test/output' with self.assertRaises(TINCSystemException) as cm: TINCSystem.make_dirs(test_dir)
def setUpClass(cls): super(TableDistrubtionPolicySnapshot, cls).setUpClass() TINCSystem.make_dirs(cls.get_ans_dir(), force_create=True)