Esempio n. 1
0
 def test_make_dirs(self):
     test_dir = os.path.join(os.path.dirname(__file__), 'test_mkdirs')
     if os.path.exists(test_dir):
         shutil.rmtree(test_dir)
     self.assertFalse(os.path.exists(test_dir))
     TINCSystem.make_dirs(test_dir)
     self.assertTrue(os.path.exists(test_dir))
Esempio n. 2
0
    def generate_config_file(self, greenplum_path_file, pgport, cc_install_dir,
                             port, mdd):
        configFile_template = local_path(
            "../gpdb/tests/utilities/commandcenter/configs/config_template.txt"
        )
        configFile = local_path(
            "../gpdb/tests/utilities/commandcenter/configs/config.txt")
        cc_source = cc_install_dir + "/gpcc_path.sh"
        transforms = {
            '%pgport%': pgport,
            '%source%': greenplum_path_file,
            '%instance_name%': cc_install_dir,
            '%port%': port,
            '%mdd%': mdd,
            '%cc_source%': cc_source
        }
        cmd_str = 'chmod 777 %s; chmod 777 %s;' % (configFile_template,
                                                   configFile)
        tinctest.logger.info(cmd_str)
        cmd = Command(name='chmod',
                      cmdStr=cmd_str,
                      ctxt=REMOTE,
                      remoteHost='localhost')

        cmd.run()

        TINCSystem.substitute_strings_in_file(configFile_template, configFile,
                                              transforms)
Esempio n. 3
0
 def setUpClass(cls):
     """
     Some unit tests here call's test_case.run() method which does not set up the out dir required (out_dir set up gets
     handled in MPPTestCase.setUpClass which will be invoked only when running through a test_suite.run().
     Therefore, we setup the out dir required here in the test itself. 
     """
     TINCSystem.make_dirs(local_path('output'), ignore_exists_error=True)
Esempio n. 4
0
 def setUpClass(cls):
     """
     Some unit tests here call's test_case.run() method which does not set up the out dir required (out_dir set up gets
     handled in MPPTestCase.setUpClass which will be invoked only when running through a test_suite.run().
     Therefore, we setup the out dir required here in the test itself. 
     """
     TINCSystem.make_dirs(local_path('output'), ignore_exists_error=True)
Esempio n. 5
0
    def loadXML(self, case_name):
        xml_dir = self.get_xml_dir()
        sql_output_dir = os.path.join(self.get_source_dir(), 'output/sql')
        TINCSystem.make_dirs(sql_output_dir, ignore_exists_error=True)

        # copy init_file from xml dir
        init_file_src = os.path.join(xml_dir, 'init_file')
        init_file_dst = os.path.join(sql_output_dir, 'init_file')
        shutil.copyfile(init_file_src, init_file_dst)

        filename = case_name + '.xml'
        xml_file = os.path.join(xml_dir, filename)

        doc = minidom.parse(xml_file)
        case_name = os.path.splitext(filename)[0]

        # extract sql and put them as files
        sql_elem = doc.getElementsByTagName('sql')
        assert(len(sql_elem) == 1)
        sql_elem = sql_elem[0]
        self.write_file(sql_elem, sql_output_dir, case_name, 'pre')
        self.write_file(sql_elem, sql_output_dir, case_name, 'trigger')
        self.write_file(sql_elem, sql_output_dir, case_name, 'post')

        ans_elem = doc.getElementsByTagName('ans')
        assert(len(ans_elem) == 1)
        ans_elem = ans_elem[0]
        self.write_file(ans_elem, sql_output_dir, case_name, 'pre')
        self.write_file(ans_elem, sql_output_dir, case_name, 'trigger')
        self.write_file(ans_elem, sql_output_dir, case_name, 'post')
Esempio n. 6
0
    def _generate_gpinit_config_files(self):
        transforms = {
            '%SEG_PREFIX%':
            self.seg_prefix,
            '%PORT_BASE%':
            self.port_base,
            '%MASTER_HOST%':
            self.master_host,  # First item in self.hosts
            '%HOSTFILE%':
            self.host_file,
            '%MASTER_PORT%':
            self.master_port,
            '%MASTER_DATA_DIR%':
            os.path.dirname(self.mdd),
            '%DATA_DIR%': (os.path.dirname(self.primary_data_dir) + ' ') *
            self.number_of_segments_per_host
        }

        # First generate host file based on number_of_hosts
        with open(self.host_file, 'w') as f:
            for host in self.hosts:
                f.write(host + '\n')
        TINCSystem.substitute_strings_in_file(self.gpinitconfig_template,
                                              self.gpinitconfig_file,
                                              transforms)
Esempio n. 7
0
 def test_make_dirs(self):
     test_dir = os.path.join(os.path.dirname(__file__), 'test_mkdirs')
     if os.path.exists(test_dir):
         shutil.rmtree(test_dir)
     self.assertFalse(os.path.exists(test_dir))
     TINCSystem.make_dirs(test_dir)
     self.assertTrue(os.path.exists(test_dir))
Esempio n. 8
0
File: mpp_tc.py Progetto: 50wu/gpdb
    def setUpClass(cls):
        """
        setUpClass of MPPTestCase does the following:
        
        -> Create out directory for the class if it does not exist.
           This is thread safe in case an MPPTestCase is used concurrently
           within a ScenarioTestCase or ConcurrencyTestCase

        -> Configures the database specified at the class level variable 'db_name'
        """
        tinctest.logger.trace_in()

        #QAINF-760 - we need to treat db_name in the class level doc string as a class level variable
        #rather than an instance level variable
        ds = cls.__doc__
        if ds:
            lines = ds.splitlines()
            for line in lines:
                line = line.strip()
                if line.find('@db_name') != 0:
                    continue
                line = line[1:]
                if len(line.split()) <= 1:
                    break
                (key, cls.db_name) = line.split(' ', 1)
                break

        super(MPPTestCase, cls).setUpClass()
        if not os.path.exists(cls.get_out_dir()):
            TINCSystem.make_dirs(cls.get_out_dir(), ignore_exists_error = True)

        if cls.db_name:
            tinctest.logger.debug("Configure database %s from MPPTestCase setUpClass." % cls.db_name)
            cls.configure_database(cls.db_name)
        tinctest.logger.trace_out()
Esempio n. 9
0
    def _generate_gpinit_config_files(self):
        transforms = {'%PORT_BASE%': self.port_base,
                      '%MASTER_HOST%': self.hosts[0], # First item in self.hosts
                      '%HOSTFILE%': self.testcase_hosts_file,
                      '%MASTER_PORT%': self.master_port,
                      '%MASTER_DATA_DIR%': self.testcase_master_dir,
                      '%DATA_DIR%': (self.testcase_primary_dir + ' ') * self.number_of_segments
                      }

        if self.mirror_enabled:
            transforms['%MIRROR_DIR%'] = (self.testcase_mirror_dir + ' ') * self.number_of_segments
            transforms['%MIRROR_PORT_BASE%'] = self.mirror_port_base
            transforms['%REP_PORT_BASE%'] = self.rep_port_base
            transforms['%MIRROR_REP_PORT_BASE%'] = self.mirror_rep_port_base

        # First generate host file based on number_of_hosts
        with open(self.testcase_hosts_file, 'w') as f:
            for host in self.hosts[1:self.number_of_hosts+1]:
                f.write(host + '\n')

        if self.mirror_enabled:
            TINCSystem.substitute_strings_in_file(self.gpinitconfig_mirror_template,
                                                  self.testcase_init_file,
                                                  transforms)
        else:
            TINCSystem.substitute_strings_in_file(self.gpinitconfig_template,
                                                  self.testcase_init_file,
                                                  transforms)
Esempio n. 10
0
    def loadXML(self, case_name):
        xml_dir = self.get_xml_dir()
        sql_output_dir = os.path.join(self.get_source_dir(), 'output/sql')
        TINCSystem.make_dirs(sql_output_dir, ignore_exists_error=True)

        # copy init_file from xml dir
        init_file_src = os.path.join(xml_dir, 'init_file')
        init_file_dst = os.path.join(sql_output_dir, 'init_file')
        shutil.copyfile(init_file_src, init_file_dst)

        filename = case_name + '.xml'
        xml_file = os.path.join(xml_dir, filename)

        doc = minidom.parse(xml_file)
        case_name = os.path.splitext(filename)[0]

        # extract sql and put them as files
        sql_elem = doc.getElementsByTagName('sql')
        assert (len(sql_elem) == 1)
        sql_elem = sql_elem[0]
        self.write_file(sql_elem, sql_output_dir, case_name, 'pre')
        self.write_file(sql_elem, sql_output_dir, case_name, 'trigger')
        self.write_file(sql_elem, sql_output_dir, case_name, 'post')

        ans_elem = doc.getElementsByTagName('ans')
        assert (len(ans_elem) == 1)
        ans_elem = ans_elem[0]
        self.write_file(ans_elem, sql_output_dir, case_name, 'pre')
        self.write_file(ans_elem, sql_output_dir, case_name, 'trigger')
        self.write_file(ans_elem, sql_output_dir, case_name, 'post')
Esempio n. 11
0
    def _do_gpinitsystem(self):
        # Takes care of initializing a cluster according to the test configuration
        TINCSystem.make_dirs(self.testcase_out_dir, force_create=True)
        TINCSystem.make_dirs(self.testcase_master_dir, force_create=True)

        segment_host_file = '/tmp/segment_hosts'
        with open(segment_host_file, 'w') as f:
            for host in self.hosts:
                f.write(host)
                f.write('\n')

        # Create primary dirs
        res = {'rc': 0, 'stdout': '', 'stderr': ''}
        run_shell_command(
            "gpssh -f %s -e 'rm -rf %s; mkdir -p %s'" %
            (segment_host_file, self.testcase_primary_dir,
             self.testcase_primary_dir), 'create segment dirs', res)
        if res['rc'] > 0:
            raise GpExpandTestCaseException(
                "Failed to create segment directories")

        # Create mirror dirs
        res = {'rc': 0, 'stdout': '', 'stderr': ''}
        run_shell_command(
            "gpssh -f %s -e 'rm -rf %s; mkdir -p %s'" %
            (segment_host_file, self.testcase_mirror_dir,
             self.testcase_mirror_dir), 'create segment dirs', res)
        if res['rc'] > 0:
            raise GpExpandTestCaseException(
                "Failed to create segment directories")

        # Create filespace dirs
        res = {'rc': 0, 'stdout': '', 'stderr': ''}
        run_shell_command(
            "gpssh -f %s -e 'rm -rf %s; mkdir -p %s'" %
            (segment_host_file, self.testcase_filespace_dir,
             self.testcase_filespace_dir), 'create segment dirs', res)
        if res['rc'] > 0:
            raise GpExpandTestCaseException(
                "Failed to create segment directories")

        # Generate the config files to initialize the cluster
        self._generate_gpinit_config_files()
        self.assertTrue(os.path.exists(self.testcase_init_file))
        self.assertTrue(os.path.exists(self.testcase_hosts_file))

        # TODO: Do some clean up before init
        # Init the system
        # Still have to figure out if it is a good idea to init for every test or not
        # Pro: easier to develop self contained tests Con: time taken for every test
        result = GpInitSystem(self.testcase_init_file).run(validate=False)

        # initsystem returns 1 for warnings and 2 for errors
        if result.rc > 1:
            tinctest.logger.error("Failed initializing the cluster: %s" %
                                  result)
            raise GPExpandTestCaseException(
                "Failed initializing the cluster. Look into gpAdminLogs for more information"
            )
Esempio n. 12
0
 def get_file_names(self, suffix):
     stem = self.method_name.replace('test_', '')
     sql_file_root = os.path.join(self.get_sql_dir(), stem  + '.sql')
     sql_file = sql_file_root + '.' + suffix
     ans_file = sql_file.replace('.sql.', '.ans.')
     out_file_name = os.path.basename(sql_file).replace('.sql.', '.out.')
     out_file = os.path.join(self.get_out_dir(), out_file_name)
     TINCSystem.make_dirs(self.get_out_dir(), ignore_exists_error=True)
     return (sql_file, ans_file, out_file)
Esempio n. 13
0
    def test_make_dirs_existing_ignore(self):
        test_dir = os.path.join(os.path.dirname(__file__), 'test_mkdirs')
        if os.path.exists(test_dir):
            shutil.rmtree(test_dir)
        self.assertFalse(os.path.exists(test_dir))
        TINCSystem.make_dirs(test_dir)
        self.assertTrue(os.path.exists(test_dir))

        # This should not fail and ignore existing error
        TINCSystem.make_dirs(test_dir, ignore_exists_error=True)
Esempio n. 14
0
 def test_make_dirs_existing_ignore(self):
     test_dir = os.path.join(os.path.dirname(__file__), 'test_mkdirs')
     if os.path.exists(test_dir):
         shutil.rmtree(test_dir)
     self.assertFalse(os.path.exists(test_dir))
     TINCSystem.make_dirs(test_dir)
     self.assertTrue(os.path.exists(test_dir))
      
     # This should not fail and ignore existing error
     TINCSystem.make_dirs(test_dir, ignore_exists_error = True)
Esempio n. 15
0
    def test_make_dirs_existing(self):
        test_dir = os.path.join(os.path.dirname(__file__), 'test_mkdirs')
        if os.path.exists(test_dir):
            shutil.rmtree(test_dir)
        self.assertFalse(os.path.exists(test_dir))
        TINCSystem.make_dirs(test_dir)
        self.assertTrue(os.path.exists(test_dir))

        # This should fail
        with self.assertRaises(OSError) as cm:
            TINCSystem.make_dirs(test_dir)
Esempio n. 16
0
 def test_make_dirs_existing(self):
     test_dir = os.path.join(os.path.dirname(__file__), 'test_mkdirs')
     if os.path.exists(test_dir):
         shutil.rmtree(test_dir)
     self.assertFalse(os.path.exists(test_dir))
     TINCSystem.make_dirs(test_dir)
     self.assertTrue(os.path.exists(test_dir))
      
     # This should fail
     with self.assertRaises(OSError) as cm:
         TINCSystem.make_dirs(test_dir)
Esempio n. 17
0
    def test_force_create_make_dirs(self):
        test_dir = os.path.join(os.path.dirname(__file__), 'test_force')
        if os.path.exists(test_dir):
            shutil.rmtree(test_dir)
        self.assertFalse(os.path.exists(test_dir))
        TINCSystem.make_dirs(test_dir)
        self.assertTrue(os.path.exists(test_dir))

        test_file = os.path.join(test_dir, 'testfile')
        with open(test_file, 'w') as f:
            f.write('test')

        self.assertTrue(os.path.exists(test_file))
        # Force create
        TINCSystem.make_dirs(test_dir, force_create = True)
        self.assertFalse(os.path.exists(test_file))
Esempio n. 18
0
    def test_force_create_make_dirs(self):
        test_dir = os.path.join(os.path.dirname(__file__), 'test_force')
        if os.path.exists(test_dir):
            shutil.rmtree(test_dir)
        self.assertFalse(os.path.exists(test_dir))
        TINCSystem.make_dirs(test_dir)
        self.assertTrue(os.path.exists(test_dir))

        test_file = os.path.join(test_dir, 'testfile')
        with open(test_file, 'w') as f:
            f.write('test')

        self.assertTrue(os.path.exists(test_file))
        # Force create
        TINCSystem.make_dirs(test_dir, force_create=True)
        self.assertFalse(os.path.exists(test_file))
Esempio n. 19
0
    def split_tbl(cls):
        """
        Split data files using split command for the testing
        where gpfdist serves multiple files.
        """

        source_dir = cls.get_source_dir()
        data_dir = os.path.join(source_dir, 'data')
        for fn in os.listdir(data_dir):
            if not fn.endswith('.tbl'):
                continue
            split_dir = os.path.join(data_dir, fn + '-dir')
            if os.path.exists(split_dir):
                continue
            TINCSystem.make_dirs(split_dir, ignore_exists_error=True)
            os.system('cd {0}; split ../{1}'.format(split_dir, fn))
Esempio n. 20
0
    def split_tbl(cls):
        """
        Split data files using split command for the testing
        where gpfdist serves multiple files.
        """

        source_dir = cls.get_source_dir()
        data_dir = os.path.join(source_dir, 'data')
        for fn in os.listdir(data_dir):
            if not fn.endswith('.tbl'):
                continue
            split_dir = os.path.join(data_dir, fn + '-dir')
            if os.path.exists(split_dir):
                continue
            TINCSystem.make_dirs(split_dir, ignore_exists_error=True)
            os.system('cd {0}; split ../{1}'.format(split_dir, fn))
Esempio n. 21
0
    def _generate_gpinit_config_files(self):
        transforms = {'%SEG_PREFIX%': self.seg_prefix,
                      '%PORT_BASE%': self.port_base,
                      '%MASTER_HOST%': self.master_host, # First item in self.hosts
                      '%HOSTFILE%': self.host_file,
                      '%MASTER_PORT%': self.master_port,
                      '%MASTER_DATA_DIR%': os.path.dirname(self.mdd),
                      '%DATA_DIR%': (os.path.dirname(self.primary_data_dir) + ' ') * self.number_of_segments_per_host 
                      }

        # First generate host file based on number_of_hosts
        with open(self.host_file, 'w') as f:
            for host in self.hosts:
                f.write(host + '\n')
        TINCSystem.substitute_strings_in_file(self.gpinitconfig_template,
                                              self.gpinitconfig_file,
                                              transforms)
Esempio n. 22
0
    def _do_gpinitsystem(self):    
        # Takes care of initializing a cluster according to the test configuration
        TINCSystem.make_dirs(self.testcase_out_dir, force_create=True)
        TINCSystem.make_dirs(self.testcase_master_dir, force_create=True)

        segment_host_file = '/tmp/segment_hosts'
        with open(segment_host_file, 'w') as f:
            for host in self.hosts:
                f.write(host)
                f.write('\n')

        # Create primary dirs
        res = {'rc': 0, 'stdout' : '', 'stderr': ''}
        run_shell_command("gpssh -f %s -e 'rm -rf %s; mkdir -p %s'" %(segment_host_file, self.testcase_primary_dir, self.testcase_primary_dir), 'create segment dirs', res)
        if res['rc'] > 0:
            raise GpExpandTestCaseException("Failed to create segment directories")

        # Create mirror dirs
        res = {'rc': 0, 'stdout' : '', 'stderr': ''}
        run_shell_command("gpssh -f %s -e 'rm -rf %s; mkdir -p %s'" %(segment_host_file, self.testcase_mirror_dir, self.testcase_mirror_dir), 'create segment dirs', res)
        if res['rc'] > 0:
            raise GpExpandTestCaseException("Failed to create segment directories")

        # Create filespace dirs
        res = {'rc': 0, 'stdout' : '', 'stderr': ''}
        run_shell_command("gpssh -f %s -e 'rm -rf %s; mkdir -p %s'" %(segment_host_file, self.testcase_filespace_dir, self.testcase_filespace_dir), 'create segment dirs', res)
        if res['rc'] > 0:
            raise GpExpandTestCaseException("Failed to create segment directories")

        # Generate the config files to initialize the cluster
        self._generate_gpinit_config_files()
        self.assertTrue(os.path.exists(self.testcase_init_file))
        self.assertTrue(os.path.exists(self.testcase_hosts_file))

        # TODO: Do some clean up before init
        # Init the system
        # Still have to figure out if it is a good idea to init for every test or not
        # Pro: easier to develop self contained tests Con: time taken for every test
        result = GpInitSystem(self.testcase_init_file).run(validate=False)

        # initsystem returns 1 for warnings and 2 for errors
        if result.rc > 1:
            tinctest.logger.error("Failed initializing the cluster: %s" %result)
            raise GPExpandTestCaseException("Failed initializing the cluster. Look into gpAdminLogs for more information")
Esempio n. 23
0
                def generate_config_file(self,greenplum_path_file,pgport,cc_install_dir,port,mdd):
                        configFile_template = local_path("../gpdb/tests/utilities/commandcenter/configs/config_template.txt")
                        configFile = local_path("../gpdb/tests/utilities/commandcenter/configs/config.txt")
                        cc_source = cc_install_dir + "/gpcc_path.sh"
                        transforms = {'%pgport%': pgport,
                              '%source%': greenplum_path_file,
                              '%instance_name%': cc_install_dir,
                              '%port%': port,
                              '%mdd%': mdd,
                              '%cc_source%': cc_source 
                              }
                        cmd_str = 'chmod 777 %s; chmod 777 %s;'% (configFile_template ,configFile)
                        tinctest.logger.info(cmd_str)
                        cmd = Command (name = 'chmod',
                                   cmdStr = cmd_str,
                                   ctxt = REMOTE, remoteHost = 'localhost')

                        cmd.run()                        
			                                 
                        TINCSystem.substitute_strings_in_file(configFile_template,
                                                          configFile,
                                                          transforms)	
Esempio n. 24
0
    def setUpClass(cls):
        """
        setUpClass of MPPTestCase does the following:
        
        -> Create out directory for the class if it does not exist.
           This is thread safe in case an MPPTestCase is used concurrently
           within a ScenarioTestCase or ConcurrencyTestCase

        -> Configures the database specified at the class level variable 'db_name'
        """
        tinctest.logger.trace_in()

        #QAINF-760 - we need to treat db_name in the class level doc string as a class level variable
        #rather than an instance level variable
        ds = cls.__doc__
        if ds:
            lines = ds.splitlines()
            for line in lines:
                line = line.strip()
                if line.find('@db_name') != 0:
                    continue
                line = line[1:]
                if len(line.split()) <= 1:
                    break
                (key, cls.db_name) = line.split(' ', 1)
                break

        super(MPPTestCase, cls).setUpClass()
        if not os.path.exists(cls.get_out_dir()):
            TINCSystem.make_dirs(cls.get_out_dir(), ignore_exists_error=True)

        if cls.db_name:
            tinctest.logger.debug(
                "Configure database %s from MPPTestCase setUpClass." %
                cls.db_name)
            cls.configure_database(cls.db_name)
        tinctest.logger.trace_out()
Esempio n. 25
0
 def setUpClass(cls):
     TINCSystem.make_dirs(cls.get_data_dir(), ignore_exists_error=True)
     super(ErrTblTestCase, cls).setUpClass()
Esempio n. 26
0
 def setUpClass(cls):
     TINCSystem.make_dirs(cls.get_data_dir(), ignore_exists_error=True)
     super(ErrorLogConcurrencyTests, cls).setUpClass()
Esempio n. 27
0
 def test_make_dirs_relative_path(self):
     test_dir = 'test/output'
     with self.assertRaises(TINCSystemException) as cm:
         TINCSystem.make_dirs(test_dir)
Esempio n. 28
0
 def setUpClass(cls):
     TINCSystem.make_dirs(cls.get_data_dir(), ignore_exists_error=True)
     super(ErrTblTestCase, cls).setUpClass()
Esempio n. 29
0
 def setUpClass(cls):
     TINCSystem.make_dirs(cls.get_data_dir(), ignore_exists_error=True)
     super(ErrorLogConcurrencyTests, cls).setUpClass()
Esempio n. 30
0
 def setUpClass(cls):
     super(TableDistrubtionPolicySnapshot, cls).setUpClass()
     TINCSystem.make_dirs(cls.get_ans_dir(), force_create=True)
Esempio n. 31
0
 def test_make_dirs_relative_path(self):
     test_dir = 'test/output'
     with self.assertRaises(TINCSystemException) as cm:
         TINCSystem.make_dirs(test_dir)
Esempio n. 32
0
 def setUpClass(cls):
     super(TableDistrubtionPolicySnapshot, cls).setUpClass()
     TINCSystem.make_dirs(cls.get_ans_dir(), force_create=True)