def __fetch_hawq_configuration(self):
        '''Fetch master hostname, segments hostname, data directory of HAWQ.'''
        self.hawq_master = config.getMasterHostName()
        self.hawq_segments = config.getSegHostNames()
        self.hawq_paths = []
        self.hawq_config = {}

        sql = "SELECT gsc.hostname, pfe.fselocation FROM gp_segment_configuration gsc, pg_filespace_entry pfe WHERE gsc.dbid = pfe.fsedbid AND pfe.fselocation NOT LIKE 'hdfs%' ORDER BY pfe.fselocation;"
        (ok, out) = psql.runcmd( dbname = 'postgres', cmd = sql , ofile = '-', flag = '-q -t' )

        if not ok:
            print out
            raise Exception("Failed to get HAWQ configuration paths.")
        else:
            for line in out:
                line = line.strip()
                if line:
                    (host, path) = line.split( '|' )
                    if not self.hawq_config.has_key(host.strip()):
                        self.hawq_config[host.strip()] = []
                    self.hawq_config[host.strip()].append(path.strip())
                        
                   # self.hawq_config.append( (host.strip(), path.strip()) )
                    self.hawq_paths.append( path.strip() ) 

        print self.hawq_master
        print self.hawq_segments
        print self.hawq_config
Exemple #2
0
    def __fetch_hawq_configuration(self):
        '''Fetch master hostname, segments hostname, data directory of HAWQ.'''
        self.hawq_master = config.getMasterHostName()
        self.hawq_segments = config.getSegHostNames()
        self.hawq_paths = []
        self.hawq_config = {}

        sql = "SELECT gsc.hostname, pfe.fselocation FROM gp_segment_configuration gsc, pg_filespace_entry pfe WHERE gsc.dbid = pfe.fsedbid AND pfe.fselocation NOT LIKE 'hdfs%' ORDER BY pfe.fselocation;"
        (ok, out) = psql.runcmd(dbname='postgres',
                                cmd=sql,
                                ofile='-',
                                flag='-q -t')

        if not ok:
            print out
            raise Exception("Failed to get HAWQ configuration paths.")
        else:
            for line in out:
                line = line.strip()
                if line:
                    (host, path) = line.split('|')
                    if not self.hawq_config.has_key(host.strip()):
                        self.hawq_config[host.strip()] = []
                    self.hawq_config[host.strip()].append(path.strip())

                    # self.hawq_config.append( (host.strip(), path.strip()) )
                    self.hawq_paths.append(path.strip())

        print self.hawq_master
        print self.hawq_segments
        print self.hawq_config
Exemple #3
0
    def _prepare_hostfile(self):
        # prep hostfile_master
        master_host_name = config.getMasterHostName()
        with open(self.hostfile_master, 'w') as f:
            f.write(str(master_host_name) + '\n')

        # prep hostfile_seg
        self.seg_hostname_list = config.getSegHostNames()
        self.seg_host_num = len(self.seg_hostname_list)
        with open(self.hostfile_seg, 'w') as f:
            f.write('\n'.join(self.seg_hostname_list))
    def _prepare_hostfile(self):
        # prep hostfile_master
        master_host_name = config.getMasterHostName()
        with open(self.hostfile_master, 'w') as f:
            f.write(str(master_host_name) + '\n')

        # prep hostfile_seg
        self.seg_hostname_list = config.getSegHostNames()
        self.seg_host_num = len(self.seg_hostname_list)
        with open(self.hostfile_seg, 'w') as f:
            f.write('\n'.join(self.seg_hostname_list))
Exemple #5
0
        self.tmp_folder = self.report_directory + os.sep + 'tmp'
        os.system('mkdir -p %s' % (self.tmp_folder))
        
    def __get_table_settings(self, workload_specification):
        ts = workload_specification['table_setting']

        # Calculate scale factor for workload
        try:
            self.data_volume_type = ts['data_volume_type'].upper()
            self.data_volume_size = ts['data_volume_size']
        except Exception, e:
            print('Please add %s option in schedule file.' % (str(e)) )
            sys.exit(2)
        
        # Need to make it univerally applicable instead of hard-code number of segments
        nnodes = len(config.getSegHostNames())
        if nnodes == 0:
            print 'getSegHostNames from gp_segment_configuration error.'
            sys.exit(2)
        
        if 'seg_num' in ts.keys():
            self.seg_num = ts['seg_num']
        self.nsegs =  nnodes * self.seg_num
        
        if self.data_volume_type == 'TOTAL':
            self.scale_factor = self.data_volume_size
        elif self.data_volume_type == 'PER_NODE':
            self.scale_factor = self.data_volume_size * nnodes
        elif self.data_volume_type == 'PER_SEGMENT':
            self.scale_factor = self.data_volume_size * self.nsegs
        else:
        self.tmp_folder = self.report_directory + os.sep + 'tmp'
        os.system('mkdir -p %s' % (self.tmp_folder))
        
    def __get_table_settings(self, workload_specification):
        ts = workload_specification['table_setting']

        # Calculate scale factor for workload
        try:
            self.data_volume_type = ts['data_volume_type'].upper()
            self.data_volume_size = ts['data_volume_size']
        except Exception, e:
            print('Please add %s option in schedule file.' % (str(e)) )
            sys.exit(2)
        
        # Need to make it univerally applicable instead of hard-code number of segments
        nnodes = len(config.getSegHostNames())
        if nnodes == 0:
            print 'getSegHostNames from gp_segment_configuration error.'
            sys.exit(2)
        
        if 'seg_num' in ts.keys():
            self.seg_num = ts['seg_num']
        self.nsegs =  nnodes * self.seg_num
        
        if self.data_volume_type == 'TOTAL':
            self.scale_factor = self.data_volume_size
        elif self.data_volume_type == 'PER_NODE':
            self.scale_factor = self.data_volume_size * nnodes
        elif self.data_volume_type == 'PER_SEGMENT':
            self.scale_factor = self.data_volume_size * self.nsegs
        else: