def __fetch_hawq_configuration(self):
'''Fetch master hostname, segments hostname, data directory of HAWQ.'''
self.hawq_master = config.getMasterHostName()
self.hawq_segments = config.getSegHostNames()
self.hawq_paths = []
self.hawq_config = {}
sql = "SELECT gsc.hostname, pfe.fselocation FROM gp_segment_configuration gsc, pg_filespace_entry pfe WHERE gsc.dbid = pfe.fsedbid AND pfe.fselocation NOT LIKE 'hdfs%' ORDER BY pfe.fselocation;"
(ok, out) = psql.runcmd( dbname = 'postgres', cmd = sql , ofile = '-', flag = '-q -t' )
if not ok:
print out
raise Exception("Failed to get HAWQ configuration paths.")
else:
for line in out:
line = line.strip()
if line:
(host, path) = line.split( '|' )
if not self.hawq_config.has_key(host.strip()):
self.hawq_config[host.strip()] = []
self.hawq_config[host.strip()].append(path.strip())
# self.hawq_config.append( (host.strip(), path.strip()) )
self.hawq_paths.append( path.strip() )
print self.hawq_master
print self.hawq_segments
print self.hawq_config
def __fetch_hawq_configuration(self):
'''Fetch master hostname, segments hostname, data directory of HAWQ.'''
self.hawq_master = config.getMasterHostName()
self.hawq_segments = config.getSegHostNames()
self.hawq_paths = []
self.hawq_config = {}
sql = "SELECT gsc.hostname, pfe.fselocation FROM gp_segment_configuration gsc, pg_filespace_entry pfe WHERE gsc.dbid = pfe.fsedbid AND pfe.fselocation NOT LIKE 'hdfs%' ORDER BY pfe.fselocation;"
(ok, out) = psql.runcmd(dbname='postgres',
cmd=sql,
ofile='-',
flag='-q -t')
if not ok:
print out
raise Exception("Failed to get HAWQ configuration paths.")
else:
for line in out:
line = line.strip()
if line:
(host, path) = line.split('|')
if not self.hawq_config.has_key(host.strip()):
self.hawq_config[host.strip()] = []
self.hawq_config[host.strip()].append(path.strip())
# self.hawq_config.append( (host.strip(), path.strip()) )
self.hawq_paths.append(path.strip())
print self.hawq_master
print self.hawq_segments
print self.hawq_config
def _prepare_hostfile(self):
# prep hostfile_master
master_host_name = config.getMasterHostName()
with open(self.hostfile_master, 'w') as f:
f.write(str(master_host_name) + '\n')
# prep hostfile_seg
self.seg_hostname_list = config.getSegHostNames()
self.seg_host_num = len(self.seg_hostname_list)
with open(self.hostfile_seg, 'w') as f:
f.write('\n'.join(self.seg_hostname_list))
def _prepare_hostfile(self):
# prep hostfile_master
master_host_name = config.getMasterHostName()
with open(self.hostfile_master, 'w') as f:
f.write(str(master_host_name) + '\n')
# prep hostfile_seg
self.seg_hostname_list = config.getSegHostNames()
self.seg_host_num = len(self.seg_hostname_list)
with open(self.hostfile_seg, 'w') as f:
f.write('\n'.join(self.seg_hostname_list))
self.tmp_folder = self.report_directory + os.sep + 'tmp'
os.system('mkdir -p %s' % (self.tmp_folder))
def __get_table_settings(self, workload_specification):
ts = workload_specification['table_setting']
# Calculate scale factor for workload
try:
self.data_volume_type = ts['data_volume_type'].upper()
self.data_volume_size = ts['data_volume_size']
except Exception, e:
print('Please add %s option in schedule file.' % (str(e)) )
sys.exit(2)
# Need to make it univerally applicable instead of hard-code number of segments
nnodes = len(config.getSegHostNames())
if nnodes == 0:
print 'getSegHostNames from gp_segment_configuration error.'
sys.exit(2)
if 'seg_num' in ts.keys():
self.seg_num = ts['seg_num']
self.nsegs = nnodes * self.seg_num
if self.data_volume_type == 'TOTAL':
self.scale_factor = self.data_volume_size
elif self.data_volume_type == 'PER_NODE':
self.scale_factor = self.data_volume_size * nnodes
elif self.data_volume_type == 'PER_SEGMENT':
self.scale_factor = self.data_volume_size * self.nsegs
else:
self.tmp_folder = self.report_directory + os.sep + 'tmp'
os.system('mkdir -p %s' % (self.tmp_folder))
def __get_table_settings(self, workload_specification):
ts = workload_specification['table_setting']
# Calculate scale factor for workload
try:
self.data_volume_type = ts['data_volume_type'].upper()
self.data_volume_size = ts['data_volume_size']
except Exception, e:
print('Please add %s option in schedule file.' % (str(e)) )
sys.exit(2)
# Need to make it univerally applicable instead of hard-code number of segments
nnodes = len(config.getSegHostNames())
if nnodes == 0:
print 'getSegHostNames from gp_segment_configuration error.'
sys.exit(2)
if 'seg_num' in ts.keys():
self.seg_num = ts['seg_num']
self.nsegs = nnodes * self.seg_num
if self.data_volume_type == 'TOTAL':
self.scale_factor = self.data_volume_size
elif self.data_volume_type == 'PER_NODE':
self.scale_factor = self.data_volume_size * nnodes
elif self.data_volume_type == 'PER_SEGMENT':
self.scale_factor = self.data_volume_size * self.nsegs
else: