def get_master_port():
logger.debug("Checking if GPHOME env variable is set.")
HAWQ_CONFIG_DIR = os.environ.get('GPHOME')
if HAWQ_CONFIG_DIR is None:
raise GpError("Environment Variable GPHOME not set!")
hawq_site = HawqXMLParser(HAWQ_CONFIG_DIR)
master_port = hawq_site.get_value_from_name('hawq_master_address_port').strip()
return master_port
def get_master_port():
logger.debug("Checking if GPHOME env variable is set.")
HAWQ_CONFIG_DIR = os.environ.get('GPHOME')
if HAWQ_CONFIG_DIR is None:
raise GpError("Environment Variable GPHOME not set!")
hawq_site = HawqXMLParser(HAWQ_CONFIG_DIR)
master_port = hawq_site.get_value_from_name(
'hawq_master_address_port').strip()
return master_port
def _get_gpdb_host_list(self):
"""
TODO: AK: Get rid of this. Program logic should not be driving host list building .
This method gets the host names
of all hosts in the gpdb array.
It sets the following variables
GpPkgProgram.master_host to master
GpPkgProgram.standby_host to standby
GpPkgProgram.segment_host_list to segment hosts
"""
logger.debug('_get_gpdb_host_list')
#Get host list
GPHOME = os.getenv('GPHOME')
if GPHOME == '' or not GPHOME:
logger.info('GPHOME is not set.')
sys.exit(1)
hawq_site = HawqXMLParser(GPHOME)
hawq_site.get_all_values()
master_port = hawq_site.hawq_dict['hawq_master_address_port']
master_host = ""
standby_host = None
segment_host_list = []
host_list = get_hawq_hostname_all(master_port)
for host, status in host_list['master'].iteritems():
master_host = host
for host, status in host_list['standby'].iteritems():
standby_host = host
for host, status in host_list['segment'].iteritems():
segment_host_list.append(host)
#Deduplicate the hosts so that we
#dont install multiple times on the same host
segment_host_list = list(set(segment_host_list))
#Segments might exist on the master host. Since we store the
#master host separately in self.master_host, storing the master_host
#in the segment_host_list is redundant.
for host in segment_host_list:
if host == master_host or host == standby_host:
segment_host_list.remove(host)
self.master_host = master_host
self.standby_host = standby_host
self.segment_host_list = segment_host_list
def _get_gpdb_host_list(self):
"""
TODO: AK: Get rid of this. Program logic should not be driving host list building .
This method gets the host names
of all hosts in the gpdb array.
It sets the following variables
GpPkgProgram.master_host to master
GpPkgProgram.standby_host to standby
GpPkgProgram.segment_host_list to segment hosts
"""
logger.debug('_get_gpdb_host_list')
#Get host list
GPHOME = os.getenv('GPHOME')
if GPHOME == '' or not GPHOME:
logger.info('GPHOME is not set.')
sys.exit(1)
hawq_site = HawqXMLParser(GPHOME)
hawq_site.get_all_values()
master_port = hawq_site.hawq_dict['hawq_master_address_port']
master_host = ""
standby_host = None
segment_host_list = []
host_list = get_hawq_hostname_all(master_port)
for host, status in host_list['master'].iteritems():
master_host = host
for host, status in host_list['standby'].iteritems():
standby_host = host
for host, status in host_list['segment'].iteritems():
segment_host_list.append(host)
#Deduplicate the hosts so that we
#dont install multiple times on the same host
segment_host_list = list(set(segment_host_list))
#Segments might exist on the master host. Since we store the
#master host separately in self.master_host, storing the master_host
#in the segment_host_list is redundant.
for host in segment_host_list:
if host == master_host or host == standby_host:
segment_host_list.remove(host)
self.master_host = master_host
self.standby_host = standby_host
self.segment_host_list = segment_host_list
def initFromCatalog(dbURL, utility=False, useAllSegmentFileSpaces=False):
"""
Factory method, initializes a HAWQArray from provided database URL
Please note that -
useAllSegmentFilespaces when set to true makes this method add *all* filespaces
to the segments of hawqarray. If false, only returns Master/Standby all filespaces
This is *hacky* and we know that it is not the right way to design methods/interfaces
We are doing this so that we do not affect behavior of existing tools like upgrade, gprecoverseg etc
"""
conn = dbconn.connect(dbURL, utility)
# Get the version from the database:
version_str = None
for row in dbconn.execSQL(conn, "SELECT version()"):
version_str = row[0]
version = GpVersion(version_str)
# Only for HAWQ 2.0
if version.getVersionRelease() in ("2.0"):
hawq_site = HawqXMLParser(GPHOME)
master_data_directory = hawq_site.get_value_from_name('hawq_master_directory')
segment_data_directory = hawq_site.get_value_from_name('hawq_segment_directory')
config_rows = dbconn.execSQL(conn, '''
SELECT sc.registration_order,
sc.role,
sc.status,
sc.hostname,
sc.address,
sc.port,
CASE
WHEN sc.registration_order <= 0 THEN '%s'
ELSE '%s'
END AS datadir
FROM pg_catalog.gp_segment_configuration sc
ORDER BY sc.registration_order;''' %
(master_data_directory, segment_data_directory))
# All of filesystem is shared storage
filesystemRows = dbconn.execSQL(conn, '''
SELECT oid, fsysname, true AS fsysshared
FROM pg_filesystem
ORDER BY fsysname
''')
filesystemArr = [HAWQFilesystemObj(fsysRow[0], fsysRow[1], fsysRow[2]) for fsysRow in filesystemRows]
filespaceRows = dbconn.execSQL(conn, '''
SELECT oid, fsname, fsfsys AS fsoid
FROM pg_filespace
WHERE oid != %d
ORDER BY fsname;
''' % (SYSTEM_FILESPACE))
filespaceArr = [HAWQFilespaceObj(fsRow[0], fsRow[1], HAWQFilesystemObj.getFilesystemObj(filesystemArr, fsRow[2])) for fsRow in filespaceRows]
else:
raise Exception("HAWQ version is invalid: %s" % version)
hawqdbs = []
print "### initFromCatalog ###"
hdb = None
for row in config_rows:
print row
# Extract fields from the row
(registration_order, role, status, hostname,
address, port, datadir) = row
# In GPSQL, only master maintain the filespace information.
# if registration_order != MASTER_REGISTRATION_ORDER and \
# fsoid != SYSTEM_FILESPACE and \
# not useAllSegmentFileSpaces:
# print "### initFromCatalog ... continue ###"
# continue
# The query returns all the filespaces for a segment on separate
# rows. If this row is the same dbid as the previous row simply
# add this filespace to the existing list, otherwise create a
# new segment.
# if seg and seg.getSegmentRegistrationOrder() == registration_order:
# seg.addSegmentFilespace(fsoid, fslocation)
# else:
# seg = HAWQDB(registration_order, role, status,
# hostname, address, port, datadir)
# segments.append(seg)
hdb = HAWQDB(registration_order, role, status,
hostname, address, port, datadir)
print "### initFromCatalog ... hdb ###"
print hdb
hawqdbs.append(hdb)
print "### initFromCatalog ... hawqdbs ###"
print hawqdbs
conn.close()
# origSegments = [seg.copy() for seg in segments]
array = HAWQArray(hawqdbs)
array.version = version
array.setFilespaces(filespaceArr)
array.setFilesystem(filesystemArr)
return array
def initFromCatalog(dbURL, utility=False, useAllSegmentFileSpaces=False):
"""
Factory method, initializes a HAWQArray from provided database URL
Please note that -
useAllSegmentFilespaces when set to true makes this method add *all* filespaces
to the segments of hawqarray. If false, only returns Master/Standby all filespaces
This is *hacky* and we know that it is not the right way to design methods/interfaces
We are doing this so that we do not affect behavior of existing tools like upgrade, gprecoverseg etc
"""
conn = dbconn.connect(dbURL, utility)
# Get the version from the database:
version_str = None
for row in dbconn.execSQL(conn, "SELECT version()"):
version_str = row[0]
version = GpVersion(version_str)
# Now only support HAWQ 2.x
hawq_major_version = version.getVersionRelease().split('.')[0]
if hawq_major_version == '2':
hawq_site = HawqXMLParser(GPHOME)
master_data_directory = hawq_site.get_value_from_name(
'hawq_master_directory')
segment_data_directory = hawq_site.get_value_from_name(
'hawq_segment_directory')
config_rows = dbconn.execSQL(
conn, '''
SELECT sc.registration_order,
sc.role,
sc.status,
sc.hostname,
sc.address,
sc.port,
CASE
WHEN sc.registration_order <= 0 THEN '%s'
ELSE '%s'
END AS datadir
FROM pg_catalog.gp_segment_configuration sc
ORDER BY sc.registration_order;''' %
(master_data_directory, segment_data_directory))
# All of filesystem is shared storage
filesystemRows = dbconn.execSQL(
conn, '''
SELECT oid, fsysname, true AS fsysshared
FROM pg_filesystem
ORDER BY fsysname
''')
filesystemArr = [
HAWQFilesystemObj(fsysRow[0], fsysRow[1], fsysRow[2])
for fsysRow in filesystemRows
]
filespaceRows = dbconn.execSQL(
conn, '''
SELECT oid, fsname, fsfsys AS fsoid
FROM pg_filespace
WHERE oid != %d
ORDER BY fsname;
''' % (SYSTEM_FILESPACE))
filespaceArr = [
HAWQFilespaceObj(
fsRow[0], fsRow[1],
HAWQFilesystemObj.getFilesystemObj(filesystemArr,
fsRow[2]))
for fsRow in filespaceRows
]
else:
raise Exception("HAWQ version is invalid: %s" % version)
hawqdbs = []
print "### initFromCatalog ###"
hdb = None
for row in config_rows:
print row
# Extract fields from the row
(registration_order, role, status, hostname, address, port,
datadir) = row
# In GPSQL, only master maintain the filespace information.
# if registration_order != MASTER_REGISTRATION_ORDER and \
# fsoid != SYSTEM_FILESPACE and \
# not useAllSegmentFileSpaces:
# print "### initFromCatalog ... continue ###"
# continue
# The query returns all the filespaces for a segment on separate
# rows. If this row is the same dbid as the previous row simply
# add this filespace to the existing list, otherwise create a
# new segment.
# if seg and seg.getSegmentRegistrationOrder() == registration_order:
# seg.addSegmentFilespace(fsoid, fslocation)
# else:
# seg = HAWQDB(registration_order, role, status,
# hostname, address, port, datadir)
# segments.append(seg)
hdb = HAWQDB(registration_order, role, status, hostname, address,
port, datadir)
print "### initFromCatalog ... hdb ###"
print hdb
hawqdbs.append(hdb)
print "### initFromCatalog ... hawqdbs ###"
print hawqdbs
conn.close()
# origSegments = [seg.copy() for seg in segments]
array = HAWQArray(hawqdbs)
array.version = version
array.setFilespaces(filespaceArr)
array.setFilesystem(filesystemArr)
return array