def orchestrator(options):
# Check that the backup set is complete
if not os.path.exists (options.backupdir):
logger.error('Backup directory you specified does not exist: ' + options.backupdir)
sys.exit(2)
if not os.path.isdir (options.backupdir):
logger.error('Backup folder you specified is not a directory: ' + options.backupdir)
sys.exit(2)
logger.info ('=== Master backup files found:')
backupset = dict()
for file in os.listdir(options.backupdir):
if os.path.isfile (os.path.join(options.backupdir, file)):
if file[:11]=='gp_dump_1_1' and options.timestamp in file and '.gz' in file:
logger.info (' ' + file)
if 'post_data' in file:
backupset['post'] = file
else:
backupset['master'] = file
logger.info ('=== Segment backup files found:')
backupset['segment'] = []
for file in os.listdir(options.backupdir):
if os.path.isfile (os.path.join(options.backupdir, file)):
if file[:9]=='gp_dump_0' and options.timestamp in file and '.gz' in file:
logger.info (' ' + file)
backupset['segment'].append(file)
if not 'post' in backupset:
logger.error ('Cannot find backup post_data file. Stopping')
sys.exit(4)
if not 'master' in backupset:
logger.error ('Cannot find backup master data file. Stopping')
sys.exit(4)
if len(backupset['segment']) == 0:
logger.error ('Cannot find backup segment files. Stopping')
sys.exit(4)
dburl = dbconn.DbURL(hostname = '127.0.0.1',
port = 5432,
dbname = 'template1',
username = '******',
password = options.password)
query = "select 1 from pg_database where datname = '%s'" % options.dbname
if execute (dburl, query) != [[1]]:
logger.error ("Database '%s' does not exist. Create it before running this script" % options.dbname)
sys.exit(4)
if not userinput.ask_yesno(None, "Confirm that database %s is empty and ready for restore?" % options.dbname, 'N'):
logger.error ("Restore terminated by user request")
sys.exit(6)
if not userinput.ask_yesno(None, "Do you want to continue with this resore?", 'N'):
logger.error ("Restore terminated by user request")
sys.exit(6)
logger.info ('=== Restoring master server backup file %s' % backupset['master'])
run_sync("cat %s/%s | /bin/gunzip -c | psql -d %s" % (options.backupdir, backupset['master'], options.dbname))
logger.info ('=== Restoring Segment Servers in %d threads' % options.nthreads)
restore_segments(backupset['segment'], options.nthreads)
logger.info ('=== Restoring master server post data file %s' % backupset['master'])
run_sync("cat %s/%s | /bin/gunzip -c | psql -d %s" % (options.backupdir, backupset['post'], options.dbname))
logger.info ('Restore complete')
def run(self):
if self.__options.parallelDegree < 1 or self.__options.parallelDegree > 64:
raise ProgramArgumentValidationException(
"Invalid parallelDegree provided with -B argument: %d" %
self.__options.parallelDegree)
self.__pool = base.WorkerPool(self.__options.parallelDegree)
gpEnv = GpMasterEnvironment(self.__options.masterDataDirectory, True)
faultProberInterface.getFaultProber().initializeProber(
gpEnv.getMasterPort())
confProvider = configInterface.getConfigurationProvider(
).initializeProvider(gpEnv.getMasterPort())
gpArray = confProvider.loadSystemConfig(useUtilityMode=False)
# check that heap_checksums is consistent across cluster, fail immediately if not
self.validate_heap_checksums(gpArray)
# check that we actually have mirrors
if gpArray.getFaultStrategy() != gparray.FAULT_STRATEGY_NONE:
raise ExceptionNoStackTraceNeeded( \
"GPDB physical mirroring cannot be added. The cluster is already configured with %s." % \
gparray.getFaultStrategyLabel(gpArray.getFaultStrategy()))
# figure out what needs to be done
mirrorBuilder = self.__getMirrorsToBuildBasedOnOptions(gpEnv, gpArray)
mirrorBuilder.checkForPortAndDirectoryConflicts(gpArray)
if self.__options.outputSampleConfigFile is not None:
# just output config file and done
self.__outputToFile(mirrorBuilder,
self.__options.outputSampleConfigFile, gpArray)
logger.info('Configuration file output to %s successfully.' %
self.__options.outputSampleConfigFile)
else:
self.__displayAddMirrors(gpEnv, mirrorBuilder, gpArray)
if self.__options.interactive:
if not userinput.ask_yesno(
None, "\nContinue with add mirrors procedure", 'N'):
raise UserAbortedException()
gpArray.setFaultStrategy(gparray.FAULT_STRATEGY_FILE_REPLICATION)
mirrorBuilder.buildMirrors("add", gpEnv, gpArray)
logger.info(
"******************************************************************"
)
logger.info(
"Mirror segments have been added; data synchronization is in progress."
)
logger.info(
"Data synchronization will continue in the background.")
logger.info("")
logger.info(
"Use gpstate -s to check the resynchronization progress.")
logger.info(
"******************************************************************"
)
return 0 # success -- exit code 0!
def _do_clean(self, token, batch_default):
if self.token is not None:
CleanVerification(token = self.token,
batch_default = self.batch_default).run()
else:
if not userinput.ask_yesno(None, "\nAre you sure you want to remove all completed verification artifacts across the cluster?", 'N'):
raise UserAbortedException()
CleanDoneVerifications(batch_default = self.batch_default).run()
def check_indexes(self, schema_name, table_name):
with dbconn.connect(dbconn.DbURL(dbname=self.database, port=self.port)) as conn:
c = execSQLForSingleton(conn, """SELECT count(*)
FROM pg_index
WHERE indrelid = (SELECT pg_class.oid
FROM pg_class, pg_namespace
WHERE pg_class.relname='{table}' AND pg_namespace.nspname='{schema}' AND pg_class.relnamespace = pg_namespace.oid)""".format(table=table_name, schema=schema_name))
if c != 0:
if self.interactive:
return ask_yesno(None,
'Table {schema}.{table} has indexes. This might slow down table reload. Do you still want to continue ?'
.format(schema=schema_name, table=table_name),
'N')
return True
def check_indexes(self, schema_name, table_name):
with dbconn.connect(dbconn.DbURL(dbname=self.database, port=self.port)) as conn:
c = execSQLForSingleton(conn, """SELECT count(*)
FROM pg_index
WHERE indrelid = (SELECT pg_class.oid
FROM pg_class, pg_namespace
WHERE pg_class.relname='{table}' AND pg_namespace.nspname='{schema}')
""".format(table=table_name, schema=schema_name))
if c != 0:
if self.interactive:
return ask_yesno(None,
'Table {schema}.{table} has indexes. This might slow down table reload. Do you still want to continue ?'
.format(schema=schema_name, table=table_name),
'N')
return True
def run(self):
if self.__options.parallelDegree < 1 or self.__options.parallelDegree > 64:
raise ProgramArgumentValidationException(
"Invalid parallelDegree provided with -B argument: %d" % self.__options.parallelDegree)
self.__pool = base.WorkerPool(self.__options.parallelDegree)
gpEnv = GpMasterEnvironment(self.__options.masterDataDirectory, True)
faultProberInterface.getFaultProber().initializeProber(gpEnv.getMasterPort())
confProvider = configInterface.getConfigurationProvider().initializeProvider(gpEnv.getMasterPort())
gpArray = confProvider.loadSystemConfig(useUtilityMode=False)
# check that heap_checksums is consistent across cluster, fail immediately if not
self.validate_heap_checksums(gpArray)
self.checkMirrorOffset(gpArray)
# check that we actually have mirrors
if gpArray.hasMirrors:
raise ExceptionNoStackTraceNeeded( \
"GPDB physical mirroring cannot be added. The cluster is already configured with Mirrors.")
# figure out what needs to be done (AND update the gpArray!)
mirrorBuilder = self.__getMirrorsToBuildBasedOnOptions(gpEnv, gpArray)
mirrorBuilder.checkForPortAndDirectoryConflicts(gpArray)
if self.__options.outputSampleConfigFile is not None:
# just output config file and done
self.__outputToFile(mirrorBuilder, self.__options.outputSampleConfigFile, gpArray)
logger.info('Configuration file output to %s successfully.' % self.__options.outputSampleConfigFile)
else:
self.__displayAddMirrors(gpEnv, mirrorBuilder, gpArray)
if self.__options.interactive:
if not userinput.ask_yesno(None, "\nContinue with add mirrors procedure", 'N'):
raise UserAbortedException()
self.config_primaries_for_replication(gpArray)
if not mirrorBuilder.buildMirrors("add", gpEnv, gpArray):
return 1
logger.info("******************************************************************")
logger.info("Mirror segments have been added; data synchronization is in progress.")
logger.info("Data synchronization will continue in the background.")
logger.info("Use gpstate -s to check the resynchronization progress.")
logger.info("******************************************************************")
return 0 # success -- exit code 0!
def run(self):
if self.__options.parallelDegree < 1 or self.__options.parallelDegree > 64:
raise ProgramArgumentValidationException(
"Invalid parallelDegree provided with -B argument: %d" % self.__options.parallelDegree
)
self.__pool = base.WorkerPool(self.__options.parallelDegree)
gpEnv = GpMasterEnvironment(self.__options.masterDataDirectory, True)
faultProberInterface.getFaultProber().initializeProber(gpEnv.getMasterPort())
confProvider = configInterface.getConfigurationProvider().initializeProvider(gpEnv.getMasterPort())
gpArray = confProvider.loadSystemConfig(useUtilityMode=False)
# check that we actually have mirrors
if gpArray.getFaultStrategy() != gparray.FAULT_STRATEGY_NONE:
raise ExceptionNoStackTraceNeeded(
"GPDB physical mirroring cannot be added. The cluster is already configured with %s."
% gparray.getFaultStrategyLabel(gpArray.getFaultStrategy())
)
# figure out what needs to be done
mirrorBuilder = self.__getMirrorsToBuildBasedOnOptions(gpEnv, gpArray)
mirrorBuilder.checkForPortAndDirectoryConflicts(gpArray)
if self.__options.outputSampleConfigFile is not None:
# just output config file and done
self.__outputToFile(mirrorBuilder, self.__options.outputSampleConfigFile, gpArray)
logger.info("Configuration file output to %s successfully." % self.__options.outputSampleConfigFile)
else:
self.__displayAddMirrors(gpEnv, mirrorBuilder, gpArray)
if self.__options.interactive:
if not userinput.ask_yesno(None, "\nContinue with add mirrors procedure", "N"):
raise UserAbortedException()
gpArray.setFaultStrategy(gparray.FAULT_STRATEGY_FILE_REPLICATION)
mirrorBuilder.buildMirrors("add", gpEnv, gpArray)
logger.info("******************************************************************")
logger.info("Mirror segments have been added; data synchronization is in progress.")
logger.info("Data synchronization will continue in the background.")
logger.info("")
logger.info("Use gpstate -s to check the resynchronization progress.")
logger.info("******************************************************************")
return 0 # success -- exit code 0!
def main():
options = parseargs()
database = find_gptext_schema()
if not database:
logger.error('Could not find schema %s.' % GPTEXT_SCHEMA)
logger.error('Use the --database option to specify the database that')
logger.error('contains the %s schema.' % GPTEXT_SCHEMA)
sys.exit(1)
url = dbconn.DbURL(dbname=database)
validate_solr_up(url)
validate_index_exists(url, options.index)
hosts = get_solr_instances(url)
if not userinput.ask_yesno(None, "\nContinue with GPText index '%s' expunge?" % options.index, 'N'):
logger.error ('User asked for termination')
sys.exit(1)
gptext_index_expunge(options.index, hosts)
logger.info('Done')
logger.info('To check the process use "ps -ef | grep curl" - it will show you all the expunge requests issued')
def main():
dburl = dbconn.DbURL(hostname="127.0.0.1", port=5432, dbname="template1", username="******")
hosts = getHostList(dburl)
logger.info("============= Cluster Parameters ==============")
logger.info("List of segment hosts:")
for h in hosts:
logger.info(" %s" % h)
segnum = getAmountOfHostsPerSegment(dburl)
logger.info("Number of segments per host is %d" % segnum)
masterram = getMasterRam()
logger.info("RAM size on master host: %d MB" % masterram)
logger.info("RAM size per segment host:")
ram = getRAMSize(hosts)
printGPTextJavaSetting(dburl)
logger.info("============== Proposed Settings ==============")
gptext_java = "-Xms120M -Xmx512M -Xloggc:logs/gc.log -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=5 -XX:GCLogFileSize=10M"
logger.info("GPText Java flags are: %s" % gptext_java)
if not userinput.ask_yesno(None, "\nContinue with GPText reconfiguration?", "N"):
logger.error("User asked for termination")
sys.exit(1)
tuneGPText(dburl, gptext_java)
logger.info("Tuning has finished. Now restart GPText to activate it")
pkg_file_list = ListFilesByPattern(GPPKG_ARCHIVE_PATH, self.remove + '*' + GPPKG_EXTENSION).run()
if len(pkg_file_list) == 0:
raise ExceptionNoStackTraceNeeded('Package %s has not been installed.' % self.remove)
# refuse to remove at all if the match is too broad, i.e., > 1
if len(pkg_file_list) > 1:
err_msg = "Remove request '%s' too broad. " \
"Multiple packages match remove request: ( %s )." % (self.remove, ", ".join(pkg_file_list))
raise ExceptionNoStackTraceNeeded(err_msg)
pkg_file = pkg_file_list[0]
pkg = Gppkg.from_package_path(os.path.join(GPPKG_ARCHIVE_PATH, pkg_file))
UninstallPackage(pkg, self.master_host, self.standby_host, self.segment_host_list).run()
elif self.update:
logger.warning('WARNING: The process of updating a package includes removing all')
logger.warning('previous versions of the system objects related to the package. For')
logger.warning('example, previous versions of shared libraries are removed.')
logger.warning('After the update process, a database function will fail when it is')
logger.warning('called if the function references a package file that has been removed.')
if self.interactive:
if not ask_yesno(None, 'Do you still want to continue ?', 'N'):
logger.info('Skipping update of gppkg based on user input')
return
pkg = Gppkg.from_package_path(self.update)
UpdatePackage(pkg, self.master_host, self.standby_host, self.segment_host_list).run()
elif self.clean:
CleanGppkg(self.standby_host, self.segment_host_list).run()
def cleanup(self):
pass
def run(self):
if self.build:
if self.filename:
BuildGppkg(self.build, self.filename).run()
else:
BuildGppkg(self.build, None).run()
return
if linux_distribution_id() == 'ubuntu':
try:
cmd = Command(name='Check for dpkg', cmdStr='dpkg --version')
cmd.run(validateAfter=True)
cmd = Command(name='Check for fakeroot',
cmdStr='fakeroot --version')
cmd.run(validateAfter=True)
except Exception as ex:
raise ExceptionNoStackTraceNeeded(
'fakeroot and dpkg are both required by gppkg')
else:
try:
cmd = Command(name='Check for rpm', cmdStr='rpm --version')
cmd.run(validateAfter=True)
results = cmd.get_results().stdout.strip()
rpm_version_string = results.split(' ')[-1]
if not rpm_version_string.startswith('4.'):
raise ExceptionNoStackTraceNeeded(
'gppkg requires rpm version 4.x')
except ExecutionError as ex:
results = ex.cmd.get_results().stderr.strip()
if len(results) != 0 and 'not found' in results:
raise ExceptionNoStackTraceNeeded(
'gppkg requires RPM to be available in PATH')
if self.coordinator_datadir is None:
self.coordinator_datadir = gp.get_coordinatordatadir()
self.coordinator_port = self._get_coordinator_port(
self.coordinator_datadir)
self._get_gpdb_host_list()
if self.migrate:
MigratePackages(from_gphome=self.migrate[0],
to_gphome=self.migrate[1],
standby_host=self.standby_host,
segment_host_list=self.segment_host_list).run()
return
if self.install:
pkg = Gppkg.from_package_path(self.install)
InstallPackage(pkg, self.coordinator_host, self.standby_host,
self.segment_host_list).run()
elif self.query:
query_type, package_path = self.query
QueryPackage(query_type, package_path).run()
elif self.remove:
# Check for exact match first, then use wildcard for what will be removed.
pkg_file_list = ListFilesByPattern(
GPPKG_ARCHIVE_PATH, self.remove + GPPKG_EXTENSION).run()
if len(pkg_file_list) == 0:
# now try wildcard
pkg_file_list = ListFilesByPattern(
GPPKG_ARCHIVE_PATH,
self.remove + '*' + GPPKG_EXTENSION).run()
if len(pkg_file_list) == 0:
raise ExceptionNoStackTraceNeeded(
'Package %s has not been installed.' % self.remove)
# refuse to remove at all if the match is too broad, i.e., > 1
if len(pkg_file_list) > 1:
err_msg = "Remove request '%s' too broad. " \
"Multiple packages match remove request: ( %s )." % (self.remove, ", ".join(pkg_file_list))
raise ExceptionNoStackTraceNeeded(err_msg)
pkg_file = pkg_file_list[0]
pkg = Gppkg.from_package_path(
os.path.join(GPPKG_ARCHIVE_PATH, pkg_file))
UninstallPackage(pkg, self.coordinator_host, self.standby_host,
self.segment_host_list).run()
elif self.update:
logger.warning(
'WARNING: The process of updating a package includes removing all'
)
logger.warning(
'previous versions of the system objects related to the package. For'
)
logger.warning(
'example, previous versions of shared libraries are removed.')
logger.warning(
'After the update process, a database function will fail when it is'
)
logger.warning(
'called if the function references a package file that has been removed.'
)
if self.interactive:
if not ask_yesno(None, 'Do you still want to continue ?', 'N'):
logger.info('Skipping update of gppkg based on user input')
return
pkg = Gppkg.from_package_path(self.update)
UpdatePackage(pkg, self.coordinator_host, self.standby_host,
self.segment_host_list).run()
elif self.clean:
CleanGppkg(self.standby_host, self.segment_host_list).run()
class GpRecoverSegmentProgram:
#
# Constructor:
#
# @param options the options as returned by the options parser
#
def __init__(self, options):
self.__options = options
self.__pool = None
self.logger = logger
def outputToFile(self, mirrorBuilder, gpArray, fileName):
lines = []
# one entry for each failure
for mirror in mirrorBuilder.getMirrorsToBuild():
output_str = ""
seg = mirror.getFailedSegment()
addr = canonicalize_address(seg.getSegmentAddress())
output_str += (
'%s:%d:%s' %
(addr, seg.getSegmentPort(), seg.getSegmentDataDirectory()))
seg = mirror.getFailoverSegment()
if seg is not None:
output_str += ' '
addr = canonicalize_address(seg.getSegmentAddress())
output_str += ('%s:%d:%s' % (addr, seg.getSegmentPort(),
seg.getSegmentDataDirectory()))
lines.append(output_str)
writeLinesToFile(fileName, lines)
def getRecoveryActionsFromConfigFile(self, gpArray):
"""
getRecoveryActionsFromConfigFile
returns: a tuple (segments in change tracking disabled mode which are unable to recover, GpMirrorListToBuild object
containing information of segments which are able to recover)
"""
# create fileData object from config file
#
filename = self.__options.recoveryConfigFile
rows = []
with open(filename) as f:
for lineno, line in line_reader(f):
fixed, flexible = parse_gprecoverseg_line(
filename, lineno, line)
rows.append(ParsedConfigFileRow(fixed, flexible, line))
fileData = ParsedConfigFile(rows)
allAddresses = [
row.getFixedValuesMap()["newAddress"]
for row in fileData.getRows()
if "newAddress" in row.getFixedValuesMap()
]
allNoneArr = [None] * len(allAddresses)
interfaceLookup = GpInterfaceToHostNameCache(self.__pool, allAddresses,
allNoneArr)
failedSegments = []
failoverSegments = []
for row in fileData.getRows():
fixedValues = row.getFixedValuesMap()
flexibleValues = row.getFlexibleValuesMap()
# find the failed segment
failedAddress = fixedValues['failedAddress']
failedPort = fixedValues['failedPort']
failedDataDirectory = normalizeAndValidateInputPath(
fixedValues['failedDataDirectory'], "config file",
row.getLine())
failedSegment = None
for segment in gpArray.getDbList():
if segment.getSegmentAddress() == failedAddress and \
str(segment.getSegmentPort()) == failedPort and \
segment.getSegmentDataDirectory() == failedDataDirectory:
if failedSegment is not None:
#
# this could be an assertion -- configuration should not allow multiple entries!
#
raise Exception((
"A segment to recover was found twice in configuration. "
"This segment is described by address:port:directory '%s:%s:%s' "
"on the input line: %s") %
(failedAddress, failedPort,
failedDataDirectory, row.getLine()))
failedSegment = segment
if failedSegment is None:
raise Exception("A segment to recover was not found in configuration. " \
"This segment is described by address:port:directory '%s:%s:%s' on the input line: %s" %
(failedAddress, failedPort, failedDataDirectory, row.getLine()))
failoverSegment = None
if "newAddress" in fixedValues:
"""
When the second set was passed, the caller is going to tell us to where we need to failover, so
build a failover segment
"""
# these two lines make it so that failoverSegment points to the object that is registered in gparray
failoverSegment = failedSegment
failedSegment = failoverSegment.copy()
address = fixedValues["newAddress"]
try:
port = int(fixedValues["newPort"])
except ValueError:
raise Exception(
'Config file format error, invalid number value in line: %s'
% (row.getLine()))
dataDirectory = normalizeAndValidateInputPath(
fixedValues["newDataDirectory"], "config file",
row.getLine())
hostName = interfaceLookup.getHostName(address)
if hostName is None:
raise Exception(
'Unable to find host name for address %s from line:%s'
% (address, row.getLine()))
# now update values in failover segment
failoverSegment.setSegmentAddress(address)
failoverSegment.setSegmentHostName(hostName)
failoverSegment.setSegmentPort(port)
failoverSegment.setSegmentDataDirectory(dataDirectory)
# this must come AFTER the if check above because failedSegment can be adjusted to
# point to a different object
failedSegments.append(failedSegment)
failoverSegments.append(failoverSegment)
peersForFailedSegments = self.findAndValidatePeersForFailedSegments(
gpArray, failedSegments)
segs = []
segs_with_persistent_mirroring_disabled = []
for index, failedSegment in enumerate(failedSegments):
peerForFailedSegment = peersForFailedSegments[index]
peerForFailedSegmentDbId = peerForFailedSegment.getSegmentDbId()
segs.append(
GpMirrorToBuild(failedSegment, peerForFailedSegment,
failoverSegments[index],
self.__options.forceFullResynchronization))
self._output_segments_with_persistent_mirroring_disabled(
segs_with_persistent_mirroring_disabled)
return GpMirrorListToBuild(segs, self.__pool, self.__options.quiet,
self.__options.parallelDegree)
def findAndValidatePeersForFailedSegments(self, gpArray, failedSegments):
dbIdToPeerMap = gpArray.getDbIdToPeerMap()
peersForFailedSegments = [
dbIdToPeerMap.get(seg.getSegmentDbId()) for seg in failedSegments
]
for i in range(len(failedSegments)):
peer = peersForFailedSegments[i]
if peer is None:
raise Exception("No peer found for dbid %s" %
failedSegments[i].getSegmentDbId())
elif peer.isSegmentDown():
raise Exception(
"Both segments for content %s are down; Try restarting Greenplum DB and running %s again."
% (peer.getSegmentContentId(), getProgramName()))
return peersForFailedSegments
def getRecoveryActionsFromConfiguration(self, gpEnv, gpArray):
"""
getRecoveryActionsFromConfiguration
returns: a tuple (segments in change tracking disabled mode which are unable to recover, GpMirrorListToBuild object
containing information of segments which are able to recover)
"""
segments = gpArray.getSegDbList()
failedSegments = [seg for seg in segments if seg.isSegmentDown()]
peersForFailedSegments = self.findAndValidatePeersForFailedSegments(
gpArray, failedSegments)
# Dictionaries used for building mapping to new hosts
recoverAddressMap = {}
recoverHostMap = {}
interfaceHostnameWarnings = []
# Check if the array is a "standard" array
(isStandardArray, _ignore) = gpArray.isStandardArray()
recoverHostIdx = 0
if self.__options.newRecoverHosts and len(
self.__options.newRecoverHosts) > 0:
for seg in failedSegments:
segAddress = seg.getSegmentAddress()
segHostname = seg.getSegmentHostName()
# Haven't seen this hostname before so we put it on a new host
if not recoverHostMap.has_key(segHostname):
try:
recoverHostMap[
segHostname] = self.__options.newRecoverHosts[
recoverHostIdx]
except:
# If we get here, not enough hosts were specified in the -p option. Need 1 new host
# per 1 failed host.
raise Exception(
'Not enough new recovery hosts given for recovery.'
)
recoverHostIdx += 1
if isStandardArray:
# We have a standard array configuration, so we'll try to use the same
# interface naming convention. If this doesn't work, we'll correct it
# below on name lookup
segInterface = segAddress[segAddress.rfind('-'):]
destAddress = recoverHostMap[segHostname] + segInterface
destHostname = recoverHostMap[segHostname]
else:
# Non standard configuration so we won't make assumptions on
# naming. Instead we'll use the hostname passed in for both
# hostname and address and flag for warning later.
destAddress = recoverHostMap[segHostname]
destHostname = recoverHostMap[segHostname]
# Save off the new host/address for this address.
recoverAddressMap[segAddress] = (destHostname, destAddress)
# Now that we've generated the mapping, look up all the addresses to make
# sure they are resolvable.
interfaces = [
address for (_ignore, address) in recoverAddressMap.values()
]
interfaceLookup = GpInterfaceToHostNameCache(
self.__pool, interfaces, [None] * len(interfaces))
for key in recoverAddressMap.keys():
(newHostname, newAddress) = recoverAddressMap[key]
try:
addressHostnameLookup = interfaceLookup.getHostName(
newAddress)
# Lookup failed so use hostname passed in for everything.
if addressHostnameLookup is None:
interfaceHostnameWarnings.append(
"Lookup of %s failed. Using %s for both hostname and address."
% (newAddress, newHostname))
newAddress = newHostname
except:
# Catch all exceptions. We will use hostname instead of address
# that we generated.
interfaceHostnameWarnings.append(
"Lookup of %s failed. Using %s for both hostname and address."
% (newAddress, newHostname))
newAddress = newHostname
# if we've updated the address to use the hostname because of lookup failure
# make sure the hostname is resolvable and up
if newHostname == newAddress:
try:
unix.Ping.local("ping new hostname", newHostname)
except:
raise Exception("Ping of host %s failed." %
newHostname)
# Save changes in map
recoverAddressMap[key] = (newHostname, newAddress)
if len(self.__options.newRecoverHosts) != recoverHostIdx:
interfaceHostnameWarnings.append(
"The following recovery hosts were not needed:")
for h in self.__options.newRecoverHosts[recoverHostIdx:]:
interfaceHostnameWarnings.append("\t%s" % h)
portAssigner = PortAssigner(gpArray)
forceFull = self.__options.forceFullResynchronization
segs = []
segs_with_persistent_mirroring_disabled = []
for i in range(len(failedSegments)):
failoverSegment = None
failedSegment = failedSegments[i]
liveSegment = peersForFailedSegments[i]
if self.__options.newRecoverHosts and len(
self.__options.newRecoverHosts) > 0:
(newRecoverHost, newRecoverAddress
) = recoverAddressMap[failedSegment.getSegmentAddress()]
# these two lines make it so that failoverSegment points to the object that is registered in gparray
failoverSegment = failedSegment
failedSegment = failoverSegment.copy()
failoverSegment.setSegmentHostName(newRecoverHost)
failoverSegment.setSegmentAddress(newRecoverAddress)
port = portAssigner.findAndReservePort(newRecoverHost,
newRecoverAddress)
failoverSegment.setSegmentPort(port)
segs.append(
GpMirrorToBuild(failedSegment, liveSegment, failoverSegment,
forceFull))
self._output_segments_with_persistent_mirroring_disabled(
segs_with_persistent_mirroring_disabled)
return GpMirrorListToBuild(segs, self.__pool, self.__options.quiet,
self.__options.parallelDegree,
interfaceHostnameWarnings)
def _output_segments_with_persistent_mirroring_disabled(
self, segs_persistent_mirroring_disabled=None):
if segs_persistent_mirroring_disabled:
self.logger.warn(
'Segments with dbid %s not recovered; persistent mirroring state is disabled.'
% (', '.join(
str(seg_id)
for seg_id in segs_persistent_mirroring_disabled)))
def getRecoveryActionsBasedOnOptions(self, gpEnv, gpArray):
if self.__options.rebalanceSegments:
return GpSegmentRebalanceOperation(gpEnv, gpArray)
elif self.__options.recoveryConfigFile is not None:
return self.getRecoveryActionsFromConfigFile(gpArray)
else:
return self.getRecoveryActionsFromConfiguration(gpEnv, gpArray)
def syncPackages(self, new_hosts):
# The design decision here is to squash any exceptions resulting from the
# synchronization of packages. We should *not* disturb the user's attempts to recover.
try:
self.logger.info('Syncing Greenplum Database extensions')
operations = [SyncPackages(host) for host in new_hosts]
ParallelOperation(operations, self.__options.parallelDegree).run()
# introspect outcomes
for operation in operations:
operation.get_ret()
except:
self.logger.exception(
'Syncing of Greenplum Database extensions has failed.')
self.logger.warning(
'Please run gppkg --clean after successful segment recovery.')
def displayRecovery(self, mirrorBuilder, gpArray):
self.logger.info('Greenplum instance recovery parameters')
self.logger.info(
'---------------------------------------------------------')
if self.__options.recoveryConfigFile:
self.logger.info('Recovery from configuration -i option supplied')
elif self.__options.newRecoverHosts is not None:
self.logger.info('Recovery type = Pool Host')
for h in self.__options.newRecoverHosts:
self.logger.info('Pool host for recovery = %s' % h)
elif self.__options.rebalanceSegments:
self.logger.info('Recovery type = Rebalance')
else:
self.logger.info('Recovery type = Standard')
if self.__options.rebalanceSegments:
i = 1
total = len(gpArray.get_unbalanced_segdbs())
for toRebalance in gpArray.get_unbalanced_segdbs():
tabLog = TableLogger()
self.logger.info(
'---------------------------------------------------------'
)
self.logger.info('Unbalanced segment %d of %d' % (i, total))
self.logger.info(
'---------------------------------------------------------'
)
programIoUtils.appendSegmentInfoForOutput(
"Unbalanced", gpArray, toRebalance, tabLog)
tabLog.info([
"Balanced role", "= Primary"
if toRebalance.preferred_role == 'p' else "= Mirror"
])
tabLog.info([
"Current role",
"= Primary" if toRebalance.role == 'p' else "= Mirror"
])
tabLog.outputTable()
i += 1
else:
i = 0
total = len(mirrorBuilder.getMirrorsToBuild())
for toRecover in mirrorBuilder.getMirrorsToBuild():
self.logger.info(
'---------------------------------------------------------'
)
self.logger.info('Recovery %d of %d' % (i + 1, total))
self.logger.info(
'---------------------------------------------------------'
)
tabLog = TableLogger()
syncMode = "Full" if toRecover.isFullSynchronization(
) else "Incremental"
tabLog.info(["Synchronization mode", "= " + syncMode])
programIoUtils.appendSegmentInfoForOutput(
"Failed", gpArray, toRecover.getFailedSegment(), tabLog)
programIoUtils.appendSegmentInfoForOutput(
"Recovery Source", gpArray, toRecover.getLiveSegment(),
tabLog)
if toRecover.getFailoverSegment() is not None:
programIoUtils.appendSegmentInfoForOutput(
"Recovery Target", gpArray,
toRecover.getFailoverSegment(), tabLog)
else:
tabLog.info(["Recovery Target", "= in-place"])
tabLog.outputTable()
i = i + 1
self.logger.info(
'---------------------------------------------------------')
def __getSimpleSegmentLabel(self, seg):
addr = canonicalize_address(seg.getSegmentAddress())
return "%s:%s" % (addr, seg.getSegmentDataDirectory())
def __displayRecoveryWarnings(self, mirrorBuilder):
for warning in self._getRecoveryWarnings(mirrorBuilder):
self.logger.warn(warning)
def _getRecoveryWarnings(self, mirrorBuilder):
"""
return an array of string warnings regarding the recovery
"""
res = []
for toRecover in mirrorBuilder.getMirrorsToBuild():
if toRecover.getFailoverSegment() is not None:
#
# user specified a failover location -- warn if it's the same host as its primary
#
src = toRecover.getLiveSegment()
dest = toRecover.getFailoverSegment()
if src.getSegmentHostName() == dest.getSegmentHostName():
res.append(
"Segment is being recovered to the same host as its primary: "
"primary %s failover target: %s" %
(self.__getSimpleSegmentLabel(src),
self.__getSimpleSegmentLabel(dest)))
for warning in mirrorBuilder.getAdditionalWarnings():
res.append(warning)
return res
def _get_dblist(self):
# template0 does not accept any connections so we exclude it
with dbconn.connect(dbconn.DbURL()) as conn:
res = dbconn.execSQL(
conn,
"SELECT datname FROM PG_DATABASE WHERE datname != 'template0'")
return res.fetchall()
def run(self):
if self.__options.parallelDegree < 1 or self.__options.parallelDegree > 64:
raise ProgramArgumentValidationException(
"Invalid parallelDegree provided with -B argument: %d" %
self.__options.parallelDegree)
self.__pool = WorkerPool(self.__options.parallelDegree)
gpEnv = GpMasterEnvironment(self.__options.masterDataDirectory, True)
# verify "where to recover" options
optionCnt = 0
if self.__options.newRecoverHosts is not None:
optionCnt += 1
if self.__options.recoveryConfigFile is not None:
optionCnt += 1
if self.__options.rebalanceSegments:
optionCnt += 1
if optionCnt > 1:
raise ProgramArgumentValidationException(
"Only one of -i, -p, and -r may be specified")
faultProberInterface.getFaultProber().initializeProber(
gpEnv.getMasterPort())
confProvider = configInterface.getConfigurationProvider(
).initializeProvider(gpEnv.getMasterPort())
gpArray = confProvider.loadSystemConfig(useUtilityMode=False)
if not gpArray.hasMirrors:
raise ExceptionNoStackTraceNeeded(
'GPDB Mirroring replication is not configured for this Greenplum Database instance.'
)
# We have phys-rep/filerep mirrors.
if self.__options.newRecoverHosts is not None:
try:
uniqueHosts = []
for h in self.__options.newRecoverHosts.split(','):
if h.strip() not in uniqueHosts:
uniqueHosts.append(h.strip())
self.__options.newRecoverHosts = uniqueHosts
except Exception, ex:
raise ProgramArgumentValidationException( \
"Invalid value for recover hosts: %s" % ex)
# If it's a rebalance operation, make sure we are in an acceptable state to do that
# Acceptable state is:
# - No segments down
# - No segments in change tracking or unsynchronized state
if self.__options.rebalanceSegments:
if len(gpArray.get_invalid_segdbs()) > 0:
raise Exception(
"Down segments still exist. All segments must be up to rebalance."
)
if len(gpArray.get_synchronized_segdbs()) != len(
gpArray.getSegDbList()):
raise Exception(
"Some segments are not yet synchronized. All segments must be synchronized to rebalance."
)
# retain list of hosts that were existing in the system prior to getRecoverActions...
# this will be needed for later calculations that determine whether
# new hosts were added into the system
existing_hosts = set(gpArray.getHostList())
# figure out what needs to be done
mirrorBuilder = self.getRecoveryActionsBasedOnOptions(gpEnv, gpArray)
if self.__options.outputSampleConfigFile is not None:
# just output config file and done
self.outputToFile(mirrorBuilder, gpArray,
self.__options.outputSampleConfigFile)
self.logger.info('Configuration file output to %s successfully.' %
self.__options.outputSampleConfigFile)
elif self.__options.rebalanceSegments:
assert (isinstance(mirrorBuilder, GpSegmentRebalanceOperation))
# Make sure we have work to do
if len(gpArray.get_unbalanced_segdbs()) == 0:
self.logger.info(
"No segments are running in their non-preferred role and need to be rebalanced."
)
else:
self.displayRecovery(mirrorBuilder, gpArray)
if self.__options.interactive:
self.logger.warn(
"This operation will cancel queries that are currently executing."
)
self.logger.warn(
"Connections to the database however will not be interrupted."
)
if not userinput.ask_yesno(
None,
"\nContinue with segment rebalance procedure",
'N'):
raise UserAbortedException()
fullRebalanceDone = mirrorBuilder.rebalance()
self.logger.info(
"******************************************************************"
)
if fullRebalanceDone:
self.logger.info(
"The rebalance operation has completed successfully.")
else:
self.logger.info(
"The rebalance operation has completed with WARNINGS."
" Please review the output in the gprecoverseg log.")
self.logger.info(
"There is a resynchronization running in the background to bring all"
)
self.logger.info("segments in sync.")
self.logger.info(
"Use gpstate -e to check the resynchronization progress.")
self.logger.info(
"******************************************************************"
)
elif len(mirrorBuilder.getMirrorsToBuild()) == 0:
self.logger.info('No segments to recover')
else:
mirrorBuilder.checkForPortAndDirectoryConflicts(gpArray)
self.validate_heap_checksum_consistency(gpArray, mirrorBuilder)
self.displayRecovery(mirrorBuilder, gpArray)
self.__displayRecoveryWarnings(mirrorBuilder)
if self.__options.interactive:
if not userinput.ask_yesno(
None, "\nContinue with segment recovery procedure",
'N'):
raise UserAbortedException()
# sync packages
current_hosts = set(gpArray.getHostList())
new_hosts = current_hosts - existing_hosts
if new_hosts:
self.syncPackages(new_hosts)
if not mirrorBuilder.buildMirrors("recover", gpEnv, gpArray):
sys.exit(1)
confProvider.sendPgElogFromMaster("Recovery of %d segment(s) has been started." % \
len(mirrorBuilder.getMirrorsToBuild()), True)
self.trigger_fts_probe(gpArray)
self.logger.info(
"******************************************************************"
)
self.logger.info("Updating segments for streaming is completed.")
self.logger.info(
"For segments updated successfully, streaming will continue in the background."
)
self.logger.info(
"Use gpstate -s to check the streaming progress.")
self.logger.info(
"******************************************************************"
)
sys.exit(0)
os.path.join(GPPKG_ARCHIVE_PATH, pkg_file))
UninstallPackage(pkg, self.master_host, self.standby_host,
self.segment_host_list).run()
elif self.update:
logger.warning(
'WARNING: The process of updating a package includes removing all'
)
logger.warning(
'previous versions of the system objects related to the package. For'
)
logger.warning(
'example, previous versions of shared libraries are removed.')
logger.warning(
'After the update process, a database function will fail when it is'
)
logger.warning(
'called if the function references a package file that has been removed.'
)
if self.interactive:
if not ask_yesno(None, 'Do you still want to continue ?', 'N'):
logger.info('Skipping update of gppkg based on user input')
return
pkg = Gppkg.from_package_path(self.update)
UpdatePackage(pkg, self.master_host, self.standby_host,
self.segment_host_list).run()
elif self.clean:
CleanGppkg(self.standby_host, self.segment_host_list).run()
def cleanup(self):
pass
def run(self):
if self.__options.batch_size < 1 or self.__options.batch_size > gp.MAX_COORDINATOR_NUM_WORKERS:
raise ProgramArgumentValidationException(
"Invalid batch_size provided with -B argument: %d" %
self.__options.batch_size)
if self.__options.segment_batch_size < 1 or self.__options.segment_batch_size > gp.MAX_SEGHOST_NUM_WORKERS:
raise ProgramArgumentValidationException(
"Invalid segment_batch_size provided with -b argument: %d" %
self.__options.segment_batch_size)
self.__pool = base.WorkerPool(self.__options.batch_size)
gpEnv = GpCoordinatorEnvironment(
self.__options.coordinatorDataDirectory, True)
faultProberInterface.getFaultProber().initializeProber(
gpEnv.getCoordinatorPort())
confProvider = configInterface.getConfigurationProvider(
).initializeProvider(gpEnv.getCoordinatorPort())
gpArray = confProvider.loadSystemConfig(useUtilityMode=False)
# check that heap_checksums is consistent across cluster, fail immediately if not
self.validate_heap_checksums(gpArray)
if self.__options.mirrorConfigFile is None:
self.checkMirrorOffset(gpArray)
# check that we actually have mirrors
if gpArray.hasMirrors:
raise ExceptionNoStackTraceNeeded( \
"GPDB physical mirroring cannot be added. The cluster is already configured with Mirrors.")
# figure out what needs to be done (AND update the gpArray!)
mirrorBuilder = self.__getMirrorsToBuildBasedOnOptions(gpEnv, gpArray)
mirrorBuilder.checkForPortAndDirectoryConflicts(gpArray)
if self.__options.outputSampleConfigFile is not None:
# just output config file and done
self.__outputToFile(mirrorBuilder,
self.__options.outputSampleConfigFile, gpArray)
logger.info('Configuration file output to %s successfully.' %
self.__options.outputSampleConfigFile)
else:
self.__displayAddMirrors(gpEnv, mirrorBuilder, gpArray)
if self.__options.interactive:
if not userinput.ask_yesno(
None, "\nContinue with add mirrors procedure", 'N'):
raise UserAbortedException()
update_pg_hba_on_segments(gpArray, self.__options.hba_hostnames,
self.__options.batch_size)
if not mirrorBuilder.buildMirrors("add", gpEnv, gpArray):
return 1
logger.info(
"******************************************************************"
)
logger.info(
"Mirror segments have been added; data synchronization is in progress."
)
logger.info(
"Data synchronization will continue in the background.")
logger.info(
"Use gpstate -s to check the resynchronization progress.")
logger.info(
"******************************************************************"
)
return 0 # success -- exit code 0!
def run(self):
if self.__options.parallelDegree < 1 or self.__options.parallelDegree > gp.MAX_COORDINATOR_NUM_WORKERS:
raise ProgramArgumentValidationException(
"Invalid parallelDegree value provided with -B argument: %d" % self.__options.parallelDegree)
if self.__options.parallelPerHost < 1 or self.__options.parallelPerHost > gp.MAX_SEGHOST_NUM_WORKERS:
raise ProgramArgumentValidationException(
"Invalid parallelPerHost value provided with -b argument: %d" % self.__options.parallelPerHost)
self.__pool = WorkerPool(self.__options.parallelDegree)
gpEnv = GpCoordinatorEnvironment(self.__options.coordinatorDataDirectory, True)
# verify "where to recover" options
optionCnt = 0
if self.__options.newRecoverHosts is not None:
optionCnt += 1
if self.__options.recoveryConfigFile is not None:
optionCnt += 1
if self.__options.rebalanceSegments:
optionCnt += 1
if optionCnt > 1:
raise ProgramArgumentValidationException("Only one of -i, -p, and -r may be specified")
faultProberInterface.getFaultProber().initializeProber(gpEnv.getCoordinatorPort())
confProvider = configInterface.getConfigurationProvider().initializeProvider(gpEnv.getCoordinatorPort())
gpArray = confProvider.loadSystemConfig(useUtilityMode=False)
if not gpArray.hasMirrors:
raise ExceptionNoStackTraceNeeded(
'GPDB Mirroring replication is not configured for this Greenplum Database instance.')
num_workers = min(len(gpArray.get_hostlist()), self.__options.parallelDegree)
hosts = set(gpArray.get_hostlist(includeCoordinator=False))
unreachable_hosts = get_unreachable_segment_hosts(hosts, num_workers)
update_unreachable_flag_for_segments(gpArray, unreachable_hosts)
# We have phys-rep/filerep mirrors.
if self.__options.newRecoverHosts is not None:
try:
uniqueHosts = []
for h in self.__options.newRecoverHosts.split(','):
if h.strip() not in uniqueHosts:
uniqueHosts.append(h.strip())
self.__options.newRecoverHosts = uniqueHosts
except Exception as ex:
raise ProgramArgumentValidationException( \
"Invalid value for recover hosts: %s" % ex)
# retain list of hosts that were existing in the system prior to getRecoverActions...
# this will be needed for later calculations that determine whether
# new hosts were added into the system
existing_hosts = set(gpArray.getHostList())
# figure out what needs to be done
mirrorBuilder = self.getRecoveryActionsBasedOnOptions(gpEnv, gpArray)
if self.__options.outputSampleConfigFile is not None:
# just output config file and done
self.outputToFile(mirrorBuilder, gpArray, self.__options.outputSampleConfigFile)
self.logger.info('Configuration file output to %s successfully.' % self.__options.outputSampleConfigFile)
elif self.__options.rebalanceSegments:
assert (isinstance(mirrorBuilder, GpSegmentRebalanceOperation))
# Make sure we have work to do
if len(gpArray.get_unbalanced_segdbs()) == 0:
self.logger.info("No segments are running in their non-preferred role and need to be rebalanced.")
else:
self.displayRecovery(mirrorBuilder, gpArray)
if self.__options.interactive:
self.logger.warn("This operation will cancel queries that are currently executing.")
self.logger.warn("Connections to the database however will not be interrupted.")
if not userinput.ask_yesno(None, "\nContinue with segment rebalance procedure", 'N'):
raise UserAbortedException()
fullRebalanceDone = mirrorBuilder.rebalance()
self.logger.info("******************************************************************")
if fullRebalanceDone:
self.logger.info("The rebalance operation has completed successfully.")
else:
self.logger.info("The rebalance operation has completed with WARNINGS."
" Please review the output in the gprecoverseg log.")
self.logger.info("******************************************************************")
elif len(mirrorBuilder.getMirrorsToBuild()) == 0:
self.logger.info('No segments to recover')
else:
#TODO this already happens in buildMirrors function
mirrorBuilder.checkForPortAndDirectoryConflicts(gpArray)
self.validate_heap_checksum_consistency(gpArray, mirrorBuilder)
self.displayRecovery(mirrorBuilder, gpArray)
self.__displayRecoveryWarnings(mirrorBuilder)
if self.__options.interactive:
if not userinput.ask_yesno(None, "\nContinue with segment recovery procedure", 'N'):
raise UserAbortedException()
# sync packages
current_hosts = set(gpArray.getHostList())
new_hosts = current_hosts - existing_hosts
if new_hosts:
self.syncPackages(new_hosts)
contentsToUpdate = [seg.getLiveSegment().getSegmentContentId() for seg in mirrorBuilder.getMirrorsToBuild()]
update_pg_hba_on_segments(gpArray, self.__options.hba_hostnames, self.__options.parallelDegree, contentsToUpdate)
if not mirrorBuilder.recover_mirrors(gpEnv, gpArray):
self.logger.error("gprecoverseg failed. Please check the output for more details.")
sys.exit(1)
self.logger.info("********************************")
self.logger.info("Segments successfully recovered.")
self.logger.info("********************************")
self.logger.info("Recovered mirror segments need to sync WAL with primary segments.")
self.logger.info("Use 'gpstate -e' to check progress of WAL sync remaining bytes")
sys.exit(0)
def main():
options = parseargs()
dburl = dbconn.DbURL(hostname = '127.0.0.1',
port = 5432,
dbname = 'template1',
username = '******')
hosts = getHostList(dburl)
logger.info ('========== Cluster Hardware Parameters =========')
logger.info ('List of segment hosts:')
for h in hosts:
logger.info (' %s' % h)
segnum = getAmountOfHostsPerSegment(dburl)
logger.info ('Number of segments per host is %d' % segnum)
masterram = getMasterRam()
logger.info ('RAM size on master host: %d MB' % masterram)
logger.info ('RAM size per segment host:')
ram = getRAMSize(hosts)
mastercores = getMasterCPUCoreNumber()
logger.info ('Number of CPU cores on master host: %d' % mastercores)
logger.info ('Number of CPU cores per segment host:')
cores = getCPUCoreNumber(hosts)
logger.info ('========== Current Greenplum Settings ==========')
printCurrentSettings(dburl)
logger.info ('========== Tune Application Constants ==========')
logger.info ('Part of the host memory allocated to GPDB and GPText: %.2f' % allocation_rate)
logger.info ('Memory split between GPDB and GPText: %.2f' % memory_split)
logger.info ('Part of the CPU allocated to GPDB: %.2f' % cpu_split)
logger.info ('============== Proposed Settings ==============')
usable_ram = int(ram*allocation_rate)
logger.info ('Amount of memory allocated to GPDB and GPText: %d MB' % usable_ram)
gptext_ram = int(float(usable_ram)*memory_split)/segnum/2
logger.info ('GPText memory setting is: %d MB' % gptext_ram)
if gptext_ram < 1000:
logger.error ('GPText has less than 1GB of RAM per process. This configuration will not work. Reduce number of segments or adjust memory split')
sys.exit(1)
if gptext_ram < 2000:
logger.warning ('GPText has less than 2GB of RAM per process. This configuration is not recommended')
gpdb_ram = int(float(usable_ram)*(1.0 - memory_split))/segnum
logger.info ('GPDB memory setting is %d MB for segment and %d MB for master' % (gpdb_ram, int(float(masterram)*allocation_rate)))
if gpdb_ram < 2000:
logger.error ('GPDB has less than 2GB of RAM per primary segment. This configuration will not work. Reduce number of segments or adjust memory split')
sys.exit(1)
if gpdb_ram < 4000:
logger.warning ('GPDB has less than 4GB of RAM per primary segment. This configuration is not recommended')
cpu_segment = float(cores) / float(segnum) * cpu_split
cpu_master = min(24, max(1, mastercores - 1))
logger.info ('GPDB cpu cores per segment is %.1f' % cpu_segment)
logger.info ('GPDB cpu cores for master is %d' % cpu_master)
if cpu_segment < 1.0:
logger.error ('GPDB has less than 1 CPU core per each primary segment. This configuration will not work. Reduce number of segments or adjust CPU split')
sys.exit(1)
if cpu_segment < 2.0:
logger.warning ('GPDB has less than 2 CPU cores per each primary segment. This configuration is not recommended')
if not options.force:
if not userinput.ask_yesno(None, "\nContinue with memory and cpu reconfiguration?", 'N'):
logger.error ('User asked for termination')
sys.exit(1)
tuneGPDBCPU (cpu_segment, cpu_master)
tuneGPDBRAM (gpdb_ram, int(float(masterram)*allocation_rate))
tuneGPTextRAM(dburl, gptext_ram)
logger.info ('Tuning finished. Now restart GPDB and GPText to activate it')
def run(self):
if self.__options.parallelDegree < 1 or self.__options.parallelDegree > 64:
raise ProgramArgumentValidationException(
"Invalid parallelDegree provided with -B argument: %d" % self.__options.parallelDegree)
self.__pool = WorkerPool(self.__options.parallelDegree)
gpEnv = GpCoordinatorEnvironment(self.__options.coordinatorDataDirectory, True)
# verify "where to recover" options
optionCnt = 0
if self.__options.newRecoverHosts is not None:
optionCnt += 1
if self.__options.recoveryConfigFile is not None:
optionCnt += 1
if self.__options.rebalanceSegments:
optionCnt += 1
if optionCnt > 1:
raise ProgramArgumentValidationException("Only one of -i, -p, and -r may be specified")
faultProberInterface.getFaultProber().initializeProber(gpEnv.getCoordinatorPort())
confProvider = configInterface.getConfigurationProvider().initializeProvider(gpEnv.getCoordinatorPort())
gpArray = confProvider.loadSystemConfig(useUtilityMode=False)
num_workers = min(len(gpArray.get_hostlist()), self.__options.parallelDegree)
hosts = set(gpArray.get_hostlist(includeCoordinator=False))
unreachable_hosts = get_unreachable_segment_hosts(hosts, num_workers)
for i, segmentPair in enumerate(gpArray.segmentPairs):
if segmentPair.primaryDB.getSegmentHostName() in unreachable_hosts:
logger.warning("Not recovering segment %d because %s is unreachable" % (segmentPair.primaryDB.dbid, segmentPair.primaryDB.getSegmentHostName()))
gpArray.segmentPairs[i].primaryDB.unreachable = True
if segmentPair.mirrorDB.getSegmentHostName() in unreachable_hosts:
logger.warning("Not recovering segment %d because %s is unreachable" % (segmentPair.mirrorDB.dbid, segmentPair.mirrorDB.getSegmentHostName()))
gpArray.segmentPairs[i].mirrorDB.unreachable = True
if not gpArray.hasMirrors:
raise ExceptionNoStackTraceNeeded(
'GPDB Mirroring replication is not configured for this Greenplum Database instance.')
# We have phys-rep/filerep mirrors.
if self.__options.newRecoverHosts is not None:
try:
uniqueHosts = []
for h in self.__options.newRecoverHosts.split(','):
if h.strip() not in uniqueHosts:
uniqueHosts.append(h.strip())
self.__options.newRecoverHosts = uniqueHosts
except Exception as ex:
raise ProgramArgumentValidationException( \
"Invalid value for recover hosts: %s" % ex)
# If it's a rebalance operation, make sure we are in an acceptable state to do that
# Acceptable state is:
# - No segments down
# - No segments in change tracking or unsynchronized state
if self.__options.rebalanceSegments:
if len(gpArray.get_invalid_segdbs()) > 0:
raise Exception("Down segments still exist. All segments must be up to rebalance.")
if len(gpArray.get_synchronized_segdbs()) != len(gpArray.getSegDbList()):
raise Exception(
"Some segments are not yet synchronized. All segments must be synchronized to rebalance.")
# retain list of hosts that were existing in the system prior to getRecoverActions...
# this will be needed for later calculations that determine whether
# new hosts were added into the system
existing_hosts = set(gpArray.getHostList())
# figure out what needs to be done
mirrorBuilder = self.getRecoveryActionsBasedOnOptions(gpEnv, gpArray)
if self.__options.outputSampleConfigFile is not None:
# just output config file and done
self.outputToFile(mirrorBuilder, gpArray, self.__options.outputSampleConfigFile)
self.logger.info('Configuration file output to %s successfully.' % self.__options.outputSampleConfigFile)
elif self.__options.rebalanceSegments:
assert (isinstance(mirrorBuilder, GpSegmentRebalanceOperation))
# Make sure we have work to do
if len(gpArray.get_unbalanced_segdbs()) == 0:
self.logger.info("No segments are running in their non-preferred role and need to be rebalanced.")
else:
self.displayRecovery(mirrorBuilder, gpArray)
if self.__options.interactive:
self.logger.warn("This operation will cancel queries that are currently executing.")
self.logger.warn("Connections to the database however will not be interrupted.")
if not userinput.ask_yesno(None, "\nContinue with segment rebalance procedure", 'N'):
raise UserAbortedException()
fullRebalanceDone = mirrorBuilder.rebalance()
self.logger.info("******************************************************************")
if fullRebalanceDone:
self.logger.info("The rebalance operation has completed successfully.")
else:
self.logger.info("The rebalance operation has completed with WARNINGS."
" Please review the output in the gprecoverseg log.")
self.logger.info("There is a resynchronization running in the background to bring all")
self.logger.info("segments in sync.")
self.logger.info("Use gpstate -e to check the resynchronization progress.")
self.logger.info("******************************************************************")
elif len(mirrorBuilder.getMirrorsToBuild()) == 0:
self.logger.info('No segments to recover')
else:
mirrorBuilder.checkForPortAndDirectoryConflicts(gpArray)
self.validate_heap_checksum_consistency(gpArray, mirrorBuilder)
self.displayRecovery(mirrorBuilder, gpArray)
self.__displayRecoveryWarnings(mirrorBuilder)
if self.__options.interactive:
if not userinput.ask_yesno(None, "\nContinue with segment recovery procedure", 'N'):
raise UserAbortedException()
# sync packages
current_hosts = set(gpArray.getHostList())
new_hosts = current_hosts - existing_hosts
if new_hosts:
self.syncPackages(new_hosts)
config_primaries_for_replication(gpArray, self.__options.hba_hostnames)
if not mirrorBuilder.buildMirrors("recover", gpEnv, gpArray):
sys.exit(1)
self.trigger_fts_probe(port=gpEnv.getCoordinatorPort())
self.logger.info("******************************************************************")
self.logger.info("Updating segments for streaming is completed.")
self.logger.info("For segments updated successfully, streaming will continue in the background.")
self.logger.info("Use gpstate -s to check the streaming progress.")
self.logger.info("******************************************************************")
sys.exit(0)
class GpPkgProgram:
""" This is the CLI entry point to package management code. """
def __init__(self, options, args):
self.master_datadir = options.masterDataDirectory
# TODO: AK: Program logic should not be dictating master, standby, and segment information
# In other words, the fundamental Operations should have APIs that preclude the need for this.
self.master_host = None
self.standby_host = None
self.segment_host_list = None
self.query = options.query
self.build = options.build
self.install = options.install
self.remove = options.remove
self.update = options.update
self.clean = options.clean
self.migrate = options.migrate
self.interactive = options.interactive
self.filename = options.filename
# only one of the following may be provided: --install, --remove, --update, --query, --build, --clean, --migrate
count = sum([
1 for opt in [
'install', 'remove', 'update', 'query', 'build', 'clean',
'migrate'
] if getattr(self, opt)
])
if count != 1:
raise ExceptionNoStackTraceNeeded(
'Exactly one of the following must be provided: --install, --remove, -update, --query, --clean, --migrate'
)
if self.query:
# gppkg -q can be supplemented with --info, --list, --all
count = sum([
1 for opt in ['info', 'list', 'all'] if options.__dict__[opt]
])
if count > 1:
raise ExceptionNoStackTraceNeeded(
'For --query, at most one of the following can be provided: --info, --list, --all'
)
# for all query options other than --all, a package path must be provided
if not options.all and len(args) != 1:
raise ExceptionNoStackTraceNeeded(
'A package must be specified for -q, -q --info, and -q --list.'
)
if options.info:
self.query = (QueryPackage.INFO, args[0])
elif options.list:
self.query = (QueryPackage.LIST, args[0])
elif options.all:
self.query = (QueryPackage.ALL, None)
else:
self.query = (None, args[0])
elif self.migrate:
if len(args) != 2:
raise ExceptionNoStackTraceNeeded(
'Invalid syntax, expecting "gppkg --migrate <from_gphome> <to_gphome>".'
)
self.migrate = (args[0], args[1])
# gppkg should check gpexpand status
check_result, msg = gp.conflict_with_gpexpand("gppkg",
refuse_phase1=True,
refuse_phase2=False)
if not check_result:
raise ExceptionNoStackTraceNeeded(msg)
@staticmethod
def create_parser():
parser = OptParser(option_class=OptChecker,
description="Greenplum Package Manager",
version='%prog version $Revision: #1 $')
parser.setHelp([])
addStandardLoggingAndHelpOptions(parser,
includeNonInteractiveOption=True)
parser.remove_option('-q')
parser.remove_option('-l')
add_to = OptionGroup(parser, 'General Options')
parser.add_option_group(add_to)
addMasterDirectoryOptionForSingleClusterProgram(add_to)
# TODO: AK: Eventually, these options may need to be flexible enough to accept mutiple packages
# in one invocation. If so, the structure of this parser may need to change.
add_to.add_option('-i',
'--install',
help='install the given gppkg',
metavar='<package>')
add_to.add_option('-u',
'--update',
help='update the given gppkg',
metavar='<package>')
add_to.add_option('-r',
'--remove',
help='remove the given gppkg',
metavar='<name>-<version>')
add_to.add_option(
'-q',
'--query',
help='query the gppkg database or a particular gppkg',
action='store_true')
add_to.add_option('-b',
'--build',
help='build a gppkg',
metavar='<directory>')
add_to.add_option('-c',
'--clean',
help='clean the cluster of the given gppkg',
action='store_true')
add_to.add_option('--migrate',
help='migrate gppkgs from a separate $GPHOME',
metavar='<from_gphome> <to_gphome>',
action='store_true',
default=False)
add_to.add_option('-f',
'--filename',
help='set specific package name',
metavar='<name>')
add_to = OptionGroup(parser, 'Query Options')
parser.add_option_group(add_to)
add_to.add_option(
'--info',
action='store_true',
help=
'print information about the gppkg including name, version, description'
)
add_to.add_option('--list',
action='store_true',
help='print all the files present in the gppkg')
add_to.add_option('--all',
action='store_true',
help='print all the gppkgs installed by gppkg')
return parser
@staticmethod
def create_program(options, args):
""" TODO: AK: This convention may be unnecessary. """
return GpPkgProgram(options, args)
def _get_gpdb_host_list(self):
"""
TODO: Perhaps the host list should be produced by gparray instead of here.
This method gets the host names
of all hosts in the gpdb array.
It sets the following variables
GpPkgProgram.master_host to master
GpPkgProgram.standby_host to standby
GpPkgProgram.segment_host_list to segment hosts
"""
logger.debug('_get_gpdb_host_list')
gparr = GpArray.initFromCatalog(dbconn.DbURL(port=self.master_port),
utility=True)
master_host = None
standby_host = None
segment_host_list = []
segs = gparr.getDbList()
for seg in segs:
if seg.isSegmentMaster(current_role=True):
master_host = seg.getSegmentHostName()
elif seg.isSegmentStandby(current_role=True):
standby_host = seg.getSegmentHostName()
else:
segment_host_list.append(seg.getSegmentHostName())
# Deduplicate the hosts so that we
# dont install multiple times on the same host
segment_host_list = list(set(segment_host_list))
# Segments might exist on the master host. Since we store the
# master host separately in self.master_host, storing the master_host
# in the segment_host_list is redundant.
for host in segment_host_list:
if host == master_host or host == standby_host:
segment_host_list.remove(host)
self.master_host = master_host
self.standby_host = standby_host
self.segment_host_list = segment_host_list
def _get_master_port(self, datadir):
'''
Obtain the master port from the pgconf file
'''
logger.debug('_get_master_port')
pgconf_dict = pgconf.readfile(os.path.join(datadir, 'postgresql.conf'))
return pgconf_dict.int('port') or os.getenv('PGPORT')
def run(self):
if self.build:
if self.filename:
BuildGppkg(self.build, self.filename).run()
else:
BuildGppkg(self.build, None).run()
return
#Check for RPM and Solaris OS
if curr_platform == SUNOS:
raise ExceptionNoStackTraceNeeded(
'gppkg is not supported on Solaris')
try:
if platform.linux_distribution()[0] == 'Ubuntu':
cmd = Command(name='Check for dpkg', cmdStr='dpkg --version')
cmd.run(validateAfter=True)
else:
cmd = Command(name='Check for rpm', cmdStr='rpm --version')
cmd.run(validateAfter=True)
results = cmd.get_results().stdout.strip()
rpm_version_string = results.split(' ')[-1]
if not rpm_version_string.startswith('4.'):
raise ExceptionNoStackTraceNeeded(
'gppkg requires rpm version 4.x')
except ExecutionError, ex:
results = ex.cmd.get_results().stderr.strip()
if len(results) != 0 and 'not found' in results:
raise ExceptionNoStackTraceNeeded(
'gppkg requires RPM to be available in PATH')
if self.master_datadir is None:
self.master_datadir = gp.get_masterdatadir()
self.master_port = self._get_master_port(self.master_datadir)
self._get_gpdb_host_list()
if self.migrate:
MigratePackages(from_gphome=self.migrate[0],
to_gphome=self.migrate[1],
standby_host=self.standby_host,
segment_host_list=self.segment_host_list).run()
return
if self.install:
pkg = Gppkg.from_package_path(self.install)
InstallPackage(pkg, self.master_host, self.standby_host,
self.segment_host_list).run()
elif self.query:
query_type, package_path = self.query
QueryPackage(query_type, package_path).run()
elif self.remove:
# Check for exact match first, then use wildcard for what will be removed.
pkg_file_list = ListFilesByPattern(
GPPKG_ARCHIVE_PATH, self.remove + GPPKG_EXTENSION).run()
if len(pkg_file_list) == 0:
# now try wildcard
pkg_file_list = ListFilesByPattern(
GPPKG_ARCHIVE_PATH,
self.remove + '*' + GPPKG_EXTENSION).run()
if len(pkg_file_list) == 0:
raise ExceptionNoStackTraceNeeded(
'Package %s has not been installed.' % self.remove)
# refuse to remove at all if the match is too broad, i.e., > 1
if len(pkg_file_list) > 1:
err_msg = "Remove request '%s' too broad. " \
"Multiple packages match remove request: ( %s )." % (self.remove, ", ".join(pkg_file_list))
raise ExceptionNoStackTraceNeeded(err_msg)
pkg_file = pkg_file_list[0]
pkg = Gppkg.from_package_path(
os.path.join(GPPKG_ARCHIVE_PATH, pkg_file))
UninstallPackage(pkg, self.master_host, self.standby_host,
self.segment_host_list).run()
elif self.update:
logger.warning(
'WARNING: The process of updating a package includes removing all'
)
logger.warning(
'previous versions of the system objects related to the package. For'
)
logger.warning(
'example, previous versions of shared libraries are removed.')
logger.warning(
'After the update process, a database function will fail when it is'
)
logger.warning(
'called if the function references a package file that has been removed.'
)
if self.interactive:
if not ask_yesno(None, 'Do you still want to continue ?', 'N'):
logger.info('Skipping update of gppkg based on user input')
return
pkg = Gppkg.from_package_path(self.update)
UpdatePackage(pkg, self.master_host, self.standby_host,
self.segment_host_list).run()
elif self.clean:
CleanGppkg(self.standby_host, self.segment_host_list).run()