def __init__(self, caller, filename, lineno, line):
"Initialize"
(self.caller, self.filename, self.lineno, self.line, self.rest,
self.error) = (caller, filename, lineno, line, line, None)
self.logger = logger
if logging_is_verbose():
self.logger.debug("%s:%s" % (filename, lineno))
def run_pg_rewind(self, rewindInfo):
"""
Run pg_rewind for incremental recovery.
"""
rewindFailedSegments = []
# Run pg_rewind on all the targets
for rewindSeg in list(rewindInfo.values()):
# Do CHECKPOINT on source to force TimeLineID to be updated in pg_control.
# pg_rewind wants that to make incremental recovery successful finally.
self.__logger.debug(
'Do CHECKPOINT on %s (port: %d) before running pg_rewind.' %
(rewindSeg.sourceHostname, rewindSeg.sourcePort))
dburl = dbconn.DbURL(hostname=rewindSeg.sourceHostname,
port=rewindSeg.sourcePort,
dbname='template1')
conn = dbconn.connect(dburl, utility=True)
dbconn.execSQL(conn, "CHECKPOINT")
conn.close()
# If the postmaster.pid still exists and another process
# is actively using that pid, pg_rewind will fail when it
# tries to start the failed segment in single-user
# mode. It should be safe to remove the postmaster.pid
# file since we do not expect the failed segment to be up.
self.remove_postmaster_pid_from_remotehost(
rewindSeg.targetSegment.getSegmentHostName(),
rewindSeg.targetSegment.getSegmentDataDirectory())
# Note the command name, we use the dbid later to
# correlate the command results with GpMirrorToBuild
# object.
cmd = gp.SegmentRewind(
'rewind dbid: %s' % rewindSeg.targetSegment.getSegmentDbId(),
rewindSeg.targetSegment.getSegmentHostName(),
rewindSeg.targetSegment.getSegmentDataDirectory(),
rewindSeg.sourceHostname,
rewindSeg.sourcePort,
verbose=gplog.logging_is_verbose())
self.__pool.addCommand(cmd)
if self.__quiet:
self.__pool.join()
else:
base.join_and_indicate_progress(self.__pool)
for cmd in self.__pool.getCompletedItems():
self.__logger.debug('pg_rewind results: %s' % cmd.results)
if not cmd.was_successful():
dbid = int(cmd.name.split(':')[1].strip())
self.__logger.debug("%s failed" % cmd.name)
self.__logger.warning(cmd.get_stdout())
self.__logger.warning(
"Incremental recovery failed for dbid %d. You must use gprecoverseg -F to recover the segment."
% dbid)
rewindFailedSegments.append(rewindInfo[dbid].targetSegment)
self.__pool.empty_completed_items()
return rewindFailedSegments
def _stop_failed_segments(self, gpEnv):
failed_reachable_segments = self._get_failed_reachable_segments()
if len(failed_reachable_segments) == 0:
return
self.__logger.info("Ensuring %d failed segment(s) are stopped" %
(len(failed_reachable_segments)))
segments = self._get_running_postgres_segments(
failed_reachable_segments)
segmentByHost = GpArray.getSegmentsByHostName(segments)
cmds = []
for hostName, segments in segmentByHost.items():
cmd = gp.GpSegStopCmd("remote segment stop on host '%s'" %
hostName,
gpEnv.getGpHome(),
gpEnv.getGpVersion(),
mode='fast',
dbs=segments,
verbose=gplog.logging_is_verbose(),
ctxt=base.REMOTE,
remoteHost=hostName,
segment_batch_size=self.__parallelPerHost)
cmds.append(cmd)
# we suppress checking for the error. This is because gpsegstop will actually error
# in many cases where the stop is actually done (that is, for example, the segment is
# running but slow to shutdown so gpsegstop errors after whacking it with a kill)
#
# Perhaps we should make it so that it so that is checks if the seg is running and only attempt stop
# if it's running? In that case, we could propagate the error
#
self.__runWaitAndCheckWorkerPoolForErrorsAndClear(
cmds, suppressErrorCheck=True)
def __ensureStopped(self, gpEnv, directives):
"""
@param directives a list of the GpStopSegmentDirectoryDirective values indicating which segments to stop
"""
if len(directives) == 0:
return
self.__logger.info("Ensuring %d failed segment(s) are stopped" % (len(directives)))
segments = [d.getSegment() for d in directives]
segments = self._get_running_postgres_segments(segments)
segmentByHost = GpArray.getSegmentsByHostName(segments)
cmds = []
for hostName, segments in segmentByHost.iteritems():
cmd = gp.GpSegStopCmd("remote segment stop on host '%s'" % hostName,
gpEnv.getGpHome(), gpEnv.getGpVersion(),
mode='fast', dbs=segments, verbose=gplog.logging_is_verbose(),
ctxt=base.REMOTE, remoteHost=hostName)
cmds.append(cmd)
# we suppress checking for the error. This is because gpsegstop will actually error
# in many cases where the stop is actually done (that is, for example, the segment is
# running but slow to shutdown so gpsegstop errors after whacking it with a kill)
#
# Perhaps we should make it so that it so that is checks if the seg is running and only attempt stop
# if it's running? In that case, we could propagate the error
#
self.__runWaitAndCheckWorkerPoolForErrorsAndClear(cmds, "stopping segments", suppressErrorCheck=True)
def __ensureStopped(self, gpEnv, directives):
"""
@param directives a list of the GpStopSegmentDirectoryDirective values indicating which segments to stop
"""
if len(directives) == 0:
return
self.__logger.info("Ensuring %d failed segment(s) are stopped" % (len(directives)))
segments = [d.getSegment() for d in directives]
segments = self._get_running_postgres_segments(segments)
segmentByHost = GpArray.getSegmentsByHostName(segments)
cmds = []
for hostName, segments in segmentByHost.iteritems():
cmd = gp.GpSegStopCmd("remote segment stop on host '%s'" % hostName,
gpEnv.getGpHome(), gpEnv.getGpVersion(),
mode='fast', dbs=segments, verbose=gplog.logging_is_verbose(),
ctxt=base.REMOTE, remoteHost=hostName)
cmds.append(cmd)
# we suppress checking for the error. This is because gpsegstop will actually error
# in many cases where the stop is actually done (that is, for example, the segment is
# running but slow to shutdown so gpsegstop errors after whacking it with a kill)
#
# Perhaps we should make it so that it so that is checks if the seg is running and only attempt stop
# if it's running? In that case, we could propagate the error
#
self.__runWaitAndCheckWorkerPoolForErrorsAndClear(cmds, "stopping segments", suppressErrorCheck=True)
def run_pg_rewind(self, rewindInfo):
"""
Run pg_rewind for incremental recovery.
"""
rewindFailedSegments = []
# Run pg_rewind on all the targets
for targetSegment, sourceHostName, sourcePort in rewindInfo:
cmd = gp.SegmentRewind('segment rewind',
targetSegment.getSegmentHostName(),
targetSegment.getSegmentDataDirectory(),
sourceHostName,
sourcePort,
verbose=gplog.logging_is_verbose())
try:
cmd.run(True)
self.__logger.debug('pg_rewind results: %s' % cmd.results)
except base.ExecutionError as e:
self.__logger.debug(
"pg_rewind failed for target directory %s." %
targetSegment.getSegmentDataDirectory())
self.__logger.warning(
"Incremental recovery failed for dbid %s. You must use gprecoverseg -F to recover the segment."
% targetSegment.getSegmentDbId())
rewindFailedSegments.append(targetSegment)
return rewindFailedSegments
def createConfigureNewSegmentCommand(hostName, cmdLabel, validationOnly):
segmentInfo = newSegmentInfo[hostName]
checkNotNone("segmentInfo for %s" % hostName, segmentInfo)
return gp.ConfigureNewSegment(cmdLabel,
segmentInfo,
tarFile=tarFileName,
newSegments=True,
verbose=gplog.logging_is_verbose(),
batchSize=self.__parallelDegree,
ctxt=gp.REMOTE,
remoteHost=hostName,
validationOnly=validationOnly)
def createConfigureNewSegmentCommand(hostName, cmdLabel, validationOnly):
segmentInfo = newSegmentInfo[hostName]
checkNotNone("segmentInfo for %s" % hostName, segmentInfo)
return gp.ConfigureNewSegment(cmdLabel,
segmentInfo,
newSegments=True,
verbose=gplog.logging_is_verbose(),
batchSize=self.__parallelDegree,
ctxt=gp.REMOTE,
remoteHost=hostName,
validationOnly=validationOnly,
forceoverwrite=self.__forceoverwrite)
def read_delimited_field(self, delimiter, name="next field", reader=read_to):
"""
Attempts to read the next field in the line up to the specified delimiter
using the specified reading method, raising any error encountered as an
exception. Returns the read field when successful.
"""
self.ensure_more_to_process(name)
value, self.rest, error = reader(delimiter, self.rest)
if error is not None:
msg = "%s (reading %s) >>%s" % (error, name, self.rest)
raise ExceptionNoStackTraceNeeded("%s:%s:%s LINE >>%s\n%s" % (self.filename, self.lineno, self.caller, self.line, msg))
if logging_is_verbose():
self.logger.debug(" name=%-30s delimiter='%s' value=%s" % (name, delimiter, value))
return value
def read_delimited_field(self, delimiter, name="next field", reader=read_to):
"""
Attempts to read the next field in the line up to the specified delimiter
using the specified reading method, raising any error encountered as an
exception. Returns the read field when successful.
"""
self.ensure_more_to_process(name)
value, self.rest, error = reader(delimiter, self.rest)
if error is not None:
msg = "%s (reading %s) >>%s" % (error, name, self.rest)
raise ExceptionNoStackTraceNeeded("%s:%s:%s LINE >>%s\n%s" % (self.filename, self.lineno, self.caller, self.line, msg))
if logging_is_verbose():
self.logger.debug(" name=%-30s delimiter='%s' value=%s" % (name, delimiter, value))
return value
def run_pg_rewind(self, rewindInfo):
"""
Run pg_rewind for incremental recovery.
"""
rewindFailedSegments = []
# Run pg_rewind on all the targets
for targetSegment, sourceHostName, sourcePort in rewindInfo:
# Do CHECKPOINT on source to force TimeLineID to be updated in pg_control.
# pg_rewind wants that to make incremental recovery successful finally.
self.__logger.debug(
'Do CHECKPOINT on %s (port: %d) before running pg_rewind.' %
(sourceHostName, sourcePort))
dburl = dbconn.DbURL(hostname=sourceHostName,
port=sourcePort,
dbname='template1')
conn = dbconn.connect(dburl, utility=True)
dbconn.execSQL(conn, "CHECKPOINT")
# If the postmaster.pid still exists and another process
# is actively using that pid, pg_rewind will fail when it
# tries to start the failed segment in single-user
# mode. It should be safe to remove the postmaster.pid
# file since we do not expect the failed segment to be up.
self.remove_postmaster_pid_from_remotehost(
targetSegment.getSegmentHostName(),
targetSegment.getSegmentDataDirectory())
# Run pg_rewind to do incremental recovery.
cmd = gp.SegmentRewind('segment rewind',
targetSegment.getSegmentHostName(),
targetSegment.getSegmentDataDirectory(),
sourceHostName,
sourcePort,
verbose=gplog.logging_is_verbose())
try:
cmd.run(True)
self.__logger.debug('pg_rewind results: %s' % cmd.results)
except base.ExecutionError as e:
self.__logger.debug(
"pg_rewind failed for target directory %s." %
targetSegment.getSegmentDataDirectory())
self.__logger.warning(
"Incremental recovery failed for dbid %s. You must use gprecoverseg -F to recover the segment."
% targetSegment.getSegmentDbId())
rewindFailedSegments.append(targetSegment)
return rewindFailedSegments
def run_pg_rewind(self, rewindInfo):
"""
Run pg_rewind for incremental recovery.
"""
rewindFailedSegments = []
# Run pg_rewind on all the targets
for targetSegment, sourceHostName, sourcePort in rewindInfo:
# Do CHECKPOINT on source to force TimeLineID to be updated in pg_control.
# pg_rewind wants that to make incremental recovery successful finally.
self.__logger.debug('Do CHECKPOINT on %s (port: %d) before running pg_rewind.' % (sourceHostName, sourcePort))
dburl = dbconn.DbURL(hostname=sourceHostName,
port=sourcePort,
dbname='template1')
conn = dbconn.connect(dburl, utility=True)
dbconn.execSQL(conn, "CHECKPOINT")
# If the postmaster.pid still exists and another process
# is actively using that pid, pg_rewind will fail when it
# tries to start the failed segment in single-user
# mode. It should be safe to remove the postmaster.pid
# file since we do not expect the failed segment to be up.
self.remove_postmaster_pid_from_remotehost(
targetSegment.getSegmentHostName(),
targetSegment.getSegmentDataDirectory())
# Run pg_rewind to do incremental recovery.
cmd = gp.SegmentRewind('segment rewind',
targetSegment.getSegmentHostName(),
targetSegment.getSegmentDataDirectory(),
sourceHostName,
sourcePort,
verbose=gplog.logging_is_verbose())
try:
cmd.run(True)
self.__logger.debug('pg_rewind results: %s' % cmd.results)
except base.ExecutionError as e:
self.__logger.debug("pg_rewind failed for target directory %s." % targetSegment.getSegmentDataDirectory())
self.__logger.warning("Incremental recovery failed for dbid %s. You must use gprecoverseg -F to recover the segment." % targetSegment.getSegmentDbId())
rewindFailedSegments.append(targetSegment)
return rewindFailedSegments
def __updateGpIdFile(self, gpEnv, gpArray, segments):
segmentByHost = GpArray.getSegmentsByHostName(segments)
newSegmentInfo = gp.ConfigureNewSegment.buildSegmentInfoForNewSegment(segments)
cmds = []
for hostName in segmentByHost.keys():
segmentInfo = newSegmentInfo[hostName]
checkNotNone("segmentInfo for %s" % hostName, segmentInfo)
cmd = gp.ConfigureNewSegment("update gpid file",
segmentInfo,
newSegments=False,
verbose=gplog.logging_is_verbose(),
batchSize=self.__parallelDegree,
ctxt=gp.REMOTE,
remoteHost=hostName,
validationOnly=False,
writeGpIdFileOnly=True)
cmds.append(cmd)
self.__runWaitAndCheckWorkerPoolForErrorsAndClear(cmds, "writing updated gpid files")
def __updateGpIdFile(self, gpEnv, gpArray, segments):
segmentByHost = GpArray.getSegmentsByHostName(segments)
newSegmentInfo = gp.ConfigureNewSegment.buildSegmentInfoForNewSegment(segments)
cmds = []
for hostName in segmentByHost.keys():
segmentInfo = newSegmentInfo[hostName]
checkNotNone("segmentInfo for %s" % hostName, segmentInfo)
cmd = gp.ConfigureNewSegment("update gpid file",
segmentInfo,
newSegments=False,
verbose=gplog.logging_is_verbose(),
batchSize=self.__parallelDegree,
ctxt=gp.REMOTE,
remoteHost=hostName,
validationOnly=False,
writeGpIdFileOnly=True)
cmds.append(cmd)
self.__runWaitAndCheckWorkerPoolForErrorsAndClear(cmds, "writing updated gpid files")
def _do_setup_for_recovery(self, recovery_info_by_host):
self.__logger.info('Setting up the required segments for recovery')
cmds = []
for host_name, recovery_info_list in recovery_info_by_host.items():
cmds.append(
gp.GpSegSetupRecovery(
'Run validation checks and setup data directories for recovery',
recoveryinfo.serialize_list(recovery_info_list),
gplog.get_logger_dir(),
verbose=gplog.logging_is_verbose(),
batchSize=self.__parallelPerHost,
remoteHost=host_name,
forceoverwrite=self.__forceoverwrite))
for cmd in cmds:
self.__pool.addCommand(cmd)
if self.__quiet:
self.__pool.join()
else:
base.join_and_indicate_progress(self.__pool)
completed_results = self.__pool.getCompletedItems()
self.__pool.empty_completed_items()
return completed_results
def run_pg_rewind(self, rewindInfo):
"""
Run pg_rewind for incremental recovery.
"""
rewindFailedSegments = []
# Run pg_rewind on all the targets
for targetSegment, sourceHostName, sourcePort in rewindInfo:
# Do CHECKPOINT on source to force TimeLineID to be updated in pg_control.
# pg_rewind wants that to make incremental recovery successful finally.
self.__logger.debug(
'Do CHECKPOINT on %s (port: %d) before running pg_rewind.' %
(sourceHostName, sourcePort))
dburl = dbconn.DbURL(hostname=sourceHostName,
port=sourcePort,
dbname='template1')
conn = dbconn.connect(dburl, utility=True)
dbconn.execSQL(conn, "CHECKPOINT")
# Run pg_rewind to do incremental recovery.
cmd = gp.SegmentRewind('segment rewind',
targetSegment.getSegmentHostName(),
targetSegment.getSegmentDataDirectory(),
sourceHostName,
sourcePort,
verbose=gplog.logging_is_verbose())
try:
cmd.run(True)
self.__logger.debug('pg_rewind results: %s' % cmd.results)
except base.ExecutionError as e:
self.__logger.debug(
"pg_rewind failed for target directory %s." %
targetSegment.getSegmentDataDirectory())
self.__logger.warning(
"Incremental recovery failed for dbid %s. You must use gprecoverseg -F to recover the segment."
% targetSegment.getSegmentDbId())
rewindFailedSegments.append(targetSegment)
return rewindFailedSegments
def _do_recovery(self, recovery_info_by_host, gpEnv):
"""
# Recover and start segments using gpsegrecovery, which will internally call either
# pg_basebackup or pg_rewind. gprecoverseg generates a log filename which is
# passed to gpsegrecovery using the confinfo parameter. gprecoverseg
# tails this file to show recovery progress to the user, and removes the
# file when done. A new file is generated for each run of gprecoverseg
# based on a timestamp.
:param gpEnv:
:param recovery_info_by_host:
:return:
"""
self.__logger.info(
'Initiating segment recovery. Upon completion, will start the successfully recovered segments'
)
cmds = []
progress_cmds = []
era = read_era(gpEnv.getCoordinatorDataDir(), logger=self.__logger)
for hostName, recovery_info_list in recovery_info_by_host.items():
for ri in recovery_info_list:
progressCmd = self._get_progress_cmd(ri.progress_file,
ri.target_segment_dbid,
hostName)
if progressCmd:
progress_cmds.append(progressCmd)
cmds.append(
gp.GpSegRecovery(
'Recover segments',
recoveryinfo.serialize_list(recovery_info_list),
gplog.get_logger_dir(),
verbose=gplog.logging_is_verbose(),
batchSize=self.__parallelPerHost,
remoteHost=hostName,
era=era,
forceoverwrite=self.__forceoverwrite))
completed_recovery_results = self.__runWaitAndCheckWorkerPoolForErrorsAndClear(
cmds, suppressErrorCheck=True, progressCmds=progress_cmds)
return completed_recovery_results
def caller():
"Return name of calling function"
if logging_is_verbose():
return sys._getframe(1).f_code.co_name + '()'
return ''
def caller():
"Return name of calling function"
if logging_is_verbose():
return sys._getframe(1).f_code.co_name + '()'
return ''
def __init__(self, caller, filename, lineno, line):
"Initialize"
(self.caller, self.filename, self.lineno, self.line, self.rest, self.error) = (caller, filename, lineno, line, line, None)
self.logger = logger
if logging_is_verbose():
self.logger.debug("%s:%s" % (filename, lineno))
