def run_pg_rewind(self, rewindInfo):
"""
Run pg_rewind for incremental recovery.
"""
rewindFailedSegments = []
# Run pg_rewind on all the targets
for targetSegment, sourceHostName, sourcePort in rewindInfo:
cmd = gp.SegmentRewind('segment rewind',
targetSegment.getSegmentHostName(),
targetSegment.getSegmentDataDirectory(),
sourceHostName,
sourcePort,
verbose=gplog.logging_is_verbose())
try:
cmd.run(True)
self.__logger.debug('pg_rewind results: %s' % cmd.results)
except base.ExecutionError as e:
self.__logger.debug(
"pg_rewind failed for target directory %s." %
targetSegment.getSegmentDataDirectory())
self.__logger.warning(
"Incremental recovery failed for dbid %s. You must use gprecoverseg -F to recover the segment."
% targetSegment.getSegmentDbId())
rewindFailedSegments.append(targetSegment)
return rewindFailedSegments
def run_pg_rewind(self, rewindInfo):
"""
Run pg_rewind for incremental recovery.
"""
rewindFailedSegments = []
# Run pg_rewind on all the targets
for rewindSeg in list(rewindInfo.values()):
# Do CHECKPOINT on source to force TimeLineID to be updated in pg_control.
# pg_rewind wants that to make incremental recovery successful finally.
self.__logger.debug(
'Do CHECKPOINT on %s (port: %d) before running pg_rewind.' %
(rewindSeg.sourceHostname, rewindSeg.sourcePort))
dburl = dbconn.DbURL(hostname=rewindSeg.sourceHostname,
port=rewindSeg.sourcePort,
dbname='template1')
conn = dbconn.connect(dburl, utility=True)
dbconn.execSQL(conn, "CHECKPOINT")
conn.close()
# If the postmaster.pid still exists and another process
# is actively using that pid, pg_rewind will fail when it
# tries to start the failed segment in single-user
# mode. It should be safe to remove the postmaster.pid
# file since we do not expect the failed segment to be up.
self.remove_postmaster_pid_from_remotehost(
rewindSeg.targetSegment.getSegmentHostName(),
rewindSeg.targetSegment.getSegmentDataDirectory())
# Note the command name, we use the dbid later to
# correlate the command results with GpMirrorToBuild
# object.
cmd = gp.SegmentRewind(
'rewind dbid: %s' % rewindSeg.targetSegment.getSegmentDbId(),
rewindSeg.targetSegment.getSegmentHostName(),
rewindSeg.targetSegment.getSegmentDataDirectory(),
rewindSeg.sourceHostname,
rewindSeg.sourcePort,
verbose=gplog.logging_is_verbose())
self.__pool.addCommand(cmd)
if self.__quiet:
self.__pool.join()
else:
base.join_and_indicate_progress(self.__pool)
for cmd in self.__pool.getCompletedItems():
self.__logger.debug('pg_rewind results: %s' % cmd.results)
if not cmd.was_successful():
dbid = int(cmd.name.split(':')[1].strip())
self.__logger.debug("%s failed" % cmd.name)
self.__logger.warning(cmd.get_stdout())
self.__logger.warning(
"Incremental recovery failed for dbid %d. You must use gprecoverseg -F to recover the segment."
% dbid)
rewindFailedSegments.append(rewindInfo[dbid].targetSegment)
self.__pool.empty_completed_items()
return rewindFailedSegments
def run_pg_rewind(self, rewindInfo):
"""
Run pg_rewind for incremental recovery.
"""
rewindFailedSegments = []
# Run pg_rewind on all the targets
for targetSegment, sourceHostName, sourcePort in rewindInfo:
# Do CHECKPOINT on source to force TimeLineID to be updated in pg_control.
# pg_rewind wants that to make incremental recovery successful finally.
self.__logger.debug(
'Do CHECKPOINT on %s (port: %d) before running pg_rewind.' %
(sourceHostName, sourcePort))
dburl = dbconn.DbURL(hostname=sourceHostName,
port=sourcePort,
dbname='template1')
conn = dbconn.connect(dburl, utility=True)
dbconn.execSQL(conn, "CHECKPOINT")
# If the postmaster.pid still exists and another process
# is actively using that pid, pg_rewind will fail when it
# tries to start the failed segment in single-user
# mode. It should be safe to remove the postmaster.pid
# file since we do not expect the failed segment to be up.
self.remove_postmaster_pid_from_remotehost(
targetSegment.getSegmentHostName(),
targetSegment.getSegmentDataDirectory())
# Run pg_rewind to do incremental recovery.
cmd = gp.SegmentRewind('segment rewind',
targetSegment.getSegmentHostName(),
targetSegment.getSegmentDataDirectory(),
sourceHostName,
sourcePort,
verbose=gplog.logging_is_verbose())
try:
cmd.run(True)
self.__logger.debug('pg_rewind results: %s' % cmd.results)
except base.ExecutionError as e:
self.__logger.debug(
"pg_rewind failed for target directory %s." %
targetSegment.getSegmentDataDirectory())
self.__logger.warning(
"Incremental recovery failed for dbid %s. You must use gprecoverseg -F to recover the segment."
% targetSegment.getSegmentDbId())
rewindFailedSegments.append(targetSegment)
return rewindFailedSegments
def run_pg_rewind(self, rewindInfo):
"""
Run pg_rewind for incremental recovery.
"""
rewindFailedSegments = []
# Run pg_rewind on all the targets
for targetSegment, sourceHostName, sourcePort in rewindInfo:
# Do CHECKPOINT on source to force TimeLineID to be updated in pg_control.
# pg_rewind wants that to make incremental recovery successful finally.
self.__logger.debug(
'Do CHECKPOINT on %s (port: %d) before running pg_rewind.' %
(sourceHostName, sourcePort))
dburl = dbconn.DbURL(hostname=sourceHostName,
port=sourcePort,
dbname='template1')
conn = dbconn.connect(dburl, utility=True)
dbconn.execSQL(conn, "CHECKPOINT")
# Run pg_rewind to do incremental recovery.
cmd = gp.SegmentRewind('segment rewind',
targetSegment.getSegmentHostName(),
targetSegment.getSegmentDataDirectory(),
sourceHostName,
sourcePort,
verbose=gplog.logging_is_verbose())
try:
cmd.run(True)
self.__logger.debug('pg_rewind results: %s' % cmd.results)
except base.ExecutionError as e:
self.__logger.debug(
"pg_rewind failed for target directory %s." %
targetSegment.getSegmentDataDirectory())
self.__logger.warning(
"Incremental recovery failed for dbid %s. You must use gprecoverseg -F to recover the segment."
% targetSegment.getSegmentDbId())
rewindFailedSegments.append(targetSegment)
return rewindFailedSegments
def run_pg_rewind(self, rewindInfo):
"""
Run pg_rewind for incremental recovery.
"""
rewindFailedSegments = []
# Run pg_rewind on all the targets
cmds = []
progressCmds = []
removeCmds = []
for rewindSeg in list(rewindInfo.values()):
# Do CHECKPOINT on source to force TimeLineID to be updated in pg_control.
# pg_rewind wants that to make incremental recovery successful finally.
self.__logger.debug(
'Do CHECKPOINT on %s (port: %d) before running pg_rewind.' %
(rewindSeg.sourceHostname, rewindSeg.sourcePort))
dburl = dbconn.DbURL(hostname=rewindSeg.sourceHostname,
port=rewindSeg.sourcePort,
dbname='template1')
conn = dbconn.connect(dburl, utility=True)
dbconn.execSQL(conn, "CHECKPOINT")
conn.close()
# If the postmaster.pid still exists and another process
# is actively using that pid, pg_rewind will fail when it
# tries to start the failed segment in single-user
# mode. It should be safe to remove the postmaster.pid
# file since we do not expect the failed segment to be up.
self.remove_postmaster_pid_from_remotehost(
rewindSeg.targetSegment.getSegmentHostName(),
rewindSeg.targetSegment.getSegmentDataDirectory())
# Note the command name, we use the dbid later to
# correlate the command results with GpMirrorToBuild
# object.
cmd = gp.SegmentRewind(
'rewind dbid: %s' % rewindSeg.targetSegment.getSegmentDbId(),
rewindSeg.targetSegment.getSegmentHostName(),
rewindSeg.targetSegment.getSegmentDataDirectory(),
rewindSeg.sourceHostname,
rewindSeg.sourcePort,
rewindSeg.progressFile,
verbose=True)
progressCmd, removeCmd = self.__getProgressAndRemoveCmds(
rewindSeg.progressFile,
rewindSeg.targetSegment.getSegmentDbId(),
rewindSeg.targetSegment.getSegmentHostName())
cmds.append(cmd)
removeCmds.append(removeCmd)
if progressCmd:
progressCmds.append(progressCmd)
completedCmds = self.__runWaitAndCheckWorkerPoolForErrorsAndClear(
cmds,
"rewinding segments",
suppressErrorCheck=True,
progressCmds=progressCmds)
self.__runWaitAndCheckWorkerPoolForErrorsAndClear(
removeCmds,
"removing rewind progress logfiles",
suppressErrorCheck=False)
rewindFailedSegments = []
for cmd in completedCmds:
self.__logger.debug('pg_rewind results: %s' % cmd.results)
if not cmd.was_successful():
dbid = int(cmd.name.split(':')[1].strip())
self.__logger.debug("%s failed" % cmd.name)
self.__logger.warning(cmd.get_stdout())
self.__logger.warning(
"Incremental recovery failed for dbid %d. You must use gprecoverseg -F to recover the segment."
% dbid)
rewindFailedSegments.append(rewindInfo[dbid].targetSegment)
return rewindFailedSegments
