def run_pg_rewind(self, rewindInfo): """ Run pg_rewind for incremental recovery. """ rewindFailedSegments = [] # Run pg_rewind on all the targets for targetSegment, sourceHostName, sourcePort in rewindInfo: cmd = gp.SegmentRewind('segment rewind', targetSegment.getSegmentHostName(), targetSegment.getSegmentDataDirectory(), sourceHostName, sourcePort, verbose=gplog.logging_is_verbose()) try: cmd.run(True) self.__logger.debug('pg_rewind results: %s' % cmd.results) except base.ExecutionError as e: self.__logger.debug( "pg_rewind failed for target directory %s." % targetSegment.getSegmentDataDirectory()) self.__logger.warning( "Incremental recovery failed for dbid %s. You must use gprecoverseg -F to recover the segment." % targetSegment.getSegmentDbId()) rewindFailedSegments.append(targetSegment) return rewindFailedSegments
def run_pg_rewind(self, rewindInfo): """ Run pg_rewind for incremental recovery. """ rewindFailedSegments = [] # Run pg_rewind on all the targets for rewindSeg in list(rewindInfo.values()): # Do CHECKPOINT on source to force TimeLineID to be updated in pg_control. # pg_rewind wants that to make incremental recovery successful finally. self.__logger.debug( 'Do CHECKPOINT on %s (port: %d) before running pg_rewind.' % (rewindSeg.sourceHostname, rewindSeg.sourcePort)) dburl = dbconn.DbURL(hostname=rewindSeg.sourceHostname, port=rewindSeg.sourcePort, dbname='template1') conn = dbconn.connect(dburl, utility=True) dbconn.execSQL(conn, "CHECKPOINT") conn.close() # If the postmaster.pid still exists and another process # is actively using that pid, pg_rewind will fail when it # tries to start the failed segment in single-user # mode. It should be safe to remove the postmaster.pid # file since we do not expect the failed segment to be up. self.remove_postmaster_pid_from_remotehost( rewindSeg.targetSegment.getSegmentHostName(), rewindSeg.targetSegment.getSegmentDataDirectory()) # Note the command name, we use the dbid later to # correlate the command results with GpMirrorToBuild # object. cmd = gp.SegmentRewind( 'rewind dbid: %s' % rewindSeg.targetSegment.getSegmentDbId(), rewindSeg.targetSegment.getSegmentHostName(), rewindSeg.targetSegment.getSegmentDataDirectory(), rewindSeg.sourceHostname, rewindSeg.sourcePort, verbose=gplog.logging_is_verbose()) self.__pool.addCommand(cmd) if self.__quiet: self.__pool.join() else: base.join_and_indicate_progress(self.__pool) for cmd in self.__pool.getCompletedItems(): self.__logger.debug('pg_rewind results: %s' % cmd.results) if not cmd.was_successful(): dbid = int(cmd.name.split(':')[1].strip()) self.__logger.debug("%s failed" % cmd.name) self.__logger.warning(cmd.get_stdout()) self.__logger.warning( "Incremental recovery failed for dbid %d. You must use gprecoverseg -F to recover the segment." % dbid) rewindFailedSegments.append(rewindInfo[dbid].targetSegment) self.__pool.empty_completed_items() return rewindFailedSegments
def run_pg_rewind(self, rewindInfo): """ Run pg_rewind for incremental recovery. """ rewindFailedSegments = [] # Run pg_rewind on all the targets for targetSegment, sourceHostName, sourcePort in rewindInfo: # Do CHECKPOINT on source to force TimeLineID to be updated in pg_control. # pg_rewind wants that to make incremental recovery successful finally. self.__logger.debug( 'Do CHECKPOINT on %s (port: %d) before running pg_rewind.' % (sourceHostName, sourcePort)) dburl = dbconn.DbURL(hostname=sourceHostName, port=sourcePort, dbname='template1') conn = dbconn.connect(dburl, utility=True) dbconn.execSQL(conn, "CHECKPOINT") # If the postmaster.pid still exists and another process # is actively using that pid, pg_rewind will fail when it # tries to start the failed segment in single-user # mode. It should be safe to remove the postmaster.pid # file since we do not expect the failed segment to be up. self.remove_postmaster_pid_from_remotehost( targetSegment.getSegmentHostName(), targetSegment.getSegmentDataDirectory()) # Run pg_rewind to do incremental recovery. cmd = gp.SegmentRewind('segment rewind', targetSegment.getSegmentHostName(), targetSegment.getSegmentDataDirectory(), sourceHostName, sourcePort, verbose=gplog.logging_is_verbose()) try: cmd.run(True) self.__logger.debug('pg_rewind results: %s' % cmd.results) except base.ExecutionError as e: self.__logger.debug( "pg_rewind failed for target directory %s." % targetSegment.getSegmentDataDirectory()) self.__logger.warning( "Incremental recovery failed for dbid %s. You must use gprecoverseg -F to recover the segment." % targetSegment.getSegmentDbId()) rewindFailedSegments.append(targetSegment) return rewindFailedSegments
def run_pg_rewind(self, rewindInfo): """ Run pg_rewind for incremental recovery. """ rewindFailedSegments = [] # Run pg_rewind on all the targets for targetSegment, sourceHostName, sourcePort in rewindInfo: # Do CHECKPOINT on source to force TimeLineID to be updated in pg_control. # pg_rewind wants that to make incremental recovery successful finally. self.__logger.debug( 'Do CHECKPOINT on %s (port: %d) before running pg_rewind.' % (sourceHostName, sourcePort)) dburl = dbconn.DbURL(hostname=sourceHostName, port=sourcePort, dbname='template1') conn = dbconn.connect(dburl, utility=True) dbconn.execSQL(conn, "CHECKPOINT") # Run pg_rewind to do incremental recovery. cmd = gp.SegmentRewind('segment rewind', targetSegment.getSegmentHostName(), targetSegment.getSegmentDataDirectory(), sourceHostName, sourcePort, verbose=gplog.logging_is_verbose()) try: cmd.run(True) self.__logger.debug('pg_rewind results: %s' % cmd.results) except base.ExecutionError as e: self.__logger.debug( "pg_rewind failed for target directory %s." % targetSegment.getSegmentDataDirectory()) self.__logger.warning( "Incremental recovery failed for dbid %s. You must use gprecoverseg -F to recover the segment." % targetSegment.getSegmentDbId()) rewindFailedSegments.append(targetSegment) return rewindFailedSegments
def run_pg_rewind(self, rewindInfo): """ Run pg_rewind for incremental recovery. """ rewindFailedSegments = [] # Run pg_rewind on all the targets cmds = [] progressCmds = [] removeCmds = [] for rewindSeg in list(rewindInfo.values()): # Do CHECKPOINT on source to force TimeLineID to be updated in pg_control. # pg_rewind wants that to make incremental recovery successful finally. self.__logger.debug( 'Do CHECKPOINT on %s (port: %d) before running pg_rewind.' % (rewindSeg.sourceHostname, rewindSeg.sourcePort)) dburl = dbconn.DbURL(hostname=rewindSeg.sourceHostname, port=rewindSeg.sourcePort, dbname='template1') conn = dbconn.connect(dburl, utility=True) dbconn.execSQL(conn, "CHECKPOINT") conn.close() # If the postmaster.pid still exists and another process # is actively using that pid, pg_rewind will fail when it # tries to start the failed segment in single-user # mode. It should be safe to remove the postmaster.pid # file since we do not expect the failed segment to be up. self.remove_postmaster_pid_from_remotehost( rewindSeg.targetSegment.getSegmentHostName(), rewindSeg.targetSegment.getSegmentDataDirectory()) # Note the command name, we use the dbid later to # correlate the command results with GpMirrorToBuild # object. cmd = gp.SegmentRewind( 'rewind dbid: %s' % rewindSeg.targetSegment.getSegmentDbId(), rewindSeg.targetSegment.getSegmentHostName(), rewindSeg.targetSegment.getSegmentDataDirectory(), rewindSeg.sourceHostname, rewindSeg.sourcePort, rewindSeg.progressFile, verbose=True) progressCmd, removeCmd = self.__getProgressAndRemoveCmds( rewindSeg.progressFile, rewindSeg.targetSegment.getSegmentDbId(), rewindSeg.targetSegment.getSegmentHostName()) cmds.append(cmd) removeCmds.append(removeCmd) if progressCmd: progressCmds.append(progressCmd) completedCmds = self.__runWaitAndCheckWorkerPoolForErrorsAndClear( cmds, "rewinding segments", suppressErrorCheck=True, progressCmds=progressCmds) self.__runWaitAndCheckWorkerPoolForErrorsAndClear( removeCmds, "removing rewind progress logfiles", suppressErrorCheck=False) rewindFailedSegments = [] for cmd in completedCmds: self.__logger.debug('pg_rewind results: %s' % cmd.results) if not cmd.was_successful(): dbid = int(cmd.name.split(':')[1].strip()) self.__logger.debug("%s failed" % cmd.name) self.__logger.warning(cmd.get_stdout()) self.__logger.warning( "Incremental recovery failed for dbid %d. You must use gprecoverseg -F to recover the segment." % dbid) rewindFailedSegments.append(rewindInfo[dbid].targetSegment) return rewindFailedSegments