def run(self): results = [] failures = [] self.logger.info("Issuing shutdown commands to local segments...") for db in self.dblist: datadir, port = db.split(':')[0:2] cmd = gp.SegmentStop('segment shutdown', datadir, mode=self.mode, timeout=self.timeout) cmd.run() res = cmd.get_results() if res.rc == 0: # MPP-15208 # cmd2 = gp.SegmentIsShutDown('check if shutdown', datadir) cmd2.run() if cmd2.is_shutdown(): status = SegStopStatus(datadir, True, "Shutdown Succeeded") results.append(status) continue # MPP-16171 # if self.mode == 'immediate': status = SegStopStatus(datadir, True, "Shutdown Immediate") results.append(status) continue # read pid and datadir from /tmp/.s.PGSQL.<port>.lock file name = "failed segment '%s'" % db (succeeded, mypid, file_datadir) = pg.ReadPostmasterTempFile.local(name,port).getResults() if succeeded and file_datadir == datadir: # now try to terminate the process, first trying with # SIGTERM and working our way up to SIGABRT sleeping # in between to give the process a moment to exit # unix.kill_sequence(mypid) if not unix.check_pid(mypid): lockfile = "/tmp/.s.PGSQL.%s" % port if os.path.exists(lockfile): self.logger.info("Clearing segment instance lock files") os.remove(lockfile) status = SegStopStatus(datadir,False,"Shutdown failed: rc: %d stdout: %s stderr: %s" % (res.rc,res.stdout,res.stderr)) failures.append(status) results.append(status) #Log the results! status = '\nCOMMAND RESULTS\n' for result in results: status += str(result) + "\n" self.logger.info(status) return 1 if failures else 0
def __convertSegments(self): """ Inform segments of their role """ if self.mirroringMode != 'quiescent': self.logger.info( "Not transitioning segments, mirroringMode is %s..." % self.mirroringMode) return self.logger.info("Transitioning segments, mirroringMode is %s..." % self.mirroringMode) transitionData = None if self.pickledTransitionData is not None: transitionData = pickle.loads( base64.urlsafe_b64decode(self.pickledTransitionData)) # send transition messages to the segments # for datadir, seg in self.overall_status.dirmap.items(): # # This cmd will deliver a message to the postmaster using gp_primarymirror # (look for the protocol message type PRIMARY_MIRROR_TRANSITION_REQUEST_CODE ) # port = seg.getSegmentPort() cmd = gp.SendFilerepTransitionMessage.buildTransitionMessageCommand( transitionData, datadir, port) self.pool.addCommand(cmd) self.pool.join() # examine the results from the segments # segments = self.overall_status.dirmap.values() dataDirToSeg = gparray.GpArray.getSegmentsGroupedByValue( segments, gparray.GpDB.getSegmentDataDirectory) toStop = [] cmds = self.pool.getCompletedItems() for cmd in cmds: res = cmd.get_results() if res.rc == 0: continue # some form of failure # stdoutFromFailure = res.stdout.replace("\n", " ").strip() stderrFromFailure = res.stderr.replace("\n", " ").strip() shouldStop = False if res.rc == gp.SendFilerepTransitionMessage.TRANSITION_ERRCODE_ERROR_SERVER_DID_NOT_RETURN_DATA: msg = "Segment did not respond to startup request; check segment logfile" reasoncode = gp.SEGSTART_ERROR_SERVER_DID_NOT_RESPOND # server crashed when sending response, should ensure it's stopped completely! shouldStop = True elif stderrFromFailure.endswith( "failure: Error: MirroringFailure"): msg = "Failure in segment mirroring; check segment logfile" reasoncode = gp.SEGSTART_ERROR_MIRRORING_FAILURE elif stderrFromFailure.endswith("failure: Error: PostmasterDied"): msg = "Segment postmaster has exited; check segment logfile" reasoncode = gp.SEGSTART_ERROR_POSTMASTER_DIED elif stderrFromFailure.endswith( "failure: Error: InvalidStateTransition"): msg = "Not a valid operation at this time; check segment logfile" reasoncode = gp.SEGSTART_ERROR_INVALID_STATE_TRANSITION # This should never happen, but if it does then we will ensure process is gone shouldStop = True elif stderrFromFailure.endswith( "failure: Error: ServerIsInShutdown"): msg = "System is shutting down" reasoncode = gp.SEGSTART_ERROR_SERVER_IS_IN_SHUTDOWN else: if res.rc == gp.SendFilerepTransitionMessage.TRANSITION_ERRCODE_ERROR_SOCKET: # Couldn't connect to server to do transition or got another problem # communicating, must make sure it's halted! shouldStop = True msg = "Start failed; check segment logfile. \"%s%s\"" % ( stdoutFromFailure, stderrFromFailure) reasoncode = gp.SEGSTART_ERROR_OTHER self.overall_status.mark_failed(cmd.dataDir, msg, reasoncode) if shouldStop: assert len( dataDirToSeg[cmd.dataDir] ) == 1, "Multiple segments with dir %s" % cmd.dataDir toStop.append(dataDirToSeg[cmd.dataDir][0]) # ensure segments in a bad state are stopped # for seg in toStop: datadir, port = (seg.getSegmentDataDirectory(), seg.getSegmentPort()) msg = "Stopping segment %s, %s because of failure sending transition" % ( datadir, port) self.logger.info(msg) cmd = gp.SegmentStop('stop segment', datadir, mode="immediate") cmd.run(validateAfter=False) res = cmd.get_results() if res.rc == 0: self.logger.info("Stop of segment succeeded") else: stdoutFromFailure = res.stdout.replace("\n", " ").strip() stderrFromFailure = res.stderr.replace("\n", " ").strip() self.logger.info("Stop of segment failed: rc: %s\nstdout:%s\nstderr:%s" % \ (res.rc, stdoutFromFailure, stderrFromFailure))
def run(self): try: self.datadir, self.port = self.get_datadir_and_port() cmd = gp.SegmentStop('segment shutdown', self.datadir, mode=self.mode, timeout=self.timeout) cmd.run() results = cmd.get_results() is_shutdown = False if results.rc == 0: cmd = gp.SegmentIsShutDown('check if shutdown', self.datadir) cmd.run() if cmd.is_shutdown(): status = SegStopStatus(self.datadir, True, "Shutdown Succeeded") self.result = status is_shutdown = True # MPP-16171 # elif self.mode == 'immediate': status = SegStopStatus(self.datadir, True, "Shutdown Immediate") self.result = status is_shutdown = True # read pid and datadir from /tmp/.s.PGSQL.<port>.lock file name = "failed segment '%s'" % self.db (succeeded, mypid, file_datadir) = pg.ReadPostmasterTempFile.local( name, self.port).getResults() if not is_shutdown: if succeeded and file_datadir == self.datadir: # now try to terminate the process, first trying with # SIGTERM and working our way up to SIGABRT sleeping # in between to give the process a moment to exit # unix.kill_sequence(mypid) if not unix.check_pid(mypid): lockfile = "/tmp/.s.PGSQL.%s" % self.port if os.path.exists(lockfile): self.logger.info( "Clearing segment instance lock files") os.remove(lockfile) status = SegStopStatus( self.datadir, True, "Forceful termination success: rc: %d stdout: %s stderr: %s." % (results.rc, results.stdout, results.stderr)) try: unix.kill_9_segment_processes(self.datadir, self.port, mypid) if unix.check_pid(mypid) and mypid != -1: status = SegStopStatus( self.datadir, False, "Failed forceful termnation: rc: %d stdout: %s stderr: %s." % (results.rc, results.stdout, results.stderr)) self.result = status except Exception as e: logger.error( 'Failed forceful termination of segment %s: (%s)' % (self.datadir, str(e))) self.result = SegStopStatus( self.datadir, False, 'Failed forceful termination of segment! (%s)' % str(e)) return self.result except Exception as e: logger.exception(e) self.result = SegStopStatus(self.datadir, False, 'Shutdown failed! %s' % str(e)) return self.result