def execute(self): entry = ValidateVerificationEntry(token = self.token).run() if not entry['verdone']: raise WrongStateError("Only finished verification tasks may be cleaned up.") path = os.path.join(get_masterdatadir(), 'pg_verify', self.token) Command('cleanup', 'rm -rf %s' % path).run(validateAfter=True) #RemoveTree(path).run() to_clean = ValidateVerification(content = entry['vercontent'], primaries_only = False).run() pool = WorkerPool(min(len(to_clean), self.batch_default)) for seg in to_clean: host = seg.getSegmentHostName() path = os.path.join(seg.getSegmentDataDirectory(), 'pg_verify', "*%s*" % self.token) cmd = Command('cleanup', 'rm -f %s' % path, remoteHost=host) pool.addCommand(cmd) logger.info('Waiting for clean commands to complete...') pool.wait_and_printdots(len(to_clean)) for cmd in pool.getCompletedItems(): res = cmd.get_results() if not res.wasSuccessful(): logger.error('Failed to send cleanup on %s' % cmd.host) logger.error('Error: %s' % res.stderr) raise CleanVerificationError() RemoveVerificationEntry(token = self.token).run() logger.info('Verification %s has been cleaned.' % self.token)
def execute(self): entry = ValidateVerificationEntry(token=self.token).run() if not entry['verdone']: raise WrongStateError( "Only finished verification tasks may be cleaned up.") path = os.path.join(get_masterdatadir(), 'pg_verify', self.token) Command('cleanup', 'rm -rf %s' % path).run(validateAfter=True) #RemoveTree(path).run() to_clean = ValidateVerification(content=entry['vercontent'], primaries_only=False).run() pool = WorkerPool(min(len(to_clean), self.batch_default)) for seg in to_clean: host = seg.getSegmentHostName() path = os.path.join(seg.getSegmentDataDirectory(), 'pg_verify', "*%s*" % self.token) cmd = Command('cleanup', 'rm -f %s' % path, remoteHost=host) pool.addCommand(cmd) logger.info('Waiting for clean commands to complete...') pool.wait_and_printdots(len(to_clean)) for cmd in pool.getCompletedItems(): res = cmd.get_results() if not res.wasSuccessful(): logger.error('Failed to send cleanup on %s' % cmd.host) logger.error('Error: %s' % res.stderr) raise CleanVerificationError() RemoveVerificationEntry(token=self.token).run() logger.info('Verification %s has been cleaned.' % self.token)
class TriggerFilerepVerifyMessages(Operation): def __init__(self, content, token, batch_default, full=None, verify_file=None, verify_dir=None, abort=None, suspend=None, resume=None, ignore_dir=None, ignore_file=None, results=None, results_level=None): self.content = content self.token = token self.full = full self.verify_file = verify_file self.verify_dir = verify_dir self.abort = abort self.suspend = suspend self.resume = resume self.ignore_dir = ignore_dir self.ignore_file = ignore_file self.results = results self.results_level = results_level self.batch_default = batch_default self.pool = None def execute(self): """ Sends arbitrary gp_primarymirror requests to the backend processes defined. """ to_trigger = ValidateVerification(content = self.content).run() logger.info('Sending gp_primarymirror requests...') self.pool = WorkerPool(min(len(to_trigger), self.batch_default)) for pseg in to_trigger: host, port = pseg.getSegmentHostName(), pseg.getSegmentPort() cmd = SendFilerepVerifyMessage(name = 'verify %s' % host, host = host, port = port, token = self.token, full = self.full, verify_file = self.verify_file, verify_dir = self.verify_dir, abort = self.abort, suspend = self.suspend, resume = self.resume, ignore_dir = self.ignore_dir, ignore_file = self.ignore_file, results = self.results, results_level = self.results_level) logger.debug("Sending request to %s:%d" % (host, port)) self.pool.addCommand(cmd) logger.info('Waiting for gp_primarymirror commands to complete...') self.pool.wait_and_printdots(len(to_trigger)) for cmd in self.pool.getCompletedItems(): res = cmd.get_results() if not res.wasSuccessful(): logger.error('Failed to send gp_primarymirror message to %s:%s' % (cmd.host, cmd.port)) logger.error('Error: %s' % res.stderr) raise TriggerGpPrimaryMirrorFailure() logger.info('gp_primarymirror messages have been triggered succesfully.')
class GatherResults(Operation): def __init__(self, master_datadir, token, content, batch_default): self.master_datadir = master_datadir self.token = token self.content = content self.batch_default = batch_default self.pool = None def execute(self): logger.info('Gathering results of verification %s...' % self.token) to_gather = ValidateVerification(content = self.content, primaries_only = False).run() dest_base = os.path.join(self.master_datadir, 'pg_verify', self.token) if CheckDir(dest_base).run(): # TODO: if end user has mucked around with artifacts on master, a regathering may # be needed; perhaps, a --force option to accompany --results? return MakeDir(dest_base).run() self.pool = WorkerPool(min(len(to_gather), self.batch_default)) for seg in to_gather: host = seg.getSegmentHostName() content = seg.getSegmentContentId() role = seg.getSegmentRole() src = os.path.join(seg.getSegmentDataDirectory(), "pg_verify", "*%s*" % self.token) dest = os.path.join(dest_base, str(content), str(role)) MakeDir(dest).run() cmd = Scp('consolidate', srcFile=src, srcHost=host, dstFile=dest) self.pool.addCommand(cmd) logger.info('Waiting for scp commands to complete...') self.pool.wait_and_printdots(len(to_gather)) self.pool.check_results() dest = os.path.join(dest_base, 'verification_%s.fix' % self.token) with open(dest, 'w') as output: for seg in to_gather: content = seg.getSegmentContentId() role = seg.getSegmentRole() src = os.path.join(dest_base, str(content), str(role), 'verification_%s.fix' % self.token) with open(src, 'r') as input: output.writelines(input.readlines())
def execute(self): """ Sends arbitrary gp_primarymirror requests to the backend processes defined. """ to_trigger = ValidateVerification(content=self.content).run() logger.info('Sending gp_primarymirror requests...') pool = WorkerPool(min(len(to_trigger), self.batch_default)) for pseg in to_trigger: host, port = pseg.getSegmentHostName(), pseg.getSegmentPort() cmd = SendFilerepVerifyMessage(name='verify %s' % host, host=host, port=port, token=self.token, full=self.full, verify_file=self.verify_file, verify_dir=self.verify_dir, abort=self.abort, suspend=self.suspend, resume=self.resume, ignore_dir=self.ignore_dir, ignore_file=self.ignore_file, results=self.results, results_level=self.results_level) logger.debug("Sending request to %s:%d" % (host, port)) pool.addCommand(cmd) logger.info('Waiting for gp_primarymirror commands to complete...') pool.wait_and_printdots(len(to_trigger)) for cmd in pool.getCompletedItems(): res = cmd.get_results() if not res.wasSuccessful(): logger.error( 'Failed to send gp_primarymirror message to %s:%s' % (cmd.host, cmd.port)) logger.error('Error: %s' % res.stderr) raise TriggerGpPrimaryMirrorFailure() logger.info( 'gp_primarymirror messages have been triggered succesfully.')
def execute(self): logger.info('Gathering results of verification %s...' % self.token) to_gather = ValidateVerification(content=self.content, primaries_only=False).run() dest_base = os.path.join(self.master_datadir, 'pg_verify', self.token) if CheckDir(dest_base).run(): # TODO: if end user has mucked around with artifacts on master, a regathering may # be needed; perhaps, a --force option to accompany --results? return MakeDir(dest_base).run() pool = WorkerPool(min(len(to_gather), self.batch_default)) for seg in to_gather: host = seg.getSegmentHostName() content = seg.getSegmentContentId() role = seg.getSegmentRole() src = os.path.join(seg.getSegmentDataDirectory(), "pg_verify", "*%s*" % self.token) dest = os.path.join(dest_base, str(content), str(role)) MakeDir(dest).run() cmd = Scp('consolidate', srcFile=src, srcHost=host, dstFile=dest) pool.addCommand(cmd) logger.info('Waiting for scp commands to complete...') pool.wait_and_printdots(len(to_gather)) pool.check_results() dest = os.path.join(dest_base, 'verification_%s.fix' % self.token) with open(dest, 'w') as output: for seg in to_gather: content = seg.getSegmentContentId() role = seg.getSegmentRole() src = os.path.join(dest_base, str(content), str(role), 'verification_%s.fix' % self.token) with open(src, 'r') as input: output.writelines(input.readlines())
def rebalance(self): # Get the unbalanced primary segments grouped by hostname # These segments are what we will shutdown. logger.info("Getting unbalanced segments") unbalanced_primary_segs = GpArray.getSegmentsByHostName(self.gpArray.get_unbalanced_primary_segdbs()) pool = WorkerPool() count = 0 try: # Disable ctrl-c signal.signal(signal.SIGINT,signal.SIG_IGN) logger.info("Stopping unbalanced primary segments...") for hostname in unbalanced_primary_segs.keys(): cmd = GpSegStopCmd("stop unbalanced primary segs", self.gpEnv.getGpHome(), self.gpEnv.getGpVersion(), 'fast', unbalanced_primary_segs[hostname], ctxt=REMOTE, remoteHost=hostname, timeout=600) pool.addCommand(cmd) count+=1 pool.wait_and_printdots(count, False) failed_count = 0 completed = pool.getCompletedItems() for res in completed: if not res.get_results().wasSuccessful(): failed_count+=1 if failed_count > 0: logger.warn("%d segments failed to stop. A full rebalance of the") logger.warn("system is not possible at this time. Please check the") logger.warn("log files, correct the problem, and run gprecoverseg -r") logger.warn("again.") logger.info("gprecoverseg will continue with a partial rebalance.") pool.empty_completed_items() # issue a distributed query to make sure we pick up the fault # that we just caused by shutting down segments conn = None try: logger.info("Triggering segment reconfiguration") dburl = dbconn.DbURL() conn = dbconn.connect(dburl) cmd = ReconfigDetectionSQLQueryCommand(conn) pool.addCommand(cmd) pool.wait_and_printdots(1, False) except Exception: # This exception is expected pass finally: if conn: conn.close() # Final step is to issue a recoverseg operation to resync segments logger.info("Starting segment synchronization") cmd = GpRecoverseg("rebalance recoverseg") pool.addCommand(cmd) pool.wait_and_printdots(1, False) except Exception, ex: raise ex
def rebalance(self): # Get the unbalanced primary segments grouped by hostname # These segments are what we will shutdown. logger.info("Getting unbalanced segments") unbalanced_primary_segs = GpArray.getSegmentsByHostName( self.gpArray.get_unbalanced_primary_segdbs()) pool = WorkerPool() count = 0 try: # Disable ctrl-c signal.signal(signal.SIGINT, signal.SIG_IGN) logger.info("Stopping unbalanced primary segments...") for hostname in unbalanced_primary_segs.keys(): cmd = GpSegStopCmd("stop unbalanced primary segs", self.gpEnv.getGpHome(), self.gpEnv.getGpVersion(), 'fast', unbalanced_primary_segs[hostname], ctxt=REMOTE, remoteHost=hostname, timeout=600) pool.addCommand(cmd) count += 1 pool.wait_and_printdots(count, False) failed_count = 0 completed = pool.getCompletedItems() for res in completed: if not res.get_results().wasSuccessful(): failed_count += 1 if failed_count > 0: logger.warn( "%d segments failed to stop. A full rebalance of the") logger.warn( "system is not possible at this time. Please check the") logger.warn( "log files, correct the problem, and run gprecoverseg -r") logger.warn("again.") logger.info( "gprecoverseg will continue with a partial rebalance.") pool.empty_completed_items() # issue a distributed query to make sure we pick up the fault # that we just caused by shutting down segments conn = None try: logger.info("Triggering segment reconfiguration") dburl = dbconn.DbURL() conn = dbconn.connect(dburl) cmd = ReconfigDetectionSQLQueryCommand(conn) pool.addCommand(cmd) pool.wait_and_printdots(1, False) except Exception: # This exception is expected pass finally: if conn: conn.close() # Final step is to issue a recoverseg operation to resync segments logger.info("Starting segment synchronization") cmd = GpRecoverseg("rebalance recoverseg") pool.addCommand(cmd) pool.wait_and_printdots(1, False) except Exception, ex: raise ex