def execute(self):
entry = ValidateVerificationEntry(token = self.token).run()
if not entry['verdone']:
raise WrongStateError("Only finished verification tasks may be cleaned up.")
path = os.path.join(get_masterdatadir(), 'pg_verify', self.token)
Command('cleanup', 'rm -rf %s' % path).run(validateAfter=True)
#RemoveTree(path).run()
to_clean = ValidateVerification(content = entry['vercontent'],
primaries_only = False).run()
pool = WorkerPool(min(len(to_clean), self.batch_default))
for seg in to_clean:
host = seg.getSegmentHostName()
path = os.path.join(seg.getSegmentDataDirectory(), 'pg_verify', "*%s*" % self.token)
cmd = Command('cleanup', 'rm -f %s' % path, remoteHost=host)
pool.addCommand(cmd)
logger.info('Waiting for clean commands to complete...')
pool.wait_and_printdots(len(to_clean))
for cmd in pool.getCompletedItems():
res = cmd.get_results()
if not res.wasSuccessful():
logger.error('Failed to send cleanup on %s' % cmd.host)
logger.error('Error: %s' % res.stderr)
raise CleanVerificationError()
RemoveVerificationEntry(token = self.token).run()
logger.info('Verification %s has been cleaned.' % self.token)
def execute(self):
entry = ValidateVerificationEntry(token=self.token).run()
if not entry['verdone']:
raise WrongStateError(
"Only finished verification tasks may be cleaned up.")
path = os.path.join(get_masterdatadir(), 'pg_verify', self.token)
Command('cleanup', 'rm -rf %s' % path).run(validateAfter=True)
#RemoveTree(path).run()
to_clean = ValidateVerification(content=entry['vercontent'],
primaries_only=False).run()
pool = WorkerPool(min(len(to_clean), self.batch_default))
for seg in to_clean:
host = seg.getSegmentHostName()
path = os.path.join(seg.getSegmentDataDirectory(), 'pg_verify',
"*%s*" % self.token)
cmd = Command('cleanup', 'rm -f %s' % path, remoteHost=host)
pool.addCommand(cmd)
logger.info('Waiting for clean commands to complete...')
pool.wait_and_printdots(len(to_clean))
for cmd in pool.getCompletedItems():
res = cmd.get_results()
if not res.wasSuccessful():
logger.error('Failed to send cleanup on %s' % cmd.host)
logger.error('Error: %s' % res.stderr)
raise CleanVerificationError()
RemoveVerificationEntry(token=self.token).run()
logger.info('Verification %s has been cleaned.' % self.token)
class TriggerFilerepVerifyMessages(Operation):
def __init__(self, content, token, batch_default, full=None, verify_file=None, verify_dir=None,
abort=None, suspend=None, resume=None, ignore_dir=None, ignore_file=None,
results=None, results_level=None):
self.content = content
self.token = token
self.full = full
self.verify_file = verify_file
self.verify_dir = verify_dir
self.abort = abort
self.suspend = suspend
self.resume = resume
self.ignore_dir = ignore_dir
self.ignore_file = ignore_file
self.results = results
self.results_level = results_level
self.batch_default = batch_default
self.pool = None
def execute(self):
"""
Sends arbitrary gp_primarymirror requests to the backend processes defined.
"""
to_trigger = ValidateVerification(content = self.content).run()
logger.info('Sending gp_primarymirror requests...')
self.pool = WorkerPool(min(len(to_trigger), self.batch_default))
for pseg in to_trigger:
host, port = pseg.getSegmentHostName(), pseg.getSegmentPort()
cmd = SendFilerepVerifyMessage(name = 'verify %s' % host, host = host, port = port,
token = self.token,
full = self.full,
verify_file = self.verify_file,
verify_dir = self.verify_dir,
abort = self.abort,
suspend = self.suspend,
resume = self.resume,
ignore_dir = self.ignore_dir,
ignore_file = self.ignore_file,
results = self.results,
results_level = self.results_level)
logger.debug("Sending request to %s:%d" % (host, port))
self.pool.addCommand(cmd)
logger.info('Waiting for gp_primarymirror commands to complete...')
self.pool.wait_and_printdots(len(to_trigger))
for cmd in self.pool.getCompletedItems():
res = cmd.get_results()
if not res.wasSuccessful():
logger.error('Failed to send gp_primarymirror message to %s:%s' % (cmd.host, cmd.port))
logger.error('Error: %s' % res.stderr)
raise TriggerGpPrimaryMirrorFailure()
logger.info('gp_primarymirror messages have been triggered succesfully.')
class GatherResults(Operation):
def __init__(self, master_datadir, token, content, batch_default):
self.master_datadir = master_datadir
self.token = token
self.content = content
self.batch_default = batch_default
self.pool = None
def execute(self):
logger.info('Gathering results of verification %s...' % self.token)
to_gather = ValidateVerification(content = self.content,
primaries_only = False).run()
dest_base = os.path.join(self.master_datadir, 'pg_verify', self.token)
if CheckDir(dest_base).run():
# TODO: if end user has mucked around with artifacts on master, a regathering may
# be needed; perhaps, a --force option to accompany --results?
return
MakeDir(dest_base).run()
self.pool = WorkerPool(min(len(to_gather), self.batch_default))
for seg in to_gather:
host = seg.getSegmentHostName()
content = seg.getSegmentContentId()
role = seg.getSegmentRole()
src = os.path.join(seg.getSegmentDataDirectory(), "pg_verify", "*%s*" % self.token)
dest = os.path.join(dest_base, str(content), str(role))
MakeDir(dest).run()
cmd = Scp('consolidate', srcFile=src, srcHost=host, dstFile=dest)
self.pool.addCommand(cmd)
logger.info('Waiting for scp commands to complete...')
self.pool.wait_and_printdots(len(to_gather))
self.pool.check_results()
dest = os.path.join(dest_base, 'verification_%s.fix' % self.token)
with open(dest, 'w') as output:
for seg in to_gather:
content = seg.getSegmentContentId()
role = seg.getSegmentRole()
src = os.path.join(dest_base, str(content), str(role), 'verification_%s.fix' % self.token)
with open(src, 'r') as input:
output.writelines(input.readlines())
def execute(self):
"""
Sends arbitrary gp_primarymirror requests to the backend processes defined.
"""
to_trigger = ValidateVerification(content=self.content).run()
logger.info('Sending gp_primarymirror requests...')
pool = WorkerPool(min(len(to_trigger), self.batch_default))
for pseg in to_trigger:
host, port = pseg.getSegmentHostName(), pseg.getSegmentPort()
cmd = SendFilerepVerifyMessage(name='verify %s' % host,
host=host,
port=port,
token=self.token,
full=self.full,
verify_file=self.verify_file,
verify_dir=self.verify_dir,
abort=self.abort,
suspend=self.suspend,
resume=self.resume,
ignore_dir=self.ignore_dir,
ignore_file=self.ignore_file,
results=self.results,
results_level=self.results_level)
logger.debug("Sending request to %s:%d" % (host, port))
pool.addCommand(cmd)
logger.info('Waiting for gp_primarymirror commands to complete...')
pool.wait_and_printdots(len(to_trigger))
for cmd in pool.getCompletedItems():
res = cmd.get_results()
if not res.wasSuccessful():
logger.error(
'Failed to send gp_primarymirror message to %s:%s' %
(cmd.host, cmd.port))
logger.error('Error: %s' % res.stderr)
raise TriggerGpPrimaryMirrorFailure()
logger.info(
'gp_primarymirror messages have been triggered succesfully.')
def execute(self):
logger.info('Gathering results of verification %s...' % self.token)
to_gather = ValidateVerification(content=self.content,
primaries_only=False).run()
dest_base = os.path.join(self.master_datadir, 'pg_verify', self.token)
if CheckDir(dest_base).run():
# TODO: if end user has mucked around with artifacts on master, a regathering may
# be needed; perhaps, a --force option to accompany --results?
return
MakeDir(dest_base).run()
pool = WorkerPool(min(len(to_gather), self.batch_default))
for seg in to_gather:
host = seg.getSegmentHostName()
content = seg.getSegmentContentId()
role = seg.getSegmentRole()
src = os.path.join(seg.getSegmentDataDirectory(), "pg_verify",
"*%s*" % self.token)
dest = os.path.join(dest_base, str(content), str(role))
MakeDir(dest).run()
cmd = Scp('consolidate', srcFile=src, srcHost=host, dstFile=dest)
pool.addCommand(cmd)
logger.info('Waiting for scp commands to complete...')
pool.wait_and_printdots(len(to_gather))
pool.check_results()
dest = os.path.join(dest_base, 'verification_%s.fix' % self.token)
with open(dest, 'w') as output:
for seg in to_gather:
content = seg.getSegmentContentId()
role = seg.getSegmentRole()
src = os.path.join(dest_base, str(content), str(role),
'verification_%s.fix' % self.token)
with open(src, 'r') as input:
output.writelines(input.readlines())
def rebalance(self):
# Get the unbalanced primary segments grouped by hostname
# These segments are what we will shutdown.
logger.info("Getting unbalanced segments")
unbalanced_primary_segs = GpArray.getSegmentsByHostName(self.gpArray.get_unbalanced_primary_segdbs())
pool = WorkerPool()
count = 0
try:
# Disable ctrl-c
signal.signal(signal.SIGINT,signal.SIG_IGN)
logger.info("Stopping unbalanced primary segments...")
for hostname in unbalanced_primary_segs.keys():
cmd = GpSegStopCmd("stop unbalanced primary segs",
self.gpEnv.getGpHome(),
self.gpEnv.getGpVersion(),
'fast',
unbalanced_primary_segs[hostname],
ctxt=REMOTE,
remoteHost=hostname,
timeout=600)
pool.addCommand(cmd)
count+=1
pool.wait_and_printdots(count, False)
failed_count = 0
completed = pool.getCompletedItems()
for res in completed:
if not res.get_results().wasSuccessful():
failed_count+=1
if failed_count > 0:
logger.warn("%d segments failed to stop. A full rebalance of the")
logger.warn("system is not possible at this time. Please check the")
logger.warn("log files, correct the problem, and run gprecoverseg -r")
logger.warn("again.")
logger.info("gprecoverseg will continue with a partial rebalance.")
pool.empty_completed_items()
# issue a distributed query to make sure we pick up the fault
# that we just caused by shutting down segments
conn = None
try:
logger.info("Triggering segment reconfiguration")
dburl = dbconn.DbURL()
conn = dbconn.connect(dburl)
cmd = ReconfigDetectionSQLQueryCommand(conn)
pool.addCommand(cmd)
pool.wait_and_printdots(1, False)
except Exception:
# This exception is expected
pass
finally:
if conn:
conn.close()
# Final step is to issue a recoverseg operation to resync segments
logger.info("Starting segment synchronization")
cmd = GpRecoverseg("rebalance recoverseg")
pool.addCommand(cmd)
pool.wait_and_printdots(1, False)
except Exception, ex:
raise ex
def rebalance(self):
# Get the unbalanced primary segments grouped by hostname
# These segments are what we will shutdown.
logger.info("Getting unbalanced segments")
unbalanced_primary_segs = GpArray.getSegmentsByHostName(
self.gpArray.get_unbalanced_primary_segdbs())
pool = WorkerPool()
count = 0
try:
# Disable ctrl-c
signal.signal(signal.SIGINT, signal.SIG_IGN)
logger.info("Stopping unbalanced primary segments...")
for hostname in unbalanced_primary_segs.keys():
cmd = GpSegStopCmd("stop unbalanced primary segs",
self.gpEnv.getGpHome(),
self.gpEnv.getGpVersion(),
'fast',
unbalanced_primary_segs[hostname],
ctxt=REMOTE,
remoteHost=hostname,
timeout=600)
pool.addCommand(cmd)
count += 1
pool.wait_and_printdots(count, False)
failed_count = 0
completed = pool.getCompletedItems()
for res in completed:
if not res.get_results().wasSuccessful():
failed_count += 1
if failed_count > 0:
logger.warn(
"%d segments failed to stop. A full rebalance of the")
logger.warn(
"system is not possible at this time. Please check the")
logger.warn(
"log files, correct the problem, and run gprecoverseg -r")
logger.warn("again.")
logger.info(
"gprecoverseg will continue with a partial rebalance.")
pool.empty_completed_items()
# issue a distributed query to make sure we pick up the fault
# that we just caused by shutting down segments
conn = None
try:
logger.info("Triggering segment reconfiguration")
dburl = dbconn.DbURL()
conn = dbconn.connect(dburl)
cmd = ReconfigDetectionSQLQueryCommand(conn)
pool.addCommand(cmd)
pool.wait_and_printdots(1, False)
except Exception:
# This exception is expected
pass
finally:
if conn:
conn.close()
# Final step is to issue a recoverseg operation to resync segments
logger.info("Starting segment synchronization")
cmd = GpRecoverseg("rebalance recoverseg")
pool.addCommand(cmd)
pool.wait_and_printdots(1, False)
except Exception, ex:
raise ex