class RecoverRemoteDumps(Operation):
def __init__(self, host, path, restore_timestamp, compress, restore_global, batch_default, master_datadir, master_port):
self.host = host
self.path = path
self.restore_timestamp = restore_timestamp
self.compress = compress
self.restore_global = restore_global
self.batch_default = batch_default
self.master_datadir = master_datadir
self.master_port = master_port
def execute(self):
gparray = GpArray.initFromCatalog(dbconn.DbURL(port=self.master_port), utility=True)
from_host, from_path = self.host, self.path
logger.info("Commencing remote database dump file recovery process, please wait...")
segs = [seg for seg in gparray.getDbList() if seg.isSegmentPrimary(current_role=True) or seg.isSegmentMaster()]
pool = WorkerPool(numWorkers = min(len(segs), self.batch_default))
for seg in segs:
if seg.isSegmentMaster():
file = '%s%s' % (MASTER_DBDUMP_PREFIX, self.restore_timestamp)
else:
file = '%s0_%d_%s' % (DBDUMP_PREFIX, seg.getSegmentDbId(), self.restore_timestamp)
if self.compress:
file += '.gz'
to_host = seg.getSegmentHostName()
to_path = os.path.join(seg.getSegmentDataDirectory(), DUMP_DIR, self.restore_timestamp[0:8])
if not CheckRemoteDir(to_path, to_host).run():
logger.info('Creating directory %s on %s' % (to_path, to_host))
try:
MakeRemoteDir(to_path, to_host).run()
except OSError, e:
raise ExceptionNoStackTraceNeeded("Failed to create directory %s on %s" % (to_path, to_host))
logger.info("Commencing remote copy from %s to %s:%s" % (from_host, to_host, to_path))
pool.addCommand(Scp('Copying dump for seg %d' % seg.getSegmentDbId(),
srcFile=os.path.join(from_path, file),
dstFile=os.path.join(to_path, file),
srcHost=from_host,
dstHost=to_host))
createdb_file = "%s%s" % (CREATEDB_PREFIX, self.restore_timestamp)
to_path = os.path.join(self.master_datadir, DUMP_DIR, self.restore_timestamp[0:8])
pool.addCommand(Scp('Copying schema dump',
srcHost=from_host,
srcFile=os.path.join(from_path, createdb_file),
dstFile=os.path.join(to_path, createdb_file)))
post_data_file = "%s%s%s" % (MASTER_DBDUMP_PREFIX, self.restore_timestamp, POST_DATA_SUFFIX)
if self.compress:
post_data_file += ".gz"
pool.addCommand(Scp('Copying post data schema dump',
srcHost=from_host,
srcFile=os.path.join(from_path, post_data_file),
dstFile=os.path.join(to_path, post_data_file)))
if self.restore_global:
global_file = "%s%s" % (GLOBAL_PREFIX, self.restore_timestamp)
pool.addCommand(Scp("Copying global dump",
srcHost=from_host,
srcFile=os.path.join(from_path, global_file),
dstFile=os.path.join(to_path, global_file)))
pool.join()
pool.check_results()
def execute(self):
gparray = GpArray.initFromCatalog(dbconn.DbURL(port=self.master_port), utility=True)
from_host, from_path = self.host, self.path
logger.info("Commencing remote database dump file recovery process, please wait...")
segs = [seg for seg in gparray.getDbList() if seg.isSegmentPrimary(current_role=True) or seg.isSegmentMaster()]
pool = WorkerPool(numWorkers = min(len(segs), self.batch_default))
for seg in segs:
if seg.isSegmentMaster():
file = '%s%s' % (MASTER_DBDUMP_PREFIX, self.restore_timestamp)
else:
file = '%s0_%d_%s' % (DBDUMP_PREFIX, seg.getSegmentDbId(), self.restore_timestamp)
if self.compress:
file += '.gz'
to_host = seg.getSegmentHostName()
to_path = os.path.join(seg.getSegmentDataDirectory(), DUMP_DIR, self.restore_timestamp[0:8])
if not CheckRemoteDir(to_path, to_host).run():
logger.info('Creating directory %s on %s' % (to_path, to_host))
try:
MakeRemoteDir(to_path, to_host).run()
except OSError, e:
raise ExceptionNoStackTraceNeeded("Failed to create directory %s on %s" % (to_path, to_host))
logger.info("Commencing remote copy from %s to %s:%s" % (from_host, to_host, to_path))
pool.addCommand(Scp('Copying dump for seg %d' % seg.getSegmentDbId(),
srcFile=os.path.join(from_path, file),
dstFile=os.path.join(to_path, file),
srcHost=from_host,
dstHost=to_host))
def test11_install_rpm_on_segments_and_uninstall(self):
"""
Try to uninstall a gppkg, when the gppkg has not been installed,
but the main comprising rpm has been installed on one of the segments.
The desired result is that the rpm should be uninstalled.
"""
gppkg_file = self.alpha_spec.get_filename()
segment_host_list = get_host_list()[1]
self.assertTrue(len(segment_host_list) > 0)
host = segment_host_list[0]
#Install the rpm
with closing(tarfile.open(self.alpha_spec.get_filename())) as tf:
tf.extract(self.A_spec.get_filename())
Scp(name="copy rpm to segment",
srcFile=self.A_spec.get_filename(),
dstFile=self.A_spec.get_filename(),
srcHost=None,
dstHost=host).run(validateAfter=True)
self.install_rpm_remotely(self.A_spec.get_filename(), host)
try:
self.remove(gppkg_file)
except ExecutionError as e:
self.uninstall_rpm_remotely(self.A_spec.get_filename(), host)
os.remove(self.A_spec.get_filename())
RemoveRemoteFile(self.A_spec.get_filename(), host).run()
self.fail("ExecutionError %s" % e)
def test08_uninstall_rpm_on_segments_and_install(self):
"""
Try to install a gppkg again, when rpm has not been
installed on one of the segments.
"""
#Use gppkg from previous test
gppkg_file = self.alpha_spec.get_filename()
self.install(gppkg_file)
segment_host_list = get_host_list()[1]
self.assertTrue(len(segment_host_list) > 0)
host = segment_host_list[0]
self.uninstall_rpm_remotely(self.A_spec.get_filename(), host)
try:
self.install(gppkg_file)
except ExecutionError as e:
#Install the rpm
with closing(tarfile.open(self.alpha_spec.get_filename())) as tf:
tf.extract(self.A_spec.get_filename())
Scp(name="copy rpm to segment",
srcFile=self.A_spec.get_filename(),
dstFile=self.A_spec.get_filename(),
srcHost=None,
dstHost=host).run(validateAfter=True)
self.install_rpm_remotely(self.A_spec.get_filename(), host)
os.remove(self.A_spec.get_filename())
RemoveRemoteFile(self.A_spec.get_filename(), host).run()
self.fail("ExecutionError %s" % e)
def scp_file_to_hosts(host_list, filename, batch_default):
pool = WorkerPool(numWorkers=min(len(host_list), batch_default))
for hname in host_list:
pool.addCommand(Scp('Copying table_filter_file to %s' % hname,
srcFile=filename,
dstFile=filename,
dstHost=hname))
pool.join()
pool.haltWork()
pool.check_results()
def execute(self):
pool = WorkerPool()
given = set(ListFiles(self.pid_dir).run())
try:
for host in self.trail:
available = ListRemoteFiles(self.pid_dir, host).run()
to_copy = [dir for dir in available if dir not in given]
for dir in to_copy:
comp_dir = os.path.join(self.pid_dir, dir)
pool.addCommand(Scp('collect coverage',
srcFile = comp_dir,
srcHost = host,
dstFile = comp_dir,
recursive = True))
finally:
pool.join()
def test04_install_rpm_on_standby_and_install(self):
gppkg_file = self.alpha_spec.get_filename()
standby = get_host_list()[0]
#Install the rpm
with closing(tarfile.open(self.alpha_spec.get_filename())) as tf:
tf.extract(self.A_spec.get_filename())
Scp(name="copy the rpm to standby",
srcFile=self.A_spec.get_filename(),
dstFile=self.A_spec.get_filename(),
srcHost=None,
dstHost=standby).run(validateAfter=True)
self.install_rpm_remotely(self.A_spec.get_filename(), standby)
self.install(gppkg_file)
def test05_delete_package_from_archive_on_standby(self):
'''
Known issue: MPP-15737
'''
gppkg_file = self.gppkg_spec.get_filename()
archive_file = os.path.join(ARCHIVE_PATH, gppkg_file)
self.install(gppkg_file)
#Remove package from standby
standby = get_host_list()[0]
RemoveRemoteFile(os.path.join(ARCHIVE_PATH, gppkg_file), standby).run()
try:
self.install(gppkg_file)
except ExecutionError, e:
Scp(name = "copy to segment", srcFile = gppkg_file, dstFile = archive_file, srcHost = None, dstHost = standby).run(validateAfter = True)
self.fail("ExecutionError %s" % e)
def test00_delete_package_from_archive_on_standby_and_install(self):
gppkg_file = self.build(self.alpha_spec, self.A_spec)
archive_file = os.path.join(ARCHIVE_PATH, gppkg_file)
self.install(gppkg_file)
#Remove package from standby
standby = get_host_list()[0]
RemoveRemoteFile(os.path.join(ARCHIVE_PATH, gppkg_file), standby).run()
try:
self.install(gppkg_file)
except ExecutionError as e:
Scp(name="copy gppkg to standby",
srcFile=gppkg_file,
dstFile=archive_file,
srcHost=None,
dstHost=standby).run(validateAfter=True)
self.fail("ExecutionError %s" % e)
def test04_delete_package_from_archive_on_segment(self):
'''
Known issue: MPP-15737
'''
gppkg_file = self.gppkg_spec.get_filename()
archive_file = os.path.join(ARCHIVE_PATH, gppkg_file)
self.install(gppkg_file)
# Remove package from archive
segment_host_list = get_host_list()[1]
self.assertTrue(len(segment_host_list) > 0)
RemoveRemoteFile(archive_file, segment_host_list[0])
try:
self.install(gppkg_file)
except ExecutionError as e:
Scp(name="copy to segment", srcFile=gppkg_file, dstFile=archive_file, srcHost=None,
dstHost=segment_host_list[0]).run(validateAfter=True)
self.fail("ExecutionError %s" % e)
def test10_install_rpm_on_segments_and_install(self):
"""
Try to install a gppkg when an rpm has already been
installed on one of the segments.
"""
gppkg_file = self.alpha_spec.get_filename()
segment_host_list = get_host_list()[1]
self.assertTrue(len(segment_host_list) > 0)
host = segment_host_list[0]
#Install the rpm
with closing(tarfile.open(self.alpha_spec.get_filename())) as tf:
tf.extract(self.A_spec.get_filename())
Scp(name="copy rpm to segment",
srcFile=self.A_spec.get_filename(),
dstFile=self.A_spec.get_filename(),
srcHost=None,
dstHost=host).run(validateAfter=True)
self.install_rpm_remotely(self.A_spec.get_filename(), host)
self.install(gppkg_file)
def execute(self):
logger.info('Gathering results of verification %s...' % self.token)
to_gather = ValidateVerification(content=self.content,
primaries_only=False).run()
dest_base = os.path.join(self.master_datadir, 'pg_verify', self.token)
if CheckDir(dest_base).run():
# TODO: if end user has mucked around with artifacts on master, a regathering may
# be needed; perhaps, a --force option to accompany --results?
return
MakeDir(dest_base).run()
pool = WorkerPool(min(len(to_gather), self.batch_default))
for seg in to_gather:
host = seg.getSegmentHostName()
content = seg.getSegmentContentId()
role = seg.getSegmentRole()
src = os.path.join(seg.getSegmentDataDirectory(), "pg_verify",
"*%s*" % self.token)
dest = os.path.join(dest_base, str(content), str(role))
MakeDir(dest).run()
cmd = Scp('consolidate', srcFile=src, srcHost=host, dstFile=dest)
pool.addCommand(cmd)
logger.info('Waiting for scp commands to complete...')
pool.wait_and_printdots(len(to_gather))
pool.check_results()
dest = os.path.join(dest_base, 'verification_%s.fix' % self.token)
with open(dest, 'w') as output:
for seg in to_gather:
content = seg.getSegmentContentId()
role = seg.getSegmentRole()
src = os.path.join(dest_base, str(content), str(role),
'verification_%s.fix' % self.token)
with open(src, 'r') as input:
output.writelines(input.readlines())
def test02_uninstall_rpm_on_standby_and_install(self):
#Use gppkg from previous test
gppkg_file = self.alpha_spec.get_filename()
self.install(gppkg_file)
standby = get_host_list()[0]
self.uninstall_rpm_remotely(self.A_spec.get_filename(), standby)
try:
self.install(gppkg_file)
except ExecutionError as e:
#Install the rpm
with closing(tarfile.open(self.alpha_spec.get_filename())) as tf:
tf.extract(self.A_spec.get_filename())
Scp(name="copy rpm to standby",
srcFile=self.A_spec.get_filename(),
dstFile=self.A_spec.get_filename(),
srcHost=None,
dstHost=standby).run(validateAfter=True)
self.install_rpm_remotely(self.A_spec.get_filename(), standby)
os.remove(self.A_spec.get_filename())
RemoveRemoteFile(self.A_spec.get_filename(), standby).run()
self.fail("ExecutionError %s" % e)
def test01_delete_package_from_archive_on_standby_and_uninstall(self):
"""
Known issue: MPP-15737
"""
gppkg_file = self.alpha_spec.get_filename()
archive_file = os.path.join(ARCHIVE_PATH, gppkg_file)
self.install(gppkg_file)
#Remove package from standby
standby = get_host_list()[0]
RemoveRemoteFile(os.path.join(ARCHIVE_PATH, gppkg_file), standby).run()
try:
self.remove(gppkg_file)
except ExecutionError as e:
GpScp(source_path=gppkg_file,
target_path=archive_file,
host_list=get_host_list()[1]).run()
Scp(name="copy gppkg to standby",
srcFile=gppkg_file,
dstFile=archive_file,
srcHost=None,
dstHost=standby).run(validateAfter=True)
self.fail("ExecutionError %s" % e)
def test06_delete_package_from_archive_on_segment_and_install(self):
"""
Delete a package from the archive on the segment
and try to install the gppkg again.
"""
gppkg_file = self.alpha_spec.get_filename()
archive_file = os.path.join(ARCHIVE_PATH, gppkg_file)
self.install(gppkg_file)
#Remove package from archive
segment_host_list = get_host_list()[1]
self.assertTrue(len(segment_host_list) > 0)
host = segment_host_list[0]
RemoveRemoteFile(archive_file, host).run()
try:
self.install(gppkg_file)
except ExecutionError as e:
Scp(name="copy gppkg to segment",
srcFile=gppkg_file,
dstFile=archive_file,
srcHost=None,
dstHost=host).run(validateAfter=True)
self.fail("ExecutionError %s" % e)
def __copySegmentDirectories(self, gpEnv, gpArray, directives):
"""
directives should be composed of GpCopySegmentDirectoryDirective values
"""
if len(directives) == 0:
return
srcSegments = [d.getSrcSegment() for d in directives]
destSegments = [d.getDestSegment() for d in directives]
isTargetReusedLocation = [
d.isTargetReusedLocation() for d in directives
]
destSegmentByHost = GpArray.getSegmentsByHostName(destSegments)
newSegmentInfo = gp.ConfigureNewSegment.buildSegmentInfoForNewSegment(
destSegments, isTargetReusedLocation)
logger.info('Building template directory')
(tempDir, blankTarFile,
tarFileName) = self.__buildTarFileForTransfer(gpEnv, gpArray.master,
srcSegments[0],
destSegments)
def createConfigureNewSegmentCommand(hostName, cmdLabel,
validationOnly):
segmentInfo = newSegmentInfo[hostName]
checkNotNone("segmentInfo for %s" % hostName, segmentInfo)
return gp.ConfigureNewSegment(cmdLabel,
segmentInfo,
tarFile=tarFileName,
newSegments=True,
verbose=gplog.logging_is_verbose(),
batchSize=self.__parallelDegree,
ctxt=gp.REMOTE,
remoteHost=hostName,
validationOnly=validationOnly)
#
# validate directories for target segments
#
logger.info('Validating remote directories')
cmds = []
for hostName in destSegmentByHost.keys():
cmds.append(
createConfigureNewSegmentCommand(hostName,
'validate blank segments',
True))
for cmd in cmds:
self.__pool.addCommand(cmd)
self.__pool.wait_and_printdots(len(cmds), self.__quiet)
validationErrors = []
for item in self.__pool.getCompletedItems():
results = item.get_results()
if not results.wasSuccessful():
if results.rc == 1:
# stdoutFromFailure = results.stdout.replace("\n", " ").strip()
lines = results.stderr.split("\n")
for line in lines:
if len(line.strip()) > 0:
validationErrors.append(
"Validation failure on host %s %s" %
(item.remoteHost, line))
else:
validationErrors.append(str(item))
self.__pool.empty_completed_items()
if validationErrors:
raise ExceptionNoStackTraceNeeded("\n" +
("\n".join(validationErrors)))
#
# copy tar from master to target hosts
#
logger.info('Copying template directory file')
cmds = []
for hostName in destSegmentByHost.keys():
cmds.append(
gp.RemoteCopy("copy segment tar", blankTarFile, hostName,
tarFileName))
self.__runWaitAndCheckWorkerPoolForErrorsAndClear(
cmds, "building and transferring basic segment directory")
#
# unpack and configure new segments
#
logger.info('Configuring new segments')
cmds = []
for hostName in destSegmentByHost.keys():
cmds.append(
createConfigureNewSegmentCommand(hostName,
'configure blank segments',
False))
self.__runWaitAndCheckWorkerPoolForErrorsAndClear(
cmds, "unpacking basic segment directory")
#
# copy dump files from old segment to new segment
#
for srcSeg in srcSegments:
for destSeg in destSegments:
if srcSeg.content == destSeg.content:
cmd = Scp('copy db_dumps from old segment to new segment',
os.path.join(srcSeg.getSegmentDataDirectory(),
'db_dumps*', '*'),
os.path.join(destSeg.getSegmentDataDirectory(),
'db_dumps'),
srcSeg.getSegmentAddress(),
destSeg.getSegmentAddress(),
recursive=True)
cmd.run(validateAfter=True)
break
#
# Clean up copied tar from each remote host
#
logger.info('Cleaning files')
cmds = []
for hostName, segments in destSegmentByHost.iteritems():
cmds.append(
unix.RemoveFiles('remove tar file',
tarFileName,
ctxt=gp.REMOTE,
remoteHost=hostName))
self.__runWaitAndCheckWorkerPoolForErrorsAndClear(
cmds, "cleaning up tar file on segment hosts")
#
# clean up the local temp directory
#
unix.RemoveFiles.local('remove temp directory', tempDir)
def __copySegmentDirectories(self, gpEnv, gpArray, directives):
"""
directives should be composed of GpCopySegmentDirectoryDirective values
"""
if len(directives) == 0:
return
srcSegments = []
destSegments = []
isTargetReusedLocation = []
timeStamp = datetime.datetime.today().strftime('%Y%m%d_%H%M%S')
for directive in directives:
srcSegment = directive.getSrcSegment()
destSegment = directive.getDestSegment()
destSegment.primaryHostname = srcSegment.getSegmentHostName()
destSegment.primarySegmentPort = srcSegment.getSegmentPort()
destSegment.progressFile = '%s/pg_basebackup.%s.dbid%s.out' % (
gplog.get_logger_dir(), timeStamp,
destSegment.getSegmentDbId())
srcSegments.append(srcSegment)
destSegments.append(destSegment)
isTargetReusedLocation.append(directive.isTargetReusedLocation())
destSegmentByHost = GpArray.getSegmentsByHostName(destSegments)
newSegmentInfo = gp.ConfigureNewSegment.buildSegmentInfoForNewSegment(
destSegments, isTargetReusedLocation)
def createConfigureNewSegmentCommand(hostName, cmdLabel,
validationOnly):
segmentInfo = newSegmentInfo[hostName]
checkNotNone("segmentInfo for %s" % hostName, segmentInfo)
return gp.ConfigureNewSegment(cmdLabel,
segmentInfo,
gplog.get_logger_dir(),
newSegments=True,
verbose=gplog.logging_is_verbose(),
batchSize=self.__parallelDegree,
ctxt=gp.REMOTE,
remoteHost=hostName,
validationOnly=validationOnly,
forceoverwrite=self.__forceoverwrite)
#
# validate directories for target segments
#
self.__logger.info('Validating remote directories')
cmds = []
for hostName in list(destSegmentByHost.keys()):
cmds.append(
createConfigureNewSegmentCommand(hostName,
'validate blank segments',
True))
for cmd in cmds:
self.__pool.addCommand(cmd)
if self.__quiet:
self.__pool.join()
else:
base.join_and_indicate_progress(self.__pool)
validationErrors = []
for item in self.__pool.getCompletedItems():
results = item.get_results()
if not results.wasSuccessful():
if results.rc == 1:
# stdoutFromFailure = results.stdout.replace("\n", " ").strip()
lines = results.stderr.split("\n")
for line in lines:
if len(line.strip()) > 0:
validationErrors.append(
"Validation failure on host %s %s" %
(item.remoteHost, line))
else:
validationErrors.append(str(item))
self.__pool.empty_completed_items()
if validationErrors:
raise ExceptionNoStackTraceNeeded("\n" +
("\n".join(validationErrors)))
# Configure a new segment
#
# Recover segments using gpconfigurenewsegment, which
# uses pg_basebackup. gprecoverseg generates a log filename which is
# passed to gpconfigurenewsegment as a confinfo parameter. gprecoverseg
# tails this file to show recovery progress to the user, and removes the
# file when one done. A new file is generated for each run of
# gprecoverseg based on a timestamp.
self.__logger.info('Configuring new segments')
cmds = []
progressCmds = []
removeCmds = []
for hostName in list(destSegmentByHost.keys()):
for segment in destSegmentByHost[hostName]:
progressCmd, removeCmd = self.__getProgressAndRemoveCmds(
segment.progressFile, segment.getSegmentDbId(), hostName)
removeCmds.append(removeCmd)
if progressCmd:
progressCmds.append(progressCmd)
cmds.append(
createConfigureNewSegmentCommand(hostName,
'configure blank segments',
False))
self.__runWaitAndCheckWorkerPoolForErrorsAndClear(
cmds,
"unpacking basic segment directory",
suppressErrorCheck=False,
progressCmds=progressCmds)
self.__runWaitAndCheckWorkerPoolForErrorsAndClear(
removeCmds,
"removing pg_basebackup progress logfiles",
suppressErrorCheck=False)
#
# copy dump files from old segment to new segment
#
for srcSeg in srcSegments:
for destSeg in destSegments:
if srcSeg.content == destSeg.content:
src_dump_dir = os.path.join(
srcSeg.getSegmentDataDirectory(), 'db_dumps')
cmd = base.Command('check existence of db_dumps directory',
'ls %s' % (src_dump_dir),
ctxt=base.REMOTE,
remoteHost=destSeg.getSegmentAddress())
cmd.run()
if cmd.results.rc == 0: # Only try to copy directory if it exists
cmd = Scp(
'copy db_dumps from old segment to new segment',
os.path.join(srcSeg.getSegmentDataDirectory(),
'db_dumps*', '*'),
os.path.join(destSeg.getSegmentDataDirectory(),
'db_dumps'),
srcSeg.getSegmentAddress(),
destSeg.getSegmentAddress(),
recursive=True)
cmd.run(validateAfter=True)
break
def __copySegmentDirectories(self, gpEnv, gpArray, directives):
"""
directives should be composed of GpCopySegmentDirectoryDirective values
"""
if len(directives) == 0:
return
srcSegments = [d.getSrcSegment() for d in directives]
destSegments = [d.getDestSegment() for d in directives]
isTargetReusedLocation = [d.isTargetReusedLocation() for d in directives]
destSegmentByHost = GpArray.getSegmentsByHostName(destSegments)
newSegmentInfo = gp.ConfigureNewSegment.buildSegmentInfoForNewSegment(destSegments, isTargetReusedLocation)
logger.info('Building template directory')
(tempDir, blankTarFile, tarFileName) = self.__buildTarFileForTransfer(gpEnv, gpArray.master, srcSegments[0], destSegments)
def createConfigureNewSegmentCommand(hostName, cmdLabel, validationOnly):
segmentInfo = newSegmentInfo[hostName]
checkNotNone("segmentInfo for %s" % hostName, segmentInfo)
return gp.ConfigureNewSegment(cmdLabel,
segmentInfo,
tarFile=tarFileName,
newSegments=True,
verbose=gplog.logging_is_verbose(),
batchSize=self.__parallelDegree,
ctxt=gp.REMOTE,
remoteHost=hostName,
validationOnly=validationOnly)
#
# validate directories for target segments
#
logger.info('Validating remote directories')
cmds = []
for hostName in destSegmentByHost.keys():
cmds.append(createConfigureNewSegmentCommand(hostName, 'validate blank segments', True))
for cmd in cmds:
self.__pool.addCommand(cmd)
self.__pool.wait_and_printdots(len(cmds), self.__quiet)
validationErrors = []
for item in self.__pool.getCompletedItems():
results = item.get_results()
if not results.wasSuccessful():
if results.rc == 1:
# stdoutFromFailure = results.stdout.replace("\n", " ").strip()
lines = results.stderr.split("\n")
for line in lines:
if len(line.strip()) > 0:
validationErrors.append("Validation failure on host %s %s" % (item.remoteHost, line))
else:
validationErrors.append(str(item))
self.__pool.empty_completed_items()
if validationErrors:
raise ExceptionNoStackTraceNeeded("\n" + ("\n".join(validationErrors)))
#
# copy tar from master to target hosts
#
logger.info('Copying template directory file')
cmds = []
for hostName in destSegmentByHost.keys():
cmds.append( gp.RemoteCopy("copy segment tar", blankTarFile, hostName, tarFileName ))
self.__runWaitAndCheckWorkerPoolForErrorsAndClear(cmds, "building and transferring basic segment directory")
#
# unpack and configure new segments
#
logger.info('Configuring new segments')
cmds = []
for hostName in destSegmentByHost.keys():
cmds.append(createConfigureNewSegmentCommand(hostName, 'configure blank segments', False))
self.__runWaitAndCheckWorkerPoolForErrorsAndClear(cmds, "unpacking basic segment directory")
#
# copy dump files from old segment to new segment
#
for srcSeg in srcSegments:
for destSeg in destSegments:
if srcSeg.content == destSeg.content:
src_dump_dir = os.path.join(srcSeg.getSegmentDataDirectory(), 'db_dumps')
cmd = base.Command('check existence of db_dumps directory', 'ls %s' % (src_dump_dir), ctxt=base.REMOTE, remoteHost=destSeg.getSegmentAddress())
cmd.run()
if cmd.results.rc == 0: # Only try to copy directory if it exists
cmd = Scp('copy db_dumps from old segment to new segment',
os.path.join(srcSeg.getSegmentDataDirectory(), 'db_dumps*', '*'),
os.path.join(destSeg.getSegmentDataDirectory(), 'db_dumps'),
srcSeg.getSegmentAddress(),
destSeg.getSegmentAddress(),
recursive=True)
cmd.run(validateAfter=True)
break
#
# Clean up copied tar from each remote host
#
logger.info('Cleaning files')
cmds = []
for hostName, segments in destSegmentByHost.iteritems():
cmds.append(unix.RemoveFiles('remove tar file', tarFileName, ctxt=gp.REMOTE, remoteHost=hostName))
self.__runWaitAndCheckWorkerPoolForErrorsAndClear(cmds, "cleaning up tar file on segment hosts")
#
# clean up the local temp directory
#
unix.RemoveFiles.local('remove temp directory', tempDir)
def __copySegmentDirectories(self, gpEnv, gpArray, directives):
"""
directives should be composed of GpCopySegmentDirectoryDirective values
"""
if len(directives) == 0:
return
srcSegments = []
destSegments = []
isTargetReusedLocation = []
for directive in directives:
srcSegment = directive.getSrcSegment()
destSegment = directive.getDestSegment()
destSegment.primaryHostname = srcSegment.getSegmentHostName()
destSegment.primarySegmentPort = srcSegment.getSegmentPort()
srcSegments.append(srcSegment)
destSegments.append(destSegment)
isTargetReusedLocation.append(directive.isTargetReusedLocation())
destSegmentByHost = GpArray.getSegmentsByHostName(destSegments)
newSegmentInfo = gp.ConfigureNewSegment.buildSegmentInfoForNewSegment(
destSegments, isTargetReusedLocation)
def createConfigureNewSegmentCommand(hostName, cmdLabel,
validationOnly):
segmentInfo = newSegmentInfo[hostName]
checkNotNone("segmentInfo for %s" % hostName, segmentInfo)
return gp.ConfigureNewSegment(cmdLabel,
segmentInfo,
newSegments=True,
verbose=gplog.logging_is_verbose(),
batchSize=self.__parallelDegree,
ctxt=gp.REMOTE,
remoteHost=hostName,
validationOnly=validationOnly,
forceoverwrite=self.__forceoverwrite)
#
# validate directories for target segments
#
self.__logger.info('Validating remote directories')
cmds = []
for hostName in destSegmentByHost.keys():
cmds.append(
createConfigureNewSegmentCommand(hostName,
'validate blank segments',
True))
for cmd in cmds:
self.__pool.addCommand(cmd)
self.__pool.wait_and_printdots(len(cmds), self.__quiet)
validationErrors = []
for item in self.__pool.getCompletedItems():
results = item.get_results()
if not results.wasSuccessful():
if results.rc == 1:
# stdoutFromFailure = results.stdout.replace("\n", " ").strip()
lines = results.stderr.split("\n")
for line in lines:
if len(line.strip()) > 0:
validationErrors.append(
"Validation failure on host %s %s" %
(item.remoteHost, line))
else:
validationErrors.append(str(item))
self.__pool.empty_completed_items()
if validationErrors:
raise ExceptionNoStackTraceNeeded("\n" +
("\n".join(validationErrors)))
#
# unpack and configure new segments
#
self.__logger.info('Configuring new segments')
cmds = []
for hostName in destSegmentByHost.keys():
cmds.append(
createConfigureNewSegmentCommand(hostName,
'configure blank segments',
False))
self.__runWaitAndCheckWorkerPoolForErrorsAndClear(
cmds, "unpacking basic segment directory")
#
# copy dump files from old segment to new segment
#
for srcSeg in srcSegments:
for destSeg in destSegments:
if srcSeg.content == destSeg.content:
src_dump_dir = os.path.join(
srcSeg.getSegmentDataDirectory(), 'db_dumps')
cmd = base.Command('check existence of db_dumps directory',
'ls %s' % (src_dump_dir),
ctxt=base.REMOTE,
remoteHost=destSeg.getSegmentAddress())
cmd.run()
if cmd.results.rc == 0: # Only try to copy directory if it exists
cmd = Scp(
'copy db_dumps from old segment to new segment',
os.path.join(srcSeg.getSegmentDataDirectory(),
'db_dumps*', '*'),
os.path.join(destSeg.getSegmentDataDirectory(),
'db_dumps'),
srcSeg.getSegmentAddress(),
destSeg.getSegmentAddress(),
recursive=True)
cmd.run(validateAfter=True)
break
def __copySegmentDirectories(self, gpEnv, gpArray, directives):
"""
directives should be composed of GpCopySegmentDirectoryDirective values
"""
if len(directives) == 0:
return
srcSegments = []
destSegments = []
isTargetReusedLocation = []
timeStamp = datetime.datetime.today().strftime('%Y%m%d_%H%M%S')
for directive in directives:
srcSegment = directive.getSrcSegment()
destSegment = directive.getDestSegment()
destSegment.primaryHostname = srcSegment.getSegmentHostName()
destSegment.primarySegmentPort = srcSegment.getSegmentPort()
destSegment.progressFile = '%s/pg_basebackup.%s.dbid%s.out' % (gplog.get_logger_dir(),
timeStamp,
destSegment.getSegmentDbId())
srcSegments.append(srcSegment)
destSegments.append(destSegment)
isTargetReusedLocation.append(directive.isTargetReusedLocation())
destSegmentByHost = GpArray.getSegmentsByHostName(destSegments)
newSegmentInfo = gp.ConfigureNewSegment.buildSegmentInfoForNewSegment(destSegments, isTargetReusedLocation)
def createConfigureNewSegmentCommand(hostName, cmdLabel, validationOnly):
segmentInfo = newSegmentInfo[hostName]
checkNotNone("segmentInfo for %s" % hostName, segmentInfo)
return gp.ConfigureNewSegment(cmdLabel,
segmentInfo,
gplog.get_logger_dir(),
newSegments=True,
verbose=gplog.logging_is_verbose(),
batchSize=self.__parallelDegree,
ctxt=gp.REMOTE,
remoteHost=hostName,
validationOnly=validationOnly,
forceoverwrite=self.__forceoverwrite)
#
# validate directories for target segments
#
self.__logger.info('Validating remote directories')
cmds = []
for hostName in destSegmentByHost.keys():
cmds.append(createConfigureNewSegmentCommand(hostName, 'validate blank segments', True))
for cmd in cmds:
self.__pool.addCommand(cmd)
if self.__quiet:
self.__pool.join()
else:
base.join_and_indicate_progress(self.__pool)
validationErrors = []
for item in self.__pool.getCompletedItems():
results = item.get_results()
if not results.wasSuccessful():
if results.rc == 1:
# stdoutFromFailure = results.stdout.replace("\n", " ").strip()
lines = results.stderr.split("\n")
for line in lines:
if len(line.strip()) > 0:
validationErrors.append("Validation failure on host %s %s" % (item.remoteHost, line))
else:
validationErrors.append(str(item))
self.__pool.empty_completed_items()
if validationErrors:
raise ExceptionNoStackTraceNeeded("\n" + ("\n".join(validationErrors)))
# Configure a new segment
#
# Recover segments using gpconfigurenewsegment, which
# uses pg_basebackup. gprecoverseg generates a log filename which is
# passed to gpconfigurenewsegment as a confinfo parameter. gprecoverseg
# tails this file to show recovery progress to the user, and removes the
# file when one done. A new file is generated for each run of
# gprecoverseg based on a timestamp.
#
# There is race between when the pg_basebackup log file is created and
# when the progress command is run. Thus, the progress command touches
# the file to ensure its present before tailing.
self.__logger.info('Configuring new segments')
cmds = []
progressCmds = []
removeCmds= []
for hostName in destSegmentByHost.keys():
for segment in destSegmentByHost[hostName]:
if self.__progressMode != GpMirrorListToBuild.Progress.NONE:
progressCmds.append(
GpMirrorListToBuild.ProgressCommand("tail the last line of the file",
"set -o pipefail; touch -a {0}; tail -1 {0} | tr '\\r' '\\n' | tail -1".format(
pipes.quote(segment.progressFile)),
segment.getSegmentDbId(),
segment.progressFile,
ctxt=base.REMOTE,
remoteHost=hostName))
removeCmds.append(
base.Command("remove file",
"rm -f %s" % pipes.quote(segment.progressFile),
ctxt=base.REMOTE,
remoteHost=hostName))
cmds.append(
createConfigureNewSegmentCommand(hostName, 'configure blank segments', False))
self.__runWaitAndCheckWorkerPoolForErrorsAndClear(cmds, "unpacking basic segment directory",
suppressErrorCheck=False,
progressCmds=progressCmds)
self.__runWaitAndCheckWorkerPoolForErrorsAndClear(removeCmds, "removing pg_basebackup progress logfiles",
suppressErrorCheck=False)
#
# copy dump files from old segment to new segment
#
for srcSeg in srcSegments:
for destSeg in destSegments:
if srcSeg.content == destSeg.content:
src_dump_dir = os.path.join(srcSeg.getSegmentDataDirectory(), 'db_dumps')
cmd = base.Command('check existence of db_dumps directory', 'ls %s' % (src_dump_dir),
ctxt=base.REMOTE, remoteHost=destSeg.getSegmentAddress())
cmd.run()
if cmd.results.rc == 0: # Only try to copy directory if it exists
cmd = Scp('copy db_dumps from old segment to new segment',
os.path.join(srcSeg.getSegmentDataDirectory(), 'db_dumps*', '*'),
os.path.join(destSeg.getSegmentDataDirectory(), 'db_dumps'),
srcSeg.getSegmentAddress(),
destSeg.getSegmentAddress(),
recursive=True)
cmd.run(validateAfter=True)
break
def __copySegmentDirectories(self, gpEnv, gpArray, directives):
"""
directives should be composed of GpCopySegmentDirectoryDirective values
"""
if len(directives) == 0:
return
srcSegments = []
destSegments = []
isTargetReusedLocation = []
for directive in directives:
srcSegment = directive.getSrcSegment()
destSegment = directive.getDestSegment()
destSegment.primaryHostname = srcSegment.getSegmentHostName()
destSegment.primarySegmentPort = srcSegment.getSegmentPort()
srcSegments.append(srcSegment)
destSegments.append(destSegment)
isTargetReusedLocation.append(directive.isTargetReusedLocation())
destSegmentByHost = GpArray.getSegmentsByHostName(destSegments)
newSegmentInfo = gp.ConfigureNewSegment.buildSegmentInfoForNewSegment(destSegments, isTargetReusedLocation)
def createConfigureNewSegmentCommand(hostName, cmdLabel, validationOnly):
segmentInfo = newSegmentInfo[hostName]
checkNotNone("segmentInfo for %s" % hostName, segmentInfo)
return gp.ConfigureNewSegment(cmdLabel,
segmentInfo,
gplog.get_logger_dir(),
newSegments=True,
verbose=gplog.logging_is_verbose(),
batchSize=self.__parallelDegree,
ctxt=gp.REMOTE,
remoteHost=hostName,
validationOnly=validationOnly,
forceoverwrite=self.__forceoverwrite)
#
# validate directories for target segments
#
self.__logger.info('Validating remote directories')
cmds = []
for hostName in destSegmentByHost.keys():
cmds.append(createConfigureNewSegmentCommand(hostName, 'validate blank segments', True))
for cmd in cmds:
self.__pool.addCommand(cmd)
if self.__quiet:
self.__pool.join()
else:
base.join_and_indicate_progress(self.__pool)
validationErrors = []
for item in self.__pool.getCompletedItems():
results = item.get_results()
if not results.wasSuccessful():
if results.rc == 1:
# stdoutFromFailure = results.stdout.replace("\n", " ").strip()
lines = results.stderr.split("\n")
for line in lines:
if len(line.strip()) > 0:
validationErrors.append("Validation failure on host %s %s" % (item.remoteHost, line))
else:
validationErrors.append(str(item))
self.__pool.empty_completed_items()
if validationErrors:
raise ExceptionNoStackTraceNeeded("\n" + ("\n".join(validationErrors)))
#
# unpack and configure new segments
#
self.__logger.info('Configuring new segments')
cmds = []
for hostName in destSegmentByHost.keys():
cmds.append(createConfigureNewSegmentCommand(hostName, 'configure blank segments', False))
self.__runWaitAndCheckWorkerPoolForErrorsAndClear(cmds, "unpacking basic segment directory")
#
# copy dump files from old segment to new segment
#
for srcSeg in srcSegments:
for destSeg in destSegments:
if srcSeg.content == destSeg.content:
src_dump_dir = os.path.join(srcSeg.getSegmentDataDirectory(), 'db_dumps')
cmd = base.Command('check existence of db_dumps directory', 'ls %s' % (src_dump_dir),
ctxt=base.REMOTE, remoteHost=destSeg.getSegmentAddress())
cmd.run()
if cmd.results.rc == 0: # Only try to copy directory if it exists
cmd = Scp('copy db_dumps from old segment to new segment',
os.path.join(srcSeg.getSegmentDataDirectory(), 'db_dumps*', '*'),
os.path.join(destSeg.getSegmentDataDirectory(), 'db_dumps'),
srcSeg.getSegmentAddress(),
destSeg.getSegmentAddress(),
recursive=True)
cmd.run(validateAfter=True)
break
