def __buildTarFileForTransfer(self, gpEnv, masterSegment, sampleSegment,
newSegments):
"""
Returns the file for the tarfile that should be transferred and used
for building the blank segment
"""
masterDir = gpEnv.getMasterDataDir()
# note that this tempdir will be left around on the system (this is what other scripts do currently)
tempDir = gp.createTempDirectoryName(gpEnv.getMasterDataDir(),
"gpbuildingsegment")
unix.MakeDirectory("create temp directory for segment",
tempDir).run(validateAfter=True)
schemaDir = tempDir + "/schema"
unix.MakeDirectory("create temp schema directory for segment",
schemaDir).run(validateAfter=True)
unix.Chmod.local('set permissions on schema dir', schemaDir,
'0700') # set perms so postgres can start
#
# Copy remote files from the sample segment to the master
#
for toCopyFromRemote in ["postgresql.conf", "pg_hba.conf"]:
cmd = gp.RemoteCopy('copying %s from a segment' % toCopyFromRemote,
os.path.join(
sampleSegment.getSegmentDataDirectory(),
toCopyFromRemote),
masterSegment.getSegmentHostName(),
schemaDir,
ctxt=base.REMOTE,
remoteHost=sampleSegment.getSegmentAddress())
cmd.run(validateAfter=True)
appendNewEntriesToHbaFile(schemaDir + "/pg_hba.conf", newSegments)
#
# Use the master's version of other files, and build
#
self.__createEmptyDirectories(schemaDir, gDatabaseDirectories)
self.__createEmptyDirectories(schemaDir, gDatabaseSubDirectories)
self.__copyFiles(masterDir, schemaDir, ["PG_VERSION", "pg_ident.conf"])
#
# Build final tar
#
tarFileName = "gp_emptySegmentSchema.tar"
tarFile = tempDir + "/" + tarFileName
cmd = gp.CreateTar('gpbuildingmirrorsegment tar segment template',
schemaDir, tarFile)
cmd.run(validateAfter=True)
return (tempDir, tarFile, tarFileName)
def __copySegmentDirectories(self, gpEnv, gpArray, directives):
"""
directives should be composed of GpCopySegmentDirectoryDirective values
"""
if len(directives) == 0:
return
srcSegments = [d.getSrcSegment() for d in directives]
destSegments = [d.getDestSegment() for d in directives]
isTargetReusedLocation = [
d.isTargetReusedLocation() for d in directives
]
destSegmentByHost = GpArray.getSegmentsByHostName(destSegments)
newSegmentInfo = gp.ConfigureNewSegment.buildSegmentInfoForNewSegment(
destSegments, isTargetReusedLocation)
logger.info('Building template directory')
# In GPSQL, we need to create a template and copy it to all of failed segments.
if gpArray.getFaultStrategy() == gparray.FAULT_STRATEGY_NONE:
tempDir = '/tmp/GPSQL'
templateDir = tempDir + '/gpsql_template' + time.strftime(
"%Y%m%d_%H%M%S")
unix.MakeDirectory("create blank directory for segment",
templateDir).run(validateAfter=True)
unix.Chmod.local('set permissions on template dir', templateDir,
'0700') # set perms so postgres can start
logger.info('Creating template')
srcSegments[0].createTemplate(templateDir)
# Don't need log files and gpperfmon files in template.
rmCmd = unix.RemoveFiles(
'gprecoverseg remove gppermfon data from template',
templateDir + '/gpperfmon/data')
rmCmd.run(validateAfter=True)
rmCmd = unix.RemoveFiles('gprecoverseg remove logs from template',
templateDir + '/pg_log/*')
rmCmd.run(validateAfter=True)
#other files not needed
rmCmd = unix.RemoveFiles(
'gprecoverseg remove postmaster.opt from template',
templateDir + '/postmaster.opts')
rmCmd.run(validateAfter=True)
rmCmd = unix.RemoveFiles(
'gprecoverseg remove postmaster.pid from template',
templateDir + '/postmaster.pid')
rmCmd.run(validateAfter=True)
# template the temporary directories file
template_temporary_directories(templateDir, srcSegments[0].content)
tarFileName = "gpsqlSegmentTemplate.tar"
blankTarFile = tempDir + "/" + tarFileName
cmd = gp.CreateTar('gpbuildingmirrorsegment tar segment template',
templateDir, blankTarFile)
cmd.run(validateAfter=True)
def createConfigureNewSegmentCommand(hostName, cmdLabel,
validationOnly):
segmentInfo = newSegmentInfo[hostName]
checkNotNone("segmentInfo for %s" % hostName, segmentInfo)
return gp.ConfigureNewSegment(cmdLabel,
segmentInfo,
tarFile=tarFileName,
newSegments=True,
verbose=gplog.logging_is_verbose(),
batchSize=self.__parallelDegree,
ctxt=gp.REMOTE,
remoteHost=hostName,
validationOnly=validationOnly)
#
# validate directories for target segments
#
logger.info('Validating remote directories')
cmds = []
for hostName in destSegmentByHost.keys():
cmds.append(
createConfigureNewSegmentCommand(hostName,
'validate blank segments',
True))
for cmd in cmds:
self.__pool.addCommand(cmd)
self.__pool.wait_and_printdots(len(cmds), self.__quiet)
validationErrors = []
for item in self.__pool.getCompletedItems():
results = item.get_results()
if not results.wasSuccessful():
if results.rc == 1:
# stdoutFromFailure = results.stdout.replace("\n", " ").strip()
lines = results.stderr.split("\n")
for line in lines:
if len(line.strip()) > 0:
validationErrors.append(
"Validation failure on host %s %s" %
(item.remoteHost, line))
else:
validationErrors.append(str(item))
self.__pool.empty_completed_items()
if validationErrors:
raise ExceptionNoStackTraceNeeded("\n" +
("\n".join(validationErrors)))
#
# copy tar from master to target hosts
#
logger.info('Copying template directory file')
cmds = []
for hostName in destSegmentByHost.keys():
cmds.append(
gp.RemoteCopy("copy segment tar", blankTarFile, hostName,
tarFileName))
self.__runWaitAndCheckWorkerPoolForErrorsAndClear(
cmds, "building and transferring basic segment directory")
#
# unpack and configure new segments
#
logger.info('Configuring new segments')
cmds = []
for hostName in destSegmentByHost.keys():
cmds.append(
createConfigureNewSegmentCommand(hostName,
'configure blank segments',
False))
self.__runWaitAndCheckWorkerPoolForErrorsAndClear(
cmds, "unpacking basic segment directory")
#
# Clean up copied tar from each remote host
#
logger.info('Cleaning files')
cmds = []
for hostName, segments in destSegmentByHost.iteritems():
cmds.append(
unix.RemoveFiles('remove tar file',
tarFileName,
ctxt=gp.REMOTE,
remoteHost=hostName))
self.__runWaitAndCheckWorkerPoolForErrorsAndClear(
cmds, "cleaning up tar file on segment hosts")
#
# clean up the local temp directory
#
unix.RemoveFiles.local('remove temp directory', tempDir)
def __createEmptyDirectories(self, dir, newDirectoryNames):
for name in newDirectoryNames:
subDir = os.path.join(dir, name)
unix.MakeDirectory("create blank directory for segment",
subDir).run(validateAfter=True)
unix.Chmod.local('set permissions on blank dir', subDir, '0700')
