def __changeuserEnv(self):
"""
function: Change user GAUSS_ENV
input : NA
output: NA
"""
# clean os user environment variable
self.logger.log("Modifying user's environmental variable $GAUSS_ENV.")
userProfile = self.mpprcFile
DefaultValue.updateUserEnvVariable(userProfile, "GAUSS_ENV", "1")
if "HOST_IP" in os.environ.keys():
g_file.deleteLine(userProfile, "^\\s*export\\s*WHITELIST_ENV=.*$")
self.logger.log("Successfully modified user's environmental"
" variable GAUSS_ENV.")
self.logger.debug("Deleting symbolic link to $GAUSSHOME if exists.")
gaussHome = DefaultValue.getInstallDir(self.user)
if gaussHome == "":
raise Exception(ErrorCode.GAUSS_518["GAUSS_51800"] % "$GAUSSHOME")
if os.path.islink(gaussHome):
self.installPath = os.path.realpath(gaussHome)
os.remove(gaussHome)
else:
self.logger.debug("symbolic link does not exists.")
self.logger.debug("Deleting bin file in installation path.")
g_file.removeDirectory("%s/bin" % self.installPath)
self.logger.debug("Successfully deleting bin file in"
" installation path.")
def cleanScript(self):
"""
function: clean script
"""
# clean lib
libPath = os.path.join(self.clusterToolPath, LIBPATH)
if os.path.exists(libPath):
g_file.removeDirectory(libPath)
# clean om script
scriptPath = os.path.join(self.clusterToolPath, SCRIPTPATH)
if os.path.exists(scriptPath):
g_file.removeDirectory(scriptPath)
# clean root script path
root_script_path = os.path.join(DefaultValue.ROOT_SCRIPTS_PATH,
self.user)
if os.path.exists(root_script_path):
g_file.removeDirectory(root_script_path)
# if /root/gauss_om has no files, delete it.
if not os.listdir(DefaultValue.ROOT_SCRIPTS_PATH):
g_file.removeDirectory(DefaultValue.ROOT_SCRIPTS_PATH)
# clean others
if os.path.exists(self.clusterToolPath):
g_file.cleanDirectoryContent(self.clusterToolPath)
if self.userHome != "":
if os.path.exists(self.userHome):
g_file.removeDirectory(self.userHome)
def __init__(self,
hostNames,
logFile=None,
timeout=DefaultValue.TIMEOUT_PSSH_COMMON,
key=""):
'''
Constructor
'''
self.hostNames = hostNames
self.__logFile = logFile
self.__pid = os.getpid()
self.__timeout = timeout + 10
self._finalizer = weakref.finalize(self, self.clenSshResultFiles)
currentTime = str(datetime.datetime.now()).replace(" ", "_").replace(
".", "_")
randomnum = ''.join(sample('0123456789', 3))
# can tmp path always access?
if key == "":
self.__hostsFile = "/tmp/gauss_hosts_file_%d_%s_%s" % (
self.__pid, currentTime, randomnum)
self.__resultFile = "/tmp/gauss_result_%d_%s_%s.log" % (
self.__pid, currentTime, randomnum)
self.__outputPath = "/tmp/gauss_output_files_%d_%s_%s" % (
self.__pid, currentTime, randomnum)
self.__errorPath = "/tmp/gauss_error_files_%d_%s_%s" % (
self.__pid, currentTime, randomnum)
else:
self.__hostsFile = "/tmp/gauss_hosts_file_%d_%s_%s_%s" % (
self.__pid, key, currentTime, randomnum)
self.__resultFile = "/tmp/gauss_result_%d_%s_%s_%s.log" % (
self.__pid, key, currentTime, randomnum)
self.__outputPath = "/tmp/gauss_output_files_%d_%s_%s_%s" % (
self.__pid, key, currentTime, randomnum)
self.__errorPath = "/tmp/gauss_error_files_%d_%s_%s_%s" % (
self.__pid, key, currentTime, randomnum)
self.__resultStatus = {}
if logFile is None:
self.__logFile = "/dev/null"
# before using, clean the old ones
g_file.removeFile(self.__hostsFile)
g_file.removeFile(self.__resultFile)
if os.path.exists(self.__outputPath):
g_file.removeDirectory(self.__outputPath)
if os.path.exists(self.__errorPath):
g_file.removeDirectory(self.__errorPath)
self.__writeHostFiles()
def cleanLocalLog(self):
"""
function: Clean default log
input : NA
output: NA
"""
self.logger.log("Deleting local node's logs.", "addStep")
try:
# clean log
path = "%s/%s" % (self.clusterInfo.logPath, self.user)
g_file.removeDirectory(path)
except Exception as e:
self.logger.logExit(ErrorCode.GAUSS_502["GAUSS_50207"] % "logs" +
" Error: \n%s." % str(e))
self.logger.log("Successfully deleted local node's logs.", "constant")
def __backupDbClusterInfo(self):
"""
function: backup DbClusterInfo.py and cluster_static_config to temp
path
input: NA
output: NA
"""
commonStaticConfigFile = "%s/bin/cluster_static_config" \
% g_opts.appPath
commonUpgradeVersionFile = "%s/bin/upgrade_version" % g_opts.appPath
commonDbClusterInfoModule = \
"%s/bin/script/gspylib/common/DbClusterInfo.py" % g_opts.appPath
bakPath = self.__getTmpDir()
# backup DbClusterInfo.py
oldDbClusterInfoModule = "%s/OldDbClusterInfo.py" % bakPath
cmd = "cp -p '%s' '%s'" % (commonDbClusterInfoModule,
oldDbClusterInfoModule)
cmd += " && cp -rp '%s/bin/script/' '%s'" % (g_opts.appPath, bakPath)
(status, output) = subprocess.getstatusoutput(cmd)
if (status != 0):
g_logger.logExit(ErrorCode.GAUSS_514["GAUSS_51400"] % cmd +
"\nOutput:%s" % output)
# backup cluster_static_config
cmd = "cp -p '%s' '%s'/" % (commonStaticConfigFile, bakPath)
(status, output) = subprocess.getstatusoutput(cmd)
if (status != 0):
g_logger.logExit(ErrorCode.GAUSS_514["GAUSS_51400"] % cmd +
"\nOutput:%s" % output)
upgradeBakPath = self.__getBackupDir()
try:
# backup upgrade_version
oldUpgradeVersionFile = "%s/old_upgrade_version" % upgradeBakPath
cmd = "cp -p %s %s" % (commonUpgradeVersionFile,
oldUpgradeVersionFile)
(status, output) = subprocess.getstatusoutput(cmd)
if (status != 0):
raise Exception(ErrorCode.GAUSS_514["GAUSS_51400"] % cmd +
"\nOutput:%s" % output)
except Exception as e:
g_logger.debug("Backup failed.ERROR:%s\nClean backup path." %
str(e))
if (os.path.isdir(upgradeBakPath)):
g_file.removeDirectory(upgradeBakPath)
def clenSshResultFiles(self):
"""
function: Delete file
input : NA
output: NA
"""
if os.path.exists(self.__hostsFile):
g_file.removeFile(self.__hostsFile)
if os.path.exists(self.__resultFile):
g_file.removeFile(self.__resultFile)
if os.path.exists(self.__outputPath):
g_file.removeDirectory(self.__outputPath)
if os.path.exists(self.__errorPath):
g_file.removeDirectory(self.__errorPath)
def cleanPath(self):
"""
function: clean path
input: NA
output: NA
"""
self.logger.debug("Begin clean path")
if os.path.exists(self.clusterInfo.appPath):
self.logger.debug("Deleting the install directory.")
cleanPath = os.path.join(self.clusterInfo.appPath, "./*")
g_file.removeDirectory(cleanPath)
self.logger.debug("Successfully deleted the install directory.")
for i in self.component:
i.cleanPath()
gsdbHomePath = "/home/%s/gsdb_home" % self.user
if os.path.exists(gsdbHomePath):
self.logger.debug("Deleting the gsdb home path.")
g_file.removeDirectory(gsdbHomePath)
self.logger.debug("Successfully deleted the gsdb home path.")
self.logger.debug("Clean Path successfully.")
def doRebuildConf(self):
"""
generating static configuration files for all nodes
input:NA
output:NA
"""
try:
self.logger.log(
"Generating static configuration files for all nodes.")
# Initialize the cluster information according to the XML file
self.context.clusterInfo = dbClusterInfo()
self.context.clusterInfo.initFromXml(self.context.g_opts.confFile)
# 1.create a tmp dir
self.logger.log(
"Creating temp directory to store static configuration files.")
dirName = os.path.dirname(os.path.realpath(__file__))
tmpDirName = os.path.realpath("%s/../../static_config_files" %
dirName)
cmd = "mkdir -p -m %s '%s'" % (DefaultValue.KEY_DIRECTORY_MODE,
tmpDirName)
(status, output) = subprocess.getstatusoutput(cmd)
if (status != 0):
raise Exception(ErrorCode.GAUSS_502["GAUSS_50208"] %
"temporary directory" +
"\nCommand:%s\nError: %s" % (cmd, output))
self.logger.log("Successfully created the temp directory.")
# create static files
self.logger.log("Generating static configuration files.")
for dbNode in self.context.clusterInfo.dbNodes:
staticConfigPath = "%s/cluster_static_config_%s" % (
tmpDirName, dbNode.name)
self.context.clusterInfo.saveToStaticConfig(
staticConfigPath, dbNode.id)
self.logger.log(
"Successfully generated static configuration files.")
self.logger.log(
"Static configuration files for all nodes are saved in %s." %
tmpDirName)
# check if need send static config files
if not self.context.g_opts.distribute:
self.logger.debug(
"No need to distribute static configuration files "
"to installation directory.")
return
# distribute static config file
self.logger.log(
"Distributing static configuration files to all nodes.")
for dbNode in self.context.clusterInfo.dbNodes:
if (dbNode.name != DefaultValue.GetHostIpOrName()):
cmd = \
"pscp -H %s '%s'/cluster_static_config_%s '%s'" \
"/bin/cluster_static_config" % (
dbNode.name, tmpDirName,
dbNode.name, self.context.clusterInfo.appPath)
else:
cmd = \
"cp '%s'/cluster_static_config_%s '%s'" \
"/bin/cluster_static_config" % (
tmpDirName,
dbNode.name, self.context.clusterInfo.appPath)
(status, output) = subprocess.getstatusoutput(cmd)
if (status != 0):
raise Exception(ErrorCode.GAUSS_502["GAUSS_50216"] %
"static configuration file" +
"Node: %s.\nCommand: \n%s\nError: \n%s" %
(dbNode.name, cmd, output))
self.logger.log(
"Successfully distributed static configuration files.")
except Exception as e:
g_file.removeDirectory(tmpDirName)
raise Exception(str(e))
def cleanLocalNodeEnvSoftware(self):
"""
function: clean local node environment software and variable
input : NA
output: NA
in this function, Gauss-MPPDB* & sctp_patch is came from R5 upgrade R7
"""
self.logger.log("Deleting software packages "
"and environmental variables of the local node.")
try:
self.clusterToolPath = DefaultValue.getClusterToolPath()
# clean local node environment software
path = "%s/%s" % (self.clusterToolPath, PSSHDIR)
g_file.removeDirectory(path)
path = "%s/upgrade.sh" % self.clusterToolPath
g_file.removeFile(path)
path = "%s/version.cfg" % self.clusterToolPath
g_file.removeFile(path)
path = "%s/GaussDB.py" % self.clusterToolPath
g_file.removeFile(path)
path = "%s/libcgroup" % self.clusterToolPath
g_file.removeDirectory(path)
path = "%s/unixodbc" % self.clusterToolPath
g_file.removeDirectory(path)
path = "%s/server.key.cipher" % self.clusterToolPath
g_file.removeFile(path)
path = "%s/server.key.rand" % self.clusterToolPath
g_file.removeFile(path)
path = "%s/%s*" % (self.clusterToolPath, VersionInfo.PRODUCT_NAME)
g_file.removeDirectory(path)
path = "%s/server.key.rand" % self.clusterToolPath
g_file.removeFile(path)
path = "%s/Gauss*" % (self.clusterToolPath)
g_file.removeDirectory(path)
path = "%s/sctp_patch" % (self.clusterToolPath)
g_file.removeDirectory(path)
self.logger.debug(
"Deleting environmental software of local nodes.")
# clean local node environment variable
cmd = "(if [ -s '%s' ]; then " % PROFILE_FILE
cmd += "sed -i -e '/^export GPHOME=%s$/d' %s " % (
self.clusterToolPath.replace('/', '\/'), PROFILE_FILE)
cmd += \
"-e '/^export PATH=\$GPHOME\/pssh-2.3.1\/bin:" \
"\$GPHOME\/script:\$PATH$/d' %s " % PROFILE_FILE
cmd += \
"-e '/^export PATH=\$GPHOME\/script\/gspylib\/pssh\/bin:" \
"\$GPHOME\/script:\$PATH$/d' %s " % PROFILE_FILE
cmd += \
"-e '/^export LD_LIBRARY_PATH=\$GPHOME\/script" \
"\/gspylib\/clib:\$LD_LIBRARY_PATH$/d' %s " % PROFILE_FILE
cmd += \
"-e '/^export LD_LIBRARY_PATH=\$GPHOME\/lib:" \
"\$LD_LIBRARY_PATH$/d' %s " % PROFILE_FILE
cmd += \
"-e '/^export PATH=\/root\/gauss_om\/%s\/script:" \
"\$PATH$/d' %s " % (self.user, PROFILE_FILE)
cmd += \
"-e '/^export PYTHONPATH=\$GPHOME\/lib$/d' %s; fi) " \
% PROFILE_FILE
self.logger.debug("Command for deleting environment variable: %s" %
cmd)
(status, output) = subprocess.getstatusoutput(cmd)
if (status != 0):
self.logger.logExit(ErrorCode.GAUSS_502["GAUSS_50207"] %
"environment variables of the local node" +
" Error: \n%s" % output)
# check if user profile exist
userProfile = ""
if (self.mpprcFile is not None and self.mpprcFile != ""):
userProfile = self.mpprcFile
else:
userProfile = "/home/%s/.bashrc" % self.user
if (not os.path.exists(userProfile)):
self.logger.debug("The %s does not exist. "
"Please skip to clean $GAUSS_ENV." %
userProfile)
return
# clean user's environmental variable
DefaultValue.cleanUserEnvVariable(userProfile,
cleanGAUSS_WARNING_TYPE=True)
# clean $GAUSS_ENV
if (not self.deleteUser):
envContent = "^\\s*export\\s*GAUSS_ENV=.*$"
g_file.deleteLine(userProfile, envContent)
self.logger.debug("Command for deleting $GAUSS_ENV: %s" % cmd,
"constant")
except Exception as e:
self.logger.logExit(str(e))
self.logger.log("Successfully deleted software packages "
"and environmental variables of the local nodes.")
def removeTbsDir(self, tbsDir):
"""
"""
g_file.removeDirectory(tbsDir)
def cleanEnvSoftware(self):
"""
function: clean environment software and variable
Gauss-MPPDB* & sctp_patch is came from R5 upgrade R7
input : NA
output: NA
"""
self.logger.debug("Cleaning the environmental software and variable.")
# clean environment software
path = "%s/%s" % (self.clusterToolPath, PSSHDIR)
g_file.removeDirectory(path)
path = "%s/lib" % self.clusterToolPath
g_file.removeDirectory(path)
path = "%s/script" % self.clusterToolPath
g_file.removeDirectory(path)
path = "%s/sudo" % self.clusterToolPath
g_file.removeDirectory(path)
path = "%s/upgrade.sh" % self.clusterToolPath
g_file.removeFile(path)
path = "%s/version.cfg" % self.clusterToolPath
g_file.removeFile(path)
path = "%s/GaussDB.py" % self.clusterToolPath
g_file.removeFile(path)
path = "%s/libcgroup" % self.clusterToolPath
g_file.removeDirectory(path)
path = "%s/server.key.cipher" % self.clusterToolPath
g_file.removeFile(path)
path = "%s/server.key.rand" % self.clusterToolPath
g_file.removeFile(path)
path = "%s/%s*" % (self.clusterToolPath, VersionInfo.PRODUCT_NAME)
g_file.removeDirectory(path)
path = "%s/Gauss*" % (self.clusterToolPath)
g_file.removeDirectory(path)
path = "%s/sctp_patch" % (self.clusterToolPath)
g_file.removeDirectory(path)
path = "%s/unixodbc" % self.clusterToolPath
g_file.removeDirectory(path)
path = "%s/%s" % (self.clusterToolPath, Const.UPGRADE_SQL_FILE)
g_file.removeFile(path)
path = "%s/%s" % (self.clusterToolPath, Const.UPGRADE_SQL_SHA)
g_file.removeFile(path)
self.logger.debug(
"Successfully cleaned the environmental software and variable.")
self.logger.debug("Cleaning environmental software.")
# clean environment variable
cmd = "(if [ -s '%s' ]; then " % PROFILE_FILE
cmd += "sed -i -e '/^export GPHOME=%s$/d' %s " % (
self.clusterToolPath.replace('/', '\/'), PROFILE_FILE)
cmd += \
"-e '/^export PATH=\$GPHOME\/pssh-2.3.1\/bin:" \
"\$GPHOME\/script:\$PATH$/d' %s " % PROFILE_FILE
cmd += \
"-e '/^export PATH=\$GPHOME\/script\/gspylib\/pssh\/bin:" \
"\$GPHOME\/script:\$PATH$/d' %s " % PROFILE_FILE
cmd += \
"-e '/^export LD_LIBRARY_PATH=\$GPHOME\/lib:" \
"\$LD_LIBRARY_PATH$/d' %s " % PROFILE_FILE
cmd += \
"-e '/^export PYTHONPATH=\$GPHOME\/lib$/d' %s; fi) " % PROFILE_FILE
self.logger.debug("Command for cleaning environment variable: %s." %
cmd)
(status, output) = subprocess.getstatusoutput(cmd)
if (status != 0):
self.logger.logExit(ErrorCode.GAUSS_514["GAUSS_51400"] % cmd +
" Error:\n%s" % output)
self.logger.debug(
"Successfully cleaned environmental software and variable.")
def doRestore(self):
"""
function: restore files
Restoring binary files:
1.decompress tar file
2.Check binary files
3.Create installation path
4.Restore binary files to install path
Restoring parameter files:
1.decompress tar file
2.delete temporary directory
3.extract parameter files to the temporary directory
4.check hostname and parameter
5.Restore parameter files
6.Remove the temporary directory
input : NA
output: NA
"""
self.logger.log("Restoring files.")
if self.restoreBin:
self.logger.log("Restoring binary files.")
try:
# decompress tar file
self.decompressTarFile("binary")
# Checking binary files
self.logger.debug("Checking if binary files exist.")
tarName = os.path.join(self.restoreDir, self.binTarName)
if (not os.path.exists(tarName)):
raise Exception(ErrorCode.GAUSS_502["GAUSS_50201"] %
"Binary files")
# Creating installation path
self.logger.debug(
"Creating installation path if did not exist.")
if (not os.path.exists(self.installPath)):
os.makedirs(self.installPath,
DefaultValue.KEY_DIRECTORY_PERMISSION)
# Restore binary files to install path.
self.logger.debug("Restore binary files to install path.")
g_file.cleanDirectoryContent(self.installPath)
cmd = g_file.SHELL_CMD_DICT["decompressTarFile"] % (
self.restoreDir, tarName)
cmd += " && "
cmd += g_file.SHELL_CMD_DICT["copyFile"] % (
"'%s'/*" % self.binExtractName, self.installPath)
self.logger.debug("Command for restoring binary files:%s." %
cmd)
(status, output) = subprocess.getstatusoutput(cmd)
if (status != 0):
raise Exception(ErrorCode.GAUSS_502["GAUSS_50220"] % (
"binary files to install path[%s]" % \
self.installPath) + " Error: \n%s" % output)
g_file.removeDirectory(
os.path.join(self.restoreDir, self.binExtractName))
except Exception as e:
raise Exception(str(e))
self.logger.log("Successfully restored binary files.")
if self.restorePara:
self.logger.log("Restoring parameter files.")
# Re-obtaining clusterInfo because the restoreBin succeeded
if self.dbNodeInfo is None:
self.clusterInfo.initFromStaticConfig(self.user, g_staticFile)
hostName = DefaultValue.GetHostIpOrName()
self.dbNodeInfo = self.clusterInfo.getDbNodeByName(hostName)
if self.dbNodeInfo is None:
self.logger.logExit(ErrorCode.GAUSS_516["GAUSS_51619"] %
hostName)
# Restoring parameter files.
try:
# decompress tar file
self.decompressTarFile("parameter")
# delete temporary directory
self.logger.debug(
"Delete temporary directory if it has existed.")
temp_dir = os.path.join(self.restoreDir,
"parameter_%s" % HOSTNAME)
if (os.path.exists(temp_dir)):
g_file.removeDirectory(temp_dir)
# extract parameter files to the temporary directory
self.logger.debug(
"Extract parameter files to the temporary directory.")
tarName = os.path.join(self.restoreDir, self.paraTarName)
if (not os.path.exists(tarName)):
if (g_ignoreMiss):
self.logger.error(ErrorCode.GAUSS_502["GAUSS_50201"] %
"parameter files")
sys.exit(0)
else:
raise Exception(ErrorCode.GAUSS_502["GAUSS_50201"] %
"parameter files")
cmd = g_file.SHELL_CMD_DICT["decompressTarFile"] % (
self.restoreDir, tarName)
(status, output) = subprocess.getstatusoutput(cmd)
if (status != 0):
raise Exception(ErrorCode.GAUSS_514["GAUSS_51400"] % cmd +
" Error: \n%s" % output)
# check hostname
self.logger.debug("Checking hostname.")
self.__checkHostName("%s/%s" %
(temp_dir, self.hostnameFileName))
# check parameter
self.logger.debug("Checking parameter files.")
paraFileList = []
self.__checkParaFiles(temp_dir, paraFileList)
self.logger.debug("Restoring parameter files.")
paraFileNum = len(paraFileList)
for i in range(paraFileNum):
tarFileName, paraFilePath = paraFileList[i].split('|')
g_file.cpFile(os.path.join(temp_dir, tarFileName),
paraFilePath)
self.logger.debug("Remove the temporary directory.")
g_file.removeDirectory(temp_dir)
except Exception as e:
g_file.removeDirectory(temp_dir)
raise Exception(str(e))
self.logger.log("Successfully restored parameter files.")
self.logger.log("Successfully restored files.")
def cleanDir(self, instDir):
"""
function: Clean the dirs
input : instDir
output: NA
"""
if (not os.path.exists(instDir)):
return
dataDir = []
dataDir = os.listdir(instDir)
if (os.getuid() == 0):
pglDir = '%s/pg_location' % instDir
isPglDirEmpty = False
if (os.path.exists(pglDir) and len(os.listdir(pglDir)) == 0):
isPglDirEmpty = True
if (len(dataDir) == 0 or isPglDirEmpty):
g_file.cleanDirectoryContent(instDir)
else:
for info in dataDir:
if (str(info) == "pg_location"):
resultMount = []
resultFile = []
resultDir = []
pglDir = '%s/pg_location' % instDir
# delete all files in the mount point
cmd = "%s | %s '%s' | %s '{printf $3}'" % \
(g_Platform.getMountCmd(), g_Platform.getGrepCmd(),
pglDir, g_Platform.getAwkCmd())
(status, outputMount) = subprocess.getstatusoutput(cmd)
if (status != 0):
raise Exception(ErrorCode.GAUSS_502["GAUSS_50207"] %
instDir +
" Error:\n%s." % str(outputMount) +
"The cmd is %s" % cmd)
else:
if (len(outputMount) > 0):
resultMount = str(outputMount).split()
for infoMount in resultMount:
g_file.cleanDirectoryContent(infoMount)
else:
g_file.cleanDirectoryContent(instDir)
continue
# delete file in the pg_location directory
if (not os.path.exists(pglDir)):
continue
cmd = "cd '%s'" % pglDir
(status, output) = subprocess.getstatusoutput(cmd)
if (status != 0):
raise Exception(ErrorCode.GAUSS_514["GAUSS_51400"] %
cmd + " Error: \n%s " % output)
outputFile = g_file.findFile(".", "f", "type")
if (len(outputFile) > 0):
for infoFile in outputFile:
tmpinfoFile = pglDir + infoFile[1:]
for infoMount in resultMount:
if (tmpinfoFile.find(infoMount) < 0
and infoMount.find(tmpinfoFile) < 0):
realFile = "'%s/%s'" % (pglDir, infoFile)
g_file.removeFile(realFile, "shell")
# delete directory in the pg_location directory
cmd = "if [ -d '%s' ]; then cd '%s' && find -type d; fi" \
% \
(pglDir, pglDir)
(status, outputDir) = subprocess.getstatusoutput(cmd)
if (status != 0):
raise Exception(ErrorCode.GAUSS_502["GAUSS_50207"] %
instDir +
" Error:\n%s." % str(outputDir) +
"The cmd is %s" % cmd)
else:
resultDir = g_file.findFile(".", "d", "type")
resultDir.remove(".")
if (len(resultDir) > 0):
for infoDir in resultDir:
tmpinfoDir = pglDir + infoDir[1:]
for infoMount in resultMount:
if (tmpinfoDir.find(infoMount) < 0 and
infoMount.find(tmpinfoDir) < 0):
realPath = "'%s/%s'" % (pglDir,
infoDir)
g_file.removeDirectory(realPath)
cmd = "if [ -d '%s' ];then cd '%s' && find . ! -name " \
"'pg_location' " \
"! -name '..' ! -name '.' -print0 |xargs -r -0 -n100 rm " \
"-rf; " \
"fi" % (instDir, instDir)
(status, output) = subprocess.getstatusoutput(cmd)
if (status != 0):
raise Exception(ErrorCode.GAUSS_502["GAUSS_50207"] % instDir +
" Error:\n%s." % str(output) +
"The cmd is %s" % cmd)
def doReplaceSSLCert(self):
"""
function: replace ssl cert files
input: NA
output: NA
"""
try:
# Initialize the cluster information according to the xml file
self.context.clusterInfo = dbClusterInfo()
self.context.clusterInfo.initFromStaticConfig(
g_OSlib.getPathOwner(self.context.g_opts.certFile)[0])
self.sshTool = SshTool(
self.context.clusterInfo.getClusterNodeNames(),
self.logger.logFile)
except Exception as e:
raise Exception(str(e))
try:
self.logger.log("Starting ssl cert files replace.", "addStep")
tempDir = os.path.join(DefaultValue.getTmpDirFromEnv(),
"tempCertDir")
# unzip files to temp directory
if (os.path.exists(tempDir)):
g_file.removeDirectory(tempDir)
g_file.createDirectory(tempDir, True,
DefaultValue.KEY_DIRECTORY_MODE)
g_file.decompressZipFiles(self.context.g_opts.certFile, tempDir)
realCertList = DefaultValue.CERT_FILES_LIST
clientCertList = DefaultValue.CLIENT_CERT_LIST
# check file exists
for clientCert in clientCertList:
sslFile = os.path.join(tempDir, clientCert)
if (not os.path.isfile(sslFile)):
raise Exception(
(ErrorCode.GAUSS_502["GAUSS_50201"] % sslFile) + \
"Missing SSL client cert file in ZIP file.")
certList = []
dnDict = self.getDnNodeDict()
for cert in realCertList:
sslFile = os.path.join(tempDir, cert)
if (not os.path.isfile(sslFile)
and cert != DefaultValue.SSL_CRL_FILE):
raise Exception(
(ErrorCode.GAUSS_502["GAUSS_50201"] % sslFile) + \
"Missing SSL server cert file in ZIP file.")
if (os.path.isfile(sslFile)):
certList.append(cert)
# distribute cert files to datanodes
self.doDNBackup()
self.distributeDNCert(certList, dnDict)
# clear temp directory
g_file.removeDirectory(tempDir)
if (not self.context.g_opts.localMode):
self.logger.log(
"Successfully distributed cert files on all nodes.")
except Exception as e:
g_file.removeDirectory(tempDir)
raise Exception(str(e))
def __cleanInstallProgram(self):
"""
function: Clean install program
input : NA
output: NA
"""
if (not os.path.exists(self.installPath)):
self.logger.log("The installation directory does not exist. ")
return
realLink = self.installPath
if os.path.islink(self.installPath):
realLink = os.readlink(self.installPath)
# delete upgrade directory
self.logger.debug("Starting delete other installation directory.")
try:
recordVersionFile = os.path.realpath(
os.path.join(self.installPath, "record_app_directory"))
if os.path.isfile(recordVersionFile):
with open(recordVersionFile, 'r') as fp:
retLines = fp.readlines()
if len(retLines) != 2:
raise Exception(ErrorCode.GAUSS_502["GAUSS_50222"]
% recordVersionFile)
oldPath = retLines[0].strip()
newPath = retLines[1].strip()
if os.path.normcase(oldPath) == os.path.normcase(realLink):
g_file.removeDirectory(newPath)
else:
g_file.removeDirectory(oldPath)
self.logger.debug("Successfully deleted other installation"
" path need to delete.")
else:
self.logger.debug("No other installation path need"
" to delete.")
except Exception as e:
raise Exception(ErrorCode.GAUSS_502["GAUSS_50209"]
% "other installation"
+ " Can not delete other installation"
" directory: %s." % str(e))
self.logger.log("Removing the installation directory.")
try:
fileList = os.listdir(self.installPath)
for fileName in fileList:
fileName = fileName.replace("/", "").replace("..", "")
filePath = os.path.join(os.path.realpath(self.installPath),
fileName)
if os.path.isfile(filePath):
os.remove(filePath)
elif os.path.isdir(filePath):
if (fileName == "bin"):
binFileList = os.listdir(filePath)
for binFile in binFileList:
fileInBinPath = os.path.join(filePath, binFile)
if os.path.isfile(fileInBinPath) and \
binFile != "cluster_static_config":
os.remove(fileInBinPath)
elif os.path.islink(fileInBinPath):
os.remove(fileInBinPath)
elif os.path.isdir(fileInBinPath):
g_file.removeDirectory(fileInBinPath)
else:
g_file.removeDirectory(filePath)
self.logger.debug("Remove path:%s." % filePath)
self.logger.debug("Successfully deleted bin file"
" in installation path.")
except Exception as e:
raise Exception(ErrorCode.GAUSS_502["GAUSS_50209"]
% "installation"
+ " Can not delete installation directory: %s."
% str(e))
# regular match delete empty directory
self.logger.debug("Starting delete empty installation directory.")
try:
removeflag = False
namePrefix = os.path.basename(self.installPath)
gaussPath = os.path.realpath(os.path.dirname(self.installPath))
curInstallName = os.path.basename(realLink)
fileList = os.listdir(gaussPath)
for fileName in fileList:
if fileName.strip() != curInstallName.strip():
filePath = os.path.join(os.path.realpath(gaussPath),
fileName)
if os.path.isdir(filePath) \
and not os.listdir(filePath) and "_" in fileName:
fileNameElement = fileName.split("_", 1)
if namePrefix.strip() == fileNameElement[0].strip():
res = re.search(
'^(?![0-9]+$)(?![a-zA-Z]+$)[0-9A-Za-z]{8}$',
fileNameElement[1].strip())
if res:
removeflag = True
g_file.removeDirectory(filePath)
if removeflag:
self.logger.debug("Successfully deleted empty"
" installation path.")
else:
self.logger.debug("No empty installation path need"
" to delete.")
except Exception as e:
raise Exception(ErrorCode.GAUSS_502["GAUSS_50209"]
% "other installation"
+ " Can not delete empty installation"
" directory: %s." % str(e))
self.logger.log("Successfully deleted installation directory.")
def doDNSSLCertRollback(self):
"""
function: rollback SSL cert file in DN instance directory
input: NA
output: NA
"""
self.context.clusterInfo = dbClusterInfo()
self.context.clusterInfo.initFromStaticConfig(
pwd.getpwuid(os.getuid()).pw_name)
self.sshTool = SshTool(self.context.clusterInfo.getClusterNodeNames(),
self.logger.logFile)
backupList = DefaultValue.CERT_FILES_LIST[:]
allDnNodeDict = self.getDnNodeDict()
noBackupList = []
temp = "tempDir"
if self.context.g_opts.localMode:
if ((DefaultValue.GetHostIpOrName() in allDnNodeDict.keys())
and os.path.isfile(
os.path.join(
allDnNodeDict[DefaultValue.GetHostIpOrName()],
DefaultValue.CERT_BACKUP_FILE))):
localDnDir = allDnNodeDict[DefaultValue.GetHostIpOrName()]
tempDir = os.path.join(localDnDir, temp)
if (os.path.exists(tempDir)):
g_file.removeDirectory(tempDir)
os.mkdir(tempDir, DefaultValue.KEY_DIRECTORY_PERMISSION)
for certFile in backupList:
realCertFile = os.path.join(localDnDir, certFile)
if (os.path.exists(realCertFile)):
g_file.moveFile(realCertFile, tempDir)
cmd = "cd '%s' && if [ -f '%s' ];then tar -zxvf %s;fi" % \
(localDnDir, DefaultValue.CERT_BACKUP_FILE,
DefaultValue.CERT_BACKUP_FILE)
(status, output) = subprocess.getstatusoutput(cmd)
if (status != 0):
cmd = "cp '%s'/* '%s' && rm -rf '%s'" % (
tempDir, localDnDir, tempDir)
(status, output) = subprocess.getstatusoutput(cmd)
raise Exception((ErrorCode.GAUSS_514["GAUSS_51400"] %
cmd) +
"Failed uncompression SSL backup file." +
"Error: \n%s" % output)
# remove temp directory
if (os.path.exists(tempDir)):
g_file.removeDirectory(tempDir)
# set guc option
if (os.path.isfile(
os.path.join(localDnDir, DefaultValue.SSL_CRL_FILE))):
cmd = \
"gs_guc set -D %s " \
"-c \"ssl_crl_file=\'%s\'\"" \
% (localDnDir, DefaultValue.SSL_CRL_FILE)
else:
cmd = \
"gs_guc set -D %s " \
"-c \"ssl_crl_file=\'\'\"" % localDnDir
(status, output) = subprocess.getstatusoutput(cmd)
if (status != 0):
raise Exception(ErrorCode.GAUSS_514["GAUSS_51400"] % cmd +
"Error: \n%s" % output)
if (os.path.isfile(os.path.join(localDnDir, EMPTY_CERT))):
os.remove(os.path.join(localDnDir, EMPTY_CERT))
self.logger.log(
"Successfully rollback SSL cert files with local mode.")
return
else:
self.logger.log("There is not exists backup files.")
return
# 1.check backup file "gsql_cert_backup.tar.gz" on all dbnodes.
for node in allDnNodeDict.keys():
backupGzFile = os.path.join(allDnNodeDict[node],
DefaultValue.CERT_BACKUP_FILE)
status = self.sshTool.checkRemoteFileExist(
node, backupGzFile, self.context.g_opts.mpprcFile)
if not status:
noBackupList.append(node)
if (len(noBackupList) > 0):
raise Exception((ErrorCode.GAUSS_502["GAUSS_50201"] %
DefaultValue.CERT_BACKUP_FILE) +
"Can't rollback SSL cert files on %s." %
noBackupList)
# 2.perform rollback on all dbnodes.
for node in allDnNodeDict.keys():
backupGzFile = os.path.join(allDnNodeDict[node],
DefaultValue.CERT_BACKUP_FILE)
# 2-1.move SSL cert files in dn directory to temp directory.
sshcmd = "cd '%s' && if [ -d '%s' ];then rm -rf '%s'" \
" && mkdir '%s';else mkdir '%s';fi" % \
(allDnNodeDict[node], temp, temp, temp, temp)
self.sshTool.executeCommand(sshcmd, "Make temp directory.",
DefaultValue.SUCCESS, \
[node], self.context.g_opts.mpprcFile)
for certFile in backupList:
realCertFile = os.path.join(allDnNodeDict[node], certFile)
sshcmd = " %s && " % g_Platform.getCdCmd(
os.path.join(allDnNodeDict[node], temp))
sshcmd += g_file.SHELL_CMD_DICT["renameFile"] % (
realCertFile, realCertFile, "./")
self.sshTool.executeCommand(
sshcmd, "Backup cert files to temp directory.",
DefaultValue.SUCCESS, [node],
self.context.g_opts.mpprcFile)
# 2-2.uncompression "gsql_cert_backup.tar.gz" file
sshcmd = "cd '%s' && if [ -f '%s' ];then tar -zxvf %s;fi" % \
(allDnNodeDict[node], DefaultValue.CERT_BACKUP_FILE,
DefaultValue.CERT_BACKUP_FILE)
self.sshTool.executeCommand(sshcmd, "Unzip backup file.",
DefaultValue.SUCCESS, [node],
self.context.g_opts.mpprcFile)
# 2-3.clear temp directory
sshcmd = " %s && " % g_Platform.getCdCmd(allDnNodeDict[node])
sshcmd += g_file.SHELL_CMD_DICT["deleteDir"] % (temp, temp)
self.sshTool.executeCommand(sshcmd, "Clear backup cert files.",
DefaultValue.SUCCESS, [node],
self.context.g_opts.mpprcFile)
# 2-4.is have "sslcrl-file.crl",config 'ssl_crl_file' option
status = self.sshTool.checkRemoteFileExist(
node,
os.path.join(allDnNodeDict[node], DefaultValue.SSL_CRL_FILE),
self.context.g_opts.mpprcFile)
# exists 'sslcrl-file.crl' file ,config option of 'postgresql.conf'
if (status):
if node == DefaultValue.GetHostIpOrName():
sshcmd = \
"gs_guc set -D %s " \
"-c \"ssl_crl_file='%s'\"" \
% (allDnNodeDict[node], DefaultValue.SSL_CRL_FILE)
else:
sshcmd = "gs_guc set -D %s " \
"-c \"ssl_crl_file=\\\\\\'%s\\\\\\'\"" \
% (allDnNodeDict[node], DefaultValue.SSL_CRL_FILE)
self.sshTool.executeCommand(sshcmd, "Exist 'ssl_crl_file'",
DefaultValue.SUCCESS, [node],
self.context.g_opts.mpprcFile)
else:
if (node == DefaultValue.GetHostIpOrName()):
sshcmd = "gs_guc set " \
"-D %s -c \"ssl_crl_file=''\"" % (
allDnNodeDict[node])
else:
sshcmd = "gs_guc set " \
"-D %s -c \"ssl_crl_file=\\\\\\'\\\\\\'\"" \
% (allDnNodeDict[node])
self.sshTool.executeCommand(sshcmd, "No exist 'ssl_crl_file'",
DefaultValue.SUCCESS, [node],
self.context.g_opts.mpprcFile)
# Clear empty file.
if (self.isDnEmpty(node)):
sshcmd = g_file.SHELL_CMD_DICT["deleteFile"] % (os.path.join(
allDnNodeDict[node],
EMPTY_CERT), os.path.join(allDnNodeDict[node], EMPTY_CERT))
self.sshTool.executeCommand(sshcmd, "Clear empty file.",
DefaultValue.SUCCESS, [node],
self.context.g_opts.mpprcFile)
self.logger.log("Successfully rollback SSL cert files on [%s]." %
node)
def doBackup(self):
"""
function: 1.back up binary files
2.back up parameter files
input : NA
output: NA
"""
self.logger.log("Backing up files.")
if self.backupBin:
self.logger.log("Backing up binary files.")
try:
self.logger.debug(
"Installation path is %s." % self.installPath)
if (len(os.listdir(self.installPath)) == 0):
raise Exception(ErrorCode.GAUSS_502["GAUSS_50203"] % (
"installation path [%s]" % self.installPath))
self.__tarDir(self.installPath, self.binTarName)
except Exception as e:
raise Exception(str(e))
self.logger.log("Successfully backed up binary files.")
if self.backupPara:
self.logger.log("Backing up parameter files.")
try:
self.logger.debug(
"Creating temporary directory for all parameter files.")
temp_dir = os.path.join(self.tmpBackupDir,
"parameter_%s" % HOSTNAME)
self.logger.debug("Temporary directory path: %s." % temp_dir)
if (os.path.exists(temp_dir)):
file_list = os.listdir(temp_dir)
if (len(file_list) != 0):
self.logger.debug(
"The temporary directory "
"is not empty.\n%s\nRemove all files silently."
% file_list)
g_file.cleanDirectoryContent(temp_dir)
else:
os.makedirs(temp_dir,
DefaultValue.KEY_DIRECTORY_PERMISSION)
self.logger.debug("Creating hostname file.")
hostnameFile = os.path.join(temp_dir, self.hostnameFileName)
self.logger.debug(
"Register hostname file path: %s." % hostnameFile)
g_file.createFileInSafeMode(hostnameFile)
with open(hostnameFile, "w") as self.__hostnameFile:
hostName = DefaultValue.GetHostIpOrName()
self.__hostnameFile.write("%s" % hostName)
self.logger.debug("Flush hostname file.")
self.__hostnameFile.flush()
self.__hostnameFile = None
os.chmod(hostnameFile, DefaultValue.KEY_FILE_PERMISSION)
self.logger.debug("Collecting parameter files.")
for inst in self.dbNodeInfo.datanodes:
self.__collectParaFilesToTempDir(inst, temp_dir)
self.logger.debug(
"Generating parameter files to be compressed.")
self.__tarDir(temp_dir, self.paraTarName, True)
self.logger.debug("Removing temporary directory.")
g_file.removeDirectory(temp_dir)
except Exception as e:
g_file.removeDirectory(temp_dir)
raise Exception(str(e))
self.logger.log("Successfully backed up parameter files.")
self.logger.log("Successfully backed up files.")
