def initGlobalInfos():
"""
function: init global infos
input: NA
output: NA
"""
global g_logger
global g_clusterInfo
g_logger = GaussLog(g_opts.logFile, "CheckUpgrade")
try:
if g_opts.action in [Const.ACTION_CHECK_VERSION]:
g_logger.log("No need to init cluster info under action %s" %
g_opts.action)
return
g_clusterInfo = dbClusterInfo()
if g_opts.xmlFile == "" or not os.path.exists(g_opts.xmlFile):
if g_opts.appPath == "" or not os.path.exists(g_opts.appPath):
raise Exception(ErrorCode.GAUSS_500["GAUSS_50001"] % "R")
staticConfigFile = "%s/bin/cluster_static_config" % g_opts.appPath
g_clusterInfo.initFromStaticConfig(g_opts.user, staticConfigFile)
else:
g_clusterInfo.initFromXml(g_opts.xmlFile)
except Exception as e:
g_logger.log(traceback.format_exc())
g_logger.logExit(str(e))
class CleanOsUser(LocalBaseOM):
'''
This class is for cleaning os user, it will not cleaning group.
'''
def __init__(self):
'''
Constructor
'''
self.userProfile = ""
self.user = ""
self.logger = None
##########################################################################
# Help context.
##########################################################################
def usage(self):
"""
function: usage
input : NA
output : NA
"""
print("CleanOsUser.py is a utility to clean OS user.")
print(" ")
print("Usage:")
print(" python3 CleanOsUser.py --help")
print(" python3 CleanOsUser.py -U user")
print(" ")
print("Common options:")
print(" -U the database program and cluster owner")
print(" --help show this help, then exit")
print(" ")
def __checkParameters(self):
"""
function: Check parameter from command line
input : NA
output: NA
"""
try:
opts, args = getopt.getopt(sys.argv[1:], "U:l:", ["help"])
except getopt.GetoptError as e:
GaussLog.exitWithError(ErrorCode.GAUSS_500["GAUSS_50000"] % str(e))
if (len(args) > 0):
GaussLog.exitWithError(ErrorCode.GAUSS_500["GAUSS_50000"] %
str(args[0]))
logFile = ""
for key, value in opts:
if (key == "-U"):
self.user = value
elif (key == "-l"):
logFile = value
elif (key == "--help"):
self.usage()
sys.exit(0)
else:
GaussLog.exitWithError(ErrorCode.GAUSS_500["GAUSS_50000"] %
key)
Parameter.checkParaVaild(key, value)
if (self.user == ""):
GaussLog.exitWithError(ErrorCode.GAUSS_500["GAUSS_50001"] % 'U' +
".")
try:
DefaultValue.checkUser(self.user, False)
except Exception as e:
GaussLog.exitWithError(str(e))
if (logFile == ""):
logFile = DefaultValue.getOMLogPath(DefaultValue.LOCAL_LOG_FILE,
self.user, "")
self.logger = GaussLog(logFile, "CleanOsUser")
self.logger.ignoreErr = True
##########################################################################
# This is the main clean OS user flow.
##########################################################################
def cleanOsUser(self):
"""
function: Clean OS user
input : NA
output: NA
"""
self.__checkParameters()
self.logger.log("Cleaning crash OS user.")
try:
# clean semaphore
subprocess.getstatusoutput("ipcs -s|awk '/ %s /{print $2}'|"
"xargs -n1 ipcrm -s" % self.user)
# get install path
cmd = "su - %s -c 'echo $GAUSSHOME' 2>/dev/null" % self.user
(status, output) = subprocess.getstatusoutput(cmd)
if (status != 0):
self.logger.logExit(ErrorCode.GAUSS_518["GAUSS_51802"] %
"$GAUSSHOME" + " Error:\n%s" % output)
gaussHome = output.strip()
if (gaussHome == ""):
self.logger.debug("$GAUSSHOME is null. This means you may "
"must clean crash install path manually.")
self.logger.debug("The installation path is %s." % gaussHome)
# delete user
status, output = subprocess.getstatusoutput("userdel -f %s" %
self.user)
if (status != 0):
self.logger.logExit(ErrorCode.GAUSS_503["GAUSS_50314"] %
self.user + " Error: \n%s" % output)
# delete path
status, output = subprocess.getstatusoutput("rm -rf '%s'" %
gaussHome)
if (status != 0):
self.logger.logExit(ErrorCode.GAUSS_502["GAUSS_50209"] %
gaussHome + " Error: \n%s" % output)
except Exception as e:
self.logger.logExit(str(e))
self.logger.log("Successfully cleaned OS user.")
def removeAllowUsers(self):
"""
function: Remove the specific user from 'AllowUsers'
in /etc/ssh/sshd_config
input : NA
output: NA
"""
sshd_config = "/etc/ssh/sshd_config"
try:
cmd = "cat %s | grep -E '\\<AllowUsers\\>'" % sshd_config
(status, output) = subprocess.getstatusoutput(cmd)
# Not found, or there is an error.
if status != 0:
if output is None or len(output.lstrip()) == 0:
self.logger.debug("No 'AllowUsers' configuration found"
" in %s" % sshd_config)
else:
# Error occurred, but there is no need to report.
self.logger.debug("Failed to get 'AllowUsers' from %s" %
sshd_config)
return
allowUsersLineBefore = output.lstrip()
userList = allowUsersLineBefore.split()
userList.remove(self.user)
allowUsersLineRemoved = ' '.join(userList)
cmd = "sed -i 's/%s/%s/g' %s" % (
allowUsersLineBefore, allowUsersLineRemoved, sshd_config)
(status, output) = subprocess.getstatusoutput(cmd)
# Not found, or there is an error.
if status != 0:
self.logger.debug(
"Failed to remove user '%s' from "
"'AllowUsers' in %s. Command: %s, Error: %s" %
(self.user, sshd_config, cmd, output))
except Exception as e:
self.logger.debug("Failed to remove user '%s' from 'AllowUsers'"
" in %s. Error: %s" %
(self.user, sshd_config, str(e)))
class CheckUninstall:
'''
classdocs
'''
def __init__(self):
'''
Constructor
'''
self.installPath = ""
self.user = ""
self.cleanUser = False
self.cleanData = False
self.logger = None
##########################################################################
# Help context. U:R:oC:v:
##########################################################################
def usage(self):
"""
function: usage
input : NA
output : NA
"""
print("CheckUninstall.py is a utility to check Gauss MPP Database"
" status .")
print(" ")
print("Usage:")
print(" python3 CheckUninstall.py --help")
print(" python3 CheckUninstall.py -R installpath -U user [-d] [-u]"
" [-l log]")
print(" ")
print("Common options:")
print(" -U the database program and cluster owner")
print(" -R the database program path")
print(" -d clean data path")
print(" -u clean user")
print(" -l log directory")
print(" --help show this help, then exit")
print(" ")
##########################################################################
# check uninstall
##########################################################################
def checkUninstall(self):
"""
function:
Check all kinds of environment. It includes:
1. Input parameters.
2. OS version.
3. User Info
4. If it has a old install.
input : NA
output: NA
"""
self.__checkParameters()
self.__checkOSVersion()
self.__checkOsUser()
self.__checkInstanllPath()
self.logger.closeLog()
def __checkParameters(self):
"""
function: check input parameters
input : NA
output: NA
"""
try:
opts, args = getopt.getopt(sys.argv[1:], "U:R:l:du", ["help"])
except getopt.GetoptError as e:
GaussLog.exitWithError(ErrorCode.GAUSS_500["GAUSS_50000"] % str(e))
if (len(args) > 0):
GaussLog.exitWithError(ErrorCode.GAUSS_500["GAUSS_50000"] %
str(args[0]))
logFile = ""
for key, value in opts:
if (key == "-U"):
self.user = value
elif (key == "-R"):
self.installPath = value
elif (key == "-l"):
logFile = value
elif (key == "-d"):
self.cleanData = True
elif (key == "-u"):
self.cleanUser = True
elif (key == "--help"):
self.usage()
sys.exit(0)
else:
GaussLog.exitWithError(ErrorCode.GAUSS_500["GAUSS_50000"] %
key)
Parameter.checkParaVaild(key, value)
if (self.user == ""):
GaussLog.exitWithError(ErrorCode.GAUSS_500["GAUSS_50001"] % 'U' +
".")
if (self.installPath == ""):
raise Exception(ErrorCode.GAUSS_500["GAUSS_50001"] % 'R' + ".")
if (logFile == ""):
logFile = DefaultValue.getOMLogPath(DefaultValue.LOCAL_LOG_FILE,
"", self.installPath, "")
self.logger = GaussLog(logFile, "CheckUninstall")
self.logger.debug("The installation path of program: " +
self.installPath)
self.logger.debug("The parameter of clean user is: %s." %
self.cleanUser)
self.logger.debug("The parameter of clean data is: %s." %
self.cleanData)
def __checkOSVersion(self):
"""
function: Check operator system version, install binary file version.
input : NA
output: NA
"""
self.logger.log("Checking OS version.")
try:
if (not DefaultValue.checkOsVersion()):
raise Exception(ErrorCode.GAUSS_519["GAUSS_51900"] +
"The current system is: %s." %
platform.platform())
except Exception as e:
raise Exception(str(e))
self.logger.log("Successfully checked OS version.")
def __checkOsUser(self):
"""
function: Check if user exists and get $GAUSSHOME
input : NA
output: NA
"""
if not self.cleanUser:
self.logger.log("Skipping user check. ")
return
self.logger.log("Checking OS user.")
try:
DefaultValue.checkUser(self.user, False)
except Exception as e:
raise Exception(str(e))
# Get GAUSSHOME
cmd = "echo $GAUSSHOME 2>/dev/null"
(status, output) = subprocess.getstatusoutput(cmd)
if (status != 0):
self.logger.debug("The cmd is %s " % cmd)
raise Exception(ErrorCode.GAUSS_518["GAUSS_51802"] % "$GAUSSHOME" +
" Error:\n%s" % output)
gaussHome = output.strip()
if (gaussHome == ""):
raise Exception(ErrorCode.GAUSS_518["GAUSS_51800"] % "$GAUSSHOME")
if (gaussHome != self.installPath):
self.logger.debug("$GAUSSHOME: %s." % gaussHome)
self.logger.debug("Installation path parameter: %s." %
self.installPath)
raise Exception(ErrorCode.GAUSS_518["GAUSS_51807"])
self.logger.log("Successfully checked OS user.")
def __checkInstanllPath(self):
"""
function: Check if path exists and get owner
input : NA
output: NA
"""
self.logger.log("Checking installation path.")
if (not os.path.exists(self.installPath)):
self.logger.log("Installation path does not exist: %s." %
self.installPath)
if (not self.cleanData and not self.cleanUser):
raise Exception(ErrorCode.GAUSS_502["GAUSS_50219"] %
"installation path")
else:
# Get owner
cmd = "stat -c '%%U:%%G' %s" % self.installPath
(status, output) = subprocess.getstatusoutput(cmd)
if (status != 0):
self.logger.debug("The cmd is %s " % cmd)
raise Exception(ErrorCode.GAUSS_503["GAUSS_50308"] +
" Error: \n%s" % str(output))
owerInfo = output.strip()
(user, group) = owerInfo.split(':')
if (self.user != user.strip()):
self.logger.debug("The owner information of installation"
" path: %s." % owerInfo)
self.logger.debug("User parameter : %s." % self.user)
raise Exception(ErrorCode.GAUSS_503["GAUSS_50315"] %
(self.user, self.installPath))
self.logger.log("Successfully checked installation path.")
class Uninstall(LocalBaseOM):
"""
uninstall the cluster
"""
def __init__(self):
"""
Constructor
"""
super(Uninstall, self).__init__()
self.installPath = ""
self.user = ""
self.keepDir = False
self.mpprcFile = ""
self.logFile = ""
self.logger = None
self.installflag = False
self.clusterInfo = None
self.localNode = None
self.keepData = True
self.method = ""
self.action = ""
##########################################################################
# Help context. U:R:oC:v:
##########################################################################
def usage(self):
"""
function: usage
"""
print("Uninstall.py is a utility to uninstall Gauss MPP Database.")
print(" ")
print("Usage:")
print(" python3 Uninstall.py --help")
print(" python3 Uninstall.py -U user -R installpath [-c] [-l log]")
print(" ")
print("Common options:")
print(" -U the database program and cluster owner")
print(" -R the database program install path")
print(" -l the log path")
print(" --help show this help, then exit")
print(" ")
##########################################################################
# This is the main uninstall flow.
##########################################################################
def uninstall(self):
"""
function: Remove install path content, which depend on $GAUSSHOME
input : NA
output: NA
"""
try:
self.logger.debug("OLAP's local uninstall.")
self.__cleanMonitor()
self.__cleanInstallProgram()
self.__changeuserEnv()
self.logger.closeLog()
except Exception as e:
raise Exception(str(e))
def __changeuserEnv(self):
"""
function: Change user GAUSS_ENV
input : NA
output: NA
"""
# clean os user environment variable
self.logger.log("Modifying user's environmental variable $GAUSS_ENV.")
userProfile = self.mpprcFile
DefaultValue.updateUserEnvVariable(userProfile, "GAUSS_ENV", "1")
if "HOST_IP" in os.environ.keys():
g_file.deleteLine(userProfile, "^\\s*export\\s*WHITELIST_ENV=.*$")
self.logger.log("Successfully modified user's environmental"
" variable GAUSS_ENV.")
self.logger.debug("Deleting symbolic link to $GAUSSHOME if exists.")
gaussHome = DefaultValue.getInstallDir(self.user)
if gaussHome == "":
raise Exception(ErrorCode.GAUSS_518["GAUSS_51800"] % "$GAUSSHOME")
if os.path.islink(gaussHome):
self.installPath = os.path.realpath(gaussHome)
os.remove(gaussHome)
else:
self.logger.debug("symbolic link does not exists.")
self.logger.debug("Deleting bin file in installation path.")
g_file.removeDirectory("%s/bin" % self.installPath)
self.logger.debug("Successfully deleting bin file in"
" installation path.")
def __cleanMonitor(self):
"""
function: clean om_monitor process and delete cron
input : NA
output: NA
"""
self.logger.log("Deleting monitor.")
try:
# get all content by crontab command
(status, output) = g_OSlib.getAllCrontab()
# overwrit crontabFile, make it empty.
crontabFile = "%s/gauss_crontab_file_%d" \
% (DefaultValue.getTmpDirFromEnv(), os.getpid())
g_file.createFile(crontabFile, True)
content_CronTabFile = [output]
g_file.writeFile(crontabFile, content_CronTabFile)
g_file.deleteLine(crontabFile, "\/bin\/om_monitor")
g_OSlib.execCrontab(crontabFile)
g_file.removeFile(crontabFile)
# clean om_monitor,cm_agent,cm_server process
for progname in ["om_monitor", "cm_agent", "cm_server"]:
g_OSlib.killallProcess(self.user, progname, '9')
except Exception as e:
if os.path.exists(crontabFile):
g_file.removeFile(crontabFile)
raise Exception(str(e))
self.logger.log("Successfully deleted OMMonitor.")
def checkParameters(self):
"""
function: Check input parameters
input : NA
output: NA
"""
try:
opts, args = getopt.getopt(sys.argv[1:], "t:U:R:l:X:M:T",
["help", "delete-data"])
except getopt.GetoptError as e:
GaussLog.exitWithError(ErrorCode.GAUSS_500["GAUSS_50000"]
% str(e))
if (len(args) > 0):
GaussLog.exitWithError(ErrorCode.GAUSS_500["GAUSS_50000"]
% str(args[0]))
for key, value in opts:
if (key == "-U"):
self.user = value
elif (key == "-R"):
self.installPath = value
elif (key == "-l"):
self.logFile = value
elif (key == "--help"):
self.usage()
sys.exit(0)
elif (key == "-T"):
self.installflag = True
elif key == "--delete-data":
self.keepData = False
elif key == "-M":
self.method = value
elif key == "-t":
self.action = value
else:
GaussLog.exitWithError(ErrorCode.GAUSS_500["GAUSS_50000"]
% key)
Parameter.checkParaVaild(key, value)
if (self.user == ""):
GaussLog.exitWithError(ErrorCode.GAUSS_500["GAUSS_50001"]
% 'U' + ".")
if (self.installPath == ""):
GaussLog.exitWithError(ErrorCode.GAUSS_500["GAUSS_50001"]
% 'R' + ".")
self.mpprcFile = DefaultValue.getMpprcFile()
if (self.logFile == ""):
self.logFile = DefaultValue.getOMLogPath(
DefaultValue.LOCAL_LOG_FILE, self.user, self.installPath)
def __initLogger(self):
"""
function: Init logger
input : NA
output: NA
"""
self.logger = GaussLog(self.logFile, "UninstallApp")
def __cleanInstallProgram(self):
"""
function: Clean install program
input : NA
output: NA
"""
if (not os.path.exists(self.installPath)):
self.logger.log("The installation directory does not exist. ")
return
realLink = self.installPath
if os.path.islink(self.installPath):
realLink = os.readlink(self.installPath)
# delete upgrade directory
self.logger.debug("Starting delete other installation directory.")
try:
recordVersionFile = os.path.realpath(
os.path.join(self.installPath, "record_app_directory"))
if os.path.isfile(recordVersionFile):
with open(recordVersionFile, 'r') as fp:
retLines = fp.readlines()
if len(retLines) != 2:
raise Exception(ErrorCode.GAUSS_502["GAUSS_50222"]
% recordVersionFile)
oldPath = retLines[0].strip()
newPath = retLines[1].strip()
if os.path.normcase(oldPath) == os.path.normcase(realLink):
g_file.removeDirectory(newPath)
else:
g_file.removeDirectory(oldPath)
self.logger.debug("Successfully deleted other installation"
" path need to delete.")
else:
self.logger.debug("No other installation path need"
" to delete.")
except Exception as e:
raise Exception(ErrorCode.GAUSS_502["GAUSS_50209"]
% "other installation"
+ " Can not delete other installation"
" directory: %s." % str(e))
self.logger.log("Removing the installation directory.")
try:
fileList = os.listdir(self.installPath)
for fileName in fileList:
fileName = fileName.replace("/", "").replace("..", "")
filePath = os.path.join(os.path.realpath(self.installPath),
fileName)
if os.path.isfile(filePath):
os.remove(filePath)
elif os.path.isdir(filePath):
if (fileName == "bin"):
binFileList = os.listdir(filePath)
for binFile in binFileList:
fileInBinPath = os.path.join(filePath, binFile)
if os.path.isfile(fileInBinPath) and \
binFile != "cluster_static_config":
os.remove(fileInBinPath)
elif os.path.islink(fileInBinPath):
os.remove(fileInBinPath)
elif os.path.isdir(fileInBinPath):
g_file.removeDirectory(fileInBinPath)
else:
g_file.removeDirectory(filePath)
self.logger.debug("Remove path:%s." % filePath)
self.logger.debug("Successfully deleted bin file"
" in installation path.")
except Exception as e:
raise Exception(ErrorCode.GAUSS_502["GAUSS_50209"]
% "installation"
+ " Can not delete installation directory: %s."
% str(e))
# regular match delete empty directory
self.logger.debug("Starting delete empty installation directory.")
try:
removeflag = False
namePrefix = os.path.basename(self.installPath)
gaussPath = os.path.realpath(os.path.dirname(self.installPath))
curInstallName = os.path.basename(realLink)
fileList = os.listdir(gaussPath)
for fileName in fileList:
if fileName.strip() != curInstallName.strip():
filePath = os.path.join(os.path.realpath(gaussPath),
fileName)
if os.path.isdir(filePath) \
and not os.listdir(filePath) and "_" in fileName:
fileNameElement = fileName.split("_", 1)
if namePrefix.strip() == fileNameElement[0].strip():
res = re.search(
'^(?![0-9]+$)(?![a-zA-Z]+$)[0-9A-Za-z]{8}$',
fileNameElement[1].strip())
if res:
removeflag = True
g_file.removeDirectory(filePath)
if removeflag:
self.logger.debug("Successfully deleted empty"
" installation path.")
else:
self.logger.debug("No empty installation path need"
" to delete.")
except Exception as e:
raise Exception(ErrorCode.GAUSS_502["GAUSS_50209"]
% "other installation"
+ " Can not delete empty installation"
" directory: %s." % str(e))
self.logger.log("Successfully deleted installation directory.")
def init(self):
"""
function: constuctor
"""
self.__initLogger()
class Postuninstall(LocalBaseOM):
"""
execute unPreInstall
"""
def __init__(self):
self.action = ""
self.userInfo = ""
self.user = ""
self.group = ""
self.clusterConfig = ""
self.preparePath = ""
self.checkEmpty = False
self.envParams = []
self.userProfile = ""
self.logFile = ""
self.clusterToolPath = ""
self.tmpFile = ""
self.component = []
self.clusterComponent = []
self.logger = None
self.userHome = ""
def initGlobals(self):
"""
init global variables
input : NA
output: NA
"""
global g_nodeInfo
self.logger = GaussLog(self.logFile, self.action)
if self.clusterConfig != "":
if os.path.isfile(self.clusterConfig):
self.clusterToolPath = DefaultValue.getPreClusterToolPath(
self.user, self.clusterConfig)
self.readConfigInfoByXML()
hostName = DefaultValue.GetHostIpOrName()
g_nodeInfo = self.clusterInfo.getDbNodeByName(hostName)
if (g_nodeInfo is None):
self.logger.logExit(
ErrorCode.GAUSS_516["GAUSS_51620"] % "local" +
" There is no host named %s!" % hostName)
else:
self.logger.logExit(ErrorCode.GAUSS_502["GAUSS_50210"] %
("config file [%s]" % self.clusterConfig))
elif self.action != ACTION_CLEAN_DEPENDENCY:
try:
self.clusterToolPath = DefaultValue.getClusterToolPath(
self.user)
except Exception as e:
self.logger.logExit(ErrorCode.GAUSS_502["GAUSS_50219"] %
"the cluster tool path" +
" Error: \n%s" % str(e))
if not self.clusterToolPath:
self.logger.logExit(ErrorCode.GAUSS_502["GAUSS_50219"] %
"cluster tool path")
# make sure if we are using env seperate version,
# and get the right profile
# we can not check mppenvfile exists here
mppenvFile = DefaultValue.getEnv(DefaultValue.MPPRC_FILE_ENV)
if (mppenvFile != "" and mppenvFile is not None):
self.userProfile = mppenvFile
else:
self.userProfile = "/home/%s/.bashrc" % self.user
def usage(self):
"""
Usage:
python3 UnPreInstallUtility.py -t action -u user [-X xmlfile] [-l log]
Common options:
-t the type of action
-u the os user of cluster
-X the xml file path
-l the path of log file
--help show this help, then exit
"""
print(self.usage.__doc__)
def parseCommandLine(self):
"""
function: Check parameter from command line
input : NA
output: NA
"""
try:
opts, args = getopt.getopt(sys.argv[1:], "t:u:X:l:f:Q:P:",
["help"])
except Exception as e:
self.usage()
GaussLog.exitWithError(ErrorCode.GAUSS_500["GAUSS_50000"] % str(e))
if (len(args) > 0):
GaussLog.exitWithError(ErrorCode.GAUSS_500["GAUSS_50000"] %
str(args[0]))
for (key, value) in opts:
if (key == "--help"):
self.usage()
sys.exit(0)
elif (key == "-t"):
self.action = value
elif (key == "-u"):
self.user = value
elif (key == "-X"):
self.clusterConfig = value
elif (key == "-l"):
self.logFile = os.path.realpath(value)
elif (key == "-f"):
self.tmpFile = value
elif key == "-Q":
self.clusterToolPath = value
elif key == "-P":
self.userHome = value
else:
GaussLog.exitWithError(ErrorCode.GAUSS_500["GAUSS_50000"] %
key)
Parameter.checkParaVaild(key, value)
def checkParameter(self):
"""
function: Check parameter from command line
input : NA
output: NA
"""
if self.action == "":
GaussLog.exitWithError(ErrorCode.GAUSS_500["GAUSS_50001"] % "t" +
".")
if self.logFile == "":
self.logFile = DefaultValue.getOMLogPath(
DefaultValue.LOCAL_LOG_FILE, self.user, "")
if self.user == "" and self.action != ACTION_CLEAN_DEPENDENCY:
GaussLog.exitWithError(ErrorCode.GAUSS_500["GAUSS_50001"] % "u" +
".")
def getSyslogType(self):
"""
function: judge syslog type
input : NA
output: str
"""
self.logger.debug("Judging the syslog type is rsyslog or syslog-ng.")
if (os.path.isfile(RSYSLOG_CONFIG_FILE)):
return RSYSLOG
elif (os.path.isfile(SYSLOG_NG_CONFIG_FILE)):
return SYSLOG_NG
else:
self.logger.logExit(ErrorCode.GAUSS_502["GAUSS_50219"] %
"rsyslog or syslog-ng" +
" \nError: Failed to judge the syslog type.")
def cleanWarningConfig(self):
"""
function: clean syslog-ng/rsyslog config
input : NA
output: NA
"""
self.logger.debug("Cleaning syslog-ng configuration.")
# judge the installed syslog type on the local host is rsyslog
# or syslog-ng
syslogType = self.getSyslogType()
if (syslogType == SYSLOG_NG):
self.cleanWarningConfigForSyslogng()
elif (syslogType == RSYSLOG):
self.cleanWarningConfigForRsyslog()
self.logger.debug("Successfully cleaned system log.")
def cleanWarningConfigForSyslogng(self):
"""
function: clean syslog-ng config
input : NA
output: NA
"""
# clean client syslog-ng configure
cmd = "(if [ -s '%s' ]; then " % SYSLOG_NG_CONFIG_FILE
cmd += \
"sed -i -e '/^filter f_gaussdb.*$/d' %s " % SYSLOG_NG_CONFIG_FILE
cmd += "-e '/^destination d_gaussdb.*$/d' %s " % SYSLOG_NG_CONFIG_FILE
cmd += \
"-e '/^log { source(src); filter(f_gaussdb); " \
"destination(d_gaussdb); };$/d' %s;fi;) " % SYSLOG_NG_CONFIG_FILE
self.logger.debug("Command for cleaning client system log: %s" % cmd)
(status, output) = subprocess.getstatusoutput(cmd)
if (status != 0):
self.logger.logExit(ErrorCode.GAUSS_514["GAUSS_51400"] % cmd +
" Error:\n%s" % output)
# clean server syslog-ng configure
cmd = "(if [ -s '%s' ]; then " % SYSLOG_NG_CONFIG_FILE
cmd += \
"sed -i -e '/^template t_gaussdb.*$/d' %s " % SYSLOG_NG_CONFIG_FILE
cmd += "-e '/^source s_gaussdb.*$/d' %s " % SYSLOG_NG_CONFIG_FILE
cmd += "-e '/^filter f_gaussdb.*$/d' %s " % SYSLOG_NG_CONFIG_FILE
cmd += "-e '/^destination d_gaussdb.*$/d' %s " % SYSLOG_NG_CONFIG_FILE
cmd += \
"-e '/^log { source(s_gaussdb); " \
"filter(f_gaussdb); destination(d_gaussdb); };$/d' %s;" \
"fi; " % SYSLOG_NG_CONFIG_FILE
cmd += "if [ -s '%s' ]; then " % SYSLOG_NG_CONFIG_FILE_SERVER
cmd += \
"sed -i -e '/^SYSLOGD_OPTIONS=\\\"-r -m 0\\\"/d' %s " \
% SYSLOG_NG_CONFIG_FILE_SERVER
cmd += "-e '/^KLOGD_OPTIONS=\\\"-x\\\"/d' %s; " \
% SYSLOG_NG_CONFIG_FILE_SERVER
cmd += "fi) "
self.logger.debug("Command for cleaning server system log: %s" % cmd)
(status, output) = subprocess.getstatusoutput(cmd)
if (status != 0):
self.logger.logExit(ErrorCode.GAUSS_514["GAUSS_51400"] % cmd +
" Error:\n%s" % output)
# restart the syslog service
(status, output) = g_service.manageOSService("syslog", "restart")
if (status != 0):
self.logger.logExit(ErrorCode.GAUSS_508["GAUSS_50802"] %
"restart syslog" + " Error: \n%s" % output)
def cleanWarningConfigForRsyslog(self):
"""
function: clean rsyslog config
input : NA
output: NA
"""
# clean rsyslog config on client and server
cmd = "(if [ -s '%s' ]; then " % RSYSLOG_CONFIG_FILE
cmd += \
"sed -i -e '/^$ModLoad imjournal.*$/d' %s " % RSYSLOG_CONFIG_FILE
cmd += "-e '/^$ModLoad imudp.*$/d' %s " % RSYSLOG_CONFIG_FILE
cmd += "-e '/^$UDPServerRun 514.*$/d' %s " % RSYSLOG_CONFIG_FILE
cmd += \
"-e '/^$imjournalRatelimitInterval.*$/d' %s " % RSYSLOG_CONFIG_FILE
cmd += "-e '/^$imjournalRatelimitBurst.*$/d' %s " % RSYSLOG_CONFIG_FILE
cmd += "-e '/^%s.*$/d' %s; " % (AP_RSYSLOG_FACILITY_LEVEL,
RSYSLOG_CONFIG_FILE)
cmd += "fi) "
self.logger.debug("Command for cleaning crash rsyslog: %s." % cmd)
(status, output) = subprocess.getstatusoutput(cmd)
if (status != 0):
self.logger.logExit(ErrorCode.GAUSS_502["GAUSS_50207"] %
'crash rsyslog' + " Error: \n%s" % output)
# restart the rsyslog service
(status, output) = g_service.manageOSService("rsyslog", "restart")
if (status != 0):
self.logger.logExit(ErrorCode.GAUSS_508["GAUSS_50802"] %
"restart rsyslog" + " Error: \n%s" % output)
def cleanEnvSoftware(self):
"""
function: clean environment software and variable
Gauss-MPPDB* & sctp_patch is came from R5 upgrade R7
input : NA
output: NA
"""
self.logger.debug("Cleaning the environmental software and variable.")
# clean environment software
path = "%s/%s" % (self.clusterToolPath, PSSHDIR)
g_file.removeDirectory(path)
path = "%s/lib" % self.clusterToolPath
g_file.removeDirectory(path)
path = "%s/script" % self.clusterToolPath
g_file.removeDirectory(path)
path = "%s/sudo" % self.clusterToolPath
g_file.removeDirectory(path)
path = "%s/upgrade.sh" % self.clusterToolPath
g_file.removeFile(path)
path = "%s/version.cfg" % self.clusterToolPath
g_file.removeFile(path)
path = "%s/GaussDB.py" % self.clusterToolPath
g_file.removeFile(path)
path = "%s/libcgroup" % self.clusterToolPath
g_file.removeDirectory(path)
path = "%s/server.key.cipher" % self.clusterToolPath
g_file.removeFile(path)
path = "%s/server.key.rand" % self.clusterToolPath
g_file.removeFile(path)
path = "%s/%s*" % (self.clusterToolPath, VersionInfo.PRODUCT_NAME)
g_file.removeDirectory(path)
path = "%s/Gauss*" % (self.clusterToolPath)
g_file.removeDirectory(path)
path = "%s/sctp_patch" % (self.clusterToolPath)
g_file.removeDirectory(path)
path = "%s/unixodbc" % self.clusterToolPath
g_file.removeDirectory(path)
path = "%s/%s" % (self.clusterToolPath, Const.UPGRADE_SQL_FILE)
g_file.removeFile(path)
path = "%s/%s" % (self.clusterToolPath, Const.UPGRADE_SQL_SHA)
g_file.removeFile(path)
self.logger.debug(
"Successfully cleaned the environmental software and variable.")
self.logger.debug("Cleaning environmental software.")
# clean environment variable
cmd = "(if [ -s '%s' ]; then " % PROFILE_FILE
cmd += "sed -i -e '/^export GPHOME=%s$/d' %s " % (
self.clusterToolPath.replace('/', '\/'), PROFILE_FILE)
cmd += \
"-e '/^export PATH=\$GPHOME\/pssh-2.3.1\/bin:" \
"\$GPHOME\/script:\$PATH$/d' %s " % PROFILE_FILE
cmd += \
"-e '/^export PATH=\$GPHOME\/script\/gspylib\/pssh\/bin:" \
"\$GPHOME\/script:\$PATH$/d' %s " % PROFILE_FILE
cmd += \
"-e '/^export LD_LIBRARY_PATH=\$GPHOME\/lib:" \
"\$LD_LIBRARY_PATH$/d' %s " % PROFILE_FILE
cmd += \
"-e '/^export PYTHONPATH=\$GPHOME\/lib$/d' %s; fi) " % PROFILE_FILE
self.logger.debug("Command for cleaning environment variable: %s." %
cmd)
(status, output) = subprocess.getstatusoutput(cmd)
if (status != 0):
self.logger.logExit(ErrorCode.GAUSS_514["GAUSS_51400"] % cmd +
" Error:\n%s" % output)
self.logger.debug(
"Successfully cleaned environmental software and variable.")
def checkUnPreInstall(self):
"""
function: check whether do uninstall before unpreinstall
input : NA
output: NA
"""
self.logger.debug("Checking UnPreInstall.")
# check if user exist
try:
DefaultValue.getUserId(self.user)
except Exception as e:
self.logger.logExit(str(e))
# check if user profile exist
if (not os.path.exists(self.userProfile)):
self.logger.debug("The %s does not exist." % self.userProfile +
" Please skip to check UnPreInstall.")
return
# check $GAUSSHOME
cmd = "su - %s -c 'source %s && echo $GAUSS_ENV' 2>/dev/null" % (
self.user, self.userProfile)
self.logger.debug("Command for getting $GAUSSHOME: %s" % cmd)
(status, output) = subprocess.getstatusoutput(cmd)
if (status != 0):
self.logger.logExit(ErrorCode.GAUSS_514["GAUSS_51400"] % cmd +
" Error:\n%s" % output)
gaussEnv = output.strip()
if (gaussEnv == "2"):
self.logger.logExit(ErrorCode.GAUSS_525["GAUSS_52501"] %
"gs_uninstall")
# check $GAUSS_ENV
cmd = "su - %s -c 'source %s && echo $GAUSS_ENV' 2>/dev/null" % (
self.user, self.userProfile)
self.logger.debug("Command for getting $GAUSS_ENV: %s" % cmd)
(status, output) = subprocess.getstatusoutput(cmd)
if (status != 0):
self.logger.logExit(ErrorCode.GAUSS_514["GAUSS_51400"] % cmd +
" Error:\n%s" % output)
gaussEnv = output.strip()
if (str(gaussEnv) != "1"):
self.logger.logExit(ErrorCode.GAUSS_525["GAUSS_52501"] %
"gs_preinstall")
self.logger.debug("Successfully checked UnPreInstall.")
def cleanGaussEnv(self):
"""
function: clean $GAUSS_ENV
input : NA
output: NA
"""
self.logger.debug("Cleaning $GAUSS_ENV.")
# check if user profile exist
if (self.userProfile is not None and self.userProfile != ""):
userProfile = self.userProfile
else:
userProfile = "/home/%s/.bashrc" % self.user
if (not os.path.exists(userProfile)):
self.logger.debug("The %s does not exist." % userProfile +
" Please skip to clean $GAUSS_ENV.")
return
# clean user's environmental variable
DefaultValue.cleanUserEnvVariable(userProfile,
cleanGAUSS_WARNING_TYPE=True)
# clean $GAUSS_ENV
envContent = "^\\s*export\\s*GAUSS_ENV=.*$"
g_file.deleteLine(userProfile, envContent)
self.logger.debug("Cleaned $GAUSS_ENV.")
def cleanNetworkfile(self, backIpNIC, virtualIp):
"""
function: clean configured IP in Network file
input : NA
output: NA
"""
self.logger.debug("Cleaning network file.")
try:
# read information from networkfile
networkfile = "/etc/sysconfig/network/ifcfg-" + backIpNIC
networkinfo = []
# check if the file is a link
g_OSlib.checkLink(networkfile)
with open(networkfile, "r") as fp:
networkinfo = fp.readlines()
LABEL = self.getLABEL(virtualIp, networkfile)
if (LABEL is not None):
# init linenum for delete
del_1 = 0
del_2 = 0
linenum = 1
for line in networkinfo:
if (line.split("=")[1].strip() == virtualIp):
# find if the netmask exist, if exist, delete this line
cmd_g = "grep -n 'NETMASK_%s=' %s" % (LABEL,
networkfile)
(status, output) = subprocess.getstatusoutput(cmd_g)
if (status == 0):
linenum_net = int(output.split(":")[0])
if (linenum + 1 == linenum_net):
del_1 = linenum_net
# find if the LABEL number exist,
# if exist, delete this line
cmd_g = "grep -n 'LABEL_%s=' %s " % (LABEL,
networkfile)
(status, output) = subprocess.getstatusoutput(cmd_g)
if (status == 0):
linenum_net = int(output.split(":")[0])
if (linenum + 2 == linenum_net):
del_2 = linenum_net
# delete issues which exist
if (del_1 != 0 and del_2 != 0):
cmd = "sed -i '%dd;%dd;%dd' %s" % (
linenum, del_1, del_2, networkfile)
elif (del_1 != 0 and del_2 == 0):
cmd = "sed -i '%dd;%dd' %s" % (linenum, del_1,
networkfile)
elif (del_1 == 0 and del_2 != 0):
cmd = "sed -i '%dd;%dd' %s" % (linenum, del_2,
networkfile)
else:
cmd = "sed -i '%dd' %s" % (linenum, networkfile)
(status, output) = subprocess.getstatusoutput(cmd)
if (status != 0):
raise Exception(
ErrorCode.GAUSS_514["GAUSS_51400"] % cmd +
"Error:\n%s" % output)
linenum += 1
self.logger.log(
"Successfully clean virtual Ip from network file")
else:
raise Exception(ErrorCode.GAUSS_502["GAUSS_50204"] %
("the LABEL number of %s " % virtualIp))
self.logger.debug("Successfully cleaned network file.")
except Exception as e:
self.logger.log("Error: Write networkfile failed." + str(e))
def IsSuSE12SP0(self):
"""
function:Check is OS SuSE12.0
input :NA
output :bool
"""
if (os.path.isfile("/etc/SuSE-release")):
cmd = "grep -i 'PATCHLEVEL' /etc/SuSE-release " \
"| awk -F '=' '{print $2}'"
(status, output) = subprocess.getstatusoutput(cmd)
if (status == 0 and output != ""):
if (output.strip().isdigit() and int(output.strip()) == 0):
return True
else:
raise Exception(ErrorCode.GAUSS_514["GAUSS_51400"] % cmd +
" Error: \n%s " % output)
return False
def getLABEL(self, virtualIp, networkfile):
"""
function: get LABEL number of virtual ip from network file
input : fp, virtualIp
output: int
"""
# check if the file is a link
g_OSlib.checkLink(networkfile)
with open(networkfile, "r") as fp:
for line in fp:
if line.split("=")[1].strip() == virtualIp:
if line.split("IPADDR_")[1].split("=%s" % virtualIp)[0]:
return line.split("IPADDR_")[1].split("=%s" %
virtualIp)[0]
else:
return None
return None
def cleanGroup(self):
"""
function: clean group
input : NA
output: NA
"""
self.logger.debug("Cleaning user group.")
hostName = DefaultValue.GetHostIpOrName()
groupname = self.user
try:
groupid = grp.getgrnam(groupname).gr_gid
except Exception:
self.logger.debug("group %s has been deleted." % groupname)
sys.exit(0)
cmd = "cat /etc/passwd | awk -F [:] '{print $1 \" \"$4}'" \
"|grep ' %s$'" % groupid
(status, output) = subprocess.getstatusoutput(cmd)
if status == 0:
self.logger.logExit(
"Warning: There are other users in the group %s on %s,"
" skip to delete group." % (groupname, hostName))
elif status == 1:
cmd = "groupdel %s" % groupname
(status, output) = subprocess.getstatusoutput(cmd)
if status != 0:
self.logger.logExit("Warning: Failed to delete group "
"%s by cmd:%s. Error: \n%s" %
(groupname, cmd, output))
else:
self.logger.logExit("Warning: Failed to delete group "
"%s by cmd:%s. Error: \n%s" %
(groupname, cmd, output))
self.logger.debug("Successfully cleaned user group.")
def cleanScript(self):
"""
function: clean script
"""
# clean lib
libPath = os.path.join(self.clusterToolPath, LIBPATH)
if os.path.exists(libPath):
g_file.removeDirectory(libPath)
# clean om script
scriptPath = os.path.join(self.clusterToolPath, SCRIPTPATH)
if os.path.exists(scriptPath):
g_file.removeDirectory(scriptPath)
# clean root script path
root_script_path = os.path.join(DefaultValue.ROOT_SCRIPTS_PATH,
self.user)
if os.path.exists(root_script_path):
g_file.removeDirectory(root_script_path)
# if /root/gauss_om has no files, delete it.
if not os.listdir(DefaultValue.ROOT_SCRIPTS_PATH):
g_file.removeDirectory(DefaultValue.ROOT_SCRIPTS_PATH)
# clean others
if os.path.exists(self.clusterToolPath):
g_file.cleanDirectoryContent(self.clusterToolPath)
if self.userHome != "":
if os.path.exists(self.userHome):
g_file.removeDirectory(self.userHome)
def cleanEnv(self):
"""
function: clean envriment variable
"""
self.logger.debug("Begin clean envrionment variable")
if not self.userProfile:
self.logger.logExit("Clean Env failed: can not get user profile.")
for comp in self.clusterComponent:
comp.cleanEnv(self.userProfile)
# clean user's environment variable
self.logger.debug("Clean user environment variable.")
DefaultValue.cleanUserEnvVariable(self.userProfile,
cleanGAUSS_WARNING_TYPE=True)
# clean GAUSS_ENV
self.logger.debug("Clean GAUSS_ENV.")
g_file.deleteLine(self.userProfile, "^\\s*export\\s*GAUSS_ENV=.*$")
self.logger.debug("Clean envrionment variable successfully.")
def cleanPath(self):
"""
function: clean path
input: NA
output: NA
"""
self.logger.debug("Begin clean path")
if os.path.exists(self.clusterInfo.appPath):
self.logger.debug("Deleting the install directory.")
cleanPath = os.path.join(self.clusterInfo.appPath, "./*")
g_file.removeDirectory(cleanPath)
self.logger.debug("Successfully deleted the install directory.")
for i in self.component:
i.cleanPath()
gsdbHomePath = "/home/%s/gsdb_home" % self.user
if os.path.exists(gsdbHomePath):
self.logger.debug("Deleting the gsdb home path.")
g_file.removeDirectory(gsdbHomePath)
self.logger.debug("Successfully deleted the gsdb home path.")
self.logger.debug("Clean Path successfully.")
def run(self):
try:
self.parseCommandLine()
self.checkParameter()
self.initGlobals()
except Exception as e:
GaussLog.exitWithError(str(e))
try:
if (self.action == ACTION_CLEAN_SYSLOG_CONFIG):
self.cleanWarningConfig()
elif (self.action == ACTION_CLEAN_TOOL_ENV):
self.cleanEnvSoftware()
elif (self.action == ACTION_CHECK_UNPREINSTALL):
self.checkUnPreInstall()
elif (self.action == ACTION_CLEAN_GAUSS_ENV):
self.cleanGaussEnv()
elif (self.action == ACTION_DELETE_GROUP):
self.cleanGroup()
elif (self.action == ACTION_CLEAN_DEPENDENCY):
self.cleanScript()
elif (self.action == ACTION_CLEAN_ENV):
self.cleanEnv()
elif (self.action == ACTION_CLEAN_INSTANCE_PATHS):
self.cleanPath()
else:
self.logger.logExit(ErrorCode.GAUSS_500["GAUSS_50000"] %
self.action)
except Exception as e:
self.logger.logExit(str(e))
class ParallelBaseOM(object):
"""
Base class of parallel command
"""
ACTION_INSTALL = "install"
ACTION_CONFIG = "config"
ACTION_START = "start"
ACTION_REDISTRIBUTE = "redistribute"
ACTION_HEALTHCHECK = "healthcheck"
HEALTH_CHECK_BEFORE = "before"
HEALTH_CHECK_AFTER = "after"
"""
Base class for parallel command
"""
def __init__(self):
'''
Constructor
'''
self.logger = None
self.clusterInfo = None
self.oldClusterInfo = None
self.sshTool = None
self.action = ""
# Cluster config file.
self.xmlFile = ""
self.oldXmlFile = ""
self.logType = DefaultValue.LOCAL_LOG_FILE
self.logFile = ""
self.localLog = ""
self.user = ""
self.group = ""
self.mpprcFile = ""
# Temporary catalog for install
self.operateStepDir = TempfileManagement.getTempDir(
"%s_step" % self.__class__.__name__.lower())
# Temporary files for install step
self.operateStepFile = "%s/%s_step.dat" % (
self.operateStepDir, self.__class__.__name__.lower())
self.initStep = ""
self.dws_mode = False
self.rollbackCommands = []
self.etcdCons = []
self.cmCons = []
self.gtmCons = []
self.cnCons = []
self.dnCons = []
# localMode is same as isSingle in all OM script, expect for
# gs_preinstall.
# in gs_preinstall, localMode means local mode for master-standby
# cluster.
# in gs_preinstall, localMode also means local mode for single
# cluster(will not create os user).
# in gs_preinstall, isSingle means single cluster, it will create
# os user.
# not isSingle and not localMode : master-standby cluster global
# mode(will create os user).
# not isSingle and localMode : master-standby cluster local
# mode(will not create os user).
# isSingle and not localMode : single cluster(will create os user).
# isSingle and localMode : single cluster(will not create os user).
self.localMode = False
self.isSingle = False
# Indicates whether there is a logical cluster.
# If elastic_group exists, the current cluster is a logical cluster.
# Otherwise, it is a large physical cluster.
self.isElasticGroup = False
self.isAddElasticGroup = False
self.lcGroup_name = ""
# Lock the cluster mode, there are two modes: exclusive lock and
# wait lock mode,
# the default exclusive lock
self.lockMode = "exclusiveLock"
# SinglePrimaryMultiStandby support binary upgrade, inplace upgrade
self.isSinglePrimaryMultiStandby = False
# Adapt to 200 and 300
self.productVersion = None
def initComponent(self):
"""
function: Init component
input : NA
output: NA
"""
for nodeInfo in self.clusterInfo.dbNodes:
self.initKernelComponent(nodeInfo)
def initComponentAttributes(self, component):
"""
function: Init component attributes on current node
input : Object component
output: NA
"""
component.logger = self.logger
component.binPath = "%s/bin" % self.clusterInfo.appPath
component.dwsMode = self.dws_mode
def initKernelComponent(self, nodeInfo):
"""
function: Init kernel component
input : Object nodeInfo
output: NA
"""
for inst in nodeInfo.datanodes:
component = DN_OLAP()
# init component cluster type
component.clusterType = self.clusterInfo.clusterType
component.instInfo = inst
self.initComponentAttributes(component)
self.dnCons.append(component)
def initLogger(self, module=""):
"""
function: Init logger
input : module
output: NA
"""
# log level
LOG_DEBUG = 1
self.logger = GaussLog(self.logFile, module, LOG_DEBUG)
dirName = os.path.dirname(self.logFile)
self.localLog = os.path.join(dirName, DefaultValue.LOCAL_LOG_FILE)
def initClusterInfo(self, refreshCN=True):
"""
function: Init cluster info
input : NA
output: NA
"""
try:
self.clusterInfo = dbClusterInfo()
if (refreshCN):
static_config_file = "%s/bin/cluster_static_config" % \
DefaultValue.getInstallDir(self.user)
self.clusterInfo.initFromXml(self.xmlFile, static_config_file)
else:
self.clusterInfo.initFromXml(self.xmlFile)
except Exception as e:
raise Exception(str(e))
self.logger.debug("Instance information of cluster:\n%s." %
str(self.clusterInfo))
def initClusterInfoFromStaticFile(self, user, flag=True):
"""
function: Function to init clusterInfo from static file
input : user
output: NA
"""
try:
self.clusterInfo = dbClusterInfo()
self.clusterInfo.initFromStaticConfig(user)
except Exception as e:
raise Exception(str(e))
if flag:
self.logger.debug("Instance information of cluster:\n%s." %
str(self.clusterInfo))
def initSshTool(self, nodeNames, timeout=0):
"""
function: Init ssh tool
input : nodeNames, timeout
output: NA
"""
self.sshTool = SshTool(nodeNames, self.logger.logFile, timeout)
def check_cluster_version_consistency(self, clusterNodes, newNodes=None):
"""
"""
self.logger.log("Check cluster version consistency.")
if newNodes is None:
newNodes = []
dic_version_info = {}
# check version.cfg on every node.
gp_home = DefaultValue.getEnv("GPHOME")
gauss_home = DefaultValue.getEnv("GAUSSHOME")
if not (os.path.exists(gp_home) and os.path.exists(gauss_home)):
GaussLog.exitWithError(ErrorCode.GAUSS_502["GAUSS_50201"] %
("%s", "or %s") % (gp_home, gauss_home))
for ip in clusterNodes:
if ip in newNodes:
cmd = "pssh -s -H %s 'cat %s/version.cfg'" % \
(ip, DefaultValue.getEnv("GPHOME"))
else:
cmd = "pssh -s -H %s 'cat %s/bin/upgrade_version'" % \
(ip, DefaultValue.getEnv("GAUSSHOME"))
status, output = subprocess.getstatusoutput(cmd)
if (status != 0):
raise Exception(ErrorCode.GAUSS_514["GAUSS_51400"] % cmd +
" Error:\n%s" % str(output))
if len(output.strip().split()) < 3:
raise Exception(ErrorCode.GAUSS_516["GAUSS_51623"])
dic_version_info[ip] = ",".join(output.strip().split()[1:])
self.logger.debug("The cluster version on every node.")
for check_ip, version_info in dic_version_info.items():
self.logger.debug("%s : %s" % (check_ip, version_info))
if len(set(dic_version_info.values())) != 1:
L_inconsistent = list(set(dic_version_info.values()))
self.logger.debug("The package version on some nodes are "
"inconsistent\n%s" % str(L_inconsistent))
raise Exception("The package version on some nodes are "
"inconsistent,%s" % str(L_inconsistent))
self.logger.log("Successfully checked cluster version.")
def checkBaseFile(self, checkXml=True):
"""
function: Check xml file and log file
input : checkXml
output: NA
"""
if (checkXml):
if (self.xmlFile == ""):
raise Exception(ErrorCode.GAUSS_500["GAUSS_50001"] % 'X' + ".")
if (not os.path.exists(self.xmlFile)):
raise Exception(ErrorCode.GAUSS_502["GAUSS_50201"] %
("configuration file [%s]" % self.xmlFile))
if (not os.path.isabs(self.xmlFile)):
raise Exception(ErrorCode.GAUSS_502["GAUSS_50213"] %
("configuration file [%s]" % self.xmlFile))
else:
self.xmlFile = ""
if (self.logFile == ""):
self.logFile = DefaultValue.getOMLogPath(self.logType, self.user,
"", self.xmlFile)
if (not os.path.isabs(self.logFile)):
raise Exception(ErrorCode.GAUSS_502["GAUSS_50213"] % "log")
def initSignalHandler(self):
"""
function: Function to init signal handler
input : NA
output: NA
"""
signal.signal(signal.SIGINT, signal.SIG_IGN)
signal.signal(signal.SIGQUIT, signal.SIG_IGN)
signal.signal(signal.SIGTERM, signal.SIG_IGN)
signal.signal(signal.SIGALRM, signal.SIG_IGN)
signal.signal(signal.SIGHUP, signal.SIG_IGN)
signal.signal(signal.SIGUSR1, signal.SIG_IGN)
signal.signal(signal.SIGUSR2, signal.SIG_IGN)
def print_signal_stack(self, frame):
"""
function: Function to print signal stack
input : frame
output: NA
"""
if (self.logger is None):
return
try:
import inspect
stacks = inspect.getouterframes(frame)
for curr in range(len(stacks)):
stack = stacks[curr]
self.logger.debug("Stack level: %d. File: %s. Function: "
"%s. LineNo: %d." %
(curr, stack[1], stack[3], stack[2]))
self.logger.debug("Code: %s." %
(stack[4][0].strip().strip("\n")))
except Exception as e:
self.logger.debug("Failed to print signal stack. Error: \n%s" %
str(e))
def raise_handler(self, signal_num, frame):
"""
function: Function to raise handler
input : signal_num, frame
output: NA
"""
if (self.logger is not None):
self.logger.debug("Received signal[%d]." % (signal_num))
self.print_signal_stack(frame)
raise Exception(ErrorCode.GAUSS_516["GAUSS_51614"] % (signal_num))
def setupTimeoutHandler(self):
"""
function: Function to set up time out handler
input : NA
output: NA
"""
signal.signal(signal.SIGALRM, self.timeout_handler)
def setTimer(self, timeout):
"""
function: Function to set timer
input : timeout
output: NA
"""
self.logger.debug("Set timer. The timeout: %d." % timeout)
signal.signal(signal.SIGALRM, self.timeout_handler)
signal.alarm(timeout)
def resetTimer(self):
"""
function: Reset timer
input : NA
output: NA
"""
signal.signal(signal.SIGALRM, signal.SIG_IGN)
self.logger.debug("Reset timer. Left time: %d." % signal.alarm(0))
def timeout_handler(self, signal_num, frame):
"""
function: Received the timeout signal
input : signal_num, frame
output: NA
"""
if (self.logger is not None):
self.logger.debug("Received the timeout signal: [%d]." %
(signal_num))
self.print_signal_stack(frame)
raise Timeout("Time out.")
def waitProcessStop(self, processKeywords, hostname):
"""
function: Wait the process stop
input : process name
output: NA
"""
count = 0
while (True):
psCmd = "ps ux|grep -v grep |awk '{print \$11}'|grep '%s' " % \
processKeywords.strip()
(status,
output) = self.sshTool.getSshStatusOutput(psCmd, [hostname])
# Determine whether the process can be found.
if (status[hostname] != DefaultValue.SUCCESS):
self.logger.debug("The %s process stopped." % processKeywords)
break
count += 1
if (count % 20 == 0):
self.logger.debug("The %s process exists." % processKeywords)
time.sleep(3)
def managerOperateStepDir(self, action='create', nodes=None):
"""
function: manager operate step directory
input : NA
output: currentStep
"""
if nodes is None:
nodes = []
try:
# Creating the backup directory
if (action == "create"):
cmd = "(if [ ! -d '%s' ];then mkdir -p '%s' -m %s;fi)" % (
self.operateStepDir, self.operateStepDir,
DefaultValue.KEY_DIRECTORY_MODE)
else:
cmd = "(if [ -d '%s' ];then rm -rf '%s';fi)" % (
self.operateStepDir, self.operateStepDir)
DefaultValue.execCommandWithMode(cmd,
"%s temporary directory" % action,
self.sshTool, self.localMode
or self.isSingle, "", nodes)
except Exception as e:
raise Exception(str(e))
def readOperateStep(self):
"""
function: read operate step signal
input : NA
output: currentStep
"""
currentStep = self.initStep
if not os.path.exists(self.operateStepFile):
self.logger.debug("The %s does not exits." % self.operateStepFile)
return currentStep
if not os.path.isfile(self.operateStepFile):
self.logger.debug("The %s must be a file." % self.operateStepFile)
return currentStep
with open(self.operateStepFile, "r") as fp:
line = fp.readline().strip()
if line is not None and line != "":
currentStep = line
return currentStep
def writeOperateStep(self, stepName, nodes=None):
"""
function: write operate step signal
input : step
output: NA
"""
if nodes is None:
nodes = []
try:
# write the step into INSTALL_STEP
# open the INSTALL_STEP
with open(self.operateStepFile, "w") as g_DB:
# write the INSTALL_STEP
g_DB.write(stepName)
g_DB.write(os.linesep)
g_DB.flush()
# change the INSTALL_STEP permissions
g_file.changeMode(DefaultValue.KEY_FILE_MODE, self.operateStepFile)
# distribute file to all nodes
cmd = "mkdir -p -m %s '%s'" % (DefaultValue.KEY_DIRECTORY_MODE,
self.operateStepDir)
DefaultValue.execCommandWithMode(
cmd, "create backup directory "
"on all nodes", self.sshTool, self.localMode or self.isSingle,
"", nodes)
if not self.localMode and not self.isSingle:
self.sshTool.scpFiles(self.operateStepFile,
self.operateStepDir, nodes)
except Exception as e:
# failed to write the step into INSTALL_STEP
raise Exception(str(e))
def distributeFiles(self):
"""
function: distribute package to every host
input : NA
output: NA
"""
self.logger.debug("Distributing files.")
try:
# get the all nodes
hosts = self.clusterInfo.getClusterNodeNames()
if DefaultValue.GetHostIpOrName() not in hosts:
raise Exception(ErrorCode.GAUSS_516["GAUSS_51619"] %
DefaultValue.GetHostIpOrName())
hosts.remove(DefaultValue.GetHostIpOrName())
# Send xml file to every host
DefaultValue.distributeXmlConfFile(self.sshTool, self.xmlFile,
hosts, self.mpprcFile)
# Successfully distributed files
self.logger.debug("Successfully distributed files.")
except Exception as e:
# failed to distribute package to every host
raise Exception(str(e))
def checkPreInstall(self, user, flag, nodes=None):
"""
function: check if have done preinstall on given nodes
input : user, nodes
output: NA
"""
if nodes is None:
nodes = []
try:
cmd = "%s -U %s -t %s" % (
OMCommand.getLocalScript("Local_Check_PreInstall"), user, flag)
DefaultValue.execCommandWithMode(cmd, "check preinstall",
self.sshTool, self.localMode
or self.isSingle, "", nodes)
except Exception as e:
raise Exception(str(e))
def checkNodeInstall(self,
nodes=None,
checkParams=None,
strictUserCheck=True):
"""
function: Check node install
input : nodes, checkParams, strictUserCheck
output: NA
"""
if nodes is None:
nodes = []
if checkParams is None:
checkParams = []
validParam = ["shared_buffers", "max_connections"]
cooGucParam = ""
for param in checkParams:
entry = param.split("=")
if (len(entry) != 2):
raise Exception(ErrorCode.GAUSS_500["GAUSS_50009"])
if (entry[0].strip() in validParam):
cooGucParam += " -C \\\"%s\\\"" % param
self.logger.log("Checking installation environment on all nodes.")
cmd = "%s -U %s:%s -R %s %s -l %s -X '%s'" % (
OMCommand.getLocalScript("Local_Check_Install"), self.user,
self.group, self.clusterInfo.appPath, cooGucParam, self.localLog,
self.xmlFile)
if (not strictUserCheck):
cmd += " -O"
self.logger.debug("Checking the install command: %s." % cmd)
DefaultValue.execCommandWithMode(cmd, "check installation environment",
self.sshTool, self.localMode
or self.isSingle, "", nodes)
def cleanNodeConfig(self, nodes=None, datadirs=None):
"""
function: Clean instance
input : nodes, datadirs
output: NA
"""
self.logger.log("Deleting instances from all nodes.")
if nodes is None:
nodes = []
if datadirs is None:
datadirs = []
cmdParam = ""
for datadir in datadirs:
cmdParam += " -D %s " % datadir
cmd = "%s -U %s %s -l %s" % (OMCommand.getLocalScript(
"Local_Clean_Instance"), self.user, cmdParam, self.localLog)
DefaultValue.execCommandWithMode(cmd, "clean instance", self.sshTool,
self.localMode or self.isSingle, "",
nodes)
self.logger.log("Successfully deleted instances from all nodes.")
@staticmethod
def getPrepareKeysCmd(key_file,
user,
confFile,
destPath,
logfile,
userProfile="",
localMode=False):
"""
function: get etcd communication keys command
input: key_file, user, confFile, destPath, localMode:do not scp keys
output: NA
"""
if (not os.path.exists(key_file)):
raise Exception(ErrorCode.GAUSS_502["GAUSS_50201"] % key_file)
if (not userProfile):
userProfile = DefaultValue.getMpprcFile()
# create the directory on all nodes
cmd = "source %s; %s -U %s -X %s --src-file=%s --dest-path=%s -l %s" \
% (userProfile, OMCommand.getLocalScript("Local_PrepareKeys"),
user, confFile, key_file, destPath, logfile)
# if local mode, only prepare keys, do not scp keys to cluster nodes
if (localMode):
cmd += " -L"
return cmd
def getClusterRings(self, clusterInfo):
"""
function: get clusterRings from cluster info
input: DbclusterInfo() instance
output: list
"""
hostPerNodeList = self.getDNHostnamesPerNode(clusterInfo)
# Loop the hostname list on each node where the master and slave
# of the DB instance.
for i in range(len(hostPerNodeList)):
# Loop the list after the i-th list
for perNodelist in hostPerNodeList[i + 1:len(hostPerNodeList)]:
# Define a tag
flag = 0
# Loop the elements of each perNodelist
for hostNameElement in perNodelist:
# If elements on the i-th node, each element of the
# list are joined in hostPerNodeList[i
if hostNameElement in hostPerNodeList[i]:
flag = 1
for element in perNodelist:
if element not in hostPerNodeList[i]:
hostPerNodeList[i].append(element)
if (flag == 1):
hostPerNodeList.remove(perNodelist)
return hostPerNodeList
def getDNHostnamesPerNode(self, clusterInfo):
"""
function: get DB hostnames per node
input: DbclusterInfo() instance
output: list
"""
hostPerNodeList = []
for dbNode in clusterInfo.dbNodes:
nodeDnlist = []
# loop per node
for dnInst in dbNode.datanodes:
if (dnInst.instanceType == DefaultValue.MASTER_INSTANCE):
if dnInst.hostname not in nodeDnlist:
nodeDnlist.append(dnInst.hostname)
# get other standby and dummy hostname
instances = clusterInfo.getPeerInstance(dnInst)
for inst in instances:
if inst.hostname not in nodeDnlist:
nodeDnlist.append(inst.hostname)
if nodeDnlist != []:
hostPerNodeList.append(nodeDnlist)
return hostPerNodeList
# for olap function
def checkIsElasticGroupExist(self, dbNodes):
"""
function: Check if elastic_group exists.
input : NA
output: NA
"""
self.logger.debug("Checking if elastic group exists.")
self.isElasticGroup = False
coorNode = []
# traverse old nodes
for dbNode in dbNodes:
if (len(dbNode.coordinators) >= 1):
coorNode.append(dbNode.coordinators[0])
break
# check elastic group
CHECK_GROUP_SQL = "SELECT count(*) FROM pg_catalog.pgxc_group " \
"WHERE group_name='elastic_group' " \
"and group_kind='e'; "
(checkstatus, checkoutput) = ClusterCommand.remoteSQLCommand(
CHECK_GROUP_SQL, self.user, coorNode[0].hostname, coorNode[0].port)
if (checkstatus != 0 or not checkoutput.isdigit()):
raise Exception(ErrorCode.GAUSS_502["GAUSS_50219"] % "node group" +
" Error:\n%s" % str(checkoutput))
elif (checkoutput.strip() == '1'):
self.isElasticGroup = True
elif (checkoutput.strip() == '0'):
self.isElasticGroup = False
else:
raise Exception(ErrorCode.GAUSS_502["GAUSS_50219"] %
"the number of node group")
self.logger.debug("Successfully checked if elastic group exists.")
def checkHostnameIsLoop(self, nodenameList):
"""
function: check if hostname is looped
input : NA
output: NA
"""
isRing = True
# 1.get ring information in the cluster
clusterRings = self.getClusterRings(self.clusterInfo)
nodeRing = ""
nodenameRings = []
# 2.Check if the node is in the ring
for num in iter(clusterRings):
ringNodeList = []
for nodename in nodenameList:
if (nodename in num):
ringNodeList.append(nodename)
if (len(ringNodeList) != 0 and len(ringNodeList) == len(num)):
nodenameRings.append(ringNodeList)
if (len(ringNodeList) != 0 and len(ringNodeList) != len(num)):
isRing = False
break
else:
continue
if not isRing:
raise Exception(ErrorCode.GAUSS_500["GAUSS_50004"] % "h" +
" The hostname (%s) specified by the -h parameter "
"must be looped." % nodeRing)
return (clusterRings, nodenameRings)
def getDNinstanceByNodeName(self, hostname, isMaster=True):
"""
function: Get the DB instance of the node based on the node name.
input : hostname
isMaster: get master DB instance
output: NA
"""
masterdnInsts = []
standbydnInsts = []
# notice
for dbNode in self.clusterInfo.dbNodes:
if (dbNode.name == hostname):
for dbInst in dbNode.datanodes:
# get master DB instance
if (dbInst.instanceType == DefaultValue.MASTER_INSTANCE):
masterdnInsts.append(dbInst)
# get standby or dummy DB instance
else:
standbydnInsts.append(dbInst)
if (isMaster):
return masterdnInsts
else:
return standbydnInsts
def getSQLResultList(self, sql, user, hostname, port, database="postgres"):
"""
"""
(status,
output) = ClusterCommand.remoteSQLCommand(sql, user, hostname, port,
False, database)
if status != 0 or ClusterCommand.findErrorInSql(output):
raise Exception(ErrorCode.GAUSS_514["GAUSS_51400"] % sql +
" Error:\n%s" % str(output))
# split the output string with '\n'
resultList = output.split("\n")
return resultList
def getCooInst(self):
"""
function: get CN instance
input : NA
output: CN instance
"""
coorInst = []
# get CN on nodes
for dbNode in self.clusterInfo.dbNodes:
if (len(dbNode.coordinators) >= 1):
coorInst.append(dbNode.coordinators[0])
# check if contain CN on nodes
if (len(coorInst) == 0):
raise Exception(ErrorCode.GAUSS_526["GAUSS_52602"])
else:
return coorInst
def getGroupName(self, fieldName, fieldVaule):
"""
function: Get nodegroup name by field name and field vaule.
input : field name and field vaule
output: node group name
"""
# 1.get CN instance info from cluster
cooInst = self.getCooInst()
# 2.obtain the node group
OBTAIN_SQL = "select group_name from pgxc_group where %s = %s; " % \
(fieldName, fieldVaule)
# execute the sql command
(status, output) = ClusterCommand.remoteSQLCommand(OBTAIN_SQL,
self.user,
cooInst[0].hostname,
cooInst[0].port,
ignoreError=False)
if (status != 0):
raise Exception(ErrorCode.GAUSS_514["GAUSS_51400"] % OBTAIN_SQL +
" Error:\n%s" % str(output))
return output.strip()
def killKernalSnapshotThread(self, dnInst):
"""
function: kill snapshot thread in Kernel,
avoid dead lock with redistribution)
input : NA
output: NA
"""
self.logger.debug("Stopping snapshot thread in database node Kernel.")
killSnapshotSQL = "select * from kill_snapshot();"
(status, output) = ClusterCommand.remoteSQLCommand(
killSnapshotSQL, self.user, dnInst.hostname, dnInst.port, False,
DefaultValue.DEFAULT_DB_NAME)
if (status != 0):
raise Exception(ErrorCode.GAUSS_514["GAUSS_51400"] %
killSnapshotSQL + " Error:\n%s" % str(output))
self.logger.debug("Successfully stopped snapshot "
"thread in database node Kernel.")
def createServerCa(self, hostList=None):
"""
function: create grpc ca file
input : NA
output: NA
"""
self.logger.debug("Generating CA files.")
if hostList is None:
hostList = []
appPath = DefaultValue.getInstallDir(self.user)
caPath = os.path.join(appPath, "share/sslcert/om")
self.logger.debug("The ca file dir is: %s." % caPath)
if (len(hostList) == 0):
for dbNode in self.clusterInfo.dbNodes:
hostList.append(dbNode.name)
# Create CA dir and prepare files for using.
self.logger.debug("Create CA file directory.")
try:
DefaultValue.createCADir(self.sshTool, caPath, hostList)
self.logger.debug("Add hostname to config file.")
DefaultValue.createServerCA(DefaultValue.SERVER_CA, caPath,
self.logger)
# Clean useless files, and change permission of ca file to 600.
DefaultValue.cleanServerCaDir(caPath)
self.logger.debug("Scp CA files to all nodes.")
except Exception as e:
certFile = caPath + "/demoCA/cacert.pem"
if os.path.exists(certFile):
g_file.removeFile(certFile)
DefaultValue.cleanServerCaDir(caPath)
raise Exception(str(e))
if not self.isSingle:
# localhost no need scp files
for certFile in DefaultValue.SERVER_CERT_LIST:
scpFile = os.path.join(caPath, "%s" % certFile)
self.sshTool.scpFiles(scpFile, caPath, hostList)
self.logger.debug("Successfully generated server CA files.")
def createGrpcCa(self, hostList=None):
"""
function: create grpc ca file
input : NA
output: NA
"""
self.logger.debug("Generating grpc CA files.")
if hostList is None:
hostList = []
appPath = DefaultValue.getInstallDir(self.user)
caPath = os.path.join(appPath, "share/sslcert/grpc")
self.logger.debug("The ca file dir is: %s." % caPath)
if (len(hostList) == 0):
for dbNode in self.clusterInfo.dbNodes:
hostList.append(dbNode.name)
# Create CA dir and prepare files for using.
self.logger.debug("Create CA file directory.")
try:
DefaultValue.createCADir(self.sshTool, caPath, hostList)
self.logger.debug("Add hostname to config file.")
configPath = os.path.join(appPath,
"share/sslcert/grpc/openssl.cnf")
self.logger.debug("The ca file dir is: %s." % caPath)
# Add hostname to openssl.cnf file.
DefaultValue.changeOpenSslConf(configPath, hostList)
self.logger.debug("Generate CA files.")
DefaultValue.createCA(DefaultValue.GRPC_CA, caPath)
# Clean useless files, and change permission of ca file to 600.
DefaultValue.cleanCaDir(caPath)
self.logger.debug("Scp CA files to all nodes.")
except Exception as e:
certFile = caPath + "/demoCA/cacertnew.pem"
if os.path.exists(certFile):
g_file.removeFile(certFile)
DefaultValue.cleanCaDir(caPath)
raise Exception(str(e))
for certFile in DefaultValue.GRPC_CERT_LIST:
scpFile = os.path.join(caPath, "%s" % certFile)
self.sshTool.scpFiles(scpFile, caPath, hostList)
self.logger.debug("Successfully generated grpc CA files.")
def genCipherAndRandFile(self, hostList=None, initPwd=None):
self.logger.debug("Encrypting cipher and rand files.")
if hostList is None:
hostList = []
appPath = DefaultValue.getInstallDir(self.user)
binPath = os.path.join(appPath, "bin")
retry = 0
while True:
if not initPwd:
sshpwd = getpass.getpass("Please enter password for database:")
sshpwd_check = getpass.getpass("Please repeat for database:")
else:
sshpwd = sshpwd_check = initPwd
if sshpwd_check != sshpwd:
sshpwd = ""
sshpwd_check = ""
self.logger.error(ErrorCode.GAUSS_503["GAUSS_50306"] %
"database" +
"The two passwords are different, "
"please enter password again.")
else:
cmd = "%s/gs_guc encrypt -M server -K %s -D %s " % (
binPath, sshpwd, binPath)
(status, output) = subprocess.getstatusoutput(cmd)
sshpwd = ""
sshpwd_check = ""
initPwd = ""
if status != 0:
self.logger.error(ErrorCode.GAUSS_503["GAUSS_50322"] %
"database" + "Error:\n %s" % output)
else:
break
if retry >= 2:
raise Exception(ErrorCode.GAUSS_503["GAUSS_50322"] %
"database")
retry += 1
g_file.changeMode(DefaultValue.KEY_FILE_MODE,
"'%s'/server.key.cipher" % binPath)
g_file.changeMode(DefaultValue.KEY_FILE_MODE,
"'%s'/server.key.rand" % binPath)
if len(hostList) == 0:
for dbNode in self.clusterInfo.dbNodes:
hostList.append(dbNode.name)
if not self.isSingle:
# localhost no need scp files
for certFile in DefaultValue.BIN_CERT_LIST:
scpFile = os.path.join(binPath, "%s" % certFile)
self.sshTool.scpFiles(scpFile, binPath, hostList)
self.logger.debug("Successfully encrypted cipher and rand files.")