def getDiskPath(self):
nodeDirs = []
# get PGHOST Dir
tmpDir = DefaultValue.getEnv("PGHOST")
nodeDirs.append(tmpDir)
# get gphome dir
gphome_path = DefaultValue.getEnv("GPHOME")
nodeDirs.append(gphome_path)
# get log dir
log_path = DefaultValue.getEnv("GAUSSLOG")
nodeDirs.append(log_path)
# get gausshome dir
gausshome_path = DefaultValue.getEnv("GAUSSHOME")
nodeDirs.append(os.path.realpath(gausshome_path))
hostName = DefaultValue.GetHostIpOrName()
dbNode = self.cluster.getDbNodeByName(hostName)
# including dn
for dbInst in dbNode.datanodes:
nodeDirs.append(dbInst.datadir)
return nodeDirs
def __init__(self, dropnode):
"""
"""
self.context = dropnode
self.user = self.context.user
self.userProfile = self.context.userProfile
self.group = self.context.group
self.backupFilePrimary = ''
self.localhostname = DefaultValue.GetHostIpOrName()
self.logger = self.context.logger
self.resultDictOfPrimary = []
self.replSlot = ''
envFile = DefaultValue.getEnv("MPPDB_ENV_SEPARATE_PATH")
if envFile:
self.envFile = envFile
else:
self.envFile = "/etc/profile"
gphomepath = DefaultValue.getEnv("GPHOME")
if gphomepath:
self.gphomepath = gphomepath
else:
(status, output) = subprocess.getstatusoutput("which gs_om")
if "no gs_om in" in output:
raise Exception(ErrorCode.GAUSS_518["GAUSS_51800"] % "$GPHOME")
self.gphomepath = os.path.normpath(output.replace("/gs_om", ""))
self.appPath = self.context.clusterInfo.appPath
self.gsql_path = "source %s;%s/bin/gsql" % (self.userProfile,
self.appPath)
currentTime = str(datetime.datetime.now()).replace(" ", "_").replace(
".", "_")
self.dnIdForDel = []
for hostDelName in self.context.hostMapForDel.keys():
self.dnIdForDel += self.context.hostMapForDel[hostDelName]['dn_id']
self.commonOper = OperCommon(dropnode)
def doCheck(self):
global g_chList
global g_result
resultList = []
g_result = []
nodeInfo = self.cluster.getDbNodeByName(self.host)
tmpDir = DefaultValue.getEnv("PGHOST")
logDir = DefaultValue.getEnv("GAUSSLOG")
toolDir = DefaultValue.getEnv("GPHOME")
(intervalLen, instList) = self.obtainDataDirLength(nodeInfo)
if intervalLen < len(self.cluster.appPath):
intervalLen = len(self.cluster.appPath)
if intervalLen < len(logDir):
intervalLen = len(logDir)
INDENTATION_VALUE_INT = intervalLen + 44
# Check the permissions for appPath
resultList.append(
self.checkSingleDirectoryPermission(self.cluster.appPath,
"AppPath",
INDENTATION_VALUE_INT))
g_chList.append(self.cluster.appPath)
# Check the permissions for tmpPath
resultList.append(
self.checkSingleDirectoryPermission(tmpDir, "Tmp",
INDENTATION_VALUE_INT))
# Check the permissions for logPath
g_chList.append(tmpDir)
resultList.append(
self.checkSingleDirectoryPermission(logDir, "Log",
INDENTATION_VALUE_INT))
# Check the permissions for logPath
g_chList.append(logDir)
resultList.append(
self.checkSingleDirectoryPermission(toolDir, "ToolPath",
INDENTATION_VALUE_INT))
# Check the permissions for all CMserver
g_chList.append(toolDir)
# Check the permissions for all DB instance
for inst in nodeInfo.datanodes:
resultList.append(
self.checkSingleDirectoryPermission(inst.datadir, "DN",
INDENTATION_VALUE_INT))
# Check the xlog permissions for all DB instance
xlogDir = "%s/pg_xlog" % inst.datadir
resultList.append(
self.checkSingleDirectoryPermission(xlogDir, "DN Xlog",
INDENTATION_VALUE_INT))
g_chList.append(inst.datadir)
g_chList.append(xlogDir)
if (-1 in resultList):
self.result.rst = ResultStatus.NG
else:
self.result.rst = ResultStatus.OK
self.result.val = ""
for detail in g_result:
self.result.val = self.result.val + '%s\n' % detail
def obtainDataDir(self, nodeInfo):
dataDirList = []
for inst in nodeInfo.datanodes:
dataDirList.append(inst.datadir)
dataDirList.append(DefaultValue.getEnv("PGHOST"))
dataDirList.append(DefaultValue.getEnv("GPHOME"))
dataDirList.append(DefaultValue.getEnv("GAUSSHOME"))
dataDirList.append(DefaultValue.getEnv("GAUSSLOG"))
dataDirList.append("/tmp")
return dataDirList
def ignorePath(self, path):
# Part of the root path and file permissions need to be ignored
ignorePathList = []
toolPath = DefaultValue.getEnv("GPHOME")
sudoPath = os.path.join(toolPath, "sudo")
inspectionPath = os.path.join(toolPath, "script/inspection")
ignorePathList.append("%s/script/gs_preinstall" % toolPath)
ignorePathList.append("%s/script/gs_postuninstall" % toolPath)
ignorePathList.append("%s/script/gs_checkos" % toolPath)
scriptPath = os.path.join(toolPath, "script")
scriptDirList = scriptPath.split('/')
inspectionDirList = inspectionPath.split('/')
# ignore own special files
if (path in ignorePathList or os.path.dirname(path) == sudoPath):
return True
else:
(filename, suffix) = os.path.splitext(path)
pathDirList = path.split('/')
# ignore .pyc file in GPHOME/script
if (path.find(scriptPath) == 0
and pathDirList[:len(scriptDirList)] == scriptDirList
and suffix == ".pyc"):
return True
# ignore GPHOME/script/inspection dir
elif (path.find(inspectionPath) == 0 and
pathDirList[:len(inspectionDirList)] == inspectionDirList):
return True
else:
return False
def checkParameter():
"""
function: check parameter
"""
if (g_opts.clusterUser == ""):
GaussLog.exitWithError(ErrorCode.GAUSS_500["GAUSS_50001"] % 'U' + ".")
if g_opts.ignorepgHbaMiss:
gaussHome = DefaultValue.getEnv("GAUSSHOME")
if not gaussHome:
GaussLog.exitWithError(ErrorCode.GAUSS_518["GAUSS_51802"] %
"GAUSSHOME")
staticConfigfile = "%s/bin/cluster_static_config" % gaussHome
if (not os.path.isfile(staticConfigfile)):
GaussLog.exitWithError(ErrorCode.GAUSS_502["GAUSS_50210"] %
staticConfigfile)
if (g_opts.clusterConf != ""):
if (not os.path.exists(g_opts.clusterConf)):
GaussLog.exitWithError(ErrorCode.GAUSS_502["GAUSS_50201"] %
g_opts.clusterConf)
if (g_opts.logFile == ""):
g_opts.logFile = DefaultValue.getOMLogPath(DefaultValue.LOCAL_LOG_FILE,
g_opts.clusterUser, "")
def __generateXml(self, backIp):
"""
"""
nodeName = self.context.backIpNameMap[backIp]
nodeInfo = self.context.clusterInfoDict[nodeName]
backIp = nodeInfo["backIp"]
sshIp = nodeInfo["sshIp"]
port = nodeInfo["port"]
dataNode = nodeInfo["dataNode"]
appPath = self.context.clusterInfoDict["appPath"]
logPath = self.context.clusterInfoDict["logPath"]
corePath = self.context.clusterInfoDict["corePath"]
toolPath = self.context.clusterInfoDict["toolPath"]
mppdbconfig = ""
tmpMppdbPath = DefaultValue.getEnv("PGHOST")
if tmpMppdbPath:
mppdbconfig = '<PARAM name="tmpMppdbPath" value="%s" />' % tmpMppdbPath
xmlConfig = """\
<?xml version="1.0" encoding="UTF-8"?>
<ROOT>
<CLUSTER>
<PARAM name="clusterName" value="dbCluster" />
<PARAM name="nodeNames" value="{nodeName}" />
<PARAM name="backIp1s" value="{backIp}"/>
<PARAM name="gaussdbAppPath" value="{appPath}" />
<PARAM name="gaussdbLogPath" value="{logPath}" />
<PARAM name="gaussdbToolPath" value="{toolPath}" />
{mappdbConfig}
<PARAM name="corePath" value="{corePath}"/>
<PARAM name="clusterType" value="single-inst"/>
</CLUSTER>
<DEVICELIST>
<DEVICE sn="1000001">
<PARAM name="name" value="{nodeName}"/>
<PARAM name="azName" value="{azName}"/>
<PARAM name="azPriority" value="1"/>
<PARAM name="backIp1" value="{backIp}"/>
<PARAM name="sshIp1" value="{sshIp}"/>
<!--dbnode-->
<PARAM name="dataNum" value="1"/>
<PARAM name="dataPortBase" value="{port}"/>
<PARAM name="dataNode1" value="{dataNode}"/>
</DEVICE>
</DEVICELIST>
</ROOT>
""".format(nodeName=nodeName,
backIp=backIp,
appPath=appPath,
logPath=logPath,
toolPath=toolPath,
corePath=corePath,
sshIp=sshIp,
port=port,
dataNode=dataNode,
azName=self.context.azName,
mappdbConfig=mppdbconfig)
return xmlConfig
def checkPreEnv(self):
"""
function: Check if LD path and path in preinstall had been changed.
input : NA
output: NA
"""
g_opts.logger.log("Checking preinstall enviroment value.")
# Check $GAUSS_ENV.
try:
# get mpp file by env parameter MPPDB_ENV_SEPARATE_PATH
mpprcFile = DefaultValue.getEnv(DefaultValue.MPPRC_FILE_ENV)
if (mpprcFile != "" and mpprcFile is not None):
userProfile = mpprcFile
if (not os.path.isabs(userProfile)):
raise Exception(ErrorCode.GAUSS_512["GAUSS_51206"] %
userProfile)
if (not os.path.exists(userProfile)):
raise Exception(ErrorCode.GAUSS_502["GAUSS_50201"] %
userProfile)
else:
userpath = pwd.getpwnam(self.user).pw_dir
userProfile = os.path.join(userpath, ".bashrc")
reEnvList = g_file.readFile(userProfile)
checkList = [
"export PATH=$GPHOME/script/gspylib/pssh/bin:$GPHOME/script"
":$PATH", "export LD_LIBRARY_PATH=$GPHOME/lib:$LD_LIBRARY_PATH"
]
for check in checkList:
if (check not in reEnvList
and (check + '\n') not in reEnvList):
self.logger.logExit(ErrorCode.GAUSS_518["GAUSS_51802"] %
check)
except Exception as e:
g_opts.logger.logExit(str(e))
g_opts.logger.log("Successfully checked preinstall enviroment value.")
def parseResult(self):
"""
function : get resCount and resSet from result
input:NA
output:NA
"""
try:
libpath = os.path.join(DefaultValue.getEnv("GAUSSHOME"), "lib")
sys.path.append(libpath)
libc = cdll.LoadLibrary("libpq.so.5.5")
libc.PQntuples.argtypes = [c_void_p]
libc.PQntuples.restype = c_int
libc.PQnfields.argtypes = [c_void_p]
libc.PQnfields.restype = c_int
libc.PQgetvalue.restype = c_char_p
ntups = libc.PQntuples(self.result)
nfields = libc.PQnfields(self.result)
libc.PQgetvalue.argtypes = [c_void_p, c_int, c_int]
self.resCount = ntups
for i in range(ntups):
tmpString = []
for j in range(nfields):
paramValue = libc.PQgetvalue(self.result, i, j)
if (paramValue is not None):
tmpString.append(string_at(paramValue).decode())
else:
tmpString.append("")
self.resSet.append(tmpString)
except Exception as e:
raise Exception("%s" % str(e))
def obtainDataDir(self, nodeInfo):
dataDirList = {}
dataDirList[DefaultValue.getEnv("PGHOST")] = [
"PGHOST", self.diskVailPGHOST
]
dataDirList[DefaultValue.getEnv("GPHOME")] = [
"GPHOME", self.diskVailGPHOME
]
dataDirList[DefaultValue.getEnv("GAUSSHOME")] = \
["GAUSSHOME", self.diskVailGAUSSHOME]
dataDirList[DefaultValue.getEnv("GAUSSLOG")] = [
"GAUSSLOG", self.diskVailGAUSSLOG
]
dataDirList["/tmp"] = ["OS_TMP", self.diskVailOS_TMP]
for inst in nodeInfo.datanodes:
dataDirList[inst.datadir] = ["DN", self.diskVailDATA]
return dataDirList
def doCheck(self):
flag = True
parRes = ""
# Determine if it is an ELK environment
elk_env = DefaultValue.getEnv("ELK_SYSTEM_TABLESPACE")
if (elk_env):
expand_value = 640000
else:
expand_value = 1000000
# Check system open files parameter
output = g_OSlib.getUserLimits('open files')
self.result.raw = output
if (output != ""):
self.result.val += output + "\n"
resList = output.split(' ')
limitValue = resList[-1].strip()
# Unlimited check is passed
if limitValue == 'unlimited':
pass
# Open file parameter value is less than 640000 will not pass
if int(limitValue) < int(expand_value):
flag = False
else:
pass
# Write check results
parRes += "Max open files: %s\n" % limitValue
else:
#
flag = False
parRes += "Failed to get system open files parameter.\n"
# Check cluster process open files parameter
if (self.cluster):
pidList = g_OSlib.getProcess(
os.path.join(self.cluster.appPath, 'bin/gaussdb'))
for pid in pidList:
if (not os.path.isfile("/proc/%s/limits" % pid)
or not os.access("/proc/%s/limits" % pid, os.R_OK)):
continue
openFileInfo = \
g_file.readFile('/proc/%s/limits' % pid, 'Max open files')[
0]
if (openFileInfo):
value = openFileInfo.split()[3]
if (int(value.strip()) < expand_value):
flag = False
parRes += "The value of " \
"max open files is %s on pid %s. " \
"it must not be less than %d.\n" % (
value.strip(), pid, expand_value)
if (flag):
self.result.rst = ResultStatus.OK
else:
self.result.rst = ResultStatus.NG
self.result.val = parRes
def doCheck(self):
flag = "Normal"
path = DefaultValue.getEnv(
"PGHOST",
os.path.join(DefaultValue.getEnv("GPHOME"),
"%s_mppdb" % self.user))
# Check space usage
rateNum = g_disk.getDiskSpaceUsage(path)
self.result.raw += "[%s] space usage: %s%%\n" % (path, rateNum)
if (rateNum > int(self.thresholdDn)):
self.result.val += \
"Path(%s) space usage(%d%%) Abnormal reason: " \
"The usage of the device disk space cannot be " \
"greater than %s%%.\n" % (
path, rateNum, self.thresholdDn)
flag = "Error"
# Check inode usage
diskName = g_disk.getMountPathByDataDir(path)
diskType = g_disk.getDiskMountType(diskName)
if (not diskType in ["xfs", "ext3", "ext4"]):
self.result.val = \
"Path(%s) inodes usage(%s) Warning reason: " \
"The file system type [%s] is unrecognized or not support. " \
"Please check it.\n" % (
path, 0, diskType)
self.result.raw = "[%s] disk type: %s\n" % (path, diskType)
self.result.rst = ResultStatus.WARNING
return
rateNum = g_disk.getDiskInodeUsage(path)
self.result.raw += "[%s] inode usage: %s%%\n" % (path, rateNum)
if (rateNum > int(self.thresholdDn)):
self.result.val += \
"Path(%s) inode usage(%d%%) Abnormal reason: " \
"The usage of the device disk inode cannot be " \
"greater than %s%%.\n" % (
path, rateNum, self.thresholdDn)
flag = "Error"
if (flag == "Normal"):
self.result.rst = ResultStatus.OK
self.result.val = "Tmp disk space are sufficient.\n"
else:
self.result.rst = ResultStatus.NG
def obtainDataDir(self, nodeInfo):
dataDirList = []
for inst in nodeInfo.datanodes:
dataDirList.append(inst.datadir)
for inst in nodeInfo.cmservers:
dataDirList.append(inst.datadir)
for inst in nodeInfo.coordinators:
dataDirList.append(inst.datadir)
for inst in nodeInfo.gtms:
dataDirList.append(inst.datadir)
if (hasattr(nodeInfo, 'etcds')):
for inst in nodeInfo.etcds:
dataDirList.append(inst.datadir)
dataDirList.append(DefaultValue.getEnv("PGHOST"))
dataDirList.append(DefaultValue.getEnv("GPHOME"))
dataDirList.append(DefaultValue.getEnv("GAUSSHOME"))
dataDirList.append(DefaultValue.getEnv("GAUSSLOG"))
dataDirList.append("/tmp")
return dataDirList
def check_cluster_version_consistency(self, clusterNodes, newNodes=None):
"""
"""
self.logger.log("Check cluster version consistency.")
if newNodes is None:
newNodes = []
dic_version_info = {}
# check version.cfg on every node.
gp_home = DefaultValue.getEnv("GPHOME")
gauss_home = DefaultValue.getEnv("GAUSSHOME")
if not (os.path.exists(gp_home) and os.path.exists(gauss_home)):
GaussLog.exitWithError(ErrorCode.GAUSS_502["GAUSS_50201"] %
("%s", "or %s") % (gp_home, gauss_home))
for ip in clusterNodes:
if ip in newNodes:
cmd = "pssh -s -H %s 'cat %s/version.cfg'" % \
(ip, DefaultValue.getEnv("GPHOME"))
else:
cmd = "pssh -s -H %s 'cat %s/bin/upgrade_version'" % \
(ip, DefaultValue.getEnv("GAUSSHOME"))
status, output = subprocess.getstatusoutput(cmd)
if (status != 0):
raise Exception(ErrorCode.GAUSS_514["GAUSS_51400"] % cmd +
" Error:\n%s" % str(output))
if len(output.strip().split()) < 3:
raise Exception(ErrorCode.GAUSS_516["GAUSS_51623"])
dic_version_info[ip] = ",".join(output.strip().split()[1:])
self.logger.debug("The cluster version on every node.")
for check_ip, version_info in dic_version_info.items():
self.logger.debug("%s : %s" % (check_ip, version_info))
if len(set(dic_version_info.values())) != 1:
L_inconsistent = list(set(dic_version_info.values()))
self.logger.debug("The package version on some nodes are "
"inconsistent\n%s" % str(L_inconsistent))
raise Exception("The package version on some nodes are "
"inconsistent,%s" % str(L_inconsistent))
self.logger.log("Successfully checked cluster version.")
def getUserOSProfile(self, env_file=""):
"""
function: get user os profile
input : env_file
output: mpprcFile, userProfile, osProfile
"""
if env_file != "":
mpprcFile = env_file
else:
mpprcFile = DefaultValue.getEnv(DefaultValue.MPPRC_FILE_ENV)
if mpprcFile != "" and mpprcFile is not None:
userProfile = mpprcFile
else:
userProfile = "~/.bashrc"
osProfile = "/etc/profile"
return mpprcFile, userProfile, osProfile
def initGlobals(self):
"""
init global variables
input : NA
output: NA
"""
global g_nodeInfo
self.logger = GaussLog(self.logFile, self.action)
if self.clusterConfig != "":
if os.path.isfile(self.clusterConfig):
self.clusterToolPath = DefaultValue.getPreClusterToolPath(
self.user, self.clusterConfig)
self.readConfigInfoByXML()
hostName = DefaultValue.GetHostIpOrName()
g_nodeInfo = self.clusterInfo.getDbNodeByName(hostName)
if (g_nodeInfo is None):
self.logger.logExit(
ErrorCode.GAUSS_516["GAUSS_51620"] % "local" +
" There is no host named %s!" % hostName)
else:
self.logger.logExit(ErrorCode.GAUSS_502["GAUSS_50210"] %
("config file [%s]" % self.clusterConfig))
elif self.action != ACTION_CLEAN_DEPENDENCY:
try:
self.clusterToolPath = DefaultValue.getClusterToolPath(
self.user)
except Exception as e:
self.logger.logExit(ErrorCode.GAUSS_502["GAUSS_50219"] %
"the cluster tool path" +
" Error: \n%s" % str(e))
if not self.clusterToolPath:
self.logger.logExit(ErrorCode.GAUSS_502["GAUSS_50219"] %
"cluster tool path")
# make sure if we are using env seperate version,
# and get the right profile
# we can not check mppenvfile exists here
mppenvFile = DefaultValue.getEnv(DefaultValue.MPPRC_FILE_ENV)
if (mppenvFile != "" and mppenvFile is not None):
self.userProfile = mppenvFile
else:
self.userProfile = "/home/%s/.bashrc" % self.user
def changeObsLogSetting(self):
"""
function: change the obs log setting file distribute package
input : NA
output: NA
"""
obspathNum = self.clusterInfo.appPath.count("/")
"""
obs path is the relative path between log path and app path.
if app path is /test/app and log path is /test/log
then the relative path from app to log is '..'+'/..'*(num-1)+logpath
the relative path from obs to log is '../../..'+'/..'*(num-1)+logpath
"""
username = DefaultValue.getEnv("LOGNAME")
DefaultValue.checkPathVaild(username)
obspath = "LogPath=../.." + "/.." * obspathNum + "%s/" \
% self.clusterInfo.logPath + "%s" % username + "/bin/gs_obs"
cmd = "mkdir -p '%s/%s/bin/gs_obs' -m %s" \
% (self.clusterInfo.logPath, username,
DefaultValue.KEY_DIRECTORY_MODE)
(status, output) = subprocess.getstatusoutput(cmd)
if (status != 0):
self.logger.debug("The cmd is %s " % cmd)
raise Exception(ErrorCode.GAUSS_502["GAUSS_50208"] % "obs log" +
" Error: \n%s " % output)
obsLogName = "gs_obs"
obsinifile = "%s/lib/OBS.ini" % self.clusterInfo.appPath
if not os.path.exists(obsinifile):
self.logger.logExit(ErrorCode.GAUSS_502["GAUSS_50201"] %
obsinifile)
try:
with open(obsinifile, 'r') as fp:
lines = fp.readlines()
flen = len(lines) - 1
for i in range(flen):
if "sdkname=eSDK-OBS-API-Linux-C" in lines[i]:
lines[i] = lines[i].replace("sdkname=eSDK-OBS-API-Linux-C",
"sdkname=gs_obs")
if "LogPath=../logs" in lines[i]:
lines[i] = lines[i].replace("LogPath=../logs", obspath)
with open(obsinifile, 'w') as fpw:
fpw.writelines(lines)
except Exception as e:
self.logger.logExit(str(e))
def CleanRackFile(self):
"""
function: clean rack information file
input : NA
output: NA
"""
gp_home = DefaultValue.getEnv("GPHOME")
if os.path.exists(gp_home):
gp_home = os.path.realpath(gp_home)
rack_conf_file = os.path.realpath(
os.path.join(gp_home, "script/gspylib/etc/conf/rack_info.conf"))
if os.path.isfile(rack_conf_file):
cmd = "rm -f %s" % rack_conf_file
DefaultValue.execCommandWithMode(cmd,
"Deleted rack information file.",
self.sshTool, self.localMode,
mpprcFile=self.mpprcFile)
self.logger.debug("Successfully deleted rack information file.")
def __init__(self, expansion):
"""
"""
self.context = expansion
self.user = self.context.user
self.group = self.context.group
self.logger = self.context.logger
self.envFile = DefaultValue.getEnv("MPPDB_ENV_SEPARATE_PATH")
currentTime = str(datetime.datetime.now()).replace(" ", "_").replace(
".", "_")
self.commonGsCtl = GsCtlCommon(expansion)
self.tempFileDir = "/tmp/gs_expansion_%s" % (currentTime)
self.logger.debug("tmp expansion dir is %s ." % self.tempFileDir)
def doSet(self):
self.result.val = ""
self.result.raw = ""
limitPath = '/etc/security/limits.d/'
if (os.path.isfile(os.path.join(limitPath, '91-nofile.conf'))):
limitFile = '91-nofile.conf'
else:
limitFile = '90-nofile.conf'
elk_env = DefaultValue.getEnv("ELK_SYSTEM_TABLESPACE")
if (elk_env):
expand_value = 640000
else:
expand_value = 1000000
errMsg = SharedFuncs.SetLimitsConf(["soft", "hard"], "nofile",
expand_value,
os.path.join(limitPath, limitFile))
if errMsg != "Success":
self.result.val = "%s\n" % errMsg
else:
self.result.val = "Success to set openfile to %d\n" % expand_value
def __init__(self, expansion):
"""
"""
self.context = expansion
self.user = self.context.user
self.group = self.context.group
self.logger = self.context.logger
envFile = DefaultValue.getEnv("MPPDB_ENV_SEPARATE_PATH")
if envFile:
self.envFile = envFile
else:
self.envFile = "/etc/profile"
currentTime = str(datetime.datetime.now()).replace(" ", "_").replace(
".", "_")
self.commonGsCtl = GsCtlCommon(expansion)
self.tempFileDir = "/tmp/gs_expansion_%s" % (currentTime)
self.logger.debug("tmp expansion dir is %s ." % self.tempFileDir)
self._finalizer = weakref.finalize(self, self.clearTmpFile)
def main():
"""
function: main function
input : NA
output: NA
"""
try:
opts, args = getopt.getopt(sys.argv[1:], "U:P:l:pbhifs:", [
"position=", "parameter", "binary_file", "logpath=", "help",
"ingore_miss", "force", "static_file="
])
except getopt.GetoptError as e:
GaussLog.exitWithError(ErrorCode.GAUSS_500["GAUSS_50000"] % e.msg)
if (len(args) > 0):
GaussLog.exitWithError(ErrorCode.GAUSS_500["GAUSS_50000"] %
str(args[0]))
global g_clusterUser
global g_ignoreMiss
global g_staticFile
global g_forceRestore
restoreDir = ""
restorePara = False
restoreBin = False
logFile = ""
for key, value in opts:
if (key == "-h" or key == "--help"):
usage()
sys.exit(0)
elif (key == "-U"):
g_clusterUser = value.strip()
elif (key == "-P" or key == "--position"):
restoreDir = value.strip()
elif (key == "-p" or key == "--parameter"):
restorePara = True
elif (key == "-b" or key == "--binary_file"):
restoreBin = True
elif (key == "-i" or key == "--ingore_miss"):
g_ignoreMiss = True
elif (key == "-s" or key == "--static_file"):
g_staticFile = value.strip()
elif (key == "-l" or key == "--logpath"):
logFile = value
elif (key == "-f" or key == "--force"):
g_forceRestore = True
else:
GaussLog.exitWithError(ErrorCode.GAUSS_500["GAUSS_50000"] % value)
Parameter.checkParaVaild(key, value)
if (g_ignoreMiss):
gaussHome = DefaultValue.getEnv("GAUSSHOME")
if not gaussHome:
return
# check if user exist and is the right user
checkUserExist()
# check log file
checkLogFile(logFile)
# check -p and -b
checkRestorePara(restorePara, restoreBin)
# check -P
checkRestoreDir(restoreDir)
try:
LocalRestorer = LocalRestore(logFile, g_clusterUser, restoreDir,
restorePara, restoreBin)
LocalRestorer.run()
except Exception as e:
GaussLog.exitWithError(str(e))
def doCheck(self):
g_envProfileDist["GAUSSHOME"] = DefaultValue.getEnv("GAUSSHOME")
g_envProfileDist["PATH"] = DefaultValue.getEnv("PATH")
g_envProfileDist["LD_LIBRARY_PATH"] = DefaultValue.getEnv(
"LD_LIBRARY_PATH")
self.result.val = ""
ProcessList = []
ProcessDisk = {}
abnormal_flag = False
if (g_envProfileDist["GAUSSHOME"] == ""):
abnormal_flag = True
self.result.val += "The environmental variable " \
"GAUSSHOME is empty.\n"
else:
self.result.val += "GAUSSHOME %s\n" % g_envProfileDist[
"GAUSSHOME"]
libPath = "%s/lib" % g_envProfileDist["GAUSSHOME"]
if (libPath not in g_envProfileDist["LD_LIBRARY_PATH"].split(':')):
abnormal_flag = True
self.result.val += \
VersionInfo.PRODUCT_NAME + \
" lib path does not exist in LD_LIBRARY_PATH.\n"
else:
self.result.val += "LD_LIBRARY_PATH %s\n" % libPath
binPath = "%s/bin" % g_envProfileDist["GAUSSHOME"]
# Whether the environment variable bin is in path
if (binPath not in g_envProfileDist["PATH"].split(':')):
abnormal_flag = True
self.result.val += VersionInfo.PRODUCT_NAME + \
" bin path does not exist in PATH.\n"
else:
self.result.val += "PATH %s\n" % binPath
if abnormal_flag:
self.result.rst = ResultStatus.NG
return
# Gets the current node information
nodeInfo = self.cluster.getDbNodeByName(self.host)
# check the number of instances
if len(nodeInfo.datanodes) > 0:
ProcessList.append("gaussdb")
# Query process
for Process in ProcessList:
cmd = "ps ux | grep '%s/bin/%s' | grep -v 'grep' |" \
" awk '{print $2}'" % (self.cluster.appPath, Process)
output = SharedFuncs.runShellCmd(cmd, self.user, self.mpprcFile)
if (output != ""):
if (len(output.split('\n')) > 1):
for ProcessNum in output.split('\n'):
ProcessDisk[ProcessNum] = [Process]
else:
ProcessDisk[output] = [Process]
else:
self.result.val += "The process %s is not exist.\n" % Process
abnormal_flag = True
for ProcessNum in ProcessDisk.keys():
# Get the process environment variables
result = self.getProcessEnv(ProcessNum, ProcessDisk[ProcessNum])
if not abnormal_flag:
abnormal_flag = result
if abnormal_flag:
self.result.rst = ResultStatus.NG
else:
self.result.rst = ResultStatus.OK
def getLocalScript(script):
"""
function: get local script by GPHOME
input : script, path
output: path
"""
Current_Path = os.path.dirname(os.path.realpath(__file__))
if os.getgid() != 0:
gp_home = DefaultValue.getEnv("GPHOME")
Current_Path = os.path.join(gp_home, "script/gspylib/common")
LocalScript = {
"Local_Backup":
os.path.normpath(Current_Path + "/../../local/Backup.py"),
"Local_Check_Config":
os.path.normpath(Current_Path + "/../../local/CheckConfig.py"),
"Local_Check_Install":
os.path.normpath(Current_Path + "/../../local/CheckInstall.py"),
"Local_Check_Uninstall":
os.path.normpath(Current_Path + "/../../local/CheckUninstall.py"),
"Local_Clean_Instance":
os.path.normpath(Current_Path + "/../../local/CleanInstance.py"),
"Local_Clean_OsUser":
os.path.normpath(Current_Path + "/../../local/CleanOsUser.py"),
"Local_Config_Hba":
os.path.normpath(Current_Path + "/../../local/ConfigHba.py"),
"Local_Config_Instance":
os.path.normpath(Current_Path + "/../../local/ConfigInstance.py"),
"Local_Init_Instance":
os.path.normpath(Current_Path + "/../../local/InitInstance.py"),
"Local_Install":
os.path.normpath(Current_Path + "/../../local/Install.py"),
"Local_Restore":
os.path.normpath(Current_Path + "/../../local/Restore.py"),
"Local_Uninstall":
os.path.normpath(Current_Path + "/../../local/Uninstall.py"),
"Local_PreInstall":
os.path.normpath(Current_Path +
"/../../local/PreInstallUtility.py"),
"Local_Check_PreInstall":
os.path.normpath(Current_Path + "/../../local/CheckPreInstall.py"),
"Local_UnPreInstall":
os.path.normpath(Current_Path +
"/../../local/UnPreInstallUtility.py"),
"Local_Roach":
os.path.normpath(Current_Path + "/../../local/LocalRoach.py"),
"Gauss_UnInstall":
os.path.normpath(Current_Path + "/../../gs_uninstall"),
"Gauss_Backup":
os.path.normpath(Current_Path + "/../../gs_backup"),
"Local_CheckOS":
os.path.normpath(Current_Path + "/../../local/LocalCheckOS.py"),
"Local_Check":
os.path.normpath(Current_Path + "/../../local/LocalCheck.py"),
"LOCAL_PERFORMANCE_CHECK":
os.path.normpath(Current_Path +
"/../../local/LocalPerformanceCheck.py"),
"Gauss_CheckOS":
os.path.normpath(Current_Path + "/../../gs_checkos"),
"Gauss_PreInstall":
os.path.normpath(Current_Path + "/../../gs_preinstall"),
"Gauss_Replace":
os.path.normpath(Current_Path + "/../../gs_replace"),
"Gauss_Om":
os.path.normpath(Current_Path + "/../../gs_om"),
"UTIL_GAUSS_STAT":
os.path.normpath(Current_Path +
"/../../gspylib/common/GaussStat.py"),
"Gauss_Check":
os.path.normpath(Current_Path + "/../../gs_check"),
"Local_Collect":
os.path.normpath(Current_Path + "/../../local/LocalCollect.py"),
"Local_Kerberos":
os.path.normpath(Current_Path + "/../../local/KerberosUtility.py"),
"Local_Execute_Sql":
os.path.normpath(Current_Path + "/../../local/ExecuteSql.py"),
"Local_StartInstance":
os.path.normpath(Current_Path + "/../../local/StartInstance.py"),
"Local_StopInstance":
os.path.normpath(Current_Path + "/../../local/StopInstance.py"),
"Local_Check_Upgrade":
os.path.normpath(Current_Path + "/../../local/CheckUpgrade.py"),
"Local_Upgrade_Utility":
os.path.normpath(Current_Path + "/../../local/UpgradeUtility.py")
}
return "python3 '%s'" % LocalScript[script]
def doCheck(self):
self.result.val = ""
nodeInfo = self.cluster.getDbNodeByName(self.host)
clusterPathList = self.getClusterDirectorys(nodeInfo)
clusterPathList.append(self.cluster.appPath)
clusterPathList.append(self.cluster.logPath)
clusterPathList.append(DefaultValue.getEnv('GPHOME'))
clusterPathList.append(DefaultValue.getEnv('PGHOST'))
nodeInfo = self.cluster.getDbNodeByName(self.host)
if self.cluster.isSingleInstCluster():
dirPath = nodeInfo.datanodes[0].datadir
else:
dirPath = nodeInfo.coordinators[0].datadir
tableSpaceDir = os.path.join(dirPath, "pg_tblspc")
tableSpaceList = os.listdir(tableSpaceDir)
tablespacePaths = []
if (len(tableSpaceList)):
for filename in tableSpaceList:
if (os.path.islink(os.path.join(tableSpaceDir, filename))):
linkDir = os.readlink(os.path.join(tableSpaceDir,
filename))
if (os.path.isdir(linkDir)):
tablespacePaths.append(linkDir)
flag = "Normal"
for tableSpace in tablespacePaths:
if (tableSpace.find(' ') >= 0):
flag = "Error"
self.result.val += "Table space path[%s] contains spaces.\n" \
% tableSpace
# Support create tablespace in pg_location dir for V1R7
if (tableSpace.find(os.path.join(dirPath, "pg_location")) == 0):
continue
tableSpaces = tableSpace.split('/')
for clusterPath in clusterPathList:
clusterPaths = clusterPath.split('/')
if (tableSpace.find(clusterPath) == 0
and tableSpaces[:len(clusterPaths)] == clusterPaths):
if (flag == "Normal"):
flag = "Warning"
self.result.val += "Table space path[%s] and cluster " \
"path[%s] are nested.\n" % (
tableSpace, clusterPath)
elif (clusterPath.find(tableSpace) == 0
and clusterPaths[:len(tableSpaces)] == tableSpaces):
flag = "Error"
self.result.val += "Table space path[%s] and cluster " \
"path[%s] are nested.\n" % (tableSpace,
clusterPath)
else:
continue
for tableSpace1 in tablespacePaths:
tableSpaces1 = tableSpace1.split('/')
for tableSpace2 in tablespacePaths:
if (tableSpace1 == tableSpace2):
continue
tableSpaces2 = tableSpace2.split('/')
if (tableSpace1.find(tableSpace2) == 0
and tableSpaces1[:len(tableSpaces2)] == tableSpaces2):
flag = "Error"
self.result.val += "Table space path[%s] and table space" \
" path[%s] are nested.\n" \
% (tableSpace1, tableSpace2)
if (flag == "Error"):
self.result.rst = ResultStatus.NG
elif (flag == "Warning"):
self.result.rst = ResultStatus.WARNING
else:
self.result.rst = ResultStatus.OK
self.result.val = "All table space path is normal."
def scpFiles(self,
srcFile,
targetDir,
hostList=None,
env_file="",
gp_path="",
parallel_num=300):
"""
function: copy files to other path
input : srcFile, targetDir, hostList, env_file, gp_path, parallel_num
output: NA
"""
scpCmd = "source /etc/profile"
outputCollect = ""
if hostList is None:
hostList = []
try:
if env_file != "":
mpprcFile = env_file
else:
mpprcFile = DefaultValue.getEnv(DefaultValue.MPPRC_FILE_ENV)
if mpprcFile != "" and mpprcFile is not None:
scpCmd += " && source %s" % mpprcFile
if gp_path == "":
cmdpre = "%s && echo $GPHOME" % scpCmd
(status, output) = subprocess.getstatusoutput(cmdpre)
if status != 0 or not output or output.strip() == "":
raise Exception(ErrorCode.GAUSS_518["GAUSS_51802"] %
"GPHOME" + "The cmd is %s" % cmdpre)
GPHOME = output.strip()
else:
GPHOME = gp_path.strip()
pscppre = "python3 %s/script/gspylib/pssh/bin/pscp" % GPHOME
if len(hostList) == 0:
scpCmd += " && %s -r -v -t %s -p %s -h %s -o %s -e %s %s %s" \
" 2>&1 | tee %s" % (pscppre, self.__timeout,
parallel_num, self.__hostsFile,
self.__outputPath,
self.__errorPath, srcFile,
targetDir, self.__resultFile)
hostList = self.hostNames
else:
scpCmd += " && %s -r -v -t %s -p %s -H %s -o %s -e %s %s %s" \
" 2>&1 | tee %s" % (pscppre, self.__timeout,
parallel_num,
" -H ".join(hostList),
self.__outputPath,
self.__errorPath, srcFile,
targetDir, self.__resultFile)
(status, output) = subprocess.getstatusoutput(scpCmd)
if status != 0:
raise Exception(ErrorCode.GAUSS_502["GAUSS_50216"] %
("file [%s]" % srcFile) +
" To directory: %s." % targetDir +
" Error:\n%s" % output)
if output.find("Timed out") > 0:
raise Exception(ErrorCode.GAUSS_514["GAUSS_51400"] % scpCmd +
" Error:\n%s" % output)
# ip and host name should match here
resultMap, outputCollect = self.parseSshResult(hostList)
except Exception as e:
self.clenSshResultFiles()
raise Exception(str(e))
for host in hostList:
if resultMap.get(host) != DefaultValue.SUCCESS:
raise Exception(ErrorCode.GAUSS_502["GAUSS_50216"] %
("file [%s]" % srcFile) +
" To directory: %s." % targetDir +
" Command: %s.\nError:\n%s" %
(scpCmd, outputCollect))
def main():
"""
function: main function
1.parse command line
2.check if user exist and is the right user
3.check log file
4.check backupPara and backupBin
5.check tmpBackupDir
6.do backup
input : NA
output: NA
"""
try:
opts, args = getopt.getopt(sys.argv[1:], "U:P:B:l:pbhi",
["position=", "backupdir=", \
"nodeName=", "parameter", "binary_file",
"logpath=", "help", "ingore_miss"])
except getopt.GetoptError as e:
GaussLog.exitWithError(ErrorCode.GAUSS_500["GAUSS_50000"] % str(e))
if (len(args) > 0):
GaussLog.exitWithError(
ErrorCode.GAUSS_500["GAUSS_50000"] % str(args[0]))
global g_clusterUser
global g_ignoreMiss
tmpBackupDir = ""
backupDir = ""
backupPara = False
backupBin = False
logFile = ""
nodeName = ""
for key, value in opts:
if (key == "-h" or key == "--help"):
usage()
sys.exit(0)
elif (key == "-U"):
g_clusterUser = value.strip()
elif (key == "-P" or key == "--position"):
tmpBackupDir = value.strip()
elif (key == "-B" or key == "--backupdir"):
backupDir = value.strip()
elif (key == "-p" or key == "--parameter"):
backupPara = True
elif (key == "-b" or key == "--binary_file"):
backupBin = True
elif (key == "-i" or key == "--ingore_miss"):
g_ignoreMiss = True
elif (key == "-l" or key == "--logpath"):
logFile = value.strip()
elif (key == "--nodeName"):
nodeName = value.strip()
else:
GaussLog.exitWithError(ErrorCode.GAUSS_500["GAUSS_50000"] % value)
Parameter.checkParaVaild(key, value)
if (g_ignoreMiss):
gaussHome = DefaultValue.getEnv("GAUSSHOME")
if not gaussHome:
return
# check if user exist and is the right user
checkUserParameter()
DefaultValue.checkUser(g_clusterUser, False)
# check log file
checkLogFile(logFile)
# check backupPara and backupBin
checkBackupPara(backupPara, backupBin)
# check tmpBackupDir
checkTmpBackupDir(tmpBackupDir)
try:
LocalBackuper = LocalBackup(logFile, g_clusterUser, tmpBackupDir,
backupDir, backupPara, backupBin, nodeName)
LocalBackuper.run()
except Exception as e:
GaussLog.exitWithError(str(e))
# ----------------------------------------------------------------------------
# Description : ExecuteSql.py is a utility to execute sql by using libpq.
#############################################################################
import getopt
import sys
import os
import json
import subprocess
sys.path.append(sys.path[0] + "/../")
from gspylib.common.GaussLog import GaussLog
from gspylib.common.ParameterParsecheck import Parameter
from gspylib.common.Common import DefaultValue, ClusterCommand
libpath = os.path.join(DefaultValue.getEnv("GAUSSHOME"), "lib")
sys.path.append(libpath)
from gspylib.common.ErrorCode import ErrorCode
from gspylib.os.gsfile import g_file
def usage():
"""
Usage:
python3 CheckCNStatus.py -h|--help
python3 CheckCNStatus.py -p port -S sql -f outputfile -s snapid -d database
General options:
-p cn port
-S SQL senned to be executed
-f, result output file
def installDatabaseOnHosts(self):
"""
install database on each standby node
"""
hostList = self.context.newHostList
envfile = DefaultValue.getEnv(DefaultValue.MPPRC_FILE_ENV)
tempXmlFile = "%s/clusterconfig.xml" % self.tempFileDir
installCmd = "source {envfile} ; gs_install -X {xmlfile} \
2>&1".format(envfile=envfile,xmlfile=tempXmlFile)
statusArr = []
for newHost in hostList:
self.logger.log("\ninstalling database on node %s:" % newHost)
self.logger.debug(installCmd)
hostName = self.context.backIpNameMap[newHost]
sshIp = self.context.clusterInfoDict[hostName]["sshIp"]
self.initSshConnect(sshIp, self.user)
stdin, stdout, stderr = self.sshClient.exec_command(installCmd,
get_pty=True)
channel = stdout.channel
echannel = stderr.channel
while not channel.exit_status_ready():
try:
recvOut = channel.recv(1024)
outDecode = recvOut.decode("utf-8");
outStr = outDecode.strip()
if(len(outStr) == 0):
continue
if(outDecode.endswith("\r\n")):
self.logger.log(outStr)
else:
value = ""
if re.match(r".*yes.*no.*", outStr):
value = input(outStr)
while True:
# check the input
if (
value.upper() != "YES"
and value.upper() != "NO"
and value.upper() != "Y"
and value.upper() != "N"):
value = input("Please type 'yes' or 'no': ")
continue
break
else:
value = getpass.getpass(outStr)
stdin.channel.send("%s\r\n" %value)
stdin.flush()
stdout.flush()
except Exception as e:
sys.exit(1)
pass
if channel.exit_status_ready() and \
not channel.recv_stderr_ready() and \
not channel.recv_ready():
channel.close()
break
stdout.close()
stderr.close()
status = channel.recv_exit_status()
statusArr.append(status)
isBothSuccess = True
for status in statusArr:
if status != 0:
isBothSuccess = False
break
if isBothSuccess:
self.logger.log("\nSuccessfully install database on node %s" %
hostList)
else:
sys.exit(1)