def getClusterNodesPackagesInstalledSummary(self): """ Returns a string that contains information about the cluster/cluster-storage packages installed for each node. @return: A string that contains information about the cluster/cluster-storage package installed for each node. @rtype: String """ rstring = "" stringUtil = StringUtil() for clusternode in self.getClusterNodes(): if (len(rstring) > 0): rstring += "\n" rstring += "%s:" % (clusternode.getHostname()) # Verify cluster packages # Combine the packages of cluster and storage packages = dict(clusternode.getClusterPackagesVersion(), **clusternode.getClusterModulePackagesVersion()) # Create a simple list of packages that are kernel modules to # compare with later. mPackages = [] for cPackages in clusternode.getClusterModulePackagesVersion( ).values(): for cPackage in cPackages: mPackages.append(cPackage) keys = packages.keys() keys.sort() index = 0 fsTable = [] currentTable = [] for key in keys: cPackages = packages[key] cPackages.sort() for cPackage in cPackages: if (cPackage in mPackages): cPackage = "*%s" % (cPackage) if (index % 2 == 0): if (len(currentTable) > 0): fsTable.append(currentTable) currentTable = [] currentTable.append("%s " % (cPackage)) else: currentTable.append(cPackage) index += 1 if (len(currentTable) > 0): startIndex = len(currentTable) for i in range(len(currentTable), 2): currentTable.append(" ") fsTable.append(currentTable) if (len(fsTable) > 0): packageTableString = stringUtil.toTableString(fsTable) rstring += ("\n%s\n") % (packageTableString) else: rstring += "\nThere was no High Availability or Resilient Storage Packages Found.\n" # Remove an extra newline rstring = rstring.rstrip("\n") return rstring
def getClusterNodesNetworkSummary(self): rstring = "" stringUtil = StringUtil() for clusternode in self.getClusterNodes(): networkMaps = clusternode.getNetworkMaps() hbNetworkMap = clusternode.getHeartbeatNetworkMap() if (not len(networkMaps.getListOfNetworkMaps()) > 0): # Skip if there is no maps continue if (len(rstring) > 0): rstring += "\n" # Get the hb interface so we can flag interface and if # bonded its slave interfaces are put into a list. hbInterfaceBondedSlaves = [] for slaveInterface in hbNetworkMap.getBondedSlaveInterfaces(): hbInterfaceBondedSlaves.append(slaveInterface.getInterface()) # Find alias if there is one and if parent alias is bond # then add to the list slave interfaces. parentAliasInterface = "" if (not hbNetworkMap.getParentAliasNetworkMap() == None): parentAliasInterface = hbNetworkMap.getParentAliasNetworkMap( ).getInterface() if (hbNetworkMap.getParentAliasNetworkMap(). isBondedMasterInterface()): for slaveInterface in hbNetworkMap.getParentAliasNetworkMap( ).getBondedSlaveInterfaces(): hbInterfaceBondedSlaves.append( slaveInterface.getInterface()) # Add netork informaton to string that will be returned. rstring += "%s:\n" % (clusternode.getHostname()) networkInterfaceTable = [] for networkMap in networkMaps.getListOfNetworkMaps(): isHBInterface = "" if (networkMap.getInterface().strip() == hbNetworkMap.getInterface().strip()): isHBInterface = "*" elif ( (networkMap.getInterface().strip() == parentAliasInterface) and (parentAliasInterface > 0)): isHBInterface = "***" elif (networkMap.getInterface().strip() in hbInterfaceBondedSlaves): isHBInterface = "**" networkInterfaceTable.append([ networkMap.getInterface(), networkMap.getNetworkInterfaceModule(), networkMap.getHardwareAddress(), networkMap.getIPv4Address(), isHBInterface ]) tableHeader = [ "device", "module", "hw_addr", "ipv4_addr", "hb_interface" ] rstring += "%s\n" % (stringUtil.toTableString( networkInterfaceTable, tableHeader)) return rstring.strip("\n")
def __getProcessesSummary(self) : stringUtil = StringUtil() rString = "" for glusterPeerNode in self.__glusterPeerNodes.getGlusterPeerNodes(): pTable = [] for process in glusterPeerNode.getGlusterProcesses(): command = process.getCommand() if (len(command) > 70): endStringIndex = len(command.split()[0]) + 50 command = command[0:endStringIndex] pTable.append([process.getPID(), process.getCPUPercentage(), process.getMemoryPercentage(), "%s ...." %(command)]) if (len(pTable) > 0): rString += "%s(%d processes):\n%s\n\n" %(glusterPeerNode.getHostname(), len(pTable), stringUtil.toTableString(pTable, ["pid", "cpu%", "mem%", "command"])) return rString
def evaluate(self): rstring = "" storageData = self.getStorageData() bdt = storageData.getBlockDeviceTree() # ################################################################### # Find out if multipath bindging file is located on a fs /var. # ################################################################### fsVarExist = False # Find out if /var is its own filesystem for filesysMount in bdt.getFilesysMountList(): if (filesysMount.getMountPoint() == "/var"): fsVarExist = True break if (fsVarExist): for line in storageData.getMultipathConfData(): lineSplit = line.strip().split() if (len(lineSplit) == 2): if ((lineSplit[0].strip() == "bindings_file") and (lineSplit[1].strip().startswith("/var"))): description = "The binding file for multipath is located on /var." urls = ["https://access.redhat.com/solutions/17643"] rstring += StringUtil.formatBulletString( description, urls) break # ################################################################### # Check if emc and dmm module are loaded at same time. # ################################################################### foundEMC = False foundDMMultipath = False for lsmod in storageData.getLSMod(): if (lsmod.getModuleName() == "emcp"): foundEMC = True elif (lsmod.getModuleName() == "dm_multipath"): foundDMMultipath = True if (foundEMC and foundDMMultipath): description = "The modules dm-emc and scsi_dh_emc should never be loaded at the same time. One of these packages should be remove: emc or device-mapper-multipath." urls = ["https://access.redhat.com/solutions/45197"] rstring += StringUtil.formatBulletString(description, urls) # ################################################################### # Add newline to separate the node stanzas # ################################################################### if (len(rstring) > 0): rstring += "\n" # ################################################################### return rstring
def __getPeerNodesSummary(self) : stringUtil = StringUtil() pTable = [] rString = "" for glusterPeerNode in self.__glusterPeerNodes.getGlusterPeerNodes(): pTable = [] for peerNodeMap in glusterPeerNode.getPeerNodes(): pnHostname1 = "" if (peerNodeMap.has_key("hostname1")): pnHostname1 = peerNodeMap.get("hostname1") pnUUID = "" if (peerNodeMap.has_key("uuid")): pnUUID = peerNodeMap.get("uuid") pnState = "" if (peerNodeMap.has_key("state")): pnState = peerNodeMap.get("state") pTable.append([pnHostname1, pnUUID, pnState]) if (len(pTable) > 0): rString += "%s(%d peers):\n%s\n\n" %(glusterPeerNode.getHostname(), len(pTable), stringUtil.toTableString(pTable, ["hostname1", "uuid", "state"])) return rString
def getClusterStorageSummary(self) : """ Returns a string that contains information about the GFS1 and GFS2 filesystems found. @return: A string that contains information about the GFS1 and GFS2 filesystems found. @rtype: String """ fsMap = {} for clusternode in self.__cnc.getClusterNodes(): clusternodeName = clusternode.getClusterNodeName() csFilesystemList = clusternode.getClusterStorageFilesystemList() for fs in csFilesystemList: locationFound = "" if (fs.isEtcFstabMount()): locationFound += "F" if (fs.isFilesysMount()): locationFound += "M" if (fs.isClusterConfMount()): locationFound += "C" if (not fsMap.has_key(clusternodeName)): fsMap[clusternodeName] = [] fsMap.get(clusternodeName).append([fs.getDeviceName(), fs.getMountPoint(), fs.getFSType(), locationFound]) rString = "" fsListHeader = ["device", "mount_point", "fs_type", "location_found"] stringUtil = StringUtil() for clusternodeName in self.__cnc.getClusterNodeNames(): # In the future I should probably add a way to only print once if they are all the same . if (fsMap.has_key(clusternodeName)): listOfFileystems = fsMap.get(clusternodeName) if (len(listOfFileystems) > 0): tableString = "%s(%d mounted GFS or GFS2 file-systems)\n%s\n\n" %(clusternodeName, len(listOfFileystems), stringUtil.toTableString(listOfFileystems, fsListHeader)) rString += tableString if (len(rString) > 0): description = "All GFS or GFS2 filesystem are required to be created on a clustered lvm(clvm) device. All GFS or GFS2 filesystems " description += "should be verified that they meet this requirement. The following article describes this requirement:" urls = ["https://access.redhat.com/solutions/46637"] legend = "C = file-system is in /etc/cluster/cluster.conf\nF = file-system is in /etc/fstab\nM = file-system is mounted\n" rString = "%s\n%s\n%s" %(StringUtil.wrapParagraphURLs(description, urls), legend, rString) return rString.strip()
def __comparePackagesToString(self, comparePackages): stringUtil = StringUtil() rString = "" missingPackagesMap = comparePackages.getMissingPackagesMap() if (len(missingPackagesMap.keys()) > 0): description = "The following hosts did not have certain cluster packages installed(whereas other hosts did have the packages installed):" keys = missingPackagesMap.keys() keys.sort() missingPackagesTable = [] for key in keys: reportNames = missingPackagesMap.get(key) reportNames.sort() if (len(reportNames) > 0): currentHostnames = "" for reportName in reportNames: currentHostnames += "%s " % (reportName) missingPackagesTable.append([key, currentHostnames]) tableHeader = ["Package Name", "Hostname(s)"] tableOfStrings = stringUtil.toTableStringsList( missingPackagesTable, tableHeader) rString += StringUtil.formatBulletString(description, [], tableOfStrings) differentPackagesVersionMap = comparePackages.getDiffernetPackagesVersionMap( ) if (len(differentPackagesVersionMap.keys()) > 0): description = "The following hosts had a different package version installed:" keys = differentPackagesVersionMap.keys() keys.sort() differentPackageVersionsTable = [] for key in keys: reportNames = differentPackagesVersionMap.get(key) reportNames.sort() if (len(reportNames) > 0): currentHostnames = "" for reportName in reportNames: currentHostnames += "%s " % (reportName) differentPackageVersionsTable.append( [key, currentHostnames]) tableHeader = ["Package Name", "Hostname(s)"] tableOfStrings = stringUtil.toTableStringsList( differentPackageVersionsTable, tableHeader) rString += StringUtil.formatBulletString(description, [], tableOfStrings) if (len(rString) > 0): rString = "%s\n%s" % (comparePackages, rString) return rString
def __compareDataToString(self, compareData): stringUtil = StringUtil() rString = "" nonBaseCompareMap = compareData.getNonBaseCompareMap() if (not len(nonBaseCompareMap.keys()) > 0): return rString description = "The following hosts had similar compared values:" baseCompareMap = compareData.getBaseCompareMap() keys = baseCompareMap.keys() keys.sort() compareTable = [] for key in keys: reportNames = baseCompareMap.get(key) reportNames.sort() currentHostnames = "" for reportName in reportNames: currentHostnames += "%s " % (reportName) compareTable.append([key, currentHostnames]) tableHeader = ["Compared String", "Hostname(s)"] tableOfStrings = stringUtil.toTableStringsList(compareTable, tableHeader) rString += StringUtil.formatBulletString(description, [], tableOfStrings) description = "The following hosts had different compared values than the above compared values:" keys = nonBaseCompareMap.keys() keys.sort() compareTable = [] for key in keys: reportNames = nonBaseCompareMap.get(key) reportNames.sort() currentHostnames = "" for reportName in reportNames: currentHostnames += "%s " % (reportName) compareTable.append([key, currentHostnames]) tableHeader = ["Compared String", "Hostname(s)"] tableOfStrings = stringUtil.toTableStringsList(compareTable, tableHeader) rString += StringUtil.formatBulletString(description, [], tableOfStrings) if (len(rString) > 0): rString = "%s\n%s" % (compareData, rString) return rString
def report(self): """ This function will write the data that was analyzed to a file. """ message = "Generating report for plugin: %s" % (self.getName()) logging.getLogger(sx.MAIN_LOGGER_NAME).status(message) stringUtil = StringUtil() if (len(self.__listOfNetworkingData) > 0): # Since we are going to run the plugin and create files in # the plugins report directory then we will first remove # all the existing files. self.clean() for networkingData in self.__listOfNetworkingData: message = "Writing the network report for: %s." % ( networkingData.getHostname()) logging.getLogger(sx.MAIN_LOGGER_NAME).debug(message) # could be a problem if they are all using localhost. ar = AnalysisReport( "networking_summary-%s" % (networkingData.getHostname()), "Network Summary") self.addAnalysisReport(ar) arSectionSystemSummary = ARSection("networking-system_summary", "System Summary") ar.add(arSectionSystemSummary) arSectionSystemSummary.add( ARSectionItem(networkingData.getHostname(), networkingData.getSummary())) # Get all the network maps that were built. networkMaps = networkingData.getNetworkMaps() # Bonded Interface Summary bondedInterfaceList = networkMaps.getListOfBondedNetworkMaps() bondedInterfaceTable = [] for bondedInterface in bondedInterfaceList: bondingNumber = bondedInterface.getBondedModeNumber() bondingName = bondedInterface.getBondedModeName() slaveInterfaces = "" for slaveInterface in bondedInterface.getBondedSlaveInterfaces( ): slaveInterfaces += " %s(%s) |" % ( slaveInterface.getInterface(), slaveInterface.getNetworkInterfaceModule()) slaveInterfaces = slaveInterfaces.rstrip("|") bondedInterfaceTable.append([ bondedInterface.getInterface(), bondingNumber, bondingName, slaveInterfaces, bondedInterface.getIPv4Address() ]) if (len(bondedInterfaceTable) > 0): tableHeader = [ "device", "mode_#", "mode_name", "slave_interfaces", "ipv4_address" ] arSectionBondedSummary = ARSection( "networking-bonding_summary", "Bonding Summary") ar.add(arSectionBondedSummary) arSectionBondedSummary.add( ARSectionItem( networkingData.getHostname(), stringUtil.toTableString(bondedInterfaceTable, tableHeader))) # Bridged Inteface Summary bridgedInterfaceTable = [] for networkMap in networkMaps.getListOfBridgedNetworkMaps(): virtualBridgeNetworkMap = networkMap.getVirtualBridgedNetworkMap( ) if (not virtualBridgeNetworkMap == None): bridgedInterfaceTable.append([ networkMap.getInterface(), virtualBridgeNetworkMap.getInterface(), virtualBridgeNetworkMap.getIPv4Address() ]) if (len(bridgedInterfaceTable) > 0): tableHeader = [ "bridge_device", "virtual_bridge_device", "ipv4_addr" ] arSectionBridgedInterfacesSummary = ARSection( "networking-bridged_interfaces_summary", "Bridged Interfaces Summary") ar.add(arSectionBridgedInterfacesSummary) arSectionBridgedInterfacesSummary.add( ARSectionItem( networkingData.getHostname(), stringUtil.toTableString(bridgedInterfaceTable, tableHeader))) # Aliases Interface Summary aliasesInterfaceTable = [] networkInterfaceAliasMap = networkMaps.getNetworkInterfaceAliasMap( ) for key in networkInterfaceAliasMap.keys(): for key in networkInterfaceAliasMap.keys(): aliasInterfacesString = "" for networkMap in networkInterfaceAliasMap[key]: aliasInterfacesString += " %s |" % ( networkMap.getInterface()) aliasInterfacesString = aliasInterfacesString.rstrip("|") aliasesInterfaceTable.append([key, aliasInterfacesString]) if (len(aliasesInterfaceTable) > 0): tableHeader = ["device", "alias_interfaces"] arSectionNetworkingAliasesSummary = ARSection( "networking-networking_aliases_summary", "Networking Aliases Summary") ar.add(arSectionNetworkingAliasesSummary) arSectionNetworkingAliasesSummary.add( ARSectionItem( networkingData.getHostname(), stringUtil.toTableString(aliasesInterfaceTable, tableHeader))) # Network Summary networkInterfaceTable = [] for networkMap in networkMaps.getListOfNetworkMaps(): networkInterfaceTable.append([ networkMap.getInterface(), networkMap.getNetworkInterfaceModule(), networkMap.getHardwareAddress(), networkMap.getIPv4Address() ]) if (len(networkInterfaceTable) > 0): tableHeader = ["device", "module", "hw_addr", "ipv4_addr"] arSectionNetworkingSummary = ARSection( "networking-networking_summary", "Networking Summary") ar.add(arSectionNetworkingSummary) arSectionNetworkingSummary.add( ARSectionItem( networkingData.getHostname(), stringUtil.toTableString(networkInterfaceTable, tableHeader))) # Wrtite the output to a file. self.write("%s.txt" % (ar.getName()), "%s\n" % (str(ar)))
def __generateReport(self, cnc): # Name of the file that will be used to write the report. if (not len(cnc.getClusterNodes()) > 0): message = "There were no cluster nodes found for this cluster." logging.getLogger(sx.MAIN_LOGGER_NAME).warn(message) else: baseClusterNode = cnc.getBaseClusterNode() if (baseClusterNode == None): # Should never occur since node count should be checked first. return cca = ClusterHAConfAnalyzer(baseClusterNode.getPathToClusterConf()) # List of clusternodes in cluster.conf that do not have # corresponding sosreport/sysreport. filename = "%s-summary.txt" % (cca.getClusterName()) missingNodesList = cnc.listClusterNodesMissingReports() missingNodesMessage = "" if (len(missingNodesList) > 0): missingNodesMessage = "The following cluster nodes could not be matched to a report that was analyzed:" for nodeName in missingNodesList: missingNodesMessage += "\n\t %s" % (nodeName) logging.getLogger( sx.MAIN_LOGGER_NAME).warn(missingNodesMessage) self.write(filename, "%s\n" % (missingNodesMessage)) # ################################################################### # Summary of each node in collection # ################################################################### result = cnc.getClusterNodesSystemSummary() if (len(result) > 0): self.writeSeperator( filename, "Cluster Nodes Summary (%s - %d Cluster Nodes)" % (cca.getClusterName(), len(cca.getClusterNodeNames()))) self.write(filename, result.rstrip()) self.write(filename, "") # Write qdisk information if it exists. result = cca.getQuorumdSummary() if (len(result) > 0): self.writeSeperator(filename, "Cluster Quorum Disk Summary") self.write(filename, result.rstrip()) self.write(filename, "") result = cnc.getClusterNodesPackagesInstalledSummary() if (len(result) > 0): self.writeSeperator( filename, "Cluster/Cluster-Storage Packages Installed") self.write( filename, "The list of installed High Availability and Resilient Storage packages installed. Kernel Module \npackages will have an asterick(*) after their name.\n" ) self.write(filename, result.rstrip()) self.write(filename, "") result = cnc.getClusterNodesNetworkSummary() if (len(result) > 0): self.writeSeperator(filename, "Cluster Nodes Network Summary") self.write( filename, "* = heartbeat network\n** = bonded slave interfaces\n*** = parent of alias interface\n" ) self.write(filename, result) self.write(filename, "") clusterHAStorage = ClusterHAStorage(cnc) result = clusterHAStorage.getSummary() if (len(result) > 0): self.write(filename, result.rstrip()) self.write(filename, "") # ################################################################### # Check the cluster node services summary # ################################################################### listOfServicesforClusterNodes = [] # Get the list of services for clusternode in cnc.getClusterNodes(): chkConfigClusterServiceList = clusternode.getChkConfigClusterServicesStatus( ) if (len(chkConfigClusterServiceList) > 0): #sortedChkConfigClusterServicesList = sorted(chkConfigClusterServiceList, key=lambda k: k.getStartOrderNumber()) sortedChkConfigClusterServicesList = sorted( chkConfigClusterServiceList, key=lambda k: k.getName()) currentListOfServices = list( set( map(lambda m: m.getName(), sortedChkConfigClusterServicesList))) listOfServicesforClusterNodes = list( set(listOfServicesforClusterNodes) | set(currentListOfServices)) # Just sort alpha and not worry with order. listOfServicesforClusterNodes.sort() clusternodeServicesTable = [] for clusternode in cnc.getClusterNodes(): currentTable = [clusternode.getClusterNodeName()] #sortedChkConfigClusterServicesList = sorted(chkConfigClusterServiceList, key=lambda k: k.getStartOrderNumber()) sortedChkConfigClusterServicesList = sorted( chkConfigClusterServiceList, key=lambda k: k.getName()) for serviceName in listOfServicesforClusterNodes: serviceStatus = "-" for chkConfigClusterService in sortedChkConfigClusterServicesList: if (chkConfigClusterService.getName() == serviceName): if (chkConfigClusterService.isEnabledRunlevel3( ) and chkConfigClusterService.isEnabledRunlevel4() and chkConfigClusterService. isEnabledRunlevel5()): serviceStatus = "E" else: serviceStatus = "D" currentTable.append(serviceStatus) clusternodeServicesTable.append(currentTable) if (len(clusternodeServicesTable) > 0): stringUtil = StringUtil() tableHeader = ["hostame"] + listOfServicesforClusterNodes self.writeSeperator(filename, "Cluster Services Summary") header = "List of Clustered Services that are enabled for all of these runlevel 3, 4, and 5:\n" header += "- https://access.redhat.com/solutions/5898\n\nE = Enabled\nD = Disabled\n- = Unknown Status\n" self.write(filename, header) self.write( filename, "%s\n" % (stringUtil.toTableString( clusternodeServicesTable, tableHeader))) # Write a summary of the cluster.conf services #filename = "%s-services.txt" %(cca.getClusterName()) clusteredServicesList = cca.getClusteredServices() clusteredServicesString = "" clusteredVMServicesString = "" regServiceCount = 0 vmServiceCount = 0 for clusteredService in clusteredServicesList: if (not clusteredService.isVirtualMachineService()): sIndex = str(regServiceCount + 1) if ((regServiceCount + 1) < 10): sIndex = " %d" % (regServiceCount + 1) # clusteredServicesString += "%s. %s\n\n" %(sIndex, str(clusteredService).rstrip()) regServiceCount = regServiceCount + 1 elif (clusteredService.isVirtualMachineService()): sIndex = str(vmServiceCount + 1) if ((vmServiceCount + 1) < 10): sIndex = " %d" % (vmServiceCount + 1) # clusteredVMServicesString += "%s. %s\n\n" %(sIndex, str(clusteredService).rstrip()) vmServiceCount = vmServiceCount + 1 if (regServiceCount > 0): self.writeSeperator(filename, "Clustered Services Summary") self.write( filename, "There was %d clustered services managed by rgmanager.\n" % (regServiceCount)) #self.write(filename, "%s\n" %(clusteredServicesString.rstrip())) if (vmServiceCount > 0): self.writeSeperator( filename, "Clustered Virtual Machine Services Summary") self.write( filename, "There was %d clustered virtual machine services managed by rgmanager.\n" % (vmServiceCount)) #self.write(filename, "%s\n" %(clusteredVMServicesString.rstrip())) # ################################################################### # Verify the cluster node configuration # ################################################################### filenameCE = "%s-evaluator.txt" % (cca.getClusterName()) clusterEvaluator = ClusterEvaluator(cnc) evaluatorResult = clusterEvaluator.evaluate() if (len(evaluatorResult) > 0): if (len(missingNodesList) > 0): self.write(filenameCE, "%s\n\n" % (missingNodesMessage)) self.writeSeperator( filenameCE, "Known Issues with Cluster (%s - %d Cluster Nodes)" % (cca.getClusterName(), len(cca.getClusterNodeNames()))) self.write( filenameCE, "NOTE: The known issues below may or may not be related to solving the current issue or preventing" ) self.write( filenameCE, " a issue. These are meant to be a guide in making sure that the cluster is happy and healthy" ) self.write( filenameCE, " healthy all the time. Please use report as a guide in reviewing the cluster.\n" ) self.write(filenameCE, evaluatorResult.rstrip()) self.write(filenameCE, "") # ################################################################### # Evaluate the reports as stretch cluster if that option is enabled. # ################################################################### isStretchCluster = self.getOptionValue("isStretchCluster") if (not isStretchCluster == "0"): filenameCE = "%s-stretch_evaluator.txt" % ( cca.getClusterName()) clusterHAStretchEvaluator = ClusterHAStretchEvaluator(cnc) evaluatorResult = clusterHAStretchEvaluator.evaluate() if (len(evaluatorResult) > 0): if (len(missingNodesList) > 0): self.write(filenameCE, "%s\n\n" % (missingNodesMessage)) self.writeSeperator(filenameCE, "Known Issues with Stretch Cluster") self.write( filenameCE, "NOTE: The known issues below may or may not be releated to solving" ) self.write( filenameCE, " the current issue or preventing a issue. These are meant to" ) self.write( filenameCE, " be a guide in making sure that the cluster is happy and" ) self.write( filenameCE, " healthy all the time. Please use report as a guide in" ) self.write(filenameCE, " reviewing the cluster.\n") self.write(filenameCE, evaluatorResult.rstrip()) self.write(filenameCE, "") # ################################################################### # Verify the cluster node configuration # ################################################################### filenameCE = "%s-clusternode_compare.txt" % (cca.getClusterName()) clusternodeCompare = ClusternodeCompare(cnc) compareResult = clusternodeCompare.compare() if (len(compareResult) > 0): if (len(missingNodesList) > 0): self.write(filenameCE, "%s\n\n" % (missingNodesMessage)) self.writeSeperator(filenameCE, "Clusternode Comparison") self.write( filenameCE, "The following section will show the difference between the reports when a value in a file is" ) self.write( filenameCE, "compared against all the reports. The section will list the reports with similar values" ) self.write( filenameCE, "and ones that are different then the similar values. The similar values is the highest number" ) self.write(filenameCE, "of reports with the same value.\n") self.write(filenameCE, compareResult.rstrip()) self.write(filenameCE, "")
def report(self): """ This function will write the data that was analyzed to a file. """ message = "Generating report for plugin: %s" % (self.getName()) logging.getLogger(sx.MAIN_LOGGER_NAME).status(message) if (len(self.__listOfStorageData) > 0): # Since we are going to run the plugin and create files in # the plugins report directory then we will first remove # all the existing files. self.clean() stringUtil = StringUtil() for storageData in self.__listOfStorageData: message = "Writing the storage report for: %s." % ( storageData.getHostname()) logging.getLogger(sx.MAIN_LOGGER_NAME).debug(message) # could be a problem if they are all using localhost. ar = AnalysisReport( "storage_summary-%s" % (storageData.getHostname()), "Storage Summary") self.addAnalysisReport(ar) arSectionSystemSummary = ARSection("storage-system_summary", "System Summary") ar.add(arSectionSystemSummary) arSectionSystemSummary.add( ARSectionItem(storageData.getHostname(), storageData.getSummary())) # The block device tree has some of the information that # is needed to report on. bdt = storageData.getBlockDeviceTree() # Get all the mounted filesystems. mountedFSList = bdt.getFilesysMountList() if (len(mountedFSList) > 0): fsTable = [] for fs in mountedFSList: fsTable.append([ fs.getDeviceName(), fs.getMountPoint(), fs.getFSType(), fs.getFSAttributes(), fs.getMountOptions() ]) tableHeader = [ "device", "mount_point", "fs_type", "fs_attributes", "fs_options" ] arSectionMountedFS = ARSection("storage-mounted_fs", "Mounted Filesystems") ar.add(arSectionMountedFS) arSectionMountedFS.add( ARSectionItem( storageData.getHostname(), stringUtil.toTableString(fsTable, tableHeader))) # Write out any multipath data blockDeviceMap = bdt.generateDMBlockDeviceMap() multipathMap = bdt.getTargetTypeMap(blockDeviceMap, "multipath") if (len(multipathMap.keys()) > 0): multipathSummary = "" for key in multipathMap.keys(): multipathSummary += "%s\n" % (str( multipathMap.get(key)).strip()) arSectionMultipathSummary = ARSection( "storage-multipath_summary", "Multipath Summary") ar.add(arSectionMultipathSummary) arSectionMultipathSummary.add( ARSectionItem(storageData.getHostname(), multipathSummary.strip().rstrip())) # ################################################################### # Run the evaluator to look for know issues # ################################################################### storageEvaluator = StorageEvaluator(storageData) rstring = storageEvaluator.evaluate() if (len(rstring) > 0): arSectionKnownIssues = ARSection("storage-known_issues", "Known Issues with Storage") ar.add(arSectionKnownIssues) arSectionKnownIssues.add( ARSectionItem(storageData.getHostname(), rstring)) # Wrtite the output to a file. self.write("%s.txt" % (ar.getName()), "%s\n" % (str(ar))) # ################################################################### # Create the blockDeviceTree file # ################################################################### blockDeviceMap = bdt.generateDMBlockDeviceMap() if (len(blockDeviceMap.keys())): # Print a summary of the devicemapper devices. Group by target type. arBDT = AnalysisReport( "storage_block_device_tree-%s" % (storageData.getHostname()), "Block Device Tree") self.addAnalysisReport(arBDT) for targetType in bdt.getValidTargetTypes(): currentBlockDeviceMap = bdt.getTargetTypeMap( blockDeviceMap, targetType) if (len(currentBlockDeviceMap) > 0): arSectionBDT = ARSection( "storage_block_device_tree-block_device_target_types", "Block Device for Target Type: %s (%d targets)" % (targetType, len(currentBlockDeviceMap))) arBDT.add(arSectionBDT) for key in currentBlockDeviceMap.keys(): currentBlockDevice = currentBlockDeviceMap[key] arSectionBDT.add( ARSectionItem( "%s-%s" % (storageData.getHostname(), targetType), "%s" % (str(currentBlockDevice)))) self.write("%s.txt" % (arBDT.getName()), "%s\n" % (str(arBDT)))
def evaluateClusteredFilesystems(self): # Is active/active nfs supported? Sorta # urls = ["https://access.redhat.com/solutions/59498"] rString = "" baseClusterNode = self.__cnc.getBaseClusterNode() if (baseClusterNode == None): return rString cca = ClusterHAConfAnalyzer(baseClusterNode.getPathToClusterConf()) if ((cca.getTransportMode() == "broadcast") or (cca.getTransportMode() == "udpu")): for clusternode in self.__cnc.getClusterNodes(): if (len(clusternode.getClusterStorageFilesystemList()) > 0): description = "There is known limitations for GFS2 filesystem when using the " description += "following transports: \"%s\"." %(cca.getTransportMode()) urls = ["https://access.redhat.com/solutions/162193", "https://access.redhat.com/articles/146163", "https://access.redhat.com/solutions/459243"] rString += "%s\n" %(StringUtil.formatBulletString(description, urls)) break; for clusternode in self.__cnc.getClusterNodes(): stringUtil = StringUtil() clusterNodeEvalString = "" # ################################################################### # Distro Specific evaluations # ################################################################### # The distro release of this node distroRelease = clusternode.getDistroRelease() if ((distroRelease.getDistroName() == "RHEL") and (distroRelease.getMajorVersion() == 5)): # Check if GFS2 module should be removed on RH5 nodes if (self.__doesGFS2ModuleNeedRemoval(clusternode.getUnameA(), clusternode.getClusterModulePackagesVersion())) : description = "The kmod-gfs2 is installed on a running kernel >= 2.6.18-128. This module should be removed since the module is included in the kernel." urls = ["https://access.redhat.com/solutions/17832"] clusterNodeEvalString += StringUtil.formatBulletString(description, urls) # ################################################################### # Analyze the Clustered Storage # ################################################################### listOfClusterStorageFilesystems = clusternode.getClusterStorageFilesystemList() # ################################################################### # Verify that GFS/GFS2 filesystem is using lvm with cluster bit set # ################################################################### fsTable = [] # Verify the locking_type is set to 3 cause built-in cluster locking is required. if (len(listOfClusterStorageFilesystems) > 0): devicemapperCommandsMap = self.__cnc.getStorageData(clusternode.getClusterNodeName()).getDMCommandsMap() lvm = LVM(DeviceMapperParser.parseVGSVData(devicemapperCommandsMap.get("vgs_-v")), DeviceMapperParser.parseLVSAODevicesData(devicemapperCommandsMap.get("lvs_-a_-o_devices")), self.__cnc.getStorageData(clusternode.getClusterNodeName()).getLVMConfData()) if (not lvm.isLockingTypeClustering()): description = "The locking_type is not set to type 3 for built-in cluster locking. A GFS/GFS2 filesystem requires the filesystem be on a " description += "clustered LVM volume with locking_type 3 enabled in the /etc/lvm/lvm.conf." urls = ["https://access.redhat.com/solutions/46637"] clusterNodeEvalString += StringUtil.formatBulletString(description, urls) # Disabling this check for now cause still working on how to do it. """ # Verify that the clustered filesystem has clusterbit set on the vg. # Verify the locking_type is set to 3 cause built-in cluster locking is required. fsTable = [] if (len(listOfClusterStorageFilesystems) > 0): devicemapperCommandsMap = self.__cnc.getStorageData(clusternode.getClusterNodeName()).getDMCommandsMap() lvm = LVM(DeviceMapperParser.parseVGSVData(devicemapperCommandsMap.get("vgs_-v")), DeviceMapperParser.parseLVSAODevicesData(devicemapperCommandsMap.get("lvs_-a_-o_devices")), self.__cnc.getStorageData(clusternode.getClusterNodeName()).getLVMConfData()) for csFilesystem in listOfClusterStorageFilesystems: pathToDevice = str(csFilesystem.getDeviceName().strip().rstrip()) if (not lvm.isClusteredLVMDevice(pathToDevice)): currentFS = [pathToDevice, csFilesystem.getMountPoint(), csFilesystem.getFSType()] if (not currentFS in fsTable): fsTable.append(currentFS) if (len(fsTable) > 0): stringUtil = StringUtil() description = "The following filesystems appears not to be on a clustered LVM volume. A clustered LVM volume is required for GFS/GFS2 fileystems." tableHeader = ["device_name", "mount_point", "fs_type"] tableOfStrings = stringUtil.toTableStringsList(fsTable, tableHeader) urls = ["https://access.redhat.com/solutions/46637"] clusterNodeEvalString += StringUtil.formatBulletString(description, urls, tableOfStrings) """ # ################################################################### # Verify they are exporting a gfs/gfs2 fs via samba and nfs correctly # ################################################################### tableHeader = ["device_name", "mount_point", "nfs_mp", "smb_mp"] fsTable = [] for csFilesystem in listOfClusterStorageFilesystems: # There are 4 ways of mounting gfs via nfs/smb at same time that # needs to be checked: # 1) nfs mount via /etc/exports and smb mount via /etc/samba/smb.conf # 2) nfs mount via /etc/cluster/cluster.conf and smb mount via /etc/cluster/cluster.conf # 3) nfs mount via /etc/cluster/cluster.conf and smb mount via /etc/samba/smb.conf. # 4) nfs mount via /etc/exports and smb mount via /etc/cluster/cluster.conf if (csFilesystem.isEtcExportMount() and csFilesystem.isSMBSectionMount()): # 1) nfs mount via /etc/exports and smb mount via /etc/samba/smb.conf #print "1: %s" %(csFilesystem.getMountPoint()) nfsMP = csFilesystem.getEtcExportMount().getMountPoint() smbSectionList = csFilesystem.getSMBSectionMountList() if (len(smbSectionList) > 0): smbMP = smbSectionList.pop().getOptionValue("path").strip() fsTable.append([csFilesystem.getDeviceName(), csFilesystem.getMountPoint(), "%s(EN)" %(nfsMP), "%s(ES)" %(smbMP)]) for smbSection in smbSectionList: smbMP = smbSection.getOptionValue("path").strip() fsTable.append(["", "", "", "%s(ES)" %(smbMP)]) elif ((not csFilesystem.isEtcExportMount()) and (not csFilesystem.isSMBSectionMount())): # 2) nfs mount via /etc/cluster/cluster.conf and smb mount via /etc/cluster/cluster.conf #print "2: %s" %(csFilesystem.getMountPoint()) if((self.__isNFSChildOfClusterStorageResource(cca, csFilesystem)) and (len(csFilesystem.getClusteredSMBNames()) > 0)): nfsMP = csFilesystem.getMountPoint() smbPaths = [] for name in csFilesystem.getClusteredSMBNames(): for smbSection in csFilesystem.getClusteredSMBSectionList(name): currentPath = smbSection.getOptionValue("path").strip() if (len(currentPath) > 0): smbPaths.append(currentPath) if ((len(nfsMP) > 0) and (len(smbPaths) > 0)): # Pop the first one off the list. smbMP = smbPaths.pop() fsTable.append([csFilesystem.getDeviceName(), csFilesystem.getMountPoint(), "%s(CN)" %(nfsMP), "%s(CS)" %(smbMP)]) # IF there any left add those with some blanks. for smbMP in smbPaths: fsTable.append(["", "", "", "%s(CS)" %(smbMP)]) elif ((csFilesystem.isSMBSectionMount()) and (self.__isNFSChildOfClusterStorageResource(cca, csFilesystem))): # 3) nfs mount via /etc/cluster/cluster.conf and smb mount via /etc/samba/smb.conf. #print "3: %s" %(csFilesystem.getMountPoint()) nfsMP = csFilesystem.getMountPoint() smbSectionList = csFilesystem.getSMBSectionMountList() if (len(smbSectionList) > 0): smbMP = smbSectionList.pop().getOptionValue("path").strip() fsTable.append([csFilesystem.getDeviceName(), csFilesystem.getMountPoint(), "%s(CN)" %(nfsMP), "%s(ES)" %(smbMP)]) for smbSection in smbSectionList: smbMP = smbSection.getOptionValue("path").strip() fsTable.append(["", "", "", "%s(ES)" %(smbMP)]) elif ((csFilesystem.isEtcExportMount()) and (len(csFilesystem.getClusteredSMBNames()) > 0)): # 4) nfs mount via /etc/exports and smb mount via /etc/cluster/cluster.conf # print "4: %s" %(csFilesystem.getMountPoint()) smbSectionList = [] for name in csFilesystem.getClusteredSMBNames(): smbSectionList += csFilesystem.getClusteredSMBSectionList(name) if (len(smbSectionList) > 0): smbMP = smbSectionList.pop().getOptionValue("path").strip() fsTable.append([csFilesystem.getDeviceName(), csFilesystem.getMountPoint(), "%s(EN)" %(nfsMP), "%s(CS)" %(smbMP)]) for smbSection in smbSectionList: smbMP = smbSection.getOptionValue("path").strip() fsTable.append(["", "", "", "%s(CS)" %(smbMP)]) # Write the table if it is not empty. if (len(fsTable) > 0): description = "The following GFS/GFS2 filesystem(s) are being exported by NFS and SMB(samba) which is unsupported. " description += "The mount point(s) that were found will be noted with these symbols below: " description += "nfs export via /etc/exports (EN) " description += "nfs export via /etc/cluster/cluster.conf (CN) " description += "samba export via /etc/exports for samba (ES) " description += "samba export via /etc/cluster/cluster.conf for samba (CS)" urls = ["https://access.redhat.com/solutions/39855"] tableOfStrings = stringUtil.toTableStringsList(fsTable, tableHeader) clusterNodeEvalString += StringUtil.formatBulletString(description, urls, tableOfStrings) # ################################################################### # Check for localflocks if they are exporting nfs. # ################################################################### fsTable = [] for csFilesystem in listOfClusterStorageFilesystems: # If a GFS or GFS2 fs is in /etc/exports or has a child that is # nfsexport then localflocks required. if ((csFilesystem.isEtcExportMount()) or (self.__isNFSChildOfClusterStorageResource(cca, csFilesystem))): csFilesystemOptions = csFilesystem.getAllMountOptions() if (not csFilesystemOptions.find("localflocks") >= 0): fsTable.append([csFilesystem.getDeviceName(), csFilesystem.getMountPoint()]) # Write the table if it is not empty. if (len(fsTable) > 0): tableHeader = ["device_name", "mount_point"] description = "Any GFS/GFS2 filesystem that is exported with NFS should have the option \"localflocks\" set." description += "The following GFS/GFS2 filesystem do not have the option set." tableOfStrings = stringUtil.toTableStringsList(fsTable, tableHeader) urls = ["https://access.redhat.com/solutions/20327", "http://docs.redhat.com/docs/en-US/Red_Hat_Enterprise_Linux/5/html-single/Configuration_Example_-_NFS_Over_GFS/index.html#locking_considerations"] clusterNodeEvalString += StringUtil.formatBulletString(description, urls, tableOfStrings) # ################################################################### # Check to see if the GFS/GFS2 fs has certain mount options enabled. # ################################################################### fsTable = [] for csFilesystem in listOfClusterStorageFilesystems: csFilesystemOptions = csFilesystem.getAllMountOptions() if (not csFilesystemOptions.find("noatime") >= 0): fsTable.append([csFilesystem.getDeviceName(), csFilesystem.getMountPoint()]) if (len(fsTable) > 0): # Verified that noatime implies nodiratime, so nodiratime check # does not need to be done. description = "There were GFS/GFS2 file-systems that did not have the mount option \"noatime\"(no \"nodiratime\" is implied. " description += "when noatime is set) enabled. Unless atime support is essential, Red Hat recommends setting the mount option " description += "\"noatime\" on every GFS/GFS2 mount point. This will significantly improve performance since it prevents " description += "reads from turning into writes because the access time attribute will not be updated." urls = ["https://access.redhat.com/articles/628093#slowdown_due_to_system_configuration_issues"] clusterNodeEvalString += StringUtil.formatBulletString(description, urls) # ################################################################### # Make sure GFS/GFS2 filesystems dont have fsck option enable # ################################################################### for csFilesystem in listOfClusterStorageFilesystems: if (csFilesystem.isEtcFstabMount()): if (not csFilesystem.getEtcFstabMount().getFSFsck() == "0"): description = "There were GFS/GFS2 file-systems that had the fsck option enabled in the /etc/fstab file. This option " description += "should be disabled(set value to 0) or corruption will occur eventually." urls = ["https://access.redhat.com/solutions/766393"] clusterNodeEvalString += StringUtil.formatBulletString(description, urls) # ################################################################### # Add to string with the hostname and header if needed. # ################################################################### if (len(clusterNodeEvalString) > 0): rString += "%s(Cluster Node ID: %s):\n%s\n\n" %(clusternode.getClusterNodeName(), clusternode.getClusterNodeID(), clusterNodeEvalString.rstrip()) # Return the string if (len(rString) > 0): sectionHeader = "%s\nCluster Storage Configuration Known Issues\n%s" %(self.__seperator, self.__seperator) rString = "%s\n%s" %(sectionHeader, rString) return rString
def evaluateNonClusteredFilesystems(self): """ This functions verifies that all fs resources are using HALVM. This checks to see if clusterbit on lvm vg is set or if they are using "volume_list" method in /etc/lvm/lvm.conf. Do note that tags in "volume_list" option are not checked. """ rString = "" baseClusterNode = self.__cnc.getBaseClusterNode() if (baseClusterNode == None): return rString cca = ClusterHAConfAnalyzer(baseClusterNode.getPathToClusterConf()) fsTable = [] filesystemResourcesList = cca.getFilesystemResourcesList() if (len(filesystemResourcesList) > 0): for clusterConfMount in filesystemResourcesList: currentFS = [clusterConfMount.getDeviceName(), clusterConfMount.getMountPoint(), clusterConfMount.getFSType()] if (not currentFS in fsTable): fsTable.append(currentFS) if (len(fsTable) > 0): stringUtil = StringUtil() sectionHeader = "%s\nFilesystem and Clustered-Filesystem cluster.conf Summary\n%s" %(self.__seperator, self.__seperator) description = "There was %d filesystems resources(fs.sh) found in the cluster.conf. It is recommended that all filesystem resource\'s(fs.sh) " %(len(fsTable)) description += "underlying storage device is using one(not both) of the 2 methods for HALVM as described in article below for the underlying " description += "storage device. The following article describes these procedures:" urls = ["https://access.redhat.com/solutions/3067"] rString += StringUtil.formatBulletString(description, urls) # Disabled for now and will return an empty string """ baseClusterNode = self.__cnc.getBaseClusterNode() if (baseClusterNode == None): return rString cca = ClusterHAConfAnalyzer(baseClusterNode.getPathToClusterConf()) fsTable = [] filesystemResourcesList = cca.getFilesystemResourcesList() for clusternode in self.__cnc.getClusterNodes(): clusterNodeEvalString = "" # Check to see if volume_list and locking_type 3 is set for cluster # locking. if (len(filesystemResourcesList) > 0): devicemapperCommandsMap = self.__cnc.getStorageData(clusternode.getClusterNodeName()).getDMCommandsMap() lvm = LVM(DeviceMapperParser.parseVGSVData(devicemapperCommandsMap.get("vgs_-v")), DeviceMapperParser.parseLVSAODevicesData(devicemapperCommandsMap.get("lvs_-a_-o_devices")), self.__cnc.getStorageData(clusternode.getClusterNodeName()).getLVMConfData()) if (lvm.isVolumeListEnabled() and lvm.isLockingTypeClustering()): description = "The option \"volume_list\" and \"locking_type = 3\" in /etc/lvm/lvm.conf are both in use. Using " description += "both options at the same time is supported, but not recommended except for certain configurations. " description += "This configuration should be reviewed to verify that configuration is correct." urls = ["https://access.redhat.com/solutions/3067"] clusterNodeEvalString += StringUtil.formatBulletString(description, urls) # Check to see if device is either has volume_list set or has # cluster bit set on each fs resource. for clusterConfMount in filesystemResourcesList: pathToDevice = str(clusterConfMount.getDeviceName().strip().rstrip()) if (not lvm.isLVMVolumeHALVM(pathToDevice)): currentFS = [clusterConfMount.getDeviceName(), clusterConfMount.getMountPoint(), clusterConfMount.getFSType()] if (not currentFS in fsTable): fsTable.append(currentFS) if (len(fsTable) > 0): # Should flag for vgs with no c bit set and no lvm on device path. stringUtil = StringUtil() description = "The following filesystems appears to not be on a HALVM volume using one of the methods outlined in the article below. " description += "A HALVM volume is recommoned for all fs resources. Do note that LVM tags were not searched and compared if they were " description += "used in the \"volume_list\" option of the /etc/lvm/lvm.conf file:" tableHeader = ["device_name", "mount_point", "fs_type"] tableOfStrings = stringUtil.toTableStringsList(fsTable, tableHeader) urls = ["https://access.redhat.com/solutions/3067"] rString += StringUtil.formatBulletString(description, urls, tableOfStrings) """ return rString
def evaluate(self): """ * If two node cluster, check if hb and fence on same network. warn qdisk required if not or fence delay. """ # Return string for evaluation. rstring = "" # Nodes that are in cluster.conf, so should have report of all these baseClusterNode = self.__cnc.getBaseClusterNode() if (baseClusterNode == None): # Should never occur since node count should be checked first. return "" cca = ClusterHAConfAnalyzer(baseClusterNode.getPathToClusterConf()) for clusternode in self.__cnc.getClusterNodes(): clusterNodeEvalString = "" if (not clusternode.isClusterNode()): continue # The distro release of this node distroRelease = clusternode.getDistroRelease() # The clusternode name in /etc/cluster/cluster.conf clusterNodeName = clusternode.getClusterNodeName() if (not (distroRelease.getDistroName() == "RHEL") and ((distroRelease.getMajorVersion() == 5) or (distroRelease.getMajorVersion() == 6))): message = "Stretch Clusters are only supported on RHEL 5 and RHEL6." logging.getLogger(sx.MAIN_LOGGER_NAME).error(message) else: # ################################################################### # CLVMD and cmirror cannot be enabled on stretch clusters. # ################################################################### serviceName = "clvmd" serviceRunlevelEnabledString = "" for chkConfigItem in clusternode.getChkConfigList(): if (chkConfigItem.getName() == serviceName): if(chkConfigItem.isEnabledRunlevel3()): serviceRunlevelEnabledString += "3 " if(chkConfigItem.isEnabledRunlevel4()): serviceRunlevelEnabledString += "4 " if(chkConfigItem.isEnabledRunlevel5()): serviceRunlevelEnabledString += "5 " if (len(serviceRunlevelEnabledString) > 0): description = "The service %s should be disabled if this is cluster node is part of a stretch cluster. The service %s is not supported in stretch clusters." %(serviceName, serviceName) description += "The following runlevels have %s enabled: %s." %(serviceName, serviceRunlevelEnabledString.strip()) urls = ["https://access.redhat.com/solutions/163833"] clusterNodeEvalString += StringUtil.formatBulletString(description, urls) serviceName = "cmirror" serviceRunlevelEnabledString = "" for chkConfigItem in clusternode.getChkConfigList(): if (chkConfigItem.getName() == serviceName): if(chkConfigItem.isEnabledRunlevel3()): serviceRunlevelEnabledString += "3 " if(chkConfigItem.isEnabledRunlevel4()): serviceRunlevelEnabledString += "4 " if(chkConfigItem.isEnabledRunlevel5()): serviceRunlevelEnabledString += "5 " if (len(serviceRunlevelEnabledString) > 0): description = "The service %s should be disabled if this is cluster node is part of a stretch cluster. The service %s is not supported in stretch clusters." %(serviceName, serviceName) description += "The following runlevels have %s enabled: %s." %(serviceName, serviceRunlevelEnabledString.strip()) urls = ["https://access.redhat.com/solutions/163833"] clusterNodeEvalString += StringUtil.formatBulletString(description, urls) # ################################################################### # Add newline to separate the node stanzas # ################################################################### if (len(clusterNodeEvalString) > 0): rstring += "%s(Cluster Node ID: %s):\n%s\n" %(clusterNodeName, clusternode.getClusterNodeID(), clusterNodeEvalString) # ################################################################### return rstring