def computeNodeInitialize(self, computeNodeResources,
versionInformationLogOnly, updateOSHarddrives):
logger = logging.getLogger(self.loggerName)
'''
This is returned with any error messages that are encountered along with whether or not the
compute node needs to be updated.
'''
resultDict = {'updateNeeded': False, 'errorMessages': []}
#Get system model.
command = "dmidecode -s system-product-name"
result = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
out, err = result.communicate()
out = out.strip()
logger.debug("The output of the command (" + command +
") used to get the system's model was: " + out)
if result.returncode != 0:
logger.error("Unable to get the system's model information.\n" +
err)
resultDict['errorMessages'].append(
"Unable to get the system's model information.")
return resultDict
try:
systemModel = (re.match('[a-z,0-9]+\s+(.*)', out,
re.IGNORECASE).group(1)).replace(' ', '')
except AttributeError as err:
logger.error(
"There was a system model match error when trying to match against '"
+ out + "':\n" + str(err) + ".")
resultDict['errorMessages'].append(
"There was a system model match error.")
return resultDict
try:
if systemModel not in self.csurResourceDict[
'supportedComputeNodeModels']:
logger.error("The system's model (" + systemModel +
") is not supported by this CSUR bundle.")
resultDict['errorMessages'].append(
"The system's model is not supported by this CSUR bundle.")
return resultDict
except KeyError as err:
logger.error(
"The resource key (" + str(err) +
") was not present in the application's esource file.")
resultDict['errorMessages'].append(
"A resource key error was encountered.")
return resultDict
logger.debug("The system's model was determined to be: " +
systemModel + ".")
self.computeNodeDict['systemModel'] = systemModel
#Get the system's OS distribution version information.
command = "cat /proc/version"
result = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
out, err = result.communicate()
logger.debug("The output of the command (" + command +
") used to get the OS distribution information was: " +
out.strip())
if result.returncode != 0:
logger.error(
"Unable to get the system's OS distribution version information.\n"
+ err)
resultDict['errorMessages'].append(
"Unable to get the system's OS distribution version information."
)
return resultDict
#Change version information to lowercase before checking for OS type.
versionInfo = out.lower()
if 'suse' in versionInfo:
OSDist = 'SLES'
command = "cat /etc/SuSE-release"
else:
OSDist = 'RHEL'
command = "cat /etc/redhat-release"
result = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
out, err = result.communicate()
if result.returncode != 0:
logger.error(
"Unable to get the system's OS distribution level.\n" + err)
resultDict['errorMessages'].append(
"Unable to get the system's OS distribution level.")
return resultDict
else:
releaseInfo = out.replace('\n', ' ')
if OSDist == 'SLES':
try:
slesVersion = re.match('.*version\s*=\s*([1-4]{2})',
releaseInfo, re.IGNORECASE).group(1)
except AttributeError as err:
logger.error(
"There was SLES OS version match error when trying to match against '"
+ releaseInfo + "':\n" + str(err) + ".")
resultDict['errorMessages'].append(
"There was a SLES OS version match error.")
return resultDict
try:
slesPatchLevel = re.match('.*patchlevel\s*=\s*([1-4]{1})',
releaseInfo,
re.IGNORECASE).group(1)
except AttributeError as err:
logger.error(
"There was SLES patch level match error when trying to match against '"
+ releaseInfo + "':\n" + str(err) + ".")
resultDict['errorMessages'].append(
"There was a SLES patch level match error.")
return resultDict
osDistLevel = OSDist + slesVersion + '.' + slesPatchLevel
else:
try:
rhelVersion = re.match('.*release\s+([6-7]{1}.[0-9]{1}).*',
releaseInfo, re.IGNORECASE).group(1)
except AttributeError as err:
logger.error(
"There was RHEL OS version match error when trying to match against '"
+ releaseInfo + "':\n" + str(err) + ".")
resultDict['errorMessages'].append(
"There was a RHEL OS version match error.")
return resultDict
osDistLevel = OSDist + rhelVersion
try:
if osDistLevel not in self.csurResourceDict[
'supportedDistributionLevels']:
logger.error("The system's OS distribution level (" +
osDistLevel +
") is not supported by this CSUR bundle.")
resultDict['errorMessages'].append(
"The system's OS distribution level is not supported by this CSUR bundle."
)
return resultDict
except KeyError as err:
logger.error("The resource key (" + str(err) +
") was not present in the resource file.")
resultDict['errorMessages'].append(
"A resource key error was encountered.")
return resultDict
logger.debug(
"The system's OS distribution level was determined to be: " +
osDistLevel + ".")
self.computeNodeDict['osDistLevel'] = osDistLevel
'''
Whenever any type of update is being performed we ensure certain prerequisites are met, e.g. the HANA application
is not running.
'''
if not versionInformationLogOnly:
#On NFS servers we want to check and notify if the cluster is still running.
if 'DL380' in systemModel:
command = '/opt/cmcluster/bin/cmviewcl -f line'
result = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
out, err = result.communicate()
logger.debug(
"The output of the command (" + command +
") used to check if the cluster is running was: " +
out.strip())
if result.returncode != 0:
logger.warn(
"Unable to check if the cluster is running.\n" + err)
resultDict['errorMessages'].append(
"Unable to check if the cluster is running.")
clusterView = out.splitlines()
for line in clusterView:
if re.search('^status=', line):
if re.match('status=up', line):
logger.warn(
"It appears that the cluster is still running.\n"
+ out)
resultDict['errorMessages'].append(
"It appears that the cluster is still running."
)
'''
On servers that are not Serviceguard we check if SAP HANA is running and notify if it is running.
We can assume that processes are not running is we get a return code of 1.
'''
if not ('DL380' in systemModel or 'DL320' in systemModel):
command = 'ps -C hdbnameserver,hdbcompileserver,hdbindexserver,hdbpreprocessor,hdbxsengine,hdbwebdispatcher'
result = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
out, err = result.communicate()
logger.debug("The output of the command (" + command +
") used to check if SAP is running was: " +
out.strip())
if result.returncode == 0:
logger.warn(
"It appears that SAP HANA is still running.\n" + out)
resultDict['errorMessages'].append(
"It appears that SAP HANA is still running.")
'''
Additional kernel version and processorType is needed for FusionIO systems when the driver is rebuilt.
Also, need to confirm the system is already at a supported firmware version for automatic upgrades.
'''
if systemModel == 'DL580G7' or systemModel == 'DL980G7':
#Check if /hana/log or /HANA/IMDB-log is mounted.
command = 'mount'
result = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
out, err = result.communicate()
logger.debug(
"The output of the command (" + command +
") used to check if the log partition is mounted was: " +
out.strip())
if result.returncode != 0:
logger.error(
"Unable to check if the log partition is mounted.\n" +
err)
resultDict['errorMessages'].append(
"Unable to check if the log partition is mounted.")
return resultDict
if re.search('/hana/log|/HANA/IMDB-log', out,
re.MULTILINE | re.DOTALL) != None:
logger.error("The log partition is still mounted.")
resultDict['errorMessages'].append(
"The log partition needs to be unmounted before the system is updated."
)
return resultDict
#Get the currently used kernel and processor type, which is used as part of the FusionIO driver RPM name.
command = 'uname -r'
result = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
out, err = result.communicate()
kernel = out.strip()
logger.debug("The output of the command (" + command +
") used to get the currently used kernel was: " +
kernel)
if result.returncode != 0:
logger.error(
"Unable to get the system's current kernel information.\n"
+ err)
resultDict['errorMessages'].append(
"Unable to get the system's current kernel information."
)
return resultDict
logger.debug(
"The currently used kernel was determined to be: " +
kernel + ".")
self.computeNodeDict['kernel'] = kernel
command = 'uname -p'
result = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
out, err = result.communicate()
processorType = out.strip()
logger.debug(
"The output of the command (" + command +
") used to get the compute node's processor type was: " +
processorType)
if result.returncode != 0:
logger.error(
"Unable to get the system's processor type.\n" + err)
resultDict['errorMessages'].append(
"Unable to get the system's processor type.")
return resultDict
logger.debug(
"The compute node's processor type was determined to be: "
+ processorType + ".")
self.computeNodeDict['processorType'] = processorType
try:
if not checkFusionIOFirmwareUpgradeSupport(
self.
csurResourceDict['fusionIOFirmwareVersionList'],
self.loggerName):
resultDict['errorMessages'].append(
"The fusionIO firmware is not at a supported version for an automatic upgrade."
)
return resultDict
except KeyError as err:
logger.error("The resource key (" + str(err) +
") was not present in the resource file.")
resultDict['errorMessages'].append(
"A resource key error was encountered.")
return resultDict
#Don't need to check drivers if only doing an OS hard drive firmware update.
if not updateOSHarddrives:
#Confirm that the drivers for the system being updated are loaded.
result = self.__checkDrivers(computeNodeResources, systemModel,
osDistLevel)
if result != '':
resultDict['errorMessages'].append(result)
return resultDict
#Instantiate the compute node inventory class which is used to get a list of compute node components that need to be updated.
try:
if (systemModel == 'DL580G7' or systemModel == 'DL980G7'):
computeNodeInventory = Gen1ScaleUpComputeNodeInventory(
self.computeNodeDict.copy(),
self.csurResourceDict['noPMCFirmwareUpdateModels'],
computeNodeResources, self.
csurResourceDict['fusionIOSoftwareInstallPackageList'])
elif systemModel == 'DL380pGen8' and 'systemGeneration' in self.csurResourceDict and self.csurResourceDict[
'systemGeneration'] == 'Gen1.x':
computeNodeInventory = ComputeNodeInventory(
self.computeNodeDict.copy(),
self.csurResourceDict['noPMCFirmwareUpdateModels'],
computeNodeResources,
systemGeneration='Gen1.x')
else:
computeNodeInventory = ComputeNodeInventory(
self.computeNodeDict.copy(),
self.csurResourceDict['noPMCFirmwareUpdateModels'],
computeNodeResources)
except KeyError as err:
logger.error("The resource key (" + str(err) +
") was not present in the resource file.")
resultDict['errorMessages'].append(
"A resource key error was encountered.")
return resultDict
#Inventory the compute node to determine which components need to be updated.
if not updateOSHarddrives:
computeNodeInventory.getComponentUpdateInventory()
else:
hardDrivesLocal = computeNodeInventory.getLocalHardDriveFirmwareInventory(
)
if hardDrivesLocal != None and not hardDrivesLocal:
resultDict['errorMessages'].append(
"There are no local hard drives to update, since there were no controllers detected."
)
return resultDict
if computeNodeInventory.getInventoryStatus():
resultDict['errorMessages'].append(
"Errors were encountered during the compute node's inventory.")
return resultDict
#The resultDict will not be used, but the inventory is done which generated the version report.
if versionInformationLogOnly:
return resultDict
componentUpdateDict = computeNodeInventory.getComponentUpdateDict()
'''
This gets a list of the dictionary sizes (Software, Drivers, Firmwware) so that
it can be determined whether or not an update is needed.
'''
if updateOSHarddrives:
if len(componentUpdateDict['Firmware']) != 0:
self.computeNodeDict[
'componentUpdateDict'] = componentUpdateDict
resultDict['updateNeeded'] = True
else:
componentDictSizes = [
len(dict) for dict in componentUpdateDict.values()
]
if any(x != 0 for x in componentDictSizes):
self.computeNodeDict[
'componentUpdateDict'] = componentUpdateDict
resultDict['updateNeeded'] = True
if 'FusionIO' in componentUpdateDict['Firmware']:
self.computeNodeDict[
'busList'] = computeNodeInventory.getFusionIOBusList()
self.computeNodeDict[
'externalStoragePresent'] = computeNodeInventory.isExternalStoragePresent(
)
return resultDict
def init(self, csurBasePath, loggerName):
if os.geteuid() != 0:
print RED + "You must be root to run this program; exiting program execution.\n" + RESETCOLORS
exit(1)
usage = 'usage: %prog [[-d] [-h] [-v]]'
parser = optparse.OptionParser(usage=usage)
parser.add_option(
'-d',
action='store_true',
default=False,
help=
'This option is used when problems are encountered and additional debug information is needed.'
)
parser.add_option(
'-v',
action='store_true',
default=False,
help='This option is used to display the application\'s version.')
(options, args) = parser.parse_args()
self.__printHeader()
print GREEN + "Phase 1: Initializing for the system update.\n" + RESETCOLORS
self.__getConfigurationType()
csurResourceDict = {}
#Save the csur base path to the csur resource dict as it will be needed later.
csurResourceDict['csurBasePath'] = csurBasePath
csurAppResourceFile = csurBasePath + '/resourceFiles/csurAppResourceFile'
csurDataResourceFile = csurBasePath + '/resourceFiles/csurDataResourceFile'
#Get csur application's resource file data and save it to a dictionary (hash).
try:
with open(csurAppResourceFile) as f:
for line in f:
line = line.strip()
#Remove quotes from resources.
line = re.sub('[\'"]', '', line)
#Ignore commented and blank lines.
if len(line) == 0 or re.match("^#", line):
continue
else:
(key, val) = line.split('=')
key = re.sub('\s+', '', key)
csurResourceDict[key] = val.lstrip()
except IOError as err:
print RED + "Unable to access the application's resource file " + csurAppResourceFile + "; exiting program execution.\n" + str(
err) + "\n" + RESETCOLORS
exit(1)
#Get the csur's resource data (software, firmware, drivers) and save it to a list (array).
try:
with open(csurDataResourceFile) as f:
csurData = f.read().splitlines()
except IOError:
print RED + "Unable to open " + csurDataResourceFile + " for reading; exiting program execution.\n" + RESETCOLORS
exit(1)
csurResourceDict['csurData'] = csurData
#Get the application's log file information.
try:
logBaseDir = csurBasePath + '/log'
csurApplicationLog = logBaseDir + '/' + csurResourceDict[
'csurApplicationLog']
except KeyError as err:
print RED + "The resource key (" + str(
err
) + ") was not present in the resource file; exiting program execution.\n" + RESETCOLORS
exit(1)
try:
#Always start with an empty log directory when performing a new update.
logList = os.listdir(logBaseDir)
for log in logList:
os.remove(logBaseDir + '/' + log)
except OSError as err:
print RED + "Unable to remove old logs in " + logBaseDir + "; exiting program execution.\n" + str(
err) + "\n" + RESETCOLORS
exit(1)
#Configure logging
handler = logging.FileHandler(csurApplicationLog)
logger = logging.getLogger(loggerName)
if options.d:
logger.setLevel(logging.DEBUG)
else:
logger.setLevel(logging.INFO)
formatter = logging.Formatter('%(asctime)s:%(levelname)s:%(message)s',
datefmt='%m/%d/%Y %H:%M:%S')
handler.setFormatter(formatter)
logger.addHandler(handler)
#Get the system's OS distribution version information.
command = "cat /proc/version"
result = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
out, err = result.communicate()
#Change version information to lowercase before checking for OS type.
versionInfo = out.lower()
if result.returncode != 0:
logger.error(
"Unable to get the system's OS distribution version information.\n"
+ str(err))
print RED + "Unable to get the system's OS distribution version information; check the log file for errors; exiting program execution.\n" + RESETCOLORS
exit(1)
if 'suse' in versionInfo:
OSDist = 'SLES'
command = "cat /etc/SuSE-release"
else:
OSDist = 'RHEL'
command = "cat /etc/redhat-release"
result = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
out, err = result.communicate()
if result.returncode != 0:
logger.error(
"Unable to get the system's OS distribution level.\n" +
str(err))
print RED + "Unable to get the system's OS distribution level; check the log file for errors; exiting program execution.\n" + RESETCOLORS
exit(1)
else:
releaseInfo = out.replace('\n', ' ')
if OSDist == 'SLES':
slesVersion = re.match('.*version\s*=\s*([1-4]{2})',
releaseInfo, re.IGNORECASE).group(1)
slesPatchLevel = re.match('.*patchlevel\s*=\s*([1-4]{1})',
releaseInfo, re.IGNORECASE).group(1)
osDistLevel = OSDist + slesVersion + '.' + slesPatchLevel
else:
rhelVersion = re.match('.*release\s+([6-7]{1}.[0-9]{1}).*',
releaseInfo, re.IGNORECASE).group(1)
osDistLevel = OSDist + rhelVersion
csurResourceDict['osDistLevel'] = osDistLevel
if osDistLevel not in csurResourceDict['supportedDistributionLevels']:
logger.error("The system's OS distribution level (" + osDistLevel +
") is not supported by this CSUR bundle.")
logger.info("The supported OS distribution levels are (" +
csurResourceDict['supportedDistributionLevels'] + ").")
print RED + "The system's OS distribution level is not supported by this CSUR bundle; check the log file for errors; exiting program execution.\n" + RESETCOLORS
exit(1)
#Get system model.
command = "dmidecode -s system-product-name"
result = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
out, err = result.communicate()
if result.returncode != 0:
logger.error("Unable to get the system's model information.\n" +
str(err))
print RED + "Unable to get the system's model information; check the log file for errors; exiting program execution.\n" + RESETCOLORS
exit(1)
systemModel = (re.match('[a-z,0-9]+\s+(.*)', out,
re.IGNORECASE).group(1)).replace(' ', '')
if systemModel not in csurResourceDict['supportedComputeNodeModels']:
logger.error("The system's model (" + systemModel +
") is not supported by this CSUR bundle.")
logger.info("The supported supported models are (" +
csurResourceDict['supportedComputeNodeModels'] + ").")
print RED + "The system's model is not supported by this CSUR bundle; check the log file for errors; exiting program execution.\n" + RESETCOLORS
exit(1)
csurResourceDict['systemModel'] = systemModel
#FusionIO should only be installed on Gen 1.0 Scale-up systems.
if systemModel == 'DL580G7' or systemModel == 'DL980G7':
fioStatus = '/usr/bin/fio-status'
if os.path.exists(fioStatus):
command = fioStatus + ' -c'
result = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
out, err = result.communicate()
logger.debug(
"The output of the command (" + command +
") used to determine the number of FusionIO cards was: " +
out.strip())
if result.returncode == 0:
if out.strip() == 0:
logger.info(
"fio-status was present, but it appears the system does not have any FusionIO cards.\n"
)
csurResourceDict['FusionIOSubsystem'] = 'no'
else:
csurResourceDict['FusionIOSubsystem'] = 'yes'
else:
logger.error(
"Unable to determine the number of FusionIO cards installed.\n"
+ str(err))
print RED + "Unable to determine the number of FusionIO cards installed; check the log file for errors; exiting program execution.\n" + RESETCOLORS
exit(1)
else:
logger.info(
"The compute node was a " + systemModel +
", however its FusionIO status could not be determined (/usr/bin/fio-status missing)."
)
csurResourceDict['FusionIOSubsystem'] = 'no'
if csurResourceDict['FusionIOSubsystem'] == 'yes':
#Get the currently used kernel and processor type, which is used as part of the FusionIO driver RPM name.
command = 'uname -r'
result = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
out, err = result.communicate()
logger.debug("The output of the command (" + command +
") used to get the currently used kernel was: " +
out.strip())
if result.returncode != 0:
logger.error(
"Unable to get the system's current kernel information.\n"
+ err)
print RED + "Unable to get the system's current kernel information; check the log file for errors; exiting program execution.\n" + RESETCOLORS
else:
kernel = out.strip()
csurResourceDict['kernel'] = kernel
command = 'uname -p'
result = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
out, err = result.communicate()
logger.debug(
"The output of the command (" + command +
") used to get the system's processor type was: " +
out.strip())
if result.returncode != 0:
logger.error(
"Unable to get the system's processor type.\n" + err)
print RED + "Unable to get the system's processor type; check the log file for errors; exiting program execution.\n" + RESETCOLORS
else:
processorType = out.strip()
csurResourceDict['processorType'] = processorType
if not checkFusionIOFirmwareUpgradeSupport(
csurResourceDict['fusionIOFirmwareVersionList'],
loggerName):
print RED + "The fusionIO firmware is not at a supported version for an automatic upgrade; upgrade the firmware manually and try again; exiting program execution.\n" + RESETCOLORS
return csurResourceDict
def computeNodeInitialize(self, computeNodeResources,
versionInformationLogOnly, updateOSHarddrives):
logger = logging.getLogger(self.loggerName)
resultDict = {'updateNeeded': False, 'errorMessages': []}
command = 'dmidecode -s system-product-name'
result = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
out, err = result.communicate()
out = out.strip()
logger.debug('The output of the command (' + command +
") used to get the system's model was: " + out)
if result.returncode != 0:
logger.error("Unable to get the system's model information.\n" +
err)
resultDict['errorMessages'].append(
"Unable to get the system's model information.")
return resultDict
else:
try:
systemModel = re.match('[a-z,0-9]+\\s+(.*)', out,
re.IGNORECASE).group(1).replace(
' ', '')
except AttributeError as err:
logger.error(
"There was a system model match error when trying to match against '"
+ out + "':\n" + str(err) + '.')
resultDict['errorMessages'].append(
'There was a system model match error.')
return resultDict
try:
if systemModel not in self.healthResourceDict[
'supportedComputeNodeModels']:
logger.error("The system's model (" + systemModel +
') is not supported by this CSUR bundle.')
resultDict['errorMessages'].append(
"The system's model is not supported by this CSUR bundle."
)
return resultDict
except KeyError as err:
logger.error(
'The resource key (' + str(err) +
") was not present in the application's esource file.")
resultDict['errorMessages'].append(
'A resource key error was encountered.')
return resultDict
logger.debug("The system's model was determined to be: " +
systemModel + '.')
self.computeNodeDict['systemModel'] = systemModel
if systemModel == 'DL580Gen9':
command = 'dmidecode -s processor-version'
result = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
out, err = result.communicate()
out = out.strip()
logger.debug('The output of the command (' + command +
') used to get the processor version was: ' + out)
if result.returncode != 0:
logger.error('Unable to get the processor version.\n' +
err)
resultDict['errorMessages'].append(
'Unable to get the processor version information.')
return resultDict
try:
processorVersion = re.search(
'CPU (E\\d-\\s*\\d{4}\\w* v\\d)',
out).group(1).replace(' ', '')
except AttributeError as err:
logger.error(
"There was a processor match error when trying to match against '"
+ out + "':\n" + str(err) + '.')
resultDict['errorMessages'].append(
'There was a processor match error.')
return resultDict
logger.debug("The processor's version was determined to be: " +
processorVersion + '.')
else:
processorVersion = ''
self.computeNodeDict['processorVersion'] = processorVersion
command = 'cat /proc/version'
result = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
out, err = result.communicate()
logger.debug(
'The output of the command (' + command +
') used to get the OS distribution information was: ' +
out.strip())
if result.returncode != 0:
logger.error(
"Unable to get the system's OS distribution version information.\n"
+ err)
resultDict['errorMessages'].append(
"Unable to get the system's OS distribution version information."
)
return resultDict
versionInfo = out.lower()
if 'suse' in versionInfo:
OSDist = 'SLES'
command = 'cat /etc/SuSE-release'
else:
OSDist = 'RHEL'
command = 'cat /etc/redhat-release'
result = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
out, err = result.communicate()
if result.returncode != 0:
logger.error(
"Unable to get the system's OS distribution level.\n" +
err)
resultDict['errorMessages'].append(
"Unable to get the system's OS distribution level.")
return resultDict
releaseInfo = out.replace('\n', ' ')
if OSDist == 'SLES':
try:
slesVersion = re.match('.*version\\s*=\\s*([1-4]{2})',
releaseInfo, re.IGNORECASE).group(1)
except AttributeError as err:
logger.error(
"There was SLES OS version match error when trying to match against '"
+ releaseInfo + "':\n" + str(err) + '.')
resultDict['errorMessages'].append(
'There was a SLES OS version match error.')
return resultDict
try:
slesPatchLevel = re.match(
'.*patchlevel\\s*=\\s*([1-4]{1})', releaseInfo,
re.IGNORECASE).group(1)
except AttributeError as err:
logger.error(
"There was SLES patch level match error when trying to match against '"
+ releaseInfo + "':\n" + str(err) + '.')
resultDict['errorMessages'].append(
'There was a SLES patch level match error.')
return resultDict
osDistLevel = OSDist + slesVersion + '.' + slesPatchLevel
else:
try:
rhelVersion = re.match(
'.*release\\s+([6-7]{1}.[0-9]{1}).*', releaseInfo,
re.IGNORECASE).group(1)
except AttributeError as err:
logger.error(
"There was RHEL OS version match error when trying to match against '"
+ releaseInfo + "':\n" + str(err) + '.')
resultDict['errorMessages'].append(
'There was a RHEL OS version match error.')
return resultDict
osDistLevel = OSDist + rhelVersion
try:
if osDistLevel not in self.healthResourceDict[
'supportedDistributionLevels']:
if osDistLevel not in self.healthResourceDict[
'unsupportedUpgradableDistLevels']:
logger.error("The system's OS distribution level (" +
osDistLevel +
') is not supported by this CSUR bundle.')
resultDict['errorMessages'].append(
"The system's OS distribution level is not supported by this CSUR bundle."
)
return resultDict
except KeyError as err:
logger.error('The resource key (' + str(err) +
') was not present in the resource file.')
resultDict['errorMessages'].append(
'A resource key error was encountered.')
return resultDict
logger.debug(
"The system's OS distribution level was determined to be: " +
osDistLevel + '.')
self.computeNodeDict['osDistLevel'] = osDistLevel
if not versionInformationLogOnly:
if 'DL380' in systemModel:
command = '/opt/cmcluster/bin/cmviewcl -f line'
result = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
out, err = result.communicate()
logger.debug(
'The output of the command (' + command +
') used to check if the cluster is running was: ' +
out.strip())
if result.returncode != 0:
logger.warn(
'Unable to check if the cluster is running.\n' +
err)
resultDict['errorMessages'].append(
'Unable to check if the cluster is running.')
clusterView = out.splitlines()
for line in clusterView:
if re.search('^status=', line):
if re.match('status=up', line):
logger.warn(
'It appears that the cluster is still running.\n'
+ out)
resultDict['errorMessages'].append(
'It appears that the cluster is still running.'
)
if not 'DL380' in systemModel:
if not 'DL320' in systemModel:
command = 'ps -C hdbnameserver,hdbcompileserver,hdbindexserver,hdbpreprocessor,hdbxsengine,hdbwebdispatcher'
result = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
out, err = result.communicate()
logger.debug(
'The output of the command (' + command +
') used to check if SAP is running was: ' +
out.strip())
if result.returncode == 0:
logger.warn(
'It appears that SAP HANA is still running.\n'
+ out)
resultDict['errorMessages'].append(
'It appears that SAP HANA is still running.')
if systemModel == 'DL580G7' or systemModel == 'DL980G7':
command = 'mount'
result = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
out, err = result.communicate()
logger.debug(
'The output of the command (' + command +
') used to check if the log partition is mounted was: '
+ out.strip())
if result.returncode != 0:
logger.error(
'Unable to check if the log partition is mounted.\n'
+ err)
resultDict['errorMessages'].append(
'Unable to check if the log partition is mounted.'
)
return resultDict
if re.search('/hana/log|/HANA/IMDB-log', out,
re.MULTILINE | re.DOTALL) != None:
logger.error('The log partition is still mounted.')
resultDict['errorMessages'].append(
'The log partition needs to be unmounted before the system is updated.'
)
return resultDict
command = 'uname -r'
result = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
out, err = result.communicate()
kernel = out.strip()
logger.debug(
'The output of the command (' + command +
') used to get the currently used kernel was: ' +
kernel)
if result.returncode != 0:
logger.error(
"Unable to get the system's current kernel information.\n"
+ err)
resultDict['errorMessages'].append(
"Unable to get the system's current kernel information."
)
return resultDict
logger.debug(
'The currently used kernel was determined to be: '
+ kernel + '.')
self.computeNodeDict['kernel'] = kernel
command = 'uname -p'
result = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
out, err = result.communicate()
processorType = out.strip()
logger.debug(
'The output of the command (' + command +
") used to get the compute node's processor type was: "
+ processorType)
if result.returncode != 0:
logger.error(
"Unable to get the system's processor type.\n"
+ err)
resultDict['errorMessages'].append(
"Unable to get the system's processor type.")
return resultDict
logger.debug(
"The compute node's processor type was determined to be: "
+ processorType + '.')
self.computeNodeDict['processorType'] = processorType
try:
if not checkFusionIOFirmwareUpgradeSupport(
self.healthResourceDict[
'fusionIOFirmwareVersionList'],
self.loggerName):
resultDict['errorMessages'].append(
'The fusionIO firmware is not at a supported version for an automatic upgrade.'
)
return resultDict
except KeyError as err:
logger.error(
'The resource key (' + str(err) +
') was not present in the resource file.')
resultDict['errorMessages'].append(
'A resource key error was encountered.')
return resultDict
if not updateOSHarddrives:
result = self.__checkDrivers(computeNodeResources,
systemModel, osDistLevel)
if result != '':
resultDict['errorMessages'].append(result)
return resultDict
try:
if systemModel == 'DL580G7' or systemModel == 'DL980G7':
computeNodeInventory = Gen1ScaleUpComputeNodeInventory(
self.computeNodeDict.copy(), self.
healthResourceDict['noPMCFirmwareUpdateModels'],
computeNodeResources,
self.healthResourceDict.copy())
else:
computeNodeInventory = ComputeNodeInventory(
self.computeNodeDict.copy(), self.
healthResourceDict['noPMCFirmwareUpdateModels'],
computeNodeResources,
self.healthResourceDict.copy())
except KeyError as err:
logger.error('The resource key (' + str(err) +
') was not present in the resource file.')
resultDict['errorMessages'].append(
'A resource key error was encountered.')
return resultDict
if not updateOSHarddrives:
computeNodeInventory.getComponentUpdateInventory()
else:
computeNodeInventory.getLocalHardDriveFirmwareInventory()
if computeNodeInventory.getInventoryStatus():
resultDict['errorMessages'].append(
"Errors were encountered during the compute node's inventory."
)
return resultDict
if versionInformationLogOnly:
if os.path.isfile(self.versionInformationLog):
try:
shutil.copy(self.versionInformationLog,
self.healthBasePath)
except IOError as err:
self.logger.error('I/O Error while copy of ' +
self.versionInformationLog +
' to ' + self.healthBasePath)
return resultDict
componentUpdateDict = computeNodeInventory.getComponentUpdateDict(
)
if updateOSHarddrives:
self.computeNodeDict['componentUpdateDict'] = len(
componentUpdateDict['Firmware']
) != 0 and componentUpdateDict
resultDict['updateNeeded'] = True
else:
componentDictSizes = [
len(dict) for dict in componentUpdateDict.values()
]
if any((x != 0 for x in componentDictSizes)):
self.computeNodeDict[
'componentUpdateDict'] = componentUpdateDict
resultDict['updateNeeded'] = True
if 'FusionIO' in componentUpdateDict['Firmware']:
self.computeNodeDict[
'busList'] = computeNodeInventory.getFusionIOBusList(
)
self.computeNodeDict[
'externalStoragePresent'] = computeNodeInventory.isExternalStoragePresent(
)
return resultDict
def computeNodeInitialize(self, computeNodeResources,
versionInformationLogOnly, updateOSHarddrives):
logger = logging.getLogger(self.loggerName)
resultDict = {
'updateNeeded': False,
'errorMessages': [],
'hardDrivesMissingFirmware': ''
}
command = 'dmidecode -s system-product-name'
result = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
out, err = result.communicate()
out = out.strip()
logger.info('The output of the command (' + command +
") used to get the system's model was: " + out)
if result.returncode != 0:
logger.error("Unable to get the system's model information.\n" +
err)
resultDict['errorMessages'].append(
"Unable to get the system's model information.")
return resultDict
else:
try:
systemModel = re.match('[a-z,0-9]+\\s+(.*)', out,
re.IGNORECASE).group(1).replace(
' ', '')
except AttributeError as err:
logger.error(
"There was a system model match error when trying to match against '"
+ out + "':\n" + str(err) + '.')
resultDict['errorMessages'].append(
'There was a system model match error.')
return resultDict
try:
if systemModel not in self.csurResourceDict[
'supportedComputeNodeModels']:
logger.error("The system's model (" + systemModel +
') is not supported by this CSUR bundle.')
resultDict['errorMessages'].append(
"The system's model is not supported by this CSUR bundle."
)
return resultDict
except KeyError as err:
logger.error(
'The resource key (' + str(err) +
") was not present in the application's resource file.")
resultDict['errorMessages'].append(
'A resource key error was encountered.')
return resultDict
logger.info("The system's model was determined to be: " +
systemModel + '.')
if '16s' in systemModel:
logger.info(
'The system was determined to be a Superdome2 compute node. Checking its CPU model.'
)
processorDict = {
'62': 'ivybridge',
'63': 'haswell',
'79': 'broadwell'
}
command = 'cat /proc/cpuinfo'
result = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
out, err = result.communicate()
logger.info('The output of the command (' + command +
") used to get the system's CPU model was: " +
out.strip())
if result.returncode != 0:
logger.error(
'There was a problem getting the cpu information.\n' +
err + '\n' + out)
resultDict['errorMessages'].append(
'Unable to get the cpu information.')
cpudata = out.splitlines()
for line in cpudata:
if re.match('\\s*model\\s+:\\s+[2-9]{2}', line) != None:
try:
processor = processorDict[re.match(
'\\s*model\\s+:\\s+([2-9]{2})', line).group(1)]
except AttributeError as err:
logger.error(
"There was a match error when trying to match against '"
+ line + "'.\n" + str(err))
resultDict['errorMessages'].append(
'There was a CPU model match error.')
break
logger.info(
"The server's processor type was determined to be: " +
processor + '.')
if processor == 'ivybridge':
systemModel = systemModel + 'Gen8'
elif processor == 'haswell' or processor == 'broadwell':
systemModel = systemModel + 'Gen9'
logger.info("The system's generation was determined to be: " +
systemModel + '.')
self.computeNodeDict['systemModel'] = systemModel
command = 'cat /proc/version'
result = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
out, err = result.communicate()
logger.info('The output of the command (' + command +
') used to get the OS distribution information was: ' +
out.strip())
if result.returncode != 0:
logger.error(
"Unable to get the system's OS distribution version information.\n"
+ err)
resultDict['errorMessages'].append(
"Unable to get the system's OS distribution version information."
)
return resultDict
versionInfo = out.lower()
if 'suse' in versionInfo:
OSDist = 'SLES'
command = 'cat /etc/SuSE-release'
else:
OSDist = 'RHEL'
command = 'cat /etc/redhat-release'
result = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
out, err = result.communicate()
if result.returncode != 0:
logger.error(
"Unable to get the system's OS distribution level.\n" +
err)
resultDict['errorMessages'].append(
"Unable to get the system's OS distribution level.")
return resultDict
releaseInfo = out.replace('\n', ' ')
if OSDist == 'SLES':
try:
slesVersion = re.match('.*version\\s*=\\s*([1-4]{2})',
releaseInfo, re.IGNORECASE).group(1)
except AttributeError as err:
logger.error(
"There was SLES OS version match error when trying to match against '"
+ releaseInfo + "':\n" + str(err) + '.')
resultDict['errorMessages'].append(
'There was a SLES OS version match error.')
return resultDict
try:
slesPatchLevel = re.match(
'.*patchlevel\\s*=\\s*([1-4]{1})', releaseInfo,
re.IGNORECASE).group(1)
except AttributeError as err:
logger.error(
"There was SLES patch level match error when trying to match against '"
+ releaseInfo + "':\n" + str(err) + '.')
resultDict['errorMessages'].append(
'There was a SLES patch level match error.')
return resultDict
osDistLevel = OSDist + slesVersion + '.' + slesPatchLevel
else:
try:
rhelVersion = re.match(
'.*release\\s+([6-7]{1}.[0-9]{1}).*', releaseInfo,
re.IGNORECASE).group(1)
except AttributeError as err:
logger.error(
"There was RHEL OS version match error when trying to match against '"
+ releaseInfo + "':\n" + str(err) + '.')
resultDict['errorMessages'].append(
'There was a RHEL OS version match error.')
return resultDict
osDistLevel = OSDist + rhelVersion
try:
if osDistLevel not in self.csurResourceDict[
'supportedDistributionLevels']:
logger.error("The system's OS distribution level (" +
osDistLevel +
') is not supported by this CSUR bundle.')
resultDict['errorMessages'].append(
"The system's OS distribution level is not supported by this CSUR bundle."
)
return resultDict
except KeyError as err:
logger.error('The resource key (' + str(err) +
') was not present in the resource file.')
resultDict['errorMessages'].append(
'A resource key error was encountered.')
return resultDict
logger.info(
"The system's OS distribution level was determined to be: " +
osDistLevel + '.')
self.computeNodeDict['osDistLevel'] = osDistLevel
if not versionInformationLogOnly:
if 'DL380' in systemModel or 'DL360' in systemModel:
if 'SLES' in osDistLevel:
sgBinPath = '/opt/cmcluster/bin'
else:
sgBinPath = '/usr/local/cmcluster/bin'
command = sgBinPath + '/cmviewcl -f line -l cluster'
result = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
out, err = result.communicate()
logger.info(
'The output of the command (' + command +
') used to check if the cluster is running was: ' +
out.strip())
if result.returncode != 0:
logger.error(
'Unable to check if the cluster is running.\n' +
err)
resultDict['errorMessages'].append(
'Unable to check if the cluster is running.')
return resultDict
clusterView = out.splitlines()
for line in clusterView:
if re.search('^status=', line):
if re.match('status=up', line):
logger.error(
'It appears that the cluster is still running.\n'
+ out.strip())
resultDict['errorMessages'].append(
'It appears that the cluster is still running.'
)
return resultDict
command = sgBinPath + '/cmversion'
result = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
out, err = result.communicate()
out = out.strip()
logger.info('The output of the command (' + command +
") used to get Serviceguard's version was: " +
out)
if result.returncode != 0:
logger.error(
"Unable to get Serviceguard's version.\n" + err)
resultDict['errorMessages'].append(
"Unable to get Serviceguard's version.")
return resultDict
sgVersion = out[0:7]
try:
if sgVersion not in self.csurResourceDict[
'supportedServiceguardLevels']:
logger.error(
'The current version of Serviceguard ' + out +
' is not supported for an upgrade.')
resultDict['errorMessages'].append(
'The current version of Serviceguard is not supported.'
)
return resultDict
except KeyError as err:
logger.error('The resource key (' + str(err) +
') was not present in the resource file.')
resultDict['errorMessages'].append(
'A resource key error was encountered.')
return resultDict
if not 'DL380' in systemModel:
if not 'DL320' in systemModel:
if not 'DL360' in systemModel:
command = 'ps -C hdbnameserver,hdbcompileserver,hdbindexserver,hdbpreprocessor,hdbxsengine,hdbwebdispatcher'
result = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
out, err = result.communicate()
logger.info(
'The output of the command (' + command +
') used to check if SAP is running was: ' +
out.strip())
if result.returncode == 0:
logger.warn(
'It appears that SAP HANA is still running.\n'
+ out)
resultDict['errorMessages'].append(
'It appears that SAP HANA is still running.'
)
return resultDict
if systemModel == 'DL580G7' or systemModel == 'DL980G7':
command = 'mount'
result = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
out, err = result.communicate()
logger.info(
'The output of the command (' + command +
') used to check if the log partition is mounted was: '
+ out.strip())
if result.returncode != 0:
logger.error(
'Unable to check if the log partition is mounted.\n'
+ err)
resultDict['errorMessages'].append(
'Unable to check if the log partition is mounted.'
)
return resultDict
if re.search('/hana/log|/HANA/IMDB-log', out,
re.MULTILINE | re.DOTALL) != None:
logger.error(
'The log partition is still mounted.')
resultDict['errorMessages'].append(
'The log partition needs to be unmounted before the system is updated.'
)
return resultDict
command = 'uname -r'
result = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
out, err = result.communicate()
kernel = out.strip()
logger.info(
'The output of the command (' + command +
') used to get the currently used kernel was: '
+ kernel)
if result.returncode != 0:
logger.error(
"Unable to get the system's current kernel information.\n"
+ err)
resultDict['errorMessages'].append(
"Unable to get the system's current kernel information."
)
return resultDict
logger.info(
'The currently used kernel was determined to be: '
+ kernel + '.')
self.computeNodeDict['kernel'] = kernel
command = 'uname -p'
result = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
out, err = result.communicate()
processorType = out.strip()
logger.info(
'The output of the command (' + command +
") used to get the compute node's processor type was: "
+ processorType)
if result.returncode != 0:
logger.error(
"Unable to get the system's processor type.\n"
+ err)
resultDict['errorMessages'].append(
"Unable to get the system's processor type."
)
return resultDict
logger.info(
"The compute node's processor type was determined to be: "
+ processorType + '.')
self.computeNodeDict[
'processorType'] = processorType
try:
if not checkFusionIOFirmwareUpgradeSupport(
self.csurResourceDict[
'fusionIOFirmwareVersionList'],
self.loggerName):
resultDict['errorMessages'].append(
'The fusionIO firmware is not at a supported version for an automatic upgrade.'
)
return resultDict
except KeyError as err:
logger.error(
'The resource key (' + str(err) +
') was not present in the resource file.')
resultDict['errorMessages'].append(
'A resource key error was encountered.')
return resultDict
if not updateOSHarddrives and '16s' not in systemModel:
result = self.__checkDrivers(computeNodeResources,
systemModel, osDistLevel)
if result != '':
resultDict['errorMessages'].append(result)
return resultDict
try:
if systemModel == 'DL580G7' or systemModel == 'DL980G7':
computeNodeInventory = Gen1ScaleUpComputeNodeInventory(
self.computeNodeDict.copy(), self.
csurResourceDict['noPMCFirmwareUpdateModels'],
computeNodeResources)
elif systemModel == 'DL380pGen8' and 'systemGeneration' in self.csurResourceDict and self.csurResourceDict[
'systemGeneration'] == 'Gen1.x':
computeNodeInventory = ComputeNodeInventory(
self.computeNodeDict.copy(),
self.
csurResourceDict['noPMCFirmwareUpdateModels'],
computeNodeResources,
systemGeneration='Gen1.x')
else:
computeNodeInventory = ComputeNodeInventory(
self.computeNodeDict.copy(), self.
csurResourceDict['noPMCFirmwareUpdateModels'],
computeNodeResources)
except KeyError as err:
logger.error('The resource key (' + str(err) +
') was not present in the resource file.')
resultDict['errorMessages'].append(
'A resource key error was encountered.')
return resultDict
if not updateOSHarddrives:
computeNodeInventory.getComponentUpdateInventory(
self.csurResourceDict.copy())
else:
hardDrivesLocal = computeNodeInventory.getLocalHardDriveFirmwareInventory(
)
if hardDrivesLocal != None and not hardDrivesLocal:
resultDict['errorMessages'].append(
'There are no local hard drives to update, since there were no controllers detected.'
)
return resultDict
if computeNodeInventory.getInventoryStatus():
resultDict['errorMessages'].append(
"Errors were encountered during the compute node's inventory."
)
return resultDict
if computeNodeInventory.getHardDriveFirmwareStatus():
resultDict[
'hardDrivesMissingFirmware'] = computeNodeInventory.getHardDrivesMissingFirmware(
)
if versionInformationLogOnly:
return resultDict
componentUpdateDict = computeNodeInventory.getComponentUpdateDict(
)
self.computeNodeDict[
'componentUpdateDict'] = updateOSHarddrives and len(
componentUpdateDict['Firmware']) != 0 and len(
resultDict['hardDrivesMissingFirmware']
) == 0 and componentUpdateDict
resultDict['updateNeeded'] = True
else:
logger.error(
'The local hard drives are not being updated, since firmware was missing for the following hard drives: '
+ resultDict['hardDrivesMissingFirmware'] + '.')
else:
componentDictSizes = [
len(dict) for dict in componentUpdateDict.values()
]
if any((x != 0 for x in componentDictSizes)):
self.computeNodeDict[
'componentUpdateDict'] = componentUpdateDict
resultDict['updateNeeded'] = True
self.computeNodeDict[
'mellanoxBusList'] = computeNodeInventory.getMellanoxBusList(
)
if 'FusionIO' in componentUpdateDict['Firmware']:
self.computeNodeDict[
'busList'] = computeNodeInventory.getFusionIOBusList(
)
self.computeNodeDict[
'externalStoragePresent'] = computeNodeInventory.isExternalStoragePresent(
)
return resultDict