def main():
if os.geteuid() != 0:
print RED + 'You must be root to run this program; exiting program execution.' + RESETCOLORS
exit(1)
programVersion = '1.4-1'
usage = 'usage: %prog [[-h] [-r] [-s] [-u] [-v]]'
parser = optparse.OptionParser(usage = usage)
parser.add_option('-r', action = 'store_true', default = False, help = 'This option is used to generate a version report.')
parser.add_option('-s', action = 'store_true', default = False, help = 'This option is used when upgrading Servicegaurd nodes to indicate which node is the primary node; the secondary node should already be upgraded before envoking this option.')
parser.add_option('-u', action = 'store_true', default = False, help = 'This option is used to update the local OS hard drives before the mirror is split.')
parser.add_option('-v', action = 'store_true', default = False, help = "This option is used to display the application's version.")
(options, args) = parser.parse_args()
if options.v:
print os.path.basename(sys.argv[0]) + ' ' + programVersion
exit(0)
if not options.r or options.u:
if (options.r or options.s or options.s) and options.u:
print RED + "Options 'r', 's', and 'u' are mutually exclusive; please try again; exiting program execution." + RESETCOLORS
exit(1)
if options.r:
versionInformationLogOnly = True
else:
versionInformationLogOnly = False
if options.u:
updateOSHarddrives = True
else:
updateOSHarddrives = False
if options.s:
command = 'dmidecode -s system-product-name'
result = subprocess.Popen(command, stdout = subprocess.PIPE, stderr = subprocess.PIPE, shell = True)
(out, err) = result.communicate()
out = out.strip()
if result.returncode != 0:
print RED + "Unable to get the system's model information (dmidecode -s system-product-name); exiting program execution." + RESETCOLORS
exit(1)
try:
systemModel = re.match('[a-z,0-9]+\\s+(.*)', out, re.IGNORECASE).group(1).replace(' ', '')
except AttributeError:
err = None
print RED + "There was a system model match error when trying to match against '" + out + "':\n" + str(err) + '.' + RESETCOLORS
exit(1)
if systemModel != 'DL380pGen8' and systemModel != 'DL360pGen8':
print RED + "The '-s' option can only be used on NFS Serviceguard systems (DL380pGen8 or DL360pGen8)." + RESETCOLORS
exit(1)
csurBasePath = '/hp/support/csur'
currentLogDir = datetime.datetime.now().strftime('Date_%d%H%M%S%b%Y')
logBaseDir = csurBasePath + '/log/' + currentLogDir + '/'
sessionScreenLog = logBaseDir + 'sessionScreenLog.log'
cursesLog = logBaseDir + 'cursesLog.log'
try:
os.mkdir(logBaseDir)
except OSError:
err = None
print RED + 'Unable to create the current log directory ' + logBaseDir + '; fix the problem and try again; exiting program execution.\n' + str(err) + RESETCOLORS
exit(1)
try:
cursesThread = CursesThread(sessionScreenLog, cursesLog)
cursesThread.daemon = True
cursesThread.start()
initialize = Initialize(cursesThread)
csurResourceDict = initialize.init(csurBasePath, logBaseDir, programVersion, versionInformationLogOnly, updateOSHarddrives)
if options.s:
csurResourceDict['sgNode1'] = True
else:
csurResourceDict['sgNode1'] = False
if options.r:
cursesThread.insertMessage([
'informative',
'The system version report has been created and is in the log directory.'])
cursesThread.insertMessage([
'info',
' '])
if len(csurResourceDict['hardDrivesMissingFirmware']) != 0:
cursesThread.insertMessage([
'warning',
'Hard drive firmware was missing for the following hard drives: ' + csurResourceDict['hardDrivesMissingFirmware'] + '; make sure to file a bug report.'])
cursesThread.insertMessage([
'info',
' '])
elif len(csurResourceDict['componentListDict']['computeNodeList']) == 0 and len(csurResourceDict['hardDrivesMissingFirmware']) != 0:
cursesThread.insertMessage([
'error',
"The compute node's local OS hard drives are not being updated, since firmware for the hard drives was missing."])
cursesThread.insertMessage([
'info',
' '])
cursesThread.insertMessage([
'error',
'Hard drive firmware was missing for the following hard drives: ' + csurResourceDict['hardDrivesMissingFirmware'] + '; make sure to file a bug report.'])
elif len(csurResourceDict['componentListDict']['computeNodeList']) == 0:
cursesThread.insertMessage([
'informative',
'The compute node is already up to date; no action taken.'])
else:
timerThreadLocation = 1
computeNodeDict = csurResourceDict['componentListDict']['computeNodeList'][0].getComputeNodeDict()
hostname = computeNodeDict['hostname']
if updateOSHarddrives:
cursesThread.insertMessage([
'informative',
'Phase 2: Updating compute node ' + hostname + "'s hard drives that need to be updated."])
elif len(csurResourceDict['hardDrivesMissingFirmware']) != 0:
cursesThread.insertMessage([
'warning',
'Phase 2: Updating the compute node ' + hostname + "'s components that need to be updated, however, hard drive firmware was missing for the following hard drives: " + csurResourceDict['hardDrivesMissingFirmware'] + '; make sure to file a bug report.'])
else:
cursesThread.insertMessage([
'informative',
'Phase 2: Updating the compute node ' + hostname + "'s components that need to be updated."])
computeNodeUpdate = ComputeNodeUpdate(cursesThread, csurResourceDict.copy(), timerThreadLocation)
original_sigint_handler = signal.getsignal(signal.SIGINT)
original_sigquit_handler = signal.getsignal(signal.SIGQUIT)
s = SignalHandler(cursesThread)
signal.signal(signal.SIGINT, s.signal_handler)
signal.signal(signal.SIGQUIT, s.signal_handler)
workerThread = Thread(target = computeNodeUpdate.updateComputeNodeComponents)
workerThread.start()
while None:
if not workerThread.is_alive():
break
response = s.getResponse()
if response != '' or response == 'y':
computeNodeUpdate.endTask()
cursesThread.join()
exit(1)
signal.signal(signal.SIGINT, original_sigint_handler)
signal.signal(signal.SIGQUIT, original_sigquit_handler)
componentProblemDict = computeNodeUpdate.getUpdateComponentProblemDict()
if not updateOSHarddrives:
updateVersionInformationFileResult = updateVersionInformationFile(csurResourceDict.copy())
cursesThread.insertMessage([
'info',
' '])
if len(componentProblemDict['Software']) == 0 and len(componentProblemDict['Drivers']) == 0 and len(componentProblemDict['Firmware']) == 0:
if not updateOSHarddrives:
if not updateVersionInformationFileResult:
cursesThread.insertMessage([
'warning',
'The update of compute node ' + hostname + ' completed succesfully, however, the version information file update failed; check the log file for errors and update the file manually.'])
cursesThread.insertMessage([
'info',
' '])
if computeNodeDict['externalStoragePresent']:
cursesThread.insertMessage([
'final',
'Once the version information file is updated, power cycle the attached storage controller(s) and reboot the system for the changes to take effect.'])
else:
cursesThread.insertMessage([
'final',
'Once the version information file is updated, reboot the system for the changes to take effect.'])
else:
cursesThread.insertMessage([
'informative',
'The update of compute node ' + hostname + ' completed succesfully.'])
cursesThread.insertMessage([
'info',
' '])
if computeNodeDict['externalStoragePresent']:
cursesThread.insertMessage([
'final',
'Power cycle the attached storage controller(s) and reboot the system for the changes to take effect.'])
else:
cursesThread.insertMessage([
'final',
'Reboot the system for the changes to take effect.'])
else:
cursesThread.insertMessage([
'informative',
'The update of compute node ' + hostname + ' completed succesfully.'])
cursesThread.insertMessage([
'info',
' '])
cursesThread.insertMessage([
'final',
'Reboot the system for the changes to take effect.'])
elif not updateOSHarddrives:
errorMessage = 'The following components encountered errors during the update of compute node ' + hostname + '; check the log file for errors:\n'
if len(componentProblemDict['Software']) != 0:
if 'rpmRemovalFailure' not in componentProblemDict['Software']:
errorMessage += 'Software: ' + ', '.join(componentProblemDict['Software'].keys()) + '\n'
elif len(componentProblemDict['Software']) == 1:
errorMessage += 'Software: ' + componentProblemDict['Software']['rpmRemovalFailure'] + '\n'
else:
errorMessage += 'Software: ' + componentProblemDict['Software']['rpmRemovalFailure'] + ', '
del componentProblemDict['Software']['rpmRemovalFailure']
errorMessage += ', '.join(componentProblemDict['Software'].keys()) + '\n'
if len(componentProblemDict['Drivers']) != 0:
errorMessage += 'Drivers: ' + ', '.join(componentProblemDict['Drivers'].keys()) + '\n'
if len(componentProblemDict['Firmware']) != 0:
errorMessage += 'Firmware: ' + ', '.join(componentProblemDict['Firmware'].keys()) + '\n'
if not updateVersionInformationFileResult:
errorMessage += 'Also, the version information file update failed; check the log file for errors and update the file manually.'
cursesThread.insertMessage([
'error',
errorMessage])
else:
cursesThread.insertMessage([
'error',
'Errors were encountered while updating compute node ' + hostname + "'s hard drive firmware; check the log file for errors."])
cursesThread.insertMessage([
'info',
' '])
cursesThread.getUserInput([
'informative',
'Press enter to exit.'])
while not cursesThread.isUserInputReady():
time.sleep(0.1)
except Exception:
cursesThread.join()
traceback.print_exc()
exit(1)
finally:
cursesThread.join()
def main():
programVersion = '2018.03-0'
addOnSoftwareInstalled = False
hanaPartitionsRestored = False
scaleUpSystem = False
errorMessageList = []
if os.geteuid() != 0:
print RED + 'You must be root to run this program; exiting program execution.' + RESETCOLORS
exit(1)
usage = 'usage: %prog [[-h] [-p] [-v]]'
parser = optparse.OptionParser(usage = usage)
parser.add_option('-p', action = 'store_true', default = False, help = 'This option should be ran from the primary NFS Serviceguard node after the upgrade of both nodes when the upgrade caused the NIC cards to be renamed on one or both of the NFS Serviceguard nodes.')
parser.add_option('-v', action = 'store_true', default = False, help = "This option is used to display the application's version.")
(options, args) = parser.parse_args()
programName = os.path.basename(sys.argv[0])
if options.v:
print programName + ' ' + programVersion
exit(0)
(osDist, osDistLevel) = getOSDistribution()
upgradeLogDir = '/var/log/CoESapHANA_' + osDist + '_UpgradeLogDir'
if not os.path.isdir(upgradeLogDir):
try:
os.mkdir(upgradeLogDir)
except OSError:
err = None
print RED + "Unable to create the post upgrade log directory '" + upgradeLogDir + "'; fix the problem and try again; exiting program execution.\n" + str(err) + RESETCOLORS
exit(1)
dateTimestamp = datetime.datetime.now().strftime('%d%H%M%b%Y')
logFile = upgradeLogDir + '/' + osDist + '_Post_Upgrade_' + dateTimestamp + '.log'
handler = logging.FileHandler(logFile)
logger = logging.getLogger('coeOSUpgradeLogger')
logger.setLevel(logging.INFO)
formatter = logging.Formatter('%(asctime)s:%(levelname)s:%(message)s', datefmt = '%m/%d/%Y %H:%M:%S')
handler.setFormatter(formatter)
logger.addHandler(handler)
logger.info('The current version of the program is: ' + programVersion + '.')
serverModel = getServerModel()
if 'DL580' in serverModel:
processor = getProcessorType()
if options.p:
if not 'DL380p' in serverModel or 'DL360p' in serverModel:
print RED + "The '-p' option is only for Serviceguard NFS servers; exiting program execution." + RESETCOLORS
exit(1)
updateClusterCfg()
else:
try:
programParentDir = ''
cwd = os.getcwd()
programParentDir = os.path.dirname(os.path.realpath(__file__))
if cwd != programParentDir:
os.chdir(programParentDir)
except OSError:
err = None
logger.error("Unable to change into the programs parent directory '" + programParentDir + "'.\n" + str(err))
print RED + "Unable to change into the programs parent directory '" + programParentDir + "'; fix the problem and try again; exiting program execution." + RESETCOLORS
exit(1)
if osDist == 'SLES':
checkNetworkConfiguration = checkSLESNetworkConfiguration
import modules.upgradeMisc
else:
checkNetworkConfiguration = checkRHELNetworkConfiguration
import modules.upgradeMisc
if 'DL580' in serverModel or serverModel == 'Superdome':
if osDist == 'SLES':
updateOSSettings = updateSLESOSSettings
import modules.sapHANANodeSpec
else:
updateOSSettings = updateRHELOSSettings
import modules.sapHANANodeSpec
if 'DL580' in serverModel:
command = 'cat /proc/scsi/scsi'
result = subprocess.Popen(command, stdout = subprocess.PIPE, stderr = subprocess.PIPE, shell = True)
(out, err) = result.communicate()
out = out.strip()
if result.returncode != 0:
logger.error('Unable to determine if the DL580 is a Scale-up.\n' + str(out) + '\n' + str(err))
print RED + 'Unable to determine if the DL580 is a Scale-up; fix the problem and try again; exiting program execution.' + RESETCOLORS
exit(1)
searchResult = re.search('P830i', out, re.MULTILINE | re.DOTALL)
if not searchResult == None:
globalIni = 'global.ini'
globalIniList = []
scaleOutSystem = False
for (root, dirs, files) in os.walk(programParentDir):
if globalIni in files:
globalIniList.append(root + '/' + globalIni)
continue
for file in globalIniList:
with open(file, 'r') as f:
data = f.read()
if re.match('.*log|data_{1,}wwid\\s*=\\s*360002ac0+', data, re.MULTILINE | re.DOTALL) != None:
scaleOutSystem = True
multipathArguments = [
'enable',
'start']
for argument in multipathArguments:
command = 'systemctl ' + argument + ' multipathd'
result = subprocess.Popen(command, stdout = subprocess.PIPE, stderr = subprocess.PIPE, shell = True)
(out, err) = result.communicate()
out = out.strip()
if result.returncode != 0:
logger.error('Unable to ' + argument + ' multipathd.\n' + str(out) + '\n' + str(err))
print RED + 'Unable to ' + argument + ' multipathd; fix the problem and try again; exiting program execution.' + RESETCOLORS
exit(1)
time.sleep(0.1)
break
if not scaleOutSystem:
scaleUpSystem = True
upgradeResourceDict = { }
try:
with open('upgradeResourceFile') as f:
for line in f:
line = line.strip()
if len(line) == 0 and re.match('^\\s*#', line) or re.match('^\\s+$', line):
continue
continue
line = re.sub('[\'"]', '', line)
(key, val) = line.split('=')
key = key.strip()
upgradeResourceDict[key] = re.sub('\\s*,\\s*', ',', val).strip()
except IOError:
err = None
logger.error("Unable to access the application's resource file 'upgradeResourceFile'.\n" + str(err))
print RED + "Unable to access the application's resource file 'upgradeResourceFile'; fix the problem and try again; exiting program execution." + RESETCOLORS
exit(1)
sessionScreenLog = upgradeLogDir + '/sessionScreenLog.log'
cursesLog = upgradeLogDir + '/cursesLog.log'
try:
cursesThread = CursesThread(sessionScreenLog, cursesLog)
cursesThread.daemon = True
cursesThread.start()
osRestorationArchiveErrorFile = upgradeLogDir + '/.osRestorationArchiveError'
if not os.path.isfile(osRestorationArchiveErrorFile):
try:
os.mknod(osRestorationArchiveErrorFile)
except OSError:
err = None
logger.error("Unable to create the OS restoration archive error file '" + osRestorationArchiveErrorFile + "'.\n" + str(err))
displayErrorMessage("Unable to create the OS restoration archive error file '" + osRestorationArchiveErrorFile + "'; fix the problem and try again.", cursesThread)
progressStatusFile = upgradeLogDir + '/.restorationProgress'
try:
if os.path.isfile(progressStatusFile):
with open(progressStatusFile) as f:
progressDict = dict.fromkeys(lambda .0: continue(f))
else:
progressDict = { }
if os.path.isfile(progressStatusFile):
f = open(progressStatusFile, 'a')
else:
f = open(progressStatusFile, 'w')
osRestorationArchiveExtracted = False
try:
if os.stat(osRestorationArchiveErrorFile).st_size < 50 and 'osRestorationArchiveExtracted' not in progressDict:
cursesThread.insertMessage([
'informative',
'Checking and extracting the OS restoration archive file.'])
cursesThread.insertMessage([
'informative',
''])
osRestorationArchive = checkOSRestorationArchive(programParentDir, osRestorationArchiveErrorFile, osDist, cursesThread)
extractOSRestorationArchive(osRestorationArchive, osRestorationArchiveErrorFile, cursesThread)
f.write('osRestorationArchiveExtracted\n')
osRestorationArchiveExtracted = True
except OSError:
err = None
logger.error("Unable to access the OS restoration archive error file '" + osRestorationArchiveErrorFile + "'.\n" + str(err))
displayErrorMessage("Unable to access the OS restoration archive error file '" + osRestorationArchiveErrorFile + "'; fix the problem and try again.", cursesThread)
if os.path.isdir(programParentDir + '/addOnSoftwareRPMS'):
if 'installaddOnSoftware' not in progressDict:
f.write('installaddOnSoftware\n')
if osDist == 'SLES':
(errorMessage, addOnSoftwareInstalled) = installSLESAddOnSoftware(programParentDir, cursesThread)
else:
(errorMessage, addOnSoftwareInstalled) = installRHELAddOnSoftware(programParentDir, cursesThread)
if len(errorMessage) > 0:
errorMessageList.append(errorMessage)
else:
errorMessageList.append('The additional RPMs were not installed, since an attempt to add them was already made.')
else:
addOnSoftwareInstalled = True
logger.info('There were no additional RPMs present for the post-upgrade.')
if 'setTimezone' not in progressDict:
f.write('setTimezone\n')
errorMessage = setTimezone(programParentDir, cursesThread)
if len(errorMessage) > 0:
errorMessageList.append(errorMessage)
else:
errorMessageList.append('The time zone was not set, since an attempt to set it was already made.')
if 'checkNetworkConfiguration' not in progressDict:
if 'osRestorationArchiveExtracted' in progressDict or osRestorationArchiveExtracted:
f.write('checkNetworkConfiguration\n')
if (serverModel == 'ProLiant DL320e Gen8 v2' or serverModel == 'ProLiant DL360 Gen9') and osDistLevel == '11.4':
errorMessage = checkNetworkConfiguration(programParentDir, osDist, cursesThread, osDistLevel = osDistLevel)
else:
errorMessage = checkNetworkConfiguration(programParentDir, osDist, cursesThread)
if len(errorMessage) > 0:
errorMessageList.append(errorMessage)
else:
errorMessageList.append('The network configuration was not checked; either because it was already checked or the OS restoration archive was not successfully extracted.')
if not (serverModel == 'ProLiant DL320e Gen8 v2' or serverModel == 'ProLiant DL360 Gen9') and osDistLevel == '11.4':
if 'setHostname' not in progressDict:
f.write('setHostname\n')
errorMessage = setHostname(programParentDir, cursesThread)
if len(errorMessage) > 0:
errorMessageList.append(errorMessage)
else:
errorMessageList.append('The host name was not set, since an attempt to set it was already made.')
if osDist == 'SLES':
if 'updateNTPConf' not in progressDict:
f.write('updateNTPConf\n')
errorMessage = updateNTPConf(cursesThread)
if len(errorMessage) > 0:
errorMessageList.append(errorMessage)
else:
errorMessageList.append('The ntp configuration was not updated, since an attempt to update it was already made.')
if 'configureSnapper' not in progressDict:
f.write('configureSnapper\n')
errorMessage = configureSnapper(osDistLevel, cursesThread)
if len(errorMessage) > 0:
errorMessageList.append(errorMessage)
else:
errorMessageList.append('The snapper configuration was not updated, since an attempt to update it was already made.')
if serverModel != 'Superdome':
if 'updateMultipathConf' not in progressDict:
f.write('updateMultipathConf\n')
errorMessage = updateMultipathConf(cursesThread)
if len(errorMessage) > 0:
errorMessageList.append(errorMessage)
else:
errorMessageList.append('The multipath blacklist configuration was not updated, since an attempt to update it was already made.')
if 'DL580' in serverModel or serverModel == 'Superdome':
if 'DL580' in serverModel:
if 'updateBIOS' not in progressDict and addOnSoftwareInstalled:
f.write('updateBIOS\n')
errorMessage = updateBIOS(programParentDir, cursesThread)
if len(errorMessage) > 0:
errorMessageList.append(errorMessage)
else:
errorMessageList.append('The BIOS was not updated; either because it was already updated or the additional RPMs failed to install.')
if os.path.isdir(programParentDir + '/addOnFileArchive'):
if 'installAddOnFiles' not in progressDict:
f.write('installAddOnFiles\n')
errorMessage = installAddOnFiles(programParentDir, upgradeResourceDict.copy(), osDist, cursesThread, processor = processor)
if len(errorMessage) > 0:
errorMessageList.append(errorMessage)
else:
errorMessageList.append('The files from the add on files archive were not installed, since an attempt to add them was already made.')
if serverModel == 'Superdome':
if os.path.isdir(programParentDir + '/addOnFileArchive'):
if 'installAddOnFiles' not in progressDict:
f.write('installAddOnFiles\n')
errorMessage = installAddOnFiles(programParentDir, upgradeResourceDict.copy(), osDist, cursesThread, serverModel = serverModel)
if len(errorMessage) > 0:
errorMessageList.append(errorMessage)
else:
errorMessageList.append('The files from the add on files archive were not installed, since an attempt to add them was already made.')
if osDist == 'SLES':
if 'updateKdump' not in progressDict:
f.write('updateKdump\n')
errorMessage = updateKdump(cursesThread)
if len(errorMessage) > 0:
errorMessageList.append(errorMessage)
else:
errorMessageList.append('The update of kdump.service to address bug 1008170 was not performed, since an attempt to update the service was already made.')
if 'restoreHANAUserAccounts' not in progressDict:
f.write('restoreHANAUserAccounts\n')
errorMessage = restoreHANAUserAccounts(programParentDir, cursesThread)
if len(errorMessage) > 0:
errorMessageList.append(errorMessage)
else:
errorMessageList.append('The SAP HANA user account data was not restored, since an attempt to restore the account data was already made.')
if 'restoreHANAPartitionData' not in progressDict:
f.write('restoreHANAPartitionData\n')
(errorMessage, hanaPartitionsRestored) = restoreHANAPartitionData(programParentDir, cursesThread)
if len(errorMessage) > 0:
errorMessageList.append(errorMessage)
else:
errorMessageList.append('The SAP HANA parition data was not restored, since an attempt to restore the data was already made.')
if 'sapRestorationArchiveExtracted' not in progressDict and hanaPartitionsRestored:
f.write('sapRestorationArchiveExtracted\n')
errorMessage = extractSAPRestorationArchive(programParentDir, osDist, cursesThread)
if len(errorMessage) > 0:
errorMessageList.append(errorMessage)
else:
errorMessageList.append('The SAP restoration archive was not extracted; either because it was already extracted or the SAP HANA partition data was not successfully restored.')
if not scaleUpSystem:
if 'prepareSAPHanaDirs' not in progressDict:
f.write('prepareSAPHanaDirs\n')
errorMessage = prepareSAPHanaDirs(programParentDir, cursesThread)
if len(errorMessage) > 0:
errorMessageList.append(errorMessage)
else:
errorMessageList.append('The /hana/data/SID and /hana/log/SID directories were not created; since an attempt to create them was already made.')
if 'DL380p' in serverModel or 'DL360p' in serverModel:
if 'configureXinetd' not in progressDict:
f.write('configureXinetd\n')
errorMessage = configureXinetd(cursesThread)
if len(errorMessage) > 0:
errorMessageList.append(errorMessage)
else:
errorMessageList.append('xinetd was not configured; since an attempt to configure it was already made.')
if 'updateLvmConf' not in progressDict:
f.write('updateLvmConf\n')
errorMessage = updateLvmConf(cursesThread)
if len(errorMessage) > 0:
errorMessageList.append(errorMessage)
else:
errorMessageList.append('/etc/lvm.conf was not updated; since an attempt to enable it was already made.')
if 'updateNFSListeners' not in progressDict:
f.write('updateNFSListeners\n')
errorMessage = updateNFSListeners(cursesThread)
if len(errorMessage) > 0:
errorMessageList.append(errorMessage)
else:
errorMessageList.append("The number of NFS listeners in '/opt/cmcluster/nfstoolkit/hananfs.sh' was not updated; since an attempt to update the number was already made.")
if 'createNFSMountPoint' not in progressDict:
f.write('createNFSMountPoint\n')
errorMessage = createNFSMountPoint(cursesThread)
if len(errorMessage) > 0:
errorMessageList.append(errorMessage)
else:
errorMessageList.append('The hananfs mount point was not created; since an attempt to create it was already made.')
if 'buildDeadmanDriver' not in progressDict:
f.write('buildDeadmanDriver\n')
errorMessage = buildDeadmanDriver(cursesThread)
if len(errorMessage) > 0:
errorMessageList.append(errorMessage)
else:
errorMessageList.append('The deadman driver was not built and installed; since an attempt to build and install it was already made.')
if serverModel == 'ProLiant DL320e Gen8 v2' or serverModel == 'ProLiant DL360 Gen9':
if 'configureServiceguardQS' not in progressDict:
f.write('configureServiceguardQS\n')
errorMessage = configureServiceguardQS(programParentDir, osDistLevel, cursesThread)
if len(errorMessage) > 0:
errorMessageList.append(errorMessage)
else:
errorMessageList.append('Serviceguard QS was not configured; since an attempt to configure it was already made.')
except IOError:
err = None
logger.error('Could not access ' + progressStatusFile + '.\n' + str(err))
displayErrorMessage("Could not access '" + progressStatusFile + "'; fix the problem and try again.", cursesThread)
f.close()
if len(errorMessageList) > 0:
cursesThread.insertMessage([
'warning',
"Restoration of the server's configuration has completed with the errors shown below; check the log file for additional information before continuing with the re-installation process:"])
cursesThread.insertMessage([
'warning',
''])
for message in errorMessageList:
cursesThread.insertMessage([
'warning',
message])
cursesThread.insertMessage([
'warning',
''])
else:
cursesThread.insertMessage([
'informative',
"Restoration of the server's configuration has completed; continue with the next step in the re-installation process."])
cursesThread.insertMessage([
'informative',
''])
cursesThread.getUserInput([
'informative',
'Press enter to exit the program.'])
cursesThread.insertMessage([
'informative',
''])
while not cursesThread.isUserInputReady():
time.sleep(0.1)
exit(0)
except Exception:
err = None
cursesThread.join()
logger.error('An unexpected error was encountered:\n' + traceback.format_exc())
print RED + "An unexpected error was encountered; collect the application's log file and report the error back to the SAP HANA CoE DevOps team." + RESETCOLORS
print RED + 'Error: ' + str(err) + RESETCOLORS
exit(1)
finally:
cursesThread.join()
def main():
programVersion = '2017.05-rc1'
hpeUtilitySoftwareInstalled = False
'''
The errorMessageList keeps track of errors encountered during the restoration process so that
one knows what will have to be done manually if necessary.
'''
errorMessageList = []
#The program can only be ran by root.
if os.geteuid() != 0:
print(RED + 'You must be root to run this program; exiting program execution.' + RESETCOLORS)
exit(1)
#Change into directory containing the program, which should be the mount point of the ISO image containing the files needed for the post upgrade configuration/update.
try:
programParentDir = ''
cwd = os.getcwd()
programParentDir = os.path.dirname(os.path.realpath(__file__))
if cwd != programParentDir:
os.chdir(programParentDir)
except OSError as err:
print(RED + 'Unable to change into the programs parent directory \'' + programParentDir + '\'; fix the problem and try again; exiting program execution.\n' + str(err) + RESETCOLORS)
exit(1)
#Get the server's OS distribution version information.
command = 'cat /proc/version'
result = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
out, err = result.communicate()
if result.returncode != 0:
print(RED + 'Unable to get the server\'s OS distribution information; fix the problem and try again; exiting program execution.\n' + err + '\n' + out + RESETCOLORS)
exit(1)
#Change version information to lowercase before checking for OS type.
versionInfo = out.lower()
if not 'suse' in versionInfo:
print(RED + 'The OS distribution \'' + versionInfo + '\' is not supported; fix the problem and try again; exiting program execution.' + RESETCOLORS)
exit(1)
#Set and create the upgrade working directory if it does not exist.
upgradeLogDir = '/var/log/CoE_SAP_HANA_SLES_Upgrade_Log_Dir'
if not os.path.isdir(upgradeLogDir):
try:
os.mkdir(upgradeLogDir)
except OSError as err:
print(RED + 'Unable to create the post upgrade log directory \'' + upgradeLogDir + '\'; fix the problem and try again; exiting program execution.\n' + str(err) + RESETCOLORS)
exit(1)
#Configure logging.
dateTimestamp = (datetime.datetime.now()).strftime('%d%H%M%b%Y')
logFile = upgradeLogDir + '/SLES_Post_Upgrade_' + dateTimestamp + '.log'
handler = logging.FileHandler(logFile)
logger = logging.getLogger('coeOSUpgradeLogger')
logger.setLevel(logging.INFO)
formatter = logging.Formatter('%(asctime)s:%(levelname)s:%(message)s', datefmt='%m/%d/%Y %H:%M:%S')
handler.setFormatter(formatter)
logger.addHandler(handler)
logger.info('The current version of the program is: ' + programVersion + '.')
#Only Ivy Bridge and Haswell systems are supported for an upgrade.
processorDict = {'62' : 'ivybridge', '63' : 'haswell'}
command = 'cat /proc/cpuinfo'
result = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
out, err = result.communicate()
if result.returncode != 0:
logger.error('There was a problem getting the cpu information.\n' + err + '\n' + out)
print(RED + 'There was a problem getting the cpu information; fix the problem and try again; exiting program execution.' + RESETCOLORS)
exit(1)
cpudata = out.splitlines()
for line in cpudata:
if re.match('\s*model\s+:\s+[2-9]{2}', line) != None:
try:
processor = processorDict[re.match('\s*model\s+:\s+([2-9]{2})', line).group(1)]
except AttributeError as err:
logger.error('There was a match error when trying to match against ' + line + '.\n' + str(err))
print(RED + 'There was a match error matching against \'' + line + '\'; fix the problem and try again; exiting program execution.' + RESETCOLORS)
exit(1)
except KeyError as err:
logger.error('The resource key (' + str(err) + ') was not present in the processor dictionary.')
print(RED + 'The server is not supported for an upgrade, since it is not using a Haswell or Ivy Bridge processor; fix the problem and try again; exiting program execution.' + RESETCOLORS)
exit(1)
break
#This dictionary holds the resource files resources.
upgradeResourceDict = {}
#Get the upgrade resource file data and save it to a dictionary (hash).
try:
with open('upgradeResourceFile') as f:
for line in f:
line = line.strip()
#Ignore commented and blank lines.
if len(line) == 0 or re.match("^\s*#", line) or re.match('^\s+$', line):
continue
else:
#Remove quotes from resources.
line = re.sub('[\'"]', '', line)
(key, val) = line.split('=')
key = key.strip()
upgradeResourceDict[key] = re.sub('\s+', '', val)
except IOError as err:
logger.error('Unable to access the application\'s resource file \'upgradeResourceFile\'.\n' + str(err))
print(RED + 'Unable to access the application\'s resource file \'upgradeResourceFile\'; fix the problem and try again; exiting program execution.' + RESETCOLORS)
exit(1)
#Setup curses, so that messages can be scrolled during program execution.
sessionScreenLog = upgradeLogDir + '/sessionScreenLog.log'
cursesLog = upgradeLogDir + '/cursesLog.log'
try:
'''
The application uses curses to manage the display so that we have two
windows. One for feedback and one for conversing. Also, scrolling is implemented.
'''
cursesThread = CursesThread(sessionScreenLog, cursesLog)
cursesThread.daemon = True
cursesThread.start()
'''
Check to make sure the restoration archive file from the preupgrade is not corrupt.
Three attempts will be made before using the restoration archive is abandoned.
'''
restorationArchiveErrorFile = upgradeLogDir + '/.restorationArchiveError'
if not os.path.isfile(restorationArchiveErrorFile):
try:
os.mknod(restorationArchiveErrorFile)
except OSError as err:
logger.error('Unable to create the restoration archive error file \'' + restorationArchiveErrorFile + '\'.\n' + str(err))
displayErrorMessage('Unable to create the restoration archive error file \'' + restorationArchiveErrorFile + '\'; fix the problem and try again.', cursesThread)
'''
The progress status file is used to keep track of what was already done in case the
program is ran again. Thus, only those functions that were note already ran will be run.
'''
progressStatusFile = upgradeLogDir + '/.restorationProgress'
try:
if os.path.isfile(progressStatusFile):
with open(progressStatusFile) as f:
progressDict = dict.fromkeys(x.strip() for x in f)
else:
progressDict = {}
if os.path.isfile(progressStatusFile):
f = open(progressStatusFile, 'a')
else:
f = open(progressStatusFile, 'w')
'''
This variable is used to identify if the restoration archive was successfully extracted,
since it may not be in the progress status file yet, while attempts to extract the archive
are still being made.
'''
restorationArchiveExtracted = False
try:
#Check and extract the restoration archive; three attempts are allowed.
if os.stat(restorationArchiveErrorFile).st_size < 50 and not 'restorationArchiveExtracted' in progressDict:
cursesThread.insertMessage(['informative', 'Checking and extracting the restoration archive file.'])
cursesThread.insertMessage(['informative', ''])
preUpgradeArchive = checkPreupgradeArchive(programParentDir, restorationArchiveErrorFile, cursesThread)
if extractPreupgradeArchive(preUpgradeArchive, restorationArchiveErrorFile, cursesThread):
f.write('restorationArchiveExtracted\n')
restorationArchiveExtracted = True
except OSError as err:
logger.error('Unable to access the restoration archive error file \'' + restorationArchiveErrorFile + '\'.\n' + str(err))
displayErrorMessage('Unable to access the restoration archive error file \'' + restorationArchiveErrorFile + '\'; fix the problem and try again.', cursesThread)
if not 'setHostname' in progressDict:
f.write('setHostname\n')
errorMessage = setHostname(programParentDir, cursesThread)
if len(errorMessage) > 0:
errorMessageList.append(errorMessage)
if not 'installHPEUtilitySoftware' in progressDict:
f.write('installHPEUtilitySoftware\n')
(errorMessage, hpeUtilitySoftwareInstalled) = installHPEUtilitySoftware(programParentDir, cursesThread)
if len(errorMessage) > 0:
errorMessageList.append(errorMessage)
if not 'updateBIOS' in progressDict and hpeUtilitySoftwareInstalled:
f.write('updateBIOS\n')
errorMessage = updateBIOS(programParentDir, cursesThread)
if len(errorMessage) > 0:
errorMessageList.append(errorMessage)
if not 'updateOSSettings' in progressDict:
f.write('updateOSSettings\n')
osErrorMessageList = updateOSSettings(cursesThread)
if len(osErrorMessageList) > 0:
errorMessageList += osErrorMessageList
if not 'installAddOnFiles' in progressDict:
f.write('installAddOnFiles\n')
errorMessage = installAddOnFiles(programParentDir, processor, cursesThread)
if len(errorMessage) > 0:
errorMessageList.append(errorMessage)
if not 'checkNetworkConfiguration' in progressDict and ('restorationArchiveExtracted' in progressDict or restorationArchiveExtracted):
f.write('checkNetworkConfiguration\n')
errorMessage = checkNetworkConfiguration(programParentDir, cursesThread)
if len(errorMessage) > 0:
errorMessageList.append(errorMessage)
if not 'restoreHANAUserAccounts' in progressDict:
f.write('restoreHANAUserAccounts\n')
errorMessage = restoreHANAUserAccounts(programParentDir, cursesThread)
if len(errorMessage) > 0:
errorMessageList.append(errorMessage)
if not 'restoreHANAPartitionData' in progressDict:
f.write('restoreHANAPartitionData\n')
errorMessage = restoreHANAPartitionData(programParentDir, cursesThread)
if len(errorMessage) > 0:
errorMessageList.append(errorMessage)
except IOError as err:
logger.error('Could not access ' + progressStatusFile + '.\n' + str(err))
displayErrorMessage('Could not access \'' + progressStatusFile + '\'; fix the problem and try again.', cursesThread)
f.close()
if len(errorMessageList) > 0:
cursesThread.insertMessage(['warning', 'Restoration of the server has completed with the errors shown below; check the log file for additional information:'])
cursesThread.insertMessage(['warning', ''])
for message in errorMessageList:
cursesThread.insertMessage(['warning', message])
cursesThread.insertMessage(['warning', ''])
else:
cursesThread.insertMessage(['informative', 'Restoration of the server has completed; check log files and perform a functional check before turning the server back over to the customer.'])
cursesThread.insertMessage(['informative', ''])
cursesThread.getUserInput(['informative', 'Press enter to exit the program.'])
cursesThread.insertMessage(['informative', ''])
while not cursesThread.isUserInputReady():
time.sleep(0.1)
exit(0)
except Exception:
cursesThread.join()
traceback.print_exc()
exit(1)
finally:
cursesThread.join()
def main():
if os.geteuid() != 0:
print RED + 'You must be root to run this program; exiting program execution.' + RESETCOLORS
exit(1)
programVersion = '1.0-0'
usage = 'usage: %prog [[-d] [-h] [-v]]'
parser = optparse.OptionParser(usage=usage)
parser.add_option(
'-d',
action='store_true',
default=False,
help=
'This option is used when problems are encountered and additional debug information is needed.'
)
parser.add_option(
'-v',
action='store_true',
default=False,
help="This option is used to display the application's version.")
options, args = parser.parse_args()
if options.v:
print os.path.basename(sys.argv[0]) + ' ' + programVersion
exit(0)
if options.d:
debug = True
else:
debug = False
versionInformationLogOnly = True
updateOSHarddrives = False
healthBasePath = '/hp/support/health'
currentLogDir = datetime.datetime.now().strftime(
'Date_%d%b%Y_Time_%H:%M:%S')
logBaseDir = healthBasePath + '/log/' + currentLogDir + '/'
sessionScreenLog = logBaseDir + 'sessionScreenLog.log'
cursesLog = logBaseDir + 'cursesLog.log'
try:
os.mkdir(logBaseDir)
except OSError as err:
print RED + 'Unable to create the current log directory ' + logBaseDir + '; fix the problem and try again; exiting program execution.\n' + str(
err) + RESETCOLORS
exit(1)
try:
cursesThread = CursesThread(sessionScreenLog, cursesLog)
cursesThread.daemon = True
cursesThread.start()
initialize = Initialize(cursesThread)
healthResourceDict = initialize.init(healthBasePath, logBaseDir, debug,
programVersion,
versionInformationLogOnly,
updateOSHarddrives)
cursesThread.insertMessage([
'informative',
'The Health Check report has been created and is in the ' +
healthBasePath + ' directory.'
])
cursesThread.insertMessage(['info', ' '])
cursesThread.getUserInput(['informative', 'Press enter to exit.'])
while not cursesThread.isUserInputReady():
time.sleep(0.1)
except Exception:
cursesThread.join()
traceback.print_exc()
exit(1)
finally:
cursesThread.join()
versionInformationLog = logBaseDir + 'HealthCheckReport.txt'
if os.path.isfile(versionInformationLog):
try:
with open(versionInformationLog) as f:
for line in f:
print line.strip()
print '\n'
except IOError as err:
print 'I/O Error while trying to print ' + versionInformationLog + '.\n' + err + '\n'
def main():
if os.geteuid() != 0:
print(
RED +
"You must be root to run this program; exiting program execution."
+ RESETCOLORS)
exit(1)
programVersion = '1.2-rc1'
#Parse options before setting up curses mode.
usage = 'usage: %prog [[-h] [-r] [-u] [-v]]'
parser = optparse.OptionParser(usage=usage)
parser.add_option('-r',
action='store_true',
default=False,
help='This option is used to generate a version report.')
parser.add_option(
'-u',
action='store_true',
default=False,
help=
'This option is used to update the local OS hard drives before the mirror is split.'
)
parser.add_option(
'-v',
action='store_true',
default=False,
help='This option is used to display the application\'s version.')
(options, args) = parser.parse_args()
if options.v:
print(os.path.basename(sys.argv[0]) + ' ' + programVersion)
exit(0)
count = 0
for option in options:
if options[option]:
count += 1
if count > 1:
print(
RED +
"Options are mutually exclusive; please try again; exiting program execution."
+ RESETCOLORS)
exit(1)
if options.r:
versionInformationLogOnly = True
else:
versionInformationLogOnly = False
if options.u:
updateOSHarddrives = True
else:
updateOSHarddrives = False
#This is the location of the csur application.
csurBasePath = '/hp/support/csur'
'''
These log files have their name hardcoded, since their logging starts before the
csur application's resource file is read.
'''
currentLogDir = datetime.datetime.now().strftime(
"Date_%d%b%Y_Time_%H:%M:%S")
logBaseDir = csurBasePath + '/log/' + currentLogDir + '/'
sessionScreenLog = logBaseDir + 'sessionScreenLog.log'
cursesLog = logBaseDir + 'cursesLog.log'
try:
os.mkdir(logBaseDir)
except OSError as err:
print(RED + 'Unable to create the current log directory ' +
logBaseDir +
'; fix the problem and try again; exiting program execution.\n' +
str(err) + RESETCOLORS)
exit(1)
try:
'''
The application uses curses to manage the display so that we have two
windows. One for feedback and one for conversing. Also, scrolling is implemented.
'''
cursesThread = CursesThread(sessionScreenLog, cursesLog)
cursesThread.daemon = True
cursesThread.start()
initialize = Initialize(cursesThread)
csurResourceDict = initialize.init(csurBasePath, logBaseDir,
programVersion,
versionInformationLogOnly,
updateOSHarddrives)
if options.r:
cursesThread.insertMessage([
'informative',
'The system version report has been created and is in the log directory.'
])
cursesThread.insertMessage(['info', ' '])
else:
#The computeNodeList will be empty if the node did not need to be updated.
if len(csurResourceDict['componentListDict']
['computeNodeList']) == 0:
cursesThread.insertMessage([
'informative',
'The compute node is already up to date; no action taken.'
])
else:
'''
For a Scale-up system/Compute Nodes only there will only be two timer threads. One for the initial initialization
and a second for the update.
'''
timerThreadLocation = 1
computeNodeDict = csurResourceDict['componentListDict'][
'computeNodeList'][0].getComputeNodeDict()
hostname = computeNodeDict['hostname']
if updateOSHarddrives:
cursesThread.insertMessage([
'informative', "Phase 2: Updating compute node " +
hostname + "'s hard drives that need to be updated."
])
else:
cursesThread.insertMessage([
'informative', "Phase 2: Updating the compute node " +
hostname + "'s components that need to be updated."
])
#Instantiate the computeNode update class, which will do the work of updating the compute node.
computeNodeUpdate = ComputeNodeUpdate(cursesThread,
csurResourceDict.copy(),
timerThreadLocation)
#Get the current signal handlers so that they can be restored after the update is completed.
original_sigint_handler = signal.getsignal(signal.SIGINT)
original_sigquit_handler = signal.getsignal(signal.SIGQUIT)
'''
Setup signal handler to intercept SIGINT and SIGQUIT.
'''
s = SignalHandler(cursesThread)
signal.signal(signal.SIGINT, s.signal_handler)
signal.signal(signal.SIGQUIT, s.signal_handler)
#Create and start the worker thread.
workerThread = Thread(
target=computeNodeUpdate.updateComputeNodeComponents)
workerThread.start()
#Wait for the thread to either stop or get interrupted.
while 1:
time.sleep(0.1)
if not workerThread.is_alive():
break
'''
The response will be an empty string unless a signal was received. If a signal is
received then response is either 'n' (Don't cancel the update) or 'y' (Cancel the update.).
'''
response = s.getResponse()
if response != '':
if response == 'y':
computeNodeUpdate.endTask()
cursesThread.join()
exit(1)
#Restore the original signal handlers.
signal.signal(signal.SIGINT, original_sigint_handler)
signal.signal(signal.SIGQUIT, original_sigquit_handler)
componentProblemDict = computeNodeUpdate.getUpdateComponentProblemDict(
)
#Update the CSUR version information file.
if not updateOSHarddrives:
updateVersionInformationFileResult = updateVersionInformationFile(
csurResourceDict.copy())
#Move cursor down for formatting purposes.
cursesThread.insertMessage(['info', ' '])
if len(componentProblemDict['Software']) == 0 and len(
componentProblemDict['Drivers']) == 0 and len(
componentProblemDict['Firmware']) == 0:
if not updateOSHarddrives:
if not updateVersionInformationFileResult:
cursesThread.insertMessage([
'warning',
'The update of compute node ' + hostname +
' completed succesfully, however, the version information file update failed; check the log file for errors and update the file manually.'
])
cursesThread.insertMessage(['info', ' '])
if computeNodeDict['externalStoragePresent']:
cursesThread.insertMessage([
'final',
'Once the version information file is updated, power cycle the attached storage controller(s) and reboot the system for the changes to take effect.'
])
else:
cursesThread.insertMessage([
'final',
'Once the version information file is updated, reboot the system for the changes to take effect.'
])
else:
cursesThread.insertMessage([
'informative', 'The update of compute node ' +
hostname + ' completed succesfully.'
])
cursesThread.insertMessage(['info', ' '])
if computeNodeDict['externalStoragePresent']:
cursesThread.insertMessage([
'final',
'Power cycle the attached storage controller(s) and reboot the system for the changes to take effect.'
])
else:
cursesThread.insertMessage([
'final',
'Reboot the system for the changes to take effect.'
])
else:
cursesThread.insertMessage([
'informative', 'The update of compute node ' +
hostname + ' completed succesfully.'
])
cursesThread.insertMessage(['info', ' '])
cursesThread.insertMessage([
'final',
'Reboot the system for the changes to take effect.'
])
else:
if not updateOSHarddrives:
errorMessage = 'The following components encountered errors during the update of compute node ' + hostname + '; check the log file for errors:\n'
if len(componentProblemDict['Software']) != 0:
'''
Print the keys which are the names of the software package(s) that had issues while being installed. Else
print the names of the software package(s) that had issues while being removed/installed.
'''
if not 'rpmRemovalFailure' in componentProblemDict[
'Software']:
errorMessage += 'Software: ' + ', '.join(
componentProblemDict['Software'].keys(
)) + '\n'
else:
if len(componentProblemDict['Software']) == 1:
errorMessage += 'Software: ' + componentProblemDict[
'Software']['rpmRemovalFailure'] + '\n'
else:
errorMessage += 'Software: ' + componentProblemDict[
'Software']['rpmRemovalFailure'] + ', '
del componentProblemDict['Software'][
'rpmRemovalFailure']
errorMessage += ', '.join(
componentProblemDict['Software'].keys(
)) + '\n'
if len(componentProblemDict['Drivers']) != 0:
errorMessage += 'Drivers: ' + ', '.join(
componentProblemDict['Drivers'].keys()) + '\n'
if len(componentProblemDict['Firmware']) != 0:
errorMessage += 'Firmware: ' + ', '.join(
componentProblemDict['Firmware'].keys()) + '\n'
if not updateVersionInformationFileResult:
errorMessage += 'Also, the version information file update failed; check the log file for errors and update the file manually.'
cursesThread.insertMessage(['error', errorMessage])
else:
cursesThread.insertMessage([
'error',
"Errors were encountered while updating compute node "
+ hostname +
"'s hard drive firmware; check the log file for errors."
])
cursesThread.insertMessage(['info', ' '])
cursesThread.getUserInput(['informative', 'Press enter to exit.'])
while not cursesThread.isUserInputReady():
time.sleep(0.1)
except Exception:
cursesThread.join()
traceback.print_exc()
exit(1)
finally:
cursesThread.join()
def main():
if os.geteuid() != 0:
print(
RED +
"You must be root to run this program; exiting program execution."
+ RESETCOLORS)
exit(1)
programVersion = '1.0-0'
#Parse options before setting up curses mode.
usage = 'usage: %prog [[-d] [-h] [-v]]'
parser = optparse.OptionParser(usage=usage)
parser.add_option(
'-d',
action='store_true',
default=False,
help=
'This option is used when problems are encountered and additional debug information is needed.'
)
parser.add_option(
'-v',
action='store_true',
default=False,
help='This option is used to display the application\'s version.')
(options, args) = parser.parse_args()
if options.v:
print(os.path.basename(sys.argv[0]) + ' ' + programVersion)
exit(0)
if options.d:
debug = True
else:
debug = False
#This is the location of the application.
csurBasePath = '/hp/support/csur'
'''
These log files have their name hardcoded, since their logging starts before the
application's resource file is read.
'''
logBaseDir = csurBasePath + '/log/'
sessionScreenLog = logBaseDir + 'sessionScreenLog.log'
cursesLog = logBaseDir + 'cursesLog.log'
#Always start with an empty log directory when performing a new update.
try:
logList = os.listdir(logBaseDir)
for log in logList:
os.remove(logBaseDir + log)
except OSError as err:
print(RED + 'Unable to remove old logs in ' + logBaseDir +
'; fix the problem and try again; exiting program execution.\n' +
str(err) + RESETCOLORS)
exit(1)
try:
'''
The application uses curses to manage the display so that we have two
windows. One for feedback and one for conversing. Also, scrolling is implemented.
'''
cursesThread = CursesThread(sessionScreenLog, cursesLog)
cursesThread.daemon = True
cursesThread.start()
initialize = Initialize(cursesThread)
csurResourceDict = initialize.init(csurBasePath, debug, programVersion)
if 'Scale-up' in csurResourceDict['systemType']:
#The computeNodeList will be empty if the node did not need to be updated.
if len(csurResourceDict['componentListDict']
['computeNodeList']) == 0:
cursesThread.insertMessage([
'informative',
'The compute node is already up to date; no action taken.'
])
else:
'''
For a Scale-up system there will only be two timer threads. One for the initial initialization
and a second for the update.
'''
timerThreadLocation = 1
computeNodeDict = csurResourceDict['componentListDict'][
'computeNodeList'][0].getComputeNodeDict()
cursesThread.insertMessage([
'informative',
'Phase 2: Updating the compute node components that need to be updated.'
])
#Instantiate the computeNode update class, which will do the work of updating the compute node.
computeNodeUpdate = ComputeNodeUpdate(cursesThread,
csurResourceDict.copy(),
timerThreadLocation)
#Get the current signal handlers so that they can be restored after the update is completed.
original_sigint_handler = signal.getsignal(signal.SIGINT)
original_sigquit_handler = signal.getsignal(signal.SIGQUIT)
'''
Setup signal handler to intercept SIGINT and SIGQUIT.
'''
s = SignalHandler(cursesThread)
signal.signal(signal.SIGINT, s.signal_handler)
signal.signal(signal.SIGQUIT, s.signal_handler)
#Create and start the worker thread.
workerThread = Thread(
target=computeNodeUpdate.updateComputeNodeComponents)
workerThread.start()
#Wait for the thread to either stop or get interrupted.
while 1:
time.sleep(0.1)
if not workerThread.is_alive():
break
'''
The response will be an empty string unless a signal was received. If a signal is
received then response is either 'n' (Don't cancel the update) or 'y' (Cancel the update.).
'''
response = s.getResponse()
if response != '':
if response == 'y':
computeNodeUpdate.endTask()
cursesThread.join()
exit(1)
#Restore the original signal handlers.
signal.signal(signal.SIGINT, original_sigint_handler)
signal.signal(signal.SIGQUIT, original_sigquit_handler)
componentProblemDict = computeNodeUpdate.getUpdateComponentProblemDict(
)
#Update the CSUR version information file.
updateVersionInformationFileResult = updateVersionInformationFile(
csurResourceDict.copy())
#Move cursor down for formatting purposes.
cursesThread.insertMessage(['info', ' '])
if len(componentProblemDict['Software']) == 0 and len(
componentProblemDict['Drivers']) == 0 and len(
componentProblemDict['Firmware']) == 0:
if not updateVersionInformationFileResult:
cursesThread.insertMessage([
'warning',
'The compute node update completed succesfully, however, the version information file update failed; check the log file for errors and update the file manually.'
])
cursesThread.insertMessage(['info', ' '])
if computeNodeDict['externalStoragePresent']:
cursesThread.insertMessage([
'final',
'Once the version information file is updated, power cycle the attached storage controller(s) and reboot the system for the changes to take effect.'
])
else:
cursesThread.insertMessage([
'final',
'Once the version information file is updated, reboot the system for the changes to take effect.'
])
else:
cursesThread.insertMessage([
'informative',
'The compute node update completed succesfully.'
])
cursesThread.insertMessage(['info', ' '])
if computeNodeDict['externalStoragePresent']:
cursesThread.insertMessage([
'final',
'Power cycle the attached storage controller(s) and reboot the system for the changes to take effect.'
])
else:
cursesThread.insertMessage([
'final',
'Reboot the system for the changes to take effect.'
])
else:
errorMessage = 'The following components encountered errors during the update; check the log file for errors:\n'
if len(componentProblemDict['Software']) != 0:
errorMessage += 'Software: ' + ', '.join(
componentProblemDict['Software'].keys()) + '\n'
if len(componentProblemDict['Drivers']) != 0:
errorMessage += 'Drivers: ' + ', '.join(
componentProblemDict['Drivers'].keys()) + '\n'
if len(componentProblemDict['Firmware']) != 0:
errorMessage += 'Firmware: ' + ', '.join(
componentProblemDict['Firmware'].keys()) + '\n'
if not updateVersionInformationFileResult:
errorMessage += 'Also, the version information file update failed; check the log file for errors and update the file manually.'
cursesThread.insertMessage(['error', errorMessage])
else:
pass
cursesThread.insertMessage(['info', ' '])
cursesThread.getUserInput(['informative', 'Press enter to exit.'])
while not cursesThread.isUserInputReady():
time.sleep(0.1)
except Exception:
cursesThread.join()
traceback.print_exc()
exit(1)
finally:
cursesThread.join()