def backupPreparation(upgradeWorkingDir, programParentDir, osDistVersion):
logger = logging.getLogger('coeOSUpgradeLogger')
debugLogger = logging.getLogger('coeOSUpgradeDebugLogger')
resourceFile = 'upgradeResourceFile'
ctrlFileImg = ''
hanaSharedWithSID = False
osDist = osDistVersion[:4]
dl360NFSNode = True
mellanoxPresent = False
bootFromSAN = False
if osDist == 'SLES':
previousUpgradeDir = '/var/log/CoESapHANA_SLES_UpgradeLogDir'
else:
previousUpgradeDir = '/var/log/CoESapHANA_RHEL_UpgradeLogDir'
upgradeResourceDict = {}
try:
with open(resourceFile, 'r') as f:
for line in f:
line = line.strip()
if len(line) == 0 or re.match('^\\s*#', line) or re.match(
'^\\s+$', line):
continue
else:
line = re.sub('[\'"]', '', line)
key, val = line.split('=')
key = key.strip()
upgradeResourceDict[key] = re.sub('\\s*,\\s*', ',',
val).strip()
except IOError as err:
debugLogger.error(
"Unable to access the application's resource file '" +
resourceFile + "'.\n" + str(err))
print RED + "Unable to access the application's resource file '" + resourceFile + "'; fix the problem and try again; exiting program execution." + RESETCOLORS
exit(1)
try:
shutil.copy2(resourceFile, upgradeWorkingDir)
except OSError as err:
debugLogger.error("Unable to copy the upgrade resource file to '" +
upgradeWorkingDir + "'.\n" + str(err))
print RED + "Unable to copy the upgrade resource file to '" + upgradeWorkingDir + "'; fix the problem and try again; exiting program execution." + RESETCOLORS
exit(1)
if osDist == 'SLES':
initialOSDict = dict.fromkeys(
(x
for x in upgradeResourceDict['initialSLESOSVersions'].split(',')))
else:
initialOSDict = dict.fromkeys(
(x
for x in upgradeResourceDict['initialRHELOSVersions'].split(',')))
debugLogger.info('The initial OS version dictionary is: ' +
str(initialOSDict))
if osDistVersion not in initialOSDict:
debugLogger.error("The server's OS distribution (" + osDistVersion +
') is not supported for an upgrade.')
print RED + "The server's OS distribution (" + osDistVersion + ') is not supported for an upgrade; exiting program execution.' + RESETCOLORS
exit(1)
serverModel = getServerModel()
if 'DL580' in serverModel:
serverProcessor = getProcessorType()
if osDist == 'SLES':
if serverModel not in upgradeResourceDict['slesSupportedServerList']:
debugLogger.error('The ' + serverModel +
' is not supported. Supported models are: ' +
upgradeResourceDict['supportedServerList'] + '.')
print RED + 'The ' + serverModel + ' is not supported; exiting program execution.' + RESETCOLORS
exit(1)
elif serverModel not in upgradeResourceDict['rhelSupportedServerList']:
debugLogger.error('The ' + serverModel +
' is not supported. Supported models are: ' +
upgradeResourceDict['supportedServerList'] + '.')
print RED + 'The ' + serverModel + ' is not supported; exiting program execution.' + RESETCOLORS
exit(1)
if serverModel == 'ProLiant DL360 Gen9':
command = 'rpm -q serviceguard-A.*'
result = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
out, err = result.communicate()
if result.returncode != 0 and 'package serviceguard-A.* is not installed' not in out:
debugLogger.error(
'Unable to determine if the server is a Serviceguard quorum server or Serviceguard NFS server:\n'
+ err + '\n' + out)
print RED + 'Unable to determine if the server is a Serviceguard quorum server or Serviceguard NFS server; fix the problem and try again; exiting program execution.' + RESETCOLORS
exit(1)
elif 'package servicguard-A.* is not installed' in out:
debugLogger.error(
'The ' + serverModel +
' is not supported, since it is not a NFS Serviceguard server.\n'
+ out)
print RED + 'The ' + serverModel + ' is not supported, since it is not a NFS Serviceguard server; exiting program execution.' + RESETCOLORS
exit(1)
try:
if osDist == 'SLES':
supportedOSVersions = dict(
(re.sub('\\s*:\\s*', ':', x).split(':') for x in
upgradeResourceDict['supportedSLESOSVersions'].split(',')))
else:
supportedOSVersions = dict(
(re.sub('\\s*:\\s*', ':', x).split(':') for x in
upgradeResourceDict['supportedRHELOSVersions'].split(',')))
except KeyError as err:
debugLogger.error(
'The resource key ' + str(err) +
" was not present in the application's resource file.")
print RED + 'The resource key ' + str(
err
) + " was not present in the application's resource file; fix the problem and try again; exiting program execution." + RESETCOLORS
exit(1)
osUpgradeVersion = supportedOSVersions[osDistVersion][5:]
debugLogger.info('The server is being upgraded to ' + osUpgradeVersion +
'.')
if osDist == 'RHEL':
logDir = '/var/log/CoESapHANA_' + osDist + '_UpgradeLogDir'
if not os.path.isdir(logDir):
try:
os.mkdir(logDir)
except OSError as err:
debugLogger.error(
"Unable to create the upgrade log directory '" + logDir +
"'.\n" + str(err))
print RED + 'Unable to create the upgrade log directory; fix the problem and try again; exiting program execution.' + RESETCOLORS
exit(1)
else:
fileList = glob.glob(logDir + '/*')
for file in fileList:
try:
os.remove(logDir + '/' + file)
except OSError as err:
debugLogger.error("Unable to remove the file '" + logDir +
'/' + file + "'.\n" + str(err))
print RED + "Unable to remove the file '" + logDir + '/' + file + "'; fix the problem and try again; exiting program execution." + RESETCOLORS
exit(1)
time.sleep(1.0)
try:
with open(logDir + '/upgradeVersionReference.log', 'w') as f:
f.write(osUpgradeVersion)
except OSError as err:
debugLogger.error(
'Could not access ' + logDir +
'/upgradeVersionReference.log to write out the OS upgrade version.\n'
+ str(err))
print RED + 'Could not access ' + logDir + '/upgradeVersionReference.log to write out the OS upgrade version; fix the problem and try again; exiting program execution.' + RESETCOLORS
exit(1)
if osDist == 'SLES':
osVersion = osDist + osUpgradeVersion[:2]
else:
osVersion = osDist + osUpgradeVersion[:1]
if os.path.isdir(previousUpgradeDir):
try:
shutil.rmtree(previousUpgradeDir)
except OSError as err:
debugLogger.error(
"Problems were encountered while removing a previous upgrade directory '"
+ previousUpgradeDir + "'.\n" + str(err))
print RED + 'Problems were encountered while removing a previous upgrade directory; fix the problem and try again; exiting program execution.' + RESETCOLORS
exit(1)
if 'DL580' in serverModel or serverModel == 'Superdome':
if osDist == 'RHEL':
supportedSAPHANARevisionList = dict(
((x.split(':')[0].strip(), re.sub('\\s+', '',
x.split(':')[1]))
for x in upgradeResourceDict['rhelSupportedSAPHANARevision'].
split(',')))['RHEL ' + osUpgradeVersion].split('|')
else:
supportedSAPHANARevisionList = dict(
((x.split(':')[0].strip(), re.sub('\\s+', '',
x.split(':')[1]))
for x in upgradeResourceDict['slesSupportedSAPHANARevision'].
split(',')))['SLES ' + osUpgradeVersion].split('|')
debugLogger.info('The supported SAP HANA revision list was: ' +
str(supportedSAPHANARevisionList))
checkSAPHANA(upgradeWorkingDir, serverModel,
supportedSAPHANARevisionList)
if serverModel == 'ProLiant DL360 Gen9' and dl360NFSNode or serverModel == 'ProLiant DL380p Gen8' or serverModel == 'ProLiant DL360p Gen8':
checkClusterStatus()
removeServiceguardExtras(
upgradeResourceDict['extraServiceguardPackages'])
stageServiceguardRPMS(
programParentDir, upgradeWorkingDir, osVersion,
upgradeResourceDict[osVersion.lower() + 'NFSServiceguardRPMS'])
if serverModel == 'ProLiant DL360 Gen9' and not dl360NFSNode or serverModel == 'ProLiant DL320e Gen8 v2':
stageServiceguardRPMS(
programParentDir, upgradeWorkingDir, osVersion,
upgradeResourceDict[osVersion.lower() + 'QSServiceguardRPMS'])
if osDist == 'SLES':
postUpgradeScripts = programParentDir + '/postUpgradeScriptFiles/SLES/' + osUpgradeVersion + '/*'
else:
postUpgradeScripts = programParentDir + '/postUpgradeScriptFiles/RHEL/' + osUpgradeVersion + '/*'
command = 'cp -r ' + postUpgradeScripts + ' ' + upgradeWorkingDir
result = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
out, err = result.communicate()
if result.returncode != 0:
debugLogger.error(
"Unable to copy the post-upgrade script files from '" +
postUpgradeScripts + "' to '" + upgradeWorkingDir + ':\n' + err +
'\n' + out)
print RED + 'Unable to copy the post-upgrade script files to the upgrade working directory; fix the problem and try again; exiting program execution.' + RESETCOLORS
exit(1)
if osDist == 'RHEL':
command = 'cp ' + programParentDir + '/rhelRPMReferenceList/* ' + upgradeWorkingDir
result = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
out, err = result.communicate()
if result.returncode != 0:
debugLogger.error("Unable to copy the RPM reference file from '" +
programParentDir +
"/rhelRPMReferenceList' to '" +
upgradeWorkingDir + ':\n' + err + '\n' + out)
print RED + 'Unable to copy the the RPM reference file to the upgrade working directory; fix the problem and try again; exiting program execution.' + RESETCOLORS
exit(1)
filesystemRPMDir = '/tmp/rhelFilesystemRPM'
if not os.path.isdir(filesystemRPMDir):
try:
os.mkdir(filesystemRPMDir)
except OSError as err:
print RED + "Unable to create the directory '" + filesystemRPMDir + "'; fix the problem and try again; exiting program execution." + RESETCOLORS
exit(1)
else:
filesystemRPMS = glob.glob(filesystemRPMDir + '/filesystem*.rpm')
for file in filesystemRPMS:
try:
os.remove(filesystemRPMDir + '/' + file)
except OSError as err:
print RED + "Unable to remove the file '" + filesystemRPMDir + '/' + file + "'; fix the problem and try again; exiting program execution." + RESETCOLORS
exit(1)
time.sleep(1.0)
command = 'cp ' + programParentDir + '/rhelFilesystemRPM/filesystem*.rpm ' + filesystemRPMDir
result = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
out, err = result.communicate()
if result.returncode != 0:
debugLogger.error("Unable to copy the filesystem RPM from '" +
programParentDir + "/rhelFilesystemRPM' to '" +
filesystemRPMDir + ':\n' + err + '\n' + out)
print RED + "Unable to copy the the filesystem RPM to '" + filesystemRPMDir + "'; fix the problem and try again; exiting program execution." + RESETCOLORS
exit(1)
if 'DL580' in serverModel:
conrepDir = upgradeWorkingDir + '/conrepFile'
getConrepFile(programParentDir, conrepDir)
if osDist == 'SLES':
if 'DL580' in serverModel or serverModel == 'ProLiant DL360 Gen9' and dl360NFSNode:
command = 'lsblk -l'
result = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
out, err = result.communicate()
if result.returncode != 0:
debugLogger.error(
'Unable to get the partition information for the root file system.\n'
+ err + '\n' + out)
print RED + 'Unable to get the partition information for the root file system; fix the problem and try again; exiting program execution.' + RESETCOLORS
exit(1)
if re.match('.*(360002ac0+.*)[_-]part.*/boot/efi.*$', out,
re.DOTALL | re.MULTILINE):
debugLogger.info(
'The ' + serverModel +
"'s boot from SAN partition information was: " +
out.strip() + '.')
bootFromSAN = True
if 'DL580' in serverModel and serverProcessor == 'broadwell' or serverModel == 'Superdome':
prepareFstab(upgradeWorkingDir)
if 'DL580' in serverModel or serverModel == 'Superdome':
backupLimitsFile(upgradeWorkingDir)
if 'DL580' in serverModel and serverProcessor != 'ivybridge':
mellanoxPresent = checkForMellanox(programParentDir, upgradeWorkingDir,
osDist, osUpgradeVersion)
try:
if serverModel == 'ProLiant DL360 Gen9' and dl360NFSNode or serverModel == 'ProLiant DL380p Gen8' or serverModel == 'ProLiant DL360p Gen8':
archiveBackup = re.split(
',', upgradeResourceDict['osArchiveBackup']) + re.split(
',', upgradeResourceDict['sglxArchiveBackup'])
else:
archiveBackup = re.split(
',', upgradeResourceDict['osArchiveBackup']) + re.split(
',', upgradeResourceDict['sapArchiveBackup'])
except KeyError as err:
debugLogger.error(
'The resource key ' + str(err) +
" was not present in the application's resource file.")
print RED + 'The resource key ' + str(
err
) + " was not present in the application's resource file; fix the problem and try again; exiting program execution." + RESETCOLORS
exit(1)
debugLogger.info('The archive backup list is: ' + str(archiveBackup) + '.')
getOriginalRPMList(upgradeWorkingDir)
archiveBackup, fileRemoved = confirmBackupDataList(archiveBackup)
if fileRemoved:
print YELLOW + 'File(s) were removed from the backup, since they were not present; review the log file for additional information.' + RESETCOLORS
checkDiskspace(archiveBackup)
if osDist == 'SLES':
removeRear()
disableMultiversionSupport()
if serverModel == 'Superdome' or serverModel == 'ProLiant DL380p Gen8' or serverModel == 'ProLiant DL360p Gen8':
ctrlFileImg = updateUpgradeCtrlFile(programParentDir,
upgradeWorkingDir,
osUpgradeVersion, serverModel)
elif bootFromSAN:
ctrlFileImg = updateUpgradeCtrlFile(
programParentDir,
upgradeWorkingDir,
osUpgradeVersion,
serverModel,
bootFromSAN='bootFromSAN',
mellanoxPresent=mellanoxPresent)
else:
ctrlFileImg = updateUpgradeCtrlFile(
programParentDir,
upgradeWorkingDir,
osUpgradeVersion,
serverModel,
mellanoxPresent=mellanoxPresent)
createNetworkInformationFile(upgradeWorkingDir, osDist)
return (archiveBackup, ctrlFileImg)
def main():
programVersion = '1.0-0'
programName = os.path.basename(sys.argv[0]).split('.')[0]
programParentDir = os.path.dirname(os.path.realpath(__file__))
errorsEncountered = False
upgradeFeedbackThread = None
if os.geteuid() != 0:
print RED + 'You must be root to run this program; exiting program execution.' + RESETCOLORS
exit(1)
usage = 'usage: %prog [-h] [-v]'
parser = optparse.OptionParser(usage=usage)
parser.add_option('-v', action='store_true', default=False, help="This option is used to display the application's version.")
options, args = parser.parse_args()
if options.v:
print programName + ' ' + programVersion
exit(0)
osDist, _ = getOSDistribution()
logDir = '/var/log/CoESapHANA_' + osDist + '_UpgradeLogDir'
if not os.path.isdir(logDir):
print RED + "The upgrade log directory '" + logDir + "' is missing, it was created by the preupgrade script; fix the problem and try again; exiting program execution.\n" + str(err) + RESETCOLORS
exit(1)
upgradeLogFile = logDir + '/upgrade.log'
try:
with open(logDir + '/upgradeVersionReference.log', 'r') as f:
osUpgradeVersion = f.readline().strip()
except OSError as err:
upgradeLogger.error("Errors were encountered while reading the OS upgrade version log file '" + logDir + "/upgradeVersionReference.log'.\n" + str(err))
print RED + 'Errors were encountered while reading the OS upgrade version log file; fix the problem and try again; exiting program execution.' + RESETCOLORS
exit(1)
upgradeHandler = logging.FileHandler(upgradeLogFile)
formatter = logging.Formatter('%(asctime)s:%(levelname)s:%(message)s', datefmt='%m/%d/%Y %H:%M:%S')
upgradeHandler.setFormatter(formatter)
upgradeLogger = logging.getLogger('coeOSUpgradeLogger')
upgradeLogger.setLevel(logging.INFO)
upgradeLogger.addHandler(upgradeHandler)
upgradeLogger.info("The program's version is: " + programVersion + '.')
upgradeLogger.info('Upgrading the server to ' + osUpgradeVersion + '.')
print GREEN + 'Upgrading the server to ' + osUpgradeVersion + '.' + RESETCOLORS
serverModel = getServerModel()
if 'DL580' in serverModel:
motdInfo = 'HPE SAP HANA CS500 Converged System'
releaseInfo = 'HPE SAP HANA CoE CS500'
elif serverModel == 'ProLiant DL360 Gen9' or serverModel == 'ProLiant DL380p Gen8':
motdInfo = 'HPE SAP HANA Serviceguard NFS Server'
releaseInfo = 'HPE SAP HANA CoE Serviceguard NFS Server'
elif serverModel == 'Superdome':
motdInfo = 'HPE SAP HANA CS900 Converged System'
releaseInfo = 'HPE SAP HANA CoE CS900'
progressStatusFile = logDir + '/progress.log'
try:
if os.path.isfile(progressStatusFile):
f = open(progressStatusFile, 'a+')
progressDict = dict.fromkeys((x.strip() for x in f))
else:
f = open(progressStatusFile, 'w')
progressDict = {}
if 'cleanupGrubCfg' not in progressDict:
cleanupGrubCfg()
f.write('cleanupGrubCfg\n')
if 'filesystemRPMUpgraded' not in progressDict:
upgradeLogger.info('Updating the filesystem package.')
command = 'rpm -Uvh /tmp/rhelFilesystemRPM/filesystem*.rpm'
result = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
out, err = result.communicate()
if result.returncode != 0:
upgradeLogger.error('Failed to upgrade the filesystem RPM (/tmp/rhelFilesystemRPM/filesystem*.rpm):\n' + err + '\n' + out)
print RED + 'Failed to upgrade the filesystem RPM (/tmp/rhelFilesystemRPM/filesystem*.rpm); fix the problem and try again; exiting program execution.' + RESETCOLORS
exit(1)
else:
f.write('filesystemRPMUpgraded\n')
imageDevice = '/dev/sr0'
while True:
command = 'mount -o loop ' + imageDevice + ' /mnt'
result = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
out, err = result.communicate()
if result.returncode != 0:
if 'is already mounted' not in err.lower():
upgradeLogger.error('Failed to mount the RHEL DVD on /mnt.\n' + err + '\n' + out)
upgradeLogger.info('The command used to mount the RHEL DVD on /mnt was: ' + command)
upgradeLogger.info("Prompting for the DVD's scsci device file.")
print RED + 'Failed to mount the RHEL DVD using device file ' + imageDevice + '.\n' + RESETCOLORS
response = raw_input("Please provide the correct device file for the RHEL DVD (hint check dmesg (tail dmesg)) or 'q' to quit: ")
response = response.strip()
if response.lower() == 'q':
upgradeLogger.info('The upgrade was cancelled.')
exit(1)
else:
imageDevice = response
else:
upgradeLogger.info('An attempt to mount the DVD reported that it was already mounted; thus, continuing the upgrade.\n' + err + '\n' + out)
break
else:
break
try:
command = 'yum --disablerepo=* clean all'
result = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
out, err = result.communicate()
if result.returncode != 0:
if 'there are no enabled repos' not in err.lower():
upgradeLogger.error('Errors were encountered while disabling the current repositories.\n' + err + '\n' + out)
upgradeLogger.info('The command used to disable the current repositories was: ' + command)
print RED + 'Errors were encountered while disabling the current repositories; fix the problem and try again; exiting program execution.' + RESETCOLORS
exit(1)
if os.path.isfile('/etc/yum.repos.d/HPECoERHELUpgrade.repo'):
try:
os.remove('/etc/yum.repos.d/HPECoERHELUpgrade.repo')
time.sleep(1.0)
except OSError as err:
upgradeLogger.error('Errors were encountered while removing the upgrade repository /etc/yum.repos.d/HPECoERHELUpgrade.repo.\n' + str(err))
print RED + 'Errors were encountered while removing the upgrade repository; the repository will need to be removed manually.' + RESETCOLORS
exit(1)
if not os.path.isdir('/tmp/CoEYumRepoBackupDir'):
try:
os.mkdir('/tmp/CoEYumRepoBackupDir')
except OSError as err:
upgradeLogger.error('Errors were encountered while creating the yum repository backup directory /tmp/CoEYumRepoBackupDir.\n' + str(err))
print RED + 'Errors were encountered while creating the yum repository backup directory; fix the problem and try again; exiting program execution.' + RESETCOLORS
exit(1)
try:
repoFiles = os.listdir('/etc/yum.repos.d')
for file in repoFiles:
shutil.move('/etc/yum.repos.d/' + file, '/tmp/CoEYumRepoBackupDir/' + file)
except (OSError, IOError, Error) as err:
upgradeLogger.error('Errors were encountered while moving the yum repository files from /etc/yum.repos.d to backup directory /tmp/CoEYumRepoBackupDir.\n' + str(err))
print RED + 'Errors were encountered while moving the yum repository files to the backup directory; fix the problem and try again; exiting program execution.' + RESETCOLORS
exit(1)
command = 'cp /mnt/media.repo /etc/yum.repos.d/HPECoERHELUpgrade.repo'
result = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
out, err = result.communicate()
if result.returncode != 0:
upgradeLogger.error('Failed to get a copy of the RHEL DVD repo file.\n' + err + '\n' + out)
upgradeLogger.info('The command used to get a copy of the RHEL DVD repo file was: ' + command)
print RED + 'Failed to get a copy of the RHEL DVD repo file; fix the problem and try again; exiting program execution.' + RESETCOLORS
exit(1)
if 'serverUpgraded' not in progressDict:
upgradeLogger.info('Updating the baseurl for the install media in the upgrade repo file /etc/yum.repos.d/HPECoERHELUpgrade.repo. Also, adding the SAPHana repository to the file.')
try:
with open('/etc/yum.repos.d/HPECoERHELUpgrade.repo', 'a') as repoFile:
repoFile.write('baseurl=file:///mnt\n')
repoFile.write('enabled=1\n')
repoFile.write('\n')
repoFile.write('[SAP_HANA_RPMS]\n')
repoFile.write('name=SAP HANA RPMS\n')
repoFile.write('baseurl=file:///mnt/addons/SAPHana\n')
repoFile.write('enabled=1\n')
repoFile.write('gpgcheck=0\n')
except OSError as err:
upgradeLogger.error('Errors were encountered while updating the upgrade repo file (/etc/yum.repos.d/HPECoERHELUpgrade.repo).\n' + str(err))
print RED + 'Errors were encountered while updating the upgrade repo file; fix the problem and try again; exiting program execution.' + RESETCOLORS
exit(1)
upgradeLogger.info('Applying updates to upgrade the server.')
upgradeFeedbackThread = UpgradeFeedbackThread()
upgradeFeedbackThread.setMessage('Applying updates to upgrade the server.')
upgradeFeedbackThread.start()
for output in processRPMS('upgrade'):
upgradeLogger.info(output)
f.write('serverUpgraded\n')
if 'installedAdditionalRPMS' not in progressDict:
command = 'rpm -qa --qf "%{NAME}\n" | sort > /tmp/currentRHEL7.4RPMList'
result = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
out, err = result.communicate()
if result.returncode != 0:
upgradeLogger.error("Errors were encountered while getting the server's RPM list.\n" + err + '\n' + out)
upgradeLogger.info("The command used to get the server's RPM list was: " + command)
print RED + "\nErrors were encountered while getting the server's RPM list; fix the problem and try again; exiting program execution." + RESETCOLORS
exit(1)
command = 'comm -23 ' + programParentDir + '/rhelRPMReferenceList/rhel7.4RPMList /tmp/currentRHEL7.4RPMList'
result = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
out, err = result.communicate()
if result.returncode != 0:
upgradeLogger.error('Errors were encountered while getting the list of RPMS that were not installed.\n' + err + '\n' + out)
upgradeLogger.info('The command used to get the list of RPMS that were not installed was: ' + command)
print RED + '\nErrors were encountered while getting the list of RPMS that were not installed; fix the problem and try again; exiting program execution.' + RESETCOLORS
exit(1)
rpmsNotInstalledList = out.splitlines()
if len(rpmsNotInstalledList) > 0:
upgradeLogger.info('Installing additional RPMS to enable uniformity amongst ConvergedSystems.')
if upgradeFeedbackThread == None:
upgradeFeedbackThread = UpgradeFeedbackThread()
upgradeFeedbackThread.setMessage('Installing additional RPMS to enable uniformity amongst ConvergedSystems.')
upgradeFeedbackThread.start()
for output in processRPMS('install'):
upgradeLogger.info(output)
else:
upgradeLogger.info('A comparison of the RPMS installed and what was listed in RPM reference list showed that no additional RPMS needed to be installed.')
f.write('installedAdditionalRPMS\n')
if 'appliedSecurityPatches' not in progressDict:
try:
with open('/etc/yum.repos.d/HPECoERHELUpgrade.repo', 'a') as repoFile:
repoFile.write('baseurl=file:///mnt\n')
repoFile.write('enabled=1\n')
repoFile.write('\n')
repoFile.write('[Security_Patches]\n')
repoFile.write('name=Security Patches\n')
repoFile.write('baseurl=file:///mnt/addons/SecurityPatches\n')
repoFile.write('enabled=1\n')
repoFile.write('gpgcheck=0\n')
except OSError as err:
upgradeLogger.error('Errors were encountered while updating the upgrade repo file with security patch information.\n' + str(err))
print RED + 'Errors were encountered while updating the upgrade repo file; fix the problem and try again; exiting program execution.' + RESETCOLORS
exit(1)
if upgradeFeedbackThread == None:
upgradeFeedbackThread = UpgradeFeedbackThread()
upgradeLogger.info('Applying security patches to the server.')
upgradeFeedbackThread.setMessage('Applying security patches to the server.')
upgradeFeedbackThread.start()
for output in processRPMS('install'):
upgradeLogger.info(output)
f.write('appliedSecurityPatches\n')
finally:
if not upgradeFeedbackThread == None:
upgradeFeedbackThread.stopThread()
command = 'umount /mnt'
result = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
out, err = result.communicate()
if result.returncode != 0:
upgradeLogger.error('Errors were encountered while unmounting the OS DVD.\n' + err + '\n' + out)
except IOError as err:
upgradeLogger.error('Could not access ' + progressStatusFile + '.\n' + str(err))
print RED + "Could not access '" + progressStatusFile + "'; fix the problem and try again." + RESETCOLORS
finally:
f.close()
upgradeLogger.info('Removing the upgrade repository and restoring the original repositories.')
try:
os.remove('/etc/yum.repos.d/HPECoERHELUpgrade.repo')
except OSError as err:
upgradeLogger.error('Errors were encountered while removing the upgrade repository /etc/yum.repos.d/HPECoERHELUpgrade.repo.\n' + str(err))
print RED + 'Errors were encountered while removing the upgrade repository; the repository will need to be removed manually.' + RESETCOLORS
try:
repoFiles = os.listdir('/tmp/CoEYumRepoBackupDir')
for file in repoFiles:
shutil.move('/tmp/CoEYumRepoBackupDir/' + file, '/etc/yum.repos.d/' + file)
os.rmdir('/tmp/CoEYumRepoBackupDir/')
except (OSError, IOError, Error) as err:
upgradeLogger.error('Errors were encountered while moving the repository files in /tmp/CoEYumRepoBackupDir back to /etc/yum.repos.d and then deleting the directory.\n' + str(err))
print RED + 'Errors were encountered while moving the repository files back in place; the files will need to be moved manually and the backup directory removed.' + RESETCOLORS
upgradeLogger.info('Updating /etc/motd with the message:\n\tThis is RHEL for SAP Applications on a ' + motdInfo + '.')
try:
with open('/etc/motd', 'w') as motdFile:
motdFile.write('This is RHEL for SAP Applications on a ' + motdInfo + '.')
except OSError as err:
upgradeLogger.error('Errors were encountered while updating the motd file.\n' + str(err))
upgradeLogger.info('The following was being written to /etc/motd:\n' + 'This is RHEL for SAP Applications on a ' + motdInfo + '.')
print RED + 'Errors were encountered while updating the motd file; the file will have to be updated manually, see the log file for the message.' + RESETCOLORS
date = datetime.datetime.now()
upgradeLogger.info('Updating /etc/hpe-release with the release information:\n\t' + releaseInfo + ' Upgrade Release ' + programVersion + '.\n\tUpgraded on ' + date.strftime('%b %d, %Y @ %H:%M:%S') + '.')
try:
with open('/etc/hpe-release', 'w') as releaseFile:
releaseFile.write(releaseInfo + ' Upgrade Release ' + programVersion + '.\n')
releaseFile.write('Upgraded on ' + date.strftime('%b %d, %Y @ %H:%M:%S') + '.')
except OSError as err:
upgradeLogger.error('Errors were encountered while updating the hpe-release file.\n' + str(err))
upgradeLogger.info('The following was being written to /etc/hpe-release:\n' + releaseInfo + ' Upgrade Release ' + programVersion + '.\nUpgraded on ' + date.strftime('%b %d, %Y @ %H:%M:%S') + '.')
print RED + 'Errors were encountered while updating the hpe-releae file; the file will have to be updated manually, see the log file for its contents.' + RESETCOLORS
upgradeLogger.info('Disabling kdump and openwsmand if they are enabled.')
serviceList = ['kdump.service', 'openwsmand.service']
for service in serviceList:
command = 'systemctl is-enabled ' + service
result = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
out, err = result.communicate()
if result.returncode != 0 and out.strip() != 'disabled':
upgradeLogger.error('Errors were encountered while checking to see if ' + service + ' is enabled.\n' + err + '\n' + out)
upgradeLogger.info('The command used to check if ' + service + ' is enabled was: ' + command)
print RED + 'Errors were encountered while checking to see if ' + service + ' is enabled before disabling the service; the service will have to be checked and disabled manually.' + RESETCOLORS
elif out.strip() == 'enabled':
command = 'systemctl disable ' + service
result = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
out, err = result.communicate()
if result.returncode != 0:
upgradeLogger.error('Errors were encountered while disabling ' + service + '.\n' + err + '\n' + out)
upgradeLogger.info('The command used to disable ' + service + ' was: ' + command)
print RED + 'Errors were encountered while disabling ' + service + '; the service will have to be disabled manually.' + RESETCOLORS
upgradeLogger.info('Done upgrading the server to ' + osUpgradeVersion + '.')
print GREEN + 'Done upgrading the server to ' + osUpgradeVersion + '; proceed with the next steps in the upgrade process.' + RESETCOLORS
return
def main():
programVersion = '2018.03-0'
addOnSoftwareInstalled = False
hanaPartitionsRestored = False
scaleUpSystem = False
errorMessageList = []
if os.geteuid() != 0:
print RED + 'You must be root to run this program; exiting program execution.' + RESETCOLORS
exit(1)
usage = 'usage: %prog [[-h] [-p] [-v]]'
parser = optparse.OptionParser(usage = usage)
parser.add_option('-p', action = 'store_true', default = False, help = 'This option should be ran from the primary NFS Serviceguard node after the upgrade of both nodes when the upgrade caused the NIC cards to be renamed on one or both of the NFS Serviceguard nodes.')
parser.add_option('-v', action = 'store_true', default = False, help = "This option is used to display the application's version.")
(options, args) = parser.parse_args()
programName = os.path.basename(sys.argv[0])
if options.v:
print programName + ' ' + programVersion
exit(0)
(osDist, osDistLevel) = getOSDistribution()
upgradeLogDir = '/var/log/CoESapHANA_' + osDist + '_UpgradeLogDir'
if not os.path.isdir(upgradeLogDir):
try:
os.mkdir(upgradeLogDir)
except OSError:
err = None
print RED + "Unable to create the post upgrade log directory '" + upgradeLogDir + "'; fix the problem and try again; exiting program execution.\n" + str(err) + RESETCOLORS
exit(1)
dateTimestamp = datetime.datetime.now().strftime('%d%H%M%b%Y')
logFile = upgradeLogDir + '/' + osDist + '_Post_Upgrade_' + dateTimestamp + '.log'
handler = logging.FileHandler(logFile)
logger = logging.getLogger('coeOSUpgradeLogger')
logger.setLevel(logging.INFO)
formatter = logging.Formatter('%(asctime)s:%(levelname)s:%(message)s', datefmt = '%m/%d/%Y %H:%M:%S')
handler.setFormatter(formatter)
logger.addHandler(handler)
logger.info('The current version of the program is: ' + programVersion + '.')
serverModel = getServerModel()
if 'DL580' in serverModel:
processor = getProcessorType()
if options.p:
if not 'DL380p' in serverModel or 'DL360p' in serverModel:
print RED + "The '-p' option is only for Serviceguard NFS servers; exiting program execution." + RESETCOLORS
exit(1)
updateClusterCfg()
else:
try:
programParentDir = ''
cwd = os.getcwd()
programParentDir = os.path.dirname(os.path.realpath(__file__))
if cwd != programParentDir:
os.chdir(programParentDir)
except OSError:
err = None
logger.error("Unable to change into the programs parent directory '" + programParentDir + "'.\n" + str(err))
print RED + "Unable to change into the programs parent directory '" + programParentDir + "'; fix the problem and try again; exiting program execution." + RESETCOLORS
exit(1)
if osDist == 'SLES':
checkNetworkConfiguration = checkSLESNetworkConfiguration
import modules.upgradeMisc
else:
checkNetworkConfiguration = checkRHELNetworkConfiguration
import modules.upgradeMisc
if 'DL580' in serverModel or serverModel == 'Superdome':
if osDist == 'SLES':
updateOSSettings = updateSLESOSSettings
import modules.sapHANANodeSpec
else:
updateOSSettings = updateRHELOSSettings
import modules.sapHANANodeSpec
if 'DL580' in serverModel:
command = 'cat /proc/scsi/scsi'
result = subprocess.Popen(command, stdout = subprocess.PIPE, stderr = subprocess.PIPE, shell = True)
(out, err) = result.communicate()
out = out.strip()
if result.returncode != 0:
logger.error('Unable to determine if the DL580 is a Scale-up.\n' + str(out) + '\n' + str(err))
print RED + 'Unable to determine if the DL580 is a Scale-up; fix the problem and try again; exiting program execution.' + RESETCOLORS
exit(1)
searchResult = re.search('P830i', out, re.MULTILINE | re.DOTALL)
if not searchResult == None:
globalIni = 'global.ini'
globalIniList = []
scaleOutSystem = False
for (root, dirs, files) in os.walk(programParentDir):
if globalIni in files:
globalIniList.append(root + '/' + globalIni)
continue
for file in globalIniList:
with open(file, 'r') as f:
data = f.read()
if re.match('.*log|data_{1,}wwid\\s*=\\s*360002ac0+', data, re.MULTILINE | re.DOTALL) != None:
scaleOutSystem = True
multipathArguments = [
'enable',
'start']
for argument in multipathArguments:
command = 'systemctl ' + argument + ' multipathd'
result = subprocess.Popen(command, stdout = subprocess.PIPE, stderr = subprocess.PIPE, shell = True)
(out, err) = result.communicate()
out = out.strip()
if result.returncode != 0:
logger.error('Unable to ' + argument + ' multipathd.\n' + str(out) + '\n' + str(err))
print RED + 'Unable to ' + argument + ' multipathd; fix the problem and try again; exiting program execution.' + RESETCOLORS
exit(1)
time.sleep(0.1)
break
if not scaleOutSystem:
scaleUpSystem = True
upgradeResourceDict = { }
try:
with open('upgradeResourceFile') as f:
for line in f:
line = line.strip()
if len(line) == 0 and re.match('^\\s*#', line) or re.match('^\\s+$', line):
continue
continue
line = re.sub('[\'"]', '', line)
(key, val) = line.split('=')
key = key.strip()
upgradeResourceDict[key] = re.sub('\\s*,\\s*', ',', val).strip()
except IOError:
err = None
logger.error("Unable to access the application's resource file 'upgradeResourceFile'.\n" + str(err))
print RED + "Unable to access the application's resource file 'upgradeResourceFile'; fix the problem and try again; exiting program execution." + RESETCOLORS
exit(1)
sessionScreenLog = upgradeLogDir + '/sessionScreenLog.log'
cursesLog = upgradeLogDir + '/cursesLog.log'
try:
cursesThread = CursesThread(sessionScreenLog, cursesLog)
cursesThread.daemon = True
cursesThread.start()
osRestorationArchiveErrorFile = upgradeLogDir + '/.osRestorationArchiveError'
if not os.path.isfile(osRestorationArchiveErrorFile):
try:
os.mknod(osRestorationArchiveErrorFile)
except OSError:
err = None
logger.error("Unable to create the OS restoration archive error file '" + osRestorationArchiveErrorFile + "'.\n" + str(err))
displayErrorMessage("Unable to create the OS restoration archive error file '" + osRestorationArchiveErrorFile + "'; fix the problem and try again.", cursesThread)
progressStatusFile = upgradeLogDir + '/.restorationProgress'
try:
if os.path.isfile(progressStatusFile):
with open(progressStatusFile) as f:
progressDict = dict.fromkeys(lambda .0: continue(f))
else:
progressDict = { }
if os.path.isfile(progressStatusFile):
f = open(progressStatusFile, 'a')
else:
f = open(progressStatusFile, 'w')
osRestorationArchiveExtracted = False
try:
if os.stat(osRestorationArchiveErrorFile).st_size < 50 and 'osRestorationArchiveExtracted' not in progressDict:
cursesThread.insertMessage([
'informative',
'Checking and extracting the OS restoration archive file.'])
cursesThread.insertMessage([
'informative',
''])
osRestorationArchive = checkOSRestorationArchive(programParentDir, osRestorationArchiveErrorFile, osDist, cursesThread)
extractOSRestorationArchive(osRestorationArchive, osRestorationArchiveErrorFile, cursesThread)
f.write('osRestorationArchiveExtracted\n')
osRestorationArchiveExtracted = True
except OSError:
err = None
logger.error("Unable to access the OS restoration archive error file '" + osRestorationArchiveErrorFile + "'.\n" + str(err))
displayErrorMessage("Unable to access the OS restoration archive error file '" + osRestorationArchiveErrorFile + "'; fix the problem and try again.", cursesThread)
if os.path.isdir(programParentDir + '/addOnSoftwareRPMS'):
if 'installaddOnSoftware' not in progressDict:
f.write('installaddOnSoftware\n')
if osDist == 'SLES':
(errorMessage, addOnSoftwareInstalled) = installSLESAddOnSoftware(programParentDir, cursesThread)
else:
(errorMessage, addOnSoftwareInstalled) = installRHELAddOnSoftware(programParentDir, cursesThread)
if len(errorMessage) > 0:
errorMessageList.append(errorMessage)
else:
errorMessageList.append('The additional RPMs were not installed, since an attempt to add them was already made.')
else:
addOnSoftwareInstalled = True
logger.info('There were no additional RPMs present for the post-upgrade.')
if 'setTimezone' not in progressDict:
f.write('setTimezone\n')
errorMessage = setTimezone(programParentDir, cursesThread)
if len(errorMessage) > 0:
errorMessageList.append(errorMessage)
else:
errorMessageList.append('The time zone was not set, since an attempt to set it was already made.')
if 'checkNetworkConfiguration' not in progressDict:
if 'osRestorationArchiveExtracted' in progressDict or osRestorationArchiveExtracted:
f.write('checkNetworkConfiguration\n')
if (serverModel == 'ProLiant DL320e Gen8 v2' or serverModel == 'ProLiant DL360 Gen9') and osDistLevel == '11.4':
errorMessage = checkNetworkConfiguration(programParentDir, osDist, cursesThread, osDistLevel = osDistLevel)
else:
errorMessage = checkNetworkConfiguration(programParentDir, osDist, cursesThread)
if len(errorMessage) > 0:
errorMessageList.append(errorMessage)
else:
errorMessageList.append('The network configuration was not checked; either because it was already checked or the OS restoration archive was not successfully extracted.')
if not (serverModel == 'ProLiant DL320e Gen8 v2' or serverModel == 'ProLiant DL360 Gen9') and osDistLevel == '11.4':
if 'setHostname' not in progressDict:
f.write('setHostname\n')
errorMessage = setHostname(programParentDir, cursesThread)
if len(errorMessage) > 0:
errorMessageList.append(errorMessage)
else:
errorMessageList.append('The host name was not set, since an attempt to set it was already made.')
if osDist == 'SLES':
if 'updateNTPConf' not in progressDict:
f.write('updateNTPConf\n')
errorMessage = updateNTPConf(cursesThread)
if len(errorMessage) > 0:
errorMessageList.append(errorMessage)
else:
errorMessageList.append('The ntp configuration was not updated, since an attempt to update it was already made.')
if 'configureSnapper' not in progressDict:
f.write('configureSnapper\n')
errorMessage = configureSnapper(osDistLevel, cursesThread)
if len(errorMessage) > 0:
errorMessageList.append(errorMessage)
else:
errorMessageList.append('The snapper configuration was not updated, since an attempt to update it was already made.')
if serverModel != 'Superdome':
if 'updateMultipathConf' not in progressDict:
f.write('updateMultipathConf\n')
errorMessage = updateMultipathConf(cursesThread)
if len(errorMessage) > 0:
errorMessageList.append(errorMessage)
else:
errorMessageList.append('The multipath blacklist configuration was not updated, since an attempt to update it was already made.')
if 'DL580' in serverModel or serverModel == 'Superdome':
if 'DL580' in serverModel:
if 'updateBIOS' not in progressDict and addOnSoftwareInstalled:
f.write('updateBIOS\n')
errorMessage = updateBIOS(programParentDir, cursesThread)
if len(errorMessage) > 0:
errorMessageList.append(errorMessage)
else:
errorMessageList.append('The BIOS was not updated; either because it was already updated or the additional RPMs failed to install.')
if os.path.isdir(programParentDir + '/addOnFileArchive'):
if 'installAddOnFiles' not in progressDict:
f.write('installAddOnFiles\n')
errorMessage = installAddOnFiles(programParentDir, upgradeResourceDict.copy(), osDist, cursesThread, processor = processor)
if len(errorMessage) > 0:
errorMessageList.append(errorMessage)
else:
errorMessageList.append('The files from the add on files archive were not installed, since an attempt to add them was already made.')
if serverModel == 'Superdome':
if os.path.isdir(programParentDir + '/addOnFileArchive'):
if 'installAddOnFiles' not in progressDict:
f.write('installAddOnFiles\n')
errorMessage = installAddOnFiles(programParentDir, upgradeResourceDict.copy(), osDist, cursesThread, serverModel = serverModel)
if len(errorMessage) > 0:
errorMessageList.append(errorMessage)
else:
errorMessageList.append('The files from the add on files archive were not installed, since an attempt to add them was already made.')
if osDist == 'SLES':
if 'updateKdump' not in progressDict:
f.write('updateKdump\n')
errorMessage = updateKdump(cursesThread)
if len(errorMessage) > 0:
errorMessageList.append(errorMessage)
else:
errorMessageList.append('The update of kdump.service to address bug 1008170 was not performed, since an attempt to update the service was already made.')
if 'restoreHANAUserAccounts' not in progressDict:
f.write('restoreHANAUserAccounts\n')
errorMessage = restoreHANAUserAccounts(programParentDir, cursesThread)
if len(errorMessage) > 0:
errorMessageList.append(errorMessage)
else:
errorMessageList.append('The SAP HANA user account data was not restored, since an attempt to restore the account data was already made.')
if 'restoreHANAPartitionData' not in progressDict:
f.write('restoreHANAPartitionData\n')
(errorMessage, hanaPartitionsRestored) = restoreHANAPartitionData(programParentDir, cursesThread)
if len(errorMessage) > 0:
errorMessageList.append(errorMessage)
else:
errorMessageList.append('The SAP HANA parition data was not restored, since an attempt to restore the data was already made.')
if 'sapRestorationArchiveExtracted' not in progressDict and hanaPartitionsRestored:
f.write('sapRestorationArchiveExtracted\n')
errorMessage = extractSAPRestorationArchive(programParentDir, osDist, cursesThread)
if len(errorMessage) > 0:
errorMessageList.append(errorMessage)
else:
errorMessageList.append('The SAP restoration archive was not extracted; either because it was already extracted or the SAP HANA partition data was not successfully restored.')
if not scaleUpSystem:
if 'prepareSAPHanaDirs' not in progressDict:
f.write('prepareSAPHanaDirs\n')
errorMessage = prepareSAPHanaDirs(programParentDir, cursesThread)
if len(errorMessage) > 0:
errorMessageList.append(errorMessage)
else:
errorMessageList.append('The /hana/data/SID and /hana/log/SID directories were not created; since an attempt to create them was already made.')
if 'DL380p' in serverModel or 'DL360p' in serverModel:
if 'configureXinetd' not in progressDict:
f.write('configureXinetd\n')
errorMessage = configureXinetd(cursesThread)
if len(errorMessage) > 0:
errorMessageList.append(errorMessage)
else:
errorMessageList.append('xinetd was not configured; since an attempt to configure it was already made.')
if 'updateLvmConf' not in progressDict:
f.write('updateLvmConf\n')
errorMessage = updateLvmConf(cursesThread)
if len(errorMessage) > 0:
errorMessageList.append(errorMessage)
else:
errorMessageList.append('/etc/lvm.conf was not updated; since an attempt to enable it was already made.')
if 'updateNFSListeners' not in progressDict:
f.write('updateNFSListeners\n')
errorMessage = updateNFSListeners(cursesThread)
if len(errorMessage) > 0:
errorMessageList.append(errorMessage)
else:
errorMessageList.append("The number of NFS listeners in '/opt/cmcluster/nfstoolkit/hananfs.sh' was not updated; since an attempt to update the number was already made.")
if 'createNFSMountPoint' not in progressDict:
f.write('createNFSMountPoint\n')
errorMessage = createNFSMountPoint(cursesThread)
if len(errorMessage) > 0:
errorMessageList.append(errorMessage)
else:
errorMessageList.append('The hananfs mount point was not created; since an attempt to create it was already made.')
if 'buildDeadmanDriver' not in progressDict:
f.write('buildDeadmanDriver\n')
errorMessage = buildDeadmanDriver(cursesThread)
if len(errorMessage) > 0:
errorMessageList.append(errorMessage)
else:
errorMessageList.append('The deadman driver was not built and installed; since an attempt to build and install it was already made.')
if serverModel == 'ProLiant DL320e Gen8 v2' or serverModel == 'ProLiant DL360 Gen9':
if 'configureServiceguardQS' not in progressDict:
f.write('configureServiceguardQS\n')
errorMessage = configureServiceguardQS(programParentDir, osDistLevel, cursesThread)
if len(errorMessage) > 0:
errorMessageList.append(errorMessage)
else:
errorMessageList.append('Serviceguard QS was not configured; since an attempt to configure it was already made.')
except IOError:
err = None
logger.error('Could not access ' + progressStatusFile + '.\n' + str(err))
displayErrorMessage("Could not access '" + progressStatusFile + "'; fix the problem and try again.", cursesThread)
f.close()
if len(errorMessageList) > 0:
cursesThread.insertMessage([
'warning',
"Restoration of the server's configuration has completed with the errors shown below; check the log file for additional information before continuing with the re-installation process:"])
cursesThread.insertMessage([
'warning',
''])
for message in errorMessageList:
cursesThread.insertMessage([
'warning',
message])
cursesThread.insertMessage([
'warning',
''])
else:
cursesThread.insertMessage([
'informative',
"Restoration of the server's configuration has completed; continue with the next step in the re-installation process."])
cursesThread.insertMessage([
'informative',
''])
cursesThread.getUserInput([
'informative',
'Press enter to exit the program.'])
cursesThread.insertMessage([
'informative',
''])
while not cursesThread.isUserInputReady():
time.sleep(0.1)
exit(0)
except Exception:
err = None
cursesThread.join()
logger.error('An unexpected error was encountered:\n' + traceback.format_exc())
print RED + "An unexpected error was encountered; collect the application's log file and report the error back to the SAP HANA CoE DevOps team." + RESETCOLORS
print RED + 'Error: ' + str(err) + RESETCOLORS
exit(1)
finally:
cursesThread.join()
def backupPreparation(upgradeWorkingDir, programParentDir, osDist):
logger = logging.getLogger('coeOSUpgradeLogger')
ctrlFileImg = ''
cs500HSWScaleOut = False
getHostname(upgradeWorkingDir)
serverModel = getServerModel()
if not 'DL380' in serverModel:
if not 'DL360' in serverModel:
if not 'DL320' in serverModel:
processor = getProcessorType()
upgradeResourceDict = {}
try:
with open('upgradeResourceFile') as f:
for line in f:
line = line.strip()
if len(line) == 0 or re.match(
'^\\s*#', line) or re.match('^\\s+$', line):
continue
else:
line = re.sub('[\'"]', '', line)
key, val = line.split('=')
key = key.strip()
upgradeResourceDict[key] = re.sub('\\s+', '', val)
except IOError as err:
logger.error(
"Unable to access the application's resource file 'upgradeResourceFile'.\n"
+ str(err))
print RED + "Unable to access the application's resource file 'upgradeResourceFile'; fix the problem and try again; exiting program execution." + RESETCOLORS
exit(1)
try:
if 'DL380' in serverModel or 'DL360' in serverModel:
archiveBackup = re.split(
'\\s*,\\s*',
upgradeResourceDict['osArchiveBackup']) + re.split(
'\\s*,\\s*',
upgradeResourceDict['sglxArchiveBackup'])
if osDist == 'SLES':
restorationBackup = re.split(
'\\s*,\\s*',
upgradeResourceDict['slesOSRestorationBackup']
) + re.split(
'\\s*,\\s*',
upgradeResourceDict['sglxRestorationBackup'])
else:
restorationBackup = re.split(
'\\s*,\\s*',
upgradeResourceDict['rhelOSRestorationBackup']
) + re.split(
'\\s*,\\s*',
upgradeResourceDict['sglxRestorationBackup'])
elif 'DL320' in serverModel:
pass
else:
archiveBackup = re.split(
'\\s*,\\s*', upgradeResourceDict['osArchiveBackup'])
if osDist == 'SLES':
restorationBackup = re.split(
'\\s*,\\s*',
upgradeResourceDict['slesOSRestorationBackup'])
sapRestorationBackup = re.split(
'\\s*,\\s*',
upgradeResourceDict['sapRestorationBackup'])
if 'DL580' in serverModel:
if processor == 'ivybridge':
addOnFileArchive = programParentDir + '/addOnFileArchives/' + upgradeResourceDict[
'ivyBridgeSLESAddOnFileArchive']
else:
addOnFileArchive = programParentDir + '/addOnFileArchives/' + upgradeResourceDict[
'haswellSLESAddOnFileArchive']
elif serverModel == 'Superdome':
addOnFileArchive = programParentDir + '/addOnFileArchives/' + upgradeResourceDict[
'cs900SLESAddOnFileArchive']
else:
restorationBackup = re.split(
'\\s*,\\s*',
upgradeResourceDict['rhelOSRestorationBackup'])
sapRestorationBackup = re.split(
'\\s*,\\s*',
upgradeResourceDict['sapRestorationBackup'])
if 'DL580' in serverModel:
if processor == 'ivybridge':
addOnFileArchive = programParentDir + '/addOnFileArchives/' + upgradeResourceDict[
'ivyBridgeRHELAddOnFileArchive']
else:
addOnFileArchive = programParentDir + '/addOnFileArchives/' + upgradeResourceDict[
'haswellRHELAddOnFileArchive']
elif serverModel == 'Superdome':
addOnFileArchive = programParentDir + '/addOnFileArchives/' + upgradeResourceDict[
'cs900RHELAddOnFileArchive']
fstabHANAEntries = upgradeResourceDict[
'fstabHANAEntries'].split(',')
except KeyError as err:
logger.error(
'The resource key ' + str(err) +
" was not present in the application's resource file.")
print RED + 'The resource key ' + str(
err
) + " was not present in the application's resource file; fix the problem and try again; exiting program execution." + RESETCOLORS
exit(1)
stageAddOnRPMS(programParentDir, upgradeWorkingDir, serverModel,
osDist)
if osDist == 'RHEL':
copyRepositoryTemplate(programParentDir, upgradeWorkingDir)
if not 'DL380' in serverModel:
if not 'DL320' in serverModel:
if not 'DL360' in serverModel:
if 'DL580' in serverModel or serverModel == 'Superdome':
addOnFileArchiveDir = upgradeWorkingDir + '/addOnFileArchive'
try:
os.mkdir(addOnFileArchiveDir)
shutil.copy2(addOnFileArchive,
addOnFileArchiveDir)
except (OSError, OIError) as err:
logger.error(
"Unable to copy the add on files archive '"
+ addOnFileArchive + "' to '" +
addOnFileArchiveDir + "'.\n" + str(err))
print RED + "Unable to copy the add on files archive '" + addOnFileArchive + "' to '" + addOnFileArchiveDir + "'; fix the problem and try again; exiting program execution." + RESETCOLORS
exit(1)
hanaSharedWithSID = getHANAFstabData(
upgradeWorkingDir, fstabHANAEntries, serverModel,
processor)
sidList, tenantUserDict = checkSAPHANA(
upgradeWorkingDir, hanaSharedWithSID, serverModel)
if len(tenantUserDict) != 0:
homeDirList = getSAPUserLoginData(
upgradeWorkingDir,
sidList,
tenantUserDict=tenantUserDict)
else:
homeDirList = getSAPUserLoginData(
upgradeWorkingDir, sidList)
restorationBackup += homeDirList
restorationBackup += getSapInitBackupList()
multipathConfList, fibrePresent = getMultipathConf()
if len(multipathConfList) > 0:
restorationBackup += multipathConfList
if 'DL580' in serverModel:
conrepDir = upgradeWorkingDir + '/conrepFile'
getConrepFile(programParentDir, conrepDir,
processor)
if serverModel == 'Superdome' and osDist == 'SLES':
ctrlFileImg = updateInstallCtrlFile(
programParentDir, upgradeWorkingDir, processor)
ctrlFileImg = 'DL580' in serverModel and processor == 'haswell' and fibrePresent and updateInstallCtrlFile(
programParentDir,
upgradeWorkingDir,
processor,
cs500ScaleOut='cs500ScaleOut')
cs500HSWScaleOut = True
backupItems = 'DL380' in serverModel or 'DL320' in serverModel or 'DL360' in serverModel or ' '.join(
restorationBackup) + ' ' + ' '.join(
archiveBackup) + ' ' + ' '.join(sapRestorationBackup)
backupItems = [
restorationBackup, sapRestorationBackup, archiveBackup
]
else:
backupItems = ' '.join(restorationBackup) + ' ' + ' '.join(
archiveBackup)
backupItems = [restorationBackup, archiveBackup]
checkDiskspace(backupItems)
createNetworkInformationFile(upgradeWorkingDir, osDist)
'DL580' in serverModel and 'Gen9' in serverModel and checkForMellanox(
programParentDir, upgradeWorkingDir, osDist)
return 'DL380' in serverModel or 'DL320' in serverModel or 'DL360' in serverModel or (
[restorationBackup, archiveBackup, sapRestorationBackup
], serverModel, ctrlFileImg, cs500HSWScaleOut)
else:
return ([restorationBackup,
archiveBackup], serverModel, ctrlFileImg, cs500HSWScaleOut)
def backupPreparation(upgradeWorkingDir, programParentDir, osDist,
osDistLevel):
logger = logging.getLogger('coeOSUpgradeLogger')
resourceFile = 'upgradeResourceFile'
ctrlFileImg = ''
hanaSharedWithSID = False
upgradeResourceDict = {}
try:
with open(resourceFile) as f:
for line in f:
line = line.strip()
if len(line) == 0 or re.match('^\\s*#', line) or re.match(
'^\\s+$', line):
continue
else:
line = re.sub('[\'"]', '', line)
key, val = line.split('=')
key = key.strip()
upgradeResourceDict[key] = re.sub('\\s*,\\s*', ',',
val).strip()
except IOError as err:
logger.error("Unable to access the application's resource file '" +
resourceFile + "'.\n" + str(err))
print RED + "Unable to access the application's resource file '" + resourceFile + "'; fix the problem and try again; exiting program execution." + RESETCOLORS
exit(1)
try:
shutil.copy2(resourceFile, upgradeWorkingDir)
except OSError as err:
print RED + "Unable to copy the upgrade resource file to '" + upgradeWorkingDir + "'; fix the problem and try again; exiting program execution." + RESETCOLORS
exit(1)
serverModel = getServerModel()
if serverModel not in upgradeResourceDict['supportedServerList']:
logger.error('The ' + serverModel +
' is not supported. Supported models are: ' +
upgradeResourceDict['supportedServerList'] + '.')
print RED + 'The ' + serverModel + ' is not supported; exiting program execution.' + RESETCOLORS
exit(1)
if serverModel == 'ProLiant DL360 Gen9':
command = 'rpm -q serviceguard-qs-*'
result = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
out, err = result.communicate()
if result.returncode != 0 and 'package servicguard-qs-* is not installed' not in out:
logger.error(
'Unable to determine if the server is a Serviceguard quorum server or Serviceguard NFS server:\n'
+ err + '\n' + out)
print RED + 'Unable to determine if the server is a Serviceguard quorum server or Serviceguard NFS server; fix the problem and try again; exiting program execution.' + RESETCOLORS
exit(1)
elif 'package servicguard-qs-* is not installed' in out:
logger.error(
'The ' + serverModel +
' is not supported, since it is not a Serviceguard Quorum server.\n'
+ out)
print RED + 'The ' + serverModel + ' is not supported, since it is not a Serviceguard Quorum server; exiting program execution.' + RESETCOLORS
exit(1)
try:
if osDist == 'RHEL':
supportedOSVersions = upgradeResourceDict[
'supportedRHELOSVersions'].split(',')
else:
supportedOSVersions = upgradeResourceDict[
'supportedSLESOSVersions'].split(',')
osUpgradeVersion = getOSUpgradeVersion(supportedOSVersions)
except KeyError as err:
logger.error('The resource key ' + str(err) +
" was not present in the application's resource file.")
print RED + 'The resource key ' + str(
err
) + " was not present in the application's resource file; fix the problem and try again; exiting program execution." + RESETCOLORS
exit(1)
if osUpgradeVersion == '11.4':
if serverModel == 'ProLiant DL320e Gen8 v2' or not serverModel == 'ProLiant DL360 Gen9':
logger.error(
'The ' + serverModel +
' is not supported for a SLES 11.4 wipe and replace upgrade.')
print RED + 'The ' + serverModel + ' is not supported for a SLES 11.4 wipe and replace upgrade; exiting program execution.' + RESETCOLORS
exit(1)
if serverModel == 'ProLiant DL320e Gen8 v2' or serverModel == 'ProLiant DL360 Gen9':
checkForQSRPMPresence(upgradeWorkingDir, osDist, osUpgradeVersion)
if osDist == 'SLES':
postUpgradeScripts = programParentDir + '/postUpgradeScriptFiles/SLES/' + osUpgradeVersion + '/*'
else:
postUpgradeScripts = programParentDir + '/postUpgradeScriptFiles/RHEL/' + osUpgradeVersion + '/*'
command = 'cp -r ' + postUpgradeScripts + ' ' + upgradeWorkingDir
result = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
out, err = result.communicate()
if result.returncode != 0:
logger.error(
"Unable to copy the post-upgrade script files from '" +
postUpgradeScripts + "' to '" + upgradeWorkingDir + ':\n' +
err + '\n' + out)
print RED + 'Unable to copy the post-upgrade script files to the upgrade working directory; fix the problem and try again; exiting program execution.' + RESETCOLORS
exit(1)
if not osUpgradeVersion == '11.4':
getHostname(upgradeWorkingDir)
getLocalTimeLink(upgradeWorkingDir)
processor = ('DL580' in serverModel
or serverModel == 'Superdome') and getProcessorType()
try:
if 'DL380p' in serverModel or 'DL360p' in serverModel:
archiveBackup = re.split(
',', upgradeResourceDict['osArchiveBackup']) + re.split(
',', upgradeResourceDict['sglxArchiveBackup'])
if osDist == 'SLES':
restorationBackup = re.split(
',',
upgradeResourceDict['slesOSRestorationBackup']) + re.split(
',', upgradeResourceDict['sglxRestorationBackup'])
else:
restorationBackup = re.split(
',',
upgradeResourceDict['rhelOSRestorationBackup']) + re.split(
',', upgradeResourceDict['sglxRestorationBackup'])
elif serverModel == 'ProLiant DL320e Gen8 v2' or serverModel == 'ProLiant DL360 Gen9':
archiveBackup = re.split(
',', upgradeResourceDict['osArchiveBackup']) + re.split(
',', upgradeResourceDict['sgQsArchiveBackup'])
if osDist == 'SLES':
restorationBackup = re.split(
',',
upgradeResourceDict['slesOSRestorationBackup']) + re.split(
',', upgradeResourceDict['sgQsRestorationBackup'])
else:
restorationBackup = re.split(
',',
upgradeResourceDict['rhelOSRestorationBackup']) + re.split(
',', upgradeResourceDict['sgQsRestorationBackup'])
else:
archiveBackup = re.split(',',
upgradeResourceDict['osArchiveBackup'])
sapRestorationBackup = re.split(
',', upgradeResourceDict['sapRestorationBackup'])
if osDist == 'SLES':
restorationBackup = re.split(
',', upgradeResourceDict['slesOSRestorationBackup'])
if 'DL580' in serverModel:
if processor == 'ivybridge':
addOnFileArchive = programParentDir + '/addOnFileArchives/' + upgradeResourceDict[
'ivyBridgeSLESAddOnFileArchive']
else:
addOnFileArchive = programParentDir + '/addOnFileArchives/' + upgradeResourceDict[
'haswellSLESAddOnFileArchive']
else:
addOnFileArchive = programParentDir + '/addOnFileArchives/' + upgradeResourceDict[
'cs900SLESAddOnFileArchive']
else:
restorationBackup = re.split(
',', upgradeResourceDict['rhelOSRestorationBackup'])
if 'DL580' in serverModel:
if processor == 'ivybridge':
addOnFileArchive = programParentDir + '/addOnFileArchives/' + upgradeResourceDict[
'ivyBridgeRHELAddOnFileArchive']
else:
addOnFileArchive = programParentDir + '/addOnFileArchives/' + upgradeResourceDict[
'haswellRHELAddOnFileArchive']
else:
addOnFileArchive = programParentDir + '/addOnFileArchives/' + upgradeResourceDict[
'cs900RHELAddOnFileArchive']
fstabHANAEntries = upgradeResourceDict['fstabHANAEntries'].split(
',')
except KeyError as err:
logger.error('The resource key ' + str(err) +
" was not present in the application's resource file.")
print RED + 'The resource key ' + str(
err
) + " was not present in the application's resource file; fix the problem and try again; exiting program execution." + RESETCOLORS
exit(1)
stageAddOnRPMS(programParentDir, upgradeWorkingDir, osDist,
osUpgradeVersion)
if not serverModel == 'Superdome':
if not 'DL580' in serverModel:
ctrlFileImg = updateSGInstallCtrlFile(programParentDir,
upgradeWorkingDir,
osUpgradeVersion,
serverModel)
if 'DL580' in serverModel or serverModel == 'Superdome':
if re.match('.*/$', addOnFileArchive) == None:
addOnFileArchiveDir = upgradeWorkingDir + '/addOnFileArchive'
try:
os.mkdir(addOnFileArchiveDir)
shutil.copy2(addOnFileArchive, addOnFileArchiveDir)
except (OSError, IOError) as err:
logger.error("Unable to copy the add on files archive '" +
addOnFileArchive + "' to '" +
addOnFileArchiveDir + "'.\n" + str(err))
print RED + "Unable to copy the add on files archive '" + addOnFileArchive + "' to '" + addOnFileArchiveDir + "'; fix the problem and try again; exiting program execution." + RESETCOLORS
exit(1)
if serverModel == 'Superdome' and processor != 'ivybridge':
logger.info(
'The system is a Superdome with the SID as part of /hana/shared in the mount point.'
)
hanaSharedWithSID = True
getHANAFstabData(upgradeWorkingDir, fstabHANAEntries)
sidList, tenantUserDict = checkSAPHANA(upgradeWorkingDir,
hanaSharedWithSID)
if len(tenantUserDict) != 0:
homeDirList = getSAPUserLoginData(
upgradeWorkingDir, sidList, tenantUserDict=tenantUserDict)
else:
homeDirList = getSAPUserLoginData(upgradeWorkingDir, sidList)
restorationBackup += homeDirList
restorationBackup += getSapInitBackupList()
if 'DL580' in serverModel:
conrepDir = upgradeWorkingDir + '/conrepFile'
getConrepFile(programParentDir, conrepDir)
if (serverModel == 'Superdome'
or 'DL580' in serverModel) and osDist == 'SLES':
if serverModel == 'Superdome':
ctrlFileImg = updateInstallCtrlFile(
programParentDir, upgradeWorkingDir, osUpgradeVersion)
else:
command = 'lsblk -l'
result = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
out, err = result.communicate()
if result.returncode != 0:
logger.error(
'Unable to get the partition information for the root file system.\n'
+ err + '\n' + out)
print RED + 'Unable to get the partition information for the root file system; fix the problem and try again; exiting program execution.' + RESETCOLORS
exit(1)
if re.match('.*(360002ac0+.*)[_-]part.*/boot/efi.*$', out,
re.DOTALL | re.MULTILINE):
ctrlFileImg = updateInstallCtrlFile(
programParentDir,
upgradeWorkingDir,
osUpgradeVersion,
serverModel='DL580',
cs500ScaleOut='cs500ScaleOut')
else:
ctrlFileImg = updateInstallCtrlFile(
programParentDir,
upgradeWorkingDir,
osUpgradeVersion,
serverModel='DL580')
if 'DL580' in serverModel or serverModel == 'Superdome':
backupItems = ' '.join(restorationBackup) + ' ' + ' '.join(
archiveBackup) + ' ' + ' '.join(sapRestorationBackup)
backupItems = [
restorationBackup, sapRestorationBackup, archiveBackup
]
else:
backupItems = ' '.join(restorationBackup) + ' ' + ' '.join(
archiveBackup)
backupItems = [restorationBackup, archiveBackup]
checkDiskspace(backupItems)
createNetworkInformationFile(upgradeWorkingDir, osDist)
if 'DL580' in serverModel and processor != 'ivybridge':
checkForMellanox(programParentDir, upgradeWorkingDir, osDist,
osUpgradeVersion)
return ('DL580' in serverModel or serverModel == 'Superdome') and ([
restorationBackup, archiveBackup, sapRestorationBackup
], ctrlFileImg)
else:
return ([restorationBackup, archiveBackup], ctrlFileImg)
return
def main():
programVersion = '1.0-0'
programName = os.path.basename(sys.argv[0]).split('.')[0]
errorsEncountered = False
if os.geteuid() != 0:
print RED + 'You must be root to run this program; exiting program execution.' + RESETCOLORS
exit(1)
usage = 'usage: %prog [-h] [-p] [-v]'
parser = optparse.OptionParser(usage=usage)
parser.add_option('-p', action='store_true', default=False, help='This option is used after an upgrade to check services and SAP Note settings.')
parser.add_option('-v', action='store_true', default=False, help="This option is used to display the application's version.")
options, args = parser.parse_args()
if options.v:
print programName + ' ' + programVersion
exit(0)
osDist, osDistLevel = getOSDistribution()
logDir = '/var/log/CoESapHANA_' + osDist + '_UpgradeLogDir'
if not os.path.isdir(logDir):
try:
os.mkdir(logDir)
except OSError as err:
print RED + "Unable to create the post upgrade log directory '" + logDir + "'; fix the problem and try again; exiting program execution.\n" + str(err) + RESETCOLORS
exit(1)
statusLogFile = logDir + '/status.log'
debugLogFile = logDir + '/debug.log'
statusHandler = logging.FileHandler(statusLogFile)
debugHandler = logging.FileHandler(debugLogFile)
formatter = logging.Formatter('%(asctime)s:%(levelname)s:%(message)s', datefmt='%m/%d/%Y %H:%M:%S')
statusHandler.setFormatter(formatter)
debugHandler.setFormatter(formatter)
logger = logging.getLogger('coeOSUpgradeLogger')
logger.setLevel(logging.INFO)
logger.addHandler(statusHandler)
debugLogger = logging.getLogger('coeOSUpgradeDebugLogger')
debugLogger.setLevel(logging.INFO)
debugLogger.addHandler(debugHandler)
debugLogger.info("The program's version is: " + programVersion + '.')
serverModel = getServerModel()
try:
programParentDir = ''
cwd = os.getcwd()
programParentDir = os.path.dirname(os.path.realpath(__file__))
if cwd != programParentDir:
os.chdir(programParentDir)
except OSError as err:
debugLogger.error("Unable to change into the programs parent directory '" + programParentDir + "'.\n" + str(err))
print RED + "Unable to change into the programs parent directory '" + programParentDir + "'; fix the problem and try again; exiting program execution." + RESETCOLORS
exit(1)
upgradeResourceDict = {}
try:
with open('upgradeResourceFile') as f:
for line in f:
line = line.strip()
if len(line) == 0 or re.match('^\\s*#', line) or re.match('^\\s+$', line):
continue
else:
line = re.sub('[\'"]', '', line)
key, val = line.split('=')
key = key.strip()
upgradeResourceDict[key] = re.sub('\\s*,\\s*', ',', val).strip()
except IOError as err:
debugLogger.error("Unable to access the application's resource file 'upgradeResourceFile'.\n" + str(err))
print RED + "Unable to access the application's resource file 'upgradeResourceFile'; fix the problem and try again; exiting program execution." + RESETCOLORS
exit(1)
if options.p:
if serverModel == 'Superdome' or 'DL580' in serverModel:
if serverModel == 'Superdome':
errors = checkSAPHANAConfiguration(serverModel, osDist)
elif osDist == 'SLES':
errors = checkSAPHANAConfiguration(serverModel, osDist, upgradeResourceDict['slesKernelParameters'])
else:
errors = checkSAPHANAConfiguration(serverModel, osDist, upgradeResourceDict['rhelKernelParameters'])
if errors:
if not errorsEncountered:
errorsEncountered = True
errors = checkServices()
if errors:
if not errorsEncountered:
errorsEncountered = True
else:
progressStatusFile = logDir + '/.restorationProgress'
try:
if os.path.isfile(progressStatusFile):
with open(progressStatusFile) as f:
progressDict = dict.fromkeys((x.strip() for x in f))
else:
progressDict = {}
if os.path.isfile(progressStatusFile):
f = open(progressStatusFile, 'a')
else:
f = open(progressStatusFile, 'w')
if serverModel == 'ProLiant DL360 Gen9' or serverModel == 'ProLiant DL380p Gen8' or serverModel == 'ProLiant DL360p Gen8' or serverModel == 'ProLiant DL320e Gen8 v2':
serviceguardRPMDir = programParentDir + '/serviceguardRPMS'
if os.path.isdir(serviceguardRPMDir):
if 'updateServiceguardPackages' not in progressDict:
errors = updateServiceguardPackages(serviceguardRPMDir)
if errors:
if not errorsEncountered:
errorsEncountered = True
else:
f.write('updateServiceguardPackages\n')
else:
print YELLOW + 'The Serviceguard packages were not updated, since they were already updated.' + RESETCOLORS
fstabFile = programParentDir + '/fstabFile/fstab'
if os.path.isfile(fstabFile):
if 'restoreFstab' not in progressDict:
errors = restoreFstab(fstabFile)
if errors:
if not errorsEncountered:
errorsEncountered = True
else:
f.write('restoreFstab\n')
else:
print YELLOW + 'The fstab file was not restored, since it was already restored.' + RESETCOLORS
if osDist == 'SLES':
limitsFile = programParentDir + '/limitsFile/limits.conf'
if os.path.isfile(limitsFile):
if 'configureNofile' not in progressDict:
errors = configureNofile(limitsFile, upgradeResourceDict['sapsysNofileValue'])
if errors:
if not errorsEncountered:
errorsEncountered = True
else:
f.write('configureNofile\n')
else:
print YELLOW + 'hard/soft nofile limits for @sapsys, were not configured, since they were already configured.' + RESETCOLORS
if 'DL580' in serverModel:
if 'updateBIOS' not in progressDict:
errors = updateBIOS(programParentDir)
if errors:
if not errorsEncountered:
errorsEncountered = True
else:
f.write('updateBIOS\n')
else:
print YELLOW + 'The BIOS was not updated, since it was already updated.' + RESETCOLORS
if os.path.isdir(programParentDir + '/mellanoxDriver'):
if 'configureMellanox' not in progressDict:
errors = configureMellanox(programParentDir)
if errors:
if not errorsEncountered:
errorsEncountered = True
else:
f.write('configureMellanox\n')
else:
print YELLOW + 'The Mellanox driver was not installed, since it was already installed.' + RESETCOLORS
if os.path.isfile('/etc/sysctl.conf'):
if 'updateSysctl' not in progressDict:
if osDist == 'SLES':
errors = updateSysctl(upgradeResourceDict['slesKernelParameters'])
else:
errors = updateSysctl(upgradeResourceDict['rhelKernelParameters'])
if errors:
if not errorsEncountered:
errorsEncountered = True
else:
f.write('updateSysctl\n')
else:
print YELLOW + '/etc/sysctl.conf was not updated, since an attempt to update it was already made.' + RESETCOLORS
if osDist == 'SLES':
if 'configureSnapper' not in progressDict:
errors = configureSnapper()
if errors:
errorsEncountered = True
else:
f.write('configureSnapper\n')
else:
print YELLOW + 'The snapper configuration was not updated, since it was already updated.' + RESETCOLORS
if 'enableMultiversionSupport' not in progressDict:
errors = enableMultiversionSupport()
if errors:
if not errorsEncountered:
errorsEncountered = True
else:
f.write('enableMultiversionSupport\n')
else:
print YELLOW + 'Kernel multiversion support was not enabled, since it was already enabled.' + RESETCOLORS
if 'cleanupGrubCfg' not in progressDict:
errors = cleanupGrubCfg()
if errors:
if not errorsEncountered:
errorsEncountered = True
else:
f.write('cleanupGrubCfg\n')
else:
print YELLOW + '/etc/default/grub was not configured, since it was already configured.' + RESETCOLORS
if 'disableServices' not in progressDict:
disableServiceList = dict(((x.split(':')[0].strip(), re.sub('\\s+', '', x.split(':')[1])) for x in upgradeResourceDict['slesServices'].split(',')))['SLES ' + osDistLevel].split('|')
errors = disableServices(disableServiceList)
if errors:
if not errorsEncountered:
errorsEncountered = True
else:
f.write('disableServices\n')
else:
print YELLOW + 'Systemd services were not disabled, since they were already disabled.' + RESETCOLORS
except IOError as err:
debugLogger.error('Could not access ' + progressStatusFile + '.\n' + str(err))
print RED + "Could not access '" + progressStatusFile + "'; fix the problem and try again; exiting program execution." + RESETCOLORS
exit(1)
finally:
f.close()
if errorsEncountered:
print RED + 'Errors were encountered while performing post upgrade tasks; check the debug log file for additional information before continuing.' + RESETCOLORS
else:
print GREEN + 'The post upgrade tasks have completed; continue with the next step in the upgrade process.' + RESETCOLORS
def backupPreparation(upgradeWorkingDir, programParentDir, osDistVersion):
debugLogger = logging.getLogger('coeOSUpgradeDebugLogger')
resourceFile = 'upgradeResourceFile'
ctrlFileImg = ''
osDist = osDistVersion[:4]
mellanoxPresent = False
upgradeResourceDict = {}
try:
with open(resourceFile) as f:
for line in f:
line = line.strip()
if len(line) == 0 or re.match('^\\s*#', line) or re.match(
'^\\s+$', line):
continue
else:
line = re.sub('[\'"]', '', line)
key, val = line.split('=')
key = key.strip()
upgradeResourceDict[key] = re.sub('\\s*,\\s*', ',',
val).strip()
except IOError as err:
debugLogger.error(
"Unable to access the application's resource file '" +
resourceFile + "'.\n" + str(err))
print RED + "Unable to access the application's resource file; fix the problem and try again; exiting program execution." + RESETCOLORS
exit(1)
try:
shutil.copy2(resourceFile, upgradeWorkingDir)
except OSError as err:
debugLogger.error("Unable to copy the upgrade resource file to '" +
upgradeWorkingDir + "'.\n" + str(err))
print RED + 'Unable to copy the upgrade resource file to the upgrade working directory; fix the problem and try again; exiting program execution.' + RESETCOLORS
exit(1)
if osDist == 'SLES':
initialOSDict = dict.fromkeys(
(x
for x in upgradeResourceDict['initialSLESOSVersions'].split(',')))
debugLogger.info('The initial OS version dictionary is: ' +
str(initialOSDict))
if osDistVersion not in initialOSDict:
debugLogger.error(
"The server's OS distribution (" + osDistVersion +
') is not supported for a wipe and replace upgrade.')
print RED + "The server's OS distribution is not supported for a wipe and replace upgrade; exiting program execution." + RESETCOLORS
exit(1)
else:
initialOSDict = dict.fromkeys(
(x
for x in upgradeResourceDict['initialRHELOSVersions'].split(',')))
debugLogger.info('The initial OS version dictionary is: ' +
str(initialOSDict))
if osDistVersion not in initialOSDict:
debugLogger.error(
"The server's OS version (" + osDistVersion +
') is not supported for a wipe and replace upgrade.')
print RED + "The server's OS version is not supported for a wipe and replace upgrade; exiting program execution." + RESETCOLORS
exit(1)
serverModel = getServerModel()
if 'DL580' in serverModel or serverModel == 'Superdome':
processor = getProcessorType()
if osDist == 'RHEL' and serverModel == 'Superdome' and processor == 'ivybridge':
debugLogger.error(
'The ' + serverModel +
' is not supported, since it is a CS900 Ivy Bridge configuration.')
print RED + 'The server is not supported; exiting program execution.' + RESETCOLORS
exit(1)
if osDist == 'SLES':
supportedServerList = upgradeResourceDict['slesSupportedServerList']
else:
supportedServerList = upgradeResourceDict['rhelSupportedServerList']
if serverModel not in supportedServerList:
debugLogger.error('The ' + serverModel +
' is not supported. Supported models are: ' +
supportedServerList + '.')
print RED + 'The server is not supported; exiting program execution.' + RESETCOLORS
exit(1)
if serverModel == 'ProLiant DL360 Gen9':
command = 'rpm -q serviceguard-qs-*'
result = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
out, err = result.communicate()
if result.returncode != 0 and 'package servicguard-qs-* is not installed' not in out:
debugLogger.error(
'Unable to determine if the server is a Serviceguard quorum server or Serviceguard NFS server:\n'
+ err + '\n' + out)
print RED + 'Unable to determine if the server is a Serviceguard quorum server or Serviceguard NFS server; fix the problem and try again; exiting program execution.' + RESETCOLORS
exit(1)
elif 'package servicguard-qs-* is not installed' in out:
debugLogger.error(
'The ' + serverModel +
' is not supported, since it is not a Serviceguard Quorum server.\n'
+ out)
print RED + 'The ' + serverModel + ' is not supported, since it is not a Serviceguard Quorum server; exiting program execution.' + RESETCOLORS
exit(1)
if serverModel == 'Superdome':
systemType = 'CS900'
elif serverModel == 'ProLiant DL360 Gen9' or serverModel == 'ProLiant DL320e Gen8 v2':
systemType = 'SGQS'
elif serverModel == 'ProLiant DL380p Gen8' or serverModel == 'ProLiant DL360p Gen8':
systemType = 'SGLX'
else:
systemType = 'CS500'
try:
if osDist == 'RHEL':
supportedOSVersions = upgradeResourceDict[
'supportedRHELOSVersions'].split(',')
else:
supportedOSVersions = upgradeResourceDict[
'supportedSLESOSVersions'].split(',')
except KeyError as err:
debugLogger.error(
'The resource key ' + str(err) +
" was not present in the application's resource file.")
print RED + 'The resource ' + str(
err
) + " was not present in the application's resource file; fix the problem and try again; exiting program execution." + RESETCOLORS
exit(1)
debugLogger.info('The supported OS Versions to upgrade to are: ' +
str(supportedOSVersions))
if len(supportedOSVersions) > 1:
osUpgradeVersion = getOSUpgradeVersion(supportedOSVersions,
serverModel,
upgradeResourceDict.copy())
else:
osUpgradeVersion = supportedOSVersions[0][5:]
debugLogger.info('The server is being upgraded to ' + osUpgradeVersion +
'.')
if osDist == 'SLES':
postUpgradeScripts = programParentDir + '/postUpgradeScriptFiles/SLES/' + osUpgradeVersion + '/*'
else:
postUpgradeScripts = programParentDir + '/postUpgradeScriptFiles/RHEL/' + osUpgradeVersion + '/*'
command = 'cp -r ' + postUpgradeScripts + ' ' + upgradeWorkingDir
result = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
out, err = result.communicate()
if result.returncode != 0:
debugLogger.error(
"Unable to copy the post-upgrade script files from '" +
postUpgradeScripts + "' to '" + upgradeWorkingDir + ':\n' + err +
'\n' + out)
print RED + 'Unable to copy the post-upgrade script files to the upgrade working directory; fix the problem and try again; exiting program execution.' + RESETCOLORS
exit(1)
if not osUpgradeVersion == '11.4':
getHostname(upgradeWorkingDir)
getLocalTimeLink(upgradeWorkingDir)
try:
if serverModel == 'ProLiant DL380p Gen8' or serverModel == 'ProLiant DL360p Gen8':
archiveBackup = re.split(
',', upgradeResourceDict['osArchiveBackup']) + re.split(
',', upgradeResourceDict['sglxArchiveBackup'])
if osDist == 'SLES':
restorationBackup = re.split(
',',
upgradeResourceDict['slesOSRestorationBackup']) + re.split(
',', upgradeResourceDict['sglxRestorationBackup'])
else:
restorationBackup = re.split(
',',
upgradeResourceDict['rhelOSRestorationBackup']) + re.split(
',', upgradeResourceDict['sglxRestorationBackup'])
elif serverModel == 'ProLiant DL320e Gen8 v2' or serverModel == 'ProLiant DL360 Gen9':
archiveBackup = re.split(
',', upgradeResourceDict['osArchiveBackup']) + re.split(
',', upgradeResourceDict['sgQsArchiveBackup'])
restorationBackup = re.split(
',',
upgradeResourceDict['slesOSRestorationBackup']) + re.split(
',', upgradeResourceDict['sgQsRestorationBackup'])
else:
archiveBackup = re.split(',',
upgradeResourceDict['osArchiveBackup'])
sapRestorationBackup = re.split(
',', upgradeResourceDict['sapRestorationBackup'])
if osDist == 'SLES':
restorationBackup = re.split(
',', upgradeResourceDict['slesOSRestorationBackup'])
if 'DL580' in serverModel:
if processor == 'ivybridge':
addOnFileArchive = programParentDir + '/addOnFileArchives/' + upgradeResourceDict[
'ivyBridgeSLESAddOnFileArchive']
else:
addOnFileArchive = programParentDir + '/addOnFileArchives/' + upgradeResourceDict[
'haswellSLESAddOnFileArchive']
else:
addOnFileArchive = programParentDir + '/addOnFileArchives/' + upgradeResourceDict[
'cs900SLESAddOnFileArchive']
else:
restorationBackup = re.split(
',', upgradeResourceDict['rhelOSRestorationBackup'])
if 'DL580' in serverModel:
if processor == 'ivybridge':
addOnFileArchive = programParentDir + '/addOnFileArchives/' + upgradeResourceDict[
'ivyBridgeRHELAddOnFileArchive']
else:
addOnFileArchive = programParentDir + '/addOnFileArchives/' + upgradeResourceDict[
'haswellRHELAddOnFileArchive']
else:
addOnFileArchive = programParentDir + '/addOnFileArchives/' + upgradeResourceDict[
'cs900RHELAddOnFileArchive']
fstabHANAEntries = upgradeResourceDict['fstabHANAEntries'].split(
',')
except KeyError as err:
debugLogger.error(
'The resource key ' + str(err) +
" was not present in the application's resource file.")
print RED + 'The resource ' + str(
err
) + " was not present in the application's resource file; fix the problem and try again; exiting program execution." + RESETCOLORS
exit(1)
stageAddOnRPMS(programParentDir, upgradeWorkingDir, osDist,
osUpgradeVersion, systemType)
if osDist == 'RHEL':
copyRepositoryTemplate(programParentDir, upgradeWorkingDir)
if not serverModel == 'Superdome':
if not 'DL580' in serverModel and osDist == 'SLES':
ctrlFileImg = updateSGInstallCtrlFile(programParentDir,
upgradeWorkingDir,
osUpgradeVersion,
serverModel)
elif not serverModel == 'Superdome':
sgNFSCtrlFileImg = not 'DL580' in serverModel and osDist == 'RHEL' and programParentDir + '/installCtrlFiles/sgNFS_ks.img'
installCtrlFileDir = upgradeWorkingDir + '/installCtrlFile'
ctrlFileImg = installCtrlFileDir + '/sgNFS_ks.img'
try:
os.mkdir(installCtrlFileDir)
shutil.copy2(sgNFSCtrlFileImg, ctrlFileImg)
except OSError as err:
debugLogger.error(
"Unable to create install control file directory '" +
installCtrlFileDir + "'.\n" + str(err))
print RED + 'Unable to create the install control file directory; fix the problem and try again; exiting program execution.' + RESETCOLORS
exit(1)
except IOError as err:
debugLogger.error(
'Unable to copy the NFS Serviceguard control image file ('
+ sgNFSCtrlFileImg + ") to '" + ctrlFileImg + "'.\n" +
str(err))
print RED + 'Unable to copy the NFS Serviceguard control image file; fix the problem and try again; exiting program execution.' + RESETCOLORS
exit(1)
if 'DL580' in serverModel or serverModel == 'Superdome':
if re.match('.*/$', addOnFileArchive) == None:
addOnFileArchiveDir = upgradeWorkingDir + '/addOnFileArchive'
try:
os.mkdir(addOnFileArchiveDir)
shutil.copy2(addOnFileArchive, addOnFileArchiveDir)
except (OSError, IOError) as err:
debugLogger.error(
"Unable to copy the add on files archive '" +
addOnFileArchive + "' to '" + addOnFileArchiveDir +
"'.\n" + str(err))
print RED + 'Unable to copy the add on files archive; fix the problem and try again; exiting program execution.' + RESETCOLORS
exit(1)
getHANAFstabData(upgradeWorkingDir, fstabHANAEntries)
backupLimitsFile(upgradeWorkingDir)
if osDist == 'RHEL':
supportedSAPHANARevisionList = dict(
((x.split(':')[0].strip(),
re.sub('\\s+', '',
x.split(':')[1])) for x in
upgradeResourceDict['rhelSupportedSAPHANARevision'].split(
',')))['RHEL ' + osUpgradeVersion].split('|')
else:
supportedSAPHANARevisionList = dict(
((x.split(':')[0].strip(),
re.sub('\\s+', '',
x.split(':')[1])) for x in
upgradeResourceDict['slesSupportedSAPHANARevision'].split(
',')))['SLES ' + osUpgradeVersion].split('|')
debugLogger.info('The supported SAP HANA revision list was: ' +
str(supportedSAPHANARevisionList))
sidList, tenantUserDict = checkSAPHANA(
upgradeWorkingDir, serverModel, supportedSAPHANARevisionList)
if len(tenantUserDict) != 0:
homeDirList = getSAPUserLoginData(
upgradeWorkingDir, sidList, tenantUserDict=tenantUserDict)
else:
homeDirList = getSAPUserLoginData(upgradeWorkingDir, sidList)
restorationBackup += homeDirList
restorationBackup += getSapInitBackupList()
if 'DL580' in serverModel:
conrepDir = upgradeWorkingDir + '/conrepFile'
getConrepFile(programParentDir, conrepDir)
if 'DL580' in serverModel and processor != 'ivybridge':
mellanoxPresent = checkForMellanox(programParentDir,
upgradeWorkingDir, osDist,
osUpgradeVersion)
if osDist == 'SLES':
if serverModel == 'Superdome':
ctrlFileImg = updateInstallCtrlFile(
programParentDir, upgradeWorkingDir, osUpgradeVersion)
else:
command = 'lsblk -l'
result = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
out, err = result.communicate()
if result.returncode != 0:
debugLogger.error(
'Unable to get the partition information for the root file system.\n'
+ err + '\n' + out)
print RED + 'Unable to get the partition information for the root file system; fix the problem and try again; exiting program execution.' + RESETCOLORS
exit(1)
if re.match('.*(360002ac0+.*)[_-]part.*/boot/efi.*$', out,
re.DOTALL | re.MULTILINE):
ctrlFileImg = updateInstallCtrlFile(
programParentDir,
upgradeWorkingDir,
osUpgradeVersion,
mellanoxPresent=mellanoxPresent,
serverModel='DL580',
bootFromSAN='bootFromSAN')
else:
ctrlFileImg = updateInstallCtrlFile(
programParentDir,
upgradeWorkingDir,
osUpgradeVersion,
mellanoxPresent=mellanoxPresent,
serverModel='DL580')
elif serverModel == 'Superdome':
ctrlFileImg = updateRHELInstallCtrlFile(
programParentDir, upgradeWorkingDir)
else:
cs500CtrlFileImg = programParentDir + '/installCtrlFiles/cs500_ks.img'
installCtrlFileDir = upgradeWorkingDir + '/installCtrlFile'
ctrlFileImg = installCtrlFileDir + '/cs500_ks.img'
try:
os.mkdir(installCtrlFileDir)
shutil.copy2(cs500CtrlFileImg, ctrlFileImg)
except OSError as err:
debugLogger.error(
"Unable to create install control file directory '" +
installCtrlFileDir + "'.\n" + str(err))
print RED + 'Unable to create the install control file directory; fix the problem and try again; exiting program execution.' + RESETCOLORS
exit(1)
except IOError as err:
debugLogger.error(
'Unable to copy the CS500 control image file (' +
cs500CtrlFileImg + ") to '" + ctrlFileImg + "'.\n" +
str(err))
print RED + 'Unable to copy the CS500 control image file; fix the problem and try again; exiting program execution.' + RESETCOLORS
exit(1)
if osDist == 'RHEL':
if os.path.isfile('/etc/sudoers'):
command = "sed -i 's/\\s*Defaults requiretty/#Defaults requiretty/g' /etc/sudoers"
result = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
out, err = result.communicate()
if result.returncode != 0:
debugLogger.error(
'There was a problem updating /etc/sudoers.\n' +
err + '\n' + out)
debugLogger.info(
'The command used to update /etc/sudoers was: ' +
command)
print RED + 'There was a problem updating /etc/sudoers; fix the problem and try again; exiting program execution.' + RESETCOLORS
exit(1)
if serverModel != 'Superdome':
print GREEN + "Changing the server's boot mode from Legacy to UEFI." + RESETCOLORS
command = 'conrep -l -f ' + programParentDir + '/conrepFile/conrepBootMode.dat'
result = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
out, err = result.communicate()
if result.returncode != 0:
debugLogger.error(
"An error was encountered while updating the server's BIOS from Legacy mode to UEFI mode.\n"
+ err + '\n' + out)
debugLogger.info(
"The command used to update the server's BIOS from Legacy mode to UEFI mode was: "
+ command)
print RED + "There was a problem updating the server's BIOS from Legacy mode to UEFI mode; fix the problem and try again; exiting program execution." + RESETCOLORS
exit(1)
if 'DL580' in serverModel or serverModel == 'Superdome':
backupItems = [
restorationBackup, sapRestorationBackup, archiveBackup
]
else:
backupItems = [restorationBackup, archiveBackup]
checkDiskSpace(backupItems, upgradeWorkingDir)
createNetworkInformationFile(upgradeWorkingDir, osDist)
return ('DL580' in serverModel or serverModel == 'Superdome') and ([
restorationBackup, archiveBackup, sapRestorationBackup
], ctrlFileImg)
else:
return ([restorationBackup, archiveBackup], ctrlFileImg)
return