def generatePatchSequence(self, patchDir, installName=None,
isUpToPatch=False):
patchOrderGen = self._patchOrderGen
patchHistInfo = self._vistaPatchInfo
patchList = []
if isUpToPatch:
patchList = patchOrderGen.generatePatchOrderTopologic(patchDir)
else:
patchList = patchOrderGen.generatePatchOrderTopologic(patchDir, installName)
self._patchSet.update([x.installName for x in patchList])
if installName and installName not in self._patchSet:
errMsg = ("Can not find patch for install name %s" % installName)
raise Exception(errMsg)
for patchInfo in patchList:
namespace = patchInfo.namespace
self.__updatePatchInfoPackageName__(patchInfo)
logger.info("Checking for patch info %s" % patchInfo.installName)
logger.debug("Checking for patch info %s" % patchInfo)
""" check to see if the patch is already installed """
if not self.__isPatchInstalled__(patchInfo):
if self.__isPatchReadyToInstall__(patchInfo, patchList):
self._outPatchList.append(patchInfo)
else:
errorMsg = ("Can not install patch %s" % patchInfo.installName)
logger.error(errorMsg)
raise Exception(errorMsg)
if isUpToPatch and installName and installName == patchInfo.installName:
break
logger.info("Total patches are %d" % len(self._outPatchList))
for patchInfo in self._outPatchList:
logger.info("%s, %s, %s" % (patchInfo.installName,
patchInfo.namespace,
patchInfo.kidsFilePath))
return self._outPatchList
def parseKIDSInfoFile(self, infoFile):
logger.debug("Parsing Info file %s" % infoFile)
patchInfo = PatchInfo()
assert os.path.exists(infoFile)
patchInfo.kidsInfoPath = infoFile
inputFile = open(infoFile, 'rb')
for line in inputFile:
line = line.rstrip(" \r\n")
if len(line) == 0:
continue
""" subject part are treated as end of parser section for now"""
if self.SUBJECT_PART_START_REGEX.search(line):
break;
ret = self.RUNDATE_DESIGNATION_REGEX.search(line)
if ret:
patchInfo.rundate = datetime.strptime(ret.group('date'),
self.RUNDATE_FORMAT_STRING)
patchInfo.installName = convertToInstallName(ret.group('design'))
continue
ret = self.PACKAGE_PRIORITY_REGEX.search(line)
if ret:
package = ret.group('name').strip()
logger.debug(package)
(namespace, name) = package.split(" - ", 1) # split on first -
patchInfo.namespace = namespace.strip()
patchInfo.package = name.strip()
patchInfo.priority = ret.group('pri').strip()
continue
ret = self.VERSION_STATUS_REGEX.search(line)
if ret:
versionInfo = ret.group('no').strip()
pos = versionInfo.find('SEQ #')
if pos >= 0:
patchInfo.version = versionInfo[:pos].strip()
patchInfo.seqNo = versionInfo[pos+5:].strip()
try: int(patchInfo.seqNo)
except ValueError as ex:
logger.error("invalid seqNo %s" % patchInfo.seqNo)
raise ex
else:
patchInfo.version = versionInfo.strip()
# fix the patch version to make sure 1 => 1.0
if (float(patchInfo.version).is_integer() and
patchInfo.version.find('.') < 0):
patchInfo.version += ".0"
patchInfo.status = ret.group('status').strip()
""" find out the dep patch info """
ret = self.ASSOCIATED_PATCH_START_REGEX.search(line)
if ret:
self.parseAssociatedPart(line[self.ASSOCIATED_PATCH_START_INDEX:],
patchInfo)
continue
ret = self.ASSOCIATED_PATCH_SECTION_REGEX.search(line)
if ret:
self.parseAssociatedPart(line.strip(), patchInfo)
continue
if patchInfo.installName == None: return None
setPatchInfoFromInstallName(patchInfo.installName,
patchInfo)
return patchInfo
def __applyIndividualPatch__(self, patchInfo):
""" double check to see if patch is already installed """
if self.__isPatchInstalled__(patchInfo):
return 0
if not self.__isPatchReadyToInstall__(patchInfo):
return -1
""" generate Sha1 Sum for patch Info """
self.generateSha1SumForPatchInfo(patchInfo)
""" install the Patch """
result = self.__installPatch__(patchInfo)
if not result:
errorMsg = ("Failed to install patch %s: KIDS %s" %
(patchInfo.installName, patchInfo.kidsFilePath))
logger.error(errorMsg)
raise Exception(errorMsg)
else:
# also need to reload the package patch hist
self.__reloadPackagePatchHistory__(patchInfo)
""" special logic to handle release code """
installed = False
namespace,ver,patch = extractInfoFromInstallName(patchInfo.installName)
if not patch:
if namespace and ver:
updVer = getPackageLatestVersionByNamespace(namespace,
self._testClient)
if updVer and updVer == ver:
installed = True
if not installed:
assert self.__isPatchInstalled__(patchInfo)
return 1
def normalInstallation(self, vistATestClient, vistATestClient2=None, reinst=True):
logger.info("Start installing %s" % self._kidsInstallName)
connection = vistATestClient.getConnection()
if vistATestClient2:
connection2 = vistATestClient2.getConnection()
self.__gotoKIDSMainMenu__(vistATestClient)
self.__loadKIDSBuild__(connection)
result = self.__handleKIDSLoadOptions__(connection, reinst)
if not result:
logger.error("Error handling KIDS Load Options %s, %s" %
(self._kidsInstallName, self._kidsFile))
return False
if self._tgOutputDir:
if self._multiBuildList is None:
self.__printTransportGlobal__(vistATestClient,[self._kidsInstallName],self._tgOutputDir)
else:
self.__printTransportGlobal__(vistATestClient,self._multiBuildList,self._tgOutputDir)
if vistATestClient2:
result = self.__handleKIDSInstallQuestions__(connection, connection2)
else:
result = self.__handleKIDSInstallQuestions__(connection)
if not result:
result = self.unloadDistribution(vistATestClient, False)
if not result:
logger.error("Unload %s failed" % self._kidsInstallName)
return False
return self.normalInstallation(vistATestClient, vistATestClient2, reinst)
self.__installationCommon__(vistATestClient)
return True
def generateKIDSPatchSequence(self, patchDir, installName=None,
isUpToPatch=False):
kidsOrderGen = self._kidsOrderGen
patchHistInfo = self._vistaPatchInfo
patchList = []
if isUpToPatch:
patchList = kidsOrderGen.generatePatchOrderTopologic(patchDir)
else:
patchList = kidsOrderGen.generatePatchOrderTopologic(patchDir, installName)
self._patchSet.update([x.installName for x in patchList])
if installName and installName not in self._patchSet:
errMsg = ("Can not find patch for install name %s" % installName)
raise Exception(errMsg)
for patchInfo in patchList:
namespace = patchInfo.namespace
self.__updatePatchInfoPackageName__(patchInfo)
logger.info("Checking for patch info %s" % patchInfo.installName)
logger.debug("Checking for patch info %s" % patchInfo)
""" check to see if the patch is already installed """
if not self.__isPatchInstalled__(patchInfo):
if self.__isPatchReadyToInstall__(patchInfo, patchList):
self._outPatchList.append(patchInfo)
else:
errorMsg = ("Can not install patch %s" % patchInfo.installName)
logger.error(errorMsg)
raise Exception(errorMsg)
if isUpToPatch and installName and installName == patchInfo.installName:
break
logger.info("Total patches are %d" % len(self._outPatchList))
for patchInfo in self._outPatchList:
logger.info("%s, %s, %s" % (patchInfo.installName,
patchInfo.namespace,
patchInfo.kidsFilePath))
def verifyTaskmanSiteParameter(self, vistAClient, autoFix=True):
retValue = True
connection = vistAClient.getConnection()
menuUtil = VistAMenuUtil(duz=1)
boxVolPair = getBoxVolPair(vistAClient)
logger.debug("Box:Vol Pair is [%s] " % boxVolPair)
menuUtil.gotoTaskmanEditParamMenu(vistAClient)
connection.send("Site Parameters Edit\r")
connection.expect("Select TASKMAN SITE PARAMETERS BOX-VOLUME PAIR: ")
connection.send("?\r")
connection.expect("Answer with TASKMAN SITE PARAMETERS BOX-VOLUME PAIR.*?:")
connection.expect("You may enter a new TASKMAN SITE PARAMETERS")
curBoxVol = connection.before.strip(' \r\n')
curBoxVol = [x.strip(' ') for x in curBoxVol.split('\r\n')]
logger.debug("Box:Vol Pair is [%s] " % curBoxVol)
if boxVolPair not in curBoxVol :
logger.error("taskman site parameter mismatch, current:[%s], correct:[%s]" %
(curBoxVol, boxVolPair))
if autoFix:
self.__fixTaskmanSiteParameter__(connection, curBoxVol[0], boxVolPair)
else:
retValue = False
connection.expect("Select TASKMAN SITE PARAMETERS BOX-VOLUME PAIR: ")
connection.send('\r')
menuUtil.exitTaskmanEditParamMenu(vistAClient)
return retValue
def __isPatchReadyToInstall__(self, patchInfo, patchList = None):
packageName = patchInfo.package
ver = patchInfo.version
patchHist = self._vistaPatchInfo.getPackagePatchHistByName(packageName, ver)
if not patchHist or not patchHist.hasPatchHistory():
logger.info("no patch hist for %s, ver: %s" % (packageName, ver))
return True # if no such an package or hist info, just return True
""" check patch sequence no to see if it is out of order """
if patchInfo.seqNo:
seqNo = patchHist.getLatestSeqNo()
if patchInfo.seqNo < seqNo:
logger.error("SeqNo out of order, %s less than latest one %s" %
(patchInfo.seqNo), seqNo)
return False
# check all the dependencies
for item in patchInfo.depKIDSPatch:
if patchList and item in self._patchSet: # we are going to install the dep patch
logger.info("We are going to install the patch %s" % item)
""" make sure installation is in the right order """
itemIndex = self.indexInPatchList(item, patchList)
patchIndex = self.indexInPatchList(patchInfo.installName, patchList)
if itemIndex >= patchIndex:
logger.warn("%s is out of order with %s" % (item, patchInfo))
return False
else:
continue
(namespace,ver,patch) = extractInfoFromInstallName(item)
if self._vistaPatchInfo.hasPatchInstalled(item, namespace, ver, patch):
logger.debug("%s is arelady installed" % item)
continue
else:
logger.error("dep %s is not installed for %s %s" %
(item, patchInfo.installName, patchInfo.kidsFilePath))
return False
return True
def generateKIDSPatchSequence(self, patchDir, patchesCSV=None):
kidsOrderGen = self._kidsOrderGen
patchHistInfo = self._vistaPatchInfo
patchList = []
if patchesCSV and os.path.exists(patchCSVFile):
patchList = kidsOrderGen.generatePatchOrderByCSV(patchDir, patchCSVFile)
else:
patchList = kidsOrderGen.generatePatchOrderTopologic(patchDir)
self._patchSet.update([x.installName for x in patchList])
for patchInfo in patchList:
namespace = patchInfo.namespace
self.__updatePatchInfoPackageName__(patchInfo)
logger.info("Checking for patch info %s" % patchInfo.installName)
logger.debug("Checking for patch info %s" % patchInfo)
""" check to see if the patch is already installed """
if self.__isPatchInstalled__(patchInfo):
continue
if self.__isPatchReadyToInstall__(patchInfo, patchList):
self._outPatchList.append(patchInfo)
else:
logger.error("Can not install patch %s" % patchInfo.installName)
break
logger.info("Total patches are %d" % len(self._outPatchList))
for patchInfo in self._outPatchList:
logger.info("%s, %s, %s" % (patchInfo.installName,
patchInfo.namespace,
patchInfo.kidsFilePath))
def __exit__(self, exc_type, exc_value, traceback):
logger.debug("__exit__ is called %s,%s,%s" % (exc_type, exc_value, traceback))
connection = self._connection
if exc_type is KeyboardInterrupt:
connection.terminate()
return True
if connection is not None and connection.isalive():
# try to close the connection gracefully
try:
for i in range(0,3):
connection.send("^\r") # get out of the MENU prompt by sending ^
connection.send("\r") # get out of the MENU prompt by sendng \r
connection.send("Q\r") # clean up any stack
self.waitForPrompt(1) # wait for VistA prompt for 1 seconds
connection.send("H\r") # Halt VistA connection
except Exception as ex:
logger.error(ex)
if isLinuxSystem():
if connection.isalive():
""" pexpect close() will close all open handlers and is non-blocking """
try:
connection.close()
except ExceptionPexpect as ose:
logger.error(ose)
connection.terminate()
return
def applyKIDSPatchSequence(self, numOfPatches = 1, restart = True):
totalPatch = len(self._outPatchList)
numOfPatch = 1
if re.match("All", str(numOfPatches), re.IGNORECASE):
numOfPatch = totalPatch
else:
numOfPatch = int(numOfPatches)
restartCache = False
taskmanUtil = VistATaskmanUtil()
""" make sure taskman is running """
taskmanUtil.startTaskman(self._testClient)
for index in range(0,min(totalPatch, numOfPatch)):
patchInfo = self._outPatchList[index]
""" double check to see if patch is already installed """
if self.__isPatchInstalled__(patchInfo):
continue
if not self.__isPatchReadyToInstall__(patchInfo):
break
""" generate Sha1 Sum for patch Info """
self.generateSha1SumForPatchInfo(patchInfo)
""" install the KIDS patch """
result = self.__installKIDSPatch__(patchInfo)
if not result:
logger.error("Failed to install patch %s: KIDS %s" %
(patchInfo.installName, patchInfo.kidsFilePath))
return
else:
# also need to reload the package patch hist
self.__reloadPackagePatchHistory__(patchInfo)
assert self.__isPatchInstalled__(patchInfo)
restartCache = True
taskmanUtil.waitTaskmanToCurrent(self._testClient)
""" restart Cache if needed """
if restart and restartCache and self._testClient.isCache():
self.__restartCache__()
def parseKIDSInfoFile(self, infoFile):
logger.debug("Parsing Info file %s" % infoFile)
patchInfo = PatchInfo()
assert os.path.exists(infoFile)
patchInfo.kidsInfoPath = infoFile
inputFile = open(infoFile, 'rb')
for line in inputFile:
line = line.rstrip(" \r\n")
if len(line) == 0:
continue
""" subject part are treated as end of parser section for now"""
if self.SUBJECT_PART_START_REGEX.search(line):
break;
ret = self.RUNDATE_DESIGNATION_REGEX.search(line)
if ret:
patchInfo.rundate = datetime.strptime(ret.group('date'),
self.RUNDATE_FORMAT_STRING)
patchInfo.installName = convertToInstallName(ret.group('design'))
continue
ret = self.PACKAGE_PRIORITY_REGEX.search(line)
if ret:
package = ret.group('name').strip()
logger.debug(package)
(namespace, name) = package.split(" - ", 1) # split on first -
patchInfo.namespace = namespace.strip()
patchInfo.package = name.strip()
patchInfo.priority = ret.group('pri').strip()
continue
ret = self.VERSION_STATUS_REGEX.search(line)
if ret:
versionInfo = ret.group('no').strip()
pos = versionInfo.find('SEQ #')
if pos >= 0:
patchInfo.version = versionInfo[:pos].strip()
patchInfo.seqNo = versionInfo[pos+5:].strip()
try: int(patchInfo.seqNo)
except ValueError as ex:
logger.error("invalid seqNo %s" % patchInfo.seqNo)
raise ex
else:
patchInfo.version = versionInfo.strip()
# fix the patch version to make sure 1 => 1.0
if (float(patchInfo.version).is_integer() and
patchInfo.version.find('.') < 0):
patchInfo.version += ".0"
patchInfo.status = ret.group('status').strip()
""" find out the dep patch info """
ret = self.ASSOCIATED_PATCH_START_REGEX.search(line)
if ret:
self.parseAssociatedPart(line[self.ASSOCIATED_PATCH_START_INDEX:],
patchInfo)
continue
ret = self.ASSOCIATED_PATCH_SECTION_REGEX.search(line)
if ret:
self.parseAssociatedPart(line.strip(), patchInfo)
continue
if patchInfo.installName == None: return None
setPatchInfoFromInstallName(patchInfo.installName,
patchInfo)
return patchInfo
def __onKIDSSectionStart__(self, section, lines, **kargs):
line = lines[0].rstrip(" \r\n")
ret = re.search('^\*\*KIDS\*\*:(?P<name>.*)\^$', line)
if ret:
self._installNameList = ret.group('name').rstrip(' ').split('^')
else:
logger.error("Invalid KIDS line [%s]" % line)
def restoreCacheData(instanceName, bzipDataFile, cacheDataDir, useSudo=False):
if not os.path.exists(bzipDataFile):
logger.error("%s does not exist" % bzipDataFile)
return False
if not os.path.exists(cacheDataDir):
logger.error("%s does not exist" % cacheDataDir)
return False
src = os.path.join(cacheDataDir, CACHE_DATA_FILE)
if not os.path.exists(src):
logger.error("%s does not exist" % src)
return False
if isInstanceRunning(instanceName):
logger.info("stop Cache instance %s" % instanceName)
if not stopCache(instanceName, useSudo=useSudo):
logger.info("force down Cache instance %s" % instanceName)
forceDownCache(instanceName, useSudo=useSudo)
if isInstanceRunning(instanceName):
logger.error("Can not stop cache instance %s" % instanceName)
return False
try:
tmpBackup = src + ".bak"
logger.info("rename %s to %s" % (src, tmpBackup))
os.rename(src, tmpBackup)
logger.info("extract %s to %s" % (bzipDataFile, cacheDataDir))
extractBZIP2Tarball(bzipDataFile, cacheDataDir)
logger.info("remove backup %s" % tmpBackup)
os.unlink(tmpBackup)
return True
except OSError as oe:
logger.error(os)
return False
def main():
parser = argparse.ArgumentParser(description='VistA KIDS Build Parser')
parser.add_argument('-i', '--inputFile', required=True,
help = 'Input KIDS Build file or Invidual Routine File')
parser.add_argument('-o', '--outputDir', default=None,
help = 'Output directory to store extracted M routines')
parser.add_argument('-c', '--checksum', default=False, action="store_true",
help = 'Print checksum of a M Routine')
parser.add_argument('-v', '--verify', default=False, action="store_true",
help = 'verify a KIDS Build')
parser.add_argument('-j', '--jsonOutputFile', default=None,
help = 'output metadata as json format')
result = parser.parse_args();
import logging
initConsoleLogging(logging.INFO)
if result.checksum:
chksum= checksum(result.inputFile)
sys.stdout.write("Checksum is: %s\n" % chksum)
elif result.verify:
kidsParser = KIDSBuildParser(result.outputDir)
ret = kidsParser.verifyKIDSBuild(result.inputFile)
if ret:
logger.info("%s is valid" % result.inputFile)
else:
logger.error("%s is invalid" % result.inputFile)
else:
kidsParser = KIDSBuildParser(result.outputDir)
kidsParser.parseKIDSBuild(result.inputFile)
kidsParser.printResult()
if result.jsonOutputFile:
outputMetaDataInJSON(kidsParser, result.jsonOutputFile)
loadMetaDataFromJSON(result.jsonOutputFile)
def parsePatchInstallDatetime(dtString):
"""
Utility function to parse Patch Install Date time
return datetime object if valid, otherwise None
"""
outDatetime = None
datetimeFmtLst = (
"%b %d, %Y@%H:%M:%S",
"%b %d, %Y@%H:%M",
"%b %d,%Y@%H:%M:%S",
"%b %d,%Y@%H:%M",
"%b %d,%Y %H:%M",
)
date_time_seperator = "@"
for fmtStr in datetimeFmtLst:
dtStr = convertAbbreviatedMonth(dtString)
try:
if dtStr.find(date_time_seperator) < 0:
if fmtStr.find(date_time_seperator) >= 0:
fmtStr = fmtStr[0:fmtStr.find(date_time_seperator)]
outDatetime = datetime.strptime(dtStr, fmtStr)
except ValueError as ve:
pass
if not outDatetime:
logger.error("Can not parse datetime %s" % dtString)
return outDatetime
def __applyIndividualPatch__(self, patchInfo):
""" double check to see if patch is already installed """
if self.__isPatchInstalled__(patchInfo):
return 0
if not self.__isPatchReadyToInstall__(patchInfo):
return -1
""" generate Sha1 Sum for patch Info """
self.generateSha1SumForPatchInfo(patchInfo)
""" install the Patch """
result = self.__installPatch__(patchInfo)
if not result:
errorMsg = ("Failed to install patch %s: KIDS %s" %
(patchInfo.installName, patchInfo.kidsFilePath))
logger.error(errorMsg)
raise Exception(errorMsg)
else:
# also need to reload the package patch hist
self.__reloadPackagePatchHistory__(patchInfo)
""" special logic to handle release code """
installed = False
namespace,ver,patch = extractInfoFromInstallName(patchInfo.installName)
if not patch:
if namespace and ver:
updVer = getPackageLatestVersionByNamespace(namespace,
self._testClient)
if updVer and updVer == ver:
installed = True
if not installed:
assert self.__isPatchInstalled__(patchInfo)
return 1
def hasPatchInstalled(self,
installName,
namespace,
version,
patchNo,
seqNo=None):
if namespace not in self._packageMapping:
logger.info("this is a new namespace %s" % namespace)
return False
packageName = self._packageMapping[namespace]
if packageName not in self._packagePatchHist:
patchHist = self.getPackagePatchHistByNamespace(namespace, version)
else:
patchHist = self._getPackageHistListByVer(packageName, version)
if not patchHist:
patchHist = self.getPackagePatchHistByNamespace(
namespace, version)
if patchHist:
if seqNo:
if patchHist.hasSeqNo(seqNo):
return True
if patchHist.version:
try:
if float(patchHist.version) != float(version):
logger.info("Diff ver %s, %s" %
(patchHist.version, version))
except Exception as ex:
logger.error(ex)
if patchNo:
return patchHist.hasPatchNo(patchNo)
status = self.getInstallationStatus(installName)
return self.isInstallCompleted(status)
def parsePackagePatchHistory(historyString, packageName, namespace, version):
allLines = historyString.split('\r\n')
patchHistoryStart = False
result = None
for line in allLines:
line = line.rstrip()
if len(line) == 0:
continue
if re.search("^-+$", line): # find line with '----'
patchHistoryStart = True
continue
if patchHistoryStart:
if not PatchInstallLog.isValidHistoryLine(line):
continue
if not result: result = PackagePatchHistory(packageName, namespace)
patchLog = PatchInstallLog(line)
result.addPatchInstallLog(patchLog)
if patchLog.hasVersion():
if version:
try:
if float(version) != float(patchLog.version):
logger.error("version mismatch, %s:%s" %
(version, patchLog.version))
except ValueError as ve:
logger.error(ve)
result.setVersion(patchLog.version)
return result
def downloadExternalDataDirectly(dwnUrl, fileToSave):
try:
urllib.urlretrieve(dwnUrl, fileToSave)
return True
except Exception as ex:
logger.error(ex)
return False
def main():
parser = argparse.ArgumentParser(description='VistA KIDS Patch Parser')
parser.add_argument('-i', '--inputFile', required=True,
help = 'Input KIDS Patch file or Invidual Routine File')
parser.add_argument('-o', '--outputDir', default=None,
help = 'Output directory to store extracted M routines')
parser.add_argument('-c', '--checksum', default=False, action="store_true",
help = 'Print checksum of a M Routine')
parser.add_argument('-v', '--verify', default=False, action="store_true",
help = 'verify a KIDS patch')
result = parser.parse_args();
import logging
initConsoleLogging(logging.INFO)
if result.checksum:
chksum= checksum(result.inputFile)
sys.stdout.write("Checksum is: %s\n" % chksum)
elif result.verify:
kidsParser = KIDSPatchParser(result.outputDir)
ret = kidsParser.verifyKidsPatch(result.inputFile)
if ret:
logger.info("%s is valid" % result.inputFile)
else:
logger.error("%s is invalid" % result.inputFile)
else:
kidsParser = KIDSPatchParser(result.outputDir)
kidsParser.parseKIDSBuild(result.inputFile)
kidsParser.printResult()
def hasPatchInstalled(self, installName, namespace, version,
patchNo, seqNo = None):
if namespace not in self._packageMapping:
logger.info("this is a new namespace %s" % namespace)
return False
packageName = self._packageMapping[namespace]
if packageName not in self._packagePatchHist:
patchHist = self.getPackagePatchHistByNamespace(namespace, version)
else:
patchHist = self._getPackageHistListByVer(packageName, version)
if not patchHist:
patchHist = self.getPackagePatchHistByNamespace(namespace, version)
if patchHist:
if seqNo:
if patchHist.hasSeqNo(seqNo):
return True
if patchHist.version:
try:
if float(patchHist.version) != float(version):
logger.info("Diff ver %s, %s" % (patchHist.version, version))
except Exception as ex:
logger.error(ex)
if patchNo:
return patchHist.hasPatchNo(patchNo)
status = self.getInstallationStatus(installName)
return self.isInstallCompleted(status)
def restoreCacheData(instanceName, bzipDataFile, cacheDataDir, useSudo=False):
if not os.path.exists(bzipDataFile):
logger.error("%s does not exist" % bzipDataFile)
return False
if not os.path.exists(cacheDataDir):
logger.error("%s does not exist" % cacheDataDir)
return False
src = os.path.join(cacheDataDir, CACHE_DATA_FILE)
if not os.path.exists(src):
logger.error("%s does not exist" % src)
return False
if isInstanceRunning(instanceName):
logger.info("stop Cache instance %s" % instanceName)
if not stopCache(instanceName, useSudo=useSudo):
logger.info("force down Cache instance %s" % instanceName)
forceDownCache(instanceName, useSudo=useSudo)
if isInstanceRunning(instanceName):
logger.error("Can not stop cache instance %s" % instanceName)
return False
try:
tmpBackup = src + ".bak"
logger.info("rename %s to %s" % (src, tmpBackup))
os.rename(src, tmpBackup)
logger.info("extract %s to %s" % (bzipDataFile, cacheDataDir))
extractBZIP2Tarball(bzipDataFile, cacheDataDir)
logger.info("remove backup %s" % tmpBackup)
os.unlink(tmpBackup)
return True
except OSError as oe:
logger.error(os)
return False
def parsePackagePatchHistory(historyString, packageName, namespace, version):
allLines = historyString.split('\r\n')
patchHistoryStart = False
result = None
for line in allLines:
line = line.rstrip()
if len(line) == 0:
continue
if re.search("^-+$", line): # find line with '----'
patchHistoryStart = True
continue
if patchHistoryStart:
if not PatchInstallLog.isValidHistoryLine(line):
continue
if not result: result = PackagePatchHistory(packageName, namespace)
patchLog = PatchInstallLog(line)
result.addPatchInstallLog(patchLog)
if patchLog.hasVersion():
if version:
try:
if float(version) != float(patchLog.version):
logger.error("version mismatch, %s:%s" %
(version, patchLog.version))
except ValueError as ve:
logger.error(ve)
result.setVersion(patchLog.version)
return result
def normalInstallation(self, vistATestClient, vistATestClient2=None, reinst=True):
logger.info("Start installing %s" % self._kidsInstallName)
connection = vistATestClient.getConnection()
if vistATestClient2:
connection2 = vistATestClient2.getConnection()
self.__gotoKIDSMainMenu__(vistATestClient)
self.__loadKIDSBuild__(connection)
result = self.__handleKIDSLoadOptions__(connection, reinst)
if not result:
logger.error("Error handling KIDS Load Options %s, %s" %
(self._kidsInstallName, self._kidsFile))
return False
if self._tgOutputDir:
if self._multiBuildList is None:
self.__printTransportGlobal__(vistATestClient,[self._kidsInstallName],self._tgOutputDir)
else:
self.__printTransportGlobal__(vistATestClient,self._multiBuildList,self._tgOutputDir)
if vistATestClient2:
result = self.__handleKIDSInstallQuestions__(connection, connection2)
else:
result = self.__handleKIDSInstallQuestions__(connection)
if not result:
result = self.unloadDistribution(vistATestClient, False)
if not result:
logger.error("Unload %s failed" % self._kidsInstallName)
return False
return self.normalInstallation(vistATestClient, vistATestClient2, reinst)
self.__installationCommon__(vistATestClient)
return True
def obtainKIDSPatchFileBySha1(filePath, sha1Sum, cacheDir):
assert cacheDir and os.path.exists(cacheDir)
rootDir = os.path.dirname(filePath)
externalFileName = generateExternalDataFileName(sha1Sum)
externalFile = os.path.join(rootDir, externalFileName)
logger.info("Checking %s" % externalFile)
if os.path.exists(externalFile):
if generateSha1Sum(externalFile) == sha1Sum:
return (True, externalFile)
else:
os.remove(externalFile)
""" try to find the file in the cache dir """
externalFile = os.path.join(cacheDir, externalFileName.replace('_','/'))
logger.info("Checking %s" % externalFile)
if os.path.exists(externalFile):
if generateSha1Sum(externalFile) == sha1Sum:
return (True, externalFile)
else:
os.remove(externalFile)
""" make sure cacheDir has the right layout """
rootDir = os.path.dirname(externalFile)
if not os.path.exists(rootDir):
os.makedirs(rootDir)
""" download from remote """
extDownloader = ExternalDataDownloader()
logger.info("Downloading from remote link")
result = extDownloader.downloadExternalDataByHash(sha1Sum, externalFile)
if not result:
logger.error("Downloading from remote failed")
if os.path.exists(externalFile):
os.remove(externalFile)
externalFile = None
logger.info("%s, %s" % (result, externalFile))
return (result, externalFile)
def downloadExternalDataDirectly(dwnUrl, fileToSave):
try:
urllib.urlretrieve(dwnUrl, fileToSave)
return True
except Exception as ex:
logger.error(ex)
return False
def __onKIDSSectionStart__(self, section, lines, **kargs):
line = lines[0].rstrip(" \r\n")
ret = re.search('^\*\*KIDS\*\*:(?P<name>.*)\^$', line)
if ret:
self._installNameList = ret.group('name').rstrip(' ').split('^')
else:
logger.error("Invalid KIDS line [%s]" % line)
def __parseAllKIDSBuildFilesList__(self):
kidsParser = KIDSPatchInfoParser()
for basename in self._kidsBuildFileDict.iterkeys():
kidsFile, sha1Path = self._kidsBuildFileDict[basename]
if kidsFile == None:
logger.error("No KIDS file available for name %s" % basename)
installNameList,seqNo = kidsParser.getKIDSBuildInstallNameSeqNo(kidsFile)
if len(installNameList) > 1:
if not self._multiBuildDict.get(kidsFile):
self._multiBuildDict[kidsFile] = installNameList
else:
assert self._multiBuildDict[kidsFile] == installNameList
elif seqNo:
if installNameList[0] not in self._installNameSeqMap:
self._installNameSeqMap[installNameList[0]] = seqNo
else:
logger.error("Duplicated KIDS build file %s" % kidsFile)
for installName in installNameList:
if installName in self._kidsInstallNameDict:
logger.warn("%s is already in the dict %s" % (installName, kidsFile))
logger.debug("Added installName %s, file %s" % (installName, kidsFile))
self._kidsInstallNameDict[installName] = os.path.normpath(kidsFile)
""" handle KIDS sha1 file Path """
if sha1Path:
if installName in self._kidsInstallNameSha1Dict:
logger.warn("%s is already in the dict %s" % (installName, sha1Path))
self._kidsInstallNameSha1Dict[installName] = sha1Path
logger.debug("%s" % sorted(self._kidsInstallNameDict.keys()))
logger.info("Total # of install name %d" % len(self._kidsInstallNameDict))
def obtainKIDSBuildFileBySha1(filePath, sha1Sum, cacheDir):
assert cacheDir and os.path.exists(cacheDir)
rootDir = os.path.dirname(filePath)
externalFileName = generateExternalDataFileName(sha1Sum)
externalFile = os.path.join(rootDir, externalFileName)
logger.info("Checking %s" % externalFile)
if os.path.exists(externalFile):
if generateSha1Sum(externalFile) == sha1Sum:
return (True, externalFile)
else:
os.remove(externalFile)
""" try to find the file in the cache dir """
externalFile = os.path.join(cacheDir, externalFileName.replace('_','/'))
logger.info("Checking %s" % externalFile)
if os.path.exists(externalFile):
if generateSha1Sum(externalFile) == sha1Sum:
return (True, externalFile)
else:
os.remove(externalFile)
""" make sure cacheDir has the right layout """
rootDir = os.path.dirname(externalFile)
if not os.path.exists(rootDir):
os.makedirs(rootDir)
""" download from remote """
extDownloader = ExternalDataDownloader()
logger.info("Downloading from remote link")
result = extDownloader.downloadExternalDataByHash(sha1Sum, externalFile)
if not result:
logger.error("Downloading from remote failed")
if os.path.exists(externalFile):
os.remove(externalFile)
externalFile = None
logger.info("%s, %s" % (result, externalFile))
return (result, externalFile)
def __exit__(self, exc_type, exc_value, traceback):
logger.debug("__exit__ is called %s,%s,%s" %
(exc_type, exc_value, traceback))
connection = self._connection
if exc_type is KeyboardInterrupt:
connection.terminate()
return True
if connection is not None and connection.isalive():
# try to close the connection gracefully
try:
for i in range(0, 3):
connection.send(
"^\r") # get out of the MENU prompt by sending ^
connection.send(
"\r") # get out of the MENU prompt by sendng \r
connection.send("Q\r") # clean up any stack
self.waitForPrompt(1) # wait for VistA prompt for 1 seconds
connection.send("H\r") # Halt VistA connection
except Exception as ex:
logger.error(ex)
if isLinuxSystem():
if connection.isalive():
""" pexpect close() will close all open handlers and is non-blocking """
try:
connection.close()
except ExceptionPexpect as ose:
logger.error(ose)
connection.terminate()
return
def getFileMan22_2RoutineFile(rFile):
sha1sum = readSha1SumFromSha1File(rFile)
logger.info("sha1sum is %s" % sha1sum)
result, path = obtainKIDSBuildFileBySha1(rFile, sha1sum, filedir)
if not result:
logger.error("Could not obtain FileMan 22V2 file for %s" % rFile)
raise Exception("Error getting FileMan 22V2 file for %s" % rFile)
return path
def getFileMan22_2RoutineFile(rFile):
sha1sum = readSha1SumFromSha1File(rFile)
logger.info("sha1sum is %s" % sha1sum)
result, path = obtainKIDSBuildFileBySha1(rFile, sha1sum, filedir)
if not result:
logger.error("Could not obtain FileMan 22V2 file for %s" % rFile)
raise Exception("Error getting FileMan 22V2 file for %s" % rFile)
return path
def downloadExternalDataByHash(self, sha1Sum, fileToSave):
dwnUrl = "%s/%s" % (self._siteUrl, sha1Sum)
if not self.downloadExternalDataDirectly(dwnUrl, fileToSave):
return False
""" verify the sha1sum of downloaded file """
sha1SumDwn = generateSha1Sum(fileToSave)
if sha1Sum == sha1SumDwn:
return True
logger.error("sha1Sum mismatch %s:%s" % (sha1Sum, sha1SumDwn))
os.remove(fileToSave)
def handleKIDSInstallQuestions(self, connection, **kargs):
errorCheckTimeout = 5 # 5 seconds
try:
connection.expect("\*\*INSTALL FILE IS CORRUPTED\*\*",errorCheckTimeout)
logger.error("%s:INSTALL FILE IS CORRUPTED" % self._kidsInstallName)
connection.expect("Select Installation ", errorCheckTimeout)
connection.send('\r')
return False
except Exception as ex:
return True
def handleKIDSInstallQuestions(self, connection, **kargs):
errorCheckTimeout = 5 # 5 seconds
try:
connection.expect("\*\*INSTALL FILE IS CORRUPTED\*\*",errorCheckTimeout)
logger.error("%s:INSTALL FILE IS CORRUPTED" % self._kidsInstallName)
connection.expect("Select Installation ", errorCheckTimeout)
connection.send('\r')
return False
except Exception as ex:
return True
def onSectionStart(self, section, lines, **kargs):
assert ('kidsBuild' in kargs and kargs['kidsBuild'] != None)
kidsBuild = kargs['kidsBuild']
logger.debug("Routine Section starts %s, %s" % (lines[0], lines[1]))
if self.ROUTINE_START.search(lines[0]):
kidsBuild._totalNumRoutinesChange = int(lines[1])
else:
logger.error("Unexpected Routine Section [%s]" % lines)
return False
return True
def downloadExternalDataByHash(self, sha1Sum, fileToSave):
dwnUrl = "%s/%s" % (self._siteUrl, sha1Sum)
if not self.downloadExternalDataDirectly(dwnUrl, fileToSave):
return False
""" verify the sha1sum of downloaded file """
sha1SumDwn = generateSha1Sum(fileToSave)
if sha1Sum == sha1SumDwn:
return True
logger.error("sha1Sum mismatch %s:%s" % (sha1Sum, sha1SumDwn))
os.remove(fileToSave)
def onSectionStart(self, section, lines, **kargs):
assert ('kidsBuild' in kargs and kargs['kidsBuild'] != None)
kidsBuild = kargs['kidsBuild']
logger.debug("Routine Section starts %s, %s" % (lines[0], lines[1]))
if self.ROUTINE_START.search(lines[0]):
kidsBuild._totalNumRoutinesChange = int(lines[1])
else:
logger.error("Unexpected Routine Section [%s]" % lines)
return False
return True
def __parseAllKIDSBuildFilesList__(self):
for basename in self._kidsBuildFileDict.iterkeys():
kidsFile, sha1Path = self._kidsBuildFileDict[basename]
if kidsFile == None:
logger.error("No KIDS file available for name %s" % basename)
continue
installNameList, seqNo, kidsBuilds = None, None, None
if isValidKIDSBuildHeaderSuffix(kidsFile):
from KIDSBuildParser import loadMetaDataFromJSON
#continue
installNameList, seqNo, kidsBuilds = loadMetaDataFromJSON(kidsFile)
else:
kidsParser = KIDSBuildParser(None)
kidsParser.unregisterSectionHandler(KIDSBuildParser.ROUTINE_SECTION)
kidsParser.parseKIDSBuild(kidsFile)
installNameList = kidsParser.installNameList
logger.debug("install name list is %s" % installNameList)
seqNo = kidsParser.seqNo
kidsBuilds = kidsParser.kidsBuilds
if len(installNameList) > 1:
if not self._multiBuildDict.get(kidsFile):
self._multiBuildDict[kidsFile] = installNameList
else:
assert self._multiBuildDict[kidsFile] == installNameList
elif seqNo:
if installNameList[0] not in self._installNameSeqMap:
self._installNameSeqMap[installNameList[0]] = seqNo
else:
logger.error("Duplicated KIDS build file %s" % kidsFile)
for installName in installNameList:
if installName in self._kidsInstallNameDict:
logger.warn("%s is already in the dict %s" % (installName, kidsFile))
logger.debug("Added installName %s, file %s" % (installName, kidsFile))
self._kidsInstallNameDict[installName] = os.path.normpath(kidsFile)
""" handle KIDS sha1 file Path """
if sha1Path:
if installName in self._kidsInstallNameSha1Dict:
logger.warn("%s is already in the dict %s" % (installName, sha1Path))
self._kidsInstallNameSha1Dict[installName] = sha1Path
""" update kids dependency """
if installName in self._kidsDepBuildDict:
logger.warn("%s already has the dep map %s" %
(installName, self._kidsDepBuildDict[installName]))
if kidsBuilds:
for kidsBuild in kidsBuilds:
if kidsBuild.installName == installName:
depList = kidsBuild.dependencyList
if depList:
self._kidsDepBuildDict[installName] = set([x[0] for x in
depList])
logger.info("%s: %s" % (installName,
self._kidsDepBuildDict[installName]))
logger.debug("%s" % sorted(self._kidsInstallNameDict.keys()))
logger.info("Total # of install name %d" % len(self._kidsInstallNameDict))
def __parseAllKIDSBuildFilesList__(self):
for basename in self._kidsBuildFileDict.iterkeys():
kidsFile, sha1Path = self._kidsBuildFileDict[basename]
if kidsFile == None:
logger.error("No KIDS file available for name %s" % basename)
continue
installNameList, seqNo, kidsBuilds = None, None, None
if isValidKIDSPatchHeaderSuffix(kidsFile):
from KIDSPatchParser import loadMetaDataFromJSON
#continue
installNameList, seqNo, kidsBuilds = loadMetaDataFromJSON(kidsFile)
else:
kidsParser = KIDSPatchParser(None)
kidsParser.unregisterSectionHandler(KIDSPatchParser.ROUTINE_SECTION)
kidsParser.parseKIDSBuild(kidsFile)
installNameList = kidsParser.installNameList
logger.debug("install name list is %s" % installNameList)
seqNo = kidsParser.seqNo
kidsBuilds = kidsParser.kidsBuilds
if len(installNameList) > 1:
if not self._multiBuildDict.get(kidsFile):
self._multiBuildDict[kidsFile] = installNameList
else:
assert self._multiBuildDict[kidsFile] == installNameList
elif seqNo:
if installNameList[0] not in self._installNameSeqMap:
self._installNameSeqMap[installNameList[0]] = seqNo
else:
logger.error("Duplicated KIDS build file %s" % kidsFile)
for installName in installNameList:
if installName in self._kidsInstallNameDict:
logger.warn("%s is already in the dict %s" % (installName, kidsFile))
logger.debug("Added installName %s, file %s" % (installName, kidsFile))
self._kidsInstallNameDict[installName] = os.path.normpath(kidsFile)
""" handle KIDS sha1 file Path """
if sha1Path:
if installName in self._kidsInstallNameSha1Dict:
logger.warn("%s is already in the dict %s" % (installName, sha1Path))
self._kidsInstallNameSha1Dict[installName] = sha1Path
""" update kids dependency """
if installName in self._kidsDepBuildDict:
logger.warn("%s is already has the dep map %s" %
(installName, self._kidsDepBuildDict[installName]))
if kidsBuilds:
for kidsBuild in kidsBuilds:
if kidsBuild.installName == installName:
depList = kidsBuild.dependencyList
if depList:
self._kidsDepBuildDict[installName] = set([x[0] for x in
depList])
logger.info("%s: %s" % (installName,
self._kidsDepBuildDict[installName]))
logger.debug("%s" % sorted(self._kidsInstallNameDict.keys()))
logger.info("Total # of install name %d" % len(self._kidsInstallNameDict))
def installFileMan22_2(testClient, inputROFile):
"""
Script to initiate FileMan 22.2
"""
if not os.path.exists(inputROFile):
logger.error("File %s does not exist" % inputROFile)
return
rFile = inputROFile
if isValidRoutineSha1Suffix(inputROFile):
rFile = getFileMan22_2RoutineFile(inputROFile)
from VistATaskmanUtil import VistATaskmanUtil
outDir = None # gtm routine import out dir
#allow logon to shutdown taskman via menu
inhibitLogons(testClient, flag=False)
# stop all taskman tasks
taskManUtil = VistATaskmanUtil()
logger.info("Stop Taskman...")
taskManUtil.stopTaskman(testClient)
logger.info("Inhibit logons...")
inhibitLogons(testClient)
# remove fileman 22.2 affected routines
logger.info("Remove FileMan 22 routines")
if testClient.isCache():
deleteFileManRoutinesCache(testClient)
else:
outDir = deleteFileManRoutinesGTM()
if not outDir:
logger.info("Can not identify mumps routine directory")
return
outDir = outDir[0]
# import routines into System
from VistARoutineImport import VistARoutineImport
vistARtnImport = VistARoutineImport()
logger.info("Import FileMan 22.2 Routines from %s" % rFile)
vistARtnImport.importRoutines(testClient, rFile, outDir)
# verify integrity of the routines that just imported
logger.info("Verify FileMan 22.2 Routines...")
verifyRoutines(testClient)
# rewrite fileman routines
logger.info("Rewrite FileMan 22.2 Routines...")
if testClient.isCache():
rewriteFileManRoutineCache(testClient)
else:
rewriteFileManRoutineGTM(outDir)
# initial fileman
logger.info("Initial FileMan...")
initFileMan(testClient, None, None)
logger.info("Initial FileMan 22.2...")
initFileMan22_2(testClient)
logger.info("Enable logons...")
inhibitLogons(testClient, flag=False)
""" restart taskman """
logger.info("Restart Taskman...")
taskManUtil.startTaskman(testClient)
def __onCurrentRoutineEnd__(self, kidsBuild):
if self._curRoutine:
self._curRoutine.checkSum = self._checkSum
if self._curRoutine.expectCheckSum != self._checkSum:
logger.error("%s checksum mismatch, expected %s, real: %s" %
(self._curRoutine.name,
self._curRoutine.expectCheckSum, self._checkSum))
self._checkSum = 0
kidsBuild.addRoutine(self._curRoutine)
if self._fileHandler:
self._fileHandler.close()
def installFileMan22_2(testClient, inputROFile):
"""
Script to initiate FileMan 22.2
"""
if not os.path.exists(inputROFile):
logger.error("File %s does not exist" % inputROFile)
return
rFile = inputROFile
if isValidRoutineSha1Suffix(inputROFile):
rFile = getFileMan22_2RoutineFile(inputROFile)
from VistATaskmanUtil import VistATaskmanUtil
outDir = None # gtm routine import out dir
#allow logon to shutdown taskman via menu
inhibitLogons(testClient, flag=False)
# stop all taskman tasks
taskManUtil = VistATaskmanUtil()
logger.info("Stop Taskman...")
taskManUtil.stopTaskman(testClient)
logger.info("Inhibit logons...")
inhibitLogons(testClient)
# remove fileman 22.2 affected routines
logger.info("Remove FileMan 22 routines")
if testClient.isCache():
deleteFileManRoutinesCache(testClient)
else:
outDir = deleteFileManRoutinesGTM()
if not outDir:
logger.info("Can not identify mumps routine directory")
return
outDir = outDir[0]
# import routines into System
from VistARoutineImport import VistARoutineImport
vistARtnImport = VistARoutineImport()
logger.info("Import FileMan 22.2 Routines from %s" % rFile)
vistARtnImport.importRoutines(testClient, rFile, outDir)
# verify integrity of the routines that just imported
logger.info("Verify FileMan 22.2 Routines...")
verifyRoutines(testClient)
# rewrite fileman routines
logger.info("Rewrite FileMan 22.2 Routines...")
if testClient.isCache():
rewriteFileManRoutineCache(testClient)
else:
rewriteFileManRoutineGTM(outDir)
# initial fileman
logger.info("Initial FileMan...")
initFileMan(testClient, None, None)
logger.info("Initial FileMan 22.2...")
initFileMan22_2(testClient)
logger.info("Enable logons...")
inhibitLogons(testClient, flag=False)
""" restart taskman """
logger.info("Restart Taskman...")
taskManUtil.startTaskman(testClient)
def __onCurrentRoutineEnd__(self, kidsBuild):
if self._curRoutine:
self._curRoutine.checkSum = self._checkSum
if self._curRoutine.expectCheckSum != self._checkSum:
logger.error("%s checksum mismatch, expected %s, real: %s" %
(self._curRoutine.name, self._curRoutine.expectCheckSum,
self._checkSum))
self._checkSum = 0
kidsBuild.addRoutine(self._curRoutine)
if self._fileHandler:
self._fileHandler.close()
def _checkMultiBuildsOrder(self):
""" make sure that all the multi-build are grouped together """
multiDict = dict()
for index in range(len(self._patchOrder)):
patchInfo = self._patchOrder[index]
if patchInfo.isMultiBuilds:
if patchInfo.kidsFilePath not in multiDict:
multiDict[patchInfo.kidsFilePath] = index
if (multiDict[patchInfo.kidsFilePath] != index
and multiDict[patchInfo.kidsFilePath] != index - 1):
logger.error("Patch out of order %s" % patchInfo)
multiDict[patchInfo.kidsFilePath] = index
def _checkMultiBuildsOrder(self):
""" make sure that all the multi-build are grouped together """
multiDict = dict()
for index in range(len(self._patchOrder)):
patchInfo = self._patchOrder[index]
if patchInfo.isMultiBuilds:
if patchInfo.kidsFilePath not in multiDict:
multiDict[patchInfo.kidsFilePath] = index
if ( multiDict[patchInfo.kidsFilePath] != index and
multiDict[patchInfo.kidsFilePath] != index - 1 ):
logger.error("Patch out of order %s" % patchInfo)
multiDict[patchInfo.kidsFilePath] = index
def __installPatch__(self, patchInfo):
installName = patchInfo.installName
kidsPath = patchInfo.kidsFilePath
seqNo = patchInfo.seqNo
logFileName = self._logFileName
multiBuildsList = patchInfo.multiBuildsList
kidsInstaller = None
""" handle patch stored as external link """
if patchInfo.kidsSha1Path != None:
kidsSha1 = patchInfo.kidsSha1
(result,
resultPath) = obtainKIDSBuildFileBySha1(kidsPath, kidsSha1,
DEFAULT_CACHE_DIR)
if not result:
logger.error("Could not obtain external Patch for %s" %
kidsPath)
return result
kidsPath = resultPath # set the KIDS Path
""" get the right KIDS installer """
associateFiles = patchInfo.associatedInfoFiles
associatedGlobals = patchInfo.associatedGlobalFiles
if patchInfo.hasCustomInstaller:
""" use python imp module to load the source """
logger.info("using custom installer %s" %
patchInfo.customInstallerPath)
import imp
installerModule = imp.load_source("KIDS",
patchInfo.customInstallerPath)
from KIDS import CustomInstaller
kidsInstaller = CustomInstaller(kidsPath,
installName,
seqNo,
logFileName,
multiBuildsList,
files=associateFiles,
globals=associatedGlobals,
duz=self._duz)
else:
kidsInstaller = KIDSInstallerFactory.createKIDSInstaller(
kidsPath,
installName,
seqNo,
logFileName,
multiBuildsList,
files=associateFiles,
globals=associatedGlobals,
duz=self._duz)
logger.info("Applying Patch %s" % patchInfo)
assert kidsInstaller
return kidsInstaller.runInstallation(self._testClient,
self._testClient2)
def __applyKIDSPatchUptoIndex__(self, endIndex):
assert endIndex >= 0 and endIndex <= len(self._outPatchList)
""" make sure taskman is running """
taskmanUtil = VistATaskmanUtil()
taskmanUtil.startTaskman(self._testClient)
for index in range(0, endIndex):
patchInfo = self._outPatchList[index]
result = self.__applyIndividualKIDSPatch__(patchInfo)
if not result:
logger.error("Failed to install patch %s: KIDS %s" % (patchInfo.installName, patchInfo.kidsFilePath))
return
""" wait until taskman is current """
taskmanUtil.waitTaskmanToCurrent(self._testClient)
def setPatchInfoFromInstallName(installName, patchInfo):
(namespace, ver, patch) = extractInfoFromInstallName(installName)
if patchInfo.namespace:
if patchInfo.namespace != namespace:
logger.error("Namespace mismatch. Expected %s. Found %s" % (namespace, patchInfo.namespace))
else:
patchInfo.namespace = namespace
if patchInfo.version:
if (float(patchInfo.version) != float(ver)):
logger.error("Version mismatch. Expected %s. Found %s" % (ver, patchInfo.version))
else:
patchInfo.version = ver
patchInfo.patchNo = patch
def __restartCache__(self):
self._testClient.getConnection().terminate()
logger.info("Wait for 5 seconds")
time.sleep(5)
logger.info("Restart CACHE instance")
instanceName = self._testClient.getInstanceName()
result = stopCache(instanceName, True)
if not result:
result = forceDownCache(instanceName)
if result:
startCache(instanceName)
else:
logger.error("Can not stop cache instance %s" % instanceName)
def __isPatchReadyToInstall__(self, patchInfo, patchList=None):
packageName = patchInfo.package
ver = patchInfo.version
patchHist = self._vistaPatchInfo.getPackagePatchHistByName(
packageName, ver)
if not patchHist or not patchHist.hasPatchHistory():
logger.info("no patch hist for %s, ver: %s" % (packageName, ver))
return True # if no such an package or hist info, just return True
""" check patch sequence no to see if it is out of order """
if patchInfo.seqNo:
seqNo = patchHist.getLatestSeqNo()
if patchInfo.seqNo < seqNo:
logger.error(
"SeqNo out of order, %s less than latest one %s" %
(patchInfo.seqNo), seqNo)
return False
# check all the dependencies
for item in patchInfo.depKIDSBuild:
if patchList and item in self._patchSet: # we are going to install the dep patch
logger.info("We are going to install the patch %s" % item)
""" make sure installation is in the right order """
itemIndex = self.indexInPatchList(item, patchList)
patchIndex = self.indexInPatchList(patchInfo.installName,
patchList)
if itemIndex >= patchIndex:
logger.warn("%s is out of order with %s" %
(item, patchInfo))
return False
else:
continue
(namespace, ver, patch) = extractInfoFromInstallName(item)
if self._vistaPatchInfo.hasPatchInstalled(item, namespace, ver,
patch):
logger.debug("%s is arelady installed" % item)
continue
installStatus = self._vistaPatchInfo.getInstallationStatus(item)
if self._vistaPatchInfo.isInstallCompleted(installStatus):
continue
elif item in patchInfo.optionalDepSet:
logger.warn(
"Patch specified in KIDS info file %s is not installed for %s"
% (item, patchInfo.installName))
continue
else:
logger.error(
"dep %s is not installed for %s %s" %
(item, patchInfo.installName, patchInfo.kidsFilePath))
patchInfo.depKIDSBuild.remove(item)
patchInfo.depKIDSBuild.add(item + " <---")
return False
return True
def __applyKIDSPatchUptoIndex__(self, endIndex):
assert endIndex >= 0 and endIndex <= len(self._outPatchList)
""" make sure taskman is running """
taskmanUtil = VistATaskmanUtil()
taskmanUtil.startTaskman(self._testClient)
for index in range(0, endIndex):
patchInfo = self._outPatchList[index]
result = self.__applyIndividualKIDSPatch__(patchInfo)
if not result:
logger.error("Failed to install patch %s: KIDS %s" %
(patchInfo.installName, patchInfo.kidsFilePath))
return
""" wait until taskman is current """
taskmanUtil.waitTaskmanToCurrent(self._testClient)
def run(self):
patchApplyConfig = self._config['Patch_Apply']
isContinuous = patchApplyConfig.get('continuous')
patchLogDir = patchApplyConfig['log_dir']
if not os.path.exists(patchLogDir):
logger.error("%s does not exist" % patchLogDir)
return False
inputPatchDir = patchApplyConfig['input_patch_dir']
mExtractConfig = self._config['M_Extract']
mRepo = mExtractConfig['M_repo']
mRepoBranch = mExtractConfig.get('M_repo_branch', None)
outputDir = mExtractConfig['temp_output_dir']
if not os.path.exists(outputDir):
os.mkdirs(outputDir)
extractLogDir = mExtractConfig['log_dir']
commitMsgDir = mExtractConfig['commit_msg_dir']
if not os.path.exists(commitMsgDir):
logger.error("%s does not exist" % commitMsgDir)
return False
backupConfig = self._config.get('Backup')
startCache(self._instance, self._useSudo)
testClient = self._createTestClient()
with testClient:
patchApply = PatchSequenceApply(testClient, patchLogDir)
outPatchList = patchApply.generatePatchSequence(inputPatchDir)
if not outPatchList:
logger.info("No Patch needs to apply")
return True
if not isContinuous:
outPatchList = [outPatchList[0]]
for patchInfo in outPatchList:
logger.info(patchInfo)
result = patchApply.applyPatchSequenceByInstallName(
patchInfo.installName,
patchOnly=True)
if result < 0:
logger.error("Error installing patch %s" % patchInfo.installName)
return False
elif result == 0:
logger.info("%s is already installed" % patchInfo.installName)
continue
commitFile = getDefaultCommitMsgFileByPatchInfo(patchInfo,
dir=commitMsgDir)
generateCommitMsgFileByPatchInfo(patchInfo, commitFile)
MExtractor = VistADataExtractor(mRepo, outputDir, extractLogDir,
gitBranch=mRepoBranch)
MExtractor.extractData(testClient)
commit = MCompReposCommitter(mRepo)
commit.commit(commitFile)
if backupConfig:
backupDir = backupConfig['backup_dir']
if not os.path.exists(backupDir):
logger.error("%s does not exist" % backupDir)
return False
cacheDir = backupConfig['cache_dat_dir']
origDir = os.path.join(cacheDir, "CACHE.DAT")
backupCacheDataByGitHash(self._instance, origDir, backupDir,
mRepo, mRepoBranch, self._useSudo)
startCache(self._instance, self._useSudo)
def backupCacheDataByGitHash(instanceName,
origDataPath,
backupDir,
gitReposDir,
gitReposBranch,
useSudo=False):
if not os.path.exists(origDataPath):
logger.error("%s does not exist" % origDataPath)
return False
if not os.path.exists(gitReposDir):
logger.error("%s does not exist" % gitReposDir)
return False
if not os.path.exists(backupDir):
logger.error("%s does not exists" % backupDir)
return False
if isInstanceRunning(instanceName):
logger.info("stop Cache instance %s" % instanceName)
if not stopCache(instanceName, useSudo=useSudo):
logger.info("force down Cache instance %s" % instanceName)
forceDownCache(instanceName, useSudo=useSudo)
if isInstanceRunning(instanceName):
logger.error("Can not stop cache instance %s" % instanceName)
return False
from GitUtils import getGitRepoRevisionHash
if gitReposBranch is None:
gitReposBranch = "HEAD"
backupHash = getGitRepoRevisionHash(gitReposBranch, gitReposDir)
destFile = os.path.join(backupDir, getCacheBackupNameByHash(backupHash))
if not os.path.exists(destFile):
logger.info("Creating tar file for %s" % origDataPath)
createBZIP2Tarball(origDataPath, destFile)
else:
logger.warn("%s already exists" % destFile)
return True
def backupCacheDataByGitHash(instanceName, origDataPath, backupDir,
gitReposDir, gitReposBranch, useSudo=False):
if not os.path.exists(origDataPath):
logger.error("%s does not exist" % origDataPath)
return False
if not os.path.exists(gitReposDir):
logger.error("%s does not exist" % gitReposDir)
return False
if not os.path.exists(backupDir):
logger.error("%s does not exists" % backupDir)
return False
if isInstanceRunning(instanceName):
logger.info("stop Cache instance %s" % instanceName)
if not stopCache(instanceName, useSudo=useSudo):
logger.info("force down Cache instance %s" % instanceName)
forceDownCache(instanceName, useSudo=useSudo)
if isInstanceRunning(instanceName):
logger.error("Can not stop cache instance %s" % instanceName)
return False
from GitUtils import getGitRepoRevisionHash
backupHash = getGitRepoRevisionHash(gitReposBranch, gitReposDir)
destFile = os.path.join(backupDir, getCacheBackupNameByHash(backupHash))
if not os.path.exists(destFile):
logger.info("Creating tar file for %s" % origDataPath)
createBZIP2Tarball(origDataPath, destFile)
else:
logger.warn("%s already exists" % destFile)
return True
def __buildPatchOrderDependencyByCSV__(self, orderCSV):
patchOrderList = self._patchOrderCSVDict[orderCSV]
if patchOrderList is None: return
""" some sanity check """
outOrderList = []
for patchOrder in patchOrderList:
installName = patchOrder[0]
if installName not in self._patchInfoDict:
if (installName not in self._informationalKidsSet
and installName not in self._kidsInstallNameDict):
logger.warn("No KIDS file found for %s" % str(patchOrder))
continue
patchInfo = self._patchInfoDict[installName]
""" check the seq no """
seqNo = patchOrder[1]
if len(seqNo) > 0:
""" check the seq no match the parsing result """
if patchInfo.seqNo is not None:
if int(seqNo) != int(patchInfo.seqNo):
logger.error(
"SeqNo mismatch for %s, from csv: %s, info %s" %
(installName, seqNo, patchInfo.seqNo))
else:
logger.info("Add seqNo %s for %s" % (seqNo, installName))
patchInfo.seqNo = seqNo
""" handle the case with multibuilds kids file """
if patchInfo.isMultiBuilds:
installList = patchInfo.multiBuildsList
index = installList.index(installName)
if index > 0:
for idx in range(0, index):
prevInstallName = installList[idx]
prevPatchInfo = self._patchInfoDict[prevInstallName]
if prevInstallName in self._notInstalledKidsSet:
logger.error("%s is not installed by FOIA" %
prevInstallName)
if prevInstallName not in outOrderList:
logger.debug("Adding %s as part of multibuilds" %
prevInstallName)
outOrderList.append(prevInstallName)
else:
logger.debug("%s is already in the list" %
prevInstallName)
outOrderList.append(installName)
for idx in range(len(outOrderList) - 1, 0, -1):
installName = outOrderList[idx]
prevInstallName = outOrderList[idx - 1]
self._csvDepDict[installName] = prevInstallName
def setPatchInfoFromInstallName(installName, patchInfo):
(namespace, ver, patch) = extractInfoFromInstallName(installName)
if patchInfo.namespace:
if patchInfo.namespace != namespace:
logger.error("Namespace mismatch. Expected %s. Found %s" %
(namespace, patchInfo.namespace))
else:
patchInfo.namespace = namespace
if patchInfo.version:
if (float(patchInfo.version) != float(ver)):
logger.error("Version mismatch. Expected %s. Found %s" %
(ver, patchInfo.version))
else:
patchInfo.version = ver
patchInfo.patchNo = patch
def generatePatchOrderTopologic(self, patchDir, installName=None):
self.analyzeVistAPatchDir(patchDir)
if installName:
if installName not in self._patchInfoDict:
raise Exception("Could not find patch for %s" % installName)
self.__updatePatchDependency__((installName is None))
self.__generatePatchDependencyGraph__()
try:
self.__topologicSort__(installName)
except CycleException as e:
errorMessage = "Failed to sort patches: %s" % e
logger.error(errorMessage)
return []
logger.info("After topologic sort %d" % len(self._patchOrder))
return self._patchOrder
def __updatePatchInfoPackageName__(self, patchInfo):
patchHistInfo = self._vistaPatchInfo
packageName = patchInfo.package
namespace = patchInfo.namespace
verInfo = patchInfo.version
if namespace:
patchHistInfo.getPackagePatchHistByNamespace(namespace, verInfo)
if packageName:
vistAPackageName = patchHistInfo.getPackageName(namespace)
if vistAPackageName != packageName:
logger.error("Different PackageName for %s, %s:%s" %
(patchInfo.installName, vistAPackageName, packageName))
patchInfo.package = vistAPackageName
else:
packageName = patchHistInfo.getPackageName(namespace)
patchInfo.package = packageName