def restoreCacheData(instanceName, bzipDataFile, cacheDataDir, useSudo=False): if not os.path.exists(bzipDataFile): logger.error("%s does not exist" % bzipDataFile) return False if not os.path.exists(cacheDataDir): logger.error("%s does not exist" % cacheDataDir) return False src = os.path.join(cacheDataDir, CACHE_DATA_FILE) if not os.path.exists(src): logger.error("%s does not exist" % src) return False if isInstanceRunning(instanceName): logger.info("stop Cache instance %s" % instanceName) if not stopCache(instanceName, useSudo=useSudo): logger.info("force down Cache instance %s" % instanceName) forceDownCache(instanceName, useSudo=useSudo) if isInstanceRunning(instanceName): logger.error("Can not stop cache instance %s" % instanceName) return False try: tmpBackup = src + ".bak" logger.info("rename %s to %s" % (src, tmpBackup)) os.rename(src, tmpBackup) logger.info("extract %s to %s" % (bzipDataFile, cacheDataDir)) extractBZIP2Tarball(bzipDataFile, cacheDataDir) logger.info("remove backup %s" % tmpBackup) os.unlink(tmpBackup) return True except OSError as oe: logger.error(os) return False
def __extractRoutines__(self, vistATestClient): # do not export ZGO, ZGI and xobw.* routines for now excludeList = ROUTINE_EXTRACT_EXCLUDE_LIST exceptionList = ROUTINE_EXTRACT_EXCEPTION_LIST vistARoutineExport = VistARoutineExport() logger.info("Extracting All Routines from VistA instance to %s" % self._routineOutputFile) vistARoutineExport.exportAllRoutines(vistATestClient, self._routineOutputFile, excludeList, exceptionList)
def forceDownCache(instanceName, useSudo=False): callList = ["ccontrol", "force", instanceName, "quietly"] if useSudo: callList.insert(0, "sudo") logger.info("Force down Cache instance %s" % instanceName) rcode = subprocess.call(callList) return rcode == 0
def addPackagePatchHistory(packageName, version, seqNo, patchNo, vistAClient, inputDuz): logger.info("Adding %s, %s, %s, %s to Package Patch history" % (packageName, version, seqNo, patchNo)) connection = vistAClient.getConnection() menuUtil = VistAMenuUtil(duz=1) menuUtil.gotoFileManEditEnterEntryMenu(vistAClient) connection.send("9.4\r") # package file connection.expect("EDIT WHICH FIELD: ") connection.send("VERSION\r") connection.expect("EDIT WHICH VERSION SUB-FIELD: ") connection.send("PATCH APPLICATION HISTORY\r") connection.expect("EDIT WHICH PATCH APPLICATION HISTORY SUB-FIELD: ") connection.send("ALL\r") connection.expect("THEN EDIT VERSION SUB-FIELD: ") connection.send("\r") connection.expect("THEN EDIT FIELD: ") connection.send("\r") connection.expect("Select PACKAGE NAME: ") connection.send("%s\r" % packageName) connection.expect("Select VERSION: %s//" % version) connection.send("\r") connection.expect("Select PATCH APPLICATION HISTORY: ") connection.send("%s SEQ #%s\r" % (patchNo, seqNo)) connection.expect("Are you adding .*\? No//") connection.send("YES\r") connection.expect("DATE APPLIED: ") connection.send("T\r") connection.expect("APPLIED BY: ") connection.send("`%s\r" % inputDuz) connection.expect("DESCRIPTION:") connection.send("\r") connection.expect("Select PATCH APPLICATION HISTORY: ") connection.send("\r") connection.expect("Select PACKAGE NAME: ") connection.send("\r") menuUtil.exitFileManMenu(vistAClient)
def __parseAllKIDSBuildFilesList__(self): kidsParser = KIDSPatchInfoParser() for basename in self._kidsBuildFileDict.iterkeys(): kidsFile, sha1Path = self._kidsBuildFileDict[basename] if kidsFile == None: logger.error("No KIDS file available for name %s" % basename) installNameList,seqNo = kidsParser.getKIDSBuildInstallNameSeqNo(kidsFile) if len(installNameList) > 1: if not self._multiBuildDict.get(kidsFile): self._multiBuildDict[kidsFile] = installNameList else: assert self._multiBuildDict[kidsFile] == installNameList elif seqNo: if installNameList[0] not in self._installNameSeqMap: self._installNameSeqMap[installNameList[0]] = seqNo else: logger.error("Duplicated KIDS build file %s" % kidsFile) for installName in installNameList: if installName in self._kidsInstallNameDict: logger.warn("%s is already in the dict %s" % (installName, kidsFile)) logger.debug("Added installName %s, file %s" % (installName, kidsFile)) self._kidsInstallNameDict[installName] = os.path.normpath(kidsFile) """ handle KIDS sha1 file Path """ if sha1Path: if installName in self._kidsInstallNameSha1Dict: logger.warn("%s is already in the dict %s" % (installName, sha1Path)) self._kidsInstallNameSha1Dict[installName] = sha1Path logger.debug("%s" % sorted(self._kidsInstallNameDict.keys())) logger.info("Total # of install name %d" % len(self._kidsInstallNameDict))
def getAllPackagesPatchHistory(self): self.createAllPackageMapping() self._packagePatchHist.clear() for (namespace, package) in self._packageMapping.iteritems(): logger.info("Parsing Package %s, namespace %s" % (package, namespace)) #if not (package[0] == "PHARMACY" and package[1] == "PS"): continue self.getPackagePatchHistory(package, namespace)
def normalInstallation(self, vistATestClient, vistATestClient2=None, reinst=True): logger.info("Start installing %s" % self._kidsInstallName) connection = vistATestClient.getConnection() if vistATestClient2: connection2 = vistATestClient2.getConnection() self.__gotoKIDSMainMenu__(vistATestClient) self.__loadKIDSBuild__(connection) result = self.__handleKIDSLoadOptions__(connection, reinst) if not result: logger.error("Error handling KIDS Load Options %s, %s" % (self._kidsInstallName, self._kidsFile)) return False if self._tgOutputDir: if self._multiBuildList is None: self.__printTransportGlobal__(vistATestClient,[self._kidsInstallName],self._tgOutputDir) else: self.__printTransportGlobal__(vistATestClient,self._multiBuildList,self._tgOutputDir) if vistATestClient2: result = self.__handleKIDSInstallQuestions__(connection, connection2) else: result = self.__handleKIDSInstallQuestions__(connection) if not result: result = self.unloadDistribution(vistATestClient, False) if not result: logger.error("Unload %s failed" % self._kidsInstallName) return False return self.normalInstallation(vistATestClient, vistATestClient2, reinst) self.__installationCommon__(vistATestClient) return True
def main(): parser = argparse.ArgumentParser(description='VistA KIDS Patch Parser') parser.add_argument('-i', '--inputFile', required=True, help = 'Input KIDS Patch file or Invidual Routine File') parser.add_argument('-o', '--outputDir', default=None, help = 'Output directory to store extracted M routines') parser.add_argument('-c', '--checksum', default=False, action="store_true", help = 'Print checksum of a M Routine') parser.add_argument('-v', '--verify', default=False, action="store_true", help = 'verify a KIDS patch') result = parser.parse_args(); import logging initConsoleLogging(logging.INFO) if result.checksum: chksum= checksum(result.inputFile) sys.stdout.write("Checksum is: %s\n" % chksum) elif result.verify: kidsParser = KIDSPatchParser(result.outputDir) ret = kidsParser.verifyKidsPatch(result.inputFile) if ret: logger.info("%s is valid" % result.inputFile) else: logger.error("%s is invalid" % result.inputFile) else: kidsParser = KIDSPatchParser(result.outputDir) kidsParser.parseKIDSBuild(result.inputFile) kidsParser.printResult()
def parseKIDSBuild(self, kidsPatch): assert os.path.exists(kidsPatch) logger.info("Parsing KIDS file %s" % kidsPatch) lines = None with open(kidsPatch, 'r') as input: curLine = input.readline() lineNum = 0 while len(curLine) > 0: lineNum += 1 """ read one more line """ lines = (curLine.rstrip('\r\n'), input.readline().rstrip('\r\n')) if lineNum == 1: self.__parseKIDSHeader__(lines) else: section, parser = self.__isSectionLine__(curLine) if section == None: # could not find a valid section logger.warn("Can not parse %s" % lines[0]) self.__resetCurrentSection__(section, parser, lines) else: # find a section if section != self._curSection: self.__resetCurrentSection__(section, parser, lines) elif self._curParser: self._curParser.parseLines(section, lines, kidsBuild=self._curKidsBuild) # goto the next line # lineNum += 1 curLine = input.readline() else: # at end of file self.__resetCurrentSection__(None, None, lines)
def __init__(self, kidsFile, kidsInstallName, seqNo=None, logFile=None, multiBuildList = None, duz = DEFAULT_INSTALL_DUZ, **kargs): assert os.path.exists(kidsFile), ("kids file does not exist %s" % kidsFile) self._origKidsFile = kidsFile if len(kidsFile) >= self.KIDS_FILE_PATH_MAX_LEN: destFilename = os.path.basename(kidsFile) tempDir = tempfile.gettempdir() if isValidExternalDataFileName(kidsFile): # if read directly from inplace, need to replace the name with hash destFilename = getSha1HashFromExternalDataFileName(kidsFile) while (len(tempDir)+len(destFilename)+1) >= self.KIDS_FILE_PATH_MAX_LEN: tempDir = os.path.split(tempDir)[0] dest = os.path.join(tempDir, destFilename) shutil.copy(kidsFile, dest) self._kidsFile = os.path.normpath(dest) logger.info("new kids file is %s" % self._kidsFile) else: self._kidsFile = os.path.normpath(kidsFile) self._kidsInstallName = kidsInstallName self._logFile = logFile self._duz = duz self._updatePackageLink = False self._multiBuildList = multiBuildList # store all the globals files associated with KIDS" self._globalFiles = None if "globals" in kargs: self._globalFiles = kargs['globals'] self._tgOutputDir = None if "printTG" in kargs: self._tgOutputDir = kargs['printTG']
def hasPatchInstalled(self, installName, namespace, version, patchNo, seqNo = None): if namespace not in self._packageMapping: logger.info("this is a new namespace %s" % namespace) return False packageName = self._packageMapping[namespace] if packageName not in self._packagePatchHist: patchHist = self.getPackagePatchHistByNamespace(namespace, version) else: patchHist = self._getPackageHistListByVer(packageName, version) if not patchHist: patchHist = self.getPackagePatchHistByNamespace(namespace, version) if patchHist: if seqNo: if patchHist.hasSeqNo(seqNo): return True if patchHist.version: try: if float(patchHist.version) != float(version): logger.info("Diff ver %s, %s" % (patchHist.version, version)) except Exception as ex: logger.error(ex) if patchNo: return patchHist.hasPatchNo(patchNo) status = self.getInstallationStatus(installName) return self.isInstallCompleted(status)
def __copyAllGlobals__(self): logger.info("Copying all files from %s to %s" % (self._globalOutputDir, self._packagesDir)) zwrFiles = glob.glob(os.path.join(self._globalOutputDir, "*.zwr")) for zwrFile in zwrFiles: logger.debug("Copying %s to %s" % (zwrFile, self._packagesDir)) shutil.copy2(zwrFile, self._packagesDir)
def __updateMultiBuildDependencies__(self): patchList = self._patchInfoDict for installList in self._multiBuildDict.itervalues(): logger.info("Multi-Buids KIDS install List: %s" % (installList)) firstPatch = patchList[installList[0]] firstPatch.otherKidsInfoList = [] if firstPatch.csvDepPatch is None: """ If primary build install name is not specified in the csv file will fall back to use dependency specified in the first secondary build """ secondPatch = patchList[installList[1]] if secondPatch.csvDepPatch != firstPatch.installName: logger.info("Assign first patch CSV Dep %s" % firstPatch.installName) firstPatch.csvDepPatch = secondPatch.csvDepPatch for index in range(1,len(installList)): nextPatchInfo = patchList[installList[index]] """ just to make sure the first one has all the dependencies """ firstPatch.depKIDSBuild.update(nextPatchInfo.depKIDSBuild) firstPatch.optionalDepSet.update(nextPatchInfo.optionalDepSet) firstPatch.otherKidsInfoList.append([nextPatchInfo.kidsInfoPath, nextPatchInfo.kidsInfoSha1]) prevInstallName = installList[index - 1] if prevInstallName not in nextPatchInfo.depKIDSBuild: nextPatchInfo.depKIDSBuild.add(prevInstallName) #del patchList[installList[index]] #remove the other patch from the list logger.debug("%s:%s" % (nextPatchInfo.installName, nextPatchInfo.depKIDSBuild)) """ remove the self dependencies of the first patch """ firstPatch.depKIDSBuild.difference_update(installList) logger.debug("%s:%s" % (firstPatch.installName, firstPatch.depKIDSBuild))
def __isPatchReadyToInstall__(self, patchInfo, patchList = None): packageName = patchInfo.package ver = patchInfo.version patchHist = self._vistaPatchInfo.getPackagePatchHistByName(packageName, ver) if not patchHist or not patchHist.hasPatchHistory(): logger.info("no patch hist for %s, ver: %s" % (packageName, ver)) return True # if no such an package or hist info, just return True """ check patch sequence no to see if it is out of order """ if patchInfo.seqNo: seqNo = patchHist.getLatestSeqNo() if patchInfo.seqNo < seqNo: logger.error("SeqNo out of order, %s less than latest one %s" % (patchInfo.seqNo), seqNo) return False # check all the dependencies for item in patchInfo.depKIDSPatch: if patchList and item in self._patchSet: # we are going to install the dep patch logger.info("We are going to install the patch %s" % item) """ make sure installation is in the right order """ itemIndex = self.indexInPatchList(item, patchList) patchIndex = self.indexInPatchList(patchInfo.installName, patchList) if itemIndex >= patchIndex: logger.warn("%s is out of order with %s" % (item, patchInfo)) return False else: continue (namespace,ver,patch) = extractInfoFromInstallName(item) if self._vistaPatchInfo.hasPatchInstalled(item, namespace, ver, patch): logger.debug("%s is arelady installed" % item) continue else: logger.error("dep %s is not installed for %s %s" % (item, patchInfo.installName, patchInfo.kidsFilePath)) return False return True
def getFileMan22_2RoutineFile(rFile): sha1sum = readSha1SumFromSha1File(rFile) logger.info("sha1sum is %s" % sha1sum) result, path = obtainKIDSBuildFileBySha1(rFile, sha1sum, filedir) if not result: logger.error("Could not obtain FileMan 22V2 file for %s" % rFile) raise Exception("Error getting FileMan 22V2 file for %s" % rFile) return path
def stopHL7BackgroundFiler(self, vistAClient): connection = vistAClient.getConnection() connection.send("D LLP^HLCS2(1)\n") vistAClient.waitForPrompt() connection.send("D STOPLM^HLCSLM\n") vistAClient.waitForPrompt() logger.info("Wait 30 seconds for HL7 background filer to stop") time.sleep(30)
def stopMailManBackgroundFiler(self, vistAClient): connection = vistAClient.getConnection() connection.send("D STOP^XMKPL\n") connection.expect("Are you sure you want the Background Filers to stop delivering mail\? ") connection.send("YES\r") vistAClient.waitForPrompt() logger.info("Wait 30 seconds for Mailman background filer to stop") time.sleep(30)
def generatePatchOrderTopologic(self, patchDir): self.analyzeVistAPatchDir(patchDir) self.__updatePatchDependency__() self.__generatePatchDependencyGraph__() self.__topologicSort__() logger.info("After topologic sort %d" % len(self._patchOrder)) self.printPatchOrderList() return self._patchOrder
def applyPatchSequenceByInstallName(self, installName, patchOnly=False): if installName not in self._patchSet: raise Exception("Invalid install name: %s" % installName) if len(self._outPatchList) == 0: logger.info("No Patch to apply") return logger.info("Apply patches up to %s" % installName) self.__applyPatchUptoIndex__(len(self._outPatchList))
def __exportAllGlobals__(self, vistATestClient): """ remove all the zwr files first """ logger.info("Remove all zwr files under %s" % self._globalOutputDir) for file in glob.glob(os.path.join(self._globalOutputDir, "*.zwr")): os.remove(file) logger.info("Exporting all globals from VistA instance") vistAGlobalExport = VistAGlobalExport() vistAGlobalExport.exportAllGlobals(vistATestClient, self._globalOutputDir, self._serialExport)
def removeTaskmanFromWait(self, vistAClient): connection = vistAClient.getConnection() menuUtil = VistAMenuUtil(duz=1) menuUtil.gotoTaskmanMgrUtilMenu(vistAClient) connection.send("Remove Taskman from WAIT State\r") menuUtil.exitTaskmanMgrUtilMenu(vistAClient) logger.info("Wait 10 seconds for Taskman to start") time.sleep(10)
def startCache(instanceName, useSudo=False): if isInstanceRunning(instanceName): return True callList = ["ccontrol", "start", instanceName] if useSudo: callList.insert(0, "sudo") logger.info("Start Cache instance %s" % instanceName) rcode = subprocess.call(callList) return rcode == 0
def __copyAllClasses__(self): if not self._cache: return logger.info("Copying all classes file from %s to %s" % (self._cacheclassesDir, self._packagesDir)) xmlFiles = glob.glob(os.path.join(self._cacheclassesDir, "*.xml")) for xmlFile in xmlFiles: logger.debug("Copying %s to %s" % (xmlFile, self._packagesDir)) shutil.copy2(xmlFile, self._packagesDir)
def __exportAllCacheClasses__(self, vistATestClient): if not vistATestClient.isCache(): """ only works for Intersystem Cache """ return """ remove all the xml files first """ logger.info("Remove all xml files under %s" % self._cacheclassesDir) for file in glob.glob(os.path.join(self._globalOutputDir, "*.xml")): os.remove(file) exportAllClassesIndividual(vistATestClient, self._cacheclassesDir)
def __getAllKIDSBuildInfoAndOtherFileList__(self, patchDir): assert os.path.exists(patchDir) absPatchDir = os.path.abspath(patchDir) for (root, dirs, files) in os.walk(absPatchDir): lastDir = os.path.split(root)[-1] for fileName in files: absFilename = os.path.join(root, fileName) if not isValidPatchRelatedFiles(absFilename, True): continue """ Handle KIDS build files """ if isValidKIDSPatchSuffix(fileName): logger.debug("Adding %s KIDS file to dict" % absFilename) self.__addKidsBuildFileToDict__(fileName, absFilename, KIDS_BUILD_FILE_TYPE_KIDS) continue """ Handle KIDS build HEADER files """ if isValidKIDSPatchHeaderSuffix(fileName): logger.debug("Adding %s KIDS header to dict" % absFilename) kidsFileName = fileName[0:fileName.rfind('.')] self.__addKidsBuildFileToDict__(kidsFileName, absFilename, KIDS_BUILD_FILE_TYPE_HEADER) continue """ Handle KIDS build Sha1 files """ if isValidKIDSPatchSha1Suffix(fileName): logger.debug("Adding %s KIDS info to dict" % absFilename) kidsFileName = fileName[0:fileName.rfind('.')] self.__addKidsBuildFileToDict__(kidsFileName, absFilename, KIDS_BUILD_FILE_TYPE_SHA1) continue """ Handle KIDS Info/Sha1 files """ if ( isValidKIDSInfoSuffix(fileName) or isValidKIDSInfoSha1Suffix(fileName) ): self._kidsInfoFileList.append(absFilename) continue """ Handle Global/Sha1 Files """ if ( isValidGlobalFileSuffix(fileName) or isValidGlobalSha1Suffix(fileName) ): logger.debug("Adding %s Global files to list" % absFilename) self._globalFilesSet.add(absFilename) continue """ handle all csv files """ if isValidCSVSuffix(fileName): if isValidOrderCSVFile(absFilename): self._csvOrderFileList.append(absFilename) continue """ Handle .py files """ if isValidPythonSuffix(fileName): logger.debug("Adding %s python script to list" % absFilename) self._pythonScriptList.append(absFilename) continue logger.info("Total # of KIDS Builds are %d" % len(self._kidsBuildFileDict)) logger.info("Total # of KIDS Info are %d" % len(self._kidsInfoFileList)) logger.info("Total # of Global files are %d" % len(self._globalFilesSet)) logger.info("Total # of Python files are %d" % len(self._pythonScriptList)) logger.info("Total # of CSV files are %d" % len(self._csvOrderFileList))
def stopHL7BackgroundFiler(self, vistAClient): connection = vistAClient.getConnection() menuUtil = VistAMenuUtil(duz=1) menuUtil.gotoHL7FilerLinkMgrMenu(vistAClient) connection.send("Stop All Messaging Background Processes\r") connection.expect("Okay to shut down all Links and Filers\? ") connection.send("Yes\r") menuUtil.exitHL7FilerLinkMgrMenu(vistAClient) logger.info("Wait 30 seconds for HL7 backgroud filer to stop") time.sleep(30)
def __removePackagesTree__(self): logger.info("Removing all files under %s" % self._packagesDir) for dirEntry in os.listdir(self._packagesDir): if dirEntry == ".gitattributes": # ignore the .gitattributes continue fullPath = os.path.join(self._packagesDir, dirEntry) if os.path.isdir(fullPath): shutil.rmtree(fullPath) else: os.remove(fullPath)
def generatePatchOrderTopologic(self, patchDir, installName=None): self.analyzeVistAPatchDir(patchDir) if installName: if installName not in self._patchInfoDict: raise Exception("Could not find patch for %s" % installName) self.__updatePatchDependency__((installName is None)) self.__generatePatchDependencyGraph__() self.__topologicSort__(installName) logger.info("After topologic sort %d" % len(self._patchOrder)) return self._patchOrder
def __importZGORoutine__(self, vistATestClient): logger.info("Import ZGO routine to VistA instance") from PackRO import pack zgoOutFile = os.path.join(self._outputResultDir, "ZGO.ro") zgoRoutine = os.path.join(SCRIPTS_DIR, "ZGO.m") assert os.path.exists(zgoRoutine) pack([zgoRoutine], open(zgoOutFile, 'w')) vistARoutineImport = VistARoutineImport() vistARoutineImport.importRoutines(vistATestClient, zgoOutFile, self._routineOutDir)
def stopMailManBackgroundFiler(self, vistAClient): connection = vistAClient.getConnection() menuUtil = VistAMenuUtil(duz=1) menuUtil.gotoMailmanLocalDeliveryMgrMenu(vistAClient) connection.send("STOP background filer\r") connection.expect("Are you sure you want the Background Filers to stop delivering mail\? ") connection.send("YES\r") menuUtil.exitMailmanLocalDeliveryMgrMenu(vistAClient) logger.info("Wait 30 seconds for Mailman backgroud filer to stop") time.sleep(30)
def stopMailManBackgroundFiler(self, vistAClient): connection = vistAClient.getConnection() connection.send("D STOP^XMKPL\n") connection.expect( "Are you sure you want the Background Filers to stop delivering mail\? " ) connection.send("YES\r") vistAClient.waitForPrompt() logger.info("Wait 30 seconds for Mailman background filer to stop") time.sleep(30)
def __parseAllKIDSBuildFilesList__(self): for basename in self._kidsBuildFileDict.iterkeys(): kidsFile, sha1Path = self._kidsBuildFileDict[basename] if kidsFile == None: logger.error("No KIDS file available for name %s" % basename) continue installNameList, seqNo, kidsBuilds = None, None, None if isValidKIDSBuildHeaderSuffix(kidsFile): from KIDSBuildParser import loadMetaDataFromJSON #continue installNameList, seqNo, kidsBuilds = loadMetaDataFromJSON(kidsFile) else: kidsParser = KIDSBuildParser(None) kidsParser.unregisterSectionHandler(KIDSBuildParser.ROUTINE_SECTION) kidsParser.parseKIDSBuild(kidsFile) installNameList = kidsParser.installNameList logger.debug("install name list is %s" % installNameList) seqNo = kidsParser.seqNo kidsBuilds = kidsParser.kidsBuilds if len(installNameList) > 1: if not self._multiBuildDict.get(kidsFile): self._multiBuildDict[kidsFile] = installNameList else: assert self._multiBuildDict[kidsFile] == installNameList elif seqNo: if installNameList[0] not in self._installNameSeqMap: self._installNameSeqMap[installNameList[0]] = seqNo else: logger.error("Duplicated KIDS build file %s" % kidsFile) for installName in installNameList: if installName in self._kidsInstallNameDict: logger.warn("%s is already in the dict %s" % (installName, kidsFile)) logger.debug("Added installName %s, file %s" % (installName, kidsFile)) self._kidsInstallNameDict[installName] = os.path.normpath(kidsFile) """ handle KIDS sha1 file Path """ if sha1Path: if installName in self._kidsInstallNameSha1Dict: logger.warn("%s is already in the dict %s" % (installName, sha1Path)) self._kidsInstallNameSha1Dict[installName] = sha1Path """ update kids dependency """ if installName in self._kidsDepBuildDict: logger.warn("%s already has the dep map %s" % (installName, self._kidsDepBuildDict[installName])) if kidsBuilds: for kidsBuild in kidsBuilds: if kidsBuild.installName == installName: depList = kidsBuild.dependencyList if depList: self._kidsDepBuildDict[installName] = set([x[0] for x in depList]) logger.info("%s: %s" % (installName, self._kidsDepBuildDict[installName])) logger.debug("%s" % sorted(self._kidsInstallNameDict.keys())) logger.info("Total # of install name %d" % len(self._kidsInstallNameDict))
def convertKIDSBuildFile(self, kidsFile): from KIDSBuildParser import KIDSBuildParser, outputMetaDataInJSON assert os.path.exists(kidsFile) """ write KIDS metadata file """ kidsParser = KIDSBuildParser(None) """ do not parse the routine part """ kidsParser.unregisterSectionHandler(KIDSBuildParser.ROUTINE_SECTION) kidsParser.parseKIDSBuild(kidsFile) logger.info("output meta data as %s" % (kidsFile + ".json")) outputMetaDataInJSON(kidsParser, kidsFile + ".json") self.convertToSha1File(kidsFile)
def _removeNotInstalledKIDSBuild(self, installName): patchInfo = self._patchInfoDict.get(installName) if not patchInfo: return listToRemove = [installName] if patchInfo.isMultiBuilds: listToRemove = patchInfo.multiBuildsList self._multiBuildDict.pop(patchInfo.kidsFilePath, None) for install in listToRemove: logger.info("Removing %s" % install) self._kidsInstallNameDict.pop(install, None) self._patchInfoDict.pop(install, None)
def stopCache(instanceName, restart=False, useSudo=False): if not isInstanceRunning(instanceName): return True callList = ["ccontrol", "stop", instanceName, "quietly"] if restart: callList.append("restart") if useSudo: callList.insert(0, "sudo") # prepend sudo command logger.info("Stop Cache instance %s" % instanceName) rcode = subprocess.call(callList) return rcode == 0
def unpackRoutines(self, routinesOutputFile, outputDir): from UnpackRO import unpack assert os.path.exists(routinesOutputFile) assert os.path.exists(outputDir) absOutDir = os.path.abspath(outputDir) logfile = os.path.join(self._outputLogDir, "unpackro.log") logger.info("Unpack routines from %s to %s" % (routinesOutputFile, outputDir)) with open(routinesOutputFile, 'r') as routineFile: # open as txt with open(logfile, 'w') as logFile: unpack(routineFile, out=logFile, odir=outputDir)
def generatePatchOrderTopologic(self, patchDir, installName=None): self.analyzeVistAPatchDir(patchDir) if installName: if installName not in self._patchInfoDict: raise Exception("Could not find patch for %s" % installName) self.__updatePatchDependency__((installName is None)) self.__generatePatchDependencyGraph__() self.__topologicSort__(installName) logger.info("After topologic sort %d" % len(self._patchOrder)) self.printPatchOrderList() return self._patchOrder
def stopMailManBackgroundFiler(self, vistAClient): connection = vistAClient.getConnection() menuUtil = VistAMenuUtil(duz=1) menuUtil.gotoMailmanLocalDeliveryMgrMenu(vistAClient) connection.send("STOP background filer\r") connection.expect( "Are you sure you want the Background Filers to stop delivering mail\? " ) connection.send("YES\r") menuUtil.exitMailmanLocalDeliveryMgrMenu(vistAClient) logger.info("Wait 30 seconds for Mailman backgroud filer to stop") time.sleep(30)
def __parseAllKIDSInfoFilesList__(self): kidsParser = PatchInfoParser() for kidsInfoFile in self._kidsInfoFileList: patchInfo = kidsParser.parseKIDSInfoFile(kidsInfoFile) if not patchInfo: logger.debug("invalid kids info file %s" % kidsInfoFile) self._invalidInfoFileSet.add(kidsInfoFile) continue """ only add to list for info that is related to a Patch""" installName = patchInfo.installName if installName not in self._kidsInstallNameDict: logger.warn("no KIDS file related to %s (%s)" % (installName, kidsInfoFile)) if installName in self._missKidsBuildDict: logger.warn("duplicated kids install name") if kidsInfoFile != self._missKidsBuildDict[ installName].kidsInfoPath: logger.warn("duplicated kids info file name %s" % kidsInfoFile) continue self._missKidsBuildDict[installName] = patchInfo continue patchInfo.kidsFilePath = self._kidsInstallNameDict[installName] assert patchInfo.kidsFilePath """ update PatchInfo kidsSha1 and kidsSha1Path """ if installName in self._kidsInstallNameSha1Dict: sha1Path = self._kidsInstallNameSha1Dict[installName] patchInfo.kidsSha1Path = sha1Path patchInfo.kidsSha1 = readSha1SumFromSha1File(sha1Path) if installName in self._patchInfoDict: logger.warn("duplicated installName %s, %s, %s" % (installName, self._patchInfoDict[installName], kidsInfoFile)) """ merge the dependency if needed, also put extra dependency into optional set """ if installName in self._kidsDepBuildDict: infoDepSet = set() kidsDepSet = set() if patchInfo.depKIDSBuild: infoDepSet = patchInfo.depKIDSBuild if self._kidsDepBuildDict[installName]: kidsDepSet = self._kidsDepBuildDict[installName] diffSet = kidsDepSet ^ infoDepSet if len(diffSet): logger.info("Merging kids dependencies %s" % installName) logger.debug("kids build set is %s" % kidsDepSet) logger.debug("info build set is %s" % infoDepSet) logger.warning("difference set: %s" % diffSet) patchInfo.depKIDSBuild = infoDepSet | kidsDepSet patchInfo.optionalDepSet = infoDepSet - kidsDepSet else: patchInfo.depKIDSBuild = infoDepSet self._patchInfoDict[installName] = patchInfo
def switchBranch(branchName, gitRepoDir=None): """ Utility function to switch to a different branch @branchName: the name of the branch to switch to @gitRepoDir: git repository directory, default is current directory. if provided, will only add all changes under that directory @return: return True if success, False otherwise """ git_command_list = ["git", "checkout", branchName] result, output = _runGitCommand(git_command_list, gitRepoDir) logger.info(output) return result
def __installPatch__(self, patchInfo): installName = patchInfo.installName kidsPath = patchInfo.kidsFilePath seqNo = patchInfo.seqNo logFileName = self._logFileName multiBuildsList = patchInfo.multiBuildsList kidsInstaller = None """ handle patch stored as external link """ if patchInfo.kidsSha1Path != None: kidsSha1 = patchInfo.kidsSha1 (result, resultPath) = obtainKIDSBuildFileBySha1(kidsPath, kidsSha1, DEFAULT_CACHE_DIR) if not result: logger.error("Could not obtain external Patch for %s" % kidsPath) return result kidsPath = resultPath # set the KIDS Path """ get the right KIDS installer """ associateFiles = patchInfo.associatedInfoFiles associatedGlobals = patchInfo.associatedGlobalFiles if patchInfo.hasCustomInstaller: """ use python imp module to load the source """ logger.info("using custom installer %s" % patchInfo.customInstallerPath) import imp installerModule = imp.load_source("KIDS", patchInfo.customInstallerPath) from KIDS import CustomInstaller kidsInstaller = CustomInstaller(kidsPath, installName, seqNo, logFileName, multiBuildsList, files=associateFiles, globals=associatedGlobals, duz=self._duz) else: kidsInstaller = KIDSInstallerFactory.createKIDSInstaller( kidsPath, installName, seqNo, logFileName, multiBuildsList, files=associateFiles, globals=associatedGlobals, duz=self._duz) logger.info("Applying Patch %s" % patchInfo) assert kidsInstaller return kidsInstaller.runInstallation(self._testClient, self._testClient2)
def placeTaskmanToWait(self, vistAClient, shutdownSubMgrs=True): connection = vistAClient.getConnection() menuUtil = VistAMenuUtil(duz=1) menuUtil.gotoTaskmanMgrUtilMenu(vistAClient) connection.send("Place Taskman in a WAIT State\r") connection.expect("Should active submanagers shut down after finishing their current tasks\? ") if shutdownSubMgrs: connection.send("YES\r") else: connection.send("NO\r") menuUtil.exitTaskmanMgrUtilMenu(vistAClient) logger.info("Wait 10 seconds for Taskman to wait state") time.sleep(10)
def commitChange(commitMsgFile, gitRepoDir=None): """ Utility function to commit the change in the current branch @commitMsgFile: input commit message file for commit @gitRepoDir: git repository directory, default is current directory. @return: return True if success, False otherwise """ if not os.path.exists(commitMsgFile): return False git_command_list = ["git", "commit", "-F", commitMsgFile] result, output = _runGitCommand(git_command_list, gitRepoDir) logger.info(output) return result
def __isPatchReadyToInstall__(self, patchInfo, patchList=None): packageName = patchInfo.package ver = patchInfo.version patchHist = self._vistaPatchInfo.getPackagePatchHistByName( packageName, ver) if not patchHist or not patchHist.hasPatchHistory(): logger.info("no patch hist for %s, ver: %s" % (packageName, ver)) return True # if no such an package or hist info, just return True """ check patch sequence no to see if it is out of order """ if patchInfo.seqNo: seqNo = patchHist.getLatestSeqNo() if patchInfo.seqNo < seqNo: logger.error( "SeqNo out of order, %s less than latest one %s" % (patchInfo.seqNo), seqNo) return False # check all the dependencies for item in patchInfo.depKIDSBuild: if patchList and item in self._patchSet: # we are going to install the dep patch logger.info("We are going to install the patch %s" % item) """ make sure installation is in the right order """ itemIndex = self.indexInPatchList(item, patchList) patchIndex = self.indexInPatchList(patchInfo.installName, patchList) if itemIndex >= patchIndex: logger.warn("%s is out of order with %s" % (item, patchInfo)) return False else: continue (namespace, ver, patch) = extractInfoFromInstallName(item) if self._vistaPatchInfo.hasPatchInstalled(item, namespace, ver, patch): logger.debug("%s is arelady installed" % item) continue installStatus = self._vistaPatchInfo.getInstallationStatus(item) if self._vistaPatchInfo.isInstallCompleted(installStatus): continue elif item in patchInfo.optionalDepSet: logger.warn( "Patch specified in KIDS info file %s is not installed for %s" % (item, patchInfo.installName)) continue else: logger.error( "dep %s is not installed for %s %s" % (item, patchInfo.installName, patchInfo.kidsFilePath)) patchInfo.depKIDSBuild.remove(item) patchInfo.depKIDSBuild.add(item + " <---") return False return True
def __reportGitStatus__(self): logger.info("Reporting git status on Uncategorized dir") output = getStatus(self._packagesDir, 'Uncategorized/') logger.info(output) logger.info("Reporting git status on Packages dir") output = getStatus(self._packagesDir) logger.info(output)
def __updateCustomInstaller__(self): for pythonScript in self._pythonScriptList: installName = os.path.basename(pythonScript) installName = dirNameToInstallName(installName[:installName.rfind('.')]) if installName in self._patchInfoDict: patchInfo = self._patchInfoDict[installName] patchInfo.hasCustomInstaller = True if patchInfo.customInstallerPath: logger.warning("Duplicated installer for %s: [%s:%s]" % ( installName, patchInfo.customInstallerPath, pythonScript)) logger.info("%s: custom installer %s" % (pythonScript, installName)) self._patchInfoDict[installName].customInstallerPath = pythonScript
def __handlePatchAssociatedFiles__(self): """ handle the info files first """ """ first by name assiciation """ patchInfoList = self._patchInfoDict.values() #handle the associated files for missingKIDSBuild info patchInfoList.extend(self._missKidsBuildDict.values()) for patchInfo in patchInfoList: infoPath = patchInfo.kidsInfoPath if infoPath: infoName = os.path.basename(infoPath) associateSet = set() for infoFile in self._invalidInfoFileSet: infoFileName = os.path.basename(infoFile) if infoFileName.startswith(infoName[:infoName.rfind('.')]): patchInfo.addToAssociatedInfoList(infoFile) associateSet.add(infoFile) continue self._invalidInfoFileSet.difference_update(associateSet) """ second by mapping association """ associateSet = set() for infoFile in self._invalidInfoFileSet: installName = getAssociatedInstallName(infoFile) if installName: if installName in self._patchInfoDict: patchInfo = self._patchInfoDict[installName] #handle the associated files for missingKIDSBuild info elif installName in self._missKidsBuildDict: patchInfo = self._missKidsBuildDict[installName] else: continue patchInfo.addToAssociatedInfoList(infoFile) associateSet.add(infoFile) self._invalidInfoFileSet.difference_update(associateSet) """ handle global files """ associateSet = set() for globalFile in self._globalFilesSet: installName = getAssociatedInstallName(globalFile) if installName and installName in self._patchInfoDict: patchInfo = self._patchInfoDict[installName] patchInfo.addToAssociatedGlobalList(globalFile) associateSet.add(globalFile) self._globalFilesSet.difference_update(associateSet) logger.info("Total # of leftover info files: %s" % len(self._invalidInfoFileSet)) logger.debug(self._invalidInfoFileSet) logger.info("Total # of leftover global files: %s" % len(self._globalFilesSet)) logger.debug(self._globalFilesSet)
def preInstallationWork(self, vistATestClient, **kargs): """ ignore the multi-build patch for now """ if self._multiBuildList is not None: return globalFiles = self.__getGlobalFileList__() if globalFiles is None or len(globalFiles) == 0: return globalImport = VistAGlobalImport() for glbFile in globalFiles: logger.info("Import global file %s" % (glbFile)) fileSize = os.path.getsize(glbFile) importTimeout = DEFAULT_GLOBAL_IMPORT_TIMEOUT importTimeout += int(fileSize/GLOBAL_IMPORT_BYTE_PER_SEC) globalImport.importGlobal(vistATestClient, glbFile, timeout=importTimeout)
def __moveToExternalDir__(self, fileName, sha1Sum): assert os.path.exists(fileName) destFile = generateExternalDataFileName(sha1Sum) if self._externDir: destFile = os.path.join(self._externDir, destFile) else: destFile = os.path.join(os.path.dirname(fileName), destFile) if os.path.exists(destFile): if generateSha1Sum(destFile) == sha1Sum: os.remove(fileName) logger.info("%s already exists and is valid" % destFile) return os.remove(destFile) os.rename(fileName, destFile)
def obtainKIDSBuildFileBySha1(filePath, sha1Sum, cacheDir): assert cacheDir and os.path.exists(cacheDir) rootDir = os.path.dirname(filePath) externalFileName = generateExternalDataFileName(sha1Sum) externalFile = os.path.join(rootDir, externalFileName) logger.info("Checking %s" % externalFile) if os.path.exists(externalFile): if generateSha1Sum(externalFile) == sha1Sum: return (True, externalFile) else: os.remove(externalFile) """ try to find the file in the cache dir """ externalFile = os.path.join(cacheDir, externalFileName.replace('_','/')) logger.info("Checking %s" % externalFile) if os.path.exists(externalFile): if generateSha1Sum(externalFile) == sha1Sum: return (True, externalFile) else: os.remove(externalFile) """ make sure cacheDir has the right layout """ rootDir = os.path.dirname(externalFile) if not os.path.exists(rootDir): os.makedirs(rootDir) """ download from remote """ extDownloader = ExternalDataDownloader() logger.info("Downloading from remote link") result = extDownloader.downloadExternalDataByHash(sha1Sum, externalFile) if not result: logger.error("Downloading from remote failed") if os.path.exists(externalFile): os.remove(externalFile) externalFile = None logger.info("%s, %s" % (result, externalFile)) return (result, externalFile)
def __buildPatchOrderDependencyByCSV__(self, orderCSV): patchOrderList = self._patchOrderCSVDict[orderCSV] if patchOrderList is None: return """ some sanity check """ outOrderList = [] for patchOrder in patchOrderList: installName = patchOrder[0] if installName not in self._patchInfoDict: if (installName not in self._informationalKidsSet and installName not in self._kidsInstallNameDict): logger.warn("No KIDS file found for %s" % str(patchOrder)) continue patchInfo = self._patchInfoDict[installName] """ check the seq no """ seqNo = patchOrder[1] if len(seqNo) > 0: """ check the seq no match the parsing result """ if patchInfo.seqNo is not None: if int(seqNo) != int(patchInfo.seqNo): logger.error( "SeqNo mismatch for %s, from csv: %s, info %s" % (installName, seqNo, patchInfo.seqNo)) else: logger.info("Add seqNo %s for %s" % (seqNo, installName)) patchInfo.seqNo = seqNo """ handle the case with multibuilds kids file """ if patchInfo.isMultiBuilds: installList = patchInfo.multiBuildsList index = installList.index(installName) if index > 0: for idx in range(0, index): prevInstallName = installList[idx] prevPatchInfo = self._patchInfoDict[prevInstallName] if prevInstallName in self._notInstalledKidsSet: logger.error("%s is not installed by FOIA" % prevInstallName) if prevInstallName not in outOrderList: logger.debug("Adding %s as part of multibuilds" % prevInstallName) outOrderList.append(prevInstallName) else: logger.debug("%s is already in the list" % prevInstallName) outOrderList.append(installName) for idx in range(len(outOrderList) - 1, 0, -1): installName = outOrderList[idx] prevInstallName = outOrderList[idx - 1] self._csvDepDict[installName] = prevInstallName
def addChangeSet(gitRepoDir=None, patternList=[]): """ Utility function to add all the files changed to staging area @gitRepoDir: git repository directory, default is current directory. if provided, will only add all changes under that directory @patternList: a list of pattern for matching files. need to escape wildcard character '*' @return: return True if success, False otherwise """ git_command_list = [ "git", "diff", "--stat", "--stat-width=10000", "--", "*.zwr" ] result, output = _runGitCommand(git_command_list, gitRepoDir) test = output.split("\n") logger.info(output) """ Attempts to check each global file for more than two lines of change This will prevent every global from being updated in each commit thanks to the date change written in by ZGO. This assumes the script will be run in the "Packages" directory, which necessitates the removal of the "Packages/" string from the filename """ patternIncludeList = ["*.m"] for line in test: group = re.match( '^[ ]+Packages/(?P<filename>.+.zwr)[ ]+[|][ ]+(?P<numlines>[0-9]+) ', line) if group and (group.group('filename') and group.group('numlines')): if (int(group.group('numlines')) > 2): patternIncludeList.append(group.group('filename')) """ Now add everything that can be found or was called for""" git_command_list = ["git", "add", "--"] totalIncludeList = patternList + patternIncludeList for file in totalIncludeList: git_command = git_command_list + [file.encode('string-escape')] result, output = _runGitCommand(git_command, gitRepoDir) logger.info(output) """ Add the untracked files through checking for "other" files and then add the list """ git_command = ["git", "ls-files", "-o", "--exclude-standard"] result, lsFilesOutput = _runGitCommand(git_command, gitRepoDir) git_command_list = ["git", "add"] for file in lsFilesOutput.split("\n"): if len(file): git_command = git_command_list + [file.encode('string-escape')] result, output = _runGitCommand(git_command, gitRepoDir) return result
def generatePatchOrderTopologic(self, patchDir, installName=None): self.analyzeVistAPatchDir(patchDir) if installName: if installName not in self._patchInfoDict: raise Exception("Could not find patch for %s" % installName) self.__updatePatchDependency__((installName is None)) self.__generatePatchDependencyGraph__() try: self.__topologicSort__(installName) except CycleException as e: errorMessage = "Failed to sort patches: %s" % e logger.error(errorMessage) return [] logger.info("After topologic sort %d" % len(self._patchOrder)) return self._patchOrder
def place(src, dst): logger.info('%s => %s\n' % (src, dst)) d = os.path.dirname(dst) if d and not os.path.exists(d): try: os.makedirs(d) except OSError as ex: logger.error(ex) pass if not os.path.exists(dst): try: os.rename(src, dst) except OSError as ex: logger.error(ex) logger.error("%s => %s" % (src, dst)) pass
def applyPatchSequenceByInstallName(self, installName, patchOnly=False): """ apply the Patch in sequence order up to specified install name if patchOnly is set to True, will only apply that patch """ if installName not in self._patchSet: raise Exception("Invalid install name: %s" % installName) if len(self._outPatchList) == 0: logger.info("No Patch to apply") return 0 logger.info("Apply patches up to %s" % installName) patchIndex = self.__getPatchIndexByInstallName__(installName) endIndex = patchIndex + 1 startIndex = 0 if patchOnly: startIndex = patchIndex return self.__applyPatchUptoIndex__(endIndex, startIndex)
def loadMetaDataFromJSON(inputFileName): import json logger.info("reading KIDS metadata from %s" % inputFileName) outputJSON = json.load(open(inputFileName, 'r')) installNameList = [] kidsBuilds = [] header = outputJSON['header'] for build in outputJSON['builds']: installName = build['name'] installNameList.append(installName) kidsBuild = KIDSBuild(installName) for depBuild in build['dependency']: kidsBuild.addDependencyBuild([depBuild, None]) kidsBuilds.append(kidsBuild) seqNo = KIDSBuildParser.parseKIDSHeader(header) return (installNameList, seqNo, kidsBuilds)