def __copyAllGlobals__(self): logger.info("Copying all files from %s to %s" % (self._globalOutputDir, self._packagesDir)) zwrFiles = glob.glob(os.path.join(self._globalOutputDir, "*.zwr")) for zwrFile in zwrFiles: logger.debug("Copying %s to %s" % (zwrFile, self._packagesDir)) shutil.copy2(zwrFile, self._packagesDir)
def rewriteFileManRoutineGTM(outDir): import shutil for filename in ['DIDT','DIDTC','DIRCR']: src = os.path.join(outDir, filename + ".m") dst = os.path.join(outDir, filename.replace('DI','_') + '.m') logger.debug("Copy %s to %s" % (src, dst)) shutil.copyfile(src, dst)
def installCompleted(self, connection, **kargs): extraInfo = connection.before logger.debug(extraInfo) if re.search("No link to PACKAGE file", extraInfo): self._updatePackageLink = True logger.warn("You might have to update KIDS build %s to link" " to Package file" % (self._kidsInstallName))
def __parseAllKIDSBuildFilesList__(self): kidsParser = KIDSPatchInfoParser() for basename in self._kidsBuildFileDict.iterkeys(): kidsFile, sha1Path = self._kidsBuildFileDict[basename] if kidsFile == None: logger.error("No KIDS file available for name %s" % basename) installNameList,seqNo = kidsParser.getKIDSBuildInstallNameSeqNo(kidsFile) if len(installNameList) > 1: if not self._multiBuildDict.get(kidsFile): self._multiBuildDict[kidsFile] = installNameList else: assert self._multiBuildDict[kidsFile] == installNameList elif seqNo: if installNameList[0] not in self._installNameSeqMap: self._installNameSeqMap[installNameList[0]] = seqNo else: logger.error("Duplicated KIDS build file %s" % kidsFile) for installName in installNameList: if installName in self._kidsInstallNameDict: logger.warn("%s is already in the dict %s" % (installName, kidsFile)) logger.debug("Added installName %s, file %s" % (installName, kidsFile)) self._kidsInstallNameDict[installName] = os.path.normpath(kidsFile) """ handle KIDS sha1 file Path """ if sha1Path: if installName in self._kidsInstallNameSha1Dict: logger.warn("%s is already in the dict %s" % (installName, sha1Path)) self._kidsInstallNameSha1Dict[installName] = sha1Path logger.debug("%s" % sorted(self._kidsInstallNameDict.keys())) logger.info("Total # of install name %d" % len(self._kidsInstallNameDict))
def __updateMultiBuildDependencies__(self): patchList = self._patchInfoDict for installList in self._multiBuildDict.itervalues(): logger.info("Multi-Buids KIDS install List: %s" % (installList)) firstPatch = patchList[installList[0]] firstPatch.otherKidsInfoList = [] if firstPatch.csvDepPatch is None: """ If primary build install name is not specified in the csv file will fall back to use dependency specified in the first secondary build """ secondPatch = patchList[installList[1]] if secondPatch.csvDepPatch != firstPatch.installName: logger.info("Assign first patch CSV Dep %s" % firstPatch.installName) firstPatch.csvDepPatch = secondPatch.csvDepPatch for index in range(1,len(installList)): nextPatchInfo = patchList[installList[index]] """ just to make sure the first one has all the dependencies """ firstPatch.depKIDSBuild.update(nextPatchInfo.depKIDSBuild) firstPatch.optionalDepSet.update(nextPatchInfo.optionalDepSet) firstPatch.otherKidsInfoList.append([nextPatchInfo.kidsInfoPath, nextPatchInfo.kidsInfoSha1]) prevInstallName = installList[index - 1] # Removing May 2019: Causes cyclical dependency - J.Snyder #if prevInstallName not in nextPatchInfo.depKIDSBuild: # nextPatchInfo.depKIDSBuild.add(prevInstallName) #del patchList[installList[index]] #remove the other patch from the list logger.debug("%s:%s" % (nextPatchInfo.installName, nextPatchInfo.depKIDSBuild)) """ remove the self dependencies of the first patch """ firstPatch.depKIDSBuild.difference_update(installList) logger.debug("%s:%s" % (firstPatch.installName, firstPatch.depKIDSBuild))
def parseKIDSInfoFile(self, infoFile): logger.debug("Parsing Info file %s" % infoFile) patchInfo = PatchInfo() assert os.path.exists(infoFile) patchInfo.kidsInfoPath = infoFile inputFile = open(infoFile, 'rb') for line in inputFile: line = line.rstrip(" \r\n") if len(line) == 0: continue """ subject part are treated as end of parser section for now""" if self.SUBJECT_PART_START_REGEX.search(line): break; ret = self.RUNDATE_DESIGNATION_REGEX.search(line) if ret: patchInfo.rundate = datetime.strptime(ret.group('date'), self.RUNDATE_FORMAT_STRING) patchInfo.installName = convertToInstallName(ret.group('design')) continue ret = self.PACKAGE_PRIORITY_REGEX.search(line) if ret: package = ret.group('name').strip() logger.debug(package) (namespace, name) = package.split(" - ", 1) # split on first - patchInfo.namespace = namespace.strip() patchInfo.package = name.strip() patchInfo.priority = ret.group('pri').strip() continue ret = self.VERSION_STATUS_REGEX.search(line) if ret: versionInfo = ret.group('no').strip() pos = versionInfo.find('SEQ #') if pos >= 0: patchInfo.version = versionInfo[:pos].strip() patchInfo.seqNo = versionInfo[pos+5:].strip() try: int(patchInfo.seqNo) except ValueError as ex: logger.error("invalid seqNo %s" % patchInfo.seqNo) raise ex else: patchInfo.version = versionInfo.strip() # fix the patch version to make sure 1 => 1.0 if (float(patchInfo.version).is_integer() and patchInfo.version.find('.') < 0): patchInfo.version += ".0" patchInfo.status = ret.group('status').strip() """ find out the dep patch info """ ret = self.ASSOCIATED_PATCH_START_REGEX.search(line) if ret: self.parseAssociatedPart(line[self.ASSOCIATED_PATCH_START_INDEX:], patchInfo) continue ret = self.ASSOCIATED_PATCH_SECTION_REGEX.search(line) if ret: self.parseAssociatedPart(line.strip(), patchInfo) continue if patchInfo.installName == None: return None setPatchInfoFromInstallName(patchInfo.installName, patchInfo) return patchInfo
def __exit__(self, exc_type, exc_value, traceback): logger.debug("__exit__ is called %s,%s,%s" % (exc_type, exc_value, traceback)) connection = self._connection if exc_type is KeyboardInterrupt: connection.terminate() return True if connection is not None and connection.isalive(): # try to close the connection gracefully try: for i in range(0,3): connection.send("^\r") # get out of the MENU prompt by sending ^ connection.send("\r") # get out of the MENU prompt by sendng \r connection.send("Q\r") # clean up any stack self.waitForPrompt(1) # wait for VistA prompt for 1 seconds connection.send("H\r") # Halt VistA connection except Exception as ex: logger.error(ex) if isLinuxSystem(): if connection.isalive(): """ pexpect close() will close all open handlers and is non-blocking """ try: connection.close() except ExceptionPexpect as ose: logger.error(ose) connection.terminate() return
def getInstallationStatus(self, installName): connection = self._testClient.getConnection() result = -1 # default is not installed menuUtil = VistAMenuUtil(duz=1) menuUtil.gotoFileManInquireFileEntryMenu(self._testClient) connection.send("9.7\r") # Package file with fileman #9.7 connection.expect("Select INSTALL NAME:") connection.send("%s\r" % installName) while True: index = connection.expect([ "Select INSTALL NAME: ", re.compile("ANOTHER ONE: ", re.I), "CHOOSE 1-[0-9]+: " ]) if index == 0: connection.send("\r") break elif index == 1: txtToSearch = connection.before.replace("\r\n", "") logger.debug(txtToSearch) result = indexOfInstallStatus(txtToSearch) connection.send("^\r") break # handle if the install has multiple install status elif index == 2: linesToSearch = connection.before.split("\r\n") for line in linesToSearch: # only care the the first line line = line.strip("\r\n ") if not re.search("^1 ", line): continue result = indexOfInstallStatus(line) break connection.send("^\r") continue menuUtil.exitFileManMenu(self._testClient) return result
def generateKIDSPatchSequence(self, patchDir, patchesCSV=None): kidsOrderGen = self._kidsOrderGen patchHistInfo = self._vistaPatchInfo patchList = [] if patchesCSV and os.path.exists(patchCSVFile): patchList = kidsOrderGen.generatePatchOrderByCSV(patchDir, patchCSVFile) else: patchList = kidsOrderGen.generatePatchOrderTopologic(patchDir) self._patchSet.update([x.installName for x in patchList]) for patchInfo in patchList: namespace = patchInfo.namespace self.__updatePatchInfoPackageName__(patchInfo) logger.info("Checking for patch info %s" % patchInfo.installName) logger.debug("Checking for patch info %s" % patchInfo) """ check to see if the patch is already installed """ if self.__isPatchInstalled__(patchInfo): continue if self.__isPatchReadyToInstall__(patchInfo, patchList): self._outPatchList.append(patchInfo) else: logger.error("Can not install patch %s" % patchInfo.installName) break logger.info("Total patches are %d" % len(self._outPatchList)) for patchInfo in self._outPatchList: logger.info("%s, %s, %s" % (patchInfo.installName, patchInfo.namespace, patchInfo.kidsFilePath))
def __isPatchReadyToInstall__(self, patchInfo, patchList = None): packageName = patchInfo.package ver = patchInfo.version patchHist = self._vistaPatchInfo.getPackagePatchHistByName(packageName, ver) if not patchHist or not patchHist.hasPatchHistory(): logger.info("no patch hist for %s, ver: %s" % (packageName, ver)) return True # if no such an package or hist info, just return True """ check patch sequence no to see if it is out of order """ if patchInfo.seqNo: seqNo = patchHist.getLatestSeqNo() if patchInfo.seqNo < seqNo: logger.error("SeqNo out of order, %s less than latest one %s" % (patchInfo.seqNo), seqNo) return False # check all the dependencies for item in patchInfo.depKIDSPatch: if patchList and item in self._patchSet: # we are going to install the dep patch logger.info("We are going to install the patch %s" % item) """ make sure installation is in the right order """ itemIndex = self.indexInPatchList(item, patchList) patchIndex = self.indexInPatchList(patchInfo.installName, patchList) if itemIndex >= patchIndex: logger.warn("%s is out of order with %s" % (item, patchInfo)) return False else: continue (namespace,ver,patch) = extractInfoFromInstallName(item) if self._vistaPatchInfo.hasPatchInstalled(item, namespace, ver, patch): logger.debug("%s is arelady installed" % item) continue else: logger.error("dep %s is not installed for %s %s" % (item, patchInfo.installName, patchInfo.kidsFilePath)) return False return True
def getInstallationStatus(self, installName): connection = self._testClient.getConnection() result = -1 # default is not installed menuUtil = VistAMenuUtil(duz=1) menuUtil.gotoFileManInquireFileEntryMenu(self._testClient) connection.send("9.7\r") # Package file with fileman #9.7 connection.expect("Select INSTALL NAME:") connection.send("%s\r" % installName) while True: index = connection.expect(["Select INSTALL NAME: ", "ANOTHER ONE: ", "CHOOSE 1-[0-9]+: "]) if index == 0: connection.send("\r") break elif index == 1: txtToSearch = connection.before.replace("\r\n","") logger.debug(txtToSearch) result = indexOfInstallStatus(txtToSearch) connection.send("^\r") break # handle if the install has multiple install status elif index == 2: linesToSearch = connection.before.split("\r\n") for line in linesToSearch: # only care the the first line line = line.strip("\r\n ") if not re.search("^1 ", line): continue result = indexOfInstallStatus(line) break connection.send("^\r") continue menuUtil.exitFileManMenu(self._testClient) return result
def generateKIDSPatchSequence(self, patchDir, installName=None, isUpToPatch=False): kidsOrderGen = self._kidsOrderGen patchHistInfo = self._vistaPatchInfo patchList = [] if isUpToPatch: patchList = kidsOrderGen.generatePatchOrderTopologic(patchDir) else: patchList = kidsOrderGen.generatePatchOrderTopologic(patchDir, installName) self._patchSet.update([x.installName for x in patchList]) if installName and installName not in self._patchSet: errMsg = ("Can not find patch for install name %s" % installName) raise Exception(errMsg) for patchInfo in patchList: namespace = patchInfo.namespace self.__updatePatchInfoPackageName__(patchInfo) logger.info("Checking for patch info %s" % patchInfo.installName) logger.debug("Checking for patch info %s" % patchInfo) """ check to see if the patch is already installed """ if not self.__isPatchInstalled__(patchInfo): if self.__isPatchReadyToInstall__(patchInfo, patchList): self._outPatchList.append(patchInfo) else: errorMsg = ("Can not install patch %s" % patchInfo.installName) logger.error(errorMsg) raise Exception(errorMsg) if isUpToPatch and installName and installName == patchInfo.installName: break logger.info("Total patches are %d" % len(self._outPatchList)) for patchInfo in self._outPatchList: logger.info("%s, %s, %s" % (patchInfo.installName, patchInfo.namespace, patchInfo.kidsFilePath))
def __updateMultiBuildDependencies__(self): patchList = self._patchInfoDict for installList in self._multiBuildDict.itervalues(): logger.info("Multi-Buids KIDS install List: %s" % (installList)) firstPatch = patchList[installList[0]] firstPatch.isMultiBuilds = True firstPatch.multiBuildsList = installList firstPatch.otherKidsInfoList = [] for index in range(1, len(installList)): nextPatchInfo = patchList[installList[index]] """ just to make sure the first one has all the dependencies """ firstPatch.depKIDSPatch.update(nextPatchInfo.depKIDSPatch) firstPatch.otherKidsInfoList.append( [nextPatchInfo.kidsInfoPath, nextPatchInfo.kidsInfoSha1]) nextPatchInfo.isMultiBuilds = True nextPatchInfo.multiBuildsList = installList prevInstallName = installList[index - 1] if prevInstallName not in nextPatchInfo.depKIDSPatch: nextPatchInfo.depKIDSPatch.add(prevInstallName) #del patchList[installList[index]] #remove the other patch from the list logger.debug( "%s:%s" % (nextPatchInfo.installName, nextPatchInfo.depKIDSPatch)) """ remove the self dependencies of the first patch """ firstPatch.depKIDSPatch.difference_update(installList) logger.debug("%s:%s" % (firstPatch.installName, firstPatch.depKIDSPatch))
def installCompleted(self, connection, **kargs): extraInfo = connection.lastconnection logger.debug(extraInfo) logger.info("Installation succeeded for %s" % self._kidsInstallName) if re.search("No link to PACKAGE file", extraInfo): # Sets flag to attempt to include information when back at VistA's prompt self._updatePackageLink = True
def __updateMultiBuildDependencies__(self): patchList = self._patchInfoDict for installList in self._multiBuildDict.itervalues(): logger.info("Multi-Buids KIDS install List: %s" % (installList)) firstPatch = patchList[installList[0]] firstPatch.otherKidsInfoList = [] if firstPatch.csvDepPatch is None: """ If primary build install name is not specified in the csv file will fall back to use dependency specified in the first secondary build """ secondPatch = patchList[installList[1]] if secondPatch.csvDepPatch != firstPatch.installName: logger.info("Assign first patch CSV Dep %s" % firstPatch.installName) firstPatch.csvDepPatch = secondPatch.csvDepPatch for index in range(1,len(installList)): nextPatchInfo = patchList[installList[index]] """ just to make sure the first one has all the dependencies """ firstPatch.depKIDSBuild.update(nextPatchInfo.depKIDSBuild) firstPatch.optionalDepSet.update(nextPatchInfo.optionalDepSet) firstPatch.otherKidsInfoList.append([nextPatchInfo.kidsInfoPath, nextPatchInfo.kidsInfoSha1]) prevInstallName = installList[index - 1] if prevInstallName not in nextPatchInfo.depKIDSBuild: nextPatchInfo.depKIDSBuild.add(prevInstallName) #del patchList[installList[index]] #remove the other patch from the list logger.debug("%s:%s" % (nextPatchInfo.installName, nextPatchInfo.depKIDSBuild)) """ remove the self dependencies of the first patch """ firstPatch.depKIDSBuild.difference_update(installList) logger.debug("%s:%s" % (firstPatch.installName, firstPatch.depKIDSBuild))
def generatePatchSequence(self, patchDir, installName=None, isUpToPatch=False): patchOrderGen = self._patchOrderGen patchHistInfo = self._vistaPatchInfo patchList = [] if isUpToPatch: patchList = patchOrderGen.generatePatchOrderTopologic(patchDir) else: patchList = patchOrderGen.generatePatchOrderTopologic(patchDir, installName) self._patchSet.update([x.installName for x in patchList]) if installName and installName not in self._patchSet: errMsg = ("Can not find patch for install name %s" % installName) raise Exception(errMsg) for patchInfo in patchList: namespace = patchInfo.namespace self.__updatePatchInfoPackageName__(patchInfo) logger.info("Checking for patch info %s" % patchInfo.installName) logger.debug("Checking for patch info %s" % patchInfo) """ check to see if the patch is already installed """ if not self.__isPatchInstalled__(patchInfo): if self.__isPatchReadyToInstall__(patchInfo, patchList): self._outPatchList.append(patchInfo) else: errorMsg = ("Can not install patch %s" % patchInfo.installName) logger.error(errorMsg) raise Exception(errorMsg) if isUpToPatch and installName and installName == patchInfo.installName: break logger.info("Total patches are %d" % len(self._outPatchList)) for patchInfo in self._outPatchList: logger.info("%s, %s, %s" % (patchInfo.installName, patchInfo.namespace, patchInfo.kidsFilePath)) return self._outPatchList
def __exit__(self, exc_type, exc_value, traceback): logger.debug("__exit__ is called %s,%s,%s" % (exc_type, exc_value, traceback)) connection = self._connection if exc_type is KeyboardInterrupt: connection.terminate() return True if connection is not None and connection.isalive(): # try to close the connection gracefully try: for i in range(0, 3): connection.send( "^\r") # get out of the MENU prompt by sending ^ connection.send( "\r") # get out of the MENU prompt by sendng \r connection.send("Q\r") # clean up any stack self.waitForPrompt(1) # wait for VistA prompt for 1 seconds connection.send("H\r") # Halt VistA connection except Exception as ex: logger.error(ex) if isLinuxSystem(): if connection.isalive(): """ pexpect close() will close all open handlers and is non-blocking """ try: connection.close() except ExceptionPexpect as ose: logger.error(ose) connection.terminate() return
def verifyTaskmanSiteParameter(self, vistAClient, autoFix=True): retValue = True connection = vistAClient.getConnection() menuUtil = VistAMenuUtil(duz=1) boxVolPair = getBoxVolPair(vistAClient) logger.debug("Box:Vol Pair is [%s] " % boxVolPair) menuUtil.gotoTaskmanEditParamMenu(vistAClient) connection.send("Site Parameters Edit\r") connection.expect("Select TASKMAN SITE PARAMETERS BOX-VOLUME PAIR: ") connection.send("?\r") connection.expect("Answer with TASKMAN SITE PARAMETERS BOX-VOLUME PAIR.*?:") connection.expect("You may enter a new TASKMAN SITE PARAMETERS") curBoxVol = connection.before.strip(' \r\n') curBoxVol = [x.strip(' ') for x in curBoxVol.split('\r\n')] logger.debug("Box:Vol Pair is [%s] " % curBoxVol) if boxVolPair not in curBoxVol : logger.error("taskman site parameter mismatch, current:[%s], correct:[%s]" % (curBoxVol, boxVolPair)) if autoFix: self.__fixTaskmanSiteParameter__(connection, curBoxVol[0], boxVolPair) else: retValue = False connection.expect("Select TASKMAN SITE PARAMETERS BOX-VOLUME PAIR: ") connection.send('\r') menuUtil.exitTaskmanEditParamMenu(vistAClient) return retValue
def parseLines(self, section, lines, **kargs): assert ('kidsBuild' in kargs and kargs['kidsBuild'] != None) kidsBuild = kargs['kidsBuild'] logger.debug("Parsing Routine Line %s, %s" % (lines[0], lines[1])) result = self.NEW_ROUTINE_START.search(lines[0]) if result: return self.__onNewRoutineStart__(section, lines, result, kidsBuild) else: return self.__parseRoutineLines__(section, lines, result)
def __copyAllClasses__(self): if not self._cache: return logger.info("Copying all classes file from %s to %s" % (self._cacheclassesDir, self._packagesDir)) xmlFiles = glob.glob(os.path.join(self._cacheclassesDir, "*.xml")) for xmlFile in xmlFiles: logger.debug("Copying %s to %s" % (xmlFile, self._packagesDir)) shutil.copy2(xmlFile, self._packagesDir)
def __getAllKIDSBuildInfoAndOtherFileList__(self, patchDir): assert os.path.exists(patchDir) absPatchDir = os.path.abspath(patchDir) for (root, dirs, files) in os.walk(absPatchDir): lastDir = os.path.split(root)[-1] for fileName in files: absFilename = os.path.join(root, fileName) if not isValidPatchRelatedFiles(absFilename, True): continue """ Handle KIDS build files """ if isValidKIDSBuildSuffix(fileName): logger.debug("Adding %s KIDS file to dict" % absFilename) self.__addKidsBuildFileToDict__(fileName, absFilename, KIDS_BUILD_FILE_TYPE_KIDS) continue """ Handle KIDS build HEADER files """ if isValidKIDSBuildHeaderSuffix(fileName): logger.debug("Adding %s KIDS header to dict" % absFilename) kidsFileName = fileName[0:fileName.rfind('.')] self.__addKidsBuildFileToDict__(kidsFileName, absFilename, KIDS_BUILD_FILE_TYPE_HEADER) continue """ Handle KIDS build Sha1 files """ if isValidKIDSBuildSha1Suffix(fileName): logger.debug("Adding %s KIDS info to dict" % absFilename) kidsFileName = fileName[0:fileName.rfind('.')] self.__addKidsBuildFileToDict__(kidsFileName, absFilename, KIDS_BUILD_FILE_TYPE_SHA1) continue """ Handle KIDS Info/Sha1 files """ if ( isValidPatchInfoSuffix(fileName) or isValidPatchInfoSha1Suffix(fileName) ): self._kidsInfoFileList.append(absFilename) continue """ Handle Global/Sha1 Files """ if ( isValidGlobalFileSuffix(fileName) or isValidGlobalSha1Suffix(fileName) ): logger.debug("Adding %s Global files to list" % absFilename) self._globalFilesSet.add(absFilename) continue """ handle all csv files """ if isValidCSVSuffix(fileName): if isValidOrderCSVFile(absFilename): self._csvOrderFileList.append(absFilename) continue """ Handle .py files """ if isValidPythonSuffix(fileName): logger.debug("Adding %s python script to list" % absFilename) self._pythonScriptList.append(absFilename) continue logger.info("Total # of KIDS Builds are %d" % len(self._kidsBuildFileDict)) logger.info("Total # of KIDS Info are %d" % len(self._kidsInfoFileList)) logger.info("Total # of Global files are %d" % len(self._globalFilesSet)) logger.info("Total # of Python files are %d" % len(self._pythonScriptList)) logger.info("Total # of CSV files are %d" % len(self._csvOrderFileList))
def __getAllKIDSBuildInfoAndOtherFileList__(self, patchDir): assert os.path.exists(patchDir) absPatchDir = os.path.abspath(patchDir) for (root, dirs, files) in os.walk(absPatchDir): lastDir = os.path.split(root)[-1] for fileName in files: absFilename = os.path.join(root, fileName) if not isValidPatchRelatedFiles(absFilename, True): continue """ Handle KIDS build files """ if isValidKIDSPatchSuffix(fileName): logger.debug("Adding %s KIDS file to dict" % absFilename) self.__addKidsBuildFileToDict__(fileName, absFilename, KIDS_BUILD_FILE_TYPE_KIDS) continue """ Handle KIDS build HEADER files """ if isValidKIDSPatchHeaderSuffix(fileName): logger.debug("Adding %s KIDS header to dict" % absFilename) kidsFileName = fileName[0:fileName.rfind('.')] self.__addKidsBuildFileToDict__(kidsFileName, absFilename, KIDS_BUILD_FILE_TYPE_HEADER) continue """ Handle KIDS build Sha1 files """ if isValidKIDSPatchSha1Suffix(fileName): logger.debug("Adding %s KIDS info to dict" % absFilename) kidsFileName = fileName[0:fileName.rfind('.')] self.__addKidsBuildFileToDict__(kidsFileName, absFilename, KIDS_BUILD_FILE_TYPE_SHA1) continue """ Handle KIDS Info/Sha1 files """ if ( isValidKIDSInfoSuffix(fileName) or isValidKIDSInfoSha1Suffix(fileName) ): self._kidsInfoFileList.append(absFilename) continue """ Handle Global/Sha1 Files """ if ( isValidGlobalFileSuffix(fileName) or isValidGlobalSha1Suffix(fileName) ): logger.debug("Adding %s Global files to list" % absFilename) self._globalFilesSet.add(absFilename) continue """ handle all csv files """ if isValidCSVSuffix(fileName): if isValidOrderCSVFile(absFilename): self._csvOrderFileList.append(absFilename) continue """ Handle .py files """ if isValidPythonSuffix(fileName): logger.debug("Adding %s python script to list" % absFilename) self._pythonScriptList.append(absFilename) continue logger.info("Total # of KIDS Builds are %d" % len(self._kidsBuildFileDict)) logger.info("Total # of KIDS Info are %d" % len(self._kidsInfoFileList)) logger.info("Total # of Global files are %d" % len(self._globalFilesSet)) logger.info("Total # of Python files are %d" % len(self._pythonScriptList)) logger.info("Total # of CSV files are %d" % len(self._csvOrderFileList))
def onSectionStart(self, section, lines, **kargs): assert ('kidsBuild' in kargs and kargs['kidsBuild'] != None) kidsBuild = kargs['kidsBuild'] logger.debug("Routine Section starts %s, %s" % (lines[0], lines[1])) if self.ROUTINE_START.search(lines[0]): kidsBuild._totalNumRoutinesChange = int(lines[1]) else: logger.error("Unexpected Routine Section [%s]" % lines) return False return True
def __addMissKIDSInfoPatch__(self): for kidsInstallName in self._missKidsInfoSet: logger.debug("Installation Name: %s, does not have info file, %s" % (kidsInstallName, self._kidsInstallNameDict[kidsInstallName])) kidsPatchInfo = KIDSPatchInfo() kidsPatchInfo.installName = kidsInstallName setPatchInfoFromInstallName(kidsInstallName, kidsPatchInfo) if kidsInstallName in self._installNameSeqMap: kidsPatchInfo.seqNo = self._installNameSeqMap[kidsInstallName] kidsPatchInfo.kidsFilePath = self._kidsInstallNameDict[kidsInstallName] self._patchInfoDict[kidsInstallName] = kidsPatchInfo
def __getTaskmanStatus__(self, connection): connection.expect("Checking Taskman. ") connection.expect("Taskman is ") connection.expect("Checking the Status List:") statusString = connection.before.strip(' \r\n') logger.debug("Status String is %s" % statusString) connection.expect("Node weight status time \$J") connection.expect("Checking the Schedule List:") detailedStatus = connection.before.strip(' \r\n') logger.debug("Detailed Status String is %s" % detailedStatus) connection.expect("Enter monitor action: UPDATE//") return self.__taskmanStatusStringToEnum__(statusString, detailedStatus)
def findChoiceNumber(choiceTxt, matchString, namespace): logger.debug("txt is [%s]" % choiceTxt) choiceLines = choiceTxt.split('\r\n') for line in choiceLines: if len(line.rstrip()) == 0: continue result = re.search('^ +(?P<number>[1-9]) %s +%s$' % (matchString, namespace), line) if result: return result.group('number') else: continue return None
def __isPatchReadyToInstall__(self, patchInfo, patchList=None): packageName = patchInfo.package ver = patchInfo.version patchHist = self._vistaPatchInfo.getPackagePatchHistByName( packageName, ver) if not patchHist or not patchHist.hasPatchHistory(): logger.info("no patch hist for %s, ver: %s" % (packageName, ver)) return True # if no such an package or hist info, just return True """ check patch sequence no to see if it is out of order """ if patchInfo.seqNo: seqNo = patchHist.getLatestSeqNo() if patchInfo.seqNo < seqNo: logger.error( "SeqNo out of order, %s less than latest one %s" % (patchInfo.seqNo), seqNo) return False # check all the dependencies for item in patchInfo.depKIDSBuild: if patchList and item in self._patchSet: # we are going to install the dep patch logger.info("We are going to install the patch %s" % item) """ make sure installation is in the right order """ itemIndex = self.indexInPatchList(item, patchList) patchIndex = self.indexInPatchList(patchInfo.installName, patchList) if itemIndex >= patchIndex: logger.warn("%s is out of order with %s" % (item, patchInfo)) return False else: continue (namespace, ver, patch) = extractInfoFromInstallName(item) if self._vistaPatchInfo.hasPatchInstalled(item, namespace, ver, patch): logger.debug("%s is arelady installed" % item) continue installStatus = self._vistaPatchInfo.getInstallationStatus(item) if self._vistaPatchInfo.isInstallCompleted(installStatus): continue elif item in patchInfo.optionalDepSet: logger.warn( "Patch specified in KIDS info file %s is not installed for %s" % (item, patchInfo.installName)) continue else: logger.error( "dep %s is not installed for %s %s" % (item, patchInfo.installName, patchInfo.kidsFilePath)) patchInfo.depKIDSBuild.remove(item) patchInfo.depKIDSBuild.add(item + " <---") return False return True
def __handlePatchAssociatedFiles__(self): """ handle the info files first """ """ first by name assiciation """ patchInfoList = self._patchInfoDict.values() #handle the associated files for missingKIDSBuild info patchInfoList.extend(self._missKidsBuildDict.values()) for patchInfo in patchInfoList: infoPath = patchInfo.kidsInfoPath if infoPath: infoName = os.path.basename(infoPath) associateSet = set() for infoFile in self._invalidInfoFileSet: infoFileName = os.path.basename(infoFile) if infoFileName.startswith(infoName[:infoName.rfind('.')]): patchInfo.addToAssociatedInfoList(infoFile) associateSet.add(infoFile) continue self._invalidInfoFileSet.difference_update(associateSet) """ second by mapping association """ associateSet = set() for infoFile in self._invalidInfoFileSet: installName = getAssociatedInstallName(infoFile) if installName: if installName in self._patchInfoDict: patchInfo = self._patchInfoDict[installName] #handle the associated files for missingKIDSBuild info elif installName in self._missKidsBuildDict: patchInfo = self._missKidsBuildDict[installName] else: continue patchInfo.addToAssociatedInfoList(infoFile) associateSet.add(infoFile) self._invalidInfoFileSet.difference_update(associateSet) """ handle global files """ associateSet = set() for globalFile in self._globalFilesSet: installName = getAssociatedInstallName(globalFile) if installName and installName in self._patchInfoDict: patchInfo = self._patchInfoDict[installName] patchInfo.addToAssociatedGlobalList(globalFile) associateSet.add(globalFile) self._globalFilesSet.difference_update(associateSet) logger.info("Total # of leftover info files: %s" % len(self._invalidInfoFileSet)) logger.debug(self._invalidInfoFileSet) logger.info("Total # of leftover global files: %s" % len(self._globalFilesSet)) logger.debug(self._globalFilesSet)
def __buildPatchOrderDependencyByCSV__(self, orderCSV): patchOrderList = self._patchOrderCSVDict[orderCSV] if patchOrderList is None: return """ some sanity check """ outOrderList = [] for patchOrder in patchOrderList: installName = patchOrder[0] if installName not in self._patchInfoDict: if (installName not in self._informationalKidsSet and installName not in self._kidsInstallNameDict): logger.warn("No KIDS file found for %s" % str(patchOrder)) continue patchInfo = self._patchInfoDict[installName] """ check the seq no """ seqNo = patchOrder[1] if len(seqNo) > 0: """ check the seq no match the parsing result """ if patchInfo.seqNo is not None: if int(seqNo) != int(patchInfo.seqNo): logger.error( "SeqNo mismatch for %s, from csv: %s, info %s" % (installName, seqNo, patchInfo.seqNo)) else: logger.info("Add seqNo %s for %s" % (seqNo, installName)) patchInfo.seqNo = seqNo """ handle the case with multibuilds kids file """ if patchInfo.isMultiBuilds: installList = patchInfo.multiBuildsList index = installList.index(installName) if index > 0: for idx in range(0, index): prevInstallName = installList[idx] prevPatchInfo = self._patchInfoDict[prevInstallName] if prevInstallName in self._notInstalledKidsSet: logger.error("%s is not installed by FOIA" % prevInstallName) if prevInstallName not in outOrderList: logger.debug("Adding %s as part of multibuilds" % prevInstallName) outOrderList.append(prevInstallName) else: logger.debug("%s is already in the list" % prevInstallName) outOrderList.append(installName) for idx in range(len(outOrderList) - 1, 0, -1): installName = outOrderList[idx] prevInstallName = outOrderList[idx - 1] self._csvDepDict[installName] = prevInstallName
def __addMissKIDSInfoPatch__(self): for kidsInstallName in self._missKidsInfoSet: logger.debug("Installation Name: %s, does not have info file, %s" % (kidsInstallName, self._kidsInstallNameDict[kidsInstallName])) patchInfo = PatchInfo() patchInfo.installName = kidsInstallName setPatchInfoFromInstallName(kidsInstallName, patchInfo) if kidsInstallName in self._installNameSeqMap: patchInfo.seqNo = self._installNameSeqMap[kidsInstallName] patchInfo.kidsFilePath = self._kidsInstallNameDict[kidsInstallName] if kidsInstallName in self._kidsDepBuildDict: logger.info("update the Missing Info KIDS depencency %s" % kidsInstallName) patchInfo.depKIDSBuild = self._kidsDepBuildDict[kidsInstallName] self._patchInfoDict[kidsInstallName] = patchInfo
def __isPatchInstalled__(self, patchInfo): patchHistInfo = self._vistaPatchInfo namespace = patchInfo.namespace installName = patchInfo.installName if patchHistInfo.hasPatchInstalled( patchInfo.installName, patchInfo.namespace, patchInfo.version, patchInfo.patchNo, patchInfo.seqNo ): logger.info("%s is already installed" % installName) return True installStatus = patchHistInfo.getInstallationStatus(installName) logger.debug("%s installation status is %d" % (installName, installStatus)) if patchHistInfo.isInstallCompleted(installStatus): logger.info("%s installed completed" % installName) return True else: return False
def __addMissKIDSInfoPatch__(self): for kidsInstallName in self._missKidsInfoSet: logger.debug( "Installation Name: %s, does not have info file, %s" % (kidsInstallName, self._kidsInstallNameDict[kidsInstallName])) patchInfo = PatchInfo() patchInfo.installName = kidsInstallName setPatchInfoFromInstallName(kidsInstallName, patchInfo) if kidsInstallName in self._installNameSeqMap: patchInfo.seqNo = self._installNameSeqMap[kidsInstallName] patchInfo.kidsFilePath = self._kidsInstallNameDict[kidsInstallName] if kidsInstallName in self._kidsDepBuildDict: logger.info("update the Missing Info KIDS depencency %s" % kidsInstallName) patchInfo.depKIDSBuild = self._kidsDepBuildDict[ kidsInstallName] self._patchInfoDict[kidsInstallName] = patchInfo
def __parseHistoryLine__(self, historyLine): totalLen = len(historyLine) if totalLen < self.PATCH_VERSION_START_INDEX: return pos = historyLine.find(";Created on") # handle informal history line datetimeIndent = self.PATCH_APPLIED_DATETIME_INDEX userIndent = self.PATCH_APPLIED_USERNAME_INDEX if pos >= 0: # ignore the Created on format logger.debug(historyLine) historyLine = historyLine[:pos].rstrip() totalLen = len(historyLine) if totalLen > datetimeIndent: historyLine = historyLine.split('-')[0].rstrip() totalLen = len(historyLine) logger.debug("total len is %d" % totalLen) if totalLen > userIndent: self.userName = historyLine[userIndent:].strip() if totalLen > datetimeIndent: datetimePart = historyLine[datetimeIndent:userIndent].strip() pos = datetimePart.find(self.DATE_TIME_SEPERATOR) if pos >=0: if len(datetimePart) - pos == 3: datetimePart += ":00:00" if len(datetimePart) - pos == 6: datetimePart +=":00" self.datetime = datetime.strptime(datetimePart, self.DATETIME_FORMAT_STRING) else: self.datetime = datetime.strptime(datetimePart, self.DATE_FORMAT_STRING) if self.isVersionLine(historyLine): self.__parseVersionInfo__( historyLine[:datetimeIndent].strip()) return patchPart = historyLine[self.PATCH_VERSION_START_INDEX:datetimeIndent] seqIndex = patchPart.find("SEQ #") if seqIndex >= 0: self.patchNo = int(patchPart[:seqIndex].strip()) self.seqNo = int(patchPart[seqIndex+5:].strip()) else: try: self.patchNo = int(patchPart.strip()) except ValueError as ex: print ex logger.error("History Line is %s" % historyLine) self.patchNo = 0
def __parseHistoryLine__(self, historyLine): totalLen = len(historyLine) if totalLen < self.PATCH_VERSION_START_INDEX: return pos = historyLine.find(";Created on") # handle informal history line datetimeIndent = self.PATCH_APPLIED_DATETIME_INDEX userIndent = self.PATCH_APPLIED_USERNAME_INDEX if pos >= 0: # ignore the Created on format logger.debug(historyLine) historyLine = historyLine[:pos].rstrip() totalLen = len(historyLine) if totalLen > datetimeIndent: historyLine = historyLine.split('-')[0].rstrip() totalLen = len(historyLine) logger.debug("total len is %d" % totalLen) if totalLen > userIndent: self.userName = historyLine[userIndent:].strip() if totalLen > datetimeIndent: datetimePart = historyLine[datetimeIndent:userIndent].strip() pos = datetimePart.find(self.DATE_TIME_SEPERATOR) if pos >= 0: if len(datetimePart) - pos == 3: datetimePart += ":00:00" if len(datetimePart) - pos == 6: datetimePart += ":00" self.datetime = datetime.strptime(datetimePart, self.DATETIME_FORMAT_STRING) else: self.datetime = datetime.strptime(datetimePart, self.DATE_FORMAT_STRING) if self.isVersionLine(historyLine): self.__parseVersionInfo__(historyLine[:datetimeIndent].strip()) return patchPart = historyLine[self.PATCH_VERSION_START_INDEX:datetimeIndent] seqIndex = patchPart.find("SEQ #") if seqIndex >= 0: self.patchNo = int(patchPart[:seqIndex].strip()) self.seqNo = int(patchPart[seqIndex + 5:].strip()) else: try: self.patchNo = int(patchPart.strip()) except ValueError as ex: print ex logger.error("History Line is %s" % historyLine) self.patchNo = 0
def __buildPatchOrderDependencyByCSV__(self, orderCSV): patchOrderList = self._patchOrderCSVDict[orderCSV] if patchOrderList is None: return """ some sanity check """ outOrderList = [] for patchOrder in patchOrderList: installName = patchOrder[0] if installName not in self._patchInfoDict: if (installName not in self._informationalKidsSet and installName not in self._kidsInstallNameDict): logger.warn("No KIDS file found for %s" % str(patchOrder)) continue patchInfo = self._patchInfoDict[installName] """ check the seq no """ seqNo = patchOrder[1] if len(seqNo) > 0: """ check the seq no match the parsing result """ if patchInfo.seqNo is not None: if int(seqNo) != int(patchInfo.seqNo): logger.error("SeqNo mismatch for %s, from csv: %s, info %s" % (installName, seqNo, patchInfo.seqNo)) else: logger.info("Add seqNo %s for %s" % (seqNo, installName)) patchInfo.seqNo = seqNo """ handle the case with multibuilds kids file """ if patchInfo.isMultiBuilds: installList = patchInfo.multiBuildsList index = installList.index(installName) if index > 0: for idx in range(0, index): prevInstallName = installList[idx] prevPatchInfo = self._patchInfoDict[prevInstallName] if prevInstallName in self._notInstalledKidsSet: logger.error("%s is not installed by FOIA" % prevInstallName) if prevInstallName not in outOrderList: logger.debug("Adding %s as part of multibuilds" % prevInstallName) outOrderList.append(prevInstallName) else: logger.debug("%s is already in the list" % prevInstallName) outOrderList.append(installName) for idx in range(len(outOrderList)-1, 0, -1): installName = outOrderList[idx] prevInstallName = outOrderList[idx-1] self._csvDepDict[installName] = prevInstallName
def findChoiceNumber(choiceTxt, matchString, extraInfo=None): logger.debug("txt is [%s]" % choiceTxt) matchRegEx = None if extraInfo and len(extraInfo) > 0: matchRegEx = re.compile("^ +(?P<number>[0-9]+) %s +%s$" % (matchString, extraInfo)) else: matchRegEx = re.compile("^ +(?P<number>[0-9]+) %s$" % (matchString)) choiceLines = choiceTxt.split("\r\n") for line in choiceLines: line = line.rstrip() if len(line) == 0: continue result = matchRegEx.search(line) if result: return result.group("number") else: continue return None
def placePatchInfo(patchInfo, curDir, path): """ place the KIDS info file first if present """ logger.debug("place patch info %s" % patchInfo) destDir = os.path.join(curDir, path) infoSrc = patchInfo.kidsInfoPath if infoSrc: placeToDir(infoSrc, destDir) """ place the associated files """ placeAssociatedFiles(patchInfo.associatedInfoFiles, destDir) """ place the global files """ placeAssociatedFiles(patchInfo.associatedGlobalFiles, destDir) """ place the custom installer file """ placeToDir(patchInfo.customInstallerPath, destDir) """ ignore the multiBuilds kids file """ if patchInfo.isMultiBuilds: return placeToDir(patchInfo.kidsFilePath, destDir) """ check the KIDS Sha1 path """ placeToDir(patchInfo.kidsSha1Path, destDir)
def __isPatchInstalled__(self, patchInfo): patchHistInfo = self._vistaPatchInfo namespace = patchInfo.namespace installName = patchInfo.installName if patchHistInfo.hasPatchInstalled(patchInfo.installName, patchInfo.namespace, patchInfo.version, patchInfo.patchNo, patchInfo.seqNo): logger.info("%s is already installed" % installName) return True installStatus = patchHistInfo.getInstallationStatus(installName) logger.debug("%s installation status is %d" % (installName, installStatus)) if patchHistInfo.isInstallCompleted(installStatus): logger.info("%s installed completed" % installName) return True else: return False
def findChoiceNumber(choiceTxt, matchString, extraInfo=None): logger.debug("txt is [%s]" % choiceTxt) matchRegEx = None if extraInfo and len(extraInfo) > 0: matchRegEx = re.compile('^ +(?P<number>[0-9]+) %s +%s$' % (matchString, extraInfo)) else: matchRegEx = re.compile('^ +(?P<number>[0-9]+) %s$' % (matchString)) choiceLines = choiceTxt.split('\r\n') for line in choiceLines: line = line.rstrip() if len(line) == 0: continue result = matchRegEx.search(line) if result: return result.group('number') else: continue return None
def generatePatchSequence(self, patchDir, installName=None, isUpToPatch=False): patchOrderGen = self._patchOrderGen patchHistInfo = self._vistaPatchInfo patchList = [] if isUpToPatch: patchList = patchOrderGen.generatePatchOrderTopologic(patchDir) else: patchList = patchOrderGen.generatePatchOrderTopologic( patchDir, installName) self._patchSet.update([x.installName for x in patchList]) if installName and installName not in self._patchSet: errMsg = ("Can not find patch for install name %s" % installName) raise Exception(errMsg) dependencyIssues = {} for patchInfo in patchList: namespace = patchInfo.namespace self.__updatePatchInfoPackageName__(patchInfo) logger.info("Checking for patch info %s" % patchInfo.installName) logger.debug("Checking for patch info %s" % patchInfo) """ check to see if the patch is already installed """ if not self.__isPatchInstalled__(patchInfo): if self.__isPatchReadyToInstall__(patchInfo, patchList): self._outPatchList.append(patchInfo) else: dependencyIssues[ patchInfo.installName] = patchInfo.depKIDSBuild continue #raise Exception(errorMsg) if isUpToPatch and installName and installName == patchInfo.installName: break if len(dependencyIssues): for obj in dependencyIssues: logger.error("Problems found with dependencies of %s in: %s" % (obj, dependencyIssues[obj])) raise Exception("Dependency Errors Found") logger.info("Total patches are %d" % len(self._outPatchList)) for patchInfo in self._outPatchList: logger.info("%s, %s, %s" % (patchInfo.installName, patchInfo.namespace, patchInfo.kidsFilePath)) return self._outPatchList
def setPatchInfoFromInstallName(installName, patchInfo): (namespace, ver, patch) = extractInfoFromInstallName(installName) logger.debug((namespace, ver, patch, installName)) logger.debug((patchInfo)) if patchInfo.namespace: if patchInfo.namespace != namespace: logger.error((namespace, ver, patch)) logger.error(installName) logger.error(patchInfo) else: patchInfo.namespace = namespace if patchInfo.version: if (float(patchInfo.version) != float(ver)): logger.error((namespace, ver, patch)) logger.error(installName) logger.error(patchInfo) else: patchInfo.version = ver patchInfo.patchNo = patch
def deleteFileManRoutinesGTM(): """ first get routine directory """ from ParseGTMRoutines import extract_m_source_dirs var = os.getenv('gtmroutines') routineDirs = extract_m_source_dirs(var) if not routineDirs: return [] import glob outDir = routineDirs[0:1] for routineDir in routineDirs: for pattern in ['DI*.m', 'DD*.m', 'DM*.m']: globPtn = os.path.join(routineDir, pattern) fmFiles = glob.glob(os.path.join(routineDir, pattern)) if fmFiles: if routineDir not in outDir: outDir.append(routineDir) for fmFile in fmFiles: logger.debug("removing file %s" % fmFile) os.remove(fmFile) return outDir
def __parseGoldSection__(self, input, curLine): prevLine = curLine curLine = input.readline() goldSection = None curModule = None while len(curLine) > 0: if re.search('^\^+$', curLine): curModule = prevLine.strip(' \r\n') logger.debug("current module is %s" % curModule) elif curLine.startswith('.. table::'): routineData = self.__parseRoutineData__(input, curLine) if routineData: if not goldSection: goldSection = [] if curModule: goldSection.append((curModule, routineData)) else: goldSection.append(routineData) prevLine = curLine curLine = input.readline() return goldSection
def __parseGoldSection__(self, input, curLine): prevLine = curLine curLine = input.readline() goldSection = None curModule = None while len(curLine) > 0: if re.search('^\^+$', curLine): curModule = prevLine.strip(' \r\n') logger.debug("current module is %s" % curModule) elif curLine.startswith('.. table::'): routineData = self.__parseRoutineData__(input, curLine) if routineData: if not goldSection: goldSection=[] if curModule: goldSection.append((curModule, routineData)) else: goldSection.append(routineData) prevLine = curLine curLine = input.readline() return goldSection
def __logFunctionCallInfo__(self, funcName, section, lines, **kargs): logger.debug("Function %s is called" % funcName) logger.debug("Current Section is: %s" % section) if lines: for line in lines: logger.debug("line: [%s]" % line) self.__logKargs__(**kargs)
def __getPatchOrderListByCSV__(self, orderCSV): """INSTALLED,VERIFY_DT,STATUS,SEQ#,LABELED_AS,CATEGORY,PRODUCT_NAME""" assert os.path.exists(orderCSV) logger.info("Parsing file: %s" % orderCSV) if orderCSV not in self._patchOrderCSVDict: self._patchOrderCSVDict[orderCSV] = [] patchOrderList = self._patchOrderCSVDict[orderCSV] result = csv.DictReader(open(orderCSV, 'r')) installNameSet = set() # to check possible duplicates entry for row in result: installName = convertToInstallName(row['LABELED_AS'].strip()) if installName in installNameSet: logger.error("Ignore duplicate installName %s" % installName) continue installNameSet.add(installName) if row['INSTALLED'].strip() != "TRUE": self._notInstalledKidsSet.add(installName) if installName in self._kidsInstallNameDict: logger.error("Uninstalled patch %s found in %s: %s" % (installName, self._kidsInstallNameDict[installName], row)) self._removeNotInstalledKIDSBuild(installName) logger.debug("Ignore uninstalled patch %s" % row) continue try: verifiedTime = datetime.strptime(row['VERIFY_DT'], "%d-%b-%y") except ValueError as ex: verifiedTime = datetime.strptime(row['VERIFY_DT'], "%Y-%m-%d") """ check the seq # field """ seqNo = row['SEQ#'].strip() if len(seqNo) > 0: try: int(seqNo) except: seqNo = "" patchOrderList.append((installName, seqNo, verifiedTime)) if re.match("^Informational$", row['CATEGORY'].strip(), re.IGNORECASE): logger.debug("patch is informational %s " % row) self._informationalKidsSet.add(installName)