def _updateInstallReference(self): installData = self._glbData['9.7'] output = os.path.join(self.outDir, "install_information.json") installJSONData = {} packageList = self._crossRef.getAllPackages() patchOrderGen = PatchOrderGenerator() patchOrderGen.analyzeVistAPatchDir(self.patchDir +"/Packages") with open(output, 'w') as installDataOut: logging.warn("inside the _updateInstallReference") for ien in sorted(installData.dataEntries.keys(), key=lambda x: float(x)): installItem = {} installEntry = installData.dataEntries[ien] package = self._findInstallPackage(packageList, installEntry.name) # if this is the first time the package is found, add an entry in the install JSON data. if package not in installJSONData: installJSONData[package]={} if installEntry.name: logging.warn("Gathering info for: %s" % installEntry.name) installItem['name'] = installEntry.name installItem['ien'] = installEntry.ien installItem['label'] = installEntry.name installItem['value'] = installEntry.name installItem['parent']= package if installEntry.name in patchOrderGen._kidsDepBuildDict: installchildren = [] for child in patchOrderGen._kidsDepBuildDict[installEntry.name]: childPackage = self._findInstallPackage(packageList,child) installchildren.append({"name": child, "package": childPackage}); installItem['children'] = installchildren if '11' in installEntry.fields: installItem['installDate'] = installEntry.fields['11'].value.strftime("%Y-%m-%d") if '1' in installEntry.fields: installItem['packageLink'] = installEntry.fields['1'].value if '40' in installEntry.fields: installItem['numRoutines'] = len(installEntry.fields['40'].value.dataEntries) if '14' in installEntry.fields: installItem['numFiles'] = len(installEntry.fields['14'].value.dataEntries) # Checks for the absence of asterisks which usually denotes a package change. testMatch = re.search("\*+",installEntry.name) if testMatch is None: installItem['packageSwitch'] = True installJSONData[package][installEntry.name] = installItem installJSONData['MultiBuild']={} for multiBuildFile in patchOrderGen._multiBuildDict: multibuildItem = {} multibuildItem['name']=os.path.basename(multiBuildFile); multibuildItem['children'] = [] for installName in patchOrderGen._multiBuildDict[multiBuildFile]: package = self._findInstallPackage(packageList, installName) multibuildItem['children'].append({"name": installName, "package": package}); installJSONData['MultiBuild'][os.path.basename(multiBuildFile)] = multibuildItem logging.warn("About to dump data into %s" % output) json.dump(installJSONData,installDataOut)
def __init__(self, testClient, logFileDir): self._testClient = testClient curTimestamp = getCurrentTimestamp() logFileName = "%s.%s" % (self.DEFAULT_VISTA_LOG_FILENAME, curTimestamp) self._logFileName = os.path.join(logFileDir, logFileName) self._testClient.setLogFile(self._logFileName) self._patchOrderGen = PatchOrderGenerator() self._vistaPatchInfo = VistAPackageInfoFetcher(testClient) self._outPatchList = [] self._patchSet = set() initFileLogging(os.path.join(logFileDir, self.DEFAULT_OUTPUT_FILE_LOG))
def populate(input): packages, namespaces = populatePackageMapByCSV(input) #--------------------------------------------------------------------------- # Collect all KIDS and info files under the current directory recursively #--------------------------------------------------------------------------- curDir = os.getcwd() patchOrderGen = PatchOrderGenerator() patchOrder = patchOrderGen.generatePatchOrder(curDir) patchInfoDict = patchOrderGen.getPatchInfoDict() patchInfoSet = set(patchInfoDict.keys()) patchList = patchInfoDict.values() noKidsInfoDict = patchOrderGen.getNoKidsBuildInfoDict() noKidsInfoSet = set(noKidsInfoDict.keys()) noKidsPatchList = noKidsInfoDict.values() leftoverTxtFiles = patchOrderGen.getInvalidInfoFiles() #--------------------------------------------------------------------------- # place multiBuilds KIDS Build under MultiBuilds directory #--------------------------------------------------------------------------- multiBuildSet = set([x.installName for x in patchList if x.isMultiBuilds]) for info in multiBuildSet: logger.info("Handling Multibuilds Kids %s" % info) patchInfo = patchInfoDict[info] src = patchInfo.kidsFilePath dest = os.path.normpath(os.path.join(curDir, "MultiBuilds", os.path.basename(src))) if src != dest: place(src,dest) if isValidKIDSBuildHeaderSuffix(dest): " add to ignore list if not there" addToGitIgnoreList(dest[0:dest.rfind('.')]) src = patchInfo.kidsSha1Path if not src: continue dest = os.path.normpath(os.path.join(curDir, "MultiBuilds", os.path.basename(src))) if src != dest: place(src,dest) # Map by package namespace (prefix). for ns in sorted(namespaces.keys(),order_long_to_short): path = namespaces[ns] nsPatchList = [x.installName for x in patchList if x.namespace==ns] for patch in nsPatchList: logger.info("Handling Kids %s" % patch) patchInfo = patchInfoDict[patch] patchDir = os.path.join(path, "Patches", installNameToDirName(patch)) placePatchInfo(patchInfo, curDir, patchDir) # Map KIDS Info Files that do not have associated KIDS Build Files nsNoKidsList = [x.installName for x in noKidsPatchList if x.namespace==ns] for patch in nsNoKidsList: logger.info("Handling No Kids info File %s" % patch) patchInfo = noKidsInfoDict[patch] patchDir = os.path.join(path, "Patches", installNameToDirName(patch)) placePatchInfo(patchInfo, curDir, patchDir) patchInfoSet.difference_update(nsPatchList) noKidsInfoSet.difference_update(nsNoKidsList) # Put leftover kids files in Uncategorized package. for patch in patchInfoSet: logger.info("Handling left over Kids File %s" % patch) patchInfo = patchInfoDict[patch] placePatchInfo(patchInfo, curDir, 'Uncategorized') for patch in noKidsInfoSet: logger.info("Handling left over no Kids Info File %s" % patch) patchInfo = noKidsInfoDict[patch] placePatchInfo(patchInfo, curDir, 'Uncategorized') # Put invalid kids info files in Uncategorized package. for src in leftoverTxtFiles: logger.info("Handling left over files: %s" % src) from KIDSAssociatedFilesMapping import getAssociatedInstallName installName = getAssociatedInstallName(src) if installName == "MultiBuilds": # put in Multibuilds directory dest = os.path.normpath(os.path.join(curDir, "MultiBuilds", os.path.basename(src))) if src != dest: place(src,dest) continue dirName = os.path.dirname(src) if not dirName.endswith("Packages"): logger.debug("Do not move %s" % src) continue dest = os.path.normpath(os.path.join(curDir, 'Uncategorized', os.path.basename(src))) if src != dest: place(src,dest)
def _updateInstallReference(self): installData = self._glbData['9.7'] output = os.path.join(self.outDir, "install_information.json") installJSONData = {} packageList = self._crossRef.getAllPackages() patchOrderGen = PatchOrderGenerator() patchOrderGen.analyzeVistAPatchDir(self.patchDir + "/Packages") with open(output, 'w') as installDataOut: logging.warn("inside the _updateInstallReference") for ien in sorted(installData.dataEntries.keys(), key=lambda x: float(x)): installItem = {} installEntry = installData.dataEntries[ien] package = self._findInstallPackage(packageList, installEntry.name) # if this is the first time the package is found, add an entry in the install JSON data. if package not in installJSONData: installJSONData[package] = {} if installEntry.name: logging.warn("Gathering info for: %s" % installEntry.name) installItem['name'] = installEntry.name installItem['ien'] = installEntry.ien installItem['label'] = installEntry.name installItem['value'] = len(installJSONData[package]) if installEntry.name in patchOrderGen._kidsDepBuildDict: installchildren = [] for child in patchOrderGen._kidsDepBuildDict[ installEntry.name]: childPackage = self._findInstallPackage( packageList, child) installchildren.append({ "name": child, "package": childPackage }) installItem['children'] = installchildren if '11' in installEntry.fields: installItem['installDate'] = installEntry.fields[ '11'].value.strftime("%Y-%m-%d") if '1' in installEntry.fields: installItem['packageLink'] = installEntry.fields[ '1'].value if '40' in installEntry.fields: installItem['numRoutines'] = len( installEntry.fields['40'].value.dataEntries) if '14' in installEntry.fields: installItem['numFiles'] = len( installEntry.fields['14'].value.dataEntries) # Checks for the absence of asterisks which usually denotes a package change. testMatch = re.search("\*+", installEntry.name) if testMatch is None: installItem['packageSwitch'] = True installJSONData[package][installEntry.name] = installItem installJSONData['MultiBuild'] = {} for multiBuildFile in patchOrderGen._multiBuildDict: multibuildItem = {} multibuildItem['name'] = os.path.basename(multiBuildFile) multibuildItem['children'] = [] for installName in patchOrderGen._multiBuildDict[ multiBuildFile]: package = self._findInstallPackage(packageList, installName) multibuildItem['children'].append({ "name": installName, "package": package }) installJSONData['MultiBuild'][os.path.basename( multiBuildFile)] = multibuildItem logging.warn("About to dump data into %s" % output) json.dump(installJSONData, installDataOut)
def _updateInstallReference(self): if not os.path.exists(self.outdir + "/9_7"): os.mkdir(self.outdir + "/9_7") installData = self._glbData['9.7'] output = os.path.join(self.outdir, "install_information.json") installJSONData = {} packageList = self._crossRef.getAllPackages() patchOrderGen = PatchOrderGenerator() patchOrderGen.analyzeVistAPatchDir(self.patchDir + "/Packages") with open(output, 'w') as installDataOut: logging.warn("inside the _updateInstallReference") for ien in sorted(installData.dataEntries.keys(), key=lambda x: float(x)): installItem = {} installEntry = installData.dataEntries[ien] package = self._findInstallPackage(packageList, installEntry.name) # if this is the first time the package is found, add an entry in the install JSON data. if package not in installJSONData: installJSONData[package] = {} if installEntry.name: logging.warn("Gathering info for: %s" % installEntry.name) installItem['name'] = installEntry.name installItem['ien'] = installEntry.ien installItem['label'] = installEntry.name installItem['value'] = installEntry.name installItem['parent'] = package if installEntry.name in patchOrderGen._kidsDepBuildDict: installchildren = [] for child in patchOrderGen._kidsDepBuildDict[ installEntry.name]: childPackage = self._findInstallPackage( packageList, child) installchildren.append({ "name": child, "package": childPackage }) installItem['children'] = installchildren if '11' in installEntry.fields: installItem['installDate'] = installEntry.fields[ '11'].value.strftime("%Y-%m-%d") if '1' in installEntry.fields: installItem['packageLink'] = installEntry.fields[ '1'].value if '40' in installEntry.fields: installItem['numRoutines'] = len( installEntry.fields['40'].value.dataEntries) if '14' in installEntry.fields: installItem['numFiles'] = len( installEntry.fields['14'].value.dataEntries) # Checks for the absence of asterisks which usually denotes a package change, also make it more specific to # eliminate the multibuilds that are being marked as package changes testMatch = re.search("\*+", installEntry.name) if testMatch is None: # Assume a package switch name will be just a package name and a version capture = re.match( "(?P<packageName>[A-Z./ \&\-\']+) (?P<packageVal>[.0-9]+)", installEntry.name) if capture: checkPackage = self._findInstallPackage( packageList, capture.groups()[0], False) if (not (checkPackage == "Unknown") or (len(capture.groups()[0]) <= 4)): installItem['packageSwitch'] = True installJSONData[package][installEntry.name] = installItem installJSONData['MultiBuild'] = {} for multiBuildFile in patchOrderGen._multiBuildDict: multibuildItem = {} multibuildItem['name'] = os.path.basename(multiBuildFile) multibuildItem['children'] = [] for installName in patchOrderGen._multiBuildDict[ multiBuildFile]: package = self._findInstallPackage(packageList, installName) multibuildItem['children'].append({ "name": installName, "package": package }) installJSONData['MultiBuild'][os.path.basename( multiBuildFile)] = multibuildItem logging.warn("About to dump data into %s" % output) json.dump(installJSONData, installDataOut)
def test_generatePatchOrderTopologicSample(self): patchOrderGen = PatchOrderGenerator() patchOrder = patchOrderGen.generatePatchOrderTopologic(TEST_PACKAGE_DIR) self.verifySampleOrder(patchOrder)
def test_generatePatchOrderTopologic(self): patchOrderGen = PatchOrderGenerator() patchOrder = patchOrderGen.generatePatchOrderTopologic(PACKAGE_DIR) self.assertTrue(patchOrder, "no valid patch order is generated")
def populate(input): packages, namespaces = populatePackageMapByCSV(input) #--------------------------------------------------------------------------- # Collect all KIDS and info files under the current directory recursively #--------------------------------------------------------------------------- curDir = os.getcwd() patchOrderGen = PatchOrderGenerator() patchOrder = patchOrderGen.generatePatchOrder(curDir) patchInfoDict = patchOrderGen.getPatchInfoDict() patchInfoSet = set(patchInfoDict.keys()) patchList = patchInfoDict.values() noKidsInfoDict = patchOrderGen.getNoKidsBuildInfoDict() noKidsInfoSet = set(noKidsInfoDict.keys()) noKidsPatchList = noKidsInfoDict.values() leftoverTxtFiles = patchOrderGen.getInvalidInfoFiles() #--------------------------------------------------------------------------- # place multiBuilds KIDS Build under MultiBuilds directory #--------------------------------------------------------------------------- multiBuildSet = set([x.installName for x in patchList if x.isMultiBuilds]) for info in multiBuildSet: logger.info("Handling Multibuilds Kids %s" % info) patchInfo = patchInfoDict[info] src = patchInfo.kidsFilePath dest = os.path.normpath( os.path.join(curDir, "MultiBuilds", os.path.basename(src))) if src != dest: place(src, dest) if isValidKIDSBuildHeaderSuffix(dest): " add to ignore list if not there" addToGitIgnoreList(dest[0:dest.rfind('.')]) src = patchInfo.kidsSha1Path if not src: continue dest = os.path.normpath( os.path.join(curDir, "MultiBuilds", os.path.basename(src))) if src != dest: place(src, dest) # Map by package namespace (prefix). for ns in sorted(namespaces.keys(), order_long_to_short): path = namespaces[ns] nsPatchList = [x.installName for x in patchList if x.namespace == ns] for patch in nsPatchList: logger.info("Handling Kids %s" % patch) patchInfo = patchInfoDict[patch] patchDir = os.path.join(path, "Patches", installNameToDirName(patch)) placePatchInfo(patchInfo, curDir, patchDir) # Map KIDS Info Files that do not have associated KIDS Build Files nsNoKidsList = [ x.installName for x in noKidsPatchList if x.namespace == ns ] for patch in nsNoKidsList: logger.info("Handling No Kids info File %s" % patch) patchInfo = noKidsInfoDict[patch] patchDir = os.path.join(path, "Patches", installNameToDirName(patch)) placePatchInfo(patchInfo, curDir, patchDir) patchInfoSet.difference_update(nsPatchList) noKidsInfoSet.difference_update(nsNoKidsList) # Put leftover kids files in Uncategorized package. for patch in patchInfoSet: logger.info("Handling left over Kids File %s" % patch) patchInfo = patchInfoDict[patch] placePatchInfo(patchInfo, curDir, 'Uncategorized') for patch in noKidsInfoSet: logger.info("Handling left over no Kids Info File %s" % patch) patchInfo = noKidsInfoDict[patch] placePatchInfo(patchInfo, curDir, 'Uncategorized') # Put invalid kids info files in Uncategorized package. for src in leftoverTxtFiles: logger.info("Handling left over files: %s" % src) from KIDSAssociatedFilesMapping import getAssociatedInstallName installName = getAssociatedInstallName(src) if installName == "MultiBuilds": # put in Multibuilds directory dest = os.path.normpath( os.path.join(curDir, "MultiBuilds", os.path.basename(src))) if src != dest: place(src, dest) continue dirName = os.path.dirname(src) if not dirName.endswith("Packages"): logger.debug("Do not move %s" % src) continue dest = os.path.normpath( os.path.join(curDir, 'Uncategorized', os.path.basename(src))) if src != dest: place(src, dest)
def _updateInstallReference(self): if not os.path.exists(self.outdir+"/9_7"): os.mkdir(self.outdir+"/9_7") installData = self._glbData['9.7'] output = os.path.join(self.outdir, "install_information.json") installJSONData = {} packageList = self._crossRef.getAllPackages() patchOrderGen = PatchOrderGenerator() patchOrderGen.analyzeVistAPatchDir(self.patchDir +"/Packages") with open(output, 'w') as installDataOut: logging.warn("inside the _updateInstallReference") for ien in sorted(installData.dataEntries.keys(), key=lambda x: float(x)): installItem = {} installEntry = installData.dataEntries[ien] package = self._findInstallPackage(packageList, installEntry.name) # if this is the first time the package is found, add an entry in the install JSON data. if package not in installJSONData: installJSONData[package]={} if installEntry.name: logging.warn("Gathering info for: %s" % installEntry.name) installItem['name'] = installEntry.name installItem['ien'] = installEntry.ien installItem['label'] = installEntry.name installItem['value'] = installEntry.name installItem['parent']= package if installEntry.name in patchOrderGen._kidsDepBuildDict: installchildren = [] for child in patchOrderGen._kidsDepBuildDict[installEntry.name]: childPackage = self._findInstallPackage(packageList,child) installchildren.append({"name": child, "package": childPackage}); installItem['children'] = installchildren if '11' in installEntry.fields: installItem['installDate'] = installEntry.fields['11'].value.strftime("%Y-%m-%d") if '1' in installEntry.fields: installItem['packageLink'] = installEntry.fields['1'].value if '40' in installEntry.fields: installItem['numRoutines'] = len(installEntry.fields['40'].value.dataEntries) if '14' in installEntry.fields: installItem['numFiles'] = len(installEntry.fields['14'].value.dataEntries) # Checks for the absence of asterisks which usually denotes a package change, also make it more specific to # eliminate the multibuilds that are being marked as package changes testMatch = re.search("\*+",installEntry.name) if testMatch is None: # Assume a package switch name will be just a package name and a version capture = re.match("(?P<packageName>[A-Z./ \&\-\']+) (?P<packageVal>[.0-9]+)",installEntry.name) if capture: checkPackage = self._findInstallPackage(packageList, capture.groups()[0],False) if (not (checkPackage == "Unknown") or (len(capture.groups()[0]) <= 4 )): installItem['packageSwitch'] = True installJSONData[package][installEntry.name] = installItem installJSONData['MultiBuild']={} for multiBuildFile in patchOrderGen._multiBuildDict: multibuildItem = {} multibuildItem['name']=os.path.basename(multiBuildFile); multibuildItem['children'] = [] for installName in patchOrderGen._multiBuildDict[multiBuildFile]: package = self._findInstallPackage(packageList, installName) multibuildItem['children'].append({"name": installName, "package": package}); installJSONData['MultiBuild'][os.path.basename(multiBuildFile)] = multibuildItem logging.warn("About to dump data into %s" % output) json.dump(installJSONData,installDataOut)