def __getAllKIDSBuildInfoAndOtherFileList__(self, patchDir): assert os.path.exists(patchDir) absPatchDir = os.path.abspath(patchDir) for (root, dirs, files) in os.walk(absPatchDir): lastDir = os.path.split(root)[-1] for fileName in files: absFilename = os.path.join(root, fileName) if not isValidPatchRelatedFiles(absFilename, True): continue """ Handle KIDS build files """ if isValidKIDSPatchSuffix(fileName): logger.debug("Adding %s KIDS file to dict" % absFilename) self.__addKidsBuildFileToDict__(fileName, absFilename, KIDS_BUILD_FILE_TYPE_KIDS) continue """ Handle KIDS build HEADER files """ if isValidKIDSPatchHeaderSuffix(fileName): logger.debug("Adding %s KIDS header to dict" % absFilename) kidsFileName = fileName[0:fileName.rfind('.')] self.__addKidsBuildFileToDict__(kidsFileName, absFilename, KIDS_BUILD_FILE_TYPE_HEADER) continue """ Handle KIDS build Sha1 files """ if isValidKIDSPatchSha1Suffix(fileName): logger.debug("Adding %s KIDS info to dict" % absFilename) kidsFileName = fileName[0:fileName.rfind('.')] self.__addKidsBuildFileToDict__(kidsFileName, absFilename, KIDS_BUILD_FILE_TYPE_SHA1) continue """ Handle KIDS Info/Sha1 files """ if ( isValidKIDSInfoSuffix(fileName) or isValidKIDSInfoSha1Suffix(fileName) ): self._kidsInfoFileList.append(absFilename) continue """ Handle Global/Sha1 Files """ if ( isValidGlobalFileSuffix(fileName) or isValidGlobalSha1Suffix(fileName) ): logger.debug("Adding %s Global files to list" % absFilename) self._globalFilesSet.add(absFilename) continue """ handle all csv files """ if isValidCSVSuffix(fileName): if isValidOrderCSVFile(absFilename): self._csvOrderFileList.append(absFilename) continue """ Handle .py files """ if isValidPythonSuffix(fileName): logger.debug("Adding %s python script to list" % absFilename) self._pythonScriptList.append(absFilename) continue logger.info("Total # of KIDS Builds are %d" % len(self._kidsBuildFileDict)) logger.info("Total # of KIDS Info are %d" % len(self._kidsInfoFileList)) logger.info("Total # of Global files are %d" % len(self._globalFilesSet)) logger.info("Total # of Python files are %d" % len(self._pythonScriptList)) logger.info("Total # of CSV files are %d" % len(self._csvOrderFileList))
def __parseAllKIDSBuildFilesList__(self): for basename in self._kidsBuildFileDict.iterkeys(): kidsFile, sha1Path = self._kidsBuildFileDict[basename] if kidsFile == None: logger.error("No KIDS file available for name %s" % basename) continue installNameList, seqNo, kidsBuilds = None, None, None if isValidKIDSPatchHeaderSuffix(kidsFile): from KIDSPatchParser import loadMetaDataFromJSON #continue installNameList, seqNo, kidsBuilds = loadMetaDataFromJSON(kidsFile) else: kidsParser = KIDSPatchParser(None) kidsParser.unregisterSectionHandler(KIDSPatchParser.ROUTINE_SECTION) kidsParser.parseKIDSBuild(kidsFile) installNameList = kidsParser.installNameList logger.debug("install name list is %s" % installNameList) seqNo = kidsParser.seqNo kidsBuilds = kidsParser.kidsBuilds if len(installNameList) > 1: if not self._multiBuildDict.get(kidsFile): self._multiBuildDict[kidsFile] = installNameList else: assert self._multiBuildDict[kidsFile] == installNameList elif seqNo: if installNameList[0] not in self._installNameSeqMap: self._installNameSeqMap[installNameList[0]] = seqNo else: logger.error("Duplicated KIDS build file %s" % kidsFile) for installName in installNameList: if installName in self._kidsInstallNameDict: logger.warn("%s is already in the dict %s" % (installName, kidsFile)) logger.debug("Added installName %s, file %s" % (installName, kidsFile)) self._kidsInstallNameDict[installName] = os.path.normpath(kidsFile) """ handle KIDS sha1 file Path """ if sha1Path: if installName in self._kidsInstallNameSha1Dict: logger.warn("%s is already in the dict %s" % (installName, sha1Path)) self._kidsInstallNameSha1Dict[installName] = sha1Path """ update kids dependency """ if installName in self._kidsDepBuildDict: logger.warn("%s is already has the dep map %s" % (installName, self._kidsDepBuildDict[installName])) if kidsBuilds: for kidsBuild in kidsBuilds: if kidsBuild.installName == installName: depList = kidsBuild.dependencyList if depList: self._kidsDepBuildDict[installName] = set([x[0] for x in depList]) logger.info("%s: %s" % (installName, self._kidsDepBuildDict[installName])) logger.debug("%s" % sorted(self._kidsInstallNameDict.keys())) logger.info("Total # of install name %d" % len(self._kidsInstallNameDict))
def populate(input): packages_csv = csv.DictReader(input) # Parse packages and namespaces from CSV table on stdin. packages = [] pkg = None for fields in packages_csv: if fields['Package Name']: pkg = Package(fields['Package Name'], fields['Directory Name']) packages.append(pkg) if pkg: pkg.add_namespace(fields['Prefixes']) pkg.add_number(fields['File Numbers']) pkg.add_global(fields['Globals']) # Construct "namespace => path" map. namespaces = {} for p in packages: for ns in p.included: namespaces[ns] = p.path for ns in p.excluded: if not namespaces.has_key(ns): namespaces[ns] = None #--------------------------------------------------------------------------- # Collect all KIDS and info files under the current directory recursively #--------------------------------------------------------------------------- curDir = os.getcwd() kidsOrderGen = KIDSPatchOrderGenerator() patchOrder = kidsOrderGen.generatePatchOrder(curDir) patchInfoDict = kidsOrderGen.getPatchInfoDict() patchInfoSet = set(patchInfoDict.keys()) patchList = patchInfoDict.values() noKidsInfoDict = kidsOrderGen.getNoKidsBuildInfoDict() noKidsInfoSet = set(noKidsInfoDict.keys()) noKidsPatchList = noKidsInfoDict.values() leftoverTxtFiles = kidsOrderGen.getInvalidInfoFiles() #--------------------------------------------------------------------------- # place multiBuilds KIDS patch under MultiBuilds directory #--------------------------------------------------------------------------- multiBuildSet = set([x.installName for x in patchList if x.isMultiBuilds]) for info in multiBuildSet: logger.info("Handling Multibuilds Kids %s" % info) patchInfo = patchInfoDict[info] src = patchInfo.kidsFilePath dest = os.path.normpath(os.path.join(curDir, "MultiBuilds", os.path.basename(src))) if src != dest: place(src,dest) if isValidKIDSPatchHeaderSuffix(dest): " add to ignore list if not there" addToGitIgnoreList(dest[0:dest.rfind('.')]) src = patchInfo.kidsSha1Path if not src: continue dest = os.path.normpath(os.path.join(curDir, "MultiBuilds", os.path.basename(src))) if src != dest: place(src,dest) # Map by package namespace (prefix). for ns in sorted(namespaces.keys(),order_long_to_short): path = namespaces[ns] nsPatchList = [x.installName for x in patchList if x.namespace==ns] for patch in nsPatchList: logger.info("Handling Kids %s" % patch) patchInfo = patchInfoDict[patch] patchDir = os.path.join(path, "Patches", installNameToDirName(patch)) placePatchInfo(patchInfo, curDir, patchDir) # Map KIDS Info Files that do not have associated KIDS Build Files nsNoKidsList = [x.installName for x in noKidsPatchList if x.namespace==ns] for patch in nsNoKidsList: logger.info("Handling No Kids info File %s" % patch) patchInfo = noKidsInfoDict[patch] patchDir = os.path.join(path, "Patches", installNameToDirName(patch)) placePatchInfo(patchInfo, curDir, patchDir) patchInfoSet.difference_update(nsPatchList) noKidsInfoSet.difference_update(nsNoKidsList) # Put leftover kids files in Uncategorized package. for patch in patchInfoSet: logger.info("Handling left over Kids File %s" % patch) patchInfo = patchInfoDict[patch] placePatchInfo(patchInfo, curDir, 'Uncategorized') for patch in noKidsInfoSet: logger.info("Handling left over no Kids Info File %s" % patch) patchInfo = noKidsInfoDict[patch] placePatchInfo(patchInfo, curDir, 'Uncategorized') # Put invalid kids info files in Uncategorized package. for src in leftoverTxtFiles: logger.info("Handling left over files: %s" % src) from KIDSAssociatedFilesMapping import getAssociatedInstallName installName = getAssociatedInstallName(src) if installName == "MultiBuilds": # put in Multibuilds directory dest = os.path.normpath(os.path.join(curDir, "MultiBuilds", os.path.basename(src))) if src != dest: place(src,dest) continue dirName = os.path.dirname(src) if not dirName.endswith("Packages"): logger.debug("Do not move %s" % src) continue dest = os.path.normpath(os.path.join(curDir, 'Uncategorized', os.path.basename(src))) if src != dest: place(src,dest)
def __parseAllKIDSBuildFilesList__(self): for basename in self._kidsBuildFileDict.iterkeys(): kidsFile, sha1Path = self._kidsBuildFileDict[basename] if kidsFile == None: logger.error("No KIDS file available for name %s" % basename) continue installNameList, seqNo, kidsBuilds = None, None, None if isValidKIDSPatchHeaderSuffix(kidsFile): from KIDSPatchParser import loadMetaDataFromJSON #continue installNameList, seqNo, kidsBuilds = loadMetaDataFromJSON( kidsFile) else: kidsParser = KIDSPatchParser(None) kidsParser.unregisterSectionHandler( KIDSPatchParser.ROUTINE_SECTION) kidsParser.parseKIDSBuild(kidsFile) installNameList = kidsParser.installNameList logger.debug("install name list is %s" % installNameList) seqNo = kidsParser.seqNo kidsBuilds = kidsParser.kidsBuilds if len(installNameList) > 1: if not self._multiBuildDict.get(kidsFile): self._multiBuildDict[kidsFile] = installNameList else: assert self._multiBuildDict[kidsFile] == installNameList elif seqNo: if installNameList[0] not in self._installNameSeqMap: self._installNameSeqMap[installNameList[0]] = seqNo else: logger.error("Duplicated KIDS build file %s" % kidsFile) for installName in installNameList: if installName in self._kidsInstallNameDict: logger.warn("%s is already in the dict %s" % (installName, kidsFile)) logger.debug("Added installName %s, file %s" % (installName, kidsFile)) self._kidsInstallNameDict[installName] = os.path.normpath( kidsFile) """ handle KIDS sha1 file Path """ if sha1Path: if installName in self._kidsInstallNameSha1Dict: logger.warn("%s is already in the dict %s" % (installName, sha1Path)) self._kidsInstallNameSha1Dict[installName] = sha1Path """ update kids dependency """ if installName in self._kidsDepBuildDict: logger.warn( "%s is already has the dep map %s" % (installName, self._kidsDepBuildDict[installName])) if kidsBuilds: for kidsBuild in kidsBuilds: if kidsBuild.installName == installName: depList = kidsBuild.dependencyList if depList: self._kidsDepBuildDict[installName] = set( [x[0] for x in depList]) logger.info( "%s: %s" % (installName, self._kidsDepBuildDict[installName])) logger.debug("%s" % sorted(self._kidsInstallNameDict.keys())) logger.info("Total # of install name %d" % len(self._kidsInstallNameDict))
def __getAllKIDSBuildInfoAndOtherFileList__(self, patchDir): assert os.path.exists(patchDir) absPatchDir = os.path.abspath(patchDir) for (root, dirs, files) in os.walk(absPatchDir): lastDir = os.path.split(root)[-1] for fileName in files: absFilename = os.path.join(root, fileName) if not isValidPatchRelatedFiles(absFilename, True): continue """ Handle KIDS build files """ if isValidKIDSPatchSuffix(fileName): logger.debug("Adding %s KIDS file to dict" % absFilename) self.__addKidsBuildFileToDict__(fileName, absFilename, KIDS_BUILD_FILE_TYPE_KIDS) continue """ Handle KIDS build HEADER files """ if isValidKIDSPatchHeaderSuffix(fileName): logger.debug("Adding %s KIDS header to dict" % absFilename) kidsFileName = fileName[0:fileName.rfind('.')] self.__addKidsBuildFileToDict__( kidsFileName, absFilename, KIDS_BUILD_FILE_TYPE_HEADER) continue """ Handle KIDS build Sha1 files """ if isValidKIDSPatchSha1Suffix(fileName): logger.debug("Adding %s KIDS info to dict" % absFilename) kidsFileName = fileName[0:fileName.rfind('.')] self.__addKidsBuildFileToDict__(kidsFileName, absFilename, KIDS_BUILD_FILE_TYPE_SHA1) continue """ Handle KIDS Info/Sha1 files """ if (isValidKIDSInfoSuffix(fileName) or isValidKIDSInfoSha1Suffix(fileName)): self._kidsInfoFileList.append(absFilename) continue """ Handle Global/Sha1 Files """ if (isValidGlobalFileSuffix(fileName) or isValidGlobalSha1Suffix(fileName)): logger.debug("Adding %s Global files to list" % absFilename) self._globalFilesSet.add(absFilename) continue """ handle all csv files """ if isValidCSVSuffix(fileName): if isValidOrderCSVFile(absFilename): self._csvOrderFileList.append(absFilename) continue """ Handle .py files """ if isValidPythonSuffix(fileName): logger.debug("Adding %s python script to list" % absFilename) self._pythonScriptList.append(absFilename) continue logger.info("Total # of KIDS Builds are %d" % len(self._kidsBuildFileDict)) logger.info("Total # of KIDS Info are %d" % len(self._kidsInfoFileList)) logger.info("Total # of Global files are %d" % len(self._globalFilesSet)) logger.info("Total # of Python files are %d" % len(self._pythonScriptList)) logger.info("Total # of CSV files are %d" % len(self._csvOrderFileList))
def populate(input): packages, namespaces = populatePackageMapByCSV(input) #--------------------------------------------------------------------------- # Collect all KIDS and info files under the current directory recursively #--------------------------------------------------------------------------- curDir = os.getcwd() kidsOrderGen = KIDSPatchOrderGenerator() patchOrder = kidsOrderGen.generatePatchOrder(curDir) patchInfoDict = kidsOrderGen.getPatchInfoDict() patchInfoSet = set(patchInfoDict.keys()) patchList = patchInfoDict.values() noKidsInfoDict = kidsOrderGen.getNoKidsBuildInfoDict() noKidsInfoSet = set(noKidsInfoDict.keys()) noKidsPatchList = noKidsInfoDict.values() leftoverTxtFiles = kidsOrderGen.getInvalidInfoFiles() #--------------------------------------------------------------------------- # place multiBuilds KIDS patch under MultiBuilds directory #--------------------------------------------------------------------------- multiBuildSet = set([x.installName for x in patchList if x.isMultiBuilds]) for info in multiBuildSet: logger.info("Handling Multibuilds Kids %s" % info) patchInfo = patchInfoDict[info] src = patchInfo.kidsFilePath dest = os.path.normpath( os.path.join(curDir, "MultiBuilds", os.path.basename(src))) if src != dest: place(src, dest) if isValidKIDSPatchHeaderSuffix(dest): " add to ignore list if not there" addToGitIgnoreList(dest[0:dest.rfind('.')]) src = patchInfo.kidsSha1Path if not src: continue dest = os.path.normpath( os.path.join(curDir, "MultiBuilds", os.path.basename(src))) if src != dest: place(src, dest) # Map by package namespace (prefix). for ns in sorted(namespaces.keys(), order_long_to_short): path = namespaces[ns] nsPatchList = [x.installName for x in patchList if x.namespace == ns] for patch in nsPatchList: logger.info("Handling Kids %s" % patch) patchInfo = patchInfoDict[patch] patchDir = os.path.join(path, "Patches", installNameToDirName(patch)) placePatchInfo(patchInfo, curDir, patchDir) # Map KIDS Info Files that do not have associated KIDS Build Files nsNoKidsList = [ x.installName for x in noKidsPatchList if x.namespace == ns ] for patch in nsNoKidsList: logger.info("Handling No Kids info File %s" % patch) patchInfo = noKidsInfoDict[patch] patchDir = os.path.join(path, "Patches", installNameToDirName(patch)) placePatchInfo(patchInfo, curDir, patchDir) patchInfoSet.difference_update(nsPatchList) noKidsInfoSet.difference_update(nsNoKidsList) # Put leftover kids files in Uncategorized package. for patch in patchInfoSet: logger.info("Handling left over Kids File %s" % patch) patchInfo = patchInfoDict[patch] placePatchInfo(patchInfo, curDir, 'Uncategorized') for patch in noKidsInfoSet: logger.info("Handling left over no Kids Info File %s" % patch) patchInfo = noKidsInfoDict[patch] placePatchInfo(patchInfo, curDir, 'Uncategorized') # Put invalid kids info files in Uncategorized package. for src in leftoverTxtFiles: logger.info("Handling left over files: %s" % src) from KIDSAssociatedFilesMapping import getAssociatedInstallName installName = getAssociatedInstallName(src) if installName == "MultiBuilds": # put in Multibuilds directory dest = os.path.normpath( os.path.join(curDir, "MultiBuilds", os.path.basename(src))) if src != dest: place(src, dest) continue dirName = os.path.dirname(src) if not dirName.endswith("Packages"): logger.debug("Do not move %s" % src) continue dest = os.path.normpath( os.path.join(curDir, 'Uncategorized', os.path.basename(src))) if src != dest: place(src, dest)
def populate(input): packages, namespaces = populatePackageMapByCSV(input) #--------------------------------------------------------------------------- # Collect all KIDS and info files under the current directory recursively #--------------------------------------------------------------------------- curDir = os.getcwd() kidsOrderGen = KIDSPatchOrderGenerator() patchOrder = kidsOrderGen.generatePatchOrder(curDir) patchInfoDict = kidsOrderGen.getPatchInfoDict() patchInfoSet = set(patchInfoDict.keys()) patchList = patchInfoDict.values() noKidsInfoDict = kidsOrderGen.getNoKidsBuildInfoDict() noKidsInfoSet = set(noKidsInfoDict.keys()) noKidsPatchList = noKidsInfoDict.values() leftoverTxtFiles = kidsOrderGen.getInvalidInfoFiles() #--------------------------------------------------------------------------- # place multiBuilds KIDS patch under MultiBuilds directory #--------------------------------------------------------------------------- multiBuildSet = set([x.installName for x in patchList if x.isMultiBuilds]) for info in multiBuildSet: logger.info("Handling Multibuilds Kids %s" % info) patchInfo = patchInfoDict[info] src = patchInfo.kidsFilePath dest = os.path.normpath(os.path.join(curDir, "MultiBuilds", os.path.basename(src))) if src != dest: place(src,dest) if isValidKIDSPatchHeaderSuffix(dest): " add to ignore list if not there" addToGitIgnoreList(dest[0:dest.rfind('.')]) src = patchInfo.kidsSha1Path if not src: continue dest = os.path.normpath(os.path.join(curDir, "MultiBuilds", os.path.basename(src))) if src != dest: place(src,dest) # Map by package namespace (prefix). for ns in sorted(namespaces.keys(),order_long_to_short): path = namespaces[ns] nsPatchList = [x.installName for x in patchList if x.namespace==ns] for patch in nsPatchList: logger.info("Handling Kids %s" % patch) patchInfo = patchInfoDict[patch] patchDir = os.path.join(path, "Patches", installNameToDirName(patch)) placePatchInfo(patchInfo, curDir, patchDir) # Map KIDS Info Files that do not have associated KIDS Build Files nsNoKidsList = [x.installName for x in noKidsPatchList if x.namespace==ns] for patch in nsNoKidsList: logger.info("Handling No Kids info File %s" % patch) patchInfo = noKidsInfoDict[patch] patchDir = os.path.join(path, "Patches", installNameToDirName(patch)) placePatchInfo(patchInfo, curDir, patchDir) patchInfoSet.difference_update(nsPatchList) noKidsInfoSet.difference_update(nsNoKidsList) # Put leftover kids files in Uncategorized package. for patch in patchInfoSet: logger.info("Handling left over Kids File %s" % patch) patchInfo = patchInfoDict[patch] placePatchInfo(patchInfo, curDir, 'Uncategorized') for patch in noKidsInfoSet: logger.info("Handling left over no Kids Info File %s" % patch) patchInfo = noKidsInfoDict[patch] placePatchInfo(patchInfo, curDir, 'Uncategorized') # Put invalid kids info files in Uncategorized package. for src in leftoverTxtFiles: logger.info("Handling left over files: %s" % src) from KIDSAssociatedFilesMapping import getAssociatedInstallName installName = getAssociatedInstallName(src) if installName == "MultiBuilds": # put in Multibuilds directory dest = os.path.normpath(os.path.join(curDir, "MultiBuilds", os.path.basename(src))) if src != dest: place(src,dest) continue dirName = os.path.dirname(src) if not dirName.endswith("Packages"): logger.debug("Do not move %s" % src) continue dest = os.path.normpath(os.path.join(curDir, 'Uncategorized', os.path.basename(src))) if src != dest: place(src,dest)