def __getAllKIDSBuildInfoAndOtherFileList__(self, patchDir):
assert os.path.exists(patchDir)
absPatchDir = os.path.abspath(patchDir)
for (root, dirs, files) in os.walk(absPatchDir):
lastDir = os.path.split(root)[-1]
for fileName in files:
absFilename = os.path.join(root, fileName)
if not isValidPatchRelatedFiles(absFilename, True):
continue
""" Handle KIDS build files """
if isValidKIDSPatchSuffix(fileName):
logger.debug("Adding %s KIDS file to dict" % absFilename)
self.__addKidsBuildFileToDict__(fileName, absFilename,
KIDS_BUILD_FILE_TYPE_KIDS)
continue
""" Handle KIDS build HEADER files """
if isValidKIDSPatchHeaderSuffix(fileName):
logger.debug("Adding %s KIDS header to dict" % absFilename)
kidsFileName = fileName[0:fileName.rfind('.')]
self.__addKidsBuildFileToDict__(kidsFileName, absFilename,
KIDS_BUILD_FILE_TYPE_HEADER)
continue
""" Handle KIDS build Sha1 files """
if isValidKIDSPatchSha1Suffix(fileName):
logger.debug("Adding %s KIDS info to dict" % absFilename)
kidsFileName = fileName[0:fileName.rfind('.')]
self.__addKidsBuildFileToDict__(kidsFileName, absFilename,
KIDS_BUILD_FILE_TYPE_SHA1)
continue
""" Handle KIDS Info/Sha1 files """
if ( isValidKIDSInfoSuffix(fileName) or
isValidKIDSInfoSha1Suffix(fileName) ):
self._kidsInfoFileList.append(absFilename)
continue
""" Handle Global/Sha1 Files """
if ( isValidGlobalFileSuffix(fileName) or
isValidGlobalSha1Suffix(fileName) ):
logger.debug("Adding %s Global files to list" % absFilename)
self._globalFilesSet.add(absFilename)
continue
""" handle all csv files """
if isValidCSVSuffix(fileName):
if isValidOrderCSVFile(absFilename):
self._csvOrderFileList.append(absFilename)
continue
""" Handle .py files """
if isValidPythonSuffix(fileName):
logger.debug("Adding %s python script to list" % absFilename)
self._pythonScriptList.append(absFilename)
continue
logger.info("Total # of KIDS Builds are %d" % len(self._kidsBuildFileDict))
logger.info("Total # of KIDS Info are %d" % len(self._kidsInfoFileList))
logger.info("Total # of Global files are %d" % len(self._globalFilesSet))
logger.info("Total # of Python files are %d" % len(self._pythonScriptList))
logger.info("Total # of CSV files are %d" % len(self._csvOrderFileList))
def __parseAllKIDSBuildFilesList__(self):
for basename in self._kidsBuildFileDict.iterkeys():
kidsFile, sha1Path = self._kidsBuildFileDict[basename]
if kidsFile == None:
logger.error("No KIDS file available for name %s" % basename)
continue
installNameList, seqNo, kidsBuilds = None, None, None
if isValidKIDSPatchHeaderSuffix(kidsFile):
from KIDSPatchParser import loadMetaDataFromJSON
#continue
installNameList, seqNo, kidsBuilds = loadMetaDataFromJSON(kidsFile)
else:
kidsParser = KIDSPatchParser(None)
kidsParser.unregisterSectionHandler(KIDSPatchParser.ROUTINE_SECTION)
kidsParser.parseKIDSBuild(kidsFile)
installNameList = kidsParser.installNameList
logger.debug("install name list is %s" % installNameList)
seqNo = kidsParser.seqNo
kidsBuilds = kidsParser.kidsBuilds
if len(installNameList) > 1:
if not self._multiBuildDict.get(kidsFile):
self._multiBuildDict[kidsFile] = installNameList
else:
assert self._multiBuildDict[kidsFile] == installNameList
elif seqNo:
if installNameList[0] not in self._installNameSeqMap:
self._installNameSeqMap[installNameList[0]] = seqNo
else:
logger.error("Duplicated KIDS build file %s" % kidsFile)
for installName in installNameList:
if installName in self._kidsInstallNameDict:
logger.warn("%s is already in the dict %s" % (installName, kidsFile))
logger.debug("Added installName %s, file %s" % (installName, kidsFile))
self._kidsInstallNameDict[installName] = os.path.normpath(kidsFile)
""" handle KIDS sha1 file Path """
if sha1Path:
if installName in self._kidsInstallNameSha1Dict:
logger.warn("%s is already in the dict %s" % (installName, sha1Path))
self._kidsInstallNameSha1Dict[installName] = sha1Path
""" update kids dependency """
if installName in self._kidsDepBuildDict:
logger.warn("%s is already has the dep map %s" %
(installName, self._kidsDepBuildDict[installName]))
if kidsBuilds:
for kidsBuild in kidsBuilds:
if kidsBuild.installName == installName:
depList = kidsBuild.dependencyList
if depList:
self._kidsDepBuildDict[installName] = set([x[0] for x in
depList])
logger.info("%s: %s" % (installName,
self._kidsDepBuildDict[installName]))
logger.debug("%s" % sorted(self._kidsInstallNameDict.keys()))
logger.info("Total # of install name %d" % len(self._kidsInstallNameDict))
def populate(input):
packages_csv = csv.DictReader(input)
# Parse packages and namespaces from CSV table on stdin.
packages = []
pkg = None
for fields in packages_csv:
if fields['Package Name']:
pkg = Package(fields['Package Name'], fields['Directory Name'])
packages.append(pkg)
if pkg:
pkg.add_namespace(fields['Prefixes'])
pkg.add_number(fields['File Numbers'])
pkg.add_global(fields['Globals'])
# Construct "namespace => path" map.
namespaces = {}
for p in packages:
for ns in p.included:
namespaces[ns] = p.path
for ns in p.excluded:
if not namespaces.has_key(ns):
namespaces[ns] = None
#---------------------------------------------------------------------------
# Collect all KIDS and info files under the current directory recursively
#---------------------------------------------------------------------------
curDir = os.getcwd()
kidsOrderGen = KIDSPatchOrderGenerator()
patchOrder = kidsOrderGen.generatePatchOrder(curDir)
patchInfoDict = kidsOrderGen.getPatchInfoDict()
patchInfoSet = set(patchInfoDict.keys())
patchList = patchInfoDict.values()
noKidsInfoDict = kidsOrderGen.getNoKidsBuildInfoDict()
noKidsInfoSet = set(noKidsInfoDict.keys())
noKidsPatchList = noKidsInfoDict.values()
leftoverTxtFiles = kidsOrderGen.getInvalidInfoFiles()
#---------------------------------------------------------------------------
# place multiBuilds KIDS patch under MultiBuilds directory
#---------------------------------------------------------------------------
multiBuildSet = set([x.installName for x in patchList if x.isMultiBuilds])
for info in multiBuildSet:
logger.info("Handling Multibuilds Kids %s" % info)
patchInfo = patchInfoDict[info]
src = patchInfo.kidsFilePath
dest = os.path.normpath(os.path.join(curDir, "MultiBuilds",
os.path.basename(src)))
if src != dest:
place(src,dest)
if isValidKIDSPatchHeaderSuffix(dest):
" add to ignore list if not there"
addToGitIgnoreList(dest[0:dest.rfind('.')])
src = patchInfo.kidsSha1Path
if not src: continue
dest = os.path.normpath(os.path.join(curDir, "MultiBuilds",
os.path.basename(src)))
if src != dest:
place(src,dest)
# Map by package namespace (prefix).
for ns in sorted(namespaces.keys(),order_long_to_short):
path = namespaces[ns]
nsPatchList = [x.installName for x in patchList if x.namespace==ns]
for patch in nsPatchList:
logger.info("Handling Kids %s" % patch)
patchInfo = patchInfoDict[patch]
patchDir = os.path.join(path, "Patches", installNameToDirName(patch))
placePatchInfo(patchInfo, curDir, patchDir)
# Map KIDS Info Files that do not have associated KIDS Build Files
nsNoKidsList = [x.installName for x in noKidsPatchList if x.namespace==ns]
for patch in nsNoKidsList:
logger.info("Handling No Kids info File %s" % patch)
patchInfo = noKidsInfoDict[patch]
patchDir = os.path.join(path, "Patches", installNameToDirName(patch))
placePatchInfo(patchInfo, curDir, patchDir)
patchInfoSet.difference_update(nsPatchList)
noKidsInfoSet.difference_update(nsNoKidsList)
# Put leftover kids files in Uncategorized package.
for patch in patchInfoSet:
logger.info("Handling left over Kids File %s" % patch)
patchInfo = patchInfoDict[patch]
placePatchInfo(patchInfo, curDir, 'Uncategorized')
for patch in noKidsInfoSet:
logger.info("Handling left over no Kids Info File %s" % patch)
patchInfo = noKidsInfoDict[patch]
placePatchInfo(patchInfo, curDir, 'Uncategorized')
# Put invalid kids info files in Uncategorized package.
for src in leftoverTxtFiles:
logger.info("Handling left over files: %s" % src)
from KIDSAssociatedFilesMapping import getAssociatedInstallName
installName = getAssociatedInstallName(src)
if installName == "MultiBuilds": # put in Multibuilds directory
dest = os.path.normpath(os.path.join(curDir, "MultiBuilds",
os.path.basename(src)))
if src != dest:
place(src,dest)
continue
dirName = os.path.dirname(src)
if not dirName.endswith("Packages"):
logger.debug("Do not move %s" % src)
continue
dest = os.path.normpath(os.path.join(curDir, 'Uncategorized',
os.path.basename(src)))
if src != dest:
place(src,dest)
def __parseAllKIDSBuildFilesList__(self):
for basename in self._kidsBuildFileDict.iterkeys():
kidsFile, sha1Path = self._kidsBuildFileDict[basename]
if kidsFile == None:
logger.error("No KIDS file available for name %s" % basename)
continue
installNameList, seqNo, kidsBuilds = None, None, None
if isValidKIDSPatchHeaderSuffix(kidsFile):
from KIDSPatchParser import loadMetaDataFromJSON
#continue
installNameList, seqNo, kidsBuilds = loadMetaDataFromJSON(
kidsFile)
else:
kidsParser = KIDSPatchParser(None)
kidsParser.unregisterSectionHandler(
KIDSPatchParser.ROUTINE_SECTION)
kidsParser.parseKIDSBuild(kidsFile)
installNameList = kidsParser.installNameList
logger.debug("install name list is %s" % installNameList)
seqNo = kidsParser.seqNo
kidsBuilds = kidsParser.kidsBuilds
if len(installNameList) > 1:
if not self._multiBuildDict.get(kidsFile):
self._multiBuildDict[kidsFile] = installNameList
else:
assert self._multiBuildDict[kidsFile] == installNameList
elif seqNo:
if installNameList[0] not in self._installNameSeqMap:
self._installNameSeqMap[installNameList[0]] = seqNo
else:
logger.error("Duplicated KIDS build file %s" % kidsFile)
for installName in installNameList:
if installName in self._kidsInstallNameDict:
logger.warn("%s is already in the dict %s" %
(installName, kidsFile))
logger.debug("Added installName %s, file %s" %
(installName, kidsFile))
self._kidsInstallNameDict[installName] = os.path.normpath(
kidsFile)
""" handle KIDS sha1 file Path """
if sha1Path:
if installName in self._kidsInstallNameSha1Dict:
logger.warn("%s is already in the dict %s" %
(installName, sha1Path))
self._kidsInstallNameSha1Dict[installName] = sha1Path
""" update kids dependency """
if installName in self._kidsDepBuildDict:
logger.warn(
"%s is already has the dep map %s" %
(installName, self._kidsDepBuildDict[installName]))
if kidsBuilds:
for kidsBuild in kidsBuilds:
if kidsBuild.installName == installName:
depList = kidsBuild.dependencyList
if depList:
self._kidsDepBuildDict[installName] = set(
[x[0] for x in depList])
logger.info(
"%s: %s" %
(installName,
self._kidsDepBuildDict[installName]))
logger.debug("%s" % sorted(self._kidsInstallNameDict.keys()))
logger.info("Total # of install name %d" %
len(self._kidsInstallNameDict))
def __getAllKIDSBuildInfoAndOtherFileList__(self, patchDir):
assert os.path.exists(patchDir)
absPatchDir = os.path.abspath(patchDir)
for (root, dirs, files) in os.walk(absPatchDir):
lastDir = os.path.split(root)[-1]
for fileName in files:
absFilename = os.path.join(root, fileName)
if not isValidPatchRelatedFiles(absFilename, True):
continue
""" Handle KIDS build files """
if isValidKIDSPatchSuffix(fileName):
logger.debug("Adding %s KIDS file to dict" % absFilename)
self.__addKidsBuildFileToDict__(fileName, absFilename,
KIDS_BUILD_FILE_TYPE_KIDS)
continue
""" Handle KIDS build HEADER files """
if isValidKIDSPatchHeaderSuffix(fileName):
logger.debug("Adding %s KIDS header to dict" % absFilename)
kidsFileName = fileName[0:fileName.rfind('.')]
self.__addKidsBuildFileToDict__(
kidsFileName, absFilename, KIDS_BUILD_FILE_TYPE_HEADER)
continue
""" Handle KIDS build Sha1 files """
if isValidKIDSPatchSha1Suffix(fileName):
logger.debug("Adding %s KIDS info to dict" % absFilename)
kidsFileName = fileName[0:fileName.rfind('.')]
self.__addKidsBuildFileToDict__(kidsFileName, absFilename,
KIDS_BUILD_FILE_TYPE_SHA1)
continue
""" Handle KIDS Info/Sha1 files """
if (isValidKIDSInfoSuffix(fileName)
or isValidKIDSInfoSha1Suffix(fileName)):
self._kidsInfoFileList.append(absFilename)
continue
""" Handle Global/Sha1 Files """
if (isValidGlobalFileSuffix(fileName)
or isValidGlobalSha1Suffix(fileName)):
logger.debug("Adding %s Global files to list" %
absFilename)
self._globalFilesSet.add(absFilename)
continue
""" handle all csv files """
if isValidCSVSuffix(fileName):
if isValidOrderCSVFile(absFilename):
self._csvOrderFileList.append(absFilename)
continue
""" Handle .py files """
if isValidPythonSuffix(fileName):
logger.debug("Adding %s python script to list" %
absFilename)
self._pythonScriptList.append(absFilename)
continue
logger.info("Total # of KIDS Builds are %d" %
len(self._kidsBuildFileDict))
logger.info("Total # of KIDS Info are %d" %
len(self._kidsInfoFileList))
logger.info("Total # of Global files are %d" %
len(self._globalFilesSet))
logger.info("Total # of Python files are %d" %
len(self._pythonScriptList))
logger.info("Total # of CSV files are %d" %
len(self._csvOrderFileList))
def populate(input):
packages, namespaces = populatePackageMapByCSV(input)
#---------------------------------------------------------------------------
# Collect all KIDS and info files under the current directory recursively
#---------------------------------------------------------------------------
curDir = os.getcwd()
kidsOrderGen = KIDSPatchOrderGenerator()
patchOrder = kidsOrderGen.generatePatchOrder(curDir)
patchInfoDict = kidsOrderGen.getPatchInfoDict()
patchInfoSet = set(patchInfoDict.keys())
patchList = patchInfoDict.values()
noKidsInfoDict = kidsOrderGen.getNoKidsBuildInfoDict()
noKidsInfoSet = set(noKidsInfoDict.keys())
noKidsPatchList = noKidsInfoDict.values()
leftoverTxtFiles = kidsOrderGen.getInvalidInfoFiles()
#---------------------------------------------------------------------------
# place multiBuilds KIDS patch under MultiBuilds directory
#---------------------------------------------------------------------------
multiBuildSet = set([x.installName for x in patchList if x.isMultiBuilds])
for info in multiBuildSet:
logger.info("Handling Multibuilds Kids %s" % info)
patchInfo = patchInfoDict[info]
src = patchInfo.kidsFilePath
dest = os.path.normpath(
os.path.join(curDir, "MultiBuilds", os.path.basename(src)))
if src != dest:
place(src, dest)
if isValidKIDSPatchHeaderSuffix(dest):
" add to ignore list if not there"
addToGitIgnoreList(dest[0:dest.rfind('.')])
src = patchInfo.kidsSha1Path
if not src: continue
dest = os.path.normpath(
os.path.join(curDir, "MultiBuilds", os.path.basename(src)))
if src != dest:
place(src, dest)
# Map by package namespace (prefix).
for ns in sorted(namespaces.keys(), order_long_to_short):
path = namespaces[ns]
nsPatchList = [x.installName for x in patchList if x.namespace == ns]
for patch in nsPatchList:
logger.info("Handling Kids %s" % patch)
patchInfo = patchInfoDict[patch]
patchDir = os.path.join(path, "Patches",
installNameToDirName(patch))
placePatchInfo(patchInfo, curDir, patchDir)
# Map KIDS Info Files that do not have associated KIDS Build Files
nsNoKidsList = [
x.installName for x in noKidsPatchList if x.namespace == ns
]
for patch in nsNoKidsList:
logger.info("Handling No Kids info File %s" % patch)
patchInfo = noKidsInfoDict[patch]
patchDir = os.path.join(path, "Patches",
installNameToDirName(patch))
placePatchInfo(patchInfo, curDir, patchDir)
patchInfoSet.difference_update(nsPatchList)
noKidsInfoSet.difference_update(nsNoKidsList)
# Put leftover kids files in Uncategorized package.
for patch in patchInfoSet:
logger.info("Handling left over Kids File %s" % patch)
patchInfo = patchInfoDict[patch]
placePatchInfo(patchInfo, curDir, 'Uncategorized')
for patch in noKidsInfoSet:
logger.info("Handling left over no Kids Info File %s" % patch)
patchInfo = noKidsInfoDict[patch]
placePatchInfo(patchInfo, curDir, 'Uncategorized')
# Put invalid kids info files in Uncategorized package.
for src in leftoverTxtFiles:
logger.info("Handling left over files: %s" % src)
from KIDSAssociatedFilesMapping import getAssociatedInstallName
installName = getAssociatedInstallName(src)
if installName == "MultiBuilds": # put in Multibuilds directory
dest = os.path.normpath(
os.path.join(curDir, "MultiBuilds", os.path.basename(src)))
if src != dest:
place(src, dest)
continue
dirName = os.path.dirname(src)
if not dirName.endswith("Packages"):
logger.debug("Do not move %s" % src)
continue
dest = os.path.normpath(
os.path.join(curDir, 'Uncategorized', os.path.basename(src)))
if src != dest:
place(src, dest)
def populate(input):
packages, namespaces = populatePackageMapByCSV(input)
#---------------------------------------------------------------------------
# Collect all KIDS and info files under the current directory recursively
#---------------------------------------------------------------------------
curDir = os.getcwd()
kidsOrderGen = KIDSPatchOrderGenerator()
patchOrder = kidsOrderGen.generatePatchOrder(curDir)
patchInfoDict = kidsOrderGen.getPatchInfoDict()
patchInfoSet = set(patchInfoDict.keys())
patchList = patchInfoDict.values()
noKidsInfoDict = kidsOrderGen.getNoKidsBuildInfoDict()
noKidsInfoSet = set(noKidsInfoDict.keys())
noKidsPatchList = noKidsInfoDict.values()
leftoverTxtFiles = kidsOrderGen.getInvalidInfoFiles()
#---------------------------------------------------------------------------
# place multiBuilds KIDS patch under MultiBuilds directory
#---------------------------------------------------------------------------
multiBuildSet = set([x.installName for x in patchList if x.isMultiBuilds])
for info in multiBuildSet:
logger.info("Handling Multibuilds Kids %s" % info)
patchInfo = patchInfoDict[info]
src = patchInfo.kidsFilePath
dest = os.path.normpath(os.path.join(curDir, "MultiBuilds",
os.path.basename(src)))
if src != dest:
place(src,dest)
if isValidKIDSPatchHeaderSuffix(dest):
" add to ignore list if not there"
addToGitIgnoreList(dest[0:dest.rfind('.')])
src = patchInfo.kidsSha1Path
if not src: continue
dest = os.path.normpath(os.path.join(curDir, "MultiBuilds",
os.path.basename(src)))
if src != dest:
place(src,dest)
# Map by package namespace (prefix).
for ns in sorted(namespaces.keys(),order_long_to_short):
path = namespaces[ns]
nsPatchList = [x.installName for x in patchList if x.namespace==ns]
for patch in nsPatchList:
logger.info("Handling Kids %s" % patch)
patchInfo = patchInfoDict[patch]
patchDir = os.path.join(path, "Patches", installNameToDirName(patch))
placePatchInfo(patchInfo, curDir, patchDir)
# Map KIDS Info Files that do not have associated KIDS Build Files
nsNoKidsList = [x.installName for x in noKidsPatchList if x.namespace==ns]
for patch in nsNoKidsList:
logger.info("Handling No Kids info File %s" % patch)
patchInfo = noKidsInfoDict[patch]
patchDir = os.path.join(path, "Patches", installNameToDirName(patch))
placePatchInfo(patchInfo, curDir, patchDir)
patchInfoSet.difference_update(nsPatchList)
noKidsInfoSet.difference_update(nsNoKidsList)
# Put leftover kids files in Uncategorized package.
for patch in patchInfoSet:
logger.info("Handling left over Kids File %s" % patch)
patchInfo = patchInfoDict[patch]
placePatchInfo(patchInfo, curDir, 'Uncategorized')
for patch in noKidsInfoSet:
logger.info("Handling left over no Kids Info File %s" % patch)
patchInfo = noKidsInfoDict[patch]
placePatchInfo(patchInfo, curDir, 'Uncategorized')
# Put invalid kids info files in Uncategorized package.
for src in leftoverTxtFiles:
logger.info("Handling left over files: %s" % src)
from KIDSAssociatedFilesMapping import getAssociatedInstallName
installName = getAssociatedInstallName(src)
if installName == "MultiBuilds": # put in Multibuilds directory
dest = os.path.normpath(os.path.join(curDir, "MultiBuilds",
os.path.basename(src)))
if src != dest:
place(src,dest)
continue
dirName = os.path.dirname(src)
if not dirName.endswith("Packages"):
logger.debug("Do not move %s" % src)
continue
dest = os.path.normpath(os.path.join(curDir, 'Uncategorized',
os.path.basename(src)))
if src != dest:
place(src,dest)
