def _updateInstallReference(self):
installData = self._glbData['9.7']
output = os.path.join(self.outDir, "install_information.json")
installJSONData = {}
packageList = self._crossRef.getAllPackages()
patchOrderGen = PatchOrderGenerator()
patchOrderGen.analyzeVistAPatchDir(self.patchDir +"/Packages")
with open(output, 'w') as installDataOut:
logging.warn("inside the _updateInstallReference")
for ien in sorted(installData.dataEntries.keys(), key=lambda x: float(x)):
installItem = {}
installEntry = installData.dataEntries[ien]
package = self._findInstallPackage(packageList, installEntry.name)
# if this is the first time the package is found, add an entry in the install JSON data.
if package not in installJSONData:
installJSONData[package]={}
if installEntry.name:
logging.warn("Gathering info for: %s" % installEntry.name)
installItem['name'] = installEntry.name
installItem['ien'] = installEntry.ien
installItem['label'] = installEntry.name
installItem['value'] = installEntry.name
installItem['parent']= package
if installEntry.name in patchOrderGen._kidsDepBuildDict:
installchildren = []
for child in patchOrderGen._kidsDepBuildDict[installEntry.name]:
childPackage = self._findInstallPackage(packageList,child)
installchildren.append({"name": child, "package": childPackage});
installItem['children'] = installchildren
if '11' in installEntry.fields:
installItem['installDate'] = installEntry.fields['11'].value.strftime("%Y-%m-%d")
if '1' in installEntry.fields:
installItem['packageLink'] = installEntry.fields['1'].value
if '40' in installEntry.fields:
installItem['numRoutines'] = len(installEntry.fields['40'].value.dataEntries)
if '14' in installEntry.fields:
installItem['numFiles'] = len(installEntry.fields['14'].value.dataEntries)
# Checks for the absence of asterisks which usually denotes a package change.
testMatch = re.search("\*+",installEntry.name)
if testMatch is None:
installItem['packageSwitch'] = True
installJSONData[package][installEntry.name] = installItem
installJSONData['MultiBuild']={}
for multiBuildFile in patchOrderGen._multiBuildDict:
multibuildItem = {}
multibuildItem['name']=os.path.basename(multiBuildFile);
multibuildItem['children'] = []
for installName in patchOrderGen._multiBuildDict[multiBuildFile]:
package = self._findInstallPackage(packageList, installName)
multibuildItem['children'].append({"name": installName, "package": package});
installJSONData['MultiBuild'][os.path.basename(multiBuildFile)] = multibuildItem
logging.warn("About to dump data into %s" % output)
json.dump(installJSONData,installDataOut)
def __init__(self, testClient, logFileDir):
self._testClient = testClient
curTimestamp = getCurrentTimestamp()
logFileName = "%s.%s" % (self.DEFAULT_VISTA_LOG_FILENAME, curTimestamp)
self._logFileName = os.path.join(logFileDir, logFileName)
self._testClient.setLogFile(self._logFileName)
self._patchOrderGen = PatchOrderGenerator()
self._vistaPatchInfo = VistAPackageInfoFetcher(testClient)
self._outPatchList = []
self._patchSet = set()
initFileLogging(os.path.join(logFileDir, self.DEFAULT_OUTPUT_FILE_LOG))
def populate(input):
packages, namespaces = populatePackageMapByCSV(input)
#---------------------------------------------------------------------------
# Collect all KIDS and info files under the current directory recursively
#---------------------------------------------------------------------------
curDir = os.getcwd()
patchOrderGen = PatchOrderGenerator()
patchOrder = patchOrderGen.generatePatchOrder(curDir)
patchInfoDict = patchOrderGen.getPatchInfoDict()
patchInfoSet = set(patchInfoDict.keys())
patchList = patchInfoDict.values()
noKidsInfoDict = patchOrderGen.getNoKidsBuildInfoDict()
noKidsInfoSet = set(noKidsInfoDict.keys())
noKidsPatchList = noKidsInfoDict.values()
leftoverTxtFiles = patchOrderGen.getInvalidInfoFiles()
#---------------------------------------------------------------------------
# place multiBuilds KIDS Build under MultiBuilds directory
#---------------------------------------------------------------------------
multiBuildSet = set([x.installName for x in patchList if x.isMultiBuilds])
for info in multiBuildSet:
logger.info("Handling Multibuilds Kids %s" % info)
patchInfo = patchInfoDict[info]
src = patchInfo.kidsFilePath
dest = os.path.normpath(os.path.join(curDir, "MultiBuilds",
os.path.basename(src)))
if src != dest:
place(src,dest)
if isValidKIDSBuildHeaderSuffix(dest):
" add to ignore list if not there"
addToGitIgnoreList(dest[0:dest.rfind('.')])
src = patchInfo.kidsSha1Path
if not src: continue
dest = os.path.normpath(os.path.join(curDir, "MultiBuilds",
os.path.basename(src)))
if src != dest:
place(src,dest)
# Map by package namespace (prefix).
for ns in sorted(namespaces.keys(),order_long_to_short):
path = namespaces[ns]
nsPatchList = [x.installName for x in patchList if x.namespace==ns]
for patch in nsPatchList:
logger.info("Handling Kids %s" % patch)
patchInfo = patchInfoDict[patch]
patchDir = os.path.join(path, "Patches", installNameToDirName(patch))
placePatchInfo(patchInfo, curDir, patchDir)
# Map KIDS Info Files that do not have associated KIDS Build Files
nsNoKidsList = [x.installName for x in noKidsPatchList if x.namespace==ns]
for patch in nsNoKidsList:
logger.info("Handling No Kids info File %s" % patch)
patchInfo = noKidsInfoDict[patch]
patchDir = os.path.join(path, "Patches", installNameToDirName(patch))
placePatchInfo(patchInfo, curDir, patchDir)
patchInfoSet.difference_update(nsPatchList)
noKidsInfoSet.difference_update(nsNoKidsList)
# Put leftover kids files in Uncategorized package.
for patch in patchInfoSet:
logger.info("Handling left over Kids File %s" % patch)
patchInfo = patchInfoDict[patch]
placePatchInfo(patchInfo, curDir, 'Uncategorized')
for patch in noKidsInfoSet:
logger.info("Handling left over no Kids Info File %s" % patch)
patchInfo = noKidsInfoDict[patch]
placePatchInfo(patchInfo, curDir, 'Uncategorized')
# Put invalid kids info files in Uncategorized package.
for src in leftoverTxtFiles:
logger.info("Handling left over files: %s" % src)
from KIDSAssociatedFilesMapping import getAssociatedInstallName
installName = getAssociatedInstallName(src)
if installName == "MultiBuilds": # put in Multibuilds directory
dest = os.path.normpath(os.path.join(curDir, "MultiBuilds",
os.path.basename(src)))
if src != dest:
place(src,dest)
continue
dirName = os.path.dirname(src)
if not dirName.endswith("Packages"):
logger.debug("Do not move %s" % src)
continue
dest = os.path.normpath(os.path.join(curDir, 'Uncategorized',
os.path.basename(src)))
if src != dest:
place(src,dest)
def _updateInstallReference(self):
installData = self._glbData['9.7']
output = os.path.join(self.outDir, "install_information.json")
installJSONData = {}
packageList = self._crossRef.getAllPackages()
patchOrderGen = PatchOrderGenerator()
patchOrderGen.analyzeVistAPatchDir(self.patchDir + "/Packages")
with open(output, 'w') as installDataOut:
logging.warn("inside the _updateInstallReference")
for ien in sorted(installData.dataEntries.keys(),
key=lambda x: float(x)):
installItem = {}
installEntry = installData.dataEntries[ien]
package = self._findInstallPackage(packageList,
installEntry.name)
# if this is the first time the package is found, add an entry in the install JSON data.
if package not in installJSONData:
installJSONData[package] = {}
if installEntry.name:
logging.warn("Gathering info for: %s" % installEntry.name)
installItem['name'] = installEntry.name
installItem['ien'] = installEntry.ien
installItem['label'] = installEntry.name
installItem['value'] = len(installJSONData[package])
if installEntry.name in patchOrderGen._kidsDepBuildDict:
installchildren = []
for child in patchOrderGen._kidsDepBuildDict[
installEntry.name]:
childPackage = self._findInstallPackage(
packageList, child)
installchildren.append({
"name": child,
"package": childPackage
})
installItem['children'] = installchildren
if '11' in installEntry.fields:
installItem['installDate'] = installEntry.fields[
'11'].value.strftime("%Y-%m-%d")
if '1' in installEntry.fields:
installItem['packageLink'] = installEntry.fields[
'1'].value
if '40' in installEntry.fields:
installItem['numRoutines'] = len(
installEntry.fields['40'].value.dataEntries)
if '14' in installEntry.fields:
installItem['numFiles'] = len(
installEntry.fields['14'].value.dataEntries)
# Checks for the absence of asterisks which usually denotes a package change.
testMatch = re.search("\*+", installEntry.name)
if testMatch is None:
installItem['packageSwitch'] = True
installJSONData[package][installEntry.name] = installItem
installJSONData['MultiBuild'] = {}
for multiBuildFile in patchOrderGen._multiBuildDict:
multibuildItem = {}
multibuildItem['name'] = os.path.basename(multiBuildFile)
multibuildItem['children'] = []
for installName in patchOrderGen._multiBuildDict[
multiBuildFile]:
package = self._findInstallPackage(packageList,
installName)
multibuildItem['children'].append({
"name": installName,
"package": package
})
installJSONData['MultiBuild'][os.path.basename(
multiBuildFile)] = multibuildItem
logging.warn("About to dump data into %s" % output)
json.dump(installJSONData, installDataOut)
def _updateInstallReference(self):
if not os.path.exists(self.outdir + "/9_7"):
os.mkdir(self.outdir + "/9_7")
installData = self._glbData['9.7']
output = os.path.join(self.outdir, "install_information.json")
installJSONData = {}
packageList = self._crossRef.getAllPackages()
patchOrderGen = PatchOrderGenerator()
patchOrderGen.analyzeVistAPatchDir(self.patchDir + "/Packages")
with open(output, 'w') as installDataOut:
logging.warn("inside the _updateInstallReference")
for ien in sorted(installData.dataEntries.keys(),
key=lambda x: float(x)):
installItem = {}
installEntry = installData.dataEntries[ien]
package = self._findInstallPackage(packageList,
installEntry.name)
# if this is the first time the package is found, add an entry in the install JSON data.
if package not in installJSONData:
installJSONData[package] = {}
if installEntry.name:
logging.warn("Gathering info for: %s" % installEntry.name)
installItem['name'] = installEntry.name
installItem['ien'] = installEntry.ien
installItem['label'] = installEntry.name
installItem['value'] = installEntry.name
installItem['parent'] = package
if installEntry.name in patchOrderGen._kidsDepBuildDict:
installchildren = []
for child in patchOrderGen._kidsDepBuildDict[
installEntry.name]:
childPackage = self._findInstallPackage(
packageList, child)
installchildren.append({
"name": child,
"package": childPackage
})
installItem['children'] = installchildren
if '11' in installEntry.fields:
installItem['installDate'] = installEntry.fields[
'11'].value.strftime("%Y-%m-%d")
if '1' in installEntry.fields:
installItem['packageLink'] = installEntry.fields[
'1'].value
if '40' in installEntry.fields:
installItem['numRoutines'] = len(
installEntry.fields['40'].value.dataEntries)
if '14' in installEntry.fields:
installItem['numFiles'] = len(
installEntry.fields['14'].value.dataEntries)
# Checks for the absence of asterisks which usually denotes a package change, also make it more specific to
# eliminate the multibuilds that are being marked as package changes
testMatch = re.search("\*+", installEntry.name)
if testMatch is None:
# Assume a package switch name will be just a package name and a version
capture = re.match(
"(?P<packageName>[A-Z./ \&\-\']+) (?P<packageVal>[.0-9]+)",
installEntry.name)
if capture:
checkPackage = self._findInstallPackage(
packageList,
capture.groups()[0], False)
if (not (checkPackage == "Unknown")
or (len(capture.groups()[0]) <= 4)):
installItem['packageSwitch'] = True
installJSONData[package][installEntry.name] = installItem
installJSONData['MultiBuild'] = {}
for multiBuildFile in patchOrderGen._multiBuildDict:
multibuildItem = {}
multibuildItem['name'] = os.path.basename(multiBuildFile)
multibuildItem['children'] = []
for installName in patchOrderGen._multiBuildDict[
multiBuildFile]:
package = self._findInstallPackage(packageList,
installName)
multibuildItem['children'].append({
"name": installName,
"package": package
})
installJSONData['MultiBuild'][os.path.basename(
multiBuildFile)] = multibuildItem
logging.warn("About to dump data into %s" % output)
json.dump(installJSONData, installDataOut)
def test_generatePatchOrderTopologicSample(self): patchOrderGen = PatchOrderGenerator() patchOrder = patchOrderGen.generatePatchOrderTopologic(TEST_PACKAGE_DIR) self.verifySampleOrder(patchOrder)
def test_generatePatchOrderTopologic(self): patchOrderGen = PatchOrderGenerator() patchOrder = patchOrderGen.generatePatchOrderTopologic(PACKAGE_DIR) self.assertTrue(patchOrder, "no valid patch order is generated")
def populate(input):
packages, namespaces = populatePackageMapByCSV(input)
#---------------------------------------------------------------------------
# Collect all KIDS and info files under the current directory recursively
#---------------------------------------------------------------------------
curDir = os.getcwd()
patchOrderGen = PatchOrderGenerator()
patchOrder = patchOrderGen.generatePatchOrder(curDir)
patchInfoDict = patchOrderGen.getPatchInfoDict()
patchInfoSet = set(patchInfoDict.keys())
patchList = patchInfoDict.values()
noKidsInfoDict = patchOrderGen.getNoKidsBuildInfoDict()
noKidsInfoSet = set(noKidsInfoDict.keys())
noKidsPatchList = noKidsInfoDict.values()
leftoverTxtFiles = patchOrderGen.getInvalidInfoFiles()
#---------------------------------------------------------------------------
# place multiBuilds KIDS Build under MultiBuilds directory
#---------------------------------------------------------------------------
multiBuildSet = set([x.installName for x in patchList if x.isMultiBuilds])
for info in multiBuildSet:
logger.info("Handling Multibuilds Kids %s" % info)
patchInfo = patchInfoDict[info]
src = patchInfo.kidsFilePath
dest = os.path.normpath(
os.path.join(curDir, "MultiBuilds", os.path.basename(src)))
if src != dest:
place(src, dest)
if isValidKIDSBuildHeaderSuffix(dest):
" add to ignore list if not there"
addToGitIgnoreList(dest[0:dest.rfind('.')])
src = patchInfo.kidsSha1Path
if not src: continue
dest = os.path.normpath(
os.path.join(curDir, "MultiBuilds", os.path.basename(src)))
if src != dest:
place(src, dest)
# Map by package namespace (prefix).
for ns in sorted(namespaces.keys(), order_long_to_short):
path = namespaces[ns]
nsPatchList = [x.installName for x in patchList if x.namespace == ns]
for patch in nsPatchList:
logger.info("Handling Kids %s" % patch)
patchInfo = patchInfoDict[patch]
patchDir = os.path.join(path, "Patches",
installNameToDirName(patch))
placePatchInfo(patchInfo, curDir, patchDir)
# Map KIDS Info Files that do not have associated KIDS Build Files
nsNoKidsList = [
x.installName for x in noKidsPatchList if x.namespace == ns
]
for patch in nsNoKidsList:
logger.info("Handling No Kids info File %s" % patch)
patchInfo = noKidsInfoDict[patch]
patchDir = os.path.join(path, "Patches",
installNameToDirName(patch))
placePatchInfo(patchInfo, curDir, patchDir)
patchInfoSet.difference_update(nsPatchList)
noKidsInfoSet.difference_update(nsNoKidsList)
# Put leftover kids files in Uncategorized package.
for patch in patchInfoSet:
logger.info("Handling left over Kids File %s" % patch)
patchInfo = patchInfoDict[patch]
placePatchInfo(patchInfo, curDir, 'Uncategorized')
for patch in noKidsInfoSet:
logger.info("Handling left over no Kids Info File %s" % patch)
patchInfo = noKidsInfoDict[patch]
placePatchInfo(patchInfo, curDir, 'Uncategorized')
# Put invalid kids info files in Uncategorized package.
for src in leftoverTxtFiles:
logger.info("Handling left over files: %s" % src)
from KIDSAssociatedFilesMapping import getAssociatedInstallName
installName = getAssociatedInstallName(src)
if installName == "MultiBuilds": # put in Multibuilds directory
dest = os.path.normpath(
os.path.join(curDir, "MultiBuilds", os.path.basename(src)))
if src != dest:
place(src, dest)
continue
dirName = os.path.dirname(src)
if not dirName.endswith("Packages"):
logger.debug("Do not move %s" % src)
continue
dest = os.path.normpath(
os.path.join(curDir, 'Uncategorized', os.path.basename(src)))
if src != dest:
place(src, dest)
def test_generatePatchOrderTopologicSample(self): patchOrderGen = PatchOrderGenerator() patchOrder = patchOrderGen.generatePatchOrderTopologic(TEST_PACKAGE_DIR) self.verifySampleOrder(patchOrder)
def test_generatePatchOrderTopologic(self): patchOrderGen = PatchOrderGenerator() patchOrder = patchOrderGen.generatePatchOrderTopologic(PACKAGE_DIR) self.assertTrue(patchOrder, "no valid patch order is generated")
def _updateInstallReference(self):
if not os.path.exists(self.outdir+"/9_7"):
os.mkdir(self.outdir+"/9_7")
installData = self._glbData['9.7']
output = os.path.join(self.outdir, "install_information.json")
installJSONData = {}
packageList = self._crossRef.getAllPackages()
patchOrderGen = PatchOrderGenerator()
patchOrderGen.analyzeVistAPatchDir(self.patchDir +"/Packages")
with open(output, 'w') as installDataOut:
logging.warn("inside the _updateInstallReference")
for ien in sorted(installData.dataEntries.keys(), key=lambda x: float(x)):
installItem = {}
installEntry = installData.dataEntries[ien]
package = self._findInstallPackage(packageList, installEntry.name)
# if this is the first time the package is found, add an entry in the install JSON data.
if package not in installJSONData:
installJSONData[package]={}
if installEntry.name:
logging.warn("Gathering info for: %s" % installEntry.name)
installItem['name'] = installEntry.name
installItem['ien'] = installEntry.ien
installItem['label'] = installEntry.name
installItem['value'] = installEntry.name
installItem['parent']= package
if installEntry.name in patchOrderGen._kidsDepBuildDict:
installchildren = []
for child in patchOrderGen._kidsDepBuildDict[installEntry.name]:
childPackage = self._findInstallPackage(packageList,child)
installchildren.append({"name": child, "package": childPackage});
installItem['children'] = installchildren
if '11' in installEntry.fields:
installItem['installDate'] = installEntry.fields['11'].value.strftime("%Y-%m-%d")
if '1' in installEntry.fields:
installItem['packageLink'] = installEntry.fields['1'].value
if '40' in installEntry.fields:
installItem['numRoutines'] = len(installEntry.fields['40'].value.dataEntries)
if '14' in installEntry.fields:
installItem['numFiles'] = len(installEntry.fields['14'].value.dataEntries)
# Checks for the absence of asterisks which usually denotes a package change, also make it more specific to
# eliminate the multibuilds that are being marked as package changes
testMatch = re.search("\*+",installEntry.name)
if testMatch is None:
# Assume a package switch name will be just a package name and a version
capture = re.match("(?P<packageName>[A-Z./ \&\-\']+) (?P<packageVal>[.0-9]+)",installEntry.name)
if capture:
checkPackage = self._findInstallPackage(packageList, capture.groups()[0],False)
if (not (checkPackage == "Unknown") or (len(capture.groups()[0]) <= 4 )):
installItem['packageSwitch'] = True
installJSONData[package][installEntry.name] = installItem
installJSONData['MultiBuild']={}
for multiBuildFile in patchOrderGen._multiBuildDict:
multibuildItem = {}
multibuildItem['name']=os.path.basename(multiBuildFile);
multibuildItem['children'] = []
for installName in patchOrderGen._multiBuildDict[multiBuildFile]:
package = self._findInstallPackage(packageList, installName)
multibuildItem['children'].append({"name": installName, "package": package});
installJSONData['MultiBuild'][os.path.basename(multiBuildFile)] = multibuildItem
logging.warn("About to dump data into %s" % output)
json.dump(installJSONData,installDataOut)
