def _getListNextPackagesReadyToBuild():
for pkg in Scheduler.listOfPackagesToBuild:
if pkg in Scheduler.listOfPackagesCurrentlyBuilding:
continue
listRequiredSubPackages = list(set(SPECS.getData().getBuildRequiresForPkg(pkg) + \
SPECS.getData().getRequiresAllForPkg(pkg)))
# extend to full Requires tree
for p in listRequiredSubPackages:
reqs = SPECS.getData().getRequiresAllForPkg(p)
for r in reqs:
if r not in listRequiredSubPackages:
listRequiredSubPackages.append(r)
# convert subpackages to basepkg
listRequiredPackages = set()
for p in listRequiredSubPackages:
listRequiredPackages.add(SPECS.getData().getBasePkg(p))
canBuild = True
for reqPkg in listRequiredPackages:
if reqPkg not in Scheduler.listOfAlreadyBuiltPackages:
canBuild = False
break
if canBuild:
Scheduler.listOfPackagesNextToBuild.put((-Scheduler._getPriority(pkg), pkg))
Scheduler.logger.debug("Adding " + pkg + " to the schedule list")
def findRPMFileForGivenPackage(self, package,version="*", index=0):
cmdUtils = CommandUtils()
release = "*"
if version == "*":
version = SPECS.getData().getVersion(package, index)
release = SPECS.getData().getRelease(package, index)
listFoundRPMFiles = sum([cmdUtils.findFile(package + "-" + version + "-" + release + "." +
platform.machine()+".rpm",
constants.rpmPath),
cmdUtils.findFile(package + "-" + version + "-" + release +
".noarch.rpm",
constants.rpmPath)], [])
if constants.inputRPMSPath is not None:
listFoundRPMFiles = sum([cmdUtils.findFile(package + "-" + version + "-" + release +
"." + platform.machine()+".rpm",
constants.inputRPMSPath),
cmdUtils.findFile(package + "-" + version + "-" + release +
".noarch.rpm", constants.inputRPMSPath)],
listFoundRPMFiles)
if len(listFoundRPMFiles) == 1:
return listFoundRPMFiles[0]
if len(listFoundRPMFiles) == 0:
return None
if len(listFoundRPMFiles) > 1:
self.logger.error("Found multiple rpm files for given package in rpm directory." +
"Unable to determine the rpm file for package:" + package)
raise Exception("Multiple rpm files found")
def loadPackagesData(self):
listPackages = SPECS.getData().getListPackages()
listPackages.sort()
cmdUtils = CommandUtils()
for package in listPackages:
for version in SPECS.getData().getVersions(package):
release = SPECS.getData().getRelease(package, version)
listRPMPackages = SPECS.getData().getRPMPackages(package, version)
srpmFileName = package + "-" + version + "-" + release + ".src.rpm"
srpmFiles = cmdUtils.findFile(srpmFileName, constants.sourceRpmPath)
srpmFile = None
if len(srpmFiles) == 1:
srpmFile = srpmFiles[0]
debugrpmFileName = package + "-debuginfo-" + version + "-" + release + "*"
debugrpmFiles = cmdUtils.findFile(debugrpmFileName, constants.rpmPath)
debugrpmFile = None
if len(debugrpmFiles) == 1:
debugrpmFile = debugrpmFiles[0]
pkgUtils = PackageUtils(self.logName, self.logPath)
for rpmPkg in listRPMPackages:
rpmFile = pkgUtils.findRPMFile(rpmPkg, version)
if rpmFile is not None:
listPkgAttributes = {"sourcerpm":srpmFile, "rpm":rpmFile,
"debugrpm":debugrpmFile}
self.pkgList[rpmPkg+"-"+version] = listPkgAttributes
self.logger.debug("Added " + rpmPkg + "-" + version + " to the package info json")
else:
self.logger.debug("Missing rpm file for package:" + rpmPkg)
def _verifyShaAndGetSourcePath(self, source, package, version):
cmdUtils = CommandUtils()
# Fetch/verify sources if sha1 not None.
sha1 = SPECS.getData().getSHA1(package, version, source)
if sha1 is not None:
PullSources.get(package, source, sha1, constants.sourcePath,
constants.getPullSourcesURLs(package), self.logger)
sourcePath = cmdUtils.findFile(source, constants.sourcePath)
if not sourcePath:
sourcePath = cmdUtils.findFile(source, os.path.dirname(SPECS.getData().getSpecFile(package, version)))
if not sourcePath:
if sha1 is None:
self.logger.error("No sha1 found or missing source for " + source)
raise Exception("No sha1 found or missing source for " + source)
else:
self.logger.error("Missing source: " + source +
". Cannot find sources for package: " + package)
raise Exception("Missing source")
else:
if sha1 is None:
self.logger.error("No sha1 found for "+source)
raise Exception("No sha1 found")
if len(sourcePath) > 1:
self.logger.error("Multiple sources found for source:" + source + "\n" +
",".join(sourcePath) +"\nUnable to determine one.")
raise Exception("Multiple sources found")
return sourcePath
def buildRPMSForGivenPackage(self, package, chrootID, destLogPath=None, index=0):
self.logger.info("Building rpm's for package:" + package)
listSourcesFiles = SPECS.getData().getSources(package, index)
listPatchFiles = SPECS.getData().getPatches(package, index)
specFile = SPECS.getData().getSpecFile(package, index)
specName = SPECS.getData().getSpecName(package) + ".spec"
chrootSourcePath = chrootID + constants.topDirPath + "/SOURCES/"
chrootSpecPath = constants.topDirPath + "/SPECS/"
chrootLogsFilePath = chrootID + constants.topDirPath + "/LOGS/" + package + ".log"
chrootCmd = self.runInChrootCommand + " " + chrootID
shutil.copyfile(specFile, chrootID + chrootSpecPath + specName)
# FIXME: some sources are located in SPECS/.. how to mount?
# if os.geteuid()==0:
self._copySourcesTobuildroot(listSourcesFiles, package, chrootSourcePath, index)
self._copySourcesTobuildroot(listPatchFiles, package, chrootSourcePath, index)
macros = []
listAdditionalFiles, macros = self._getAdditionalBuildFiles(package)
self._copyAdditionalBuildFiles(listAdditionalFiles, chrootID)
#Adding rpm macros
listRPMMacros = constants.userDefinedMacros
for macroName, value in listRPMMacros.items():
macros.append(macroName + " " + value)
listRPMFiles = []
listSRPMFiles = []
try:
listRPMFiles, listSRPMFiles = self._buildRPM(chrootSpecPath + specName,
chrootLogsFilePath, chrootCmd,
package, macros)
self.logger.info("Successfully built rpm:" + package)
except Exception as e:
self.logger.error("Failed while building rpm:" + package)
raise e
finally:
if destLogPath is not None:
if (constants.rpmCheck and
package in constants.testForceRPMS and
SPECS.getData().isCheckAvailable(package)):
cmd = ("sed -i '/^Executing(%check):/,/^Processing files:/{//!b};d' " +
chrootLogsFilePath)
logFile = destLogPath + "/adjustTestFile.log"
returnVal = CommandUtils().runCommandInShell(cmd, logFile)
testLogFile = destLogPath + "/" + package + "-test.log"
shutil.copyfile(chrootLogsFilePath, testLogFile)
else:
shutil.copy2(chrootLogsFilePath, destLogPath)
self.logger.info("RPM build is successful")
for rpmFile in listRPMFiles:
self._copyRPM(chrootID + "/" + rpmFile, constants.rpmPath)
for srpmFile in listSRPMFiles:
srpmDestFile = self._copyRPM(chrootID + "/" + srpmFile, constants.sourceRpmPath)
def updateLevels(self, mapDependencies, inPkg, parent, level):
listPackages = SPECS.getData().getPackagesForPkg(inPkg)
for depPkg in SPECS.getData().getRequiresForPkg(inPkg):
if depPkg in listPackages:
continue
if depPkg in mapDependencies and mapDependencies[depPkg] < level + 1:
mapDependencies[depPkg] = level + 1
parent[depPkg] = inPkg
self.updateLevels(mapDependencies, depPkg, parent, mapDependencies[depPkg])
def getBasePackagesRequired(self, pkg):
listBasePackagesRequired=[]
listPackagesRequired = SPECS.getData().getBuildRequiresForPkg(pkg)
listPackagesRequired.extend(SPECS.getData().getRequiresAllForPkg(pkg))
for p in listPackagesRequired:
basePkg = SPECS.getData().getBasePkg(p)
if basePkg not in listBasePackagesRequired:
listBasePackagesRequired.append(basePkg)
return listBasePackagesRequired
def _checkIfPackageIsAlreadyBuilt(self):
basePkg = SPECS.getData().getSpecName(self.package)
listRPMPackages = SPECS.getData().getRPMPackages(basePkg)
packageIsAlreadyBuilt = True
pkgUtils = PackageUtils(self.logName, self.logPath)
for pkg in listRPMPackages:
if pkgUtils.findRPMFileForGivenPackage(pkg) is None:
packageIsAlreadyBuilt = False
break
return packageIsAlreadyBuilt
def findTotalWhoNeeds(self, depList, whoNeeds):
while depList:
pkg = depList.pop(0)
for depPackage in SPECS.getData().getListPackages():
for version in SPECS.getData().getVersions(depPackage):
depBasePkg = depPackage+"-"+version
if depBasePkg in whoNeeds:
continue
if pkg in self.getBasePackagesRequired(depBasePkg):
whoNeeds.append(depBasePkg)
if depBasePkg not in depList:
depList.append(depBasePkg)
def calculateSpecDependency(self, inputPackages, mapDependencies, parent):
depQue = queue.Queue()
for package in inputPackages:
if SPECS.getData().isRPMPackage(package):
version = SPECS.getData().getHighestVersion(package)
pkg = package+"-"+version
if pkg not in mapDependencies:
mapDependencies[pkg] = 0
parent[pkg] = ""
depQue.put(pkg)
self.findTotalRequires(mapDependencies, depQue, parent)
else:
self.logger.info("Could not find spec for " + package)
def buildSRPMList(srpmPath, yamlDir, blackListPkgs, dist_tag, logger, singleFile=True):
cmdUtils = CommandUtils()
yamlSrpmDir = os.path.join(yamlDir, "yaml_srpms")
if not os.path.isdir(yamlSrpmDir):
cmdUtils.runCommandInShell("mkdir -p " + yamlSrpmDir)
if singleFile:
yamlFile = open(yamlSrpmDir + "/srpm_list.yaml", "w")
listPackages = SPECS.getData().getListPackages()
listPackages.sort()
for package in listPackages:
if package in blackListPkgs:
continue
ossname = package
for ossversion in SPECS.getData().getVersions(package):
ossrelease = SPECS.getData().getRelease(package, ossversion)
srpm_file_name = "%s-%s-%s%s.src.rpm" % (ossname, ossversion, ossrelease, dist_tag)
logger.info("srpm name is %s" % (srpm_file_name))
listFoundSRPMFiles = cmdUtils.findFile(srpm_file_name, srpmPath)
srpmName = None
if len(listFoundSRPMFiles) == 1:
srpmFullPath = listFoundSRPMFiles[0]
srpmName = os.path.basename(srpmFullPath)
cpcmd = "cp " + srpmFullPath + " " + yamlSrpmDir + "/"
returnVal = cmdUtils.runCommandInShell(cpcmd)
if returnVal != 0:
logger.error("Copy SRPM File is failed for package:" + ossname)
else:
logger.error("SRPM file is not found:" + ossname)
if not singleFile:
yamlFile = open(yamlSrpmDir + "/" + ossname + "-" + ossversion + "-"
+ ossrelease + ".yaml", "w")
yamlFile.write("baseos:" + ossname + ":" + ossversion + "-" + ossrelease + dist_tag + ":\n")
yamlFile.write(" repository: BaseOS\n")
yamlFile.write(" name: '" + ossname + "'\n")
yamlFile.write(" version: '" + ossversion + "-" + ossrelease + dist_tag +"'\n")
yamlFile.write(" url: 'http://www.vmware.com'\n")
yamlFile.write(" baseos-style: rpm\n")
yamlFile.write(" baseos-source: '" + str(srpmName) + "'\n")
yamlFile.write(" baseos-osname: 'photon'\n")
yamlFile.write("\n")
if not singleFile:
yamlFile.close()
if singleFile:
yamlFile.close()
logger.debug("Generated SRPM yaml files for all packages")
def verifyShaAndGetSourcePath(self, source, package):
cmdUtils = CommandUtils()
# Fetch/verify sources if sha1 not None.
sha1 = SPECS.getData().getSHA1(package, source)
if sha1 is not None:
PullSources.get(source, sha1, constants.sourcePath,
constants.pullsourcesConfig, self.logger)
sourcePath = cmdUtils.findFile(source, constants.sourcePath)
if sourcePath is None or len(sourcePath) == 0:
sourcePath = cmdUtils.findFile(source, constants.specPath)
if sourcePath is None or len(sourcePath) == 0:
if sha1 is None:
self.logger.error("No sha1 found or missing source for " +
source)
raise Exception("No sha1 found or missing source for " +
source)
else:
self.logger.error("Missing source: " + source +
". Cannot find sources for package: " +
package)
raise Exception("Missing source")
else:
if sha1 is None:
self.logger.error("No sha1 found for " + source)
raise Exception("No sha1 found")
if len(sourcePath) > 1:
self.logger.error("Multiple sources found for source:" + source +
"\n" + ",".join(sourcePath) +
"\nUnable to determine one.")
raise Exception("Multiple sources found")
return sourcePath
def _calculateParams(self, listPackages):
self.mapCyclesToPackageList.clear()
self.mapPackageToCycle.clear()
self.sortedPackageList = []
self.listOfPackagesAlreadyBuilt = self._readAlreadyAvailablePackages()
updateBuiltRPMSList = False
while not updateBuiltRPMSList:
updateBuiltRPMSList = True
listOfPackagesAlreadyBuilt = list(self.listOfPackagesAlreadyBuilt)
for pkg in listOfPackagesAlreadyBuilt:
listDependentRpmPackages = SPECS.getData().getRequiresAllForPackage(pkg)
needToRebuild = False
for dependentPkg in listDependentRpmPackages:
if dependentPkg not in self.listOfPackagesAlreadyBuilt:
needToRebuild = True
updateBuiltRPMSList = False
if needToRebuild:
self.listOfPackagesAlreadyBuilt.remove(pkg)
listPackagesToBuild = copy.copy(listPackages)
for pkg in listPackages:
if (pkg in self.listOfPackagesAlreadyBuilt and
not constants.rpmCheck):
listPackagesToBuild.remove(pkg)
if not self._readPackageBuildData(listPackagesToBuild):
return False
return True
def adjustGCCSpecs(self, package, chrootID, logPath):
opt = " " + SPECS.getData().getSecurityHardeningOption(package)
cmdUtils = CommandUtils()
cpcmd = "cp " + self.adjustGCCSpecScript + " " + chrootID + "/tmp/" + self.adjustGCCSpecScript
cmd = "/tmp/" + self.adjustGCCSpecScript + opt
logFile = logPath + "/adjustGCCSpecScript.log"
chrootCmd = self.runInChrootCommand + " " + chrootID
returnVal = cmdUtils.runCommandInShell(cpcmd, logFile)
if not returnVal:
self.logger.error("Error during copying the file adjust gcc spec")
raise Exception("Failed while copying adjust gcc spec file")
returnVal = cmdUtils.runCommandInShell(cmd, logFile, chrootCmd)
if returnVal:
return
self.logger.debug(
cmdUtils.runCommandInShell2("ls -la " + chrootID + "/tmp/" +
self.adjustGCCSpecScript))
self.logger.debug(
cmdUtils.runCommandInShell2("lsof " + chrootID + "/tmp/" +
self.adjustGCCSpecScript))
self.logger.debug(cmdUtils.runCommandInShell2("ps ax"))
self.logger.error("Failed while adjusting gcc specs")
raise Exception("Failed while adjusting gcc specs")
def installExtraToolchainRPMS(self, sandbox, packageName, packageVersion):
listOfToolChainPkgs = SPECS.getData().getExtraBuildRequiresForPackage(
packageName, packageVersion)
if not listOfToolChainPkgs:
return
self.logger.debug("Installing package specific toolchain RPMS for " +
packageName + ": " + str(listOfToolChainPkgs))
rpmFiles = ""
packages = ""
for package in listOfToolChainPkgs:
pkgUtils = PackageUtils(self.logName, self.logPath)
if re.match("openjre*", packageName) is not None or re.match(
"openjdk*", packageName):
path = constants.prevPublishXRPMRepo
sandboxPath = "/publishxrpms"
else:
path = constants.prevPublishRPMRepo
sandboxPath = "/publishrpms"
rpmFile = self._findPublishedRPM(package, path)
if rpmFile is None:
self.logger.error("Unable to find rpm " + package +
" in current and previous versions")
raise Exception("Input Error")
rpmFiles += " " + rpmFile.replace(path, sandboxPath)
packages += " " + package
self.logger.debug("Installing custom rpms:" + packages)
cmd = (self.rpmCommand + " -i -v --nodeps --noorder --force " +
rpmFiles)
retVal = sandbox.run(cmd, logfn=self.logger.debug)
if retVal != 0:
self.logger.debug("Command Executed:" + cmd)
self.logger.error("Installing custom toolchains failed")
raise Exception("RPM installation failed")
def _calculateParams(self, listPackages):
self.mapCyclesToPackageList.clear()
self.mapPackageToCycle.clear()
self.sortedPackageList = []
self.listOfPackagesAlreadyBuilt = list(
self._readAlreadyAvailablePackages())
updateBuiltRPMSList = False
while not updateBuiltRPMSList:
updateBuiltRPMSList = True
listOfPackagesAlreadyBuilt = self.listOfPackagesAlreadyBuilt
for pkg in listOfPackagesAlreadyBuilt:
packageName, packageVersion = StringUtils.splitPackageNameAndVersion(
pkg)
listDependentRpmPackages = SPECS.getData(
).getRequiresAllForPackage(packageName, packageVersion)
needToRebuild = False
for dependentPkg in listDependentRpmPackages:
if dependentPkg not in self.listOfPackagesAlreadyBuilt:
needToRebuild = True
updateBuiltRPMSList = False
if needToRebuild:
self.listOfPackagesAlreadyBuilt.remove(pkg)
listPackagesToBuild = copy.copy(listPackages)
for pkg in listPackages:
if (pkg in self.listOfPackagesAlreadyBuilt
and not constants.rpmCheck):
listPackagesToBuild.remove(pkg)
if not self._readPackageBuildData(listPackagesToBuild):
return False
return True
def buildCoreToolChainPackages(self):
self.logger.info("Step 1 : Building the core toolchain packages.....")
self.logger.info(constants.listCoreToolChainPackages)
self.logger.info("")
chroot = None
pkgCount = 0
try:
pkgUtils = PackageUtils(self.logName, self.logPath)
coreToolChainYetToBuild = []
for package in constants.listCoreToolChainPackages:
version = SPECS.getData().getHighestVersion(package)
rpmPkg = pkgUtils.findRPMFile(package, version)
if rpmPkg is not None:
continue
else:
coreToolChainYetToBuild.append(package)
if coreToolChainYetToBuild:
self.logger.info(
"The following core toolchain packages need to be built :")
self.logger.info(coreToolChainYetToBuild)
else:
self.logger.info(
"Core toolchain packages are already available")
for package in coreToolChainYetToBuild:
self.logger.debug("Building core toolchain package : " +
package)
version = SPECS.getData().getHighestVersion(package)
destLogPath = constants.logPath + "/" + package + "-" + version
if not os.path.isdir(destLogPath):
CommandUtils.runCommandInShell("mkdir -p " + destLogPath)
chroot = Chroot(self.logger)
chroot.create(package + "-" + version)
self.installToolchainRPMS(chroot, package, version)
pkgUtils.adjustGCCSpecs(chroot, package, version)
pkgUtils.buildRPMSForGivenPackage(chroot, package, version,
destLogPath)
pkgCount += 1
chroot.destroy()
self.logger.debug("Successfully built toolchain")
self.logger.info("-" * 45 + "\n")
except Exception as e:
self.logger.error("Unable to build toolchain.")
# print stacktrace
traceback.print_exc()
raise e
return pkgCount
def _buildGivenPackages(self, listPackages, buildThreads):
# Extend listPackages from ["name1", "name2",..] to ["name1-vers1", "name2-vers2",..]
listPackageNamesAndVersions = set()
for pkg in listPackages:
base = SPECS.getData().getSpecName(pkg)
for version in SPECS.getData().getVersions(base):
listPackageNamesAndVersions.add(base + "-" + version)
returnVal = self._calculateParams(listPackageNamesAndVersions)
if not returnVal:
self.logger.error(
"Unable to set parameters. Terminating the package manager.")
raise Exception("Unable to set parameters")
statusEvent = threading.Event()
self._initializeScheduler(statusEvent)
self._initializeThreadPool(statusEvent)
for i in range(0, buildThreads):
workerName = "WorkerThread" + str(i)
ThreadPool.addWorkerThread(workerName)
ThreadPool.startWorkerThread(workerName)
statusEvent.wait()
Scheduler.stopScheduling = True
self.logger.debug("Waiting for all remaining worker threads")
ThreadPool.join_all()
setFailFlag = False
allPackagesBuilt = False
if Scheduler.isAnyPackagesFailedToBuild():
setFailFlag = True
if Scheduler.isAllPackagesBuilt():
allPackagesBuilt = True
if setFailFlag:
self.logger.error("Some of the packages failed:")
self.logger.error(Scheduler.listOfFailedPackages)
raise Exception("Failed during building package")
if not setFailFlag:
if allPackagesBuilt:
self.logger.debug("All packages built successfully")
else:
self.logger.error("Build stopped unexpectedly.Unknown error.")
raise Exception("Unknown error")
def _readAlreadyAvailablePackages(self):
listAvailablePackages = set()
pkgUtils = PackageUtils(self.logName, self.logPath)
listPackages = SPECS.getData().getListPackages()
for package in listPackages:
for version in SPECS.getData().getVersions(package):
# Mark package available only if all subpackages are available
packageIsAlreadyBuilt=True
listRPMPackages = SPECS.getData().getRPMPackages(package, version)
for rpmPkg in listRPMPackages:
if pkgUtils.findRPMFile(rpmPkg, version) is None:
packageIsAlreadyBuilt=False
break;
if packageIsAlreadyBuilt:
listAvailablePackages.add(package+"-"+version)
return listAvailablePackages
def getPackageBuildData(self, listPackages):
basePackages = []
for pkg in listPackages:
basePackages.append(SPECS.getData().getBasePkg(pkg))
self._readDependencyGraphAndCyclesForGivenPackages(basePackages)
self._getSortedBuildOrderList()
return self.__mapCyclesToPackageList, self.__mapPackageToCycle, self.__sortedPackageList
def __getRequiredTypePackages(pkg, requiresType):
listRPMPackages = []
if requiresType == "build":
listRPMPackages.extend(SPECS.getData().getBuildRequiresForPkg(pkg))
elif requiresType == "install":
listRPMPackages.extend(SPECS.getData().getRequiresAllForPkg(pkg))
# Remove duplicates.
listRPMPackages = list(set(listRPMPackages))
listPackages = set()
for reqPkg in listRPMPackages:
basePkg = SPECS.getData().getBasePkg(reqPkg)
listPackages.add(basePkg)
return list(listPackages)
def _buildGivenPackages(self, listPackages, buildThreads):
# Extend listPackages from ["name1", "name2",..] to ["name1-vers1", "name2-vers2",..]
listPackageNamesAndVersions=set()
for pkg in listPackages:
base = SPECS.getData().getSpecName(pkg)
for version in SPECS.getData().getVersions(base):
listPackageNamesAndVersions.add(base+"-"+version)
returnVal = self._calculateParams(listPackageNamesAndVersions)
if not returnVal:
self.logger.error("Unable to set parameters. Terminating the package manager.")
raise Exception("Unable to set parameters")
statusEvent = threading.Event()
self._initializeScheduler(statusEvent)
self._initializeThreadPool(statusEvent)
for i in range(0, buildThreads):
workerName = "WorkerThread" + str(i)
ThreadPool.addWorkerThread(workerName)
ThreadPool.startWorkerThread(workerName)
statusEvent.wait()
Scheduler.stopScheduling = True
self.logger.debug("Waiting for all remaining worker threads")
ThreadPool.join_all()
setFailFlag = False
allPackagesBuilt = False
if Scheduler.isAnyPackagesFailedToBuild():
setFailFlag = True
if Scheduler.isAllPackagesBuilt():
allPackagesBuilt = True
if setFailFlag:
self.logger.error("Some of the packages failed:")
self.logger.error(Scheduler.listOfFailedPackages)
raise Exception("Failed during building package")
if not setFailFlag:
if allPackagesBuilt:
self.logger.debug("All packages built successfully")
else:
self.logger.error("Build stopped unexpectedly.Unknown error.")
raise Exception("Unknown error")
def __getRequiredTypePackages(pkg, requiresType):
listRPMPackages = []
if requiresType == "build":
listRPMPackages.extend(SPECS.getData().getBuildRequiresForPkg(pkg))
elif requiresType == "install":
listRPMPackages.extend(SPECS.getData().getRequiresAllForPkg(pkg))
# Remove duplicates.
listRPMPackages = list(set(listRPMPackages))
listPackages = set()
for reqPkg in listRPMPackages:
basePkg = SPECS.getData().getBasePkg(reqPkg)
listPackages.add(basePkg)
return list(listPackages)
def _readAlreadyAvailablePackages(self):
listAvailablePackages = set()
pkgUtils = PackageUtils(self.logName, self.logPath)
listPackages = SPECS.getData().getListPackages()
for package in listPackages:
for version in SPECS.getData().getVersions(package):
# Mark package available only if all subpackages are available
packageIsAlreadyBuilt=True
listRPMPackages = SPECS.getData().getRPMPackages(package, version)
for rpmPkg in listRPMPackages:
if pkgUtils.findRPMFile(rpmPkg, version) is None:
packageIsAlreadyBuilt=False
break;
if packageIsAlreadyBuilt:
listAvailablePackages.add(package+"-"+version)
return listAvailablePackages
def buildSRPMList(srpmPath, yamlDir, blackListPkgs, logger, singleFile=True):
cmdUtils = CommandUtils()
yamlSrpmDir = os.path.join(yamlDir, "yaml_srpms")
if not os.path.isdir(yamlSrpmDir):
cmdUtils.runCommandInShell("mkdir -p "+yamlSrpmDir)
if singleFile:
yamlFile = open(yamlSrpmDir+"/srpm_list.yaml", "w")
listPackages = SPECS.getData().getListPackages()
listPackages.sort()
for package in listPackages:
if package in blackListPkgs:
continue
ossname = package
ossversion = SPECS.getData().getVersion(package)
ossrelease = SPECS.getData().getRelease(package)
listFoundSRPMFiles = cmdUtils.findFile(ossname+"-"+ossversion+"-"+ossrelease+".src.rpm",srpmPath)
srpmName = None
if len(listFoundSRPMFiles) == 1:
srpmFullPath = listFoundSRPMFiles[0];
srpmName = os.path.basename(srpmFullPath)
cpcmd = "cp "+ srpmFullPath +" "+yamlSrpmDir+"/"
returnVal = cmdUtils.runCommandInShell(cpcmd)
if not returnVal:
logger.error("Copy SRPM File is failed for package:"+ossname)
else:
logger.error("SRPM file is not found:" +ossname)
if not singleFile:
yamlFile = open(yamlSrpmDir+"/"+ossname+"-"+ossversion+"-"+ossrelease+".yaml", "w")
yamlFile.write("baseos:"+ossname+":"+ossversion+"-"+ossrelease+":\n")
yamlFile.write(" repository: BaseOS\n")
yamlFile.write(" name: '"+ossname+"'\n")
yamlFile.write(" version: '"+ossversion+"-"+ossrelease+"'\n")
yamlFile.write(" baseos-style: rpm\n")
yamlFile.write(" baseos-source: '"+str(srpmName)+"'\n")
yamlFile.write(" baseos-osname: 'photon'\n")
yamlFile.write("\n")
if not singleFile:
yamlFile.close()
if singleFile:
yamlFile.close()
logger.info("Generated srpm yaml files for all packages")
def getPackageBuildData(self, listPackages):
basePackages = set()
for pkg in listPackages:
basePackage = SPECS.getData().getSpecName(pkg)
basePackages.add(basePackage)
self._readDependencyGraphAndCyclesForGivenPackages(basePackages)
self._getSortedBuildOrderList()
return self.__mapCyclesToPackageList, self.__mapPackageToCycle, self.__sortedPackageList
def findRPMFile(self, package, version="*"):
cmdUtils = CommandUtils()
if version == "*":
version = SPECS.getData().getHighestVersion(package)
release = SPECS.getData().getRelease(package, version)
buildarch = SPECS.getData().getBuildArch(package, version)
filename = package + "-" + version + "-" + release + "." + buildarch + ".rpm"
fullpath = constants.rpmPath + "/" + buildarch + "/" + filename
if os.path.isfile(fullpath):
return fullpath
if constants.inputRPMSPath is not None:
fullpath = constants.inputRPMSPath + "/" + buildarch + "/" + filename
if os.path.isfile(fullpath):
return fullpath
return None
def findRPMFile(self, package,version="*"):
cmdUtils = CommandUtils()
if version == "*":
version = SPECS.getData().getHighestVersion(package)
release = SPECS.getData().getRelease(package, version)
buildarch=SPECS.getData().getBuildArch(package, version)
filename= package + "-" + version + "-" + release + "." + buildarch+".rpm"
fullpath = constants.rpmPath + "/" + buildarch + "/" + filename
if os.path.isfile(fullpath):
return fullpath
if constants.inputRPMSPath is not None:
fullpath = constants.inputRPMSPath + "/" + buildarch + "/" + filename
if os.path.isfile(fullpath):
return fullpath
return None
def _calculateAllRequiredPackagesPerNode():
# pkgNode contains information about spec package without deeping into subpackages.
# getRequiresTreeOfBasePkgsForPkg() creates full build time dependency list base on
# subpackages dependencies by walking the tree of:
# BuildRequires and their Requires tree + Requires and their Requires tree
# Let's keep graph simple by caching this preprocessed information per each pkgNode.
# It shouldn't add much memory overhead.
for package in Scheduler.sortedList:
pkgNode = Scheduler.mapPackagesToGraphNodes[package]
pkgNode.allRequiredPackages.extend(SPECS.getData().getRequiresTreeOfBasePkgsForPkg(package))
def process(self, inputType, inputValue, displayOption, outputFile=None):
whoNeedsList = []
inputPackages = []
whatNeedsBuild = []
mapDependencies = {}
parent = {}
if inputType == "pkg" or inputType == "json":
if inputType == "pkg":
inputPackages.append(inputValue)
else:
inputPackages = self.getAllPackageNames(inputValue)
self.calculateSpecDependency(inputPackages, mapDependencies,
parent)
if outputFile is not None:
return self.displayDependencies(displayOption, inputType,
outputFile, mapDependencies,
parent)
else:
return self.displayDependencies(displayOption, inputType,
inputValue, mapDependencies,
parent)
elif inputType == "get-upward-deps":
depList = []
for specFile in inputValue.split(":"):
if specFile in SPECS.getData().mapSpecFileNameToSpecObj:
specObj = SPECS.getData(
).mapSpecFileNameToSpecObj[specFile]
whoNeedsList.append(specObj.name + "-" + specObj.version)
depList.append(specObj.name + "-" + specObj.version)
self.findTotalWhoNeeds(depList, whoNeedsList)
return whoNeedsList
elif inputType == "who-needs":
for depPackage in SPECS.getData().mapPackageToSpec:
pkg = inputValue + "-" + SPECS.getData().getHighestVersion(
inputValue)
for version in SPECS.getData().getVersions(depPackage):
depPkg = depPackage + "-" + version
self.logger.info(depPkg)
if pkg in SPECS.getData().getRequiresForPkg(depPkg):
whoNeedsList.append(depPkg)
self.logger.info(whoNeedsList)
return whoNeedsList
elif inputType == "is-toolchain-pkg":
for specFile in inputValue.split(":"):
if specFile in SPECS.getData().mapSpecFileNameToSpecObj:
specObj = SPECS.getData(
).mapSpecFileNameToSpecObj[specFile]
if (specObj.name in constants.listCoreToolChainPackages) \
or (specObj.name in constants.listToolChainPackages):
return True
return False
def readAlreadyAvailablePackages(self):
listAvailablePackages = []
listFoundRPMPackages = []
listRPMFiles = []
listDirectorys = []
listDirectorys.append(constants.rpmPath)
if constants.inputRPMSPath is not None:
listDirectorys.append(constants.inputRPMSPath)
while len(listDirectorys) > 0:
dirPath = listDirectorys.pop()
for dirEntry in os.listdir(dirPath):
dirEntryPath = os.path.join(dirPath, dirEntry)
if os.path.isfile(dirEntryPath) and dirEntryPath.endswith(
".rpm"):
listRPMFiles.append(dirEntryPath)
elif os.path.isdir(dirEntryPath):
listDirectorys.append(dirEntryPath)
pkgUtils = PackageUtils(self.logName, self.logPath)
for rpmfile in listRPMFiles:
package, version, release = pkgUtils.findPackageInfoFromRPMFile(
rpmfile)
if SPECS.getData().isRPMPackage(package):
specVersion = SPECS.getData().getVersion(package)
specRelease = SPECS.getData().getRelease(package)
if version == specVersion and release == specRelease:
listFoundRPMPackages.append(package)
#Mark package available only if all sub packages are available
for package in listFoundRPMPackages:
basePkg = SPECS.getData().getSpecName(package)
if basePkg in listAvailablePackages:
continue
listRPMPackages = SPECS.getData().getRPMPackages(basePkg)
packageIsAlreadyBuilt = True
for rpmpkg in listRPMPackages:
if rpmpkg not in listFoundRPMPackages:
packageIsAlreadyBuilt = False
if packageIsAlreadyBuilt:
listAvailablePackages.append(package)
self.logger.info("List of Already built packages")
self.logger.info(listAvailablePackages)
return listAvailablePackages
def _createSortListForPkg(self, pkg):
runTimeDepPkgList = list(set(self.__runTimeDependencyGraph[pkg]))
runTimeDepPkgList.append(pkg)
sortListForPkg = []
for p in runTimeDepPkgList:
basePkg = SPECS.getData().getBasePkg(p)
for bPkg in self.__sortedBuildDependencyGraph[basePkg]:
if bPkg not in sortListForPkg:
sortListForPkg.append(bPkg)
return sortListForPkg
def buildPackagesForAllSpecs(logger, buildThreads, pkgInfoJsonFile, pkgBuildType):
listPackages = SPECS.getData().getListPackages()
if constants.rpmCheck:
constants.setTestForceRPMS(copy.copy(listPackages))
pkgManager = PackageManager(pkgBuildType=pkgBuildType)
pkgManager.buildPackages(listPackages, buildThreads, pkgBuildType)
# Generating package info file which is required by installer
logger.debug("Writing Package info to the file:" + pkgInfoJsonFile)
pkgInfo = PackageInfo()
pkgInfo.loadPackagesData()
pkgInfo.writePkgListToFile(pkgInfoJsonFile)
def buildSourcesList(yamlDir, blackListPkgs, logger, singleFile=True):
cmdUtils = CommandUtils()
yamlSourceDir = os.path.join(yamlDir, "yaml_sources")
if not os.path.isdir(yamlSourceDir):
cmdUtils.runCommandInShell("mkdir -p " + yamlSourceDir)
if singleFile:
yamlFile = open(yamlSourceDir + "/sources_list.yaml", "w")
listPackages = SPECS.getData().getListPackages()
listPackages.sort()
import PullSources
for package in listPackages:
if package in blackListPkgs:
continue
ossname = package
for version in SPECS.getData().getVersions(package):
modified = False
listPatches = SPECS.getData().getPatches(package, version)
if listPatches:
modified = True
url = SPECS.getData().getSourceURL(package, version)
if url is None:
url = SPECS.getData().getURL(package, version)
sourceName = None
listSourceNames = SPECS.getData().getSources(package, version)
if listSourceNames:
sourceName = listSourceNames[0]
sha1 = SPECS.getData().getSHA1(package, version, sourceName)
if sha1 is not None:
PullSources.get(package, sourceName, sha1, yamlSourceDir,
constants.getPullSourcesURLs(package),
logger)
if not singleFile:
yamlFile = open(
yamlSourceDir + "/" + ossname + "-" + version + ".yaml",
"w")
yamlFile.write("vmwsource:" + ossname + ":" + version + ":\n")
yamlFile.write(" repository: VMWsource\n")
yamlFile.write(" name: '" + ossname + "'\n")
yamlFile.write(" version: '" + version + "'\n")
yamlFile.write(" url: " + str(url) + "\n")
yamlFile.write(" license: UNKNOWN\n")
if sourceName is not None:
yamlFile.write(" vmwsource-distribution: " + str(sourceName) +
"\n")
if modified:
yamlFile.write(" modified: true\n")
yamlFile.write("\n")
if not singleFile:
yamlFile.close()
if singleFile:
yamlFile.close()
logger.debug("Generated source yaml files for all packages")
def installTargetToolchain(self, chroot, stopAtPackage=None):
self.logger.debug("Installing target toolchain RPMS.......")
pkgUtils = PackageUtils(self.logName, self.logPath)
rpmFiles = ""
packages = ""
for package in constants.listCoreToolChainPackages:
if stopAtPackage and package == stopAtPackage:
break
version = SPECS.getData().getHighestVersion(package)
basePkg = SPECS.getData().getSpecName(package)
# install all subpackages of given package
# for instance: for 'glibc' we want glibc-devel, glibc-tools,
# glibc-i18n, etc also to be installed
subpackages = SPECS.getData().getRPMPackages(basePkg, version)
for p in subpackages:
rpmFile = pkgUtils.findRPMFile(p, version,
constants.targetArch)
rpmFiles += " " + rpmFile
packages += " " + package + "-" + version
self.logger.debug(packages)
cmd = "mkdir -p " + chroot.getID(
) + "/target-" + constants.targetArch + "/var/lib/rpm"
CommandUtils.runCommandInShell(cmd, logfn=self.logger.debug)
if rpmFiles != "":
cmd = (self.rpmCommand +
" -Uvh --nodeps --ignorearch --noscripts --root " +
chroot.getID() + "/target-" + constants.targetArch +
" --define \'_dbpath /var/lib/rpm\' " + rpmFiles)
retVal = CommandUtils.runCommandInShell(cmd,
logfn=self.logger.debug)
if retVal != 0:
self.logger.debug("Command Executed:" + cmd)
self.logger.error("Installing toolchain failed")
raise Exception("RPM installation failed")
self.logger.debug(
"Successfully installed target toolchain RPMS in chroot:" +
chroot.getID())
def buildRPMSForGivenPackage(self, sandbox, package, version, destLogPath):
self.logger.info("Building package : " + package)
listSourcesFiles = SPECS.getData().getSources(package, version)
listPatchFiles = SPECS.getData().getPatches(package, version)
specFile = SPECS.getData().getSpecFile(package, version)
specName = SPECS.getData().getSpecName(package) + ".spec"
sourcePath = constants.topDirPath + "/SOURCES/"
specPath = constants.topDirPath + "/SPECS/"
if (constants.rpmCheck and
package in constants.testForceRPMS and
SPECS.getData().isCheckAvailable(package, version)):
logFilePath = destLogPath + "/" + package + "-test.log"
else:
logFilePath = destLogPath + "/" + package + ".log"
sandbox.put(specFile, specPath + specName)
sources_urls, macros = self._getAdditionalBuildOptions(package)
self.logger.debug("Extra macros for " + package + ": " + str(macros))
self.logger.debug("Extra source URLs for " + package + ": " + str(sources_urls))
constants.setExtraSourcesURLs(package, sources_urls)
self._copySources(sandbox, listSourcesFiles, package, version, sourcePath)
self._copySources(sandbox, listPatchFiles, package, version, sourcePath)
#Adding rpm macros
listRPMMacros = constants.userDefinedMacros
for macroName, value in listRPMMacros.items():
macros.append(macroName + " " + value)
listRPMFiles = []
listSRPMFiles = []
try:
listRPMFiles, listSRPMFiles = self._buildRPM(sandbox, specPath + specName,
logFilePath, package, version, macros)
logmsg = package + " build done - RPMs : [ "
for f in listRPMFiles:
logmsg += (os.path.basename(f) + " ")
logmsg += "]\n"
self.logger.info(logmsg)
except Exception as e:
self.logger.error("Failed while building rpm:" + package)
raise e
finally:
if (constants.rpmCheck and
package in constants.testForceRPMS and
SPECS.getData().isCheckAvailable(package, version)):
cmd = ("sed -i '/^Executing(%check):/,/^Processing files:/{//!b};d' " + logFilePath)
CommandUtils().runCommandInShell(cmd, logfn=self.logger.debug)
if constants.crossCompiling:
for f in listRPMFiles:
workingDir = os.getcwd()
cmd = "%s/../../tools/scripts/check-rpm-arch.sh %s" % (workingDir,
constants.rpmPath + "/" + constants.targetArch + "/" + os.path.basename(f))
retVal = CommandUtils().runCommandInShell(cmd, logfn=self.logger.debug, showOutput=True)
if retVal != 0:
raise NameError("%s failed %s architecture check" % (f, constants.targetArch))
else:
self.logger.debug("RPM passed architecture check")
self.logger.debug("RPM build is successful")
def _readAlreadyAvailablePackages(self):
listAvailablePackages = set()
listFoundRPMPackages = set()
listRPMFiles = set()
listDirectorys = set()
listDirectorys.add(constants.rpmPath)
if constants.inputRPMSPath is not None:
listDirectorys.add(constants.inputRPMSPath)
while listDirectorys:
dirPath = listDirectorys.pop()
for dirEntry in os.listdir(dirPath):
dirEntryPath = os.path.join(dirPath, dirEntry)
if os.path.isfile(dirEntryPath) and dirEntryPath.endswith(".rpm"):
listRPMFiles.add(dirEntryPath)
elif os.path.isdir(dirEntryPath):
listDirectorys.add(dirEntryPath)
pkgUtils = PackageUtils(self.logName, self.logPath)
for rpmfile in listRPMFiles:
package, version, release = pkgUtils.findPackageInfoFromRPMFile(rpmfile)
if SPECS.getData().isRPMPackage(package):
specVersion = SPECS.getData().getVersion(package)
specRelease = SPECS.getData().getRelease(package)
if version == specVersion and release == specRelease:
listFoundRPMPackages.add(package)
#Mark package available only if all sub packages are available
for package in listFoundRPMPackages:
basePkg = SPECS.getData().getSpecName(package)
if basePkg in listAvailablePackages:
continue
listRPMPackages = SPECS.getData().getRPMPackages(basePkg)
packageIsAlreadyBuilt = True
for rpmpkg in listRPMPackages:
if rpmpkg not in listFoundRPMPackages:
packageIsAlreadyBuilt = False
if packageIsAlreadyBuilt:
listAvailablePackages.add(package)
self.logger.info("List of Already built packages")
self.logger.info(listAvailablePackages)
return listAvailablePackages
def findRPMFile(self, package,version="*",arch=None, throw=False):
if not arch:
arch=constants.currentArch
if version == "*":
version = SPECS.getData(arch).getHighestVersion(package)
release = SPECS.getData(arch).getRelease(package, version)
buildarch=SPECS.getData(arch).getBuildArch(package, version)
filename= package + "-" + version + "-" + release + "." + buildarch+".rpm"
fullpath = constants.rpmPath + "/" + buildarch + "/" + filename
if os.path.isfile(fullpath):
return fullpath
if constants.inputRPMSPath is not None:
fullpath = constants.inputRPMSPath + "/" + buildarch + "/" + filename
if os.path.isfile(fullpath):
return fullpath
if throw:
raise Exception("RPM %s not found" % (filename))
return None
def installToolChainRPMSinContainer(self, containerID):
self.logger.info("Installing tool-chain RPMS in container: " +
containerID.short_id)
rpmFiles = ""
packages = ""
pkgUtils = PackageUtils(self.logName, self.logPath)
for package in constants.listToolChainRPMPkgsToInstall:
version = SPECS.getData().getHighestVersion(package)
rpmFile = pkgUtils.findRPMFileForGivenPackage(package, version)
if rpmFile is None:
# sqlite-autoconf package was renamed, but it still published as sqlite-autoconf
# if (package == "sqlite") and (platform.machine() == "x86_64"):
# package = "sqlite-autoconf"
rpmFile = self.findRPMFileInGivenLocation(
package, constants.prevPublishRPMRepo)
if rpmFile is None:
if package in constants.listOfRPMsProvidedAfterBuild:
self.logger.info(
"No old version of " + package +
" exists, skip until the new version is built")
continue
self.logger.error("Unable to find rpm " + package +
" in current and previous versions")
raise Exception("Input Error")
if rpmFile.find("stage/PUBLISHRPMS"):
rpmFile = rpmFile.replace(constants.prevPublishRPMRepo,
"/publishrpms")
if rpmFile.find("stage/PUBLISHXRPMS"):
rpmFile = rpmFile.replace(constants.prevPublishXRPMRepo,
"/publishxrpms")
if rpmFile.find("stage/RPMS"):
rpmFile = rpmFile.replace(constants.rpmPath,
constants.topDirPath + "/RPMS")
rpmFiles += " " + rpmFile
packages += " " + package
self.logger.debug("Installing tool-chain rpms: " + packages)
cmd = "/usr/bin/bash -l -c '/usr/bin/rpm -Uvh --force --nodeps " + rpmFiles + "'"
self.logger.info(
"VDBG-TCU-installToolChainRPMSinContainer: Installing rpms cmd: " +
cmd)
tcInstallLog = containerID.exec_run(cmd)
# TODO: Find a way to collect exit status of the command that was run.
if not tcInstallLog:
self.logger.error("Installing tool chain in container failed")
raise Exception("RPM installation in container failed")
self.logger.info(tcInstallLog)
self.logger.info(
"Successfully installed default tool-chain RPMS in container: " +
containerID.short_id)
def _readAlreadyAvailablePackages(self):
listAvailablePackages = set()
listFoundRPMPackages = set()
listRPMFiles = set()
listDirectorys = set()
mapPackageVersion = {}
mapPackageRelease = {}
listDirectorys.add(constants.rpmPath)
if constants.inputRPMSPath is not None:
listDirectorys.add(constants.inputRPMSPath)
while listDirectorys:
dirPath = listDirectorys.pop()
for dirEntry in os.listdir(dirPath):
dirEntryPath = os.path.join(dirPath, dirEntry)
if os.path.isfile(dirEntryPath) and dirEntryPath.endswith(
".rpm"):
listRPMFiles.add(dirEntryPath)
elif os.path.isdir(dirEntryPath):
listDirectorys.add(dirEntryPath)
pkgUtils = PackageUtils(self.logName, self.logPath)
for rpmfile in listRPMFiles:
package, version, release = pkgUtils.findPackageInfoFromRPMFile(
rpmfile)
if package in mapPackageVersion:
mapPackageVersion[package].append(version)
mapPackageRelease[package].append(release)
else:
mapPackageVersion[package] = [version]
mapPackageRelease[package] = [release]
for package in mapPackageVersion:
if SPECS.getData().isRPMPackage(package):
numVersions = SPECS.getData().getNumberOfVersions(package)
flag = True
for index in range(0, numVersions):
specVersion = SPECS.getData().getVersion(package, index)
specRelease = SPECS.getData().getRelease(package, index)
if specVersion not in mapPackageVersion[
package] and specRelease not in mapPackageRelease[
package]:
flag = False
if flag:
listFoundRPMPackages.add(package)
#Mark package available only if all sub packages are available
for package in listFoundRPMPackages:
basePkg = SPECS.getData().getSpecName(package)
if basePkg in listAvailablePackages:
continue
listRPMPackages = SPECS.getData().getRPMPackages(basePkg)
packageIsAlreadyBuilt = True
for rpmpkg in listRPMPackages:
if rpmpkg not in listFoundRPMPackages:
packageIsAlreadyBuilt = False
if packageIsAlreadyBuilt:
listAvailablePackages.add(package)
self.logger.info("List of Already built packages")
self.logger.info(listAvailablePackages)
return listAvailablePackages
def loadPackagesData(self):
listPackages = SPECS.getData().getListPackages()
listPackages.sort()
listRPMFiles = []
cmdUtils = CommandUtils()
for package in listPackages:
for version in SPECS.getData().getVersions(package):
release = SPECS.getData().getRelease(package, version)
listRPMPackages = SPECS.getData().getRPMPackages(
package, version)
srpmFileName = package + "-" + version + "-" + release + ".src.rpm"
srpmFiles = cmdUtils.findFile(srpmFileName,
constants.sourceRpmPath)
srpmFile = None
if len(srpmFiles) == 1:
srpmFile = srpmFiles[0]
debugrpmFileName = package + "-debuginfo-" + version + "-" + release + "*"
debugrpmFiles = cmdUtils.findFile(debugrpmFileName,
constants.rpmPath)
debugrpmFile = None
if len(debugrpmFiles) == 1:
debugrpmFile = debugrpmFiles[0]
pkgUtils = PackageUtils(self.logName, self.logPath)
for rpmPkg in listRPMPackages:
rpmFile = pkgUtils.findRPMFileForGivenPackage(rpmPkg)
if rpmFile is not None:
listRPMFiles.append(rpmFile)
listPkgAttributes = {
"sourcerpm": srpmFile,
"rpm": rpmFile,
"debugrpm": debugrpmFile
}
self.pkgList[rpmPkg] = listPkgAttributes
self.logger.debug("Added " + rpmPkg +
" rpm package to the list")
else:
self.logger.error("Missing rpm file for package:" +
rpmPkg)
def buildSourcesList(yamlDir, blackListPkgs, logger, singleFile=True):
cmdUtils = CommandUtils()
yamlSourceDir = os.path.join(yamlDir, "yaml_sources")
if not os.path.isdir(yamlSourceDir):
cmdUtils.runCommandInShell("mkdir -p " + yamlSourceDir)
if singleFile:
yamlFile = open(yamlSourceDir + "/sources_list.yaml", "w")
listPackages = SPECS.getData().getListPackages()
listPackages.sort()
import PullSources
for package in listPackages:
if package in blackListPkgs:
continue
ossname = package
for version in SPECS.getData().getVersions(package):
modified = False
listPatches = SPECS.getData().getPatches(package, version)
if listPatches:
modified = True
url = SPECS.getData().getSourceURL(package, version)
if url is None:
url = SPECS.getData().getURL(package, version)
sourceName = None
listSourceNames = SPECS.getData().getSources(package, version)
if listSourceNames:
sourceName = listSourceNames[0]
sha1 = SPECS.getData().getSHA1(package, version, sourceName)
if sha1 is not None:
PullSources.get(package, sourceName, sha1, yamlSourceDir,
constants.getPullSourcesURLs(package),
logger)
if not singleFile:
yamlFile = open(yamlSourceDir + "/" + ossname + "-" + version + ".yaml", "w")
yamlFile.write("vmwsource:" + ossname + ":" + version + ":\n")
yamlFile.write(" repository: VMWsource\n")
yamlFile.write(" name: '" + ossname + "'\n")
yamlFile.write(" version: '" + version + "'\n")
yamlFile.write(" url: " + str(url) + "\n")
yamlFile.write(" license: UNKNOWN\n")
if sourceName is not None:
yamlFile.write(" vmwsource-distribution: " + str(sourceName) + "\n")
if modified:
yamlFile.write(" modified: true\n")
yamlFile.write("\n")
if not singleFile:
yamlFile.close()
if singleFile:
yamlFile.close()
logger.debug("Generated source yaml files for all packages")
def _constructBuildAndRunTimeDependencyGraph(self, basePackage):
addBuildTimeGraph = True
addRunTimeGraph = True
if basePackage in self.__buildDependencyGraph:
addBuildTimeGraph = False
if basePackage in self.__runTimeDependencyGraph:
addRunTimeGraph = False
nextPackagesToConstructGraph = set()
if addBuildTimeGraph:
dependentRpmPackages = SPECS.getData().getBuildRequiresForPackage(
basePackage)
dependentPackages = set()
for rpmPkg in dependentRpmPackages:
basePkg = SPECS.getData().getSpecName(rpmPkg.package)
dependentPackages.add(basePkg)
self.__buildDependencyGraph[basePackage] = dependentPackages
nextPackagesToConstructGraph.update(dependentPackages)
if addRunTimeGraph:
rpmPackages = SPECS.getData().getPackages(basePackage)
dependentPackages = set()
dependentRpmPackagesNames = set()
for rpmPkg in rpmPackages:
dependentRpmPackages = SPECS.getData(
).getRequiresAllForPackage(rpmPkg)
for pkgName in dependentRpmPackages:
dependentRpmPackagesNames.add(pkgName.package)
self.__runTimeDependencyGraph[rpmPkg] = copy.copy(
dependentRpmPackagesNames)
for pkg in dependentRpmPackagesNames:
dependentPackages.add(SPECS.getData().getSpecName(pkg))
nextPackagesToConstructGraph.update(dependentPackages)
for pkg in nextPackagesToConstructGraph:
self._constructBuildAndRunTimeDependencyGraph(pkg)
def buildPackagesForAllSpecs(listBuildOptionPackages, pkgBuildOptionFile, logger, buildThreads, pkgInfoJsonFile, pkgBuildType):
listPackages = SPECS.getData().getListPackages()
logger.info("List of packages to build:")
logger.info(listPackages)
if constants.rpmCheck:
constants.setTestForceRPMS(listPackages[:])
pkgManager = PackageManager()
pkgManager.buildPackages(listPackages, listBuildOptionPackages, pkgBuildOptionFile, buildThreads, pkgBuildType)
#Generating package info file which is required by installer
logger.info("Writing Package info to the file:"+pkgInfoJsonFile)
pkgInfo = PackageInfo()
pkgInfo.loadPackagesData()
pkgInfo.writePkgListToFile(pkgInfoJsonFile)
def process(self, inputType, inputValue, displayOption, outputFile=None):
whoNeedsList = []
inputPackages = []
whatNeedsBuild = []
mapDependencies = {}
parent = {}
if inputType == "pkg" or inputType == "json":
if inputType == "pkg":
inputPackages.append(inputValue)
else:
inputPackages = self.getAllPackageNames(inputValue)
self.calculateSpecDependency(inputPackages, mapDependencies, parent)
if outputFile is not None:
return self.displayDependencies(displayOption, inputType, outputFile, mapDependencies, parent)
else:
return self.displayDependencies(displayOption, inputType, inputValue, mapDependencies, parent)
elif inputType == "get-upward-deps":
depList = []
for specFile in inputValue.split(":"):
if specFile in SPECS.getData().mapSpecFileNameToSpecObj:
specObj = SPECS.getData().mapSpecFileNameToSpecObj[specFile]
whoNeedsList.append(specObj.name+"-"+specObj.version)
depList.append(specObj.name+"-"+specObj.version)
self.findTotalWhoNeeds(depList, whoNeedsList)
return whoNeedsList
elif inputType == "who-needs":
for depPackage in SPECS.getData().mapPackageToSpec:
pkg=inputValue+"-"+SPECS.getData().getHighestVersion(inputValue)
for version in SPECS.getData().getVersions(depPackage):
depPkg = depPackage+"-"+version
self.logger.info(depPkg)
if pkg in SPECS.getData().getRequiresForPkg(depPkg):
whoNeedsList.append(depPkg)
self.logger.info(whoNeedsList)
return whoNeedsList
elif inputType == "is-toolchain-pkg":
for specFile in inputValue.split(":"):
if specFile in SPECS.getData().mapSpecFileNameToSpecObj:
specObj = SPECS.getData().mapSpecFileNameToSpecObj[specFile]
if (specObj.name in constants.listCoreToolChainPackages) \
or (specObj.name in constants.listToolChainPackages):
return True
return False
def adjustGCCSpecs(self, sandbox, package, version):
opt = " " + SPECS.getData().getSecurityHardeningOption(package, version)
sandbox.put(self.adjustGCCSpecScript, "/tmp")
cmd = "/tmp/" + self.adjustGCCSpecScript + opt
returnVal = sandbox.run(cmd, logfn=self.logger.debug)
if returnVal == 0:
return
# in debugging ...
sandbox.run("ls -la /tmp/" + self.adjustGCCSpecScript,
logfn=self.logger.debug)
sandbox.run("lsof /tmp/" + self.adjustGCCSpecScript,
logfn=self.logger.debug)
sandbox.run("ps ax", logfn=self.logger.debug)
self.logger.error("Failed while adjusting gcc specs")
raise Exception("Failed while adjusting gcc specs")
def buildRPMSForGivenPackage(self, sandbox, package, version, destLogPath):
self.logger.info("Building package : " + package)
listSourcesFiles = SPECS.getData().getSources(package, version)
listPatchFiles = SPECS.getData().getPatches(package, version)
specFile = SPECS.getData().getSpecFile(package, version)
specName = SPECS.getData().getSpecName(package) + ".spec"
sourcePath = constants.topDirPath + "/SOURCES/"
specPath = constants.topDirPath + "/SPECS/"
if (constants.rpmCheck and package in constants.testForceRPMS
and SPECS.getData().isCheckAvailable(package)):
logFilePath = destLogPath + "/" + package + "-test.log"
else:
logFilePath = destLogPath + "/" + package + ".log"
sandbox.put(specFile, specPath + specName)
sources_urls, macros = self._getAdditionalBuildOptions(package)
self.logger.debug("Extra macros for " + package + ": " + str(macros))
self.logger.debug("Extra source URLs for " + package + ": " +
str(sources_urls))
constants.setExtraSourcesURLs(package, sources_urls)
self._copySources(sandbox, listSourcesFiles, package, version,
sourcePath)
self._copySources(sandbox, listPatchFiles, package, version,
sourcePath)
#Adding rpm macros
listRPMMacros = constants.userDefinedMacros
for macroName, value in listRPMMacros.items():
macros.append(macroName + " " + value)
listRPMFiles = []
listSRPMFiles = []
try:
listRPMFiles, listSRPMFiles = self._buildRPM(
sandbox, specPath + specName, logFilePath, package, macros)
logmsg = package + " build done - RPMs : [ "
for f in listRPMFiles:
logmsg += (os.path.basename(f) + " ")
logmsg += "]\n"
self.logger.info(logmsg)
except Exception as e:
self.logger.error("Failed while building rpm:" + package)
raise e
finally:
if (constants.rpmCheck and package in constants.testForceRPMS
and SPECS.getData().isCheckAvailable(package)):
cmd = (
"sed -i '/^Executing(%check):/,/^Processing files:/{//!b};d' "
+ logFilePath)
CommandUtils().runCommandInShell(cmd, logfn=self.logger.debug)
self.logger.debug("RPM build is successful")
def adjustGCCSpecs(self, sandbox, package, version):
# TODO: need to harden cross compiller also
opt = " " + SPECS.getData().getSecurityHardeningOption(package, version)
sandbox.put(os.path.join(os.path.dirname(__file__), self.adjustGCCSpecScript), "/tmp")
cmd = "/tmp/" + self.adjustGCCSpecScript + opt
returnVal = sandbox.run(cmd, logfn=self.logger.debug)
if returnVal == 0:
return
# in debugging ...
sandbox.run("ls -la /tmp/" + self.adjustGCCSpecScript,
logfn=self.logger.debug)
sandbox.run("lsof /tmp/" + self.adjustGCCSpecScript,
logfn=self.logger.debug)
sandbox.run("ps ax", logfn=self.logger.debug)
self.logger.error("Failed while adjusting gcc specs")
raise Exception("Failed while adjusting gcc specs")
def adjustGCCSpecsInContainer(self, package, containerID, logPath, index=0):
opt = " " + SPECS.getData().getSecurityHardeningOption(package, index)
adjustCmd = "/" + self.adjustGCCSpecScript + opt
adjustCmd = "/bin/bash -l -c '" + adjustCmd + "'"
logFile = logPath + "/adjustGCCSpecScript.log"
#TODO: Error code from exec_run
scriptLog = containerID.exec_run(adjustCmd)
if scriptLog:
with open(logFile, 'w') as logfile:
logfile.write(scriptLog.decode())
return
self.logger.debug(containerID.exec_run("ls -la /tmp/" + self.adjustGCCSpecScript))
self.logger.debug(containerID.exec_run("lsof /tmp/" + self.adjustGCCSpecScript))
self.logger.debug(containerID.exec_run("ps ax"))
self.logger.error("Failed while adjusting gcc specs")
raise Exception("Failed while adjusting gcc specs")
def buildRPMSForGivenPackage(self, sandbox, package, version, destLogPath):
self.logger.info("Building package : " + package)
listSourcesFiles = SPECS.getData().getSources(package, version)
listPatchFiles = SPECS.getData().getPatches(package, version)
specFile = SPECS.getData().getSpecFile(package, version)
specName = SPECS.getData().getSpecName(package) + ".spec"
sourcePath = constants.topDirPath + "/SOURCES/"
specPath = constants.topDirPath + "/SPECS/"
if (constants.rpmCheck and
package in constants.testForceRPMS and
SPECS.getData().isCheckAvailable(package, version)):
logFilePath = destLogPath + "/" + package + "-test.log"
else:
logFilePath = destLogPath + "/" + package + ".log"
sandbox.put(specFile, specPath + specName)
sources_urls, macros = self._getAdditionalBuildOptions(package)
self.logger.debug("Extra macros for " + package + ": " + str(macros))
self.logger.debug("Extra source URLs for " + package + ": " + str(sources_urls))
constants.setExtraSourcesURLs(package, sources_urls)
self._copySources(sandbox, listSourcesFiles, package, version, sourcePath)
self._copySources(sandbox, listPatchFiles, package, version, sourcePath)
#Adding rpm macros
listRPMMacros = constants.userDefinedMacros
for macroName, value in listRPMMacros.items():
macros.append(macroName + " " + value)
listRPMFiles = []
listSRPMFiles = []
try:
listRPMFiles, listSRPMFiles = self._buildRPM(sandbox, specPath + specName,
logFilePath, package, version, macros)
logmsg = package + " build done - RPMs : [ "
for f in listRPMFiles:
logmsg += (os.path.basename(f) + " ")
logmsg += "]\n"
self.logger.info(logmsg)
except Exception as e:
self.logger.error("Failed while building rpm:" + package)
raise e
finally:
if (constants.rpmCheck and
package in constants.testForceRPMS and
SPECS.getData().isCheckAvailable(package, version)):
cmd = ("sed -i '/^Executing(%check):/,/^Processing files:/{//!b};d' " + logFilePath)
CommandUtils().runCommandInShell(cmd, logfn=self.logger.debug)
self.logger.debug("RPM build is successful")
def _findDependentPackagesAndInstalledRPM(self, sandbox):
listInstalledPackages, listInstalledRPMs = self._findInstalledPackages(sandbox)
self.logger.debug(listInstalledPackages)
listDependentPackages = self._findBuildTimeRequiredPackages()
listTestPackages=[]
if constants.rpmCheck and self.package in constants.testForceRPMS:
# One time optimization
if constants.listMakeCheckRPMPkgWithVersionstoInstall is None:
constants.listMakeCheckRPMPkgWithVersionstoInstall=[]
for package in constants.listMakeCheckRPMPkgtoInstall:
version = SPECS.getData().getHighestVersion(package)
constants.listMakeCheckRPMPkgWithVersionstoInstall.append(package+"-"+version)
listDependentPackages.extend(self._findBuildTimeCheckRequiredPackages())
testPackages = (set(constants.listMakeCheckRPMPkgWithVersionstoInstall) -
set(listInstalledPackages) -
set([self.package+"-"+self.version]))
listTestPackages=list(set(testPackages))
listDependentPackages = list(set(listDependentPackages))
return listDependentPackages, listTestPackages, listInstalledPackages, listInstalledRPMs
def findTotalRequires(self, mapDependencies, depQue, parent):
while not depQue.empty():
specPkg = depQue.get()
try:
listRequiredPackages = SPECS.getData().getRequiresForPkg(specPkg)
except Exception as e:
self.logger.info("Caught Exception:"+str(e))
self.logger.info(specPkg + " is missing")
raise e
for depPkg in listRequiredPackages:
if depPkg in mapDependencies:
if mapDependencies[depPkg] < mapDependencies[specPkg] + 1:
mapDependencies[depPkg] = mapDependencies[specPkg] + 1
parent[depPkg] = specPkg
self.updateLevels(mapDependencies, depPkg, parent, mapDependencies[depPkg])
else:
mapDependencies[depPkg] = mapDependencies[specPkg] + 1
parent[depPkg] = specPkg
depQue.put(depPkg)
def adjustGCCSpecs(self, package, chrootID, logPath, index=0):
opt = " " + SPECS.getData().getSecurityHardeningOption(package, index)
cmdUtils = CommandUtils()
cpcmd = ("cp " + self.adjustGCCSpecScript + " " + chrootID +
"/tmp/" + self.adjustGCCSpecScript)
cmd = "/tmp/" + self.adjustGCCSpecScript + opt
logFile = logPath + "/adjustGCCSpecScript.log"
chrootCmd = self.runInChrootCommand + " " + chrootID
returnVal = cmdUtils.runCommandInShell(cpcmd, logFile)
if not returnVal:
self.logger.error("Error during copying the file adjust gcc spec")
raise Exception("Failed while copying adjust gcc spec file")
returnVal = cmdUtils.runCommandInShell(cmd, logFile, chrootCmd)
if returnVal:
return
self.logger.debug(cmdUtils.runCommandInShell2("ls -la " + chrootID +
"/tmp/" + self.adjustGCCSpecScript))
self.logger.debug(cmdUtils.runCommandInShell2("lsof " + chrootID + "/tmp/" +
self.adjustGCCSpecScript))
self.logger.debug(cmdUtils.runCommandInShell2("ps ax"))
self.logger.error("Failed while adjusting gcc specs")
raise Exception("Failed while adjusting gcc specs")
def buildPackagesList(csvFilename):
with open(csvFilename, "w") as csvFile:
csvFile.write("Package,Version,License,URL,Sources,Patches\n")
listPackages = SPECS.getData().getListPackages()
listPackages.sort()
for package in listPackages:
name = package
for version in SPECS.getData().getVersions(package):
packagelicense = SPECS.getData().getLicense(package, version)
listPatches = SPECS.getData().getPatches(package, version)
url = SPECS.getData().getURL(package, version)
listSourceNames = SPECS.getData().getSources(package, version)
sources = ""
patches = ""
if listPatches is not None:
patches = " ".join(listPatches)
if listSourceNames is not None:
sources = " ".join(listSourceNames)
csvFile.write(name + "," + version + "," + packagelicense + "," + url + "," +
sources + "," + patches + "\n")
def buildPackagesForAllSpecs(buildThreads, pkgBuildType, pkgInfoJsonFile, logger):
listPackages = SPECS.getData().getListPackages()
buildSpecifiedPackages(listPackages, buildThreads, pkgBuildType, pkgInfoJsonFile, logger)
def _buildRPMinContainer(self, specFile, rpmLogFile, destLogFile, containerID, package, macros):
rpmBuildCmd = self.rpmbuildBinary + " " + self.rpmbuildBuildallOption
if constants.rpmCheck and package in constants.testForceRPMS:
self.logger.info("#" * (68 + 2 * len(package)))
if not SPECS.getData().isCheckAvailable(package, index):
self.logger.info("####### " + package +
" MakeCheck is not available. Skipping MakeCheck TEST for " +
package + " #######")
rpmBuildCmd = self.rpmbuildBinary + " --clean"
else:
self.logger.info("####### " + package +
" MakeCheck is available. Running MakeCheck TEST for " +
package + " #######")
rpmBuildCmd = self.rpmbuildBinary + " " + self.rpmbuildCheckOption
self.logger.info("#" * (68 + 2 * len(package)))
else:
rpmBuildCmd += " "+self.rpmbuildNocheckOption
for macro in macros:
rpmBuildCmd += ' --define \"%s\"' % macro
rpmBuildCmd += " " + specFile
rpmBuildCmd = "/bin/bash -l -c '" + rpmBuildCmd + " > " + rpmLogFile + " 2>&1'"
rpmBuildCmd = "docker exec -t " + str(containerID.short_id) + " " + rpmBuildCmd
cmdUtils = CommandUtils()
self.logger.info("Building rpm for package: " + package)
#TODO: Show running log of rpmbuildcmd
#TODO: Get exit status of rpmBuildCmd
#containerID.exec_run(rpmBuildCmd)
returnVal = cmdUtils.runCommandInShell(rpmBuildCmd)
if not os.path.isfile(destLogFile):
self.logger.error("RPM build not file not found. Building rpm failed for: " + specFile)
raise Exception("RPM Build failed")
if constants.rpmCheck and package in constants.testForceRPMS:
if not SPECS.getData().isCheckAvailable(package, index):
constants.testLogger.info(package + " : N/A")
elif returnVal:
constants.testLogger.info(package + " : PASS")
else:
constants.testLogger.error(package + " : FAIL")
if constants.rpmCheck:
if not returnVal and constants.rpmCheckStopOnError:
self.logger.error("Checking rpm is failed " + specFile)
raise Exception("RPM check failed")
else:
if not returnVal:
self.logger.error("Building rpm is failed " + specFile)
raise Exception("RPM build failed")
#Extracting rpms created from log file
listRPMFiles = []
listSRPMFiles = []
with open(destLogFile, 'r') as logfile:
rpmBuildLogLines = logfile.readlines()
for i in range(0, len(rpmBuildLogLines)):
if re.search("^Wrote:", rpmBuildLogLines[i]):
listcontents = rpmBuildLogLines[i].split()
if ((len(listcontents) == 2) and
listcontents[1].strip().endswith(".rpm") and
"/RPMS/" in listcontents[1]):
listRPMFiles.append(listcontents[1])
if ((len(listcontents) == 2) and
listcontents[1].strip().endswith(".src.rpm") and
"/SRPMS/" in listcontents[1]):
listSRPMFiles.append(listcontents[1])
#if not listRPMFiles:
# self.logger.error("Building rpm failed for " + specFile)
# raise Exception("RPM Build failed")
return listRPMFiles, listSRPMFiles
def _buildRPM(self, specFile, logFile, chrootCmd, package, macros):
rpmBuildcmd = self.rpmbuildBinary + " " + self.rpmbuildBuildallOption
if constants.rpmCheck and package in constants.testForceRPMS:
self.logger.info("#" * (68 + 2 * len(package)))
if not SPECS.getData().isCheckAvailable(package):
self.logger.info("####### " + package +
" MakeCheck is not available. Skipping MakeCheck TEST for " +
package + " #######")
rpmBuildcmd = self.rpmbuildBinary + " --clean"
else:
self.logger.info("####### " + package +
" MakeCheck is available. Running MakeCheck TEST for " +
package + " #######")
rpmBuildcmd = self.rpmbuildBinary + " " + self.rpmbuildCheckOption
self.logger.info("#" * (68 + 2 * len(package)))
else:
rpmBuildcmd += " " + self.rpmbuildNocheckOption
for macro in macros:
rpmBuildcmd += ' --define \\\"%s\\\"' % macro
rpmBuildcmd += " " + specFile
cmdUtils = CommandUtils()
self.logger.info("Building rpm....")
self.logger.info(rpmBuildcmd)
returnVal = cmdUtils.runCommandInShell(rpmBuildcmd, logFile, chrootCmd)
if constants.rpmCheck and package in constants.testForceRPMS:
if not SPECS.getData().isCheckAvailable(package):
constants.testLogger.info(package + " : N/A")
elif returnVal:
constants.testLogger.info(package + " : PASS")
else:
constants.testLogger.error(package + " : FAIL")
if constants.rpmCheck:
if not returnVal and constants.rpmCheckStopOnError:
self.logger.error("Checking rpm is failed " + specFile)
raise Exception("RPM check failed")
else:
if not returnVal:
self.logger.error("Building rpm is failed " + specFile)
raise Exception("RPM build failed")
#Extracting rpms created from log file
listRPMFiles = []
listSRPMFiles = []
with open(logFile, 'r') as logfile:
fileContents = logfile.readlines()
for i in range(0, len(fileContents)):
if re.search("^Wrote:", fileContents[i]):
listcontents = fileContents[i].split()
if ((len(listcontents) == 2) and
listcontents[1].strip().endswith(".rpm") and
"/RPMS/" in listcontents[1]):
listRPMFiles.append(listcontents[1])
if ((len(listcontents) == 2) and
listcontents[1].strip().endswith(".src.rpm") and
"/SRPMS/" in listcontents[1]):
listSRPMFiles.append(listcontents[1])
return listRPMFiles, listSRPMFiles
def buildRPMSForGivenPackageInContainer(self, package, containerID, destLogPath=None, index=0):
self.logger.info("Building rpm's for package " + package + " in container " +
containerID.short_id)
listSourcesFiles = SPECS.getData().getSources(package, index)
listPatchFiles = SPECS.getData().getPatches(package, index)
specFile = SPECS.getData().getSpecFile(package, index)
specName = SPECS.getData().getSpecName(package) + ".spec"
sourcePath = constants.topDirPath + "/SOURCES/"
specPath = constants.topDirPath + "/SPECS/"
rpmLogFile = constants.topDirPath + "/LOGS/" + package + ".log"
destLogFile = destLogPath + "/" + package + ".log"
cmdUtils = CommandUtils()
#TODO: mount it in, don't copy
cpSpecCmd = "docker cp " + specFile + " " + containerID.short_id \
+ ":" + specPath + specName
returnVal = cmdUtils.runCommandInShell(cpSpecCmd)
if not returnVal:
self.logger.error("Error copying source SPEC file to container")
raise Exception("Failed copying source SPEC to container")
#FIXME: some sources are located in SPECS/.. how to mount?
# if os.geteuid()==0:
#TODO: mount it in, don't copy
macros = []
self._copySourcesToContainer(listSourcesFiles, package, containerID, sourcePath, index)
#TODO: mount it in, don't copy
self._copySourcesToContainer(listPatchFiles, package, containerID, sourcePath, index)
listAdditionalFiles, macros = self._getAdditionalBuildFiles(package)
self._copyAdditionalBuildFilesToContainer(listAdditionalFiles, containerID)
# Add rpm macros
listRPMMacros = constants.userDefinedMacros
for macroName, value in listRPMMacros.items():
macros.append(macroName + " " + value)
# Build RPMs
listRPMFiles = []
listSRPMFiles = []
try:
listRPMFiles, listSRPMFiles = self._buildRPMinContainer(
specPath + specName,
rpmLogFile,
destLogFile,
containerID,
package,
macros)
self.logger.info("Successfully built rpm:" + package)
except Exception as e:
self.logger.error("Failed while building rpm: " + package)
raise e
finally:
if destLogPath is not None:
rpmLog = destLogPath + "/" + package + ".log"
if (constants.rpmCheck and
package in constants.testForceRPMS and
SPECS.getData().isCheckAvailable(package, index)):
cmd = "sed -i '/^Executing(%check):/,/^Processing files:/{//!b};d' " + rpmLog
logFile = destLogPath + "/adjustTestFile.log"
returnVal = CommandUtils().runCommandInShell(cmd, logFile)
testLogFile = destLogPath + "/" + package + "-test.log"
shutil.copyfile(rpmLog, testLogFile)
self.logger.info("RPM build is successful")
# Verify RPM and SRPM files exist as success criteria
for rpmFile in listRPMFiles:
rpmName = os.path.basename(rpmFile)
rpmDestDir = self._getRPMDestDir(rpmName, constants.rpmPath)
rpmDestFile = rpmDestDir + "/" + rpmName
if not os.path.isfile(rpmDestFile):
self.logger.error("Could not find RPM file: " + rpmDestFile)
raise Exception("Built RPM file not found.")
for srpmFile in listSRPMFiles:
srpmName = os.path.basename(srpmFile)
srpmDestDir = self._getRPMDestDir(os.path.basename(srpmFile), constants.sourceRpmPath)
srpmDestFile = srpmDestDir + "/" + srpmName
if not os.path.isfile(srpmDestFile):
self.logger.error("Could not find RPM file: " + srpmDestFile)
raise Exception("Built SRPM file not found.")