def __constructBuildAndRunTimeDependencyGraph(self, package):
basePackage = SPECS.getData().getSpecName(package)
addBuildTimeGraph = True
addRunTimeGraph = True
if basePackage in self.__buildDependencyGraph:
addBuildTimeGraph = False
if basePackage in self.__runTimeDependencyGraph:
addRunTimeGraph = False
nextPackagesToConstructGraph = []
if addBuildTimeGraph:
listDependentRpmPackages = SPECS.getData(
).getBuildRequiresForPackage(basePackage)
listDependentPackages = []
for rpmPkg in listDependentRpmPackages:
basePkg = SPECS.getData().getSpecName(rpmPkg)
if basePkg not in listDependentPackages:
listDependentPackages.append(basePkg)
self.__buildDependencyGraph[basePackage] = listDependentPackages
nextPackagesToConstructGraph.extend(listDependentPackages)
if addRunTimeGraph:
listRpmPackages = SPECS.getData().getPackages(basePackage)
for rpmPkg in listRpmPackages:
listDependentRpmPackages = SPECS.getData(
).getRequiresAllForPackage(rpmPkg)
self.__runTimeDependencyGraph[
rpmPkg] = listDependentRpmPackages[:]
nextPackagesToConstructGraph.extend(listDependentRpmPackages)
for pkg in nextPackagesToConstructGraph:
self.__constructBuildAndRunTimeDependencyGraph(pkg)
def buildRPMSForGivenPackage(self, package, chrootID, destLogPath=None):
self.logger.info("Building rpm's for package:" + package)
listSourcesFiles = SPECS.getData().getSources(package)
listPatchFiles = SPECS.getData().getPatches(package)
specFile = SPECS.getData().getSpecFile(package)
specName = SPECS.getData().getSpecName(package) + ".spec"
chrootSourcePath = chrootID + constants.topDirPath + "/SOURCES/"
chrootSpecPath = constants.topDirPath + "/SPECS/"
chrootLogsFilePath = chrootID + constants.topDirPath + "/LOGS/" + package + ".log"
chrootCmd = self.runInChrootCommand + " " + chrootID
shutil.copyfile(specFile, chrootID + chrootSpecPath + specName)
# FIXME: some sources are located in SPECS/.. how to mount?
# if os.geteuid()==0:
self._copySourcesTobuildroot(listSourcesFiles, package,
chrootSourcePath)
self._copySourcesTobuildroot(listPatchFiles, package, chrootSourcePath)
macros = []
listAdditionalFiles, macros = self._getAdditionalBuildFiles(package)
self._copyAdditionalBuildFiles(listAdditionalFiles, chrootID)
#Adding rpm macros
listRPMMacros = constants.userDefinedMacros
for macroName, value in listRPMMacros.items():
macros.append(macroName + " " + value)
listRPMFiles = []
listSRPMFiles = []
try:
listRPMFiles, listSRPMFiles = self._buildRPM(
chrootSpecPath + specName, chrootLogsFilePath, chrootCmd,
package, macros)
self.logger.info("Successfully built rpm:" + package)
except Exception as e:
self.logger.error("Failed while building rpm:" + package)
raise e
finally:
if destLogPath is not None:
if (constants.rpmCheck and package in constants.testForceRPMS
and SPECS.getData().isCheckAvailable(package)):
cmd = (
"sed -i '/^Executing(%check):/,/^Processing files:/{//!b};d' "
+ chrootLogsFilePath)
logFile = destLogPath + "/adjustTestFile.log"
returnVal = CommandUtils().runCommandInShell(cmd, logFile)
testLogFile = destLogPath + "/" + package + "-test.log"
shutil.copyfile(chrootLogsFilePath, testLogFile)
else:
shutil.copy2(chrootLogsFilePath, destLogPath)
self.logger.info("RPM build is successful")
for rpmFile in listRPMFiles:
self._copyRPM(chrootID + "/" + rpmFile, constants.rpmPath)
for srpmFile in listSRPMFiles:
srpmDestFile = self._copyRPM(chrootID + "/" + srpmFile,
constants.sourceRpmPath)
def _getListNextPackagesReadyToBuild():
for pkg in Scheduler.listOfPackagesToBuild:
if pkg in Scheduler.listOfPackagesCurrentlyBuilding:
continue
listRequiredSubPackages = list(set(SPECS.getData().getBuildRequiresForPkg(pkg) + \
SPECS.getData().getRequiresAllForPkg(pkg)))
# extend to full Requires tree
for p in listRequiredSubPackages:
reqs = SPECS.getData().getRequiresAllForPkg(p)
for r in reqs:
if r not in listRequiredSubPackages:
listRequiredSubPackages.append(r)
# convert subpackages to basepkg
listRequiredPackages = set()
for p in listRequiredSubPackages:
listRequiredPackages.add(SPECS.getData().getBasePkg(p))
canBuild = True
for reqPkg in listRequiredPackages:
if reqPkg not in Scheduler.listOfAlreadyBuiltPackages:
canBuild = False
break
#if canBuild and constants.currentArch == "arm":
# canBuild = pkg in constants.customPackageList
# if not canBuild:
# Scheduler.logger.debug(">>>> %s is not in custom package list" % pkg)
if canBuild:
Scheduler.listOfPackagesNextToBuild.put((-Scheduler._getPriority(pkg), pkg))
Scheduler.logger.debug("Adding " + pkg + " to the schedule list")
def loadPackagesData(self):
listPackages = SPECS.getData().getListPackages()
listPackages.sort()
cmdUtils = CommandUtils()
for package in listPackages:
for version in SPECS.getData().getVersions(package):
release = SPECS.getData().getRelease(package, version)
listRPMPackages = SPECS.getData().getRPMPackages(package, version)
srpmFileName = package + "-" + version + "-" + release + ".src.rpm"
srpmFiles = cmdUtils.findFile(srpmFileName, constants.sourceRpmPath)
srpmFile = None
if len(srpmFiles) == 1:
srpmFile = srpmFiles[0]
debugrpmFileName = package + "-debuginfo-" + version + "-" + release + "*"
debugrpmFiles = cmdUtils.findFile(debugrpmFileName, constants.rpmPath)
debugrpmFile = None
if len(debugrpmFiles) == 1:
debugrpmFile = debugrpmFiles[0]
pkgUtils = PackageUtils(self.logName, self.logPath)
for rpmPkg in listRPMPackages:
rpmFile = pkgUtils.findRPMFile(rpmPkg, version)
if rpmFile is not None:
listPkgAttributes = {"sourcerpm":srpmFile, "rpm":rpmFile,
"debugrpm":debugrpmFile}
self.pkgList[rpmPkg+"-"+version] = listPkgAttributes
self.logger.debug("Added " + rpmPkg + "-" + version + " to the package info json")
else:
self.logger.debug("Missing rpm file for package:" + rpmPkg)
def installTargetToolchain(self, chroot, stopAtPackage=None):
self.logger.debug("Installing target toolchain RPMS.......")
pkgUtils = PackageUtils(self.logName, self.logPath)
rpmFiles = ""
packages = ""
for package in constants.listCoreToolChainPackages:
if stopAtPackage and package == stopAtPackage:
break
version = SPECS.getData().getHighestVersion(package)
basePkg = SPECS.getData().getSpecName(package)
# install all subpackages of given package
# for instance: for 'glibc' we want glibc-devel, glibc-tools,
# glibc-i18n, etc also to be installed
subpackages = SPECS.getData().getRPMPackages(basePkg, version)
for p in subpackages:
rpmFile = pkgUtils.findRPMFile(p, version, constants.targetArch)
rpmFiles += " " + rpmFile
packages += " " + package+"-"+version
self.logger.debug(packages)
cmd = "mkdir -p " + chroot.getID() +"/target-"+ constants.targetArch+"/var/lib/rpm"
CommandUtils.runCommandInShell(cmd, logfn=self.logger.debug)
if rpmFiles != "":
cmd = (self.rpmCommand+" -Uvh --nodeps --ignorearch --noscripts --root "+
chroot.getID() +"/target-"+ constants.targetArch+
" --define \'_dbpath /var/lib/rpm\' "+rpmFiles)
retVal = CommandUtils.runCommandInShell(cmd, logfn=self.logger.debug)
if retVal != 0:
self.logger.debug("Command Executed:" + cmd)
self.logger.error("Installing toolchain failed")
raise Exception("RPM installation failed")
self.logger.debug("Successfully installed target toolchain RPMS in chroot:" + chroot.getID())
def _getListNextPackagesReadyToBuild():
for pkg in Scheduler.listOfPackagesToBuild:
if pkg in Scheduler.listOfPackagesCurrentlyBuilding:
continue
listRequiredSubPackages = list(set(SPECS.getData().getBuildRequiresForPkg(pkg) + \
SPECS.getData().getRequiresAllForPkg(pkg)))
# extend to full Requires tree
for p in listRequiredSubPackages:
reqs = SPECS.getData().getRequiresAllForPkg(p)
for r in reqs:
if r not in listRequiredSubPackages:
listRequiredSubPackages.append(r)
# convert subpackages to basepkg
listRequiredPackages = set()
for p in listRequiredSubPackages:
listRequiredPackages.add(SPECS.getData().getBasePkg(p))
canBuild = True
for reqPkg in listRequiredPackages:
if reqPkg not in Scheduler.listOfAlreadyBuiltPackages:
canBuild = False
break
if canBuild:
Scheduler.listOfPackagesNextToBuild.put((-Scheduler._getPriority(pkg), pkg))
Scheduler.logger.debug("Adding " + pkg + " to the schedule list")
def getListDependentPackages(self, package, version):
listBuildRequiresPkg = SPECS.getData(
constants.buildArch).getBuildRequiresForPackage(package, version)
listBuildRequiresPkg.extend(
SPECS.getData(constants.buildArch).getCheckBuildRequiresForPackage(
package, version))
return listBuildRequiresPkg
def loadPackagesData(self):
listPackages = SPECS.getData().getListPackages()
listPackages.sort()
pkgUtils = PackageUtils(self.logName, self.logPath)
for package in listPackages:
for version in SPECS.getData().getVersions(package):
srpmFile = pkgUtils.findSourceRPMFile(package, version)
debugrpmFile = pkgUtils.findDebugRPMFile(package, version)
listRPMPackages = SPECS.getData().getRPMPackages(
package, version)
for rpmPkg in listRPMPackages:
rpmFile = pkgUtils.findRPMFile(rpmPkg, version)
if rpmFile is not None:
listPkgAttributes = {
"sourcerpm": srpmFile,
"rpm": rpmFile,
"debugrpm": debugrpmFile
}
self.pkgList[rpmPkg + "-" +
version] = listPkgAttributes
self.logger.debug("Added " + rpmPkg + "-" + version +
" to the package info json")
else:
self.logger.debug("Missing rpm file for package:" +
rpmPkg)
def findRPMFileForGivenPackage(self, package, version="*"):
cmdUtils = CommandUtils()
if version == "*":
version = SPECS.getData().getHighestVersion(package)
release = SPECS.getData().getRelease(package, version)
buildarch = SPECS.getData().getBuildArch(package, version)
listFoundRPMFiles = sum([
cmdUtils.findFile(
package + "-" + version + "-" + release + "." + buildarch +
".rpm", constants.rpmPath)
], [])
if constants.inputRPMSPath is not None:
listFoundRPMFiles = sum([
cmdUtils.findFile(
package + "-" + version + "-" + release + "." + buildarch +
".rpm", constants.inputRPMSPath)
], listFoundRPMFiles)
if len(listFoundRPMFiles) == 1:
return listFoundRPMFiles[0]
if len(listFoundRPMFiles) == 0:
return None
if len(listFoundRPMFiles) > 1:
self.logger.error(
"Found multiple rpm files for given package in rpm directory."
+ "Unable to determine the rpm file for package:" + package +
" : " + str(listFoundRPMFiles))
raise Exception("Multiple rpm files found")
def _verifyShaAndGetSourcePath(self, source, package, version):
cmdUtils = CommandUtils()
# Fetch/verify sources if sha1 not None.
sha1 = SPECS.getData().getSHA1(package, version, source)
if sha1 is not None:
PullSources.get(package, source, sha1, constants.sourcePath,
constants.getPullSourcesURLs(package), self.logger)
sourcePath = cmdUtils.findFile(source, constants.sourcePath)
if not sourcePath:
sourcePath = cmdUtils.findFile(source, os.path.dirname(SPECS.getData().getSpecFile(package, version)))
if not sourcePath:
if sha1 is None:
self.logger.error("No sha1 found or missing source for " + source)
raise Exception("No sha1 found or missing source for " + source)
else:
self.logger.error("Missing source: " + source +
". Cannot find sources for package: " + package)
raise Exception("Missing source")
else:
if sha1 is None:
self.logger.error("No sha1 found for "+source)
raise Exception("No sha1 found")
if len(sourcePath) > 1:
self.logger.error("Multiple sources found for source:" + source + "\n" +
",".join(sourcePath) +"\nUnable to determine one.")
raise Exception("Multiple sources found")
return sourcePath
def _constructBuildAndRunTimeDependencyGraph(self, basePackage):
addBuildTimeGraph = True
addRunTimeGraph = True
if basePackage in self.__buildDependencyGraph:
addBuildTimeGraph = False
if basePackage in self.__runTimeDependencyGraph:
addRunTimeGraph = False
nextPackagesToConstructGraph = set()
if addBuildTimeGraph:
dependentRpmPackages = SPECS.getData().getBuildRequiresForPkg(
basePackage)
dependentPackages = set()
for dependentPkg in dependentRpmPackages:
dependentPackages.add(SPECS.getData().getBasePkg(dependentPkg))
self.__buildDependencyGraph[basePackage] = dependentPackages
nextPackagesToConstructGraph.update(dependentPackages)
if addRunTimeGraph:
dependentPackages = set()
for rpmPkg in SPECS.getData().getPackagesForPkg(basePackage):
dependentRpmPackages = SPECS.getData().getRequiresAllForPkg(
rpmPkg)
self.__runTimeDependencyGraph[rpmPkg] = copy.copy(
set(dependentRpmPackages))
for pkg in dependentRpmPackages:
dependentPackages.add(SPECS.getData().getBasePkg(pkg))
nextPackagesToConstructGraph.update(dependentPackages)
for pkg in nextPackagesToConstructGraph:
self._constructBuildAndRunTimeDependencyGraph(pkg)
def loadPackagesData(self):
listPackages = SPECS.getData().getListPackages()
listPackages.sort()
listRPMFiles = []
cmdUtils = CommandUtils()
for package in listPackages:
release = SPECS.getData().getRelease(package)
version = SPECS.getData().getVersion(package)
listRPMPackages = SPECS.getData().getRPMPackages(package)
srpmFileName = package + "-" + version + "-" + release + ".src.rpm"
srpmFiles = cmdUtils.findFile(srpmFileName, constants.sourceRpmPath)
srpmFile = None
if len(srpmFiles) == 1:
srpmFile = srpmFiles[0]
debugrpmFileName = package + "-debuginfo-" + version + "-" + release + "*"
debugrpmFiles = cmdUtils.findFile(debugrpmFileName, constants.rpmPath)
debugrpmFile = None
if len(debugrpmFiles) == 1:
debugrpmFile = debugrpmFiles[0]
pkgUtils = PackageUtils(self.logName, self.logPath)
for rpmPkg in listRPMPackages:
rpmFile = pkgUtils.findRPMFileForGivenPackage(rpmPkg)
if rpmFile is not None:
listRPMFiles.append(rpmFile)
listPkgAttributes = {"sourcerpm":srpmFile, "rpm":rpmFile,
"debugrpm":debugrpmFile}
self.pkgList[rpmPkg] = listPkgAttributes
self.logger.debug("Added " + rpmPkg + " rpm package to the list")
else:
self.logger.error("Missing rpm file for package:" + rpmPkg)
def findRPMFileForGivenPackage(self, package,version="*", index=0):
cmdUtils = CommandUtils()
release = "*"
if version == "*":
version = SPECS.getData().getVersion(package, index)
release = SPECS.getData().getRelease(package, index)
listFoundRPMFiles = sum([cmdUtils.findFile(package + "-" + version + "-" + release + "." +
platform.machine()+".rpm",
constants.rpmPath),
cmdUtils.findFile(package + "-" + version + "-" + release +
".noarch.rpm",
constants.rpmPath)], [])
if constants.inputRPMSPath is not None:
listFoundRPMFiles = sum([cmdUtils.findFile(package + "-" + version + "-" + release +
"." + platform.machine()+".rpm",
constants.inputRPMSPath),
cmdUtils.findFile(package + "-" + version + "-" + release +
".noarch.rpm", constants.inputRPMSPath)],
listFoundRPMFiles)
if len(listFoundRPMFiles) == 1:
return listFoundRPMFiles[0]
if len(listFoundRPMFiles) == 0:
return None
if len(listFoundRPMFiles) > 1:
self.logger.error("Found multiple rpm files for given package in rpm directory." +
"Unable to determine the rpm file for package:" + package)
raise Exception("Multiple rpm files found")
def _verifyShaAndGetSourcePath(self, source, package, version):
cmdUtils = CommandUtils()
# Fetch/verify sources if sha1 not None.
sha1 = SPECS.getData().getSHA1(package, version, source)
if sha1 is not None:
PullSources.get(package, source, sha1, constants.sourcePath,
constants.getPullSourcesURLs(package), self.logger)
sourcePath = cmdUtils.findFile(source, constants.sourcePath)
if not sourcePath:
sourcePath = cmdUtils.findFile(source, os.path.dirname(SPECS.getData().getSpecFile(package, version)))
if not sourcePath:
if sha1 is None:
self.logger.error("No sha1 found or missing source for " + source)
raise Exception("No sha1 found or missing source for " + source)
else:
self.logger.error("Missing source: " + source +
". Cannot find sources for package: " + package)
raise Exception("Missing source")
else:
if sha1 is None:
self.logger.error("No sha1 found for "+source)
raise Exception("No sha1 found")
if len(sourcePath) > 1:
self.logger.error("Multiple sources found for source:" + source + "\n" +
",".join(sourcePath) +"\nUnable to determine one.")
raise Exception("Multiple sources found")
return sourcePath
def buildSRPMList(srpmPath,
yamlDir,
blackListPkgs,
dist_tag,
logger,
singleFile=True):
cmdUtils = CommandUtils()
yamlSrpmDir = os.path.join(yamlDir, "yaml_srpms")
if not os.path.isdir(yamlSrpmDir):
cmdUtils.runCommandInShell("mkdir -p " + yamlSrpmDir)
if singleFile:
yamlFile = open(yamlSrpmDir + "/srpm_list.yaml", "w")
listPackages = SPECS.getData().getListPackages()
listPackages.sort()
for package in listPackages:
if package in blackListPkgs:
continue
ossname = package
for ossversion in SPECS.getData().getVersions(package):
ossrelease = SPECS.getData().getRelease(package, ossversion)
srpm_file_name = "%s-%s-%s%s.src.rpm" % (ossname, ossversion,
ossrelease, dist_tag)
logger.info("srpm name is %s" % (srpm_file_name))
listFoundSRPMFiles = cmdUtils.findFile(srpm_file_name, srpmPath)
srpmName = None
if len(listFoundSRPMFiles) == 1:
srpmFullPath = listFoundSRPMFiles[0]
srpmName = os.path.basename(srpmFullPath)
cpcmd = "cp " + srpmFullPath + " " + yamlSrpmDir + "/"
returnVal = cmdUtils.runCommandInShell(cpcmd)
if returnVal != 0:
logger.error("Copy SRPM File is failed for package:" +
ossname)
else:
logger.error("SRPM file is not found:" + ossname)
if not singleFile:
yamlFile = open(
yamlSrpmDir + "/" + ossname + "-" + ossversion + "-" +
ossrelease + ".yaml", "w")
yamlFile.write("baseos:" + ossname + ":" + ossversion + "-" +
ossrelease + ":\n")
yamlFile.write(" repository: BaseOS\n")
yamlFile.write(" name: '" + ossname + "'\n")
yamlFile.write(" version: '" + ossversion + "-" + ossrelease +
"'\n")
yamlFile.write(" url: 'http://www.vmware.com'\n")
yamlFile.write(" baseos-style: rpm\n")
yamlFile.write(" baseos-source: '" + str(srpmName) + "'\n")
yamlFile.write(" baseos-osname: 'photon'\n")
yamlFile.write("\n")
if not singleFile:
yamlFile.close()
if singleFile:
yamlFile.close()
logger.debug("Generated SRPM yaml files for all packages")
def _buildGivenPackages(self, listPackages, buildThreads):
# Extend listPackages from ["name1", "name2",..] to ["name1-vers1", "name2-vers2",..]
listPackageNamesAndVersions = []
for pkg in listPackages:
for version in SPECS.getData().getVersions(pkg):
listPackageNamesAndVersions.append(pkg + "-" + version)
if constants.rpmCheck:
listMakeCheckPackages = set()
for pkg in listPackages:
version = SPECS.getData().getHighestVersion(pkg)
listMakeCheckPackages.add(pkg + "-" + version)
alreadyBuiltRPMS = self._readAlreadyAvailablePackages()
listPackageNamesAndVersions = (list(
set(listPackageNamesAndVersions)
| (listMakeCheckPackages - alreadyBuiltRPMS)))
returnVal = self._calculateParams(listPackageNamesAndVersions)
if not returnVal:
self.logger.error(
"Unable to set paramaters. Terminating the package manager.")
raise Exception("Unable to set paramaters")
statusEvent = threading.Event()
self._initializeScheduler(statusEvent)
self._initializeThreadPool(statusEvent)
for i in range(0, buildThreads):
workerName = "WorkerThread" + str(i)
ThreadPool.addWorkerThread(workerName)
ThreadPool.startWorkerThread(workerName)
statusEvent.wait()
Scheduler.stopScheduling = True
self.logger.info("Waiting for all remaining worker threads")
ThreadPool.join_all()
setFailFlag = False
allPackagesBuilt = False
if Scheduler.isAnyPackagesFailedToBuild():
setFailFlag = True
if Scheduler.isAllPackagesBuilt():
allPackagesBuilt = True
if setFailFlag:
self.logger.error("Some of the packages failed:")
self.logger.error(Scheduler.listOfFailedPackages)
raise Exception("Failed during building package")
if not setFailFlag:
if allPackagesBuilt:
self.logger.info("All packages built successfully")
else:
self.logger.error("Build stopped unexpectedly.Unknown error.")
raise Exception("Unknown error")
self.logger.info("Terminated")
def buildRPMSForGivenPackage(self, package, chrootID, destLogPath=None, index=0):
self.logger.info("Building rpm's for package:" + package)
listSourcesFiles = SPECS.getData().getSources(package, index)
listPatchFiles = SPECS.getData().getPatches(package, index)
specFile = SPECS.getData().getSpecFile(package, index)
specName = SPECS.getData().getSpecName(package) + ".spec"
chrootSourcePath = chrootID + constants.topDirPath + "/SOURCES/"
chrootSpecPath = constants.topDirPath + "/SPECS/"
chrootLogsFilePath = chrootID + constants.topDirPath + "/LOGS/" + package + ".log"
chrootCmd = self.runInChrootCommand + " " + chrootID
shutil.copyfile(specFile, chrootID + chrootSpecPath + specName)
# FIXME: some sources are located in SPECS/.. how to mount?
# if os.geteuid()==0:
self._copySourcesTobuildroot(listSourcesFiles, package, chrootSourcePath, index)
self._copySourcesTobuildroot(listPatchFiles, package, chrootSourcePath, index)
macros = []
listAdditionalFiles, macros = self._getAdditionalBuildFiles(package)
self._copyAdditionalBuildFiles(listAdditionalFiles, chrootID)
#Adding rpm macros
listRPMMacros = constants.userDefinedMacros
for macroName, value in listRPMMacros.items():
macros.append(macroName + " " + value)
listRPMFiles = []
listSRPMFiles = []
try:
listRPMFiles, listSRPMFiles = self._buildRPM(chrootSpecPath + specName,
chrootLogsFilePath, chrootCmd,
package, macros)
self.logger.info("Successfully built rpm:" + package)
except Exception as e:
self.logger.error("Failed while building rpm:" + package)
raise e
finally:
if destLogPath is not None:
if (constants.rpmCheck and
package in constants.testForceRPMS and
SPECS.getData().isCheckAvailable(package)):
cmd = ("sed -i '/^Executing(%check):/,/^Processing files:/{//!b};d' " +
chrootLogsFilePath)
logFile = destLogPath + "/adjustTestFile.log"
returnVal = CommandUtils().runCommandInShell(cmd, logFile)
testLogFile = destLogPath + "/" + package + "-test.log"
shutil.copyfile(chrootLogsFilePath, testLogFile)
else:
shutil.copy2(chrootLogsFilePath, destLogPath)
self.logger.info("RPM build is successful")
for rpmFile in listRPMFiles:
self._copyRPM(chrootID + "/" + rpmFile, constants.rpmPath)
for srpmFile in listSRPMFiles:
srpmDestFile = self._copyRPM(chrootID + "/" + srpmFile, constants.sourceRpmPath)
def updateLevels(self, mapDependencies, inPkg, parent, level):
listPackages = SPECS.getData().getPackagesForPkg(inPkg)
for depPkg in SPECS.getData().getRequiresForPkg(inPkg):
if depPkg in listPackages:
continue
if depPkg in mapDependencies and mapDependencies[depPkg] < level + 1:
mapDependencies[depPkg] = level + 1
parent[depPkg] = inPkg
self.updateLevels(mapDependencies, depPkg, parent, mapDependencies[depPkg])
def getBasePackagesRequired(self, pkg):
listBasePackagesRequired=[]
listPackagesRequired = SPECS.getData().getBuildRequiresForPkg(pkg)
listPackagesRequired.extend(SPECS.getData().getRequiresAllForPkg(pkg))
for p in listPackagesRequired:
basePkg = SPECS.getData().getBasePkg(p)
if basePkg not in listBasePackagesRequired:
listBasePackagesRequired.append(basePkg)
return listBasePackagesRequired
def buildCoreToolChainPackages(self):
self.logger.info("Step 1 : Building the core toolchain packages.....")
self.logger.info(constants.listCoreToolChainPackages)
self.logger.info("")
chrootID = None
pkgCount = 0
try:
pkgUtils = PackageUtils(self.logName, self.logPath)
coreToolChainYetToBuild = []
for package in constants.listCoreToolChainPackages:
version = SPECS.getData().getHighestVersion(package)
rpmPkg = pkgUtils.findRPMFileForGivenPackage(package, version)
if rpmPkg is not None:
continue
else:
coreToolChainYetToBuild.append(package)
if coreToolChainYetToBuild:
self.logger.info(
"The following core toolchain packages need to be built :")
self.logger.info(coreToolChainYetToBuild)
else:
self.logger.info(
"Core toolchain packages are already available")
for package in coreToolChainYetToBuild:
version = SPECS.getData().getHighestVersion(package)
self.logger.debug("Building core toolchain package : " +
package)
chrUtils = ChrootUtils(self.logName, self.logPath)
chrootName = "build-" + package
destLogPath = constants.logPath + "/build-" + package
if not os.path.isdir(destLogPath):
cmdUtils = CommandUtils()
cmdUtils.runCommandInShell("mkdir -p " + destLogPath)
returnVal, chrootID = chrUtils.createChroot(chrootName)
if not returnVal:
self.logger.error("Creating chroot failed")
raise Exception("creating chroot failed")
self.installToolChainRPMS(package, version, chrootID,
destLogPath)
pkgUtils.adjustGCCSpecs(package, version, chrootID,
destLogPath)
pkgUtils.buildRPMSForGivenPackage(package, version, chrootID,
destLogPath)
pkgCount += 1
chrUtils.destroyChroot(chrootID)
chrootID = None
self.logger.debug("Successfully built toolchain")
self.logger.info("-" * 45 + "\n")
if chrootID is not None:
chrUtils.destroyChroot(chrootID)
except Exception as e:
self.logger.error("Unable to build tool chain.")
# print stacktrace
traceback.print_exc()
raise e
return pkgCount
def getListDependentPackageLineContent(self, index):
listBuildRequiresPkgLineContent = SPECS.getData(
).getBuildRequiresForPackage(self.package, index)
listBuildRequiresPkgLineContent.extend(
SPECS.getData().getCheckBuildRequiresForPackage(
self.package, index))
listBuildRequiresPkgLineContent = list(
set(listBuildRequiresPkgLineContent))
return listBuildRequiresPkgLineContent
def updateLevels(self, mapDependencies, inPkg, parent, level):
listPackages = SPECS.getData().getPackagesForPkg(inPkg)
for depPkg in SPECS.getData().getRequiresForPkg(inPkg):
if depPkg in listPackages:
continue
if depPkg in mapDependencies and mapDependencies[depPkg] < level + 1:
mapDependencies[depPkg] = level + 1
parent[depPkg] = inPkg
self.updateLevels(mapDependencies, depPkg, parent, mapDependencies[depPkg])
def getBasePackagesRequired(self, pkg):
listBasePackagesRequired=[]
listPackagesRequired = SPECS.getData().getBuildRequiresForPkg(pkg)
listPackagesRequired.extend(SPECS.getData().getRequiresAllForPkg(pkg))
for p in listPackagesRequired:
basePkg = SPECS.getData().getBasePkg(p)
if basePkg not in listBasePackagesRequired:
listBasePackagesRequired.append(basePkg)
return listBasePackagesRequired
def checkIfPackageIsAlreadyBuilt(self):
basePkg = SPECS.getData().getSpecName(self.package)
listRPMPackages = SPECS.getData().getRPMPackages(basePkg)
packageIsAlreadyBuilt = True
pkgUtils = PackageUtils(self.logName, self.logPath)
for pkg in listRPMPackages:
if pkgUtils.findRPMFileForGivenPackage(pkg) is None:
packageIsAlreadyBuilt = False
break
return packageIsAlreadyBuilt
def _checkIfPackageIsAlreadyBuilt(self):
basePkg = SPECS.getData().getSpecName(self.package)
listRPMPackages = SPECS.getData().getRPMPackages(basePkg)
packageIsAlreadyBuilt = True
pkgUtils = PackageUtils(self.logName, self.logPath)
for pkg in listRPMPackages:
if pkgUtils.findRPMFileForGivenPackage(pkg) is None:
packageIsAlreadyBuilt = False
break
return packageIsAlreadyBuilt
def _checkIfPackageIsAlreadyBuilt(self, package, version):
basePkg = SPECS.getData().getSpecName(package)
listRPMPackages = SPECS.getData().getRPMPackages(basePkg, version)
packageIsAlreadyBuilt = True
pkgUtils = PackageUtils()
for pkg in listRPMPackages:
if pkgUtils.findRPMFileForGivenPackage(pkg, version) is None:
packageIsAlreadyBuilt = False
break
return packageIsAlreadyBuilt
def findSourceRPMFile(self, package,version="*"):
if version == "*":
version = SPECS.getData().getHighestVersion(package)
release = SPECS.getData().getRelease(package, version)
filename= package + "-" + version + "-" + release + "src.rpm"
fullpath = constants.sourceRpmPath + "/" + filename
if os.path.isfile(fullpath):
return fullpath
return None
def _findBuildTimeRequiredPackages(self, arch):
deps = SPECS.getData(arch).getBuildRequiresForPackage(
self.package, self.version)
# Add BuildRequiresNative list
if constants.crossCompiling and arch == constants.buildArch:
deps.extend(
SPECS.getData(arch).getBuildRequiresNativeForPackage(
self.package, self.version))
return deps
def _getBuildRequiredPackages(package):
listRequiredRPMPackages = []
listRequiredRPMPackages.extend(SPECS.getData().getBuildRequiresForPackage(package))
listRequiredPackages = []
for pkg in listRequiredRPMPackages:
basePkg = SPECS.getData().getSpecName(pkg.package)
if basePkg not in listRequiredPackages:
listRequiredPackages.append(basePkg)
return listRequiredPackages
def findTotalWhoNeeds(self, depList, whoNeeds):
while depList:
pkg = depList.pop(0)
for depPackage in SPECS.getData().getListPackages():
for version in SPECS.getData().getVersions(depPackage):
depBasePkg = depPackage+"-"+version
if depBasePkg in whoNeeds:
continue
if pkg in self.getBasePackagesRequired(depBasePkg):
whoNeeds.append(depBasePkg)
if depBasePkg not in depList:
depList.append(depBasePkg)
def _getBuildRequiredPackages(pkg):
listRequiredRPMPackages = []
listRequiredRPMPackages.extend(SPECS.getData().getBuildRequiresForPkg(pkg))
listRequiredPackages = []
for reqPkg in listRequiredRPMPackages:
basePkg = SPECS.getData().getBasePkg(reqPkg)
if basePkg not in listRequiredPackages:
listRequiredPackages.append(basePkg)
return listRequiredPackages
def findTotalWhoNeeds(self, depList, whoNeeds):
while depList:
pkg = depList.pop(0)
for depPackage in SPECS.getData().getListPackages():
for version in SPECS.getData().getVersions(depPackage):
depBasePkg = depPackage+"-"+version
if depBasePkg in whoNeeds:
continue
if pkg in self.getBasePackagesRequired(depBasePkg):
whoNeeds.append(depBasePkg)
if depBasePkg not in depList:
depList.append(depBasePkg)
def process(self, inputType, inputValue, displayOption, outputFile=None):
whoNeedsList = []
inputPackages = []
whatNeedsBuild = []
mapDependencies = {}
parent = {}
if inputType == "pkg" or inputType == "json":
if inputType == "pkg":
inputPackages.append(inputValue)
else:
inputPackages = self.getAllPackageNames(inputValue)
self.calculateSpecDependency(inputPackages, mapDependencies, parent)
if outputFile is not None:
return self.displayDependencies(displayOption, inputType, outputFile, mapDependencies, parent)
else:
return self.displayDependencies(displayOption, inputType, inputValue, mapDependencies, parent)
elif inputType == "get-upward-deps":
depList = []
for specFile in inputValue.split(":"):
if specFile in SPECS.getData().mapSpecFileNameToSpecObj:
specObj = SPECS.getData().mapSpecFileNameToSpecObj[specFile]
whoNeedsList.append(specObj.name+"-"+specObj.version)
depList.append(specObj.name+"-"+specObj.version)
self.findTotalWhoNeeds(depList, whoNeedsList)
return whoNeedsList
elif inputType == "who-needs":
for depPackage in SPECS.getData().mapPackageToSpec:
pkg=inputValue+"-"+SPECS.getData().getHighestVersion(inputValue)
for version in SPECS.getData().getVersions(depPackage):
depPkg = depPackage+"-"+version
self.logger.info(depPkg)
if pkg in SPECS.getData().getRequiresForPkg(depPkg):
whoNeedsList.append(depPkg)
self.logger.info(whoNeedsList)
return whoNeedsList
elif inputType == "all-requires":
pkg=inputValue+"-"+SPECS.getData().getHighestVersion(inputValue)
requires = SPECS.getData().getRequiresTreeOfBasePkgsForPkg(pkg)
requires.sort()
self.logger.info(requires)
return requires
elif inputType == "is-toolchain-pkg":
for specFile in inputValue.split(":"):
if specFile in SPECS.getData().mapSpecFileNameToSpecObj:
specObj = SPECS.getData().mapSpecFileNameToSpecObj[specFile]
if (specObj.name in constants.listCoreToolChainPackages) \
or (specObj.name in constants.listToolChainPackages):
return True
return False
def findDebugRPMFile(self, package,version="*",arch=None):
if not arch:
arch=constants.currentArch
if version == "*":
version = SPECS.getData(arch).getHighestVersion(package)
release = SPECS.getData(arch).getRelease(package, version)
filename= package + "-debuginfo-" + version + "-" + release + "." + arch +".rpm"
fullpath = constants.rpmPath + "/" + arch + "/" + filename
if os.path.isfile(fullpath):
return fullpath
return None
def _buildGivenPackages(self, listPackages, buildThreads):
# Extend listPackages from ["name1", "name2",..] to ["name1-vers1", "name2-vers2",..]
listPackageNamesAndVersions=set()
for pkg in listPackages:
base = SPECS.getData().getSpecName(pkg)
for version in SPECS.getData().getVersions(base):
listPackageNamesAndVersions.add(base+"-"+version)
returnVal = self._calculateParams(listPackageNamesAndVersions)
if not returnVal:
self.logger.error("Unable to set parameters. Terminating the package manager.")
raise Exception("Unable to set parameters")
self._buildPackages(buildThreads)
def calculateSpecDependency(self, inputPackages, mapDependencies, parent):
depQue = queue.Queue()
for package in inputPackages:
if SPECS.getData().isRPMPackage(package):
version = SPECS.getData().getHighestVersion(package)
pkg = package+"-"+version
if pkg not in mapDependencies:
mapDependencies[pkg] = 0
parent[pkg] = ""
depQue.put(pkg)
self.findTotalRequires(mapDependencies, depQue, parent)
else:
self.logger.info("Could not find spec for " + package)
def calculateSpecDependency(self, inputPackages, mapDependencies, parent):
depQue = queue.Queue()
for package in inputPackages:
if SPECS.getData().isRPMPackage(package):
for version in SPECS.getData().getVersions(package):
pkg = package+"-"+version
if pkg not in mapDependencies:
mapDependencies[pkg] = 0
parent[pkg] = ""
depQue.put(pkg)
self.findTotalRequires(mapDependencies, depQue, parent)
else:
self.logger.info("Could not find spec for " + package)
def buildCoreToolChainPackages(self):
self.logger.info("Step 1 : Building the core toolchain packages for " + constants.currentArch)
self.logger.info(constants.listCoreToolChainPackages)
self.logger.info("")
chroot = None
pkgCount = 0
try:
pkgUtils = PackageUtils(self.logName, self.logPath)
coreToolChainYetToBuild = []
doneList = []
#self.logger.debug(">>>> Looking for available packages...")
for package in constants.listCoreToolChainPackages:
version = SPECS.getData().getHighestVersion(package)
rpmPkg = pkgUtils.findRPMFile(package, version)
#rpmPkg = pkgUtils.findRPMFile(package, version, constants.buildArch)
if rpmPkg is not None:
doneList.append(package+'-'+version)
#self.logger.debug(">>>>>> %s", package+'-'+version)
continue
else:
coreToolChainYetToBuild.append(package)
if coreToolChainYetToBuild:
self.logger.info("The following core toolchain packages need to be built :")
self.logger.info(coreToolChainYetToBuild)
else:
self.logger.info("Core toolchain packages are already available")
for package in coreToolChainYetToBuild:
self.logger.debug("Building core toolchain package : " + package)
version = SPECS.getData().getHighestVersion(package)
destLogPath = constants.logPath + "/" + package + "-" + version + "." + constants.currentArch
if not os.path.isdir(destLogPath):
CommandUtils.runCommandInShell("mkdir -p " + destLogPath)
chroot = Chroot(self.logger)
chroot.create(package + "-" + version)
#self.logger.debug(">>>>> Done List:")
#self.logger.info(doneList)
self.installToolchainRPMS(chroot, package, version, availablePackages=doneList)
pkgUtils.adjustGCCSpecs(chroot, package, version)
pkgUtils.buildRPMSForGivenPackage(chroot, package, version, destLogPath)
pkgCount += 1
chroot.destroy()
doneList.append(package+'-'+version)
self.logger.debug("Successfully built toolchain")
self.logger.info("-" * 45 + "\n")
except Exception as e:
self.logger.error("Unable to build toolchain.")
# print stacktrace
traceback.print_exc()
raise e
return pkgCount
def buildSRPMList(srpmPath, yamlDir, blackListPkgs, dist_tag, logger, singleFile=True):
cmdUtils = CommandUtils()
yamlSrpmDir = os.path.join(yamlDir, "yaml_srpms")
if not os.path.isdir(yamlSrpmDir):
cmdUtils.runCommandInShell("mkdir -p " + yamlSrpmDir)
if singleFile:
yamlFile = open(yamlSrpmDir + "/srpm_list.yaml", "w")
listPackages = SPECS.getData().getListPackages()
listPackages.sort()
for package in listPackages:
if package in blackListPkgs:
continue
ossname = package
for ossversion in SPECS.getData().getVersions(package):
ossrelease = SPECS.getData().getRelease(package, ossversion)
srpm_file_name = "%s-%s-%s%s.src.rpm" % (ossname, ossversion, ossrelease, dist_tag)
logger.info("srpm name is %s" % (srpm_file_name))
listFoundSRPMFiles = cmdUtils.findFile(srpm_file_name, srpmPath)
srpmName = None
if len(listFoundSRPMFiles) == 1:
srpmFullPath = listFoundSRPMFiles[0]
srpmName = os.path.basename(srpmFullPath)
cpcmd = "cp " + srpmFullPath + " " + yamlSrpmDir + "/"
returnVal = cmdUtils.runCommandInShell(cpcmd)
if returnVal != 0:
logger.error("Copy SRPM File is failed for package:" + ossname)
else:
logger.error("SRPM file is not found:" + ossname)
if not singleFile:
yamlFile = open(yamlSrpmDir + "/" + ossname + "-" + ossversion + "-"
+ ossrelease + ".yaml", "w")
yamlFile.write("baseos:" + ossname + ":" + ossversion + "-" + ossrelease + dist_tag + ":\n")
yamlFile.write(" repository: BaseOS\n")
yamlFile.write(" name: '" + ossname + "'\n")
yamlFile.write(" version: '" + ossversion + "-" + ossrelease + dist_tag +"'\n")
yamlFile.write(" url: 'http://www.vmware.com'\n")
yamlFile.write(" baseos-style: rpm\n")
yamlFile.write(" baseos-source: '" + str(srpmName) + "'\n")
yamlFile.write(" baseos-osname: 'photon'\n")
yamlFile.write("\n")
if not singleFile:
yamlFile.close()
if singleFile:
yamlFile.close()
logger.debug("Generated SRPM yaml files for all packages")
def _calculateParams(self, listPackages):
self.mapCyclesToPackageList.clear()
self.mapPackageToCycle.clear()
self.sortedPackageList = []
self.listOfPackagesAlreadyBuilt = self._readAlreadyAvailablePackages()
updateBuiltRPMSList = False
while not updateBuiltRPMSList:
updateBuiltRPMSList = True
listOfPackagesAlreadyBuilt = list(self.listOfPackagesAlreadyBuilt)
for pkg in listOfPackagesAlreadyBuilt:
listDependentRpmPackages = SPECS.getData().getRequiresAllForPackage(pkg)
needToRebuild = False
for dependentPkg in listDependentRpmPackages:
if dependentPkg not in self.listOfPackagesAlreadyBuilt:
needToRebuild = True
updateBuiltRPMSList = False
if needToRebuild:
self.listOfPackagesAlreadyBuilt.remove(pkg)
listPackagesToBuild = copy.copy(listPackages)
for pkg in listPackages:
if (pkg in self.listOfPackagesAlreadyBuilt and
not constants.rpmCheck):
listPackagesToBuild.remove(pkg)
if not self._readPackageBuildData(listPackagesToBuild):
return False
return True
def adjustGCCSpecs(self, package, version, chrootID, logPath):
opt = " " + SPECS.getData().getSecurityHardeningOption(
package, version)
cmdUtils = CommandUtils()
cpcmd = ("cp " + self.adjustGCCSpecScript + " " + chrootID + "/tmp/" +
self.adjustGCCSpecScript)
cmd = "/tmp/" + self.adjustGCCSpecScript + opt
logFile = logPath + "/adjustGCCSpecScript.log"
chrootCmd = self.runInChrootCommand + " " + chrootID
returnVal = cmdUtils.runCommandInShell(cpcmd, logFile)
if not returnVal:
self.logger.error("Error during copying the file adjust gcc spec")
raise Exception("Failed while copying adjust gcc spec file")
returnVal = cmdUtils.runCommandInShell(cmd, logFile, chrootCmd)
if returnVal:
return
self.logger.debug(
cmdUtils.runCommandInShell2("ls -la " + chrootID + "/tmp/" +
self.adjustGCCSpecScript))
self.logger.debug(
cmdUtils.runCommandInShell2("lsof " + chrootID + "/tmp/" +
self.adjustGCCSpecScript))
self.logger.debug(cmdUtils.runCommandInShell2("ps ax"))
self.logger.error("Failed while adjusting gcc specs")
raise Exception("Failed while adjusting gcc specs")
def _buildGivenPackages(self, listPackages, buildThreads):
# Extend listPackages from ["name1", "name2",..] to ["name1-vers1", "name2-vers2",..]
listPackageNamesAndVersions=set()
for pkg in listPackages:
base = SPECS.getData().getSpecName(pkg)
for version in SPECS.getData().getVersions(base):
listPackageNamesAndVersions.add(base+"-"+version)
returnVal = self._calculateParams(listPackageNamesAndVersions)
if not returnVal:
self.logger.error("Unable to set parameters. Terminating the package manager.")
raise Exception("Unable to set parameters")
statusEvent = threading.Event()
self._initializeScheduler(statusEvent)
self._initializeThreadPool(statusEvent)
for i in range(0, buildThreads):
workerName = "WorkerThread" + str(i)
ThreadPool.addWorkerThread(workerName)
ThreadPool.startWorkerThread(workerName)
statusEvent.wait()
Scheduler.stopScheduling = True
self.logger.debug("Waiting for all remaining worker threads")
ThreadPool.join_all()
setFailFlag = False
allPackagesBuilt = False
if Scheduler.isAnyPackagesFailedToBuild():
setFailFlag = True
if Scheduler.isAllPackagesBuilt():
allPackagesBuilt = True
if setFailFlag:
self.logger.error("Some of the packages failed:")
self.logger.error(Scheduler.listOfFailedPackages)
raise Exception("Failed during building package")
if not setFailFlag:
if allPackagesBuilt:
self.logger.debug("All packages built successfully")
else:
self.logger.error("Build stopped unexpectedly.Unknown error.")
raise Exception("Unknown error")
def _readAlreadyAvailablePackages(self):
listAvailablePackages = set()
pkgUtils = PackageUtils(self.logName, self.logPath)
listPackages = SPECS.getData().getListPackages()
for package in listPackages:
for version in SPECS.getData().getVersions(package):
# Mark package available only if all subpackages are available
packageIsAlreadyBuilt=True
listRPMPackages = SPECS.getData().getRPMPackages(package, version)
for rpmPkg in listRPMPackages:
if pkgUtils.findRPMFile(rpmPkg, version) is None:
packageIsAlreadyBuilt=False
break;
if packageIsAlreadyBuilt:
listAvailablePackages.add(package+"-"+version)
return listAvailablePackages
def __getRequiredTypePackages(pkg, requiresType):
listRPMPackages = []
if requiresType == "build":
listRPMPackages.extend(SPECS.getData().getBuildRequiresForPkg(pkg))
elif requiresType == "install":
listRPMPackages.extend(SPECS.getData().getRequiresAllForPkg(pkg))
# Remove duplicates.
listRPMPackages = list(set(listRPMPackages))
listPackages = set()
for reqPkg in listRPMPackages:
basePkg = SPECS.getData().getBasePkg(reqPkg)
listPackages.add(basePkg)
return list(listPackages)
def findRPMFile(self, package,version="*"):
cmdUtils = CommandUtils()
if version == "*":
version = SPECS.getData().getHighestVersion(package)
release = SPECS.getData().getRelease(package, version)
buildarch=SPECS.getData().getBuildArch(package, version)
filename= package + "-" + version + "-" + release + "." + buildarch+".rpm"
fullpath = constants.rpmPath + "/" + buildarch + "/" + filename
if os.path.isfile(fullpath):
return fullpath
if constants.inputRPMSPath is not None:
fullpath = constants.inputRPMSPath + "/" + buildarch + "/" + filename
if os.path.isfile(fullpath):
return fullpath
return None
def _readAlreadyAvailablePackages(self):
listAvailablePackages = set()
listFoundRPMPackages = set()
listRPMFiles = set()
listDirectorys = set()
listDirectorys.add(constants.rpmPath)
if constants.inputRPMSPath is not None:
listDirectorys.add(constants.inputRPMSPath)
while listDirectorys:
dirPath = listDirectorys.pop()
for dirEntry in os.listdir(dirPath):
dirEntryPath = os.path.join(dirPath, dirEntry)
if os.path.isfile(dirEntryPath) and dirEntryPath.endswith(".rpm"):
listRPMFiles.add(dirEntryPath)
elif os.path.isdir(dirEntryPath):
listDirectorys.add(dirEntryPath)
pkgUtils = PackageUtils(self.logName, self.logPath)
for rpmfile in listRPMFiles:
package, version, release = pkgUtils.findPackageInfoFromRPMFile(rpmfile)
if SPECS.getData().isRPMPackage(package):
specVersion = SPECS.getData().getVersion(package)
specRelease = SPECS.getData().getRelease(package)
if version == specVersion and release == specRelease:
listFoundRPMPackages.add(package)
#Mark package available only if all sub packages are available
for package in listFoundRPMPackages:
basePkg = SPECS.getData().getSpecName(package)
if basePkg in listAvailablePackages:
continue
listRPMPackages = SPECS.getData().getRPMPackages(basePkg)
packageIsAlreadyBuilt = True
for rpmpkg in listRPMPackages:
if rpmpkg not in listFoundRPMPackages:
packageIsAlreadyBuilt = False
if packageIsAlreadyBuilt:
listAvailablePackages.add(package)
self.logger.info("List of Already built packages")
self.logger.info(listAvailablePackages)
return listAvailablePackages
def buildSourcesList(yamlDir, blackListPkgs, logger, singleFile=True):
cmdUtils = CommandUtils()
yamlSourceDir = os.path.join(yamlDir, "yaml_sources")
if not os.path.isdir(yamlSourceDir):
cmdUtils.runCommandInShell("mkdir -p " + yamlSourceDir)
if singleFile:
yamlFile = open(yamlSourceDir + "/sources_list.yaml", "w")
listPackages = SPECS.getData().getListPackages()
listPackages.sort()
import PullSources
for package in listPackages:
if package in blackListPkgs:
continue
ossname = package
for version in SPECS.getData().getVersions(package):
modified = False
listPatches = SPECS.getData().getPatches(package, version)
if listPatches:
modified = True
url = SPECS.getData().getSourceURL(package, version)
if url is None:
url = SPECS.getData().getURL(package, version)
sourceName = None
listSourceNames = SPECS.getData().getSources(package, version)
if listSourceNames:
sourceName = listSourceNames[0]
sha1 = SPECS.getData().getSHA1(package, version, sourceName)
if sha1 is not None:
PullSources.get(package, sourceName, sha1, yamlSourceDir,
constants.getPullSourcesURLs(package),
logger)
if not singleFile:
yamlFile = open(yamlSourceDir + "/" + ossname + "-" + version + ".yaml", "w")
yamlFile.write("vmwsource:" + ossname + ":" + version + ":\n")
yamlFile.write(" repository: VMWsource\n")
yamlFile.write(" name: '" + ossname + "'\n")
yamlFile.write(" version: '" + version + "'\n")
yamlFile.write(" url: " + str(url) + "\n")
yamlFile.write(" license: UNKNOWN\n")
if sourceName is not None:
yamlFile.write(" vmwsource-distribution: " + str(sourceName) + "\n")
if modified:
yamlFile.write(" modified: true\n")
yamlFile.write("\n")
if not singleFile:
yamlFile.close()
if singleFile:
yamlFile.close()
logger.debug("Generated source yaml files for all packages")
def process(self, inputType, inputValue, displayOption, outputFile=None):
whoNeedsList = []
inputPackages = []
whatNeedsBuild = []
mapDependencies = {}
parent = {}
if inputType == "pkg" or inputType == "json":
if inputType == "pkg":
inputPackages.append(inputValue)
else:
inputPackages = self.getAllPackageNames(inputValue)
self.calculateSpecDependency(inputPackages, mapDependencies, parent)
if outputFile is not None:
return self.displayDependencies(displayOption, inputType, outputFile, mapDependencies, parent)
else:
return self.displayDependencies(displayOption, inputType, inputValue, mapDependencies, parent)
elif inputType == "get-upward-deps":
depList = []
for specFile in inputValue.split(":"):
if specFile in SPECS.getData().mapSpecFileNameToSpecObj:
specObj = SPECS.getData().mapSpecFileNameToSpecObj[specFile]
whoNeedsList.append(specObj.name+"-"+specObj.version)
depList.append(specObj.name+"-"+specObj.version)
self.findTotalWhoNeeds(depList, whoNeedsList)
return whoNeedsList
elif inputType == "who-needs":
for depPackage in SPECS.getData().mapPackageToSpec:
pkg=inputValue+"-"+SPECS.getData().getHighestVersion(inputValue)
for version in SPECS.getData().getVersions(depPackage):
depPkg = depPackage+"-"+version
self.logger.info(depPkg)
if pkg in SPECS.getData().getRequiresForPkg(depPkg):
whoNeedsList.append(depPkg)
self.logger.info(whoNeedsList)
return whoNeedsList
elif inputType == "is-toolchain-pkg":
for specFile in inputValue.split(":"):
if specFile in SPECS.getData().mapSpecFileNameToSpecObj:
specObj = SPECS.getData().mapSpecFileNameToSpecObj[specFile]
if (specObj.name in constants.listCoreToolChainPackages) \
or (specObj.name in constants.listToolChainPackages):
return True
return False
def adjustGCCSpecs(self, sandbox, package, version):
opt = " " + SPECS.getData().getSecurityHardeningOption(package, version)
sandbox.put(self.adjustGCCSpecScript, "/tmp")
cmd = "/tmp/" + self.adjustGCCSpecScript + opt
returnVal = sandbox.run(cmd, logfn=self.logger.debug)
if returnVal == 0:
return
# in debugging ...
sandbox.run("ls -la /tmp/" + self.adjustGCCSpecScript,
logfn=self.logger.debug)
sandbox.run("lsof /tmp/" + self.adjustGCCSpecScript,
logfn=self.logger.debug)
sandbox.run("ps ax", logfn=self.logger.debug)
self.logger.error("Failed while adjusting gcc specs")
raise Exception("Failed while adjusting gcc specs")
def adjustGCCSpecsInContainer(self, package, containerID, logPath, index=0):
opt = " " + SPECS.getData().getSecurityHardeningOption(package, index)
adjustCmd = "/" + self.adjustGCCSpecScript + opt
adjustCmd = "/bin/bash -l -c '" + adjustCmd + "'"
logFile = logPath + "/adjustGCCSpecScript.log"
#TODO: Error code from exec_run
scriptLog = containerID.exec_run(adjustCmd)
if scriptLog:
with open(logFile, 'w') as logfile:
logfile.write(scriptLog.decode())
return
self.logger.debug(containerID.exec_run("ls -la /tmp/" + self.adjustGCCSpecScript))
self.logger.debug(containerID.exec_run("lsof /tmp/" + self.adjustGCCSpecScript))
self.logger.debug(containerID.exec_run("ps ax"))
self.logger.error("Failed while adjusting gcc specs")
raise Exception("Failed while adjusting gcc specs")
def buildRPMSForGivenPackage(self, sandbox, package, version, destLogPath):
self.logger.info("Building package : " + package)
listSourcesFiles = SPECS.getData().getSources(package, version)
listPatchFiles = SPECS.getData().getPatches(package, version)
specFile = SPECS.getData().getSpecFile(package, version)
specName = SPECS.getData().getSpecName(package) + ".spec"
sourcePath = constants.topDirPath + "/SOURCES/"
specPath = constants.topDirPath + "/SPECS/"
if (constants.rpmCheck and
package in constants.testForceRPMS and
SPECS.getData().isCheckAvailable(package, version)):
logFilePath = destLogPath + "/" + package + "-test.log"
else:
logFilePath = destLogPath + "/" + package + ".log"
sandbox.put(specFile, specPath + specName)
sources_urls, macros = self._getAdditionalBuildOptions(package)
self.logger.debug("Extra macros for " + package + ": " + str(macros))
self.logger.debug("Extra source URLs for " + package + ": " + str(sources_urls))
constants.setExtraSourcesURLs(package, sources_urls)
self._copySources(sandbox, listSourcesFiles, package, version, sourcePath)
self._copySources(sandbox, listPatchFiles, package, version, sourcePath)
#Adding rpm macros
listRPMMacros = constants.userDefinedMacros
for macroName, value in listRPMMacros.items():
macros.append(macroName + " " + value)
listRPMFiles = []
listSRPMFiles = []
try:
listRPMFiles, listSRPMFiles = self._buildRPM(sandbox, specPath + specName,
logFilePath, package, version, macros)
logmsg = package + " build done - RPMs : [ "
for f in listRPMFiles:
logmsg += (os.path.basename(f) + " ")
logmsg += "]\n"
self.logger.info(logmsg)
except Exception as e:
self.logger.error("Failed while building rpm:" + package)
raise e
finally:
if (constants.rpmCheck and
package in constants.testForceRPMS and
SPECS.getData().isCheckAvailable(package, version)):
cmd = ("sed -i '/^Executing(%check):/,/^Processing files:/{//!b};d' " + logFilePath)
CommandUtils().runCommandInShell(cmd, logfn=self.logger.debug)
self.logger.debug("RPM build is successful")
def _findDependentPackagesAndInstalledRPM(self, sandbox):
listInstalledPackages, listInstalledRPMs = self._findInstalledPackages(sandbox)
self.logger.debug(listInstalledPackages)
listDependentPackages = self._findBuildTimeRequiredPackages()
listTestPackages=[]
if constants.rpmCheck and self.package in constants.testForceRPMS:
# One time optimization
if constants.listMakeCheckRPMPkgWithVersionstoInstall is None:
constants.listMakeCheckRPMPkgWithVersionstoInstall=[]
for package in constants.listMakeCheckRPMPkgtoInstall:
version = SPECS.getData().getHighestVersion(package)
constants.listMakeCheckRPMPkgWithVersionstoInstall.append(package+"-"+version)
listDependentPackages.extend(self._findBuildTimeCheckRequiredPackages())
testPackages = (set(constants.listMakeCheckRPMPkgWithVersionstoInstall) -
set(listInstalledPackages) -
set([self.package+"-"+self.version]))
listTestPackages=list(set(testPackages))
listDependentPackages = list(set(listDependentPackages))
return listDependentPackages, listTestPackages, listInstalledPackages, listInstalledRPMs
def findTotalRequires(self, mapDependencies, depQue, parent):
while not depQue.empty():
specPkg = depQue.get()
try:
listRequiredPackages = SPECS.getData().getRequiresForPkg(specPkg)
except Exception as e:
self.logger.info("Caught Exception:"+str(e))
self.logger.info(specPkg + " is missing")
raise e
for depPkg in listRequiredPackages:
if depPkg in mapDependencies:
if mapDependencies[depPkg] < mapDependencies[specPkg] + 1:
mapDependencies[depPkg] = mapDependencies[specPkg] + 1
parent[depPkg] = specPkg
self.updateLevels(mapDependencies, depPkg, parent, mapDependencies[depPkg])
else:
mapDependencies[depPkg] = mapDependencies[specPkg] + 1
parent[depPkg] = specPkg
depQue.put(depPkg)
def adjustGCCSpecs(self, package, chrootID, logPath, index=0):
opt = " " + SPECS.getData().getSecurityHardeningOption(package, index)
cmdUtils = CommandUtils()
cpcmd = ("cp " + self.adjustGCCSpecScript + " " + chrootID +
"/tmp/" + self.adjustGCCSpecScript)
cmd = "/tmp/" + self.adjustGCCSpecScript + opt
logFile = logPath + "/adjustGCCSpecScript.log"
chrootCmd = self.runInChrootCommand + " " + chrootID
returnVal = cmdUtils.runCommandInShell(cpcmd, logFile)
if not returnVal:
self.logger.error("Error during copying the file adjust gcc spec")
raise Exception("Failed while copying adjust gcc spec file")
returnVal = cmdUtils.runCommandInShell(cmd, logFile, chrootCmd)
if returnVal:
return
self.logger.debug(cmdUtils.runCommandInShell2("ls -la " + chrootID +
"/tmp/" + self.adjustGCCSpecScript))
self.logger.debug(cmdUtils.runCommandInShell2("lsof " + chrootID + "/tmp/" +
self.adjustGCCSpecScript))
self.logger.debug(cmdUtils.runCommandInShell2("ps ax"))
self.logger.error("Failed while adjusting gcc specs")
raise Exception("Failed while adjusting gcc specs")
def buildPackagesList(csvFilename):
with open(csvFilename, "w") as csvFile:
csvFile.write("Package,Version,License,URL,Sources,Patches\n")
listPackages = SPECS.getData().getListPackages()
listPackages.sort()
for package in listPackages:
name = package
for version in SPECS.getData().getVersions(package):
packagelicense = SPECS.getData().getLicense(package, version)
listPatches = SPECS.getData().getPatches(package, version)
url = SPECS.getData().getURL(package, version)
listSourceNames = SPECS.getData().getSources(package, version)
sources = ""
patches = ""
if listPatches is not None:
patches = " ".join(listPatches)
if listSourceNames is not None:
sources = " ".join(listSourceNames)
csvFile.write(name + "," + version + "," + packagelicense + "," + url + "," +
sources + "," + patches + "\n")
def buildPackagesForAllSpecs(buildThreads, pkgBuildType, pkgInfoJsonFile, logger):
listPackages = SPECS.getData().getListPackages()
buildSpecifiedPackages(listPackages, buildThreads, pkgBuildType, pkgInfoJsonFile, logger)
def _buildRPMinContainer(self, specFile, rpmLogFile, destLogFile, containerID, package, macros):
rpmBuildCmd = self.rpmbuildBinary + " " + self.rpmbuildBuildallOption
if constants.rpmCheck and package in constants.testForceRPMS:
self.logger.info("#" * (68 + 2 * len(package)))
if not SPECS.getData().isCheckAvailable(package, index):
self.logger.info("####### " + package +
" MakeCheck is not available. Skipping MakeCheck TEST for " +
package + " #######")
rpmBuildCmd = self.rpmbuildBinary + " --clean"
else:
self.logger.info("####### " + package +
" MakeCheck is available. Running MakeCheck TEST for " +
package + " #######")
rpmBuildCmd = self.rpmbuildBinary + " " + self.rpmbuildCheckOption
self.logger.info("#" * (68 + 2 * len(package)))
else:
rpmBuildCmd += " "+self.rpmbuildNocheckOption
for macro in macros:
rpmBuildCmd += ' --define \"%s\"' % macro
rpmBuildCmd += " " + specFile
rpmBuildCmd = "/bin/bash -l -c '" + rpmBuildCmd + " > " + rpmLogFile + " 2>&1'"
rpmBuildCmd = "docker exec -t " + str(containerID.short_id) + " " + rpmBuildCmd
cmdUtils = CommandUtils()
self.logger.info("Building rpm for package: " + package)
#TODO: Show running log of rpmbuildcmd
#TODO: Get exit status of rpmBuildCmd
#containerID.exec_run(rpmBuildCmd)
returnVal = cmdUtils.runCommandInShell(rpmBuildCmd)
if not os.path.isfile(destLogFile):
self.logger.error("RPM build not file not found. Building rpm failed for: " + specFile)
raise Exception("RPM Build failed")
if constants.rpmCheck and package in constants.testForceRPMS:
if not SPECS.getData().isCheckAvailable(package, index):
constants.testLogger.info(package + " : N/A")
elif returnVal:
constants.testLogger.info(package + " : PASS")
else:
constants.testLogger.error(package + " : FAIL")
if constants.rpmCheck:
if not returnVal and constants.rpmCheckStopOnError:
self.logger.error("Checking rpm is failed " + specFile)
raise Exception("RPM check failed")
else:
if not returnVal:
self.logger.error("Building rpm is failed " + specFile)
raise Exception("RPM build failed")
#Extracting rpms created from log file
listRPMFiles = []
listSRPMFiles = []
with open(destLogFile, 'r') as logfile:
rpmBuildLogLines = logfile.readlines()
for i in range(0, len(rpmBuildLogLines)):
if re.search("^Wrote:", rpmBuildLogLines[i]):
listcontents = rpmBuildLogLines[i].split()
if ((len(listcontents) == 2) and
listcontents[1].strip().endswith(".rpm") and
"/RPMS/" in listcontents[1]):
listRPMFiles.append(listcontents[1])
if ((len(listcontents) == 2) and
listcontents[1].strip().endswith(".src.rpm") and
"/SRPMS/" in listcontents[1]):
listSRPMFiles.append(listcontents[1])
#if not listRPMFiles:
# self.logger.error("Building rpm failed for " + specFile)
# raise Exception("RPM Build failed")
return listRPMFiles, listSRPMFiles
def _buildRPM(self, specFile, logFile, chrootCmd, package, macros):
rpmBuildcmd = self.rpmbuildBinary + " " + self.rpmbuildBuildallOption
if constants.rpmCheck and package in constants.testForceRPMS:
self.logger.info("#" * (68 + 2 * len(package)))
if not SPECS.getData().isCheckAvailable(package):
self.logger.info("####### " + package +
" MakeCheck is not available. Skipping MakeCheck TEST for " +
package + " #######")
rpmBuildcmd = self.rpmbuildBinary + " --clean"
else:
self.logger.info("####### " + package +
" MakeCheck is available. Running MakeCheck TEST for " +
package + " #######")
rpmBuildcmd = self.rpmbuildBinary + " " + self.rpmbuildCheckOption
self.logger.info("#" * (68 + 2 * len(package)))
else:
rpmBuildcmd += " " + self.rpmbuildNocheckOption
for macro in macros:
rpmBuildcmd += ' --define \\\"%s\\\"' % macro
rpmBuildcmd += " " + specFile
cmdUtils = CommandUtils()
self.logger.info("Building rpm....")
self.logger.info(rpmBuildcmd)
returnVal = cmdUtils.runCommandInShell(rpmBuildcmd, logFile, chrootCmd)
if constants.rpmCheck and package in constants.testForceRPMS:
if not SPECS.getData().isCheckAvailable(package):
constants.testLogger.info(package + " : N/A")
elif returnVal:
constants.testLogger.info(package + " : PASS")
else:
constants.testLogger.error(package + " : FAIL")
if constants.rpmCheck:
if not returnVal and constants.rpmCheckStopOnError:
self.logger.error("Checking rpm is failed " + specFile)
raise Exception("RPM check failed")
else:
if not returnVal:
self.logger.error("Building rpm is failed " + specFile)
raise Exception("RPM build failed")
#Extracting rpms created from log file
listRPMFiles = []
listSRPMFiles = []
with open(logFile, 'r') as logfile:
fileContents = logfile.readlines()
for i in range(0, len(fileContents)):
if re.search("^Wrote:", fileContents[i]):
listcontents = fileContents[i].split()
if ((len(listcontents) == 2) and
listcontents[1].strip().endswith(".rpm") and
"/RPMS/" in listcontents[1]):
listRPMFiles.append(listcontents[1])
if ((len(listcontents) == 2) and
listcontents[1].strip().endswith(".src.rpm") and
"/SRPMS/" in listcontents[1]):
listSRPMFiles.append(listcontents[1])
return listRPMFiles, listSRPMFiles
def buildRPMSForGivenPackageInContainer(self, package, containerID, destLogPath=None, index=0):
self.logger.info("Building rpm's for package " + package + " in container " +
containerID.short_id)
listSourcesFiles = SPECS.getData().getSources(package, index)
listPatchFiles = SPECS.getData().getPatches(package, index)
specFile = SPECS.getData().getSpecFile(package, index)
specName = SPECS.getData().getSpecName(package) + ".spec"
sourcePath = constants.topDirPath + "/SOURCES/"
specPath = constants.topDirPath + "/SPECS/"
rpmLogFile = constants.topDirPath + "/LOGS/" + package + ".log"
destLogFile = destLogPath + "/" + package + ".log"
cmdUtils = CommandUtils()
#TODO: mount it in, don't copy
cpSpecCmd = "docker cp " + specFile + " " + containerID.short_id \
+ ":" + specPath + specName
returnVal = cmdUtils.runCommandInShell(cpSpecCmd)
if not returnVal:
self.logger.error("Error copying source SPEC file to container")
raise Exception("Failed copying source SPEC to container")
#FIXME: some sources are located in SPECS/.. how to mount?
# if os.geteuid()==0:
#TODO: mount it in, don't copy
macros = []
self._copySourcesToContainer(listSourcesFiles, package, containerID, sourcePath, index)
#TODO: mount it in, don't copy
self._copySourcesToContainer(listPatchFiles, package, containerID, sourcePath, index)
listAdditionalFiles, macros = self._getAdditionalBuildFiles(package)
self._copyAdditionalBuildFilesToContainer(listAdditionalFiles, containerID)
# Add rpm macros
listRPMMacros = constants.userDefinedMacros
for macroName, value in listRPMMacros.items():
macros.append(macroName + " " + value)
# Build RPMs
listRPMFiles = []
listSRPMFiles = []
try:
listRPMFiles, listSRPMFiles = self._buildRPMinContainer(
specPath + specName,
rpmLogFile,
destLogFile,
containerID,
package,
macros)
self.logger.info("Successfully built rpm:" + package)
except Exception as e:
self.logger.error("Failed while building rpm: " + package)
raise e
finally:
if destLogPath is not None:
rpmLog = destLogPath + "/" + package + ".log"
if (constants.rpmCheck and
package in constants.testForceRPMS and
SPECS.getData().isCheckAvailable(package, index)):
cmd = "sed -i '/^Executing(%check):/,/^Processing files:/{//!b};d' " + rpmLog
logFile = destLogPath + "/adjustTestFile.log"
returnVal = CommandUtils().runCommandInShell(cmd, logFile)
testLogFile = destLogPath + "/" + package + "-test.log"
shutil.copyfile(rpmLog, testLogFile)
self.logger.info("RPM build is successful")
# Verify RPM and SRPM files exist as success criteria
for rpmFile in listRPMFiles:
rpmName = os.path.basename(rpmFile)
rpmDestDir = self._getRPMDestDir(rpmName, constants.rpmPath)
rpmDestFile = rpmDestDir + "/" + rpmName
if not os.path.isfile(rpmDestFile):
self.logger.error("Could not find RPM file: " + rpmDestFile)
raise Exception("Built RPM file not found.")
for srpmFile in listSRPMFiles:
srpmName = os.path.basename(srpmFile)
srpmDestDir = self._getRPMDestDir(os.path.basename(srpmFile), constants.sourceRpmPath)
srpmDestFile = srpmDestDir + "/" + srpmName
if not os.path.isfile(srpmDestFile):
self.logger.error("Could not find RPM file: " + srpmDestFile)
raise Exception("Built SRPM file not found.")