def buildToolChain(self):
self.logger.info("Step 1 : Building the core toolchain packages for " + constants.currentArch)
self.logger.info(constants.listCoreToolChainPackages)
self.logger.info("")
pkgCount = 0
pkgUtils = PackageUtils(self.logName, self.logPath)
coreToolChainYetToBuild = []
doneList = []
for package in constants.listCoreToolChainPackages:
version = SPECS.getData().getHighestVersion(package)
rpmPkg = pkgUtils.findRPMFile(package, version)
self.sortedPackageList.append(package+"-"+version)
if rpmPkg is not None:
doneList.append(package+'-'+version)
continue
else:
coreToolChainYetToBuild.append(package)
self.listOfPackagesAlreadyBuilt = set(doneList)
pkgCount = len(coreToolChainYetToBuild)
if coreToolChainYetToBuild:
self.logger.info("The following core toolchain packages need to be built :")
self.logger.info(coreToolChainYetToBuild)
else:
self.logger.info("Core toolchain packages are already available")
self.logger.info("")
return pkgCount
Scheduler.coreToolChainBuild = True
self._buildPackages(1)
Scheduler.coreToolChainBuild = False
self.logger.debug("Successfully built core toolchain")
self.logger.info("-" * 45 + "\n")
return pkgCount
def loadPackagesData(self):
listPackages = SPECS.getData().getListPackages()
listPackages.sort()
pkgUtils = PackageUtils(self.logName, self.logPath)
for package in listPackages:
for version in SPECS.getData().getVersions(package):
srpmFile = pkgUtils.findSourceRPMFile(package, version)
debugrpmFile = pkgUtils.findDebugRPMFile(package, version)
listRPMPackages = SPECS.getData().getRPMPackages(
package, version)
for rpmPkg in listRPMPackages:
rpmFile = pkgUtils.findRPMFile(rpmPkg, version)
if rpmFile is not None:
listPkgAttributes = {
"sourcerpm": srpmFile,
"rpm": rpmFile,
"debugrpm": debugrpmFile
}
self.pkgList[rpmPkg + "-" +
version] = listPkgAttributes
self.logger.debug("Added " + rpmPkg + "-" + version +
" to the package info json")
else:
self.logger.debug("Missing rpm file for package:" +
rpmPkg)
def installTargetToolchain(self, chroot, stopAtPackage=None):
self.logger.debug("Installing target toolchain RPMS.......")
pkgUtils = PackageUtils(self.logName, self.logPath)
rpmFiles = ""
packages = ""
for package in constants.listCoreToolChainPackages:
if stopAtPackage and package == stopAtPackage:
break
version = SPECS.getData().getHighestVersion(package)
basePkg = SPECS.getData().getSpecName(package)
# install all subpackages of given package
# for instance: for 'glibc' we want glibc-devel, glibc-tools,
# glibc-i18n, etc also to be installed
subpackages = SPECS.getData().getRPMPackages(basePkg, version)
for p in subpackages:
rpmFile = pkgUtils.findRPMFile(p, version, constants.targetArch)
rpmFiles += " " + rpmFile
packages += " " + package+"-"+version
self.logger.debug(packages)
cmd = "mkdir -p " + chroot.getID() +"/target-"+ constants.targetArch+"/var/lib/rpm"
CommandUtils.runCommandInShell(cmd, logfn=self.logger.debug)
if rpmFiles != "":
cmd = (self.rpmCommand+" -Uvh --nodeps --ignorearch --noscripts --root "+
chroot.getID() +"/target-"+ constants.targetArch+
" --define \'_dbpath /var/lib/rpm\' "+rpmFiles)
retVal = CommandUtils.runCommandInShell(cmd, logfn=self.logger.debug)
if retVal != 0:
self.logger.debug("Command Executed:" + cmd)
self.logger.error("Installing toolchain failed")
raise Exception("RPM installation failed")
self.logger.debug("Successfully installed target toolchain RPMS in chroot:" + chroot.getID())
def loadPackagesData(self):
listPackages = SPECS.getData().getListPackages()
listPackages.sort()
cmdUtils = CommandUtils()
for package in listPackages:
for version in SPECS.getData().getVersions(package):
release = SPECS.getData().getRelease(package, version)
listRPMPackages = SPECS.getData().getRPMPackages(package, version)
srpmFileName = package + "-" + version + "-" + release + ".src.rpm"
srpmFiles = cmdUtils.findFile(srpmFileName, constants.sourceRpmPath)
srpmFile = None
if len(srpmFiles) == 1:
srpmFile = srpmFiles[0]
debugrpmFileName = package + "-debuginfo-" + version + "-" + release + "*"
debugrpmFiles = cmdUtils.findFile(debugrpmFileName, constants.rpmPath)
debugrpmFile = None
if len(debugrpmFiles) == 1:
debugrpmFile = debugrpmFiles[0]
pkgUtils = PackageUtils(self.logName, self.logPath)
for rpmPkg in listRPMPackages:
rpmFile = pkgUtils.findRPMFile(rpmPkg, version)
if rpmFile is not None:
listPkgAttributes = {"sourcerpm":srpmFile, "rpm":rpmFile,
"debugrpm":debugrpmFile}
self.pkgList[rpmPkg+"-"+version] = listPkgAttributes
self.logger.debug("Added " + rpmPkg + "-" + version + " to the package info json")
else:
self.logger.debug("Missing rpm file for package:" + rpmPkg)
def buildCoreToolChainPackages(self):
self.logger.info("Step 1 : Building the core toolchain packages for " + constants.currentArch)
self.logger.info(constants.listCoreToolChainPackages)
self.logger.info("")
chroot = None
pkgCount = 0
try:
pkgUtils = PackageUtils(self.logName, self.logPath)
coreToolChainYetToBuild = []
doneList = []
#self.logger.debug(">>>> Looking for available packages...")
for package in constants.listCoreToolChainPackages:
version = SPECS.getData().getHighestVersion(package)
rpmPkg = pkgUtils.findRPMFile(package, version)
#rpmPkg = pkgUtils.findRPMFile(package, version, constants.buildArch)
if rpmPkg is not None:
doneList.append(package+'-'+version)
#self.logger.debug(">>>>>> %s", package+'-'+version)
continue
else:
coreToolChainYetToBuild.append(package)
if coreToolChainYetToBuild:
self.logger.info("The following core toolchain packages need to be built :")
self.logger.info(coreToolChainYetToBuild)
else:
self.logger.info("Core toolchain packages are already available")
for package in coreToolChainYetToBuild:
self.logger.debug("Building core toolchain package : " + package)
version = SPECS.getData().getHighestVersion(package)
destLogPath = constants.logPath + "/" + package + "-" + version + "." + constants.currentArch
if not os.path.isdir(destLogPath):
CommandUtils.runCommandInShell("mkdir -p " + destLogPath)
chroot = Chroot(self.logger)
chroot.create(package + "-" + version)
#self.logger.debug(">>>>> Done List:")
#self.logger.info(doneList)
self.installToolchainRPMS(chroot, package, version, availablePackages=doneList)
pkgUtils.adjustGCCSpecs(chroot, package, version)
pkgUtils.buildRPMSForGivenPackage(chroot, package, version, destLogPath)
pkgCount += 1
chroot.destroy()
doneList.append(package+'-'+version)
self.logger.debug("Successfully built toolchain")
self.logger.info("-" * 45 + "\n")
except Exception as e:
self.logger.error("Unable to build toolchain.")
# print stacktrace
traceback.print_exc()
raise e
return pkgCount
def _readAlreadyAvailablePackages(self):
listAvailablePackages = set()
pkgUtils = PackageUtils(self.logName, self.logPath)
listPackages = SPECS.getData().getListPackages()
for package in listPackages:
for version in SPECS.getData().getVersions(package):
# Mark package available only if all subpackages are available
packageIsAlreadyBuilt=True
listRPMPackages = SPECS.getData().getRPMPackages(package, version)
for rpmPkg in listRPMPackages:
if pkgUtils.findRPMFile(rpmPkg, version) is None:
packageIsAlreadyBuilt=False
break;
if packageIsAlreadyBuilt:
listAvailablePackages.add(package+"-"+version)
return listAvailablePackages
def _readAlreadyAvailablePackages(self):
listAvailablePackages = set()
pkgUtils = PackageUtils(self.logName, self.logPath)
listPackages = SPECS.getData().getListPackages()
for package in listPackages:
for version in SPECS.getData().getVersions(package):
# Mark package available only if all subpackages are available
packageIsAlreadyBuilt=True
listRPMPackages = SPECS.getData().getRPMPackages(package, version)
for rpmPkg in listRPMPackages:
if pkgUtils.findRPMFile(rpmPkg, version) is None:
packageIsAlreadyBuilt=False
break;
if packageIsAlreadyBuilt:
listAvailablePackages.add(package+"-"+version)
return listAvailablePackages
def installToolchainRPMS(self,
chroot,
packageName=None,
packageVersion=None,
usePublishedRPMS=True,
availablePackages=None):
self.logger.debug("Installing toolchain RPMS.......")
rpmFiles = ""
packages = ""
listBuildRequiresPackages = []
listRPMsToInstall = list(constants.listToolChainRPMsToInstall)
if constants.crossCompiling:
targetPackageName = packageName
packageName = None
packageVersion = None
listRPMsToInstall.extend([
'binutils-' + constants.targetArch + '-linux-gnu',
'gcc-' + constants.targetArch + '-linux-gnu'
])
if packageName:
listBuildRequiresPackages = self.getListDependentPackages(
packageName, packageVersion)
for package in listRPMsToInstall:
pkgUtils = PackageUtils(self.logName, self.logPath)
rpmFile = None
version = None
# Get proper package version
for depPkg in listBuildRequiresPackages:
depPkgName, depPkgVersion = StringUtils.splitPackageNameAndVersion(
depPkg)
if depPkgName == package:
version = depPkgVersion
break
if not version:
version = SPECS.getData(
constants.buildArch).getHighestVersion(package)
if availablePackages is not None:
basePkg = SPECS.getData(
constants.buildArch).getSpecName(package) + "-" + version
isAvailable = basePkg in availablePackages
else:
# if availablePackages is not provided (rear case) it is safe
# to use findRPMFile()
isAvailable = True
if constants.rpmCheck:
rpmFile = pkgUtils.findRPMFile(package, version,
constants.buildArch)
if rpmFile is None:
# Honor the toolchain list order.
# if index of depended package ('package') is more
# then index of the current package that we are
# building ('packageName'), then we _must_ use published
# `package` rpm.
if (packageName and packageName in listRPMsToInstall
and listRPMsToInstall.index(packageName) <
listRPMsToInstall.index(package)):
isAvailable = False
if isAvailable:
rpmFile = pkgUtils.findRPMFile(package, version,
constants.buildArch)
if rpmFile is None:
if not usePublishedRPMS or isAvailable or constants.crossCompiling:
raise Exception(
"%s-%s.%s not found in available packages" %
(package, version, constants.buildArch))
# Safe to use published RPM
# sqlite-autoconf package was renamed, but it still published as sqlite-autoconf
if (package == "sqlite") and (constants.buildArch == "x86_64"):
package = "sqlite-autoconf"
rpmFile = self._findPublishedRPM(package,
constants.prevPublishRPMRepo)
if rpmFile is None:
if package in constants.listOfRPMsProvidedAfterBuild:
self.logger.debug(
"No old version of " + package +
" exists, skip until the new version is built")
continue
self.logger.error("Unable to find published rpm " +
package)
raise Exception("Input Error")
rpmFiles += " " + rpmFile
packages += " " + package + "-" + version
self.logger.debug(rpmFiles)
self.logger.debug(packages)
cmd = (self.rpmCommand + " -i -v --nodeps --noorder --force --root " +
chroot.getID() + " --define \'_dbpath /var/lib/rpm\' " +
rpmFiles)
retVal = CommandUtils.runCommandInShell(cmd, logfn=self.logger.debug)
if retVal != 0:
self.logger.debug("Command Executed:" + cmd)
self.logger.error("Installing toolchain RPMS failed")
raise Exception("RPM installation failed")
self.logger.debug(
"Successfully installed default toolchain RPMS in Chroot:" +
chroot.getID())
if packageName:
self.installExtraToolchainRPMS(chroot, packageName, packageVersion)
if constants.crossCompiling:
self.installTargetToolchain(chroot, targetPackageName)
def installToolchainRPMS(self, chroot, packageName=None, packageVersion=None, usePublishedRPMS=True, availablePackages=None):
self.logger.debug("Installing toolchain RPMS.......")
rpmFiles = ""
packages = ""
listBuildRequiresPackages = []
listRPMsToInstall=list(constants.listToolChainRPMsToInstall)
if constants.crossCompiling:
targetPackageName = packageName
packageName = None
packageVersion = None
listRPMsToInstall.extend(['binutils-'+constants.targetArch+'-linux-gnu',
'gcc-'+constants.targetArch+'-linux-gnu'])
if packageName:
listBuildRequiresPackages = self.getListDependentPackages(packageName, packageVersion)
for package in listRPMsToInstall:
pkgUtils = PackageUtils(self.logName, self.logPath)
rpmFile = None
version = None
# Get proper package version
for depPkg in listBuildRequiresPackages:
depPkgName, depPkgVersion = StringUtils.splitPackageNameAndVersion(depPkg)
if depPkgName == package:
version=depPkgVersion
break
if not version:
version = SPECS.getData(constants.buildArch).getHighestVersion(package)
if availablePackages is not None:
basePkg = SPECS.getData(constants.buildArch).getSpecName(package)+"-"+version
isAvailable = basePkg in availablePackages
else:
# if availablePackages is not provided (rear case) it is safe
# to use findRPMFile()
isAvailable = True
if constants.rpmCheck:
rpmFile = pkgUtils.findRPMFile(package, version, constants.buildArch)
if rpmFile is None:
# Honor the toolchain list order.
# if index of depended package ('package') is more
# then index of the current package that we are
# building ('packageName'), then we _must_ use published
# `package` rpm.
if (packageName and
packageName in listRPMsToInstall and
listRPMsToInstall.index(packageName) <
listRPMsToInstall.index(package)):
isAvailable = False
if isAvailable:
rpmFile = pkgUtils.findRPMFile(package, version, constants.buildArch)
if rpmFile is None:
if not usePublishedRPMS or isAvailable or constants.crossCompiling:
raise Exception("%s-%s.%s not found in available packages" % (package, version, constants.buildArch))
# Safe to use published RPM
rpmFile = self._findPublishedRPM(package, constants.prevPublishRPMRepo)
if rpmFile is None:
self.logger.error("Unable to find published rpm " + package)
raise Exception("Input Error")
rpmFiles += " " + rpmFile
packages += " " + package+"-"+version
self.logger.debug(rpmFiles)
self.logger.debug(packages)
cmd = (self.rpmCommand + " -i -v --nodeps --noorder --force --root " +
chroot.getID() +" --define \'_dbpath /var/lib/rpm\' "+ rpmFiles)
# If rpm doesn't have zstd support, use rpm from photon_builder image
if constants.hostRpmIsNotUsable:
# if we are not root, make installed files owned by effective user to
# support pure non-root package building.
if os.geteuid() != 0:
cmd = cmd + "; chown -R {0}:{1} {2}".format(os.geteuid(), os.getegid(), chroot.getID())
cmd = ("docker run -i -v " + constants.prevPublishRPMRepo + ":" + constants.prevPublishRPMRepo +
" -v " + constants.rpmPath + ":" + constants.rpmPath + " -v " + chroot.getID() + ":" +
chroot.getID() + " photon_builder:latest " + "/bin/bash -c \"" + cmd + "\"")
self.logger.debug("Executing cmd: " + cmd)
retVal = CommandUtils.runCommandInShell(cmd, logfn=self.logger.debug)
if retVal != 0:
self.logger.error("Installing toolchain RPMS failed")
raise Exception("RPM installation failed")
self.logger.debug("Successfully installed default toolchain RPMS in Chroot:" + chroot.getID())
if packageName:
self.installExtraToolchainRPMS(chroot, packageName, packageVersion)
if constants.crossCompiling:
self.installTargetToolchain(chroot, targetPackageName)
