def _buildPackage(self):
chroot = None
try:
chroot = Chroot(self.logger)
chroot.create(self.package + "-" + self.version)
tUtils = ToolChainUtils(self.logName, self.logPath)
tUtils.installToolChainRPMS(chroot, self.package, self.version,
self.logPath)
listDependentPackages, listTestPackages, listInstalledPackages, listInstalledRPMs = (
self._findDependentPackagesAndInstalledRPM(chroot))
pkgUtils = PackageUtils(self.logName, self.logPath)
if listDependentPackages:
self.logger.debug(
"Installing the build time dependent packages......")
for pkg in listDependentPackages:
packageName, packageVersion = StringUtils.splitPackageNameAndVersion(
pkg)
self._installPackage(pkgUtils, packageName, packageVersion,
chroot, self.logPath,
listInstalledPackages,
listInstalledRPMs)
for pkg in listTestPackages:
flag = False
packageName, packageVersion = StringUtils.splitPackageNameAndVersion(
pkg)
for depPkg in listDependentPackages:
depPackageName, depPackageVersion = StringUtils.splitPackageNameAndVersion(
depPkg)
if depPackageName == packageName:
flag = True
break
if flag == False:
self._installPackage(pkgUtils, packageName,
packageVersion, chroot,
self.logPath,
listInstalledPackages,
listInstalledRPMs)
pkgUtils.installRPMSInOneShot(chroot)
self.logger.debug(
"Finished installing the build time dependent packages...."
)
pkgUtils.adjustGCCSpecs(chroot, self.package, self.version)
pkgUtils.buildRPMSForGivenPackage(chroot, self.package,
self.version, self.logPath)
self.logger.debug("Successfully built the package:" + self.package)
except Exception as e:
self.logger.error("Failed while building package:" + self.package)
self.logger.debug("Chroot: " + chroot.getPath() +
" not deleted for debugging.")
logFileName = os.path.join(self.logPath, self.package + ".log")
fileLog = os.popen('tail -n 100 ' + logFileName).read()
self.logger.info(fileLog)
raise e
if chroot:
chroot.destroy()
def _calculateParams(self, listPackages):
self.mapCyclesToPackageList.clear()
self.mapPackageToCycle.clear()
self.sortedPackageList = []
self.listOfPackagesAlreadyBuilt = list(
self._readAlreadyAvailablePackages())
updateBuiltRPMSList = False
while not updateBuiltRPMSList:
updateBuiltRPMSList = True
listOfPackagesAlreadyBuilt = self.listOfPackagesAlreadyBuilt
for pkg in listOfPackagesAlreadyBuilt:
packageName, packageVersion = StringUtils.splitPackageNameAndVersion(
pkg)
listDependentRpmPackages = SPECS.getData(
).getRequiresAllForPackage(packageName, packageVersion)
needToRebuild = False
for dependentPkg in listDependentRpmPackages:
if dependentPkg not in self.listOfPackagesAlreadyBuilt:
needToRebuild = True
updateBuiltRPMSList = False
if needToRebuild:
self.listOfPackagesAlreadyBuilt.remove(pkg)
listPackagesToBuild = copy.copy(listPackages)
for pkg in listPackages:
if (pkg in self.listOfPackagesAlreadyBuilt
and not constants.rpmCheck):
listPackagesToBuild.remove(pkg)
if not self._readPackageBuildData(listPackagesToBuild):
return False
return True
def _installDependentRunTimePackages(self, pkgUtils, package,
packageVersion, sandbox, destLogPath,
listInstalledPackages,
listInstalledRPMs, arch):
listRunTimeDependentPackages = self._findRunTimeRequiredRPMPackages(
package, packageVersion, arch)
if listRunTimeDependentPackages:
for pkg in listRunTimeDependentPackages:
if pkg in self.mapPackageToCycles:
continue
packageName, packageVersion = StringUtils.splitPackageNameAndVersion(
pkg)
rpmfile = pkgUtils.findRPMFile(packageName, packageVersion,
arch, True)
if rpmfile is None:
self.logger.error("No rpm file found for package: " +
packageName + "-" + packageVersion)
raise Exception("Missing rpm file")
latestPkgRPM = os.path.basename(rpmfile).replace(".rpm", "")
if pkg in listInstalledPackages and latestPkgRPM in listInstalledRPMs:
continue
self._installPackage(pkgUtils, packageName, packageVersion,
sandbox, destLogPath,
listInstalledPackages, listInstalledRPMs,
arch)
def setParams(sortedList, listOfAlreadyBuiltPackages):
Scheduler.sortedList = sortedList
Scheduler.listOfAlreadyBuiltPackages = listOfAlreadyBuiltPackages
for pkg in Scheduler.sortedList:
pkgName, pkgVersion = StringUtils.splitPackageNameAndVersion(pkg)
if (pkg not in Scheduler.listOfAlreadyBuiltPackages
or pkgName in constants.testForceRPMS):
Scheduler.listOfPackagesToBuild.append(pkg)
Scheduler.listOfPackagesCurrentlyBuilding = set()
Scheduler.listOfPackagesNextToBuild = PriorityQueue()
Scheduler.listOfFailedPackages = []
# When performing (only) make-check, package dependencies are
# irrelevant; i.e., all the packages can be "make-checked" in
# parallel. So skip building the dependency graph. This is not
# merely an optimization! A given package can define
# additional packages to be installed in its build environment
# when performing a make-check, under %if %{with_check}.
# However, these are not really build-time-dependencies in the
# usual sense; i.e., there is no ordering requirement when
# building these packages; they only make sense when running a
# `make check`. Hence, trying to build a dependency graph out
# of them will result in anomalies such as cycles in the
# graph. So skip building the graph altogether and schedule
# all the `make check`s in parallel.
skipGraphBuild = constants.rpmCheck
Scheduler._setPriorities(skipGraphBuild)
if constants.publishBuildDependencies:
# This must be called only after calling _setPriorities(),
# which builds the dependency graph.
Scheduler._publishBuildDependencies()
def _buildPackage(self):
try:
self.sandbox.create(self.package + "-" + self.version)
tUtils = ToolChainUtils(self.logName, self.logPath)
if self.sandbox.hasToolchain():
tUtils.installExtraToolchainRPMS(self.sandbox, self.package, self.version)
else:
tUtils.installToolchainRPMS(self.sandbox, self.package, self.version, availablePackages=self.doneList)
listDependentPackages, listTestPackages, listInstalledPackages, listInstalledRPMs = (
self._findDependentPackagesAndInstalledRPM(self.sandbox))
pkgUtils = PackageUtils(self.logName, self.logPath)
if listDependentPackages:
self.logger.debug("Installing the build time dependent packages......")
for pkg in listDependentPackages:
packageName, packageVersion = StringUtils.splitPackageNameAndVersion(pkg)
self._installPackage(pkgUtils, packageName, packageVersion, self.sandbox, self.logPath,listInstalledPackages, listInstalledRPMs)
for pkg in listTestPackages:
flag = False
packageName, packageVersion = StringUtils.splitPackageNameAndVersion(pkg)
for depPkg in listDependentPackages:
depPackageName, depPackageVersion = StringUtils.splitPackageNameAndVersion(depPkg)
if depPackageName == packageName:
flag = True
break;
if flag == False:
self._installPackage(pkgUtils, packageName,packageVersion, self.sandbox, self.logPath,listInstalledPackages, listInstalledRPMs)
pkgUtils.installRPMSInOneShot(self.sandbox)
self.logger.debug("Finished installing the build time dependent packages....")
pkgUtils.adjustGCCSpecs(self.sandbox, self.package, self.version)
pkgUtils.buildRPMSForGivenPackage(self.sandbox, self.package, self.version,
self.logPath)
self.logger.debug("Successfully built the package: " + self.package)
except Exception as e:
self.logger.error("Failed while building package: " + self.package)
self.logger.debug("Sandbox: " + self.sandbox.getID() +
" not deleted for debugging.")
logFileName = os.path.join(self.logPath, self.package + ".log")
fileLog = os.popen('tail -n 100 ' + logFileName).read()
self.logger.info(fileLog)
raise e
if self.sandbox:
self.sandbox.destroy()
def _createNodes():
# Create a graph node to represent every package
for package in Scheduler.sortedList:
packageName, packageVersion = StringUtils.splitPackageNameAndVersion(package)
node = DependencyGraphNode(packageName, packageVersion,
Scheduler._getWeight(package))
Scheduler.mapPackagesToGraphNodes[package] = node
if package in Scheduler.listOfAlreadyBuiltPackages:
node.built = 1
def _getWeight(package):
# Package weights are assumed to be independent of package
# version (i.e., in the case of multi-version packages such as
# Go or Kubernetes, all the versions have the same weight). So
# convert packageName-version to packageName before looking up
# the package weight.
package, _ = StringUtils.splitPackageNameAndVersion(package)
try:
return int(Scheduler.pkgWeights[package]) + 1
except KeyError:
return 1
def _getWeight(package):
# Package weights are assumed to be independent of package
# version (i.e., in the case of multi-version packages such as
# Go or Kubernetes, all the versions have the same weight). So
# convert packageName-version to packageName before looking up
# the package weight.
package, _ = StringUtils.splitPackageNameAndVersion(package)
try:
return int(Scheduler.pkgWeights[package]) + 1
except KeyError:
return 1
def _installDependentRunTimePackages(self, pkgUtils, package, packageVersion, sandbox, destLogPath,
listInstalledPackages, listInstalledRPMs):
listRunTimeDependentPackages = self._findRunTimeRequiredRPMPackages(package, packageVersion)
if listRunTimeDependentPackages:
for pkg in listRunTimeDependentPackages:
if pkg in self.mapPackageToCycles:
continue
packageName, packageVersion = StringUtils.splitPackageNameAndVersion(pkg)
latestPkgRPM = os.path.basename(
pkgUtils.findRPMFile(packageName, packageVersion)).replace(".rpm", "")
if pkg in listInstalledPackages and latestPkgRPM in listInstalledRPMs:
continue
self._installPackage(pkgUtils, packageName,packageVersion, sandbox, destLogPath,listInstalledPackages, listInstalledRPMs)
def _installDependencies(self, arch, deps=[]):
listDependentPackages, listTestPackages, listInstalledPackages, listInstalledRPMs = (
self._findDependentPackagesAndInstalledRPM(self.sandbox, arch))
# PackageUtils should be initialized here - as per arch basis
# Do not move it to __init__
pkgUtils = PackageUtils(self.logName, self.logPath)
if listDependentPackages:
self.logger.debug(
"Installing the build time dependent packages for " + arch)
for pkg in listDependentPackages:
packageName, packageVersion = StringUtils.splitPackageNameAndVersion(
pkg)
self._installPackage(pkgUtils, packageName, packageVersion,
self.sandbox, self.logPath,
listInstalledPackages, listInstalledRPMs,
arch)
for pkg in listTestPackages:
flag = False
packageName, packageVersion = StringUtils.splitPackageNameAndVersion(
pkg)
for depPkg in listDependentPackages:
depPackageName, depPackageVersion = StringUtils.splitPackageNameAndVersion(
depPkg)
if depPackageName == packageName:
flag = True
break
if flag == False:
self._installPackage(pkgUtils, packageName, packageVersion,
self.sandbox, self.logPath,
listInstalledPackages,
listInstalledRPMs, arch)
pkgUtils.installRPMSInOneShot(self.sandbox, arch)
self.logger.debug(
"Finished installing the build time dependent packages for " +
arch)
def _createGraphNodes():
# GRAPH-BUILD STEP 1: Initialize graph nodes for each package.
#
# Create a graph with a node to represent every package and all
# its dependent packages in the given list.
for package in Scheduler.sortedList:
packageName, packageVersion = StringUtils.splitPackageNameAndVersion(package)
node = DependencyGraphNode(packageName, packageVersion,
Scheduler._getWeight(package))
Scheduler.mapPackagesToGraphNodes[package] = node
if package in Scheduler.listOfAlreadyBuiltPackages:
node.built = 1
for package in Scheduler.sortedList:
pkgNode = Scheduler.mapPackagesToGraphNodes[package]
for childPackage in Scheduler._getBuildRequiredPackages(package):
childPkgNode = Scheduler.mapPackagesToGraphNodes[childPackage]
pkgNode.buildRequiresPkgNodes.add(childPkgNode)
for childPackage in Scheduler._getRequiredPackages(package):
childPkgNode = Scheduler.mapPackagesToGraphNodes[childPackage]
pkgNode.installRequiresPkgNodes.add(childPkgNode)
# GRAPH-BUILD STEP 2: Mark package dependencies in the graph.
#
# Add parent-child relationships between dependent packages.
# If a package 'A' build-requires or install-requires package 'B', then:
# - Mark 'B' as a child of 'A' in the graph.
# - Mark 'A' as a parent of 'B' in the graph.
#
# A
#
# / \
# v v
#
# B C
#
for package in Scheduler.sortedList:
pkgNode = Scheduler.mapPackagesToGraphNodes[package]
for childPkgNode in pkgNode.buildRequiresPkgNodes:
pkgNode.childPkgNodes.add(childPkgNode)
childPkgNode.parentPkgNodes.add(pkgNode)
for childPkgNode in pkgNode.installRequiresPkgNodes:
pkgNode.childPkgNodes.add(childPkgNode)
childPkgNode.parentPkgNodes.add(pkgNode)
def _installDependentRunTimePackages(self, pkgUtils, package, packageVersion, sandbox, destLogPath,
listInstalledPackages, listInstalledRPMs):
listRunTimeDependentPackages = self._findRunTimeRequiredRPMPackages(package, packageVersion)
if listRunTimeDependentPackages:
for pkg in listRunTimeDependentPackages:
if pkg in self.mapPackageToCycles:
continue
packageName, packageVersion = StringUtils.splitPackageNameAndVersion(pkg)
rpmfile = pkgUtils.findRPMFile(packageName, packageVersion)
if rpmfile is None:
self.logger.error("No rpm file found for package: " + packageName + "-" + packageVersion)
raise Exception("Missing rpm file")
latestPkgRPM = os.path.basename(rpmfile).replace(".rpm", "")
if pkg in listInstalledPackages and latestPkgRPM in listInstalledRPMs:
continue
self._installPackage(pkgUtils, packageName,packageVersion, sandbox, destLogPath,listInstalledPackages, listInstalledRPMs)
def build(self, pkg, doneList):
packageName, packageVersion = StringUtils.splitPackageNameAndVersion(pkg)
#do not build if RPM is already built
#test only if the package is in the testForceRPMS with rpmCheck
#build only if the package is not in the testForceRPMS with rpmCheck
if not constants.rpmCheck or packageName in constants.testForceRPMS:
if self._checkIfPackageIsAlreadyBuilt(packageName, packageVersion, doneList):
return
self._buildPackagePrepareFunction(packageName, packageVersion, doneList)
try:
self._buildPackage()
except Exception as e:
# TODO: self.logger might be None
self.logger.exception(e)
raise e
def build(self, pkg, doneList):
packageName, packageVersion = StringUtils.splitPackageNameAndVersion(pkg)
#do not build if RPM is already built
#test only if the package is in the testForceRPMS with rpmCheck
#build only if the package is not in the testForceRPMS with rpmCheck
if not constants.rpmCheck or packageName in constants.testForceRPMS:
if self._checkIfPackageIsAlreadyBuilt(packageName, packageVersion, doneList):
return
self._buildPackagePrepareFunction(packageName, packageVersion, doneList)
try:
self._buildPackage()
except Exception as e:
# TODO: self.logger might be None
self.logger.exception(e)
raise e
def _createGraphNodes():
# GRAPH-BUILD STEP 1: Initialize graph nodes for each package.
#
# Create a graph with a node to represent every package and all
# its dependent packages in the given list.
for package in Scheduler.sortedList:
packageName, packageVersion = StringUtils.splitPackageNameAndVersion(package)
node = DependencyGraphNode(packageName, packageVersion,
Scheduler._getWeight(package))
Scheduler.mapPackagesToGraphNodes[package] = node
for package in Scheduler.sortedList:
pkgNode = Scheduler.mapPackagesToGraphNodes[package]
for childPackage in Scheduler._getBuildRequiredPackages(package):
childPkgNode = Scheduler.mapPackagesToGraphNodes[childPackage]
pkgNode.buildRequiresPkgNodes.add(childPkgNode)
for childPackage in Scheduler._getRequiredPackages(package):
childPkgNode = Scheduler.mapPackagesToGraphNodes[childPackage]
pkgNode.installRequiresPkgNodes.add(childPkgNode)
# GRAPH-BUILD STEP 2: Mark package dependencies in the graph.
#
# Add parent-child relationships between dependent packages.
# If a package 'A' build-requires or install-requires package 'B', then:
# - Mark 'B' as a child of 'A' in the graph.
# - Mark 'A' as a parent of 'B' in the graph.
#
# A
#
# / \
# v v
#
# B C
#
for package in Scheduler.sortedList:
pkgNode = Scheduler.mapPackagesToGraphNodes[package]
for childPkgNode in pkgNode.buildRequiresPkgNodes:
pkgNode.childPkgNodes.add(childPkgNode)
childPkgNode.parentPkgNodes.add(pkgNode)
for childPkgNode in pkgNode.installRequiresPkgNodes:
pkgNode.childPkgNodes.add(childPkgNode)
childPkgNode.parentPkgNodes.add(pkgNode)
def installToolChainRPMS(self,
chroot,
packageName=None,
packageVersion=None,
logPath=None):
if logPath is None:
logPath = self.logPath
self.logger.debug("Installing Tool Chain RPMS.......")
rpmFiles = ""
packages = ""
listBuildRequiresPackages = []
if packageName:
listBuildRequiresPackages = self.getListDependentPackages(
packageName, packageVersion)
for package in constants.listToolChainRPMsToInstall:
pkgUtils = PackageUtils(self.logName, self.logPath)
rpmFile = None
version = "*"
for depPkg in listBuildRequiresPackages:
depPkgName, depPkgVersion = StringUtils.splitPackageNameAndVersion(
depPkg)
if depPkgName == package:
version = depPkgVersion
if constants.rpmCheck:
rpmFile = pkgUtils.findRPMFileForGivenPackage(package, version)
elif (packageName is None
or packageName not in constants.listToolChainRPMsToInstall
or constants.listToolChainRPMsToInstall.index(packageName) >
constants.listToolChainRPMsToInstall.index(package)):
rpmFile = pkgUtils.findRPMFileForGivenPackage(package, version)
if rpmFile is None:
# sqlite-autoconf package was renamed, but it still published as sqlite-autoconf
if (package == "sqlite") and (platform.machine() == "x86_64"):
package = "sqlite-autoconf"
rpmFile = self.findRPMFileInGivenLocation(
package, constants.prevPublishRPMRepo)
if rpmFile is None:
if package in constants.listOfRPMsProvidedAfterBuild:
self.logger.debug(
"No old version of " + package +
" exists, skip until the new version is built")
continue
self.logger.error("Unable to find rpm " + package +
" in current and previous versions")
raise Exception("Input Error")
rpmFiles += " " + rpmFile
packages += " " + package + "-" + version
self.logger.debug(packages)
cmd = (self.rpmCommand + " -i -v --nodeps --noorder --force --root " +
chroot.getPath() + " --define \'_dbpath /var/lib/rpm\' " +
rpmFiles)
retVal = CommandUtils.runCommandInShell(cmd, logfn=self.logger.debug)
if retVal != 0:
self.logger.debug("Command Executed:" + cmd)
self.logger.error("Installing tool chain failed")
raise Exception("RPM installation failed")
self.logger.debug(
"Successfully installed default Tool Chain RPMS in Chroot:" +
chroot.getPath())
if packageName:
self.installCustomToolChainRPMS(chroot, packageName,
packageVersion)
def installToolchainRPMS(self, chroot, packageName=None, packageVersion=None, usePublishedRPMS=True, availablePackages=None):
self.logger.debug("Installing toolchain RPMS.......")
rpmFiles = ""
packages = ""
listBuildRequiresPackages = []
listRPMsToInstall=list(constants.listToolChainRPMsToInstall)
if constants.crossCompiling:
targetPackageName = packageName
packageName = None
packageVersion = None
listRPMsToInstall.extend(['binutils-'+constants.targetArch+'-linux-gnu',
'gcc-'+constants.targetArch+'-linux-gnu'])
if packageName:
listBuildRequiresPackages = self.getListDependentPackages(packageName, packageVersion)
for package in listRPMsToInstall:
pkgUtils = PackageUtils(self.logName, self.logPath)
rpmFile = None
version = None
# Get proper package version
for depPkg in listBuildRequiresPackages:
depPkgName, depPkgVersion = StringUtils.splitPackageNameAndVersion(depPkg)
if depPkgName == package:
version=depPkgVersion
break
if not version:
version = SPECS.getData(constants.buildArch).getHighestVersion(package)
if availablePackages is not None:
basePkg = SPECS.getData(constants.buildArch).getSpecName(package)+"-"+version
isAvailable = basePkg in availablePackages
else:
# if availablePackages is not provided (rear case) it is safe
# to use findRPMFile()
isAvailable = True
if constants.rpmCheck:
rpmFile = pkgUtils.findRPMFile(package, version, constants.buildArch)
if rpmFile is None:
# Honor the toolchain list order.
# if index of depended package ('package') is more
# then index of the current package that we are
# building ('packageName'), then we _must_ use published
# `package` rpm.
if (packageName and
packageName in listRPMsToInstall and
listRPMsToInstall.index(packageName) <
listRPMsToInstall.index(package)):
isAvailable = False
if isAvailable:
rpmFile = pkgUtils.findRPMFile(package, version, constants.buildArch)
if rpmFile is None:
if not usePublishedRPMS or isAvailable or constants.crossCompiling:
raise Exception("%s-%s.%s not found in available packages" % (package, version, constants.buildArch))
# Safe to use published RPM
rpmFile = self._findPublishedRPM(package, constants.prevPublishRPMRepo)
if rpmFile is None:
self.logger.error("Unable to find published rpm " + package)
raise Exception("Input Error")
rpmFiles += " " + rpmFile
packages += " " + package+"-"+version
self.logger.debug(rpmFiles)
self.logger.debug(packages)
cmd = (self.rpmCommand + " -i -v --nodeps --noorder --force --root " +
chroot.getID() +" --define \'_dbpath /var/lib/rpm\' "+ rpmFiles)
# If rpm doesn't have zstd support, use rpm from photon_builder image
if constants.hostRpmIsNotUsable:
# if we are not root, make installed files owned by effective user to
# support pure non-root package building.
if os.geteuid() != 0:
cmd = cmd + "; chown -R {0}:{1} {2}".format(os.geteuid(), os.getegid(), chroot.getID())
cmd = ("docker run -i -v " + constants.prevPublishRPMRepo + ":" + constants.prevPublishRPMRepo +
" -v " + constants.rpmPath + ":" + constants.rpmPath + " -v " + chroot.getID() + ":" +
chroot.getID() + " photon_builder:latest " + "/bin/bash -c \"" + cmd + "\"")
self.logger.debug("Executing cmd: " + cmd)
retVal = CommandUtils.runCommandInShell(cmd, logfn=self.logger.debug)
if retVal != 0:
self.logger.error("Installing toolchain RPMS failed")
raise Exception("RPM installation failed")
self.logger.debug("Successfully installed default toolchain RPMS in Chroot:" + chroot.getID())
if packageName:
self.installExtraToolchainRPMS(chroot, packageName, packageVersion)
if constants.crossCompiling:
self.installTargetToolchain(chroot, targetPackageName)
def _buildPackage(self):
#should initialize a logger based on package name
containerTaskName = "build-" + self.package + "-" + self.version
container = None
try:
container = Container(self.logger)
container.create(containerTaskName)
tcUtils = ToolChainUtils(self.logName, self.logPath)
tcUtils.installCustomToolChainRPMS(container, self.package,
self.version)
listDependentPackages, listTestPackages, listInstalledPackages, listInstalledRPMs = (
self._findDependentPackagesAndInstalledRPM(container))
pkgUtils = PackageUtils(self.logName, self.logPath)
if listDependentPackages:
self.logger.debug(
"Installing the build time dependent packages......")
for pkg in listDependentPackages:
packageName, packageVersion = StringUtils.splitPackageNameAndVersion(
pkg)
self._installPackage(pkgUtils, packageName, packageVersion,
container, self.logPath,
listInstalledPackages,
listInstalledRPMs)
for pkg in listTestPackages:
flag = False
packageName, packageVersion = StringUtils.splitPackageNameAndVersion(
pkg)
for depPkg in listDependentPackages:
depPackageName, depPackageVersion = StringUtils.splitPackageNameAndVersion(
depPkg)
if depPackageName == packageName:
flag = True
break
if flag == False:
self._installPackage(pkgUtils, packageName,
packageVersion, container,
self.logPath,
listInstalledPackages,
listInstalledRPMs)
pkgUtils.installRPMSInOneShot(container)
self.logger.debug(
"Finished installing the build time dependent packages...."
)
self.logger.debug(
"BuildContainer-buildPackage: Start building the package: " +
self.package)
pkgUtils.adjustGCCSpecs(container, self.package, self.version)
pkgUtils.buildRPMSForGivenPackage(container, self.package,
self.version, self.logPath)
self.logger.debug(
"BuildContainer-buildPackage: Successfully built the package: "
+ self.package)
except Exception as e:
self.logger.error("Failed while building package:" + self.package)
if container is not None:
self.logger.debug("Container " + container.getID() +
" retained for debugging.")
logFileName = os.path.join(self.logPath, self.package + ".log")
fileLog = os.popen('tail -n 20 ' + logFileName).read()
self.logger.debug(fileLog)
raise e
# Remove the container
if container:
container.destroy()
def installToolchainRPMS(self,
chroot,
packageName=None,
packageVersion=None,
usePublishedRPMS=True,
availablePackages=None):
self.logger.debug("Installing toolchain RPMS.......")
rpmFiles = ""
packages = ""
listBuildRequiresPackages = []
listRPMsToInstall = list(constants.listToolChainRPMsToInstall)
if constants.crossCompiling:
targetPackageName = packageName
packageName = None
packageVersion = None
listRPMsToInstall.extend([
'binutils-' + constants.targetArch + '-linux-gnu',
'gcc-' + constants.targetArch + '-linux-gnu'
])
if packageName:
listBuildRequiresPackages = self.getListDependentPackages(
packageName, packageVersion)
for package in listRPMsToInstall:
pkgUtils = PackageUtils(self.logName, self.logPath)
rpmFile = None
version = None
# Get proper package version
for depPkg in listBuildRequiresPackages:
depPkgName, depPkgVersion = StringUtils.splitPackageNameAndVersion(
depPkg)
if depPkgName == package:
version = depPkgVersion
break
if not version:
version = SPECS.getData(
constants.buildArch).getHighestVersion(package)
if availablePackages is not None:
basePkg = SPECS.getData(
constants.buildArch).getSpecName(package) + "-" + version
isAvailable = basePkg in availablePackages
else:
# if availablePackages is not provided (rear case) it is safe
# to use findRPMFile()
isAvailable = True
if constants.rpmCheck:
rpmFile = pkgUtils.findRPMFile(package, version,
constants.buildArch)
if rpmFile is None:
# Honor the toolchain list order.
# if index of depended package ('package') is more
# then index of the current package that we are
# building ('packageName'), then we _must_ use published
# `package` rpm.
if (packageName and packageName in listRPMsToInstall
and listRPMsToInstall.index(packageName) <
listRPMsToInstall.index(package)):
isAvailable = False
if isAvailable:
rpmFile = pkgUtils.findRPMFile(package, version,
constants.buildArch)
if rpmFile is None:
if not usePublishedRPMS or isAvailable or constants.crossCompiling:
raise Exception(
"%s-%s.%s not found in available packages" %
(package, version, constants.buildArch))
# Safe to use published RPM
# sqlite-autoconf package was renamed, but it still published as sqlite-autoconf
if (package == "sqlite") and (constants.buildArch == "x86_64"):
package = "sqlite-autoconf"
rpmFile = self._findPublishedRPM(package,
constants.prevPublishRPMRepo)
if rpmFile is None:
if package in constants.listOfRPMsProvidedAfterBuild:
self.logger.debug(
"No old version of " + package +
" exists, skip until the new version is built")
continue
self.logger.error("Unable to find published rpm " +
package)
raise Exception("Input Error")
rpmFiles += " " + rpmFile
packages += " " + package + "-" + version
self.logger.debug(rpmFiles)
self.logger.debug(packages)
cmd = (self.rpmCommand + " -i -v --nodeps --noorder --force --root " +
chroot.getID() + " --define \'_dbpath /var/lib/rpm\' " +
rpmFiles)
retVal = CommandUtils.runCommandInShell(cmd, logfn=self.logger.debug)
if retVal != 0:
self.logger.debug("Command Executed:" + cmd)
self.logger.error("Installing toolchain RPMS failed")
raise Exception("RPM installation failed")
self.logger.debug(
"Successfully installed default toolchain RPMS in Chroot:" +
chroot.getID())
if packageName:
self.installExtraToolchainRPMS(chroot, packageName, packageVersion)
if constants.crossCompiling:
self.installTargetToolchain(chroot, targetPackageName)
def main():
usage = "Usage: %prog [options]"
parser = ArgumentParser(usage)
parser.add_argument("-i", "--input-type", dest="input_type", default=DEFAULT_INPUT_TYPE)
parser.add_argument("-p", "--pkg", dest="pkg")
parser.add_argument("-f", "--file", dest="json_file", default="packages_minimal.json")
parser.add_argument("-d", "--display-option", dest="display_option", default=DEFAULT_DISPLAY_OPTION)
parser.add_argument("-s", "--spec-path", dest="spec_path", default=SPEC_FILE_DIR)
parser.add_argument("-l", "--log-path", dest="log_path", default=LOG_FILE_DIR)
parser.add_argument("-y", "--log-level", dest="log_level", default="info")
parser.add_argument("-t", "--stage-dir", dest="stage_dir", default="../../stage")
parser.add_argument("-a", "--input-data-dir", dest="input_data_dir", default="../../common/data/")
parser.add_argument("-o", "--output-dir", dest="output_dir", default="../../stage/common/data")
options = parser.parse_args()
constants.setSpecPath(options.spec_path)
constants.setLogPath(options.log_path)
constants.setLogLevel(options.log_level)
constants.initialize()
cmdUtils = CommandUtils()
logger = Logger.getLogger("SpecDeps", options.log_path, options.log_level)
if not os.path.isdir(options.output_dir):
cmdUtils.runCommandInShell("mkdir -p "+options.output_dir)
if not options.input_data_dir.endswith('/'):
options.input_data_dir += '/'
try:
specDeps = SpecDependencyGenerator(options.log_path, options.log_level)
if options.input_type == "remove-upward-deps":
isToolChainPkg = specDeps.process("is-toolchain-pkg", options.pkg, options.display_option)
if isToolChainPkg:
logger.info("Removing all staged RPMs since toolchain packages were modified")
cmdUtils.runCommandInShell("rm -rf stage/RPMS/")
else:
whoNeedsList = specDeps.process("get-upward-deps", options.pkg, options.display_option)
logger.info("Removing upward dependencies: " + str(whoNeedsList))
for pkg in whoNeedsList:
package, version = StringUtils.splitPackageNameAndVersion(pkg)
release = SPECS.getData().getRelease(package, version)
for p in SPECS.getData().getPackages(package,version):
buildarch=SPECS.getData().getBuildArch(p, version)
rpmFile = "stage/RPMS/" + buildarch + "/" + p + "-" + version + "-" + release + ".*" + buildarch+".rpm"
cmdUtils.runCommandInShell("rm -f "+rpmFile)
elif options.input_type == "print-upward-deps":
whoNeedsList = specDeps.process("get-upward-deps", options.pkg, options.display_option)
logger.info("Upward dependencies: " + str(whoNeedsList))
# To display/print package dependencies on console
elif (options.input_type == "pkg" or
options.input_type == "who-needs"):
specDeps.process(options.input_type, options.pkg, options.display_option)
elif options.input_type == "json":
list_json_files = options.json_file.split("\n")
# Generate the expanded package dependencies json file based on package_list_file
logger.info("Generating the install time dependency list for all json files")
if list_json_files:
shutil.copy2(os.path.dirname(list_json_files[0]) + "/build_install_options_all.json", options.output_dir)
for json_file in list_json_files:
output_file = None
if options.display_option == "json":
output_file = os.path.join(options.output_dir, os.path.basename(json_file))
specDeps.process(options.input_type, json_file, options.display_option, output_file)
except Exception as e:
traceback.print_exc()
sys.stderr.write(str(e))
sys.stderr.write("Failed to generate dependency lists from spec files\n")
sys.exit(1)
sys.exit(0)
def main():
usage = "Usage: %prog [options]"
parser = ArgumentParser(usage)
parser.add_argument("-i", "--input-type", dest="input_type", default=DEFAULT_INPUT_TYPE)
parser.add_argument("-p", "--pkg", dest="pkg")
parser.add_argument("-f", "--file", dest="json_file", default="packages_minimal.json")
parser.add_argument("-d", "--display-option", dest="display_option", default=DEFAULT_DISPLAY_OPTION)
parser.add_argument("-s", "--spec-path", dest="spec_path", default=SPEC_FILE_DIR)
parser.add_argument("-l", "--log-path", dest="log_path", default=LOG_FILE_DIR)
parser.add_argument("-y", "--log-level", dest="log_level", default="info")
parser.add_argument("-t", "--stage-dir", dest="stage_dir", default="../../stage")
parser.add_argument("-a", "--input-data-dir", dest="input_data_dir", default="../../common/data/")
parser.add_argument("-o", "--output-dir", dest="output_dir", default="../../stage/common/data")
options = parser.parse_args()
constants.setSpecPath(options.spec_path)
constants.setLogPath(options.log_path)
constants.setLogLevel(options.log_level)
constants.initialize()
cmdUtils = CommandUtils()
logger = Logger.getLogger("SpecDeps", options.log_path, options.log_level)
if not os.path.isdir(options.output_dir):
cmdUtils.runCommandInShell2("mkdir -p "+options.output_dir)
if not options.input_data_dir.endswith('/'):
options.input_data_dir += '/'
try:
specDeps = SpecDependencyGenerator(options.log_path, options.log_level)
if options.input_type == "remove-upward-deps":
whoNeedsList = specDeps.process("get-upward-deps", options.pkg, options.display_option)
logger.info("Removing upward dependencies: " + str(whoNeedsList))
for pkg in whoNeedsList:
package, version = StringUtils.splitPackageNameAndVersion(pkg)
release = SPECS.getData().getRelease(package, version)
for p in SPECS.getData().getPackages(package,version):
buildarch=SPECS.getData().getBuildArch(p, version)
rpmFile = "stage/RPMS/" + buildarch + "/" + p + "-" + version + "-" + release + ".*" + buildarch+".rpm"
cmdUtils.runCommandInShell2("rm -f "+rpmFile)
elif options.input_type == "print-upward-deps":
whoNeedsList = specDeps.process("get-upward-deps", options.pkg, options.display_option)
logger.info("Upward dependencies: " + str(whoNeedsList))
# To display/print package dependencies on console
elif (options.input_type == "pkg" or
options.input_type == "who-needs"):
specDeps.process(options.input_type, options.pkg, options.display_option)
elif options.input_type == "json":
list_json_files = options.json_file.split("\n")
# Generate the expanded package dependencies json file based on package_list_file
logger.info("Generating the install time dependency list for all json files")
for json_file in list_json_files:
shutil.copy2(json_file, options.output_dir)
json_wrapper_option_list = JsonWrapper(json_file)
option_list_json = json_wrapper_option_list.read()
options_sorted = option_list_json.items()
for install_option in options_sorted:
output_file = None
input_value = os.path.join(os.path.dirname(json_file), install_option[1]["file"])
if options.display_option == "tree" and install_option[1]["title"] == "ISO Packages":
continue
if options.display_option == "json":
output_file = os.path.join(options.output_dir, install_option[1]["file"])
specDeps.process(options.input_type, input_value, options.display_option, output_file)
except Exception as e:
traceback.print_exc()
sys.stderr.write(str(e))
sys.stderr.write("Failed to generate dependency lists from spec files\n")
sys.exit(1)
sys.exit(0)
def getBasePkg(self, pkg):
package, version = StringUtils.splitPackageNameAndVersion(pkg)
return self.getSpecName(package)+"-"+version
def getPackagesForPkg(self, pkg):
pkgs = []
package, version = StringUtils.splitPackageNameAndVersion(pkg)
for p in self.getPackages(package, version):
pkgs.append(p + "-" + version)
return pkgs
def _buildPackage(self):
#should initialize a logger based on package name
containerTaskName = "build-" + self.package
containerID = None
chrootID = None
isToolChainPackage = False
if self.package in constants.listToolChainPackages:
isToolChainPackage = True
destLogPath = constants.logPath + "/build-" + self.package
try:
containerID, chrootID = self._prepareBuildContainer(
containerTaskName, self.package, self.version,
isToolChainPackage)
tcUtils = ToolChainUtils(self.logName, self.logPath)
if self.package in constants.perPackageToolChain:
self.logger.debug(constants.perPackageToolChain[self.package])
tcUtils.installCustomToolChainRPMSinContainer(
containerID,
constants.perPackageToolChain[self.package].get(
platform.machine(), []), self.package)
listDependentPackages, listTestPackages, listInstalledPackages, listInstalledRPMs = (
self._findDependentPackagesAndInstalledRPM(containerID))
pkgUtils = PackageUtils(self.logName, self.logPath)
if listDependentPackages:
self.logger.info("BuildContainer-buildPackage: " +
"Installing dependent packages..")
self.logger.info(listDependentPackages)
for pkg in listDependentPackages:
packageName, packageVersion = StringUtils.splitPackageNameAndVersion(
pkg)
self._installPackage(pkgUtils, packageName, packageVersion,
containerID, destLogPath,
listInstalledPackages,
listInstalledRPMs)
for pkg in listTestPackages:
packageName, packageVersion = StringUtils.splitPackageNameAndVersion(
pkg)
flag = False
for depPkg in listDependentPackages:
depPackageName, depPackageVersion = StringUtils.splitPackageNameAndVersion(
depPkg)
if depPackageName == packageName:
flag = True
break
if flag == False:
self._installPackage(pkgUtils, packageName,
packageVersion, containerID,
destLogPath,
listInstalledPackages,
listInstalledRPMs)
pkgUtils.installRPMSInAOneShotInContainer(
containerID, destLogPath)
self.logger.info(
"Finished installing the build time dependent packages...."
)
self.logger.info(
"BuildContainer-buildPackage: Start building the package: " +
self.package)
pkgUtils.adjustGCCSpecsInContainer(self.package, self.version,
containerID, destLogPath)
pkgUtils.buildRPMSForGivenPackageInContainer(
self.package, self.version, containerID, destLogPath)
self.logger.info(
"BuildContainer-buildPackage: Successfully built the package: "
+ self.package)
except Exception as e:
self.logger.error("Failed while building package:" + self.package)
if containerID is not None:
self.logger.debug("Container " + containerID.short_id +
" retained for debugging.")
logFileName = os.path.join(destLogPath, self.package + ".log")
fileLog = os.popen('tail -n 20 ' + logFileName).read()
self.logger.debug(fileLog)
raise e
# Remove the container
if containerID is not None:
containerID.remove(force=True)
# Remove the dummy chroot
if chrootID is not None:
chrUtils = ChrootUtils(self.logName, self.logPath)
chrUtils.destroyChroot(chrootID)
def getRequiresForPkg(self, pkg):
package, version = StringUtils.splitPackageNameAndVersion(pkg)
return self.getRequiresForPackage(package, version)
def getRequiresForPkg(self, pkg):
package, version = StringUtils.splitPackageNameAndVersion(pkg)
return self.getRequiresForPackage(package, version)
def getBasePkg(self, pkg):
package, version = StringUtils.splitPackageNameAndVersion(pkg)
return self.getSpecName(package) + "-" + version
def getPackagesForPkg(self, pkg):
pkgs=[]
package, version = StringUtils.splitPackageNameAndVersion(pkg)
for p in self.getPackages(package, version):
pkgs.append(p+"-"+version)
return pkgs
