def buildPackage(self,package):
#should initialize a logger based on package name
chrUtils = ChrootUtils(self.logName,self.logPath)
chrootName="build-"+package
chrootID=None
isToolChainPackage=False
if package in constants.listToolChainPackages:
isToolChainPackage=True
try:
chrootID = self.prepareBuildRoot(chrootName,isToolChainPackage)
destLogPath=constants.logPath+"/build-"+package
if not os.path.isdir(destLogPath):
cmdUtils = CommandUtils()
cmdUtils.runCommandInShell("mkdir -p "+destLogPath)
listInstalledPackages=self.findInstalledPackages(chrootID)
self.logger.info("List of installed packages")
self.logger.info(listInstalledPackages)
listDependentPackages=self.findBuildTimeRequiredPackages(package)
if len(listDependentPackages) != 0:
self.logger.info("Installing the build time dependent packages......")
for pkg in listDependentPackages:
self.installPackage(pkg,chrootID,destLogPath,listInstalledPackages)
self.logger.info("Finished installing the build time dependent packages......")
self.adjustGCCSpecs(package, chrootID, destLogPath)
pkgUtils = PackageUtils(self.logName,self.logPath)
pkgUtils.buildRPMSForGivenPackage(package,chrootID,destLogPath)
self.logger.info("Successfully built the package:"+package)
except Exception as e:
self.logger.error("Failed while building package:" + package)
self.logger.debug("Chroot with ID: " + chrootID + " not deleted for debugging.")
raise e
if chrootID is not None:
chrUtils.destroyChroot(chrootID)
def buildPackage(self, package):
#do not build if RPM is already built
#test only if the package is in the testForceRPMS with rpmCheck
#build only if the package is not in the testForceRPMS with rpmCheck
if self.checkIfPackageIsAlreadyBuilt(package):
if not constants.rpmCheck:
self.logger.info("Skipping building the package:" + package)
return
elif constants.rpmCheck and package not in constants.testForceRPMS:
self.logger.info("Skipping testing the package:" + package)
return
#should initialize a logger based on package name
chrUtils = ChrootUtils(self.logName, self.logPath)
chrootName = "build-" + package
chrootID = None
try:
chrootID = self.prepareBuildRoot(chrootName, package)
destLogPath = constants.logPath + "/build-" + package
if not os.path.isdir(destLogPath):
cmdUtils = CommandUtils()
cmdUtils.runCommandInShell("mkdir -p " + destLogPath)
listInstalledPackages = self.findInstalledPackages(chrootID)
listDependentPackages = self.findBuildTimeRequiredPackages(package)
if constants.rpmCheck and package in constants.testForceRPMS:
testPackages = set(
constants.listMakeCheckRPMPkgtoInstall) - set(
listInstalledPackages) - set([package])
listDependentPackages.extend(testPackages)
pkgUtils = PackageUtils(self.logName, self.logPath)
if len(listDependentPackages) != 0:
self.logger.info(
"Installing the build time dependent packages......")
for pkg in listDependentPackages:
self.installPackage(pkgUtils, pkg, chrootID, destLogPath,
listInstalledPackages)
pkgUtils.installRPMSInAOneShot(chrootID, destLogPath)
self.logger.info(
"Finished installing the build time dependent packages......"
)
pkgUtils.adjustGCCSpecs(package, chrootID, destLogPath)
pkgUtils.buildRPMSForGivenPackage(package, chrootID,
self.listBuildOptionPackages,
self.pkgBuildOptionFile,
destLogPath)
self.logger.info("Successfully built the package:" + package)
except Exception as e:
self.logger.error("Failed while building package:" + package)
self.logger.debug("Chroot with ID: " + chrootID +
" not deleted for debugging.")
logFileName = os.path.join(destLogPath, package + ".log")
fileLog = os.popen('tail -n 100 ' + logFileName).read()
self.logger.debug(fileLog)
raise e
if chrootID is not None:
chrUtils.destroyChroot(chrootID)
def _buildPackage(self):
chrUtils = ChrootUtils(self.logName, self.logPath)
chrootID = None
try:
chrootID = self._prepareBuildRoot()
listDependentPackages, listTestPackages, listInstalledPackages, listInstalledRPMs = (
self._findDependentPackagesAndInstalledRPM(chrootID))
pkgUtils = PackageUtils(self.logName, self.logPath)
if listDependentPackages:
self.logger.info(
"Installing the build time dependent packages......")
for pkg in listDependentPackages:
packageName, packageVersion = StringUtils.splitPackageNameAndVersion(
pkg)
self._installPackage(pkgUtils, packageName, packageVersion,
chrootID, self.logPath,
listInstalledPackages,
listInstalledRPMs)
for pkg in listTestPackages:
flag = False
packageName, packageVersion = StringUtils.splitPackageNameAndVersion(
pkg)
for depPkg in listDependentPackages:
depPackageName, depPackageVersion = StringUtils.splitPackageNameAndVersion(
depPkg)
if depPackageName == packageName:
flag = True
break
if flag == False:
self._installPackage(pkgUtils, packageName,
packageVersion, chrootID,
self.logPath,
listInstalledPackages,
listInstalledRPMs)
pkgUtils.installRPMSInAOneShot(chrootID, self.logPath)
self.logger.info(
"Finished installing the build time dependent packages...."
)
pkgUtils.adjustGCCSpecs(self.package, self.version, chrootID,
self.logPath)
pkgUtils.buildRPMSForGivenPackage(self.package, self.version,
chrootID, self.logPath)
self.logger.info("Successfully built the package:" + self.package)
except Exception as e:
self.logger.error("Failed while building package:" + self.package)
self.logger.debug("Chroot with ID: " + chrootID +
" not deleted for debugging.")
logFileName = os.path.join(self.logPath, self.package + ".log")
fileLog = os.popen('tail -n 100 ' + logFileName).read()
self.logger.debug(fileLog)
raise e
if chrootID is not None:
chrUtils.destroyChroot(chrootID)
def buildPackage(self,package):
#do not build if RPM is already built
#test only if the package is in the testForceRPMS with rpmCheck
#build only if the package is not in the testForceRPMS with rpmCheck
if self.checkIfPackageIsAlreadyBuilt(package):
if not constants.rpmCheck:
self.logger.info("Skipping building the package:"+package)
return
elif constants.rpmCheck and package not in constants.testForceRPMS:
self.logger.info("Skipping testing the package:"+package)
return
#should initialize a logger based on package name
chrUtils = ChrootUtils(self.logName,self.logPath)
chrootName="build-"+package
chrootID=None
isToolChainPackage=False
if package in constants.listToolChainPackages:
isToolChainPackage=True
try:
chrootID = self.prepareBuildRoot(chrootName,isToolChainPackage)
destLogPath=constants.logPath+"/build-"+package
if not os.path.isdir(destLogPath):
cmdUtils = CommandUtils()
cmdUtils.runCommandInShell("mkdir -p "+destLogPath)
listInstalledPackages=self.findInstalledPackages(chrootID)
listDependentPackages=self.findBuildTimeRequiredPackages(package)
if constants.rpmCheck and package in constants.testForceRPMS:
testPackages=set(constants.listMakeCheckRPMPkgtoInstall)-set(listInstalledPackages)-set([package])
listDependentPackages.extend(testPackages)
pkgUtils = PackageUtils(self.logName,self.logPath)
if len(listDependentPackages) != 0:
self.logger.info("Installing the build time dependent packages......")
for pkg in listDependentPackages:
self.installPackage(pkgUtils, pkg,chrootID,destLogPath,listInstalledPackages)
pkgUtils.installRPMSInAOneShot(chrootID,destLogPath)
self.logger.info("Finished installing the build time dependent packages......")
pkgUtils.adjustGCCSpecs(package, chrootID, destLogPath)
pkgUtils.buildRPMSForGivenPackage(package,chrootID,self.listBuildOptionPackages,self.pkgBuildOptionFile,destLogPath)
self.logger.info("Successfully built the package:"+package)
except Exception as e:
self.logger.error("Failed while building package:" + package)
self.logger.debug("Chroot with ID: " + chrootID + " not deleted for debugging.")
logFileName = os.path.join(destLogPath, package + ".log")
fileLog = os.popen('tail -n 100 ' + logFileName).read()
self.logger.debug(fileLog)
raise e
if chrootID is not None:
chrUtils.destroyChroot(chrootID)
def _buildPackage(self, index=0):
#do not build if RPM is already built
#test only if the package is in the testForceRPMS with rpmCheck
#build only if the package is not in the testForceRPMS with rpmCheck
if self._checkIfPackageIsAlreadyBuilt(index):
if not constants.rpmCheck:
self.logger.info("Skipping building the package:" +
self.package)
return
elif constants.rpmCheck and self.package not in constants.testForceRPMS:
self.logger.info("Skipping testing the package:" +
self.package)
return
chrUtils = ChrootUtils(self.logName, self.logPath)
chrootID = None
try:
chrootID = self._prepareBuildRoot()
listDependentPackages, listInstalledPackages, listInstalledRPMs = (
self._findDependentPackagesAndInstalledRPM(chrootID, index))
pkgUtils = PackageUtils(self.logName, self.logPath)
if listDependentPackages:
self.logger.info(
"Installing the build time dependent packages......")
for pkg in listDependentPackages:
self._installPackage(pkgUtils, pkg, chrootID, self.logPath,
listInstalledPackages,
listInstalledRPMs)
pkgUtils.installRPMSInAOneShot(chrootID, self.logPath)
self.logger.info(
"Finished installing the build time dependent packages...."
)
pkgUtils.adjustGCCSpecs(self.package, chrootID, self.logPath,
index)
pkgUtils.buildRPMSForGivenPackage(self.package, chrootID,
self.logPath, index)
self.logger.info("Successfully built the package:" + self.package)
except Exception as e:
self.logger.error("Failed while building package:" + self.package)
self.logger.debug("Chroot with ID: " + chrootID +
" not deleted for debugging.")
logFileName = os.path.join(self.logPath, self.package + ".log")
fileLog = os.popen('tail -n 100 ' + logFileName).read()
self.logger.debug(fileLog)
raise e
if chrootID is not None:
chrUtils.destroyChroot(chrootID)
def prepareBuildRoot(self,chrootName, packageName):
chrootID=None
try:
chrUtils = ChrootUtils(self.logName,self.logPath)
returnVal,chrootID = chrUtils.createChroot(chrootName)
self.logger.debug("Created new chroot: " + chrootID)
if not returnVal:
raise Exception("Unable to prepare build root")
tUtils=ToolChainUtils(self.logName,self.logPath)
tUtils.installToolChainRPMS(chrootID, packageName)
except Exception as e:
if chrootID is not None:
self.logger.debug("Deleting chroot: " + chrootID)
chrUtils.destroyChroot(chrootID)
raise e
return chrootID
def prepareBuildRoot(self, chrootName, packageName):
chrootID = None
try:
chrUtils = ChrootUtils(self.logName, self.logPath)
returnVal, chrootID = chrUtils.createChroot(chrootName)
self.logger.debug("Created new chroot: " + chrootID)
if not returnVal:
raise Exception("Unable to prepare build root")
tUtils = ToolChainUtils(self.logName, self.logPath)
tUtils.installToolChainRPMS(chrootID, packageName)
except Exception as e:
if chrootID is not None:
self.logger.debug("Deleting chroot: " + chrootID)
chrUtils.destroyChroot(chrootID)
raise e
return chrootID
def buildCoreToolChainPackages(self):
self.logger.info("Building core tool chain packages.....")
chrootID = None
try:
pkgUtils = PackageUtils(self.logName, self.logPath)
for package in constants.listCoreToolChainRPMPackages:
rpmPkg = pkgUtils.findRPMFileForGivenPackage(package)
if rpmPkg is not None:
continue
chrUtils = ChrootUtils(self.logName, self.logPath)
chrootName = "build-core-toolchain"
destLogPath = constants.logPath + "/build-" + package
if not os.path.isdir(destLogPath):
cmdUtils = CommandUtils()
cmdUtils.runCommandInShell("mkdir -p " + destLogPath)
returnVal, chrootID = chrUtils.createChroot(chrootName)
if not returnVal:
self.logger.error("Creating chroot failed")
raise Exception("creating chroot failed")
self.installToolChainRPMS(chrootID)
pkgUtils.buildRPMSForGivenPackage(package, chrootID,
destLogPath)
chrUtils.destroyChroot(chrootID)
chrootID = None
self.logger.info("Successfully built toolchain")
except Exception as e:
self.logger.error("Unable to build tool chain.")
raise e
finally:
if chrootID is not None:
chrUtils.destroyChroot(chrootID)
def buildPackage(self, package):
#should initialize a logger based on package name
chrUtils = ChrootUtils(self.logName, self.logPath)
chrootName = "build-" + package
chrootID = None
isToolChainPackage = False
if package in constants.listToolChainPackages:
isToolChainPackage = True
try:
chrootID = self.prepareBuildRoot(chrootName, isToolChainPackage)
destLogPath = constants.logPath + "/build-" + package
if not os.path.isdir(destLogPath):
cmdUtils = CommandUtils()
cmdUtils.runCommandInShell("mkdir -p " + destLogPath)
listInstalledPackages = self.findInstalledPackages(chrootID)
self.logger.info("List of installed packages")
self.logger.info(listInstalledPackages)
listDependentPackages = self.findBuildTimeRequiredPackages(package)
pkgUtils = PackageUtils(self.logName, self.logPath)
if len(listDependentPackages) != 0:
self.logger.info(
"Installing the build time dependent packages......")
for pkg in listDependentPackages:
self.installPackage(pkgUtils, pkg, chrootID, destLogPath,
listInstalledPackages)
pkgUtils.installRPMSInAOneShot(chrootID, destLogPath)
self.logger.info(
"Finished installing the build time dependent packages......"
)
pkgUtils.adjustGCCSpecs(package, chrootID, destLogPath)
pkgUtils.buildRPMSForGivenPackage(package, chrootID, destLogPath)
self.logger.info("Successfully built the package:" + package)
except Exception as e:
self.logger.error("Failed while building package:" + package)
self.logger.debug("Chroot with ID: " + chrootID +
" not deleted for debugging.")
logFileName = os.path.join(destLogPath, package + ".log")
fileLog = os.popen('tail -n 20 ' + logFileName).read()
self.logger.debug(fileLog)
raise e
if chrootID is not None:
chrUtils.destroyChroot(chrootID)
def _buildPackage(self):
#do not build if RPM is already built
#test only if the package is in the testForceRPMS with rpmCheck
#build only if the package is not in the testForceRPMS with rpmCheck
if self._checkIfPackageIsAlreadyBuilt():
if not constants.rpmCheck:
self.logger.info("Skipping building the package:" + self.package)
return
elif constants.rpmCheck and self.package not in constants.testForceRPMS:
self.logger.info("Skipping testing the package:" + self.package)
return
chrUtils = ChrootUtils(self.logName, self.logPath)
chrootID = None
try:
chrootID = self._prepareBuildRoot()
listDependentPackages, listInstalledPackages, listInstalledRPMs = (
self._findDependentPackagesAndInstalledRPM(chrootID))
pkgUtils = PackageUtils(self.logName, self.logPath)
if listDependentPackages:
self.logger.info("Installing the build time dependent packages......")
for pkg in listDependentPackages:
self._installPackage(pkgUtils, pkg, chrootID, self.logPath,
listInstalledPackages, listInstalledRPMs)
pkgUtils.installRPMSInAOneShot(chrootID, self.logPath)
self.logger.info("Finished installing the build time dependent packages....")
pkgUtils.adjustGCCSpecs(self.package, chrootID, self.logPath)
pkgUtils.buildRPMSForGivenPackage(self.package, chrootID,
self.logPath)
self.logger.info("Successfully built the package:" + self.package)
except Exception as e:
self.logger.error("Failed while building package:" + self.package)
self.logger.debug("Chroot with ID: " + chrootID +
" not deleted for debugging.")
logFileName = os.path.join(self.logPath, self.package + ".log")
fileLog = os.popen('tail -n 100 ' + logFileName).read()
self.logger.debug(fileLog)
raise e
if chrootID is not None:
chrUtils.destroyChroot(chrootID)
def buildCoreToolChainPackages(self):
self.logger.info("Step 1 : Building the core toolchain packages.....")
self.logger.info(constants.listCoreToolChainPackages)
self.logger.info("")
chrootID = None
pkgCount = 0
try:
pkgUtils = PackageUtils(self.logName, self.logPath)
coreToolChainYetToBuild = []
for package in constants.listCoreToolChainPackages:
version = SPECS.getData().getHighestVersion(package)
rpmPkg = pkgUtils.findRPMFileForGivenPackage(package, version)
if rpmPkg is not None:
continue
else:
coreToolChainYetToBuild.append(package)
if coreToolChainYetToBuild:
self.logger.info(
"The following core toolchain packages need to be built :")
self.logger.info(coreToolChainYetToBuild)
else:
self.logger.info(
"Core toolchain packages are already available")
for package in coreToolChainYetToBuild:
version = SPECS.getData().getHighestVersion(package)
self.logger.debug("Building core toolchain package : " +
package)
chrUtils = ChrootUtils(self.logName, self.logPath)
chrootName = "build-" + package
destLogPath = constants.logPath + "/build-" + package
if not os.path.isdir(destLogPath):
cmdUtils = CommandUtils()
cmdUtils.runCommandInShell("mkdir -p " + destLogPath)
returnVal, chrootID = chrUtils.createChroot(chrootName)
if not returnVal:
self.logger.error("Creating chroot failed")
raise Exception("creating chroot failed")
self.installToolChainRPMS(package, version, chrootID,
destLogPath)
pkgUtils.adjustGCCSpecs(package, version, chrootID,
destLogPath)
pkgUtils.buildRPMSForGivenPackage(package, version, chrootID,
destLogPath)
pkgCount += 1
chrUtils.destroyChroot(chrootID)
chrootID = None
self.logger.debug("Successfully built toolchain")
self.logger.info("-" * 45 + "\n")
if chrootID is not None:
chrUtils.destroyChroot(chrootID)
except Exception as e:
self.logger.error("Unable to build tool chain.")
# print stacktrace
traceback.print_exc()
raise e
return pkgCount
def buildCoreToolChainPackages(self):
self.logger.info("Building core tool chain packages.....")
chrootID=None
try:
pkgUtils=PackageUtils(self.logName,self.logPath)
for package in constants.listCoreToolChainRPMPackages:
rpmPkg=pkgUtils.findRPMFileForGivenPackage(package)
if rpmPkg is not None:
continue
chrUtils = ChrootUtils(self.logName,self.logPath)
chrootName="build-core-toolchain"
destLogPath=constants.logPath+"/build-"+package
if not os.path.isdir(destLogPath):
cmdUtils = CommandUtils()
cmdUtils.runCommandInShell("mkdir -p "+destLogPath)
returnVal,chrootID = chrUtils.createChroot(chrootName)
if not returnVal:
self.logger.error("Creating chroot failed")
raise Exception("creating chroot failed")
self.installToolChainRPMS(chrootID)
pkgUtils.adjustGCCSpecs(package, chrootID, destLogPath)
pkgUtils.buildRPMSForGivenPackage(package, chrootID,destLogPath)
chrUtils.destroyChroot(chrootID)
chrootID=None
self.logger.info("Successfully built toolchain")
if chrootID is not None:
chrUtils.destroyChroot(chrootID)
except Exception as e:
self.logger.error("Unable to build tool chain.")
# print stacktrace
traceback.print_exc()
raise e
def createBuildContainer(self):
self.logger.info("Generating photon build container..")
try:
#TODO image name constants.buildContainerImageName
self.dockerClient.images.remove("photon_build_container:latest",
force=True)
except Exception as e:
#TODO - better handling
self.logger.debug("Photon build container image not found.")
# Create toolchain chroot and install toolchain RPMs
chrootID = None
try:
#TODO: constants.tcrootname
chrUtils = ChrootUtils("toolchain-chroot", self.logPath)
returnVal, chrootID = chrUtils.createChroot("toolchain-chroot")
self.logger.debug("Created tool-chain chroot: " + chrootID)
if not returnVal:
raise Exception("Unable to prepare tool-chain chroot")
tcUtils = ToolChainUtils("toolchain-chroot", self.logPath)
tcUtils.installToolChainRPMS(chrootID, "dummy")
except Exception as e:
if chrootID is not None:
self.logger.debug("Deleting chroot: " + chrootID)
chrUtils.destroyChroot(chrootID)
raise e
self.logger.info("VDBG-PU-createBuildContainer: chrootID: " + chrootID)
# Create photon build container using toolchain chroot
#TODO: Coalesce logging
cmdUtils = CommandUtils()
cmd = "./umount-build-root.sh " + chrootID
cmdUtils.runCommandInShell(cmd,
self.logPath + "/toolchain-chroot1.log")
cmd = "cd " + chrootID + " && tar -czvf ../tcroot.tar.gz ."
cmdUtils.runCommandInShell(cmd,
self.logPath + "/toolchain-chroot2.log")
cmd = "mv " + chrootID + "/../tcroot.tar.gz ."
cmdUtils.runCommandInShell(cmd,
self.logPath + "/toolchain-chroot3.log")
#TODO: Container name, docker file name from constants.
self.dockerClient.images.build(
tag="photon_build_container:latest",
path=".",
rm=True,
dockerfile="Dockerfile.photon_build_container")
# Cleanup
cmd = "rm -f ./tcroot.tar.gz"
cmdUtils.runCommandInShell(cmd,
self.logPath + "/toolchain-chroot4.log")
chrUtils.destroyChroot(chrootID)
self.logger.info("Photon build container successfully created.")
def _createBuildContainer(self):
self.logger.info("Generating photon build container..")
try:
#TODO image name constants.buildContainerImageName
self.dockerClient.images.remove("photon_build_container:latest", force=True)
except Exception as e:
#TODO - better handling
self.logger.debug("Photon build container image not found.")
# Create toolchain chroot and install toolchain RPMs
chrootID = None
try:
#TODO: constants.tcrootname
chrUtils = ChrootUtils("toolchain-chroot", self.logPath)
returnVal, chrootID = chrUtils.createChroot("toolchain-chroot")
self.logger.debug("Created tool-chain chroot: " + chrootID)
if not returnVal:
raise Exception("Unable to prepare tool-chain chroot")
tcUtils = ToolChainUtils("toolchain-chroot", self.logPath)
tcUtils.installToolChainRPMS(chrootID, "dummy")
except Exception as e:
if chrootID is not None:
self.logger.debug("Deleting chroot: " + chrootID)
chrUtils.destroyChroot(chrootID)
raise e
self.logger.info("VDBG-PU-createBuildContainer: chrootID: " + chrootID)
# Create photon build container using toolchain chroot
#TODO: Coalesce logging
cmdUtils = CommandUtils()
cmd = "./umount-build-root.sh " + chrootID
cmdUtils.runCommandInShell(cmd, self.logPath + "/toolchain-chroot1.log")
cmd = "cd " + chrootID + " && tar -czvf ../tcroot.tar.gz ."
cmdUtils.runCommandInShell(cmd, self.logPath + "/toolchain-chroot2.log")
cmd = "mv " + chrootID + "/../tcroot.tar.gz ."
cmdUtils.runCommandInShell(cmd, self.logPath + "/toolchain-chroot3.log")
#TODO: Container name, docker file name from constants.
self.dockerClient.images.build(tag="photon_build_container:latest",
path=".",
rm=True,
dockerfile="Dockerfile.photon_build_container")
# Cleanup
cmd = "rm -f ./tcroot.tar.gz"
cmdUtils.runCommandInShell(cmd, self.logPath + "/toolchain-chroot4.log")
chrUtils.destroyChroot(chrootID)
self.logger.info("Photon build container successfully created.")
def buildCoreToolChainPackages(self, listBuildOptionPackages,
pkgBuildOptionFile):
self.logger.info("Building core toolchain packages.....")
chrootID = None
pkgCount = 0
try:
pkgUtils = PackageUtils(self.logName, self.logPath)
for package in constants.listCoreToolChainPackages:
rpmPkg = pkgUtils.findRPMFileForGivenPackage(package)
if rpmPkg is not None:
continue
self.logger.info("Building core toolchain package: " + package)
chrUtils = ChrootUtils(self.logName, self.logPath)
chrootName = "build-" + package
destLogPath = constants.logPath + "/build-" + package
if not os.path.isdir(destLogPath):
cmdUtils = CommandUtils()
cmdUtils.runCommandInShell("mkdir -p " + destLogPath)
returnVal, chrootID = chrUtils.createChroot(chrootName)
if not returnVal:
self.logger.error("Creating chroot failed")
raise Exception("creating chroot failed")
self.installToolChainRPMS(chrootID, package, destLogPath)
pkgUtils.adjustGCCSpecs(package, chrootID, destLogPath)
pkgUtils.buildRPMSForGivenPackage(package, chrootID,
listBuildOptionPackages,
pkgBuildOptionFile,
destLogPath)
pkgCount += 1
chrUtils.destroyChroot(chrootID)
chrootID = None
self.logger.info("Successfully built toolchain")
if chrootID is not None:
chrUtils.destroyChroot(chrootID)
except Exception as e:
self.logger.error("Unable to build tool chain.")
# print stacktrace
traceback.print_exc()
raise e
return pkgCount
def buildPackage(self):
#do not build if RPM is already built
#test only if the package is in the testForceRPMS with rpmCheck
#build only if the package is not in the testForceRPMS with rpmCheck
if self.base.checkIfPackageIsAlreadyBuilt():
if not constants.rpmCheck:
self.base.logger.info("Skipping building the package:" +
self.base.package)
return
elif constants.rpmCheck and self.base.package not in constants.testForceRPMS:
self.base.logger.info("Skipping testing the package:" +
self.base.package)
return
chrUtils = ChrootUtils(self.base.logName, self.base.logPath)
chrootID = None
try:
chrootID = self.prepareBuildRoot()
listInstalledPackages, listInstalledRPMs = self.base.findInstalledPackages(
chrootID)
listDependentPackages = self.base.findBuildTimeRequiredPackages()
if constants.rpmCheck and self.base.package in constants.testForceRPMS:
listDependentPackages.extend(
self.base.findBuildTimeCheckRequiredPackages())
testPackages = set(
constants.listMakeCheckRPMPkgtoInstall) - set(
listInstalledPackages) - set([self.base.package])
listDependentPackages.extend(testPackages)
listDependentPackages = list(set(listDependentPackages))
pkgUtils = PackageUtils(self.base.logName, self.base.logPath)
if len(listDependentPackages) != 0:
self.base.logger.info(
"Installing the build time dependent packages......")
for pkg in listDependentPackages:
self.base.installPackage(pkgUtils, pkg, chrootID,
self.base.logPath,
listInstalledPackages,
listInstalledRPMs)
pkgUtils.installRPMSInAOneShot(chrootID, self.base.logPath)
self.base.logger.info(
"Finished installing the build time dependent packages......"
)
pkgUtils.adjustGCCSpecs(self.base.package, chrootID,
self.base.logPath)
pkgUtils.buildRPMSForGivenPackage(
self.base.package, chrootID, self.base.listBuildOptionPackages,
self.base.pkgBuildOptionFile, self.base.logPath)
self.base.logger.info("Successfully built the package:" +
self.base.package)
except Exception as e:
self.base.logger.error("Failed while building package:" +
self.base.package)
self.base.logger.debug("Chroot with ID: " + chrootID +
" not deleted for debugging.")
logFileName = os.path.join(self.base.logPath,
self.base.package + ".log")
fileLog = os.popen('tail -n 100 ' + logFileName).read()
self.base.logger.debug(fileLog)
raise e
if chrootID is not None:
chrUtils.destroyChroot(chrootID)
def buildPackage(self):
#do not build if RPM is already built
#test only if the package is in the testForceRPMS with rpmCheck
#build only if the package is not in the testForceRPMS with rpmCheck
if self.base.checkIfPackageIsAlreadyBuilt():
if not constants.rpmCheck:
self.base.logger.info("Skipping building the package:" +
self.base.package)
return
elif constants.rpmCheck and self.base.package not in constants.testForceRPMS:
self.base.logger.info("Skipping testing the package:" +
self.base.package)
return
#should initialize a logger based on package name
containerTaskName = "build-" + self.base.package
containerID = None
chrootID = None
isToolChainPackage = False
if self.base.package in constants.listToolChainPackages:
isToolChainPackage = True
destLogPath = constants.logPath + "/build-" + self.base.package
try:
containerID, chrootID = self.prepareBuildContainer(
containerTaskName, self.base.package, isToolChainPackage)
tcUtils = ToolChainUtils(self.base.logName, self.base.logPath)
if self.base.package in constants.perPackageToolChain:
self.base.logger.debug(
constants.perPackageToolChain[self.base.package])
tcUtils.installCustomToolChainRPMSinContainer(
containerID,
constants.perPackageToolChain[self.base.package],
self.base.package)
listInstalledPackages, listInstalledRPMs = self.base.findInstalledPackages(
containerID)
self.base.logger.info(listInstalledPackages)
listDependentPackages = self.base.findBuildTimeRequiredPackages()
if constants.rpmCheck and self.base.package in constants.testForceRPMS:
listDependentPackages.extend(
self.base.findBuildTimeCheckRequiredPackages())
testPackages = set(
constants.listMakeCheckRPMPkgtoInstall) - set(
listInstalledPackages) - set([self.base.package])
listDependentPackages.extend(testPackages)
listDependentPackages = list(set(listDependentPackages))
pkgUtils = PackageUtils(self.base.logName, self.base.logPath)
if len(listDependentPackages) != 0:
self.base.logger.info(
"BuildContainer-buildPackage: Installing dependent packages.."
)
self.base.logger.info(listDependentPackages)
for pkg in listDependentPackages:
self.base.installPackage(pkgUtils, pkg, containerID,
destLogPath,
listInstalledPackages,
listInstalledRPMs)
pkgUtils.installRPMSInAOneShotInContainer(
containerID, destLogPath)
self.base.logger.info(
"Finished installing the build time dependent packages......"
)
self.base.logger.info(
"BuildContainer-buildPackage: Start building the package: " +
self.base.package)
pkgUtils.adjustGCCSpecsInContainer(self.base.package, containerID,
destLogPath)
pkgUtils.buildRPMSForGivenPackageInContainer(
self.base.package, containerID,
self.base.listBuildOptionPackages,
self.base.pkgBuildOptionFile, destLogPath)
self.base.logger.info(
"BuildContainer-buildPackage: Successfully built the package: "
+ self.base.package)
except Exception as e:
self.base.logger.error("Failed while building package:" +
self.base.package)
if containerID is not None:
self.base.logger.debug("Container " + containerID.short_id +
" retained for debugging.")
logFileName = os.path.join(destLogPath, self.base.package + ".log")
fileLog = os.popen('tail -n 20 ' + logFileName).read()
self.base.logger.debug(fileLog)
raise e
# Remove the container
if containerID is not None:
containerID.remove(force=True)
# Remove the dummy chroot
if chrootID is not None:
chrUtils = ChrootUtils(self.base.logName, self.base.logPath)
chrUtils.destroyChroot(chrootID)
def prepareBuildContainer(self,
containerTaskName,
packageName,
isToolChainPackage=False):
# Prepare an empty chroot environment to let docker use the BUILD folder.
# This avoids docker using overlayFS which will cause make check failure.
chrootName = "build-" + packageName
chrUtils = ChrootUtils(self.base.logName, self.base.logPath)
returnVal, chrootID = chrUtils.createChroot(chrootName)
if not returnVal:
raise Exception("Unable to prepare build root")
cmdUtils = CommandUtils()
cmdUtils.runCommandInShell("mkdir -p " + chrootID +
constants.topDirPath)
cmdUtils.runCommandInShell("mkdir -p " + chrootID +
constants.topDirPath + "/BUILD")
containerID = None
mountVols = {
constants.prevPublishRPMRepo: {
'bind': '/publishrpms',
'mode': 'ro'
},
constants.prevPublishXRPMRepo: {
'bind': '/publishxrpms',
'mode': 'ro'
},
constants.tmpDirPath: {
'bind': '/tmp',
'mode': 'rw'
},
constants.rpmPath: {
'bind': constants.topDirPath + "/RPMS",
'mode': 'rw'
},
constants.sourceRpmPath: {
'bind': constants.topDirPath + "/SRPMS",
'mode': 'rw'
},
constants.logPath + "/" + self.base.logName: {
'bind': constants.topDirPath + "/LOGS",
'mode': 'rw'
},
chrootID + constants.topDirPath + "/BUILD": {
'bind': constants.topDirPath + "/BUILD",
'mode': 'rw'
},
constants.dockerUnixSocket: {
'bind': constants.dockerUnixSocket,
'mode': 'rw'
}
}
containerName = containerTaskName
containerName = containerName.replace("+", "p")
try:
oldContainerID = self.dockerClient.containers.get(containerName)
if oldContainerID is not None:
oldContainerID.remove(force=True)
except docker.errors.NotFound:
sys.exc_clear()
try:
self.base.logger.info(
"BuildContainer-prepareBuildContainer: Starting build container: "
+ containerName)
#TODO: Is init=True equivalent of --sig-proxy?
privilegedDocker = False
cap_list = ['SYS_PTRACE']
if packageName in constants.listReqPrivilegedDockerForTest:
privilegedDocker = True
containerID = self.dockerClient.containers.run(
self.buildContainerImage,
detach=True,
cap_add=cap_list,
privileged=privilegedDocker,
name=containerName,
network_mode="host",
volumes=mountVols,
command="/bin/bash -l -c /wait.sh")
self.base.logger.debug("Started Photon build container for task " +
containerTaskName + " ID: " +
containerID.short_id)
if not containerID:
raise Exception(
"Unable to start Photon build container for task " +
containerTaskName)
except Exception as e:
self.base.logger.debug(
"Unable to start Photon build container for task " +
containerTaskName)
raise e
return containerID, chrootID
def _buildPackage(self, index=0):
#do not build if RPM is already built
#test only if the package is in the testForceRPMS with rpmCheck
#build only if the package is not in the testForceRPMS with rpmCheck
if self._checkIfPackageIsAlreadyBuilt(index):
if not constants.rpmCheck:
self.logger.info("Skipping building the package:" +
self.package)
return
elif constants.rpmCheck and self.package not in constants.testForceRPMS:
self.logger.info("Skipping testing the package:" +
self.package)
return
#should initialize a logger based on package name
containerTaskName = "build-" + self.package
containerID = None
chrootID = None
isToolChainPackage = False
if self.package in constants.listToolChainPackages:
isToolChainPackage = True
destLogPath = constants.logPath + "/build-" + self.package
try:
containerID, chrootID = self._prepareBuildContainer(
containerTaskName, self.package, isToolChainPackage)
tcUtils = ToolChainUtils(self.logName, self.logPath)
if self.package in constants.perPackageToolChain:
self.logger.debug(constants.perPackageToolChain[self.package])
tcUtils.installCustomToolChainRPMSinContainer(
containerID, constants.perPackageToolChain[self.package],
self.package)
listDependentPackages, listDependentPackagesParseObj, listInstalledPackages, listInstalledRPMs = (
self._findDependentPackagesAndInstalledRPM(containerID, index))
pkgUtils = PackageUtils(self.logName, self.logPath)
if listDependentPackages:
self.logger.info("BuildContainer-buildPackage: " +
"Installing dependent packages..")
self.logger.info(listDependentPackages)
for pkg in listDependentPackages:
flag = False
for objName in listDependentPackagesParseObj:
if objName.package == pkg:
properVersion = self._getProperVersion(
pkg, objName)
self._installPackage(pkgUtils, pkg, properVersion,
containerID, destLogPath,
listInstalledPackages,
listInstalledRPMs)
flag = True
break
if flag == False:
self._installPackage(pkgUtils, pkg, "*", containerID,
destLogPath,
listInstalledPackages,
listInstalledRPMs)
pkgUtils.installRPMSInAOneShotInContainer(
containerID, destLogPath)
self.logger.info(
"Finished installing the build time dependent packages...."
)
self.logger.info(
"BuildContainer-buildPackage: Start building the package: " +
self.package)
pkgUtils.adjustGCCSpecsInContainer(self.package, containerID,
destLogPath, index)
pkgUtils.buildRPMSForGivenPackageInContainer(
self.package, containerID, destLogPath, index)
self.logger.info(
"BuildContainer-buildPackage: Successfully built the package: "
+ self.package)
except Exception as e:
self.logger.error("Failed while building package:" + self.package)
if containerID is not None:
self.logger.debug("Container " + containerID.short_id +
" retained for debugging.")
logFileName = os.path.join(destLogPath, self.package + ".log")
fileLog = os.popen('tail -n 20 ' + logFileName).read()
self.logger.debug(fileLog)
raise e
# Remove the container
if containerID is not None:
containerID.remove(force=True)
# Remove the dummy chroot
if chrootID is not None:
chrUtils = ChrootUtils(self.logName, self.logPath)
chrUtils.destroyChroot(chrootID)
def _buildPackage(self):
#should initialize a logger based on package name
containerTaskName = "build-" + self.package
containerID = None
chrootID = None
isToolChainPackage = False
if self.package in constants.listToolChainPackages:
isToolChainPackage = True
destLogPath = constants.logPath + "/build-" + self.package
try:
containerID, chrootID = self._prepareBuildContainer(
containerTaskName, self.package, self.version,
isToolChainPackage)
tcUtils = ToolChainUtils(self.logName, self.logPath)
if self.package in constants.perPackageToolChain:
self.logger.debug(constants.perPackageToolChain[self.package])
tcUtils.installCustomToolChainRPMSinContainer(
containerID,
constants.perPackageToolChain[self.package].get(
platform.machine(), []), self.package)
listDependentPackages, listTestPackages, listInstalledPackages, listInstalledRPMs = (
self._findDependentPackagesAndInstalledRPM(containerID))
pkgUtils = PackageUtils(self.logName, self.logPath)
if listDependentPackages:
self.logger.info("BuildContainer-buildPackage: " +
"Installing dependent packages..")
self.logger.info(listDependentPackages)
for pkg in listDependentPackages:
packageName, packageVersion = StringUtils.splitPackageNameAndVersion(
pkg)
self._installPackage(pkgUtils, packageName, packageVersion,
containerID, destLogPath,
listInstalledPackages,
listInstalledRPMs)
for pkg in listTestPackages:
packageName, packageVersion = StringUtils.splitPackageNameAndVersion(
pkg)
flag = False
for depPkg in listDependentPackages:
depPackageName, depPackageVersion = StringUtils.splitPackageNameAndVersion(
depPkg)
if depPackageName == packageName:
flag = True
break
if flag == False:
self._installPackage(pkgUtils, packageName,
packageVersion, containerID,
destLogPath,
listInstalledPackages,
listInstalledRPMs)
pkgUtils.installRPMSInAOneShotInContainer(
containerID, destLogPath)
self.logger.info(
"Finished installing the build time dependent packages...."
)
self.logger.info(
"BuildContainer-buildPackage: Start building the package: " +
self.package)
pkgUtils.adjustGCCSpecsInContainer(self.package, self.version,
containerID, destLogPath)
pkgUtils.buildRPMSForGivenPackageInContainer(
self.package, self.version, containerID, destLogPath)
self.logger.info(
"BuildContainer-buildPackage: Successfully built the package: "
+ self.package)
except Exception as e:
self.logger.error("Failed while building package:" + self.package)
if containerID is not None:
self.logger.debug("Container " + containerID.short_id +
" retained for debugging.")
logFileName = os.path.join(destLogPath, self.package + ".log")
fileLog = os.popen('tail -n 20 ' + logFileName).read()
self.logger.debug(fileLog)
raise e
# Remove the container
if containerID is not None:
containerID.remove(force=True)
# Remove the dummy chroot
if chrootID is not None:
chrUtils = ChrootUtils(self.logName, self.logPath)
chrUtils.destroyChroot(chrootID)
def _buildPackage(self):
#do not build if RPM is already built
#test only if the package is in the testForceRPMS with rpmCheck
#build only if the package is not in the testForceRPMS with rpmCheck
if self._checkIfPackageIsAlreadyBuilt():
if not constants.rpmCheck:
self.logger.info("Skipping building the package:" + self.package)
return
elif constants.rpmCheck and self.package not in constants.testForceRPMS:
self.logger.info("Skipping testing the package:" + self.package)
return
#should initialize a logger based on package name
containerTaskName = "build-" + self.package
containerID = None
chrootID = None
isToolChainPackage = False
if self.package in constants.listToolChainPackages:
isToolChainPackage = True
destLogPath = constants.logPath + "/build-" + self.package
try:
containerID, chrootID = self._prepareBuildContainer(
containerTaskName, self.package, isToolChainPackage)
tcUtils = ToolChainUtils(self.logName, self.logPath)
if self.package in constants.perPackageToolChain:
self.logger.debug(constants.perPackageToolChain[self.package])
tcUtils.installCustomToolChainRPMSinContainer(
containerID,
constants.perPackageToolChain[self.package],
self.package)
listDependentPackages, listInstalledPackages, listInstalledRPMs = (
self._findDependentPackagesAndInstalledRPM(containerID))
pkgUtils = PackageUtils(self.logName, self.logPath)
if listDependentPackages:
self.logger.info("BuildContainer-buildPackage: " +
"Installing dependent packages..")
self.logger.info(listDependentPackages)
for pkg in listDependentPackages:
self._installPackage(pkgUtils, pkg, containerID, destLogPath,
listInstalledPackages, listInstalledRPMs)
pkgUtils.installRPMSInAOneShotInContainer(containerID, destLogPath)
self.logger.info("Finished installing the build time dependent packages....")
self.logger.info("BuildContainer-buildPackage: Start building the package: " +
self.package)
pkgUtils.adjustGCCSpecsInContainer(self.package, containerID, destLogPath)
pkgUtils.buildRPMSForGivenPackageInContainer(
self.package,
containerID,
destLogPath)
self.logger.info("BuildContainer-buildPackage: Successfully built the package: " +
self.package)
except Exception as e:
self.logger.error("Failed while building package:" + self.package)
if containerID is not None:
self.logger.debug("Container " + containerID.short_id +
" retained for debugging.")
logFileName = os.path.join(destLogPath, self.package + ".log")
fileLog = os.popen('tail -n 20 ' + logFileName).read()
self.logger.debug(fileLog)
raise e
# Remove the container
if containerID is not None:
containerID.remove(force=True)
# Remove the dummy chroot
if chrootID is not None:
chrUtils = ChrootUtils(self.logName, self.logPath)
chrUtils.destroyChroot(chrootID)
def _prepareBuildContainer(self, containerTaskName, packageName,
isToolChainPackage=False):
# Prepare an empty chroot environment to let docker use the BUILD folder.
# This avoids docker using overlayFS which will cause make check failure.
chrootName = "build-" + packageName
chrUtils = ChrootUtils(self.logName, self.logPath)
returnVal, chrootID = chrUtils.createChroot(chrootName)
if not returnVal:
raise Exception("Unable to prepare build root")
cmdUtils = CommandUtils()
cmdUtils.runCommandInShell("mkdir -p " + chrootID + constants.topDirPath)
cmdUtils.runCommandInShell("mkdir -p " + chrootID + constants.topDirPath + "/BUILD")
containerID = None
mountVols = {
constants.prevPublishRPMRepo: {'bind': '/publishrpms', 'mode': 'ro'},
constants.prevPublishXRPMRepo: {'bind': '/publishxrpms', 'mode': 'ro'},
constants.tmpDirPath: {'bind': '/tmp', 'mode': 'rw'},
constants.rpmPath: {'bind': constants.topDirPath + "/RPMS", 'mode': 'rw'},
constants.sourceRpmPath: {'bind': constants.topDirPath + "/SRPMS", 'mode': 'rw'},
constants.logPath + "/" + self.logName: {'bind': constants.topDirPath + "/LOGS",
'mode': 'rw'},
chrootID + constants.topDirPath + "/BUILD": {'bind': constants.topDirPath + "/BUILD",
'mode': 'rw'},
constants.dockerUnixSocket: {'bind': constants.dockerUnixSocket, 'mode': 'rw'}
}
containerName = containerTaskName
containerName = containerName.replace("+", "p")
try:
oldContainerID = self.dockerClient.containers.get(containerName)
if oldContainerID is not None:
oldContainerID.remove(force=True)
except docker.errors.NotFound:
try:
sys.exc_clear()
except:
pass
try:
self.logger.info("BuildContainer-prepareBuildContainer: " +
"Starting build container: " + containerName)
#TODO: Is init=True equivalent of --sig-proxy?
privilegedDocker = False
cap_list = ['SYS_PTRACE']
if packageName in constants.listReqPrivilegedDockerForTest:
privilegedDocker = True
containerID = self.dockerClient.containers.run(self.buildContainerImage,
detach=True,
cap_add=cap_list,
privileged=privilegedDocker,
name=containerName,
network_mode="host",
volumes=mountVols,
command="/bin/bash -l -c /wait.sh")
self.logger.debug("Started Photon build container for task " + containerTaskName +
" ID: " + containerID.short_id)
if not containerID:
raise Exception("Unable to start Photon build container for task " +
containerTaskName)
except Exception as e:
self.logger.debug("Unable to start Photon build container for task " +
containerTaskName)
raise e
return containerID, chrootID
