def buildCoreToolChainPackages(self): self.logger.info("Building core tool chain packages.....") chrootID = None try: pkgUtils = PackageUtils(self.logName, self.logPath) for package in constants.listCoreToolChainRPMPackages: rpmPkg = pkgUtils.findRPMFileForGivenPackage(package) if rpmPkg is not None: continue chrUtils = ChrootUtils(self.logName, self.logPath) chrootName = "build-core-toolchain" destLogPath = constants.logPath + "/build-" + package if not os.path.isdir(destLogPath): cmdUtils = CommandUtils() cmdUtils.runCommandInShell("mkdir -p " + destLogPath) returnVal, chrootID = chrUtils.createChroot(chrootName) if not returnVal: self.logger.error("Creating chroot failed") raise Exception("creating chroot failed") self.installToolChainRPMS(chrootID) pkgUtils.buildRPMSForGivenPackage(package, chrootID, destLogPath) chrUtils.destroyChroot(chrootID) chrootID = None self.logger.info("Successfully built toolchain") except Exception as e: self.logger.error("Unable to build tool chain.") raise e finally: if chrootID is not None: chrUtils.destroyChroot(chrootID)
def buildCoreToolChainPackages(self): self.logger.info("Building core tool chain packages.....") chrootID=None try: pkgUtils=PackageUtils(self.logName,self.logPath) for package in constants.listCoreToolChainRPMPackages: rpmPkg=pkgUtils.findRPMFileForGivenPackage(package) if rpmPkg is not None: continue chrUtils = ChrootUtils(self.logName,self.logPath) chrootName="build-core-toolchain" destLogPath=constants.logPath+"/build-"+package if not os.path.isdir(destLogPath): cmdUtils = CommandUtils() cmdUtils.runCommandInShell("mkdir -p "+destLogPath) returnVal,chrootID = chrUtils.createChroot(chrootName) if not returnVal: self.logger.error("Creating chroot failed") raise Exception("creating chroot failed") self.installToolChainRPMS(chrootID) pkgUtils.adjustGCCSpecs(package, chrootID, destLogPath) pkgUtils.buildRPMSForGivenPackage(package, chrootID,destLogPath) chrUtils.destroyChroot(chrootID) chrootID=None self.logger.info("Successfully built toolchain") if chrootID is not None: chrUtils.destroyChroot(chrootID) except Exception as e: self.logger.error("Unable to build tool chain.") # print stacktrace traceback.print_exc() raise e
def buildCoreToolChainPackages(self): self.logger.info("Step 1 : Building the core toolchain packages.....") self.logger.info(constants.listCoreToolChainPackages) self.logger.info("") chrootID = None pkgCount = 0 try: pkgUtils = PackageUtils(self.logName, self.logPath) coreToolChainYetToBuild = [] for package in constants.listCoreToolChainPackages: version = SPECS.getData().getHighestVersion(package) rpmPkg = pkgUtils.findRPMFileForGivenPackage(package, version) if rpmPkg is not None: continue else: coreToolChainYetToBuild.append(package) if coreToolChainYetToBuild: self.logger.info( "The following core toolchain packages need to be built :") self.logger.info(coreToolChainYetToBuild) else: self.logger.info( "Core toolchain packages are already available") for package in coreToolChainYetToBuild: version = SPECS.getData().getHighestVersion(package) self.logger.debug("Building core toolchain package : " + package) chrUtils = ChrootUtils(self.logName, self.logPath) chrootName = "build-" + package destLogPath = constants.logPath + "/build-" + package if not os.path.isdir(destLogPath): cmdUtils = CommandUtils() cmdUtils.runCommandInShell("mkdir -p " + destLogPath) returnVal, chrootID = chrUtils.createChroot(chrootName) if not returnVal: self.logger.error("Creating chroot failed") raise Exception("creating chroot failed") self.installToolChainRPMS(package, version, chrootID, destLogPath) pkgUtils.adjustGCCSpecs(package, version, chrootID, destLogPath) pkgUtils.buildRPMSForGivenPackage(package, version, chrootID, destLogPath) pkgCount += 1 chrUtils.destroyChroot(chrootID) chrootID = None self.logger.debug("Successfully built toolchain") self.logger.info("-" * 45 + "\n") if chrootID is not None: chrUtils.destroyChroot(chrootID) except Exception as e: self.logger.error("Unable to build tool chain.") # print stacktrace traceback.print_exc() raise e return pkgCount
def createBuildContainer(self): self.logger.info("Generating photon build container..") try: #TODO image name constants.buildContainerImageName self.dockerClient.images.remove("photon_build_container:latest", force=True) except Exception as e: #TODO - better handling self.logger.debug("Photon build container image not found.") # Create toolchain chroot and install toolchain RPMs chrootID = None try: #TODO: constants.tcrootname chrUtils = ChrootUtils("toolchain-chroot", self.logPath) returnVal, chrootID = chrUtils.createChroot("toolchain-chroot") self.logger.debug("Created tool-chain chroot: " + chrootID) if not returnVal: raise Exception("Unable to prepare tool-chain chroot") tcUtils = ToolChainUtils("toolchain-chroot", self.logPath) tcUtils.installToolChainRPMS(chrootID, "dummy") except Exception as e: if chrootID is not None: self.logger.debug("Deleting chroot: " + chrootID) chrUtils.destroyChroot(chrootID) raise e self.logger.info("VDBG-PU-createBuildContainer: chrootID: " + chrootID) # Create photon build container using toolchain chroot #TODO: Coalesce logging cmdUtils = CommandUtils() cmd = "./umount-build-root.sh " + chrootID cmdUtils.runCommandInShell(cmd, self.logPath + "/toolchain-chroot1.log") cmd = "cd " + chrootID + " && tar -czvf ../tcroot.tar.gz ." cmdUtils.runCommandInShell(cmd, self.logPath + "/toolchain-chroot2.log") cmd = "mv " + chrootID + "/../tcroot.tar.gz ." cmdUtils.runCommandInShell(cmd, self.logPath + "/toolchain-chroot3.log") #TODO: Container name, docker file name from constants. self.dockerClient.images.build( tag="photon_build_container:latest", path=".", rm=True, dockerfile="Dockerfile.photon_build_container") # Cleanup cmd = "rm -f ./tcroot.tar.gz" cmdUtils.runCommandInShell(cmd, self.logPath + "/toolchain-chroot4.log") chrUtils.destroyChroot(chrootID) self.logger.info("Photon build container successfully created.")
def prepareBuildRoot(self, chrootName, packageName): chrootID = None try: chrUtils = ChrootUtils(self.logName, self.logPath) returnVal, chrootID = chrUtils.createChroot(chrootName) self.logger.debug("Created new chroot: " + chrootID) if not returnVal: raise Exception("Unable to prepare build root") tUtils = ToolChainUtils(self.logName, self.logPath) tUtils.installToolChainRPMS(chrootID, packageName) except Exception as e: if chrootID is not None: self.logger.debug("Deleting chroot: " + chrootID) chrUtils.destroyChroot(chrootID) raise e return chrootID
def prepareBuildRoot(self,chrootName, packageName): chrootID=None try: chrUtils = ChrootUtils(self.logName,self.logPath) returnVal,chrootID = chrUtils.createChroot(chrootName) self.logger.debug("Created new chroot: " + chrootID) if not returnVal: raise Exception("Unable to prepare build root") tUtils=ToolChainUtils(self.logName,self.logPath) tUtils.installToolChainRPMS(chrootID, packageName) except Exception as e: if chrootID is not None: self.logger.debug("Deleting chroot: " + chrootID) chrUtils.destroyChroot(chrootID) raise e return chrootID
def _createBuildContainer(self): self.logger.info("Generating photon build container..") try: #TODO image name constants.buildContainerImageName self.dockerClient.images.remove("photon_build_container:latest", force=True) except Exception as e: #TODO - better handling self.logger.debug("Photon build container image not found.") # Create toolchain chroot and install toolchain RPMs chrootID = None try: #TODO: constants.tcrootname chrUtils = ChrootUtils("toolchain-chroot", self.logPath) returnVal, chrootID = chrUtils.createChroot("toolchain-chroot") self.logger.debug("Created tool-chain chroot: " + chrootID) if not returnVal: raise Exception("Unable to prepare tool-chain chroot") tcUtils = ToolChainUtils("toolchain-chroot", self.logPath) tcUtils.installToolChainRPMS(chrootID, "dummy") except Exception as e: if chrootID is not None: self.logger.debug("Deleting chroot: " + chrootID) chrUtils.destroyChroot(chrootID) raise e self.logger.info("VDBG-PU-createBuildContainer: chrootID: " + chrootID) # Create photon build container using toolchain chroot #TODO: Coalesce logging cmdUtils = CommandUtils() cmd = "./umount-build-root.sh " + chrootID cmdUtils.runCommandInShell(cmd, self.logPath + "/toolchain-chroot1.log") cmd = "cd " + chrootID + " && tar -czvf ../tcroot.tar.gz ." cmdUtils.runCommandInShell(cmd, self.logPath + "/toolchain-chroot2.log") cmd = "mv " + chrootID + "/../tcroot.tar.gz ." cmdUtils.runCommandInShell(cmd, self.logPath + "/toolchain-chroot3.log") #TODO: Container name, docker file name from constants. self.dockerClient.images.build(tag="photon_build_container:latest", path=".", rm=True, dockerfile="Dockerfile.photon_build_container") # Cleanup cmd = "rm -f ./tcroot.tar.gz" cmdUtils.runCommandInShell(cmd, self.logPath + "/toolchain-chroot4.log") chrUtils.destroyChroot(chrootID) self.logger.info("Photon build container successfully created.")
def buildCoreToolChainPackages(self, listBuildOptionPackages, pkgBuildOptionFile): self.logger.info("Building core toolchain packages.....") chrootID = None pkgCount = 0 try: pkgUtils = PackageUtils(self.logName, self.logPath) for package in constants.listCoreToolChainPackages: rpmPkg = pkgUtils.findRPMFileForGivenPackage(package) if rpmPkg is not None: continue self.logger.info("Building core toolchain package: " + package) chrUtils = ChrootUtils(self.logName, self.logPath) chrootName = "build-" + package destLogPath = constants.logPath + "/build-" + package if not os.path.isdir(destLogPath): cmdUtils = CommandUtils() cmdUtils.runCommandInShell("mkdir -p " + destLogPath) returnVal, chrootID = chrUtils.createChroot(chrootName) if not returnVal: self.logger.error("Creating chroot failed") raise Exception("creating chroot failed") self.installToolChainRPMS(chrootID, package, destLogPath) pkgUtils.adjustGCCSpecs(package, chrootID, destLogPath) pkgUtils.buildRPMSForGivenPackage(package, chrootID, listBuildOptionPackages, pkgBuildOptionFile, destLogPath) pkgCount += 1 chrUtils.destroyChroot(chrootID) chrootID = None self.logger.info("Successfully built toolchain") if chrootID is not None: chrUtils.destroyChroot(chrootID) except Exception as e: self.logger.error("Unable to build tool chain.") # print stacktrace traceback.print_exc() raise e return pkgCount
def prepareBuildContainer(self, containerTaskName, packageName, isToolChainPackage=False): # Prepare an empty chroot environment to let docker use the BUILD folder. # This avoids docker using overlayFS which will cause make check failure. chrootName = "build-" + packageName chrUtils = ChrootUtils(self.base.logName, self.base.logPath) returnVal, chrootID = chrUtils.createChroot(chrootName) if not returnVal: raise Exception("Unable to prepare build root") cmdUtils = CommandUtils() cmdUtils.runCommandInShell("mkdir -p " + chrootID + constants.topDirPath) cmdUtils.runCommandInShell("mkdir -p " + chrootID + constants.topDirPath + "/BUILD") containerID = None mountVols = { constants.prevPublishRPMRepo: { 'bind': '/publishrpms', 'mode': 'ro' }, constants.prevPublishXRPMRepo: { 'bind': '/publishxrpms', 'mode': 'ro' }, constants.tmpDirPath: { 'bind': '/tmp', 'mode': 'rw' }, constants.rpmPath: { 'bind': constants.topDirPath + "/RPMS", 'mode': 'rw' }, constants.sourceRpmPath: { 'bind': constants.topDirPath + "/SRPMS", 'mode': 'rw' }, constants.logPath + "/" + self.base.logName: { 'bind': constants.topDirPath + "/LOGS", 'mode': 'rw' }, chrootID + constants.topDirPath + "/BUILD": { 'bind': constants.topDirPath + "/BUILD", 'mode': 'rw' }, constants.dockerUnixSocket: { 'bind': constants.dockerUnixSocket, 'mode': 'rw' } } containerName = containerTaskName containerName = containerName.replace("+", "p") try: oldContainerID = self.dockerClient.containers.get(containerName) if oldContainerID is not None: oldContainerID.remove(force=True) except docker.errors.NotFound: sys.exc_clear() try: self.base.logger.info( "BuildContainer-prepareBuildContainer: Starting build container: " + containerName) #TODO: Is init=True equivalent of --sig-proxy? privilegedDocker = False cap_list = ['SYS_PTRACE'] if packageName in constants.listReqPrivilegedDockerForTest: privilegedDocker = True containerID = self.dockerClient.containers.run( self.buildContainerImage, detach=True, cap_add=cap_list, privileged=privilegedDocker, name=containerName, network_mode="host", volumes=mountVols, command="/bin/bash -l -c /wait.sh") self.base.logger.debug("Started Photon build container for task " + containerTaskName + " ID: " + containerID.short_id) if not containerID: raise Exception( "Unable to start Photon build container for task " + containerTaskName) except Exception as e: self.base.logger.debug( "Unable to start Photon build container for task " + containerTaskName) raise e return containerID, chrootID
def _prepareBuildContainer(self, containerTaskName, packageName, isToolChainPackage=False): # Prepare an empty chroot environment to let docker use the BUILD folder. # This avoids docker using overlayFS which will cause make check failure. chrootName = "build-" + packageName chrUtils = ChrootUtils(self.logName, self.logPath) returnVal, chrootID = chrUtils.createChroot(chrootName) if not returnVal: raise Exception("Unable to prepare build root") cmdUtils = CommandUtils() cmdUtils.runCommandInShell("mkdir -p " + chrootID + constants.topDirPath) cmdUtils.runCommandInShell("mkdir -p " + chrootID + constants.topDirPath + "/BUILD") containerID = None mountVols = { constants.prevPublishRPMRepo: {'bind': '/publishrpms', 'mode': 'ro'}, constants.prevPublishXRPMRepo: {'bind': '/publishxrpms', 'mode': 'ro'}, constants.tmpDirPath: {'bind': '/tmp', 'mode': 'rw'}, constants.rpmPath: {'bind': constants.topDirPath + "/RPMS", 'mode': 'rw'}, constants.sourceRpmPath: {'bind': constants.topDirPath + "/SRPMS", 'mode': 'rw'}, constants.logPath + "/" + self.logName: {'bind': constants.topDirPath + "/LOGS", 'mode': 'rw'}, chrootID + constants.topDirPath + "/BUILD": {'bind': constants.topDirPath + "/BUILD", 'mode': 'rw'}, constants.dockerUnixSocket: {'bind': constants.dockerUnixSocket, 'mode': 'rw'} } containerName = containerTaskName containerName = containerName.replace("+", "p") try: oldContainerID = self.dockerClient.containers.get(containerName) if oldContainerID is not None: oldContainerID.remove(force=True) except docker.errors.NotFound: try: sys.exc_clear() except: pass try: self.logger.info("BuildContainer-prepareBuildContainer: " + "Starting build container: " + containerName) #TODO: Is init=True equivalent of --sig-proxy? privilegedDocker = False cap_list = ['SYS_PTRACE'] if packageName in constants.listReqPrivilegedDockerForTest: privilegedDocker = True containerID = self.dockerClient.containers.run(self.buildContainerImage, detach=True, cap_add=cap_list, privileged=privilegedDocker, name=containerName, network_mode="host", volumes=mountVols, command="/bin/bash -l -c /wait.sh") self.logger.debug("Started Photon build container for task " + containerTaskName + " ID: " + containerID.short_id) if not containerID: raise Exception("Unable to start Photon build container for task " + containerTaskName) except Exception as e: self.logger.debug("Unable to start Photon build container for task " + containerTaskName) raise e return containerID, chrootID