def buildToolChain(self): self.logger.info("Step 1 : Building the core toolchain packages for " + constants.currentArch) self.logger.info(constants.listCoreToolChainPackages) self.logger.info("") pkgCount = 0 pkgUtils = PackageUtils(self.logName, self.logPath) coreToolChainYetToBuild = [] doneList = [] for package in constants.listCoreToolChainPackages: version = SPECS.getData().getHighestVersion(package) rpmPkg = pkgUtils.findRPMFile(package, version) self.sortedPackageList.append(package+"-"+version) if rpmPkg is not None: doneList.append(package+'-'+version) continue else: coreToolChainYetToBuild.append(package) self.listOfPackagesAlreadyBuilt = set(doneList) pkgCount = len(coreToolChainYetToBuild) if coreToolChainYetToBuild: self.logger.info("The following core toolchain packages need to be built :") self.logger.info(coreToolChainYetToBuild) else: self.logger.info("Core toolchain packages are already available") self.logger.info("") return pkgCount Scheduler.coreToolChainBuild = True self._buildPackages(1) Scheduler.coreToolChainBuild = False self.logger.debug("Successfully built core toolchain") self.logger.info("-" * 45 + "\n") return pkgCount
def loadPackagesData(self): listPackages = SPECS.getData().getListPackages() listPackages.sort() pkgUtils = PackageUtils(self.logName, self.logPath) for package in listPackages: for version in SPECS.getData().getVersions(package): srpmFile = pkgUtils.findSourceRPMFile(package, version) debugrpmFile = pkgUtils.findDebugRPMFile(package, version) listRPMPackages = SPECS.getData().getRPMPackages( package, version) for rpmPkg in listRPMPackages: rpmFile = pkgUtils.findRPMFile(rpmPkg, version) if rpmFile is not None: listPkgAttributes = { "sourcerpm": srpmFile, "rpm": rpmFile, "debugrpm": debugrpmFile } self.pkgList[rpmPkg + "-" + version] = listPkgAttributes self.logger.debug("Added " + rpmPkg + "-" + version + " to the package info json") else: self.logger.debug("Missing rpm file for package:" + rpmPkg)
def installTargetToolchain(self, chroot, stopAtPackage=None): self.logger.debug("Installing target toolchain RPMS.......") pkgUtils = PackageUtils(self.logName, self.logPath) rpmFiles = "" packages = "" for package in constants.listCoreToolChainPackages: if stopAtPackage and package == stopAtPackage: break version = SPECS.getData().getHighestVersion(package) basePkg = SPECS.getData().getSpecName(package) # install all subpackages of given package # for instance: for 'glibc' we want glibc-devel, glibc-tools, # glibc-i18n, etc also to be installed subpackages = SPECS.getData().getRPMPackages(basePkg, version) for p in subpackages: rpmFile = pkgUtils.findRPMFile(p, version, constants.targetArch) rpmFiles += " " + rpmFile packages += " " + package+"-"+version self.logger.debug(packages) cmd = "mkdir -p " + chroot.getID() +"/target-"+ constants.targetArch+"/var/lib/rpm" CommandUtils.runCommandInShell(cmd, logfn=self.logger.debug) if rpmFiles != "": cmd = (self.rpmCommand+" -Uvh --nodeps --ignorearch --noscripts --root "+ chroot.getID() +"/target-"+ constants.targetArch+ " --define \'_dbpath /var/lib/rpm\' "+rpmFiles) retVal = CommandUtils.runCommandInShell(cmd, logfn=self.logger.debug) if retVal != 0: self.logger.debug("Command Executed:" + cmd) self.logger.error("Installing toolchain failed") raise Exception("RPM installation failed") self.logger.debug("Successfully installed target toolchain RPMS in chroot:" + chroot.getID())
def loadPackagesData(self): listPackages = SPECS.getData().getListPackages() listPackages.sort() cmdUtils = CommandUtils() for package in listPackages: for version in SPECS.getData().getVersions(package): release = SPECS.getData().getRelease(package, version) listRPMPackages = SPECS.getData().getRPMPackages(package, version) srpmFileName = package + "-" + version + "-" + release + ".src.rpm" srpmFiles = cmdUtils.findFile(srpmFileName, constants.sourceRpmPath) srpmFile = None if len(srpmFiles) == 1: srpmFile = srpmFiles[0] debugrpmFileName = package + "-debuginfo-" + version + "-" + release + "*" debugrpmFiles = cmdUtils.findFile(debugrpmFileName, constants.rpmPath) debugrpmFile = None if len(debugrpmFiles) == 1: debugrpmFile = debugrpmFiles[0] pkgUtils = PackageUtils(self.logName, self.logPath) for rpmPkg in listRPMPackages: rpmFile = pkgUtils.findRPMFile(rpmPkg, version) if rpmFile is not None: listPkgAttributes = {"sourcerpm":srpmFile, "rpm":rpmFile, "debugrpm":debugrpmFile} self.pkgList[rpmPkg+"-"+version] = listPkgAttributes self.logger.debug("Added " + rpmPkg + "-" + version + " to the package info json") else: self.logger.debug("Missing rpm file for package:" + rpmPkg)
def buildCoreToolChainPackages(self): self.logger.info("Step 1 : Building the core toolchain packages for " + constants.currentArch) self.logger.info(constants.listCoreToolChainPackages) self.logger.info("") chroot = None pkgCount = 0 try: pkgUtils = PackageUtils(self.logName, self.logPath) coreToolChainYetToBuild = [] doneList = [] #self.logger.debug(">>>> Looking for available packages...") for package in constants.listCoreToolChainPackages: version = SPECS.getData().getHighestVersion(package) rpmPkg = pkgUtils.findRPMFile(package, version) #rpmPkg = pkgUtils.findRPMFile(package, version, constants.buildArch) if rpmPkg is not None: doneList.append(package+'-'+version) #self.logger.debug(">>>>>> %s", package+'-'+version) continue else: coreToolChainYetToBuild.append(package) if coreToolChainYetToBuild: self.logger.info("The following core toolchain packages need to be built :") self.logger.info(coreToolChainYetToBuild) else: self.logger.info("Core toolchain packages are already available") for package in coreToolChainYetToBuild: self.logger.debug("Building core toolchain package : " + package) version = SPECS.getData().getHighestVersion(package) destLogPath = constants.logPath + "/" + package + "-" + version + "." + constants.currentArch if not os.path.isdir(destLogPath): CommandUtils.runCommandInShell("mkdir -p " + destLogPath) chroot = Chroot(self.logger) chroot.create(package + "-" + version) #self.logger.debug(">>>>> Done List:") #self.logger.info(doneList) self.installToolchainRPMS(chroot, package, version, availablePackages=doneList) pkgUtils.adjustGCCSpecs(chroot, package, version) pkgUtils.buildRPMSForGivenPackage(chroot, package, version, destLogPath) pkgCount += 1 chroot.destroy() doneList.append(package+'-'+version) self.logger.debug("Successfully built toolchain") self.logger.info("-" * 45 + "\n") except Exception as e: self.logger.error("Unable to build toolchain.") # print stacktrace traceback.print_exc() raise e return pkgCount
def _readAlreadyAvailablePackages(self): listAvailablePackages = set() pkgUtils = PackageUtils(self.logName, self.logPath) listPackages = SPECS.getData().getListPackages() for package in listPackages: for version in SPECS.getData().getVersions(package): # Mark package available only if all subpackages are available packageIsAlreadyBuilt=True listRPMPackages = SPECS.getData().getRPMPackages(package, version) for rpmPkg in listRPMPackages: if pkgUtils.findRPMFile(rpmPkg, version) is None: packageIsAlreadyBuilt=False break; if packageIsAlreadyBuilt: listAvailablePackages.add(package+"-"+version) return listAvailablePackages
def installToolchainRPMS(self, chroot, packageName=None, packageVersion=None, usePublishedRPMS=True, availablePackages=None): self.logger.debug("Installing toolchain RPMS.......") rpmFiles = "" packages = "" listBuildRequiresPackages = [] listRPMsToInstall = list(constants.listToolChainRPMsToInstall) if constants.crossCompiling: targetPackageName = packageName packageName = None packageVersion = None listRPMsToInstall.extend([ 'binutils-' + constants.targetArch + '-linux-gnu', 'gcc-' + constants.targetArch + '-linux-gnu' ]) if packageName: listBuildRequiresPackages = self.getListDependentPackages( packageName, packageVersion) for package in listRPMsToInstall: pkgUtils = PackageUtils(self.logName, self.logPath) rpmFile = None version = None # Get proper package version for depPkg in listBuildRequiresPackages: depPkgName, depPkgVersion = StringUtils.splitPackageNameAndVersion( depPkg) if depPkgName == package: version = depPkgVersion break if not version: version = SPECS.getData( constants.buildArch).getHighestVersion(package) if availablePackages is not None: basePkg = SPECS.getData( constants.buildArch).getSpecName(package) + "-" + version isAvailable = basePkg in availablePackages else: # if availablePackages is not provided (rear case) it is safe # to use findRPMFile() isAvailable = True if constants.rpmCheck: rpmFile = pkgUtils.findRPMFile(package, version, constants.buildArch) if rpmFile is None: # Honor the toolchain list order. # if index of depended package ('package') is more # then index of the current package that we are # building ('packageName'), then we _must_ use published # `package` rpm. if (packageName and packageName in listRPMsToInstall and listRPMsToInstall.index(packageName) < listRPMsToInstall.index(package)): isAvailable = False if isAvailable: rpmFile = pkgUtils.findRPMFile(package, version, constants.buildArch) if rpmFile is None: if not usePublishedRPMS or isAvailable or constants.crossCompiling: raise Exception( "%s-%s.%s not found in available packages" % (package, version, constants.buildArch)) # Safe to use published RPM # sqlite-autoconf package was renamed, but it still published as sqlite-autoconf if (package == "sqlite") and (constants.buildArch == "x86_64"): package = "sqlite-autoconf" rpmFile = self._findPublishedRPM(package, constants.prevPublishRPMRepo) if rpmFile is None: if package in constants.listOfRPMsProvidedAfterBuild: self.logger.debug( "No old version of " + package + " exists, skip until the new version is built") continue self.logger.error("Unable to find published rpm " + package) raise Exception("Input Error") rpmFiles += " " + rpmFile packages += " " + package + "-" + version self.logger.debug(rpmFiles) self.logger.debug(packages) cmd = (self.rpmCommand + " -i -v --nodeps --noorder --force --root " + chroot.getID() + " --define \'_dbpath /var/lib/rpm\' " + rpmFiles) retVal = CommandUtils.runCommandInShell(cmd, logfn=self.logger.debug) if retVal != 0: self.logger.debug("Command Executed:" + cmd) self.logger.error("Installing toolchain RPMS failed") raise Exception("RPM installation failed") self.logger.debug( "Successfully installed default toolchain RPMS in Chroot:" + chroot.getID()) if packageName: self.installExtraToolchainRPMS(chroot, packageName, packageVersion) if constants.crossCompiling: self.installTargetToolchain(chroot, targetPackageName)
def installToolchainRPMS(self, chroot, packageName=None, packageVersion=None, usePublishedRPMS=True, availablePackages=None): self.logger.debug("Installing toolchain RPMS.......") rpmFiles = "" packages = "" listBuildRequiresPackages = [] listRPMsToInstall=list(constants.listToolChainRPMsToInstall) if constants.crossCompiling: targetPackageName = packageName packageName = None packageVersion = None listRPMsToInstall.extend(['binutils-'+constants.targetArch+'-linux-gnu', 'gcc-'+constants.targetArch+'-linux-gnu']) if packageName: listBuildRequiresPackages = self.getListDependentPackages(packageName, packageVersion) for package in listRPMsToInstall: pkgUtils = PackageUtils(self.logName, self.logPath) rpmFile = None version = None # Get proper package version for depPkg in listBuildRequiresPackages: depPkgName, depPkgVersion = StringUtils.splitPackageNameAndVersion(depPkg) if depPkgName == package: version=depPkgVersion break if not version: version = SPECS.getData(constants.buildArch).getHighestVersion(package) if availablePackages is not None: basePkg = SPECS.getData(constants.buildArch).getSpecName(package)+"-"+version isAvailable = basePkg in availablePackages else: # if availablePackages is not provided (rear case) it is safe # to use findRPMFile() isAvailable = True if constants.rpmCheck: rpmFile = pkgUtils.findRPMFile(package, version, constants.buildArch) if rpmFile is None: # Honor the toolchain list order. # if index of depended package ('package') is more # then index of the current package that we are # building ('packageName'), then we _must_ use published # `package` rpm. if (packageName and packageName in listRPMsToInstall and listRPMsToInstall.index(packageName) < listRPMsToInstall.index(package)): isAvailable = False if isAvailable: rpmFile = pkgUtils.findRPMFile(package, version, constants.buildArch) if rpmFile is None: if not usePublishedRPMS or isAvailable or constants.crossCompiling: raise Exception("%s-%s.%s not found in available packages" % (package, version, constants.buildArch)) # Safe to use published RPM rpmFile = self._findPublishedRPM(package, constants.prevPublishRPMRepo) if rpmFile is None: self.logger.error("Unable to find published rpm " + package) raise Exception("Input Error") rpmFiles += " " + rpmFile packages += " " + package+"-"+version self.logger.debug(rpmFiles) self.logger.debug(packages) cmd = (self.rpmCommand + " -i -v --nodeps --noorder --force --root " + chroot.getID() +" --define \'_dbpath /var/lib/rpm\' "+ rpmFiles) # If rpm doesn't have zstd support, use rpm from photon_builder image if constants.hostRpmIsNotUsable: # if we are not root, make installed files owned by effective user to # support pure non-root package building. if os.geteuid() != 0: cmd = cmd + "; chown -R {0}:{1} {2}".format(os.geteuid(), os.getegid(), chroot.getID()) cmd = ("docker run -i -v " + constants.prevPublishRPMRepo + ":" + constants.prevPublishRPMRepo + " -v " + constants.rpmPath + ":" + constants.rpmPath + " -v " + chroot.getID() + ":" + chroot.getID() + " photon_builder:latest " + "/bin/bash -c \"" + cmd + "\"") self.logger.debug("Executing cmd: " + cmd) retVal = CommandUtils.runCommandInShell(cmd, logfn=self.logger.debug) if retVal != 0: self.logger.error("Installing toolchain RPMS failed") raise Exception("RPM installation failed") self.logger.debug("Successfully installed default toolchain RPMS in Chroot:" + chroot.getID()) if packageName: self.installExtraToolchainRPMS(chroot, packageName, packageVersion) if constants.crossCompiling: self.installTargetToolchain(chroot, targetPackageName)