def installCoreToolChainPackages(self, chrootID): self.logger.info("Installing toolchain.....") cmdUtils = CommandUtils() self.prepareBuildRoot(chrootID) for package in constants.listToolChainRPMPkgsToBuild: pkgUtils = PackageUtils(self.logName, self.logPath) rpmFile = None if package in constants.listCoreToolChainRPMPackages: rpmFile = pkgUtils.findRPMFileForGivenPackage(package) else: rpmFile = self.findRPMFileInGivenLocation( package, constants.prevPublishRPMRepo) if rpmFile is None: self.logger.error("Unable to find rpm " + package) raise "Input Error" self.logger.debug("Installing rpm:" + rpmFile) cmd = "rpm -i --nodeps --force --root " + chrootID + " --define \'_dbpath /var/lib/rpm\' " + rpmFile process = subprocess.Popen("%s" % cmd, shell=True, stdout=subprocess.PIPE) retval = process.wait() if retval != 0: self.logger.error("Installing tool chain package " + package + " failed") raise "RPM installation failed" self.logger.info( "Installed core tool chain packages successfully on chroot:" + chrootID)
def readAlreadyAvailablePackages(self): listAvailablePackages=[] listRPMFiles=[] listDirectorys=[] listDirectorys.append(constants.rpmPath) if constants.inputRPMSPath is not None: listDirectorys.append(constants.inputRPMSPath) while len(listDirectorys) > 0: dirPath=listDirectorys.pop() for dirEntry in os.listdir(dirPath): dirEntryPath = os.path.join(dirPath, dirEntry) if os.path.isfile(dirEntryPath) and dirEntryPath.endswith(".rpm"): listRPMFiles.append(dirEntryPath) elif os.path.isdir(dirEntryPath): listDirectorys.append(dirEntryPath) pkgUtils = PackageUtils(self.logName,self.logPath) for rpmfile in listRPMFiles: package,version,release = pkgUtils.findPackageInfoFromRPMFile(rpmfile) if constants.specData.isRPMPackage(package): specVersion=constants.specData.getVersion(package) specRelease=constants.specData.getRelease(package) if version == specVersion and release == specRelease: listAvailablePackages.append(package) self.logger.info("List of Already built packages") self.logger.info(listAvailablePackages) return listAvailablePackages
def loadPackagesData(self): listPackages = SPECS.getData().getListPackages() listPackages.sort() cmdUtils = CommandUtils() for package in listPackages: for version in SPECS.getData().getVersions(package): release = SPECS.getData().getRelease(package, version) listRPMPackages = SPECS.getData().getRPMPackages(package, version) srpmFileName = package + "-" + version + "-" + release + ".src.rpm" srpmFiles = cmdUtils.findFile(srpmFileName, constants.sourceRpmPath) srpmFile = None if len(srpmFiles) == 1: srpmFile = srpmFiles[0] debugrpmFileName = package + "-debuginfo-" + version + "-" + release + "*" debugrpmFiles = cmdUtils.findFile(debugrpmFileName, constants.rpmPath) debugrpmFile = None if len(debugrpmFiles) == 1: debugrpmFile = debugrpmFiles[0] pkgUtils = PackageUtils(self.logName, self.logPath) for rpmPkg in listRPMPackages: rpmFile = pkgUtils.findRPMFile(rpmPkg, version) if rpmFile is not None: listPkgAttributes = {"sourcerpm":srpmFile, "rpm":rpmFile, "debugrpm":debugrpmFile} self.pkgList[rpmPkg+"-"+version] = listPkgAttributes self.logger.debug("Added " + rpmPkg + "-" + version + " to the package info json") else: self.logger.debug("Missing rpm file for package:" + rpmPkg)
def installToolChain(self,chrootID): self.logger.info("Installing toolchain.....") self.prepareBuildRoot(chrootID) cmdUtils = CommandUtils() rpmFiles = "" packages = "" for package in constants.listToolChainRPMPkgsToInstall: pkgUtils=PackageUtils(self.logName,self.logPath) rpmFile=pkgUtils.findRPMFileForGivenPackage(package) if rpmFile is None: rpmFile=self.findRPMFileInGivenLocation(package, constants.prevPublishRPMRepo) if rpmFile is None: if package == "rpm-build" or package == "rpm-devel": continue self.logger.error("Unable to find rpm "+ package +" in current and previous versions") raise "Input Error" rpmFiles += " " + rpmFile packages += " " + package self.logger.debug("Installing toolchain rpms:" + packages) cmd=self.rpmCommand + " -i --nodeps --force --root "+chrootID+" --define \'_dbpath /var/lib/rpm\' "+ rpmFiles process = subprocess.Popen("%s" %cmd,shell=True,stdout=subprocess.PIPE) retval = process.wait() if retval != 0: self.logger.error("Installing toolchain rpms failed") raise "RPM installation failed" self.logger.info("Installed toolchain successfully on chroot:"+chrootID)
def installToolChain(self,chrootID): self.logger.info("Installing toolchain.....") self.prepareBuildRoot(chrootID) cmdUtils = CommandUtils() rpmFiles = "" packages = "" for package in constants.listToolChainRPMPkgsToInstall: pkgUtils=PackageUtils(self.logName,self.logPath) rpmFile=pkgUtils.findRPMFileForGivenPackage(package) if rpmFile is None: rpmFile=self.findRPMFileInGivenLocation(package, constants.prevPublishRPMRepo) if rpmFile is None: if package == "util-linux-devel": self.logger.info("No old verion of util-linux-devel exists, skip until the new version is built") continue if package == "flex-devel": self.logger.info("No old verion of flex-devel exists, skip until the new version is built") continue self.logger.error("Unable to find rpm "+ package +" in current and previous versions") raise Exception("Input Error") rpmFiles += " " + rpmFile packages += " " + package self.logger.debug("Installing toolchain rpms:" + packages) cmd=self.rpmCommand + " -i --nodeps --force --root "+chrootID+" --define \'_dbpath /var/lib/rpm\' "+ rpmFiles process = subprocess.Popen("%s" %cmd,shell=True,stdout=subprocess.PIPE) retval = process.wait() if retval != 0: self.logger.error("Installing toolchain rpms failed") raise Exception("RPM installation failed") self.logger.info("Installed toolchain successfully on chroot:"+chrootID)
def installCoreToolChainPackages(self,chrootID): self.logger.info("Installing toolchain.....") cmdUtils = CommandUtils() self.prepareBuildRoot(chrootID) rpmFiles = "" packages = "" for package in constants.listToolChainRPMPkgsToBuild: pkgUtils=PackageUtils(self.logName,self.logPath) rpmFile = None if package in constants.listCoreToolChainRPMPackages: rpmFile=pkgUtils.findRPMFileForGivenPackage(package) else: rpmFile=self.findRPMFileInGivenLocation(package, constants.prevPublishRPMRepo) if rpmFile is None: self.logger.error("Unable to find rpm "+ package) raise Exception("Input Error") rpmFiles += " " + rpmFile packages += " " + package self.logger.debug("Installing core toolchain rpms:" + packages) cmd=self.rpmCommand + " -i --nodeps --force --root "+chrootID+" --define \'_dbpath /var/lib/rpm\' "+ rpmFiles process = subprocess.Popen("%s" %cmd,shell=True,stdout=subprocess.PIPE) retval = process.wait() if retval != 0: self.logger.error("Installing toolchain rpms failed") raise Exception("RPM installation failed") self.logger.info("Installed core tool chain packages successfully on chroot:"+chrootID)
def buildToolChain(self): self.logger.info("Step 1 : Building the core toolchain packages for " + constants.currentArch) self.logger.info(constants.listCoreToolChainPackages) self.logger.info("") pkgCount = 0 pkgUtils = PackageUtils(self.logName, self.logPath) coreToolChainYetToBuild = [] doneList = [] for package in constants.listCoreToolChainPackages: version = SPECS.getData().getHighestVersion(package) rpmPkg = pkgUtils.findRPMFile(package, version) self.sortedPackageList.append(package+"-"+version) if rpmPkg is not None: doneList.append(package+'-'+version) continue else: coreToolChainYetToBuild.append(package) self.listOfPackagesAlreadyBuilt = set(doneList) pkgCount = len(coreToolChainYetToBuild) if coreToolChainYetToBuild: self.logger.info("The following core toolchain packages need to be built :") self.logger.info(coreToolChainYetToBuild) else: self.logger.info("Core toolchain packages are already available") self.logger.info("") return pkgCount Scheduler.coreToolChainBuild = True self._buildPackages(1) Scheduler.coreToolChainBuild = False self.logger.debug("Successfully built core toolchain") self.logger.info("-" * 45 + "\n") return pkgCount
def buildCoreToolChainPackages(self): self.logger.info("Building core tool chain packages.....") chrootID=None try: pkgUtils=PackageUtils(self.logName,self.logPath) for package in constants.listCoreToolChainRPMPackages: rpmPkg=pkgUtils.findRPMFileForGivenPackage(package) if rpmPkg is not None: continue chrUtils = ChrootUtils(self.logName,self.logPath) chrootName="build-core-toolchain" destLogPath=constants.logPath+"/build-"+package if not os.path.isdir(destLogPath): cmdUtils = CommandUtils() cmdUtils.runCommandInShell("mkdir -p "+destLogPath) returnVal,chrootID = chrUtils.createChroot(chrootName) if not returnVal: self.logger.error("Creating chroot failed") raise Exception("creating chroot failed") self.installToolChainRPMS(chrootID) pkgUtils.buildRPMSForGivenPackage(package, chrootID,destLogPath) chrUtils.destroyChroot(chrootID) chrootID=None self.logger.info("Successfully built toolchain") except Exception as e: self.logger.error("Unable to build tool chain.") raise e finally: if chrootID is not None: chrUtils.destroyChroot(chrootID)
def installToolChainRPMS(self,chrootID): cmdUtils = CommandUtils() self.prepareBuildRoot(chrootID) self.logger.info("Installing Tool Chain RPMS.......") rpmFiles = "" packages = "" for package in constants.listToolChainRPMPkgsToBuild: pkgUtils=PackageUtils(self.logName,self.logPath) rpmFile=pkgUtils.findRPMFileForGivenPackage(package) if rpmFile is None: rpmFile=self.findRPMFileInGivenLocation(package, constants.prevPublishRPMRepo) if rpmFile is None: self.logger.error("Unable to find rpm "+ package +" in current and previous versions") raise "Input Error" rpmFiles += " " + rpmFile packages += " " + package self.logger.debug("Installing rpms:"+packages) cmd=self.rpmCommand + " -i --nodeps --force --root "+chrootID+" --define \'_dbpath /var/lib/rpm\' "+ rpmFiles process = subprocess.Popen("%s" %cmd,shell=True,stdout=subprocess.PIPE) retval = process.wait() if retval != 0: self.logger.error("Installing tool chain failed") raise "RPM installation failed" self.logger.info("Successfully installed all Tool Chain RPMS in Chroot:"+chrootID)
def installTargetToolchain(self, chroot, stopAtPackage=None): self.logger.debug("Installing target toolchain RPMS.......") pkgUtils = PackageUtils(self.logName, self.logPath) rpmFiles = "" packages = "" for package in constants.listCoreToolChainPackages: if stopAtPackage and package == stopAtPackage: break version = SPECS.getData().getHighestVersion(package) basePkg = SPECS.getData().getSpecName(package) # install all subpackages of given package # for instance: for 'glibc' we want glibc-devel, glibc-tools, # glibc-i18n, etc also to be installed subpackages = SPECS.getData().getRPMPackages(basePkg, version) for p in subpackages: rpmFile = pkgUtils.findRPMFile(p, version, constants.targetArch) rpmFiles += " " + rpmFile packages += " " + package+"-"+version self.logger.debug(packages) cmd = "mkdir -p " + chroot.getID() +"/target-"+ constants.targetArch+"/var/lib/rpm" CommandUtils.runCommandInShell(cmd, logfn=self.logger.debug) if rpmFiles != "": cmd = (self.rpmCommand+" -Uvh --nodeps --ignorearch --noscripts --root "+ chroot.getID() +"/target-"+ constants.targetArch+ " --define \'_dbpath /var/lib/rpm\' "+rpmFiles) retVal = CommandUtils.runCommandInShell(cmd, logfn=self.logger.debug) if retVal != 0: self.logger.debug("Command Executed:" + cmd) self.logger.error("Installing toolchain failed") raise Exception("RPM installation failed") self.logger.debug("Successfully installed target toolchain RPMS in chroot:" + chroot.getID())
def readAlreadyAvailablePackages(self): listAvailablePackages = [] listRPMFiles = [] listDirectorys = [] listDirectorys.append(constants.rpmPath) while len(listDirectorys) > 0: dirPath = listDirectorys.pop() for dirEntry in os.listdir(dirPath): dirEntryPath = os.path.join(dirPath, dirEntry) if os.path.isfile(dirEntryPath) and dirEntryPath.endswith( ".rpm"): listRPMFiles.append(dirEntryPath) elif os.path.isdir(dirEntryPath): listDirectorys.append(dirEntryPath) pkgUtils = PackageUtils(self.logName, self.logPath) for rpmfile in listRPMFiles: package, version, release = pkgUtils.findPackageInfoFromRPMFile( rpmfile) if constants.specData.isRPMPackage(package): specVersion = constants.specData.getVersion(package) specRelease = constants.specData.getRelease(package) if version == specVersion and release == specRelease: listAvailablePackages.append(package) self.logger.info("List of Already built packages") self.logger.info(listAvailablePackages) return listAvailablePackages
def installToolChainRPMS(self,chrootID): cmdUtils = CommandUtils() self.prepareBuildRoot(chrootID) self.logger.info("Installing Tool Chain RPMS.......") rpmFiles = "" packages = "" for package in constants.listToolChainRPMPkgsToBuild: pkgUtils=PackageUtils(self.logName,self.logPath) rpmFile=pkgUtils.findRPMFileForGivenPackage(package) if rpmFile is None: rpmFile=self.findRPMFileInGivenLocation(package, constants.prevPublishRPMRepo) if rpmFile is None: if package == "util-linux-devel": self.logger.info("No old version of util-linux-devel exists, skip until the new version is built") continue if package == "flex-devel": self.logger.info("No old version of flex-devel exists, skip until the new version is built") continue self.logger.error("Unable to find rpm "+ package +" in current and previous versions") raise Exception("Input Error") rpmFiles += " " + rpmFile packages += " " + package self.logger.debug("Installing rpms:"+packages) cmd=self.rpmCommand + " -i --nodeps --force --root "+chrootID+" --define \'_dbpath /var/lib/rpm\' "+ rpmFiles process = subprocess.Popen("%s" %cmd,shell=True,stdout=subprocess.PIPE) retval = process.wait() if retval != 0: self.logger.error("Installing tool chain failed") raise Exception("RPM installation failed") self.logger.info("Successfully installed all Tool Chain RPMS in Chroot:"+chrootID)
def buildPackage(self,package): #should initialize a logger based on package name chrUtils = ChrootUtils(self.logName,self.logPath) chrootName="build-"+package chrootID=None isToolChainPackage=False if package in constants.listToolChainPackages: isToolChainPackage=True try: chrootID = self.prepareBuildRoot(chrootName,isToolChainPackage) destLogPath=constants.logPath+"/build-"+package if not os.path.isdir(destLogPath): cmdUtils = CommandUtils() cmdUtils.runCommandInShell("mkdir -p "+destLogPath) listInstalledPackages=self.findInstalledPackages(chrootID) self.logger.info("List of installed packages") self.logger.info(listInstalledPackages) listDependentPackages=self.findBuildTimeRequiredPackages(package) if len(listDependentPackages) != 0: self.logger.info("Installing the build time dependent packages......") for pkg in listDependentPackages: self.installPackage(pkg,chrootID,destLogPath,listInstalledPackages) self.logger.info("Finished installing the build time dependent packages......") self.adjustGCCSpecs(package, chrootID, destLogPath) pkgUtils = PackageUtils(self.logName,self.logPath) pkgUtils.buildRPMSForGivenPackage(package,chrootID,destLogPath) self.logger.info("Successfully built the package:"+package) except Exception as e: self.logger.error("Failed while building package:" + package) self.logger.debug("Chroot with ID: " + chrootID + " not deleted for debugging.") raise e if chrootID is not None: chrUtils.destroyChroot(chrootID)
def loadPackagesData(self): listPackages = constants.specData.getListPackages() listPackages.sort() listRPMFiles = [] cmdUtils = CommandUtils() for package in listPackages: release = constants.specData.getRelease(package) version = constants.specData.getVersion(package) listRPMPackages = constants.specData.getRPMPackages(package) srpmFileName = package+"-"+version+"-"+release+".src.rpm" srpmFiles = cmdUtils.findFile(srpmFileName, constants.sourceRpmPath) srpmFile = None if len(srpmFiles) == 1: srpmFile = srpmFiles[0] debugrpmFileName = package+"-debuginfo-"+version+"-"+release+"*" debugrpmFiles = cmdUtils.findFile(debugrpmFileName, constants.rpmPath) debugrpmFile = None if len(debugrpmFiles) == 1: debugrpmFile = debugrpmFiles[0] pkgUtils = PackageUtils(self.logName,self.logPath) for rpmPkg in listRPMPackages: rpmFile = pkgUtils.findRPMFileForGivenPackage(rpmPkg) if rpmFile is not None: listRPMFiles.append(rpmFile) listPkgAttributes = {"sourcerpm":srpmFile, "rpm":rpmFile, "debugrpm":debugrpmFile} self.pkgList[rpmPkg] = listPkgAttributes self.logger.debug("Added "+rpmPkg +" rpm package to the list") else: self.logger.error("Missing rpm file for package:"+rpmPkg)
def loadPackagesData(self): listPackages = SPECS.getData().getListPackages() listPackages.sort() listRPMFiles = [] cmdUtils = CommandUtils() for package in listPackages: release = SPECS.getData().getRelease(package) version = SPECS.getData().getVersion(package) listRPMPackages = SPECS.getData().getRPMPackages(package) srpmFileName = package + "-" + version + "-" + release + ".src.rpm" srpmFiles = cmdUtils.findFile(srpmFileName, constants.sourceRpmPath) srpmFile = None if len(srpmFiles) == 1: srpmFile = srpmFiles[0] debugrpmFileName = package + "-debuginfo-" + version + "-" + release + "*" debugrpmFiles = cmdUtils.findFile(debugrpmFileName, constants.rpmPath) debugrpmFile = None if len(debugrpmFiles) == 1: debugrpmFile = debugrpmFiles[0] pkgUtils = PackageUtils(self.logName, self.logPath) for rpmPkg in listRPMPackages: rpmFile = pkgUtils.findRPMFileForGivenPackage(rpmPkg) if rpmFile is not None: listRPMFiles.append(rpmFile) listPkgAttributes = {"sourcerpm":srpmFile, "rpm":rpmFile, "debugrpm":debugrpmFile} self.pkgList[rpmPkg] = listPkgAttributes self.logger.debug("Added " + rpmPkg + " rpm package to the list") else: self.logger.error("Missing rpm file for package:" + rpmPkg)
def loadPackagesData(self): listPackages = SPECS.getData().getListPackages() listPackages.sort() pkgUtils = PackageUtils(self.logName, self.logPath) for package in listPackages: for version in SPECS.getData().getVersions(package): srpmFile = pkgUtils.findSourceRPMFile(package, version) debugrpmFile = pkgUtils.findDebugRPMFile(package, version) listRPMPackages = SPECS.getData().getRPMPackages( package, version) for rpmPkg in listRPMPackages: rpmFile = pkgUtils.findRPMFile(rpmPkg, version) if rpmFile is not None: listPkgAttributes = { "sourcerpm": srpmFile, "rpm": rpmFile, "debugrpm": debugrpmFile } self.pkgList[rpmPkg + "-" + version] = listPkgAttributes self.logger.debug("Added " + rpmPkg + "-" + version + " to the package info json") else: self.logger.debug("Missing rpm file for package:" + rpmPkg)
def buildCoreToolChainPackages(self): self.logger.info("Building core tool chain packages.....") chrootID = None try: pkgUtils = PackageUtils(self.logName, self.logPath) for package in constants.listCoreToolChainRPMPackages: rpmPkg = pkgUtils.findRPMFileForGivenPackage(package) if rpmPkg is not None: continue chrUtils = ChrootUtils(self.logName, self.logPath) chrootName = "build-core-toolchain" destLogPath = constants.logPath + "/build-" + package if not os.path.isdir(destLogPath): cmdUtils = CommandUtils() cmdUtils.runCommandInShell("mkdir -p " + destLogPath) returnVal, chrootID = chrUtils.createChroot(chrootName) if not returnVal: self.logger.error("Creating chroot failed") raise Exception("creating chroot failed") self.installToolChainRPMS(chrootID) pkgUtils.buildRPMSForGivenPackage(package, chrootID, destLogPath) chrUtils.destroyChroot(chrootID) chrootID = None self.logger.info("Successfully built toolchain") except Exception as e: self.logger.error("Unable to build tool chain.") raise e finally: if chrootID is not None: chrUtils.destroyChroot(chrootID)
def _readAlreadyAvailablePackages(self): listAvailablePackages = set() listFoundRPMPackages = set() listRPMFiles = set() listDirectorys = set() mapPackageVersion = {} mapPackageRelease = {} listDirectorys.add(constants.rpmPath) if constants.inputRPMSPath is not None: listDirectorys.add(constants.inputRPMSPath) while listDirectorys: dirPath = listDirectorys.pop() for dirEntry in os.listdir(dirPath): dirEntryPath = os.path.join(dirPath, dirEntry) if os.path.isfile(dirEntryPath) and dirEntryPath.endswith( ".rpm"): listRPMFiles.add(dirEntryPath) elif os.path.isdir(dirEntryPath): listDirectorys.add(dirEntryPath) pkgUtils = PackageUtils(self.logName, self.logPath) for rpmfile in listRPMFiles: package, version, release = pkgUtils.findPackageInfoFromRPMFile( rpmfile) if package in mapPackageVersion: mapPackageVersion[package].append(version) mapPackageRelease[package].append(release) else: mapPackageVersion[package] = [version] mapPackageRelease[package] = [release] for package in mapPackageVersion: if SPECS.getData().isRPMPackage(package): numVersions = SPECS.getData().getNumberOfVersions(package) flag = True for index in range(0, numVersions): specVersion = SPECS.getData().getVersion(package, index) specRelease = SPECS.getData().getRelease(package, index) if specVersion not in mapPackageVersion[ package] and specRelease not in mapPackageRelease[ package]: flag = False if flag: listFoundRPMPackages.add(package) #Mark package available only if all sub packages are available for package in listFoundRPMPackages: basePkg = SPECS.getData().getSpecName(package) if basePkg in listAvailablePackages: continue listRPMPackages = SPECS.getData().getRPMPackages(basePkg) packageIsAlreadyBuilt = True for rpmpkg in listRPMPackages: if rpmpkg not in listFoundRPMPackages: packageIsAlreadyBuilt = False if packageIsAlreadyBuilt: listAvailablePackages.add(package) self.logger.info("List of Already built packages") self.logger.info(listAvailablePackages) return listAvailablePackages
def _findInstalledPackages(self, sandbox): pkgUtils = PackageUtils(self.logName, self.logPath) listInstalledRPMs = pkgUtils.findInstalledRPMPackages(sandbox) listInstalledPackages = [] for installedRPM in listInstalledRPMs: pkg = self._findPackageNameAndVersionFromRPMFile(installedRPM) if pkg is not None: listInstalledPackages.append(pkg) return listInstalledPackages, listInstalledRPMs
def _findInstalledPackages(self, sandbox, arch): pkgUtils = PackageUtils(self.logName, self.logPath) listInstalledRPMs = pkgUtils.findInstalledRPMPackages(sandbox, arch) listInstalledPackages = [] for installedRPM in listInstalledRPMs: pkg = self._findPackageNameAndVersionFromRPMFile(installedRPM) if pkg is not None: listInstalledPackages.append(pkg) return listInstalledPackages, listInstalledRPMs
def findInstalledPackages(self,chrootID): pkgUtils = PackageUtils(self.logName,self.logPath) listInstalledRPMs=pkgUtils.findInstalledRPMPackages(chrootID) listInstalledPackages=[] for installedRPM in listInstalledRPMs: packageName=self.findPackageNameFromRPMFile(installedRPM) if packageName is not None: listInstalledPackages.append(packageName) return listInstalledPackages
def findInstalledPackages(self, chrootID): pkgUtils = PackageUtils(self.logName, self.logPath) listInstalledRPMs = pkgUtils.findInstalledRPMPackages(chrootID) listInstalledPackages = [] for installedRPM in listInstalledRPMs: packageName = self.findPackageNameFromRPMFile(installedRPM) if packageName is not None: listInstalledPackages.append(packageName) return listInstalledPackages
def installToolChain2(self,chrootID,packagesInToolChain): self.logger.info("Installing toolchain.....") self.prepareBuildRoot(chrootID) cmdUtils = CommandUtils() rpmFiles = "" packages = "" for package in packagesInToolChain: pkgUtils=PackageUtils(self.logName,self.logPath) currentRpmFile=pkgUtils.findRPMFileForGivenPackage(package) prevPublishedRpmFile=None if currentRpmFile is None or ("-devel" not in package and "-lib" not in package and "perl" not in package): prevPublishedRpmFile=self.findRPMFileInGivenLocation(package, constants.prevPublishRPMRepo) if prevPublishedRpmFile is None: if package == "util-linux-devel": self.logger.info("No old verion of util-linux-devel exists, skip until the new version is built") continue if package == "flex-devel": self.logger.info("No old verion of flex-devel exists, skip until the new version is built") continue self.logger.error("Unable to find rpm "+ package +" in current and previous versions") raise Exception("Input Error") if (currentRpmFile is not None) and (prevPublishedRpmFile is not None): temp = currentRpmFile[len(package)+ 1:] currentRpmVersion = temp[:temp.find('-')] temp = prevPublishedRpmFile[len(package)+1:] prevPublishedRpmVersion = temp[:temp.find('-')] if(currentRpmVersion != prevPublishedRpmVersion): rpmFiles += " " + prevPublishedRpmFile packages += " " + package rpmFiles += " " + currentRpmFile packages += " " + package else: if prevPublishedRpmFile is not None: rpmFiles += " " + prevPublishedRpmFile packages += " " + package if currentRpmFile is not None: rpmFiles += " " + currentRpmFile packages += " " + package self.logger.debug("Installing toolchain rpms:" + packages) self.logger.debug("Installing toolchain rpms:" + rpmFiles) cmd=self.rpmCommand + " -i --nodeps --force --root "+chrootID+" --define \'_dbpath /var/lib/rpm\' "+ rpmFiles process = subprocess.Popen("%s" %cmd,shell=True,stdout=subprocess.PIPE) retval = process.wait() if retval != 0: self.logger.error("Installing toolchain rpms failed") raise Exception("RPM installation failed") self.logger.info("Installed toolchain successfully on chroot:"+chrootID)
def checkIfPackageIsAlreadyBuilt(self): basePkg = SPECS.getData().getSpecName(self.package) listRPMPackages = SPECS.getData().getRPMPackages(basePkg) packageIsAlreadyBuilt = True pkgUtils = PackageUtils(self.logName, self.logPath) for pkg in listRPMPackages: if pkgUtils.findRPMFileForGivenPackage(pkg) is None: packageIsAlreadyBuilt = False break return packageIsAlreadyBuilt
def _checkIfPackageIsAlreadyBuilt(self, package, version): basePkg = SPECS.getData().getSpecName(package) listRPMPackages = SPECS.getData().getRPMPackages(basePkg, version) packageIsAlreadyBuilt = True pkgUtils = PackageUtils() for pkg in listRPMPackages: if pkgUtils.findRPMFileForGivenPackage(pkg, version) is None: packageIsAlreadyBuilt = False break return packageIsAlreadyBuilt
def _checkIfPackageIsAlreadyBuilt(self): basePkg = SPECS.getData().getSpecName(self.package) listRPMPackages = SPECS.getData().getRPMPackages(basePkg) packageIsAlreadyBuilt = True pkgUtils = PackageUtils(self.logName, self.logPath) for pkg in listRPMPackages: if pkgUtils.findRPMFileForGivenPackage(pkg) is None: packageIsAlreadyBuilt = False break return packageIsAlreadyBuilt
def checkIfPackageIsAlreadyBuilt(self, package): basePkg=constants.specData.getSpecName(package) listRPMPackages=constants.specData.getRPMPackages(basePkg) packageIsAlreadyBuilt=True pkgUtils = PackageUtils(self.logName,self.logPath) for pkg in listRPMPackages: if pkgUtils.findRPMFileForGivenPackage(pkg) is None: packageIsAlreadyBuilt=False break return packageIsAlreadyBuilt
def installToolChainRPMS(self, chrootID, packageName, logPath=None): if logPath is None: logPath = self.logPath cmdUtils = CommandUtils() self.prepareBuildRoot(chrootID) self.logger.info("Installing Tool Chain RPMS.......") rpmFiles = "" packages = "" for package in constants.listToolChainRPMsToInstall: pkgUtils = PackageUtils(self.logName, self.logPath) rpmFile = None if constants.rpmCheck: rpmFile = pkgUtils.findRPMFileForGivenPackage(package) else: if (packageName not in constants.listToolChainRPMsToInstall or constants.listToolChainRPMsToInstall.index(packageName) > constants.listToolChainRPMsToInstall.index(package)): rpmFile = pkgUtils.findRPMFileForGivenPackage(package) if rpmFile is None: # sqlite-autoconf package was renamed, but it still published as sqlite-autoconf if (package == "sqlite") and (platform.machine() == "x86_64"): package = "sqlite-autoconf" rpmFile = self.findRPMFileInGivenLocation( package, constants.prevPublishRPMRepo) if rpmFile is None: if package in constants.listOfRPMsProvidedAfterBuild: self.logger.info( "No old version of " + package + " exists, skip until the new version is built") continue self.logger.error("Unable to find rpm " + package + " in current and previous versions") raise Exception("Input Error") rpmFiles += " " + rpmFile packages += " " + package self.logger.debug("Installing toolchain rpms:" + packages) cmd = (self.rpmCommand + " -i -v --nodeps --noorder --force --root " + chrootID + " --define \'_dbpath /var/lib/rpm\' " + rpmFiles) retVal = cmdUtils.runCommandInShell( cmd, logPath + "/install_toolchain_rpms.log") if not retVal: self.logger.debug("Command Executed:" + cmd) self.logger.error("Installing tool chain failed") raise Exception("RPM installation failed") self.logger.info( "Successfully installed default Tool Chain RPMS in Chroot:" + chrootID) print("Building Package: ".format(packageName)) print(constants.perPackageToolChain) if packageName in constants.perPackageToolChain: print(constants.perPackageToolChain[packageName]) self.installCustomToolChainRPMS( chrootID, constants.perPackageToolChain[packageName], packageName)
def _findInstalledPackages(self, instanceID): pkgUtils = PackageUtils(self.logName, self.logPath) if self.pkgBuildType == "chroot": listInstalledRPMs = pkgUtils.findInstalledRPMPackages(instanceID) elif self.pkgBuildType == "container": listInstalledRPMs = pkgUtils.findInstalledRPMPackagesInContainer(instanceID) listInstalledPackages = [] for installedRPM in listInstalledRPMs: packageName = self._findPackageNameFromRPMFile(installedRPM) if packageName is not None: listInstalledPackages.append(packageName) return listInstalledPackages, listInstalledRPMs
def installToolChainRPMS(self, chrootID, packageName): cmdUtils = CommandUtils() self.prepareBuildRoot(chrootID) self.logger.info("Installing Tool Chain RPMS.......") rpmFiles = "" packages = "" for package in constants.listToolChainRPMsToInstall: pkgUtils = PackageUtils(self.logName, self.logPath) rpmFile = None if constants.rpmCheck: rpmFile = pkgUtils.findRPMFileForGivenPackage(package) else: if (packageName not in constants.listToolChainRPMsToInstall or constants.listToolChainRPMsToInstall.index(packageName) > constants.listToolChainRPMsToInstall.index(package)): rpmFile = pkgUtils.findRPMFileForGivenPackage(package) if rpmFile is None: # sqlite-autoconf package was renamed, but it still published as sqlite-autoconf if package == "sqlite": package = "sqlite-autoconf" rpmFile = self.findRPMFileInGivenLocation( package, constants.prevPublishRPMRepo) if rpmFile is None: if package in constants.listOfRPMsProvidedAfterBuild: self.logger.info( "No old version of " + package + " exists, skip until the new version is built") continue self.logger.error("Unable to find rpm " + package + " in current and previous versions") raise Exception("Input Error") rpmFiles += " " + rpmFile packages += " " + package self.logger.debug("Installing rpms:" + packages) cmd = self.rpmCommand + " -i --nodeps --force --root " + chrootID + " --define \'_dbpath /var/lib/rpm\' " + rpmFiles process = subprocess.Popen("%s" % cmd, shell=True, stdout=subprocess.PIPE) retval = process.wait() if retval != 0: self.logger.error("Installing tool chain failed") raise Exception("RPM installation failed") self.logger.info( "Successfully installed default Tool Chain RPMS in Chroot:" + chrootID) print "Building Package:" + packageName print constants.perPackageToolChain if packageName in constants.perPackageToolChain: print constants.perPackageToolChain[packageName] self.installCustomToolChainRPMS( chrootID, constants.perPackageToolChain[packageName], packageName)
def prepareBuildRoot(self,chrootID): self.logger.info("Preparing build environment") cmdUtils = CommandUtils() cmdUtils.runCommandInShell("mkdir -p "+chrootID+"/tmp") cmdUtils.runCommandInShell("mkdir -p "+chrootID+constants.topDirPath) cmdUtils.runCommandInShell("mkdir -p "+chrootID+constants.topDirPath+"/RPMS/x86_64") cmdUtils.runCommandInShell("mkdir -p "+chrootID+constants.topDirPath+"/RPMS/noarch") cmdUtils.runCommandInShell("mkdir -p "+chrootID+constants.topDirPath+"/SOURCES") cmdUtils.runCommandInShell("mkdir -p "+chrootID+constants.topDirPath+"/SPECS") cmdUtils.runCommandInShell("mkdir -p "+chrootID+constants.topDirPath+"/LOGS") cmdUtils.runCommandInShell("mkdir -p "+chrootID+constants.topDirPath+"/BUILD") cmdUtils.runCommandInShell("mkdir -p "+chrootID+constants.topDirPath+"/BUILDROOT") package="filesystem" pkgUtils=PackageUtils(self.logName,self.logPath) rpmFile=pkgUtils.findRPMFileForGivenPackage(package) if rpmFile is None: specFile=constants.specData.getSpecFile(package) cmd=self.rpmbuildCommand+" -ba --nocheck --define \'_topdir "+chrootID+constants.topDirPath+"\' --define \'_dbpath "+chrootID+"/var/lib/rpm\' --define \'dist "+constants.dist+"\' --define \'photon_build_number "+constants.buildNumber+"\' --define \'photon_release_version "+constants.releaseVersion+"\' "+specFile self.logger.info(cmd) cmdUtils.runCommandInShell(cmd,self.logPath+"/filesystem.log") filesystemrpmFile = cmdUtils.findFile(package+"-[0-9]*.rpm", chrootID+constants.topDirPath+"/RPMS") filesystemsrpmFile = cmdUtils.findFile(package+"-[0-9]*.src.rpm", chrootID+constants.topDirPath+"/SRPMS") if len(filesystemrpmFile) > 0: shutil.copy2(filesystemrpmFile[0],constants.rpmPath+"/x86_64/") if len(filesystemsrpmFile) > 0: shutil.copy2(filesystemsrpmFile[0],constants.sourceRpmPath+"/") rpmFile=pkgUtils.findRPMFileForGivenPackage(package) if rpmFile is None: self.logger.error("Cannot find filesystem rpm") raise Exception("Cannot find filesystem rpm") self.logger.debug("Installing filesystem rpms:" + package) if os.geteuid()==0: cmd=self.rpmCommand + " -i --nodeps --root "+chrootID+" --define '_dbpath /var/lib/rpm' "+ rpmFile else: cmd=self.rpmCommand + " -i --nodeps --badreloc --relocate /="+chrootID+" --define '_dbpath "+chrootID+"/var/lib/rpm' "+ rpmFile process = subprocess.Popen("%s" %cmd,shell=True,stdout=subprocess.PIPE) retval = process.wait() if retval != 0: self.logger.error("Installing filesystem rpm failed") raise Exception("RPM installation failed") prepareChrootCmd=self.prepareBuildRootCmd+" "+chrootID logFile=constants.logPath+"/prepareBuildRoot.log" returnVal=cmdUtils.runCommandInShell(prepareChrootCmd,logFile) if not returnVal: self.logger.error("Prepare build root script failed.Unable to prepare chroot.") raise Exception("Prepare build root script failed") self.logger.info("Successfully prepared chroot:"+chrootID)
def prepareBuildRoot(self,chrootID): self.logger.info("Preparing build environment") cmdUtils = CommandUtils() cmdUtils.runCommandInShell("mkdir -p "+chrootID+"/tmp") cmdUtils.runCommandInShell("mkdir -p "+chrootID+constants.topDirPath) cmdUtils.runCommandInShell("mkdir -p "+chrootID+constants.topDirPath+"/RPMS/x86_64") cmdUtils.runCommandInShell("mkdir -p "+chrootID+constants.topDirPath+"/RPMS/noarch") cmdUtils.runCommandInShell("mkdir -p "+chrootID+constants.topDirPath+"/SOURCES") cmdUtils.runCommandInShell("mkdir -p "+chrootID+constants.topDirPath+"/SPECS") cmdUtils.runCommandInShell("mkdir -p "+chrootID+constants.topDirPath+"/LOGS") cmdUtils.runCommandInShell("mkdir -p "+chrootID+constants.topDirPath+"/BUILD") cmdUtils.runCommandInShell("mkdir -p "+chrootID+constants.topDirPath+"/BUILDROOT") package="filesystem" pkgUtils=PackageUtils(self.logName,self.logPath) rpmFile=pkgUtils.findRPMFileForGivenPackage(package) if rpmFile is None: specFile=constants.specData.getSpecFile(package) cmd=self.rpmbuildCommand+" -ba --nocheck --define \'_topdir "+chrootID+constants.topDirPath+"\' --define \'_dbpath "+chrootID+"/var/lib/rpm\' --define \'dist "+constants.dist+"\' "+specFile self.logger.info(cmd) cmdUtils.runCommandInShell(cmd,self.logPath+"/filesystem.log") filesystemrpmFile = cmdUtils.findFile(package+"-[0-9]*.rpm", chrootID+constants.topDirPath+"/RPMS") filesystemsrpmFile = cmdUtils.findFile(package+"-[0-9]*.src.rpm", chrootID+constants.topDirPath+"/SRPMS") if len(filesystemrpmFile) > 0: shutil.copy2(filesystemrpmFile[0],constants.rpmPath+"/x86_64/") if len(filesystemsrpmFile) > 0: shutil.copy2(filesystemsrpmFile[0],constants.sourceRpmPath+"/") rpmFile=pkgUtils.findRPMFileForGivenPackage(package) if rpmFile is None: self.logger.error("Cannot find filesystem rpm") raise Exception("Cannot find filesystem rpm") self.logger.debug("Installing filesystem rpms:" + package) if os.geteuid()==0: cmd=self.rpmCommand + " -i --nodeps --root "+chrootID+" --define '_dbpath /var/lib/rpm' "+ rpmFile else: cmd=self.rpmCommand + " -i --nodeps --badreloc --relocate /="+chrootID+" --define '_dbpath "+chrootID+"/var/lib/rpm' "+ rpmFile process = subprocess.Popen("%s" %cmd,shell=True,stdout=subprocess.PIPE) retval = process.wait() if retval != 0: self.logger.error("Installing filesystem rpm failed") raise Exception("RPM installation failed") prepareChrootCmd=self.prepareBuildRootCmd+" "+chrootID logFile=constants.logPath+"/prepareBuildRoot.log" returnVal=cmdUtils.runCommandInShell(prepareChrootCmd,logFile) if not returnVal: self.logger.error("Prepare build root script failed.Unable to prepare chroot.") raise Exception("Prepare build root script failed") self.logger.info("Successfully prepared chroot:"+chrootID)
def installToolChainRPMSinContainer(self, containerID): self.logger.info("Installing tool-chain RPMS in container: " + containerID.short_id) rpmFiles = "" packages = "" pkgUtils = PackageUtils(self.logName, self.logPath) for package in constants.listToolChainRPMPkgsToInstall: version = SPECS.getData().getHighestVersion(package) rpmFile = pkgUtils.findRPMFileForGivenPackage(package, version) if rpmFile is None: # sqlite-autoconf package was renamed, but it still published as sqlite-autoconf # if (package == "sqlite") and (platform.machine() == "x86_64"): # package = "sqlite-autoconf" rpmFile = self.findRPMFileInGivenLocation( package, constants.prevPublishRPMRepo) if rpmFile is None: if package in constants.listOfRPMsProvidedAfterBuild: self.logger.info( "No old version of " + package + " exists, skip until the new version is built") continue self.logger.error("Unable to find rpm " + package + " in current and previous versions") raise Exception("Input Error") if rpmFile.find("stage/PUBLISHRPMS"): rpmFile = rpmFile.replace(constants.prevPublishRPMRepo, "/publishrpms") if rpmFile.find("stage/PUBLISHXRPMS"): rpmFile = rpmFile.replace(constants.prevPublishXRPMRepo, "/publishxrpms") if rpmFile.find("stage/RPMS"): rpmFile = rpmFile.replace(constants.rpmPath, constants.topDirPath + "/RPMS") rpmFiles += " " + rpmFile packages += " " + package self.logger.debug("Installing tool-chain rpms: " + packages) cmd = "/usr/bin/bash -l -c '/usr/bin/rpm -Uvh --force --nodeps " + rpmFiles + "'" self.logger.info( "VDBG-TCU-installToolChainRPMSinContainer: Installing rpms cmd: " + cmd) tcInstallLog = containerID.exec_run(cmd) # TODO: Find a way to collect exit status of the command that was run. if not tcInstallLog: self.logger.error("Installing tool chain in container failed") raise Exception("RPM installation in container failed") self.logger.info(tcInstallLog) self.logger.info( "Successfully installed default tool-chain RPMS in container: " + containerID.short_id)
def _buildPackage(self): chroot = None try: chroot = Chroot(self.logger) chroot.create(self.package + "-" + self.version) tUtils = ToolChainUtils(self.logName, self.logPath) tUtils.installToolChainRPMS(chroot, self.package, self.version, self.logPath) listDependentPackages, listTestPackages, listInstalledPackages, listInstalledRPMs = ( self._findDependentPackagesAndInstalledRPM(chroot)) pkgUtils = PackageUtils(self.logName, self.logPath) if listDependentPackages: self.logger.debug( "Installing the build time dependent packages......") for pkg in listDependentPackages: packageName, packageVersion = StringUtils.splitPackageNameAndVersion( pkg) self._installPackage(pkgUtils, packageName, packageVersion, chroot, self.logPath, listInstalledPackages, listInstalledRPMs) for pkg in listTestPackages: flag = False packageName, packageVersion = StringUtils.splitPackageNameAndVersion( pkg) for depPkg in listDependentPackages: depPackageName, depPackageVersion = StringUtils.splitPackageNameAndVersion( depPkg) if depPackageName == packageName: flag = True break if flag == False: self._installPackage(pkgUtils, packageName, packageVersion, chroot, self.logPath, listInstalledPackages, listInstalledRPMs) pkgUtils.installRPMSInOneShot(chroot) self.logger.debug( "Finished installing the build time dependent packages...." ) pkgUtils.adjustGCCSpecs(chroot, self.package, self.version) pkgUtils.buildRPMSForGivenPackage(chroot, self.package, self.version, self.logPath) self.logger.debug("Successfully built the package:" + self.package) except Exception as e: self.logger.error("Failed while building package:" + self.package) self.logger.debug("Chroot: " + chroot.getPath() + " not deleted for debugging.") logFileName = os.path.join(self.logPath, self.package + ".log") fileLog = os.popen('tail -n 100 ' + logFileName).read() self.logger.info(fileLog) raise e if chroot: chroot.destroy()
def buildPackage(self, package): #do not build if RPM is already built #test only if the package is in the testForceRPMS with rpmCheck #build only if the package is not in the testForceRPMS with rpmCheck if self.checkIfPackageIsAlreadyBuilt(package): if not constants.rpmCheck: self.logger.info("Skipping building the package:" + package) return elif constants.rpmCheck and package not in constants.testForceRPMS: self.logger.info("Skipping testing the package:" + package) return #should initialize a logger based on package name chrUtils = ChrootUtils(self.logName, self.logPath) chrootName = "build-" + package chrootID = None try: chrootID = self.prepareBuildRoot(chrootName, package) destLogPath = constants.logPath + "/build-" + package if not os.path.isdir(destLogPath): cmdUtils = CommandUtils() cmdUtils.runCommandInShell("mkdir -p " + destLogPath) listInstalledPackages = self.findInstalledPackages(chrootID) listDependentPackages = self.findBuildTimeRequiredPackages(package) if constants.rpmCheck and package in constants.testForceRPMS: testPackages = set( constants.listMakeCheckRPMPkgtoInstall) - set( listInstalledPackages) - set([package]) listDependentPackages.extend(testPackages) pkgUtils = PackageUtils(self.logName, self.logPath) if len(listDependentPackages) != 0: self.logger.info( "Installing the build time dependent packages......") for pkg in listDependentPackages: self.installPackage(pkgUtils, pkg, chrootID, destLogPath, listInstalledPackages) pkgUtils.installRPMSInAOneShot(chrootID, destLogPath) self.logger.info( "Finished installing the build time dependent packages......" ) pkgUtils.adjustGCCSpecs(package, chrootID, destLogPath) pkgUtils.buildRPMSForGivenPackage(package, chrootID, self.listBuildOptionPackages, self.pkgBuildOptionFile, destLogPath) self.logger.info("Successfully built the package:" + package) except Exception as e: self.logger.error("Failed while building package:" + package) self.logger.debug("Chroot with ID: " + chrootID + " not deleted for debugging.") logFileName = os.path.join(destLogPath, package + ".log") fileLog = os.popen('tail -n 100 ' + logFileName).read() self.logger.debug(fileLog) raise e if chrootID is not None: chrUtils.destroyChroot(chrootID)
def installPackage(self,package,chrootID,destLogPath,listInstalledPackages): if package in listInstalledPackages: return self.installDependentRunTimePackages(package,chrootID,destLogPath,listInstalledPackages) pkgUtils = PackageUtils(self.logName,self.logPath) noDeps=False if self.mapPackageToCycles.has_key(package): noDeps = True if package in self.listNodepsPackages: noDeps=True if package in constants.noDepsPackageList: noDeps=True pkgUtils.installRPM(package,chrootID,noDeps,destLogPath) listInstalledPackages.append(package) self.logger.info("Installed the package:"+package)
def installCustomToolChainRPMS(self, chrootID, listOfToolChainPkgs, packageName): self.logger.info("Installing package specific tool chain RPMs for " + packageName + ".......") rpmFiles = "" packages = "" for package in listOfToolChainPkgs: pkgUtils = PackageUtils(self.logName, self.logPath) print "DEBUG:" + package if "openjre8" in packageName or "openjdk8" in packageName: rpmFile = self.findRPMFileInGivenLocation( package, constants.prevPublishXRPMRepo) else: rpmFile = self.findRPMFileInGivenLocation( package, constants.prevPublishRPMRepo) if rpmFile is None: self.logger.error("Unable to find rpm " + package + " in current and previous versions") raise Exception("Input Error") rpmFiles += " " + rpmFile packages += " " + package self.logger.debug("Installing rpms:" + packages) cmd = self.rpmCommand + " -i --nodeps --force --root " + chrootID + " --define \'_dbpath /var/lib/rpm\' " + rpmFiles print "Command Executed:" + cmd process = subprocess.Popen("%s" % cmd, shell=True, stdout=subprocess.PIPE) retval = process.wait() if retval != 0: self.logger.error("Installing tool chain failed") raise Exception("RPM installation failed") self.logger.info("Successfully installed all Tool Chain X RPMS")
def installExtraToolchainRPMS(self, sandbox, packageName, packageVersion): listOfToolChainPkgs = SPECS.getData( constants.buildArch).getExtraBuildRequiresForPackage( packageName, packageVersion) if not listOfToolChainPkgs: return self.logger.debug("Installing package specific toolchain RPMs for " + packageName + ": " + str(listOfToolChainPkgs)) rpmFiles = "" packages = "" for package in listOfToolChainPkgs: pkgUtils = PackageUtils(self.logName, self.logPath) if re.match("openjre*", packageName) is not None or re.match( "openjdk*", packageName): path = constants.prevPublishXRPMRepo sandboxPath = "/publishxrpms" else: path = constants.prevPublishRPMRepo sandboxPath = "/publishrpms" rpmFile = self._findPublishedRPM(package, path) if rpmFile is None: self.logger.error("Unable to find rpm " + package + " in current and previous versions") raise Exception("Input Error") rpmFiles += " " + rpmFile.replace(path, sandboxPath) packages += " " + package self.logger.debug("Installing custom rpms:" + packages) cmd = ("rpm -i -v --nodeps --noorder --force " + rpmFiles) retVal = sandbox.run(cmd, logfn=self.logger.debug) if retVal != 0: self.logger.debug("Command Executed:" + cmd) self.logger.error("Installing custom toolchains failed") raise Exception("RPM installation failed")
def installCustomToolChainRPMS(self, chrootID, listOfToolChainPkgs, packageName): self.logger.info("Installing package specific tool chain RPMs for " + packageName + ".......") rpmFiles = "" packages = "" cmdUtils = CommandUtils() for package in listOfToolChainPkgs: pkgUtils = PackageUtils(self.logName, self.logPath) print("DEBUG:" + package) if "openjre8" in packageName or "openjdk8" in packageName: # x86_64 has openjdk/jre as a published rpms but aarch64 has openjdk8/jre8 # Remove this condition after publishxrpms for x86_^4 got updated if ((package == "openjdk" or package == "openjre") and platform.machine() == "aarch64"): package = package + "8" rpmFile = self.findRPMFileInGivenLocation(package, constants.prevPublishXRPMRepo) else: rpmFile = self.findRPMFileInGivenLocation(package, constants.prevPublishRPMRepo) if rpmFile is None: self.logger.error("Unable to find rpm "+ package + " in current and previous versions") raise Exception("Input Error") rpmFiles += " " + rpmFile packages += " " + package self.logger.debug("Installing custom rpms:" + packages) cmd = (self.rpmCommand + " -i -v --nodeps --noorder --force --root " + chrootID + " --define \'_dbpath /var/lib/rpm\' " + rpmFiles) retVal = cmdUtils.runCommandInShell(cmd, self.logPath + "/install_custom_toolchain_rpms.log") if not retVal: self.logger.debug("Command Executed:" + cmd) self.logger.error("Installing tool chain failed") raise Exception("RPM installation failed") self.logger.info("Successfully installed all Tool Chain X RPMS")
def _readAlreadyAvailablePackages(self): listAvailablePackages = set() pkgUtils = PackageUtils(self.logName, self.logPath) listPackages = SPECS.getData().getListPackages() for package in listPackages: for version in SPECS.getData().getVersions(package): # Mark package available only if all subpackages are available packageIsAlreadyBuilt=True listRPMPackages = SPECS.getData().getRPMPackages(package, version) for rpmPkg in listRPMPackages: if pkgUtils.findRPMFile(rpmPkg, version) is None: packageIsAlreadyBuilt=False break; if packageIsAlreadyBuilt: listAvailablePackages.add(package+"-"+version) return listAvailablePackages
def installPackage(self, package, chrootID, destLogPath, listInstalledPackages): if package in listInstalledPackages: return self.installDependentRunTimePackages(package, chrootID, destLogPath, listInstalledPackages) pkgUtils = PackageUtils(self.logName, self.logPath) noDeps = False if self.mapPackageToCycles.has_key(package): noDeps = True if package in self.listNodepsPackages: noDeps = True if package in constants.noDepsPackageList: noDeps = True pkgUtils.installRPM(package, chrootID, noDeps, destLogPath) listInstalledPackages.append(package) self.logger.info("Installed the package:" + package)
def readAlreadyAvailablePackages(self): listAvailablePackages=[] listRPMFiles=[] listDirectorys=[] listDirectorys.append(constants.rpmPath) while len(listDirectorys) > 0: dirPath=listDirectorys.pop() for dirEntry in os.listdir(dirPath): dirEntryPath = os.path.join(dirPath, dirEntry) if os.path.isfile(dirEntryPath) and dirEntryPath.endswith(".rpm"): listRPMFiles.append(dirEntryPath) elif os.path.isdir(dirEntryPath): listDirectorys.append(dirEntryPath) pkgUtils = PackageUtils(self.logName,self.logPath) for rpmfile in listRPMFiles: package = pkgUtils.findPackageNameFromRPMFile(rpmfile) listAvailablePackages.append(package) return listAvailablePackages
def readAlreadyAvailablePackages(self): listAvailablePackages = [] listFoundRPMPackages = [] listRPMFiles = [] listDirectorys = [] listDirectorys.append(constants.rpmPath) if constants.inputRPMSPath is not None: listDirectorys.append(constants.inputRPMSPath) while len(listDirectorys) > 0: dirPath = listDirectorys.pop() for dirEntry in os.listdir(dirPath): dirEntryPath = os.path.join(dirPath, dirEntry) if os.path.isfile(dirEntryPath) and dirEntryPath.endswith( ".rpm"): listRPMFiles.append(dirEntryPath) elif os.path.isdir(dirEntryPath): listDirectorys.append(dirEntryPath) pkgUtils = PackageUtils(self.logName, self.logPath) for rpmfile in listRPMFiles: package, version, release = pkgUtils.findPackageInfoFromRPMFile( rpmfile) if constants.specData.isRPMPackage(package): specVersion = constants.specData.getVersion(package) specRelease = constants.specData.getRelease(package) if version == specVersion and release == specRelease: listFoundRPMPackages.append(package) #Mark package available only if all sub packages are available for package in listFoundRPMPackages: basePkg = constants.specData.getSpecName(package) if basePkg in listAvailablePackages: continue listRPMPackages = constants.specData.getRPMPackages(basePkg) packageIsAlreadyBuilt = True for rpmpkg in listRPMPackages: if rpmpkg not in listFoundRPMPackages: packageIsAlreadyBuilt = False if packageIsAlreadyBuilt: listAvailablePackages.append(package) self.logger.info("List of Already built packages") self.logger.info(listAvailablePackages) return listAvailablePackages
def buildCoreToolChainPackages(self): self.logger.info("Step 1 : Building the core toolchain packages.....") self.logger.info(constants.listCoreToolChainPackages) self.logger.info("") chrootID = None pkgCount = 0 try: pkgUtils = PackageUtils(self.logName, self.logPath) coreToolChainYetToBuild = [] for package in constants.listCoreToolChainPackages: version = SPECS.getData().getHighestVersion(package) rpmPkg = pkgUtils.findRPMFileForGivenPackage(package, version) if rpmPkg is not None: continue else: coreToolChainYetToBuild.append(package) if coreToolChainYetToBuild: self.logger.info( "The following core toolchain packages need to be built :") self.logger.info(coreToolChainYetToBuild) else: self.logger.info( "Core toolchain packages are already available") for package in coreToolChainYetToBuild: version = SPECS.getData().getHighestVersion(package) self.logger.debug("Building core toolchain package : " + package) chrUtils = ChrootUtils(self.logName, self.logPath) chrootName = "build-" + package destLogPath = constants.logPath + "/build-" + package if not os.path.isdir(destLogPath): cmdUtils = CommandUtils() cmdUtils.runCommandInShell("mkdir -p " + destLogPath) returnVal, chrootID = chrUtils.createChroot(chrootName) if not returnVal: self.logger.error("Creating chroot failed") raise Exception("creating chroot failed") self.installToolChainRPMS(package, version, chrootID, destLogPath) pkgUtils.adjustGCCSpecs(package, version, chrootID, destLogPath) pkgUtils.buildRPMSForGivenPackage(package, version, chrootID, destLogPath) pkgCount += 1 chrUtils.destroyChroot(chrootID) chrootID = None self.logger.debug("Successfully built toolchain") self.logger.info("-" * 45 + "\n") if chrootID is not None: chrUtils.destroyChroot(chrootID) except Exception as e: self.logger.error("Unable to build tool chain.") # print stacktrace traceback.print_exc() raise e return pkgCount
def readAlreadyAvailablePackages(self): listAvailablePackages = [] listRPMFiles = [] listDirectorys = [] listDirectorys.append(constants.rpmPath) while len(listDirectorys) > 0: dirPath = listDirectorys.pop() for dirEntry in os.listdir(dirPath): dirEntryPath = os.path.join(dirPath, dirEntry) if os.path.isfile(dirEntryPath) and dirEntryPath.endswith( ".rpm"): listRPMFiles.append(dirEntryPath) elif os.path.isdir(dirEntryPath): listDirectorys.append(dirEntryPath) pkgUtils = PackageUtils(self.logName, self.logPath) for rpmfile in listRPMFiles: package = pkgUtils.findPackageNameFromRPMFile(rpmfile) listAvailablePackages.append(package) return listAvailablePackages
def installToolChainRPMSinContainer(self, containerID): self.logger.info("Installing tool-chain RPMS in container: " + containerID.short_id) rpmFiles = "" packages = "" pkgUtils = PackageUtils(self.logName, self.logPath) for package in constants.listToolChainRPMPkgsToInstall: rpmFile = pkgUtils.findRPMFileForGivenPackage(package) if rpmFile is None: # sqlite-autoconf package was renamed, but it still published as sqlite-autoconf # if (package == "sqlite") and (platform.machine() == "x86_64"): # package = "sqlite-autoconf" rpmFile = self.findRPMFileInGivenLocation(package, constants.prevPublishRPMRepo) if rpmFile is None: if package in constants.listOfRPMsProvidedAfterBuild: self.logger.info("No old version of " + package + " exists, skip until the new version is built") continue self.logger.error("Unable to find rpm " + package + " in current and previous versions") raise Exception("Input Error") if rpmFile.find("stage/PUBLISHRPMS"): rpmFile = rpmFile.replace(constants.prevPublishRPMRepo, "/publishrpms") if rpmFile.find("stage/PUBLISHXRPMS"): rpmFile = rpmFile.replace(constants.prevPublishXRPMRepo, "/publishxrpms") if rpmFile.find("stage/RPMS"): rpmFile = rpmFile.replace(constants.rpmPath, constants.topDirPath + "/RPMS") rpmFiles += " " + rpmFile packages += " " + package self.logger.debug("Installing tool-chain rpms: " + packages) cmd = "/usr/bin/bash -l -c '/usr/bin/rpm -Uvh --force --nodeps " + rpmFiles + "'" self.logger.info("VDBG-TCU-installToolChainRPMSinContainer: Installing rpms cmd: " + cmd) tcInstallLog = containerID.exec_run(cmd) # TODO: Find a way to collect exit status of the command that was run. if not tcInstallLog: self.logger.error("Installing tool chain in container failed") raise Exception("RPM installation in container failed") self.logger.info(tcInstallLog) self.logger.info("Successfully installed default tool-chain RPMS in container: " + containerID.short_id)
def _buildPackage(self): try: self.sandbox.create(self.package + "-" + self.version) tUtils = ToolChainUtils(self.logName, self.logPath) if self.sandbox.hasToolchain(): tUtils.installExtraToolchainRPMS(self.sandbox, self.package, self.version) else: tUtils.installToolchainRPMS(self.sandbox, self.package, self.version, availablePackages=self.doneList) self._installDependencies(constants.buildArch) if constants.crossCompiling: self._installDependencies(constants.targetArch) pkgUtils = PackageUtils(self.logName, self.logPath) pkgUtils.adjustGCCSpecs(self.sandbox, self.package, self.version) pkgUtils.buildRPMSForGivenPackage(self.sandbox, self.package, self.version, self.logPath) self.logger.debug("Successfully built the package: " + self.package) except Exception as e: self.logger.error("Failed while building package: " + self.package) self.logger.debug("Sandbox: " + self.sandbox.getID() + " not deleted for debugging.") logFileName = os.path.join(self.logPath, self.package + ".log") fileLog = os.popen('tail -n 100 ' + logFileName).read() self.logger.info(fileLog) raise e if self.sandbox: self.sandbox.destroy()
def readAlreadyAvailablePackages(self): listAvailablePackages=[] listFoundRPMPackages=[] listRPMFiles=[] listDirectorys=[] listDirectorys.append(constants.rpmPath) if constants.inputRPMSPath is not None: listDirectorys.append(constants.inputRPMSPath) while len(listDirectorys) > 0: dirPath=listDirectorys.pop() for dirEntry in os.listdir(dirPath): dirEntryPath = os.path.join(dirPath, dirEntry) if os.path.isfile(dirEntryPath) and dirEntryPath.endswith(".rpm"): listRPMFiles.append(dirEntryPath) elif os.path.isdir(dirEntryPath): listDirectorys.append(dirEntryPath) pkgUtils = PackageUtils(self.logName,self.logPath) for rpmfile in listRPMFiles: package,version,release = pkgUtils.findPackageInfoFromRPMFile(rpmfile) if constants.specData.isRPMPackage(package): specVersion=constants.specData.getVersion(package) specRelease=constants.specData.getRelease(package) if version == specVersion and release == specRelease: listFoundRPMPackages.append(package) #Mark package available only if all sub packages are available for package in listFoundRPMPackages: basePkg = constants.specData.getSpecName(package) if basePkg in listAvailablePackages: continue; listRPMPackages = constants.specData.getRPMPackages(basePkg) packageIsAlreadyBuilt = True for rpmpkg in listRPMPackages: if rpmpkg not in listFoundRPMPackages: packageIsAlreadyBuilt = False if packageIsAlreadyBuilt: listAvailablePackages.append(package) self.logger.info("List of Already built packages") self.logger.info(listAvailablePackages) return listAvailablePackages
def buildPackage(self,package): #do not build if RPM is already built #test only if the package is in the testForceRPMS with rpmCheck #build only if the package is not in the testForceRPMS with rpmCheck if self.checkIfPackageIsAlreadyBuilt(package): if not constants.rpmCheck: self.logger.info("Skipping building the package:"+package) return elif constants.rpmCheck and package not in constants.testForceRPMS: self.logger.info("Skipping testing the package:"+package) return #should initialize a logger based on package name chrUtils = ChrootUtils(self.logName,self.logPath) chrootName="build-"+package chrootID=None isToolChainPackage=False if package in constants.listToolChainPackages: isToolChainPackage=True try: chrootID = self.prepareBuildRoot(chrootName,isToolChainPackage) destLogPath=constants.logPath+"/build-"+package if not os.path.isdir(destLogPath): cmdUtils = CommandUtils() cmdUtils.runCommandInShell("mkdir -p "+destLogPath) listInstalledPackages=self.findInstalledPackages(chrootID) listDependentPackages=self.findBuildTimeRequiredPackages(package) if constants.rpmCheck and package in constants.testForceRPMS: testPackages=set(constants.listMakeCheckRPMPkgtoInstall)-set(listInstalledPackages)-set([package]) listDependentPackages.extend(testPackages) pkgUtils = PackageUtils(self.logName,self.logPath) if len(listDependentPackages) != 0: self.logger.info("Installing the build time dependent packages......") for pkg in listDependentPackages: self.installPackage(pkgUtils, pkg,chrootID,destLogPath,listInstalledPackages) pkgUtils.installRPMSInAOneShot(chrootID,destLogPath) self.logger.info("Finished installing the build time dependent packages......") pkgUtils.adjustGCCSpecs(package, chrootID, destLogPath) pkgUtils.buildRPMSForGivenPackage(package,chrootID,self.listBuildOptionPackages,self.pkgBuildOptionFile,destLogPath) self.logger.info("Successfully built the package:"+package) except Exception as e: self.logger.error("Failed while building package:" + package) self.logger.debug("Chroot with ID: " + chrootID + " not deleted for debugging.") logFileName = os.path.join(destLogPath, package + ".log") fileLog = os.popen('tail -n 100 ' + logFileName).read() self.logger.debug(fileLog) raise e if chrootID is not None: chrUtils.destroyChroot(chrootID)
def installToolChainRPMS(self, chrootID, packageName, logPath=None): if logPath is None: logPath = self.logPath cmdUtils = CommandUtils() self.prepareBuildRoot(chrootID) self.logger.info("Installing Tool Chain RPMS.......") rpmFiles = "" packages = "" for package in constants.listToolChainRPMsToInstall: pkgUtils = PackageUtils(self.logName, self.logPath) rpmFile = None if constants.rpmCheck: rpmFile = pkgUtils.findRPMFileForGivenPackage(package) else: if (packageName not in constants.listToolChainRPMsToInstall or constants.listToolChainRPMsToInstall.index(packageName) > constants.listToolChainRPMsToInstall.index(package)): rpmFile = pkgUtils.findRPMFileForGivenPackage(package) if rpmFile is None: # sqlite-autoconf package was renamed, but it still published as sqlite-autoconf if (package == "sqlite") and (platform.machine() == "x86_64"): package = "sqlite-autoconf" rpmFile = self.findRPMFileInGivenLocation(package, constants.prevPublishRPMRepo) if rpmFile is None: if package in constants.listOfRPMsProvidedAfterBuild: self.logger.info("No old version of " + package + " exists, skip until the new version is built") continue self.logger.error("Unable to find rpm " + package + " in current and previous versions") raise Exception("Input Error") rpmFiles += " " + rpmFile packages += " " + package self.logger.debug("Installing toolchain rpms:" + packages) cmd = (self.rpmCommand + " -i -v --nodeps --noorder --force --root " + chrootID +" --define \'_dbpath /var/lib/rpm\' "+ rpmFiles) retVal = cmdUtils.runCommandInShell(cmd, logPath + "/install_toolchain_rpms.log") if not retVal: self.logger.debug("Command Executed:" + cmd) self.logger.error("Installing tool chain failed") raise Exception("RPM installation failed") self.logger.info("Successfully installed default Tool Chain RPMS in Chroot:" + chrootID) print("Building Package: ".format(packageName)) print(constants.perPackageToolChain) if packageName in constants.perPackageToolChain: print(constants.perPackageToolChain[packageName]) self.installCustomToolChainRPMS(chrootID, constants.perPackageToolChain[packageName], packageName) #convert rpm db to lmdb (if required) convertLogFile = self.logPath+"/rpmdbconvert.log" pkgUtilsConvert=PackageUtils(self.logName,self.logPath) pkgUtilsConvert.ensureLMDBForRPMDB(chrootID, convertLogFile)
def _buildPackage(self): try: self.sandbox.create(self.package + "-" + self.version) tUtils = ToolChainUtils(self.logName, self.logPath) if self.sandbox.hasToolchain(): tUtils.installExtraToolchainRPMS(self.sandbox, self.package, self.version) else: tUtils.installToolchainRPMS(self.sandbox, self.package, self.version, availablePackages=self.doneList) listDependentPackages, listTestPackages, listInstalledPackages, listInstalledRPMs = ( self._findDependentPackagesAndInstalledRPM(self.sandbox)) pkgUtils = PackageUtils(self.logName, self.logPath) if listDependentPackages: self.logger.debug("Installing the build time dependent packages......") for pkg in listDependentPackages: packageName, packageVersion = StringUtils.splitPackageNameAndVersion(pkg) self._installPackage(pkgUtils, packageName, packageVersion, self.sandbox, self.logPath,listInstalledPackages, listInstalledRPMs) for pkg in listTestPackages: flag = False packageName, packageVersion = StringUtils.splitPackageNameAndVersion(pkg) for depPkg in listDependentPackages: depPackageName, depPackageVersion = StringUtils.splitPackageNameAndVersion(depPkg) if depPackageName == packageName: flag = True break; if flag == False: self._installPackage(pkgUtils, packageName,packageVersion, self.sandbox, self.logPath,listInstalledPackages, listInstalledRPMs) pkgUtils.installRPMSInOneShot(self.sandbox) self.logger.debug("Finished installing the build time dependent packages....") pkgUtils.adjustGCCSpecs(self.sandbox, self.package, self.version) pkgUtils.buildRPMSForGivenPackage(self.sandbox, self.package, self.version, self.logPath) self.logger.debug("Successfully built the package: " + self.package) except Exception as e: self.logger.error("Failed while building package: " + self.package) self.logger.debug("Sandbox: " + self.sandbox.getID() + " not deleted for debugging.") logFileName = os.path.join(self.logPath, self.package + ".log") fileLog = os.popen('tail -n 100 ' + logFileName).read() self.logger.info(fileLog) raise e if self.sandbox: self.sandbox.destroy()
def _buildPackage(self): #do not build if RPM is already built #test only if the package is in the testForceRPMS with rpmCheck #build only if the package is not in the testForceRPMS with rpmCheck if self._checkIfPackageIsAlreadyBuilt(): if not constants.rpmCheck: self.logger.info("Skipping building the package:" + self.package) return elif constants.rpmCheck and self.package not in constants.testForceRPMS: self.logger.info("Skipping testing the package:" + self.package) return chrUtils = ChrootUtils(self.logName, self.logPath) chrootID = None try: chrootID = self._prepareBuildRoot() listDependentPackages, listInstalledPackages, listInstalledRPMs = ( self._findDependentPackagesAndInstalledRPM(chrootID)) pkgUtils = PackageUtils(self.logName, self.logPath) if listDependentPackages: self.logger.info("Installing the build time dependent packages......") for pkg in listDependentPackages: self._installPackage(pkgUtils, pkg, chrootID, self.logPath, listInstalledPackages, listInstalledRPMs) pkgUtils.installRPMSInAOneShot(chrootID, self.logPath) self.logger.info("Finished installing the build time dependent packages....") pkgUtils.adjustGCCSpecs(self.package, chrootID, self.logPath) pkgUtils.buildRPMSForGivenPackage(self.package, chrootID, self.logPath) self.logger.info("Successfully built the package:" + self.package) except Exception as e: self.logger.error("Failed while building package:" + self.package) self.logger.debug("Chroot with ID: " + chrootID + " not deleted for debugging.") logFileName = os.path.join(self.logPath, self.package + ".log") fileLog = os.popen('tail -n 100 ' + logFileName).read() self.logger.debug(fileLog) raise e if chrootID is not None: chrUtils.destroyChroot(chrootID)
def _buildPackage(self): #do not build if RPM is already built #test only if the package is in the testForceRPMS with rpmCheck #build only if the package is not in the testForceRPMS with rpmCheck if self._checkIfPackageIsAlreadyBuilt(): if not constants.rpmCheck: self.logger.info("Skipping building the package:" + self.package) return elif constants.rpmCheck and self.package not in constants.testForceRPMS: self.logger.info("Skipping testing the package:" + self.package) return #should initialize a logger based on package name containerTaskName = "build-" + self.package containerID = None chrootID = None isToolChainPackage = False if self.package in constants.listToolChainPackages: isToolChainPackage = True destLogPath = constants.logPath + "/build-" + self.package try: containerID, chrootID = self._prepareBuildContainer( containerTaskName, self.package, isToolChainPackage) tcUtils = ToolChainUtils(self.logName, self.logPath) if self.package in constants.perPackageToolChain: self.logger.debug(constants.perPackageToolChain[self.package]) tcUtils.installCustomToolChainRPMSinContainer( containerID, constants.perPackageToolChain[self.package], self.package) listDependentPackages, listInstalledPackages, listInstalledRPMs = ( self._findDependentPackagesAndInstalledRPM(containerID)) pkgUtils = PackageUtils(self.logName, self.logPath) if listDependentPackages: self.logger.info("BuildContainer-buildPackage: " + "Installing dependent packages..") self.logger.info(listDependentPackages) for pkg in listDependentPackages: self._installPackage(pkgUtils, pkg, containerID, destLogPath, listInstalledPackages, listInstalledRPMs) pkgUtils.installRPMSInAOneShotInContainer(containerID, destLogPath) self.logger.info("Finished installing the build time dependent packages....") self.logger.info("BuildContainer-buildPackage: Start building the package: " + self.package) pkgUtils.adjustGCCSpecsInContainer(self.package, containerID, destLogPath) pkgUtils.buildRPMSForGivenPackageInContainer( self.package, containerID, destLogPath) self.logger.info("BuildContainer-buildPackage: Successfully built the package: " + self.package) except Exception as e: self.logger.error("Failed while building package:" + self.package) if containerID is not None: self.logger.debug("Container " + containerID.short_id + " retained for debugging.") logFileName = os.path.join(destLogPath, self.package + ".log") fileLog = os.popen('tail -n 20 ' + logFileName).read() self.logger.debug(fileLog) raise e # Remove the container if containerID is not None: containerID.remove(force=True) # Remove the dummy chroot if chrootID is not None: chrUtils = ChrootUtils(self.logName, self.logPath) chrUtils.destroyChroot(chrootID)