def _getListNextPackagesReadyToBuild(): for pkg in Scheduler.listOfPackagesToBuild: if pkg in Scheduler.listOfPackagesCurrentlyBuilding: continue listRequiredSubPackages = list(set(SPECS.getData().getBuildRequiresForPkg(pkg) + \ SPECS.getData().getRequiresAllForPkg(pkg))) # extend to full Requires tree for p in listRequiredSubPackages: reqs = SPECS.getData().getRequiresAllForPkg(p) for r in reqs: if r not in listRequiredSubPackages: listRequiredSubPackages.append(r) # convert subpackages to basepkg listRequiredPackages = set() for p in listRequiredSubPackages: listRequiredPackages.add(SPECS.getData().getBasePkg(p)) canBuild = True for reqPkg in listRequiredPackages: if reqPkg not in Scheduler.listOfAlreadyBuiltPackages: canBuild = False break if canBuild: Scheduler.listOfPackagesNextToBuild.put((-Scheduler._getPriority(pkg), pkg)) Scheduler.logger.debug("Adding " + pkg + " to the schedule list")
def findRPMFileForGivenPackage(self, package,version="*", index=0): cmdUtils = CommandUtils() release = "*" if version == "*": version = SPECS.getData().getVersion(package, index) release = SPECS.getData().getRelease(package, index) listFoundRPMFiles = sum([cmdUtils.findFile(package + "-" + version + "-" + release + "." + platform.machine()+".rpm", constants.rpmPath), cmdUtils.findFile(package + "-" + version + "-" + release + ".noarch.rpm", constants.rpmPath)], []) if constants.inputRPMSPath is not None: listFoundRPMFiles = sum([cmdUtils.findFile(package + "-" + version + "-" + release + "." + platform.machine()+".rpm", constants.inputRPMSPath), cmdUtils.findFile(package + "-" + version + "-" + release + ".noarch.rpm", constants.inputRPMSPath)], listFoundRPMFiles) if len(listFoundRPMFiles) == 1: return listFoundRPMFiles[0] if len(listFoundRPMFiles) == 0: return None if len(listFoundRPMFiles) > 1: self.logger.error("Found multiple rpm files for given package in rpm directory." + "Unable to determine the rpm file for package:" + package) raise Exception("Multiple rpm files found")
def loadPackagesData(self): listPackages = SPECS.getData().getListPackages() listPackages.sort() cmdUtils = CommandUtils() for package in listPackages: for version in SPECS.getData().getVersions(package): release = SPECS.getData().getRelease(package, version) listRPMPackages = SPECS.getData().getRPMPackages(package, version) srpmFileName = package + "-" + version + "-" + release + ".src.rpm" srpmFiles = cmdUtils.findFile(srpmFileName, constants.sourceRpmPath) srpmFile = None if len(srpmFiles) == 1: srpmFile = srpmFiles[0] debugrpmFileName = package + "-debuginfo-" + version + "-" + release + "*" debugrpmFiles = cmdUtils.findFile(debugrpmFileName, constants.rpmPath) debugrpmFile = None if len(debugrpmFiles) == 1: debugrpmFile = debugrpmFiles[0] pkgUtils = PackageUtils(self.logName, self.logPath) for rpmPkg in listRPMPackages: rpmFile = pkgUtils.findRPMFile(rpmPkg, version) if rpmFile is not None: listPkgAttributes = {"sourcerpm":srpmFile, "rpm":rpmFile, "debugrpm":debugrpmFile} self.pkgList[rpmPkg+"-"+version] = listPkgAttributes self.logger.debug("Added " + rpmPkg + "-" + version + " to the package info json") else: self.logger.debug("Missing rpm file for package:" + rpmPkg)
def _verifyShaAndGetSourcePath(self, source, package, version): cmdUtils = CommandUtils() # Fetch/verify sources if sha1 not None. sha1 = SPECS.getData().getSHA1(package, version, source) if sha1 is not None: PullSources.get(package, source, sha1, constants.sourcePath, constants.getPullSourcesURLs(package), self.logger) sourcePath = cmdUtils.findFile(source, constants.sourcePath) if not sourcePath: sourcePath = cmdUtils.findFile(source, os.path.dirname(SPECS.getData().getSpecFile(package, version))) if not sourcePath: if sha1 is None: self.logger.error("No sha1 found or missing source for " + source) raise Exception("No sha1 found or missing source for " + source) else: self.logger.error("Missing source: " + source + ". Cannot find sources for package: " + package) raise Exception("Missing source") else: if sha1 is None: self.logger.error("No sha1 found for "+source) raise Exception("No sha1 found") if len(sourcePath) > 1: self.logger.error("Multiple sources found for source:" + source + "\n" + ",".join(sourcePath) +"\nUnable to determine one.") raise Exception("Multiple sources found") return sourcePath
def buildRPMSForGivenPackage(self, package, chrootID, destLogPath=None, index=0): self.logger.info("Building rpm's for package:" + package) listSourcesFiles = SPECS.getData().getSources(package, index) listPatchFiles = SPECS.getData().getPatches(package, index) specFile = SPECS.getData().getSpecFile(package, index) specName = SPECS.getData().getSpecName(package) + ".spec" chrootSourcePath = chrootID + constants.topDirPath + "/SOURCES/" chrootSpecPath = constants.topDirPath + "/SPECS/" chrootLogsFilePath = chrootID + constants.topDirPath + "/LOGS/" + package + ".log" chrootCmd = self.runInChrootCommand + " " + chrootID shutil.copyfile(specFile, chrootID + chrootSpecPath + specName) # FIXME: some sources are located in SPECS/.. how to mount? # if os.geteuid()==0: self._copySourcesTobuildroot(listSourcesFiles, package, chrootSourcePath, index) self._copySourcesTobuildroot(listPatchFiles, package, chrootSourcePath, index) macros = [] listAdditionalFiles, macros = self._getAdditionalBuildFiles(package) self._copyAdditionalBuildFiles(listAdditionalFiles, chrootID) #Adding rpm macros listRPMMacros = constants.userDefinedMacros for macroName, value in listRPMMacros.items(): macros.append(macroName + " " + value) listRPMFiles = [] listSRPMFiles = [] try: listRPMFiles, listSRPMFiles = self._buildRPM(chrootSpecPath + specName, chrootLogsFilePath, chrootCmd, package, macros) self.logger.info("Successfully built rpm:" + package) except Exception as e: self.logger.error("Failed while building rpm:" + package) raise e finally: if destLogPath is not None: if (constants.rpmCheck and package in constants.testForceRPMS and SPECS.getData().isCheckAvailable(package)): cmd = ("sed -i '/^Executing(%check):/,/^Processing files:/{//!b};d' " + chrootLogsFilePath) logFile = destLogPath + "/adjustTestFile.log" returnVal = CommandUtils().runCommandInShell(cmd, logFile) testLogFile = destLogPath + "/" + package + "-test.log" shutil.copyfile(chrootLogsFilePath, testLogFile) else: shutil.copy2(chrootLogsFilePath, destLogPath) self.logger.info("RPM build is successful") for rpmFile in listRPMFiles: self._copyRPM(chrootID + "/" + rpmFile, constants.rpmPath) for srpmFile in listSRPMFiles: srpmDestFile = self._copyRPM(chrootID + "/" + srpmFile, constants.sourceRpmPath)
def updateLevels(self, mapDependencies, inPkg, parent, level): listPackages = SPECS.getData().getPackagesForPkg(inPkg) for depPkg in SPECS.getData().getRequiresForPkg(inPkg): if depPkg in listPackages: continue if depPkg in mapDependencies and mapDependencies[depPkg] < level + 1: mapDependencies[depPkg] = level + 1 parent[depPkg] = inPkg self.updateLevels(mapDependencies, depPkg, parent, mapDependencies[depPkg])
def getBasePackagesRequired(self, pkg): listBasePackagesRequired=[] listPackagesRequired = SPECS.getData().getBuildRequiresForPkg(pkg) listPackagesRequired.extend(SPECS.getData().getRequiresAllForPkg(pkg)) for p in listPackagesRequired: basePkg = SPECS.getData().getBasePkg(p) if basePkg not in listBasePackagesRequired: listBasePackagesRequired.append(basePkg) return listBasePackagesRequired
def _checkIfPackageIsAlreadyBuilt(self): basePkg = SPECS.getData().getSpecName(self.package) listRPMPackages = SPECS.getData().getRPMPackages(basePkg) packageIsAlreadyBuilt = True pkgUtils = PackageUtils(self.logName, self.logPath) for pkg in listRPMPackages: if pkgUtils.findRPMFileForGivenPackage(pkg) is None: packageIsAlreadyBuilt = False break return packageIsAlreadyBuilt
def findTotalWhoNeeds(self, depList, whoNeeds): while depList: pkg = depList.pop(0) for depPackage in SPECS.getData().getListPackages(): for version in SPECS.getData().getVersions(depPackage): depBasePkg = depPackage+"-"+version if depBasePkg in whoNeeds: continue if pkg in self.getBasePackagesRequired(depBasePkg): whoNeeds.append(depBasePkg) if depBasePkg not in depList: depList.append(depBasePkg)
def calculateSpecDependency(self, inputPackages, mapDependencies, parent): depQue = queue.Queue() for package in inputPackages: if SPECS.getData().isRPMPackage(package): version = SPECS.getData().getHighestVersion(package) pkg = package+"-"+version if pkg not in mapDependencies: mapDependencies[pkg] = 0 parent[pkg] = "" depQue.put(pkg) self.findTotalRequires(mapDependencies, depQue, parent) else: self.logger.info("Could not find spec for " + package)
def buildSRPMList(srpmPath, yamlDir, blackListPkgs, dist_tag, logger, singleFile=True): cmdUtils = CommandUtils() yamlSrpmDir = os.path.join(yamlDir, "yaml_srpms") if not os.path.isdir(yamlSrpmDir): cmdUtils.runCommandInShell("mkdir -p " + yamlSrpmDir) if singleFile: yamlFile = open(yamlSrpmDir + "/srpm_list.yaml", "w") listPackages = SPECS.getData().getListPackages() listPackages.sort() for package in listPackages: if package in blackListPkgs: continue ossname = package for ossversion in SPECS.getData().getVersions(package): ossrelease = SPECS.getData().getRelease(package, ossversion) srpm_file_name = "%s-%s-%s%s.src.rpm" % (ossname, ossversion, ossrelease, dist_tag) logger.info("srpm name is %s" % (srpm_file_name)) listFoundSRPMFiles = cmdUtils.findFile(srpm_file_name, srpmPath) srpmName = None if len(listFoundSRPMFiles) == 1: srpmFullPath = listFoundSRPMFiles[0] srpmName = os.path.basename(srpmFullPath) cpcmd = "cp " + srpmFullPath + " " + yamlSrpmDir + "/" returnVal = cmdUtils.runCommandInShell(cpcmd) if returnVal != 0: logger.error("Copy SRPM File is failed for package:" + ossname) else: logger.error("SRPM file is not found:" + ossname) if not singleFile: yamlFile = open(yamlSrpmDir + "/" + ossname + "-" + ossversion + "-" + ossrelease + ".yaml", "w") yamlFile.write("baseos:" + ossname + ":" + ossversion + "-" + ossrelease + dist_tag + ":\n") yamlFile.write(" repository: BaseOS\n") yamlFile.write(" name: '" + ossname + "'\n") yamlFile.write(" version: '" + ossversion + "-" + ossrelease + dist_tag +"'\n") yamlFile.write(" url: 'http://www.vmware.com'\n") yamlFile.write(" baseos-style: rpm\n") yamlFile.write(" baseos-source: '" + str(srpmName) + "'\n") yamlFile.write(" baseos-osname: 'photon'\n") yamlFile.write("\n") if not singleFile: yamlFile.close() if singleFile: yamlFile.close() logger.debug("Generated SRPM yaml files for all packages")
def verifyShaAndGetSourcePath(self, source, package): cmdUtils = CommandUtils() # Fetch/verify sources if sha1 not None. sha1 = SPECS.getData().getSHA1(package, source) if sha1 is not None: PullSources.get(source, sha1, constants.sourcePath, constants.pullsourcesConfig, self.logger) sourcePath = cmdUtils.findFile(source, constants.sourcePath) if sourcePath is None or len(sourcePath) == 0: sourcePath = cmdUtils.findFile(source, constants.specPath) if sourcePath is None or len(sourcePath) == 0: if sha1 is None: self.logger.error("No sha1 found or missing source for " + source) raise Exception("No sha1 found or missing source for " + source) else: self.logger.error("Missing source: " + source + ". Cannot find sources for package: " + package) raise Exception("Missing source") else: if sha1 is None: self.logger.error("No sha1 found for " + source) raise Exception("No sha1 found") if len(sourcePath) > 1: self.logger.error("Multiple sources found for source:" + source + "\n" + ",".join(sourcePath) + "\nUnable to determine one.") raise Exception("Multiple sources found") return sourcePath
def _calculateParams(self, listPackages): self.mapCyclesToPackageList.clear() self.mapPackageToCycle.clear() self.sortedPackageList = [] self.listOfPackagesAlreadyBuilt = self._readAlreadyAvailablePackages() updateBuiltRPMSList = False while not updateBuiltRPMSList: updateBuiltRPMSList = True listOfPackagesAlreadyBuilt = list(self.listOfPackagesAlreadyBuilt) for pkg in listOfPackagesAlreadyBuilt: listDependentRpmPackages = SPECS.getData().getRequiresAllForPackage(pkg) needToRebuild = False for dependentPkg in listDependentRpmPackages: if dependentPkg not in self.listOfPackagesAlreadyBuilt: needToRebuild = True updateBuiltRPMSList = False if needToRebuild: self.listOfPackagesAlreadyBuilt.remove(pkg) listPackagesToBuild = copy.copy(listPackages) for pkg in listPackages: if (pkg in self.listOfPackagesAlreadyBuilt and not constants.rpmCheck): listPackagesToBuild.remove(pkg) if not self._readPackageBuildData(listPackagesToBuild): return False return True
def adjustGCCSpecs(self, package, chrootID, logPath): opt = " " + SPECS.getData().getSecurityHardeningOption(package) cmdUtils = CommandUtils() cpcmd = "cp " + self.adjustGCCSpecScript + " " + chrootID + "/tmp/" + self.adjustGCCSpecScript cmd = "/tmp/" + self.adjustGCCSpecScript + opt logFile = logPath + "/adjustGCCSpecScript.log" chrootCmd = self.runInChrootCommand + " " + chrootID returnVal = cmdUtils.runCommandInShell(cpcmd, logFile) if not returnVal: self.logger.error("Error during copying the file adjust gcc spec") raise Exception("Failed while copying adjust gcc spec file") returnVal = cmdUtils.runCommandInShell(cmd, logFile, chrootCmd) if returnVal: return self.logger.debug( cmdUtils.runCommandInShell2("ls -la " + chrootID + "/tmp/" + self.adjustGCCSpecScript)) self.logger.debug( cmdUtils.runCommandInShell2("lsof " + chrootID + "/tmp/" + self.adjustGCCSpecScript)) self.logger.debug(cmdUtils.runCommandInShell2("ps ax")) self.logger.error("Failed while adjusting gcc specs") raise Exception("Failed while adjusting gcc specs")
def installExtraToolchainRPMS(self, sandbox, packageName, packageVersion): listOfToolChainPkgs = SPECS.getData().getExtraBuildRequiresForPackage( packageName, packageVersion) if not listOfToolChainPkgs: return self.logger.debug("Installing package specific toolchain RPMS for " + packageName + ": " + str(listOfToolChainPkgs)) rpmFiles = "" packages = "" for package in listOfToolChainPkgs: pkgUtils = PackageUtils(self.logName, self.logPath) if re.match("openjre*", packageName) is not None or re.match( "openjdk*", packageName): path = constants.prevPublishXRPMRepo sandboxPath = "/publishxrpms" else: path = constants.prevPublishRPMRepo sandboxPath = "/publishrpms" rpmFile = self._findPublishedRPM(package, path) if rpmFile is None: self.logger.error("Unable to find rpm " + package + " in current and previous versions") raise Exception("Input Error") rpmFiles += " " + rpmFile.replace(path, sandboxPath) packages += " " + package self.logger.debug("Installing custom rpms:" + packages) cmd = (self.rpmCommand + " -i -v --nodeps --noorder --force " + rpmFiles) retVal = sandbox.run(cmd, logfn=self.logger.debug) if retVal != 0: self.logger.debug("Command Executed:" + cmd) self.logger.error("Installing custom toolchains failed") raise Exception("RPM installation failed")
def _calculateParams(self, listPackages): self.mapCyclesToPackageList.clear() self.mapPackageToCycle.clear() self.sortedPackageList = [] self.listOfPackagesAlreadyBuilt = list( self._readAlreadyAvailablePackages()) updateBuiltRPMSList = False while not updateBuiltRPMSList: updateBuiltRPMSList = True listOfPackagesAlreadyBuilt = self.listOfPackagesAlreadyBuilt for pkg in listOfPackagesAlreadyBuilt: packageName, packageVersion = StringUtils.splitPackageNameAndVersion( pkg) listDependentRpmPackages = SPECS.getData( ).getRequiresAllForPackage(packageName, packageVersion) needToRebuild = False for dependentPkg in listDependentRpmPackages: if dependentPkg not in self.listOfPackagesAlreadyBuilt: needToRebuild = True updateBuiltRPMSList = False if needToRebuild: self.listOfPackagesAlreadyBuilt.remove(pkg) listPackagesToBuild = copy.copy(listPackages) for pkg in listPackages: if (pkg in self.listOfPackagesAlreadyBuilt and not constants.rpmCheck): listPackagesToBuild.remove(pkg) if not self._readPackageBuildData(listPackagesToBuild): return False return True
def buildCoreToolChainPackages(self): self.logger.info("Step 1 : Building the core toolchain packages.....") self.logger.info(constants.listCoreToolChainPackages) self.logger.info("") chroot = None pkgCount = 0 try: pkgUtils = PackageUtils(self.logName, self.logPath) coreToolChainYetToBuild = [] for package in constants.listCoreToolChainPackages: version = SPECS.getData().getHighestVersion(package) rpmPkg = pkgUtils.findRPMFile(package, version) if rpmPkg is not None: continue else: coreToolChainYetToBuild.append(package) if coreToolChainYetToBuild: self.logger.info( "The following core toolchain packages need to be built :") self.logger.info(coreToolChainYetToBuild) else: self.logger.info( "Core toolchain packages are already available") for package in coreToolChainYetToBuild: self.logger.debug("Building core toolchain package : " + package) version = SPECS.getData().getHighestVersion(package) destLogPath = constants.logPath + "/" + package + "-" + version if not os.path.isdir(destLogPath): CommandUtils.runCommandInShell("mkdir -p " + destLogPath) chroot = Chroot(self.logger) chroot.create(package + "-" + version) self.installToolchainRPMS(chroot, package, version) pkgUtils.adjustGCCSpecs(chroot, package, version) pkgUtils.buildRPMSForGivenPackage(chroot, package, version, destLogPath) pkgCount += 1 chroot.destroy() self.logger.debug("Successfully built toolchain") self.logger.info("-" * 45 + "\n") except Exception as e: self.logger.error("Unable to build toolchain.") # print stacktrace traceback.print_exc() raise e return pkgCount
def _buildGivenPackages(self, listPackages, buildThreads): # Extend listPackages from ["name1", "name2",..] to ["name1-vers1", "name2-vers2",..] listPackageNamesAndVersions = set() for pkg in listPackages: base = SPECS.getData().getSpecName(pkg) for version in SPECS.getData().getVersions(base): listPackageNamesAndVersions.add(base + "-" + version) returnVal = self._calculateParams(listPackageNamesAndVersions) if not returnVal: self.logger.error( "Unable to set parameters. Terminating the package manager.") raise Exception("Unable to set parameters") statusEvent = threading.Event() self._initializeScheduler(statusEvent) self._initializeThreadPool(statusEvent) for i in range(0, buildThreads): workerName = "WorkerThread" + str(i) ThreadPool.addWorkerThread(workerName) ThreadPool.startWorkerThread(workerName) statusEvent.wait() Scheduler.stopScheduling = True self.logger.debug("Waiting for all remaining worker threads") ThreadPool.join_all() setFailFlag = False allPackagesBuilt = False if Scheduler.isAnyPackagesFailedToBuild(): setFailFlag = True if Scheduler.isAllPackagesBuilt(): allPackagesBuilt = True if setFailFlag: self.logger.error("Some of the packages failed:") self.logger.error(Scheduler.listOfFailedPackages) raise Exception("Failed during building package") if not setFailFlag: if allPackagesBuilt: self.logger.debug("All packages built successfully") else: self.logger.error("Build stopped unexpectedly.Unknown error.") raise Exception("Unknown error")
def _readAlreadyAvailablePackages(self): listAvailablePackages = set() pkgUtils = PackageUtils(self.logName, self.logPath) listPackages = SPECS.getData().getListPackages() for package in listPackages: for version in SPECS.getData().getVersions(package): # Mark package available only if all subpackages are available packageIsAlreadyBuilt=True listRPMPackages = SPECS.getData().getRPMPackages(package, version) for rpmPkg in listRPMPackages: if pkgUtils.findRPMFile(rpmPkg, version) is None: packageIsAlreadyBuilt=False break; if packageIsAlreadyBuilt: listAvailablePackages.add(package+"-"+version) return listAvailablePackages
def getPackageBuildData(self, listPackages): basePackages = [] for pkg in listPackages: basePackages.append(SPECS.getData().getBasePkg(pkg)) self._readDependencyGraphAndCyclesForGivenPackages(basePackages) self._getSortedBuildOrderList() return self.__mapCyclesToPackageList, self.__mapPackageToCycle, self.__sortedPackageList
def __getRequiredTypePackages(pkg, requiresType): listRPMPackages = [] if requiresType == "build": listRPMPackages.extend(SPECS.getData().getBuildRequiresForPkg(pkg)) elif requiresType == "install": listRPMPackages.extend(SPECS.getData().getRequiresAllForPkg(pkg)) # Remove duplicates. listRPMPackages = list(set(listRPMPackages)) listPackages = set() for reqPkg in listRPMPackages: basePkg = SPECS.getData().getBasePkg(reqPkg) listPackages.add(basePkg) return list(listPackages)
def _buildGivenPackages(self, listPackages, buildThreads): # Extend listPackages from ["name1", "name2",..] to ["name1-vers1", "name2-vers2",..] listPackageNamesAndVersions=set() for pkg in listPackages: base = SPECS.getData().getSpecName(pkg) for version in SPECS.getData().getVersions(base): listPackageNamesAndVersions.add(base+"-"+version) returnVal = self._calculateParams(listPackageNamesAndVersions) if not returnVal: self.logger.error("Unable to set parameters. Terminating the package manager.") raise Exception("Unable to set parameters") statusEvent = threading.Event() self._initializeScheduler(statusEvent) self._initializeThreadPool(statusEvent) for i in range(0, buildThreads): workerName = "WorkerThread" + str(i) ThreadPool.addWorkerThread(workerName) ThreadPool.startWorkerThread(workerName) statusEvent.wait() Scheduler.stopScheduling = True self.logger.debug("Waiting for all remaining worker threads") ThreadPool.join_all() setFailFlag = False allPackagesBuilt = False if Scheduler.isAnyPackagesFailedToBuild(): setFailFlag = True if Scheduler.isAllPackagesBuilt(): allPackagesBuilt = True if setFailFlag: self.logger.error("Some of the packages failed:") self.logger.error(Scheduler.listOfFailedPackages) raise Exception("Failed during building package") if not setFailFlag: if allPackagesBuilt: self.logger.debug("All packages built successfully") else: self.logger.error("Build stopped unexpectedly.Unknown error.") raise Exception("Unknown error")
def __getRequiredTypePackages(pkg, requiresType): listRPMPackages = [] if requiresType == "build": listRPMPackages.extend(SPECS.getData().getBuildRequiresForPkg(pkg)) elif requiresType == "install": listRPMPackages.extend(SPECS.getData().getRequiresAllForPkg(pkg)) # Remove duplicates. listRPMPackages = list(set(listRPMPackages)) listPackages = set() for reqPkg in listRPMPackages: basePkg = SPECS.getData().getBasePkg(reqPkg) listPackages.add(basePkg) return list(listPackages)
def _readAlreadyAvailablePackages(self): listAvailablePackages = set() pkgUtils = PackageUtils(self.logName, self.logPath) listPackages = SPECS.getData().getListPackages() for package in listPackages: for version in SPECS.getData().getVersions(package): # Mark package available only if all subpackages are available packageIsAlreadyBuilt=True listRPMPackages = SPECS.getData().getRPMPackages(package, version) for rpmPkg in listRPMPackages: if pkgUtils.findRPMFile(rpmPkg, version) is None: packageIsAlreadyBuilt=False break; if packageIsAlreadyBuilt: listAvailablePackages.add(package+"-"+version) return listAvailablePackages
def buildSRPMList(srpmPath, yamlDir, blackListPkgs, logger, singleFile=True): cmdUtils = CommandUtils() yamlSrpmDir = os.path.join(yamlDir, "yaml_srpms") if not os.path.isdir(yamlSrpmDir): cmdUtils.runCommandInShell("mkdir -p "+yamlSrpmDir) if singleFile: yamlFile = open(yamlSrpmDir+"/srpm_list.yaml", "w") listPackages = SPECS.getData().getListPackages() listPackages.sort() for package in listPackages: if package in blackListPkgs: continue ossname = package ossversion = SPECS.getData().getVersion(package) ossrelease = SPECS.getData().getRelease(package) listFoundSRPMFiles = cmdUtils.findFile(ossname+"-"+ossversion+"-"+ossrelease+".src.rpm",srpmPath) srpmName = None if len(listFoundSRPMFiles) == 1: srpmFullPath = listFoundSRPMFiles[0]; srpmName = os.path.basename(srpmFullPath) cpcmd = "cp "+ srpmFullPath +" "+yamlSrpmDir+"/" returnVal = cmdUtils.runCommandInShell(cpcmd) if not returnVal: logger.error("Copy SRPM File is failed for package:"+ossname) else: logger.error("SRPM file is not found:" +ossname) if not singleFile: yamlFile = open(yamlSrpmDir+"/"+ossname+"-"+ossversion+"-"+ossrelease+".yaml", "w") yamlFile.write("baseos:"+ossname+":"+ossversion+"-"+ossrelease+":\n") yamlFile.write(" repository: BaseOS\n") yamlFile.write(" name: '"+ossname+"'\n") yamlFile.write(" version: '"+ossversion+"-"+ossrelease+"'\n") yamlFile.write(" baseos-style: rpm\n") yamlFile.write(" baseos-source: '"+str(srpmName)+"'\n") yamlFile.write(" baseos-osname: 'photon'\n") yamlFile.write("\n") if not singleFile: yamlFile.close() if singleFile: yamlFile.close() logger.info("Generated srpm yaml files for all packages")
def getPackageBuildData(self, listPackages): basePackages = set() for pkg in listPackages: basePackage = SPECS.getData().getSpecName(pkg) basePackages.add(basePackage) self._readDependencyGraphAndCyclesForGivenPackages(basePackages) self._getSortedBuildOrderList() return self.__mapCyclesToPackageList, self.__mapPackageToCycle, self.__sortedPackageList
def findRPMFile(self, package, version="*"): cmdUtils = CommandUtils() if version == "*": version = SPECS.getData().getHighestVersion(package) release = SPECS.getData().getRelease(package, version) buildarch = SPECS.getData().getBuildArch(package, version) filename = package + "-" + version + "-" + release + "." + buildarch + ".rpm" fullpath = constants.rpmPath + "/" + buildarch + "/" + filename if os.path.isfile(fullpath): return fullpath if constants.inputRPMSPath is not None: fullpath = constants.inputRPMSPath + "/" + buildarch + "/" + filename if os.path.isfile(fullpath): return fullpath return None
def findRPMFile(self, package,version="*"): cmdUtils = CommandUtils() if version == "*": version = SPECS.getData().getHighestVersion(package) release = SPECS.getData().getRelease(package, version) buildarch=SPECS.getData().getBuildArch(package, version) filename= package + "-" + version + "-" + release + "." + buildarch+".rpm" fullpath = constants.rpmPath + "/" + buildarch + "/" + filename if os.path.isfile(fullpath): return fullpath if constants.inputRPMSPath is not None: fullpath = constants.inputRPMSPath + "/" + buildarch + "/" + filename if os.path.isfile(fullpath): return fullpath return None
def _calculateAllRequiredPackagesPerNode(): # pkgNode contains information about spec package without deeping into subpackages. # getRequiresTreeOfBasePkgsForPkg() creates full build time dependency list base on # subpackages dependencies by walking the tree of: # BuildRequires and their Requires tree + Requires and their Requires tree # Let's keep graph simple by caching this preprocessed information per each pkgNode. # It shouldn't add much memory overhead. for package in Scheduler.sortedList: pkgNode = Scheduler.mapPackagesToGraphNodes[package] pkgNode.allRequiredPackages.extend(SPECS.getData().getRequiresTreeOfBasePkgsForPkg(package))
def process(self, inputType, inputValue, displayOption, outputFile=None): whoNeedsList = [] inputPackages = [] whatNeedsBuild = [] mapDependencies = {} parent = {} if inputType == "pkg" or inputType == "json": if inputType == "pkg": inputPackages.append(inputValue) else: inputPackages = self.getAllPackageNames(inputValue) self.calculateSpecDependency(inputPackages, mapDependencies, parent) if outputFile is not None: return self.displayDependencies(displayOption, inputType, outputFile, mapDependencies, parent) else: return self.displayDependencies(displayOption, inputType, inputValue, mapDependencies, parent) elif inputType == "get-upward-deps": depList = [] for specFile in inputValue.split(":"): if specFile in SPECS.getData().mapSpecFileNameToSpecObj: specObj = SPECS.getData( ).mapSpecFileNameToSpecObj[specFile] whoNeedsList.append(specObj.name + "-" + specObj.version) depList.append(specObj.name + "-" + specObj.version) self.findTotalWhoNeeds(depList, whoNeedsList) return whoNeedsList elif inputType == "who-needs": for depPackage in SPECS.getData().mapPackageToSpec: pkg = inputValue + "-" + SPECS.getData().getHighestVersion( inputValue) for version in SPECS.getData().getVersions(depPackage): depPkg = depPackage + "-" + version self.logger.info(depPkg) if pkg in SPECS.getData().getRequiresForPkg(depPkg): whoNeedsList.append(depPkg) self.logger.info(whoNeedsList) return whoNeedsList elif inputType == "is-toolchain-pkg": for specFile in inputValue.split(":"): if specFile in SPECS.getData().mapSpecFileNameToSpecObj: specObj = SPECS.getData( ).mapSpecFileNameToSpecObj[specFile] if (specObj.name in constants.listCoreToolChainPackages) \ or (specObj.name in constants.listToolChainPackages): return True return False
def readAlreadyAvailablePackages(self): listAvailablePackages = [] listFoundRPMPackages = [] listRPMFiles = [] listDirectorys = [] listDirectorys.append(constants.rpmPath) if constants.inputRPMSPath is not None: listDirectorys.append(constants.inputRPMSPath) while len(listDirectorys) > 0: dirPath = listDirectorys.pop() for dirEntry in os.listdir(dirPath): dirEntryPath = os.path.join(dirPath, dirEntry) if os.path.isfile(dirEntryPath) and dirEntryPath.endswith( ".rpm"): listRPMFiles.append(dirEntryPath) elif os.path.isdir(dirEntryPath): listDirectorys.append(dirEntryPath) pkgUtils = PackageUtils(self.logName, self.logPath) for rpmfile in listRPMFiles: package, version, release = pkgUtils.findPackageInfoFromRPMFile( rpmfile) if SPECS.getData().isRPMPackage(package): specVersion = SPECS.getData().getVersion(package) specRelease = SPECS.getData().getRelease(package) if version == specVersion and release == specRelease: listFoundRPMPackages.append(package) #Mark package available only if all sub packages are available for package in listFoundRPMPackages: basePkg = SPECS.getData().getSpecName(package) if basePkg in listAvailablePackages: continue listRPMPackages = SPECS.getData().getRPMPackages(basePkg) packageIsAlreadyBuilt = True for rpmpkg in listRPMPackages: if rpmpkg not in listFoundRPMPackages: packageIsAlreadyBuilt = False if packageIsAlreadyBuilt: listAvailablePackages.append(package) self.logger.info("List of Already built packages") self.logger.info(listAvailablePackages) return listAvailablePackages
def _createSortListForPkg(self, pkg): runTimeDepPkgList = list(set(self.__runTimeDependencyGraph[pkg])) runTimeDepPkgList.append(pkg) sortListForPkg = [] for p in runTimeDepPkgList: basePkg = SPECS.getData().getBasePkg(p) for bPkg in self.__sortedBuildDependencyGraph[basePkg]: if bPkg not in sortListForPkg: sortListForPkg.append(bPkg) return sortListForPkg
def buildPackagesForAllSpecs(logger, buildThreads, pkgInfoJsonFile, pkgBuildType): listPackages = SPECS.getData().getListPackages() if constants.rpmCheck: constants.setTestForceRPMS(copy.copy(listPackages)) pkgManager = PackageManager(pkgBuildType=pkgBuildType) pkgManager.buildPackages(listPackages, buildThreads, pkgBuildType) # Generating package info file which is required by installer logger.debug("Writing Package info to the file:" + pkgInfoJsonFile) pkgInfo = PackageInfo() pkgInfo.loadPackagesData() pkgInfo.writePkgListToFile(pkgInfoJsonFile)
def buildSourcesList(yamlDir, blackListPkgs, logger, singleFile=True): cmdUtils = CommandUtils() yamlSourceDir = os.path.join(yamlDir, "yaml_sources") if not os.path.isdir(yamlSourceDir): cmdUtils.runCommandInShell("mkdir -p " + yamlSourceDir) if singleFile: yamlFile = open(yamlSourceDir + "/sources_list.yaml", "w") listPackages = SPECS.getData().getListPackages() listPackages.sort() import PullSources for package in listPackages: if package in blackListPkgs: continue ossname = package for version in SPECS.getData().getVersions(package): modified = False listPatches = SPECS.getData().getPatches(package, version) if listPatches: modified = True url = SPECS.getData().getSourceURL(package, version) if url is None: url = SPECS.getData().getURL(package, version) sourceName = None listSourceNames = SPECS.getData().getSources(package, version) if listSourceNames: sourceName = listSourceNames[0] sha1 = SPECS.getData().getSHA1(package, version, sourceName) if sha1 is not None: PullSources.get(package, sourceName, sha1, yamlSourceDir, constants.getPullSourcesURLs(package), logger) if not singleFile: yamlFile = open( yamlSourceDir + "/" + ossname + "-" + version + ".yaml", "w") yamlFile.write("vmwsource:" + ossname + ":" + version + ":\n") yamlFile.write(" repository: VMWsource\n") yamlFile.write(" name: '" + ossname + "'\n") yamlFile.write(" version: '" + version + "'\n") yamlFile.write(" url: " + str(url) + "\n") yamlFile.write(" license: UNKNOWN\n") if sourceName is not None: yamlFile.write(" vmwsource-distribution: " + str(sourceName) + "\n") if modified: yamlFile.write(" modified: true\n") yamlFile.write("\n") if not singleFile: yamlFile.close() if singleFile: yamlFile.close() logger.debug("Generated source yaml files for all packages")
def installTargetToolchain(self, chroot, stopAtPackage=None): self.logger.debug("Installing target toolchain RPMS.......") pkgUtils = PackageUtils(self.logName, self.logPath) rpmFiles = "" packages = "" for package in constants.listCoreToolChainPackages: if stopAtPackage and package == stopAtPackage: break version = SPECS.getData().getHighestVersion(package) basePkg = SPECS.getData().getSpecName(package) # install all subpackages of given package # for instance: for 'glibc' we want glibc-devel, glibc-tools, # glibc-i18n, etc also to be installed subpackages = SPECS.getData().getRPMPackages(basePkg, version) for p in subpackages: rpmFile = pkgUtils.findRPMFile(p, version, constants.targetArch) rpmFiles += " " + rpmFile packages += " " + package + "-" + version self.logger.debug(packages) cmd = "mkdir -p " + chroot.getID( ) + "/target-" + constants.targetArch + "/var/lib/rpm" CommandUtils.runCommandInShell(cmd, logfn=self.logger.debug) if rpmFiles != "": cmd = (self.rpmCommand + " -Uvh --nodeps --ignorearch --noscripts --root " + chroot.getID() + "/target-" + constants.targetArch + " --define \'_dbpath /var/lib/rpm\' " + rpmFiles) retVal = CommandUtils.runCommandInShell(cmd, logfn=self.logger.debug) if retVal != 0: self.logger.debug("Command Executed:" + cmd) self.logger.error("Installing toolchain failed") raise Exception("RPM installation failed") self.logger.debug( "Successfully installed target toolchain RPMS in chroot:" + chroot.getID())
def buildRPMSForGivenPackage(self, sandbox, package, version, destLogPath): self.logger.info("Building package : " + package) listSourcesFiles = SPECS.getData().getSources(package, version) listPatchFiles = SPECS.getData().getPatches(package, version) specFile = SPECS.getData().getSpecFile(package, version) specName = SPECS.getData().getSpecName(package) + ".spec" sourcePath = constants.topDirPath + "/SOURCES/" specPath = constants.topDirPath + "/SPECS/" if (constants.rpmCheck and package in constants.testForceRPMS and SPECS.getData().isCheckAvailable(package, version)): logFilePath = destLogPath + "/" + package + "-test.log" else: logFilePath = destLogPath + "/" + package + ".log" sandbox.put(specFile, specPath + specName) sources_urls, macros = self._getAdditionalBuildOptions(package) self.logger.debug("Extra macros for " + package + ": " + str(macros)) self.logger.debug("Extra source URLs for " + package + ": " + str(sources_urls)) constants.setExtraSourcesURLs(package, sources_urls) self._copySources(sandbox, listSourcesFiles, package, version, sourcePath) self._copySources(sandbox, listPatchFiles, package, version, sourcePath) #Adding rpm macros listRPMMacros = constants.userDefinedMacros for macroName, value in listRPMMacros.items(): macros.append(macroName + " " + value) listRPMFiles = [] listSRPMFiles = [] try: listRPMFiles, listSRPMFiles = self._buildRPM(sandbox, specPath + specName, logFilePath, package, version, macros) logmsg = package + " build done - RPMs : [ " for f in listRPMFiles: logmsg += (os.path.basename(f) + " ") logmsg += "]\n" self.logger.info(logmsg) except Exception as e: self.logger.error("Failed while building rpm:" + package) raise e finally: if (constants.rpmCheck and package in constants.testForceRPMS and SPECS.getData().isCheckAvailable(package, version)): cmd = ("sed -i '/^Executing(%check):/,/^Processing files:/{//!b};d' " + logFilePath) CommandUtils().runCommandInShell(cmd, logfn=self.logger.debug) if constants.crossCompiling: for f in listRPMFiles: workingDir = os.getcwd() cmd = "%s/../../tools/scripts/check-rpm-arch.sh %s" % (workingDir, constants.rpmPath + "/" + constants.targetArch + "/" + os.path.basename(f)) retVal = CommandUtils().runCommandInShell(cmd, logfn=self.logger.debug, showOutput=True) if retVal != 0: raise NameError("%s failed %s architecture check" % (f, constants.targetArch)) else: self.logger.debug("RPM passed architecture check") self.logger.debug("RPM build is successful")
def _readAlreadyAvailablePackages(self): listAvailablePackages = set() listFoundRPMPackages = set() listRPMFiles = set() listDirectorys = set() listDirectorys.add(constants.rpmPath) if constants.inputRPMSPath is not None: listDirectorys.add(constants.inputRPMSPath) while listDirectorys: dirPath = listDirectorys.pop() for dirEntry in os.listdir(dirPath): dirEntryPath = os.path.join(dirPath, dirEntry) if os.path.isfile(dirEntryPath) and dirEntryPath.endswith(".rpm"): listRPMFiles.add(dirEntryPath) elif os.path.isdir(dirEntryPath): listDirectorys.add(dirEntryPath) pkgUtils = PackageUtils(self.logName, self.logPath) for rpmfile in listRPMFiles: package, version, release = pkgUtils.findPackageInfoFromRPMFile(rpmfile) if SPECS.getData().isRPMPackage(package): specVersion = SPECS.getData().getVersion(package) specRelease = SPECS.getData().getRelease(package) if version == specVersion and release == specRelease: listFoundRPMPackages.add(package) #Mark package available only if all sub packages are available for package in listFoundRPMPackages: basePkg = SPECS.getData().getSpecName(package) if basePkg in listAvailablePackages: continue listRPMPackages = SPECS.getData().getRPMPackages(basePkg) packageIsAlreadyBuilt = True for rpmpkg in listRPMPackages: if rpmpkg not in listFoundRPMPackages: packageIsAlreadyBuilt = False if packageIsAlreadyBuilt: listAvailablePackages.add(package) self.logger.info("List of Already built packages") self.logger.info(listAvailablePackages) return listAvailablePackages
def findRPMFile(self, package,version="*",arch=None, throw=False): if not arch: arch=constants.currentArch if version == "*": version = SPECS.getData(arch).getHighestVersion(package) release = SPECS.getData(arch).getRelease(package, version) buildarch=SPECS.getData(arch).getBuildArch(package, version) filename= package + "-" + version + "-" + release + "." + buildarch+".rpm" fullpath = constants.rpmPath + "/" + buildarch + "/" + filename if os.path.isfile(fullpath): return fullpath if constants.inputRPMSPath is not None: fullpath = constants.inputRPMSPath + "/" + buildarch + "/" + filename if os.path.isfile(fullpath): return fullpath if throw: raise Exception("RPM %s not found" % (filename)) return None
def installToolChainRPMSinContainer(self, containerID): self.logger.info("Installing tool-chain RPMS in container: " + containerID.short_id) rpmFiles = "" packages = "" pkgUtils = PackageUtils(self.logName, self.logPath) for package in constants.listToolChainRPMPkgsToInstall: version = SPECS.getData().getHighestVersion(package) rpmFile = pkgUtils.findRPMFileForGivenPackage(package, version) if rpmFile is None: # sqlite-autoconf package was renamed, but it still published as sqlite-autoconf # if (package == "sqlite") and (platform.machine() == "x86_64"): # package = "sqlite-autoconf" rpmFile = self.findRPMFileInGivenLocation( package, constants.prevPublishRPMRepo) if rpmFile is None: if package in constants.listOfRPMsProvidedAfterBuild: self.logger.info( "No old version of " + package + " exists, skip until the new version is built") continue self.logger.error("Unable to find rpm " + package + " in current and previous versions") raise Exception("Input Error") if rpmFile.find("stage/PUBLISHRPMS"): rpmFile = rpmFile.replace(constants.prevPublishRPMRepo, "/publishrpms") if rpmFile.find("stage/PUBLISHXRPMS"): rpmFile = rpmFile.replace(constants.prevPublishXRPMRepo, "/publishxrpms") if rpmFile.find("stage/RPMS"): rpmFile = rpmFile.replace(constants.rpmPath, constants.topDirPath + "/RPMS") rpmFiles += " " + rpmFile packages += " " + package self.logger.debug("Installing tool-chain rpms: " + packages) cmd = "/usr/bin/bash -l -c '/usr/bin/rpm -Uvh --force --nodeps " + rpmFiles + "'" self.logger.info( "VDBG-TCU-installToolChainRPMSinContainer: Installing rpms cmd: " + cmd) tcInstallLog = containerID.exec_run(cmd) # TODO: Find a way to collect exit status of the command that was run. if not tcInstallLog: self.logger.error("Installing tool chain in container failed") raise Exception("RPM installation in container failed") self.logger.info(tcInstallLog) self.logger.info( "Successfully installed default tool-chain RPMS in container: " + containerID.short_id)
def _readAlreadyAvailablePackages(self): listAvailablePackages = set() listFoundRPMPackages = set() listRPMFiles = set() listDirectorys = set() mapPackageVersion = {} mapPackageRelease = {} listDirectorys.add(constants.rpmPath) if constants.inputRPMSPath is not None: listDirectorys.add(constants.inputRPMSPath) while listDirectorys: dirPath = listDirectorys.pop() for dirEntry in os.listdir(dirPath): dirEntryPath = os.path.join(dirPath, dirEntry) if os.path.isfile(dirEntryPath) and dirEntryPath.endswith( ".rpm"): listRPMFiles.add(dirEntryPath) elif os.path.isdir(dirEntryPath): listDirectorys.add(dirEntryPath) pkgUtils = PackageUtils(self.logName, self.logPath) for rpmfile in listRPMFiles: package, version, release = pkgUtils.findPackageInfoFromRPMFile( rpmfile) if package in mapPackageVersion: mapPackageVersion[package].append(version) mapPackageRelease[package].append(release) else: mapPackageVersion[package] = [version] mapPackageRelease[package] = [release] for package in mapPackageVersion: if SPECS.getData().isRPMPackage(package): numVersions = SPECS.getData().getNumberOfVersions(package) flag = True for index in range(0, numVersions): specVersion = SPECS.getData().getVersion(package, index) specRelease = SPECS.getData().getRelease(package, index) if specVersion not in mapPackageVersion[ package] and specRelease not in mapPackageRelease[ package]: flag = False if flag: listFoundRPMPackages.add(package) #Mark package available only if all sub packages are available for package in listFoundRPMPackages: basePkg = SPECS.getData().getSpecName(package) if basePkg in listAvailablePackages: continue listRPMPackages = SPECS.getData().getRPMPackages(basePkg) packageIsAlreadyBuilt = True for rpmpkg in listRPMPackages: if rpmpkg not in listFoundRPMPackages: packageIsAlreadyBuilt = False if packageIsAlreadyBuilt: listAvailablePackages.add(package) self.logger.info("List of Already built packages") self.logger.info(listAvailablePackages) return listAvailablePackages
def loadPackagesData(self): listPackages = SPECS.getData().getListPackages() listPackages.sort() listRPMFiles = [] cmdUtils = CommandUtils() for package in listPackages: for version in SPECS.getData().getVersions(package): release = SPECS.getData().getRelease(package, version) listRPMPackages = SPECS.getData().getRPMPackages( package, version) srpmFileName = package + "-" + version + "-" + release + ".src.rpm" srpmFiles = cmdUtils.findFile(srpmFileName, constants.sourceRpmPath) srpmFile = None if len(srpmFiles) == 1: srpmFile = srpmFiles[0] debugrpmFileName = package + "-debuginfo-" + version + "-" + release + "*" debugrpmFiles = cmdUtils.findFile(debugrpmFileName, constants.rpmPath) debugrpmFile = None if len(debugrpmFiles) == 1: debugrpmFile = debugrpmFiles[0] pkgUtils = PackageUtils(self.logName, self.logPath) for rpmPkg in listRPMPackages: rpmFile = pkgUtils.findRPMFileForGivenPackage(rpmPkg) if rpmFile is not None: listRPMFiles.append(rpmFile) listPkgAttributes = { "sourcerpm": srpmFile, "rpm": rpmFile, "debugrpm": debugrpmFile } self.pkgList[rpmPkg] = listPkgAttributes self.logger.debug("Added " + rpmPkg + " rpm package to the list") else: self.logger.error("Missing rpm file for package:" + rpmPkg)
def buildSourcesList(yamlDir, blackListPkgs, logger, singleFile=True): cmdUtils = CommandUtils() yamlSourceDir = os.path.join(yamlDir, "yaml_sources") if not os.path.isdir(yamlSourceDir): cmdUtils.runCommandInShell("mkdir -p " + yamlSourceDir) if singleFile: yamlFile = open(yamlSourceDir + "/sources_list.yaml", "w") listPackages = SPECS.getData().getListPackages() listPackages.sort() import PullSources for package in listPackages: if package in blackListPkgs: continue ossname = package for version in SPECS.getData().getVersions(package): modified = False listPatches = SPECS.getData().getPatches(package, version) if listPatches: modified = True url = SPECS.getData().getSourceURL(package, version) if url is None: url = SPECS.getData().getURL(package, version) sourceName = None listSourceNames = SPECS.getData().getSources(package, version) if listSourceNames: sourceName = listSourceNames[0] sha1 = SPECS.getData().getSHA1(package, version, sourceName) if sha1 is not None: PullSources.get(package, sourceName, sha1, yamlSourceDir, constants.getPullSourcesURLs(package), logger) if not singleFile: yamlFile = open(yamlSourceDir + "/" + ossname + "-" + version + ".yaml", "w") yamlFile.write("vmwsource:" + ossname + ":" + version + ":\n") yamlFile.write(" repository: VMWsource\n") yamlFile.write(" name: '" + ossname + "'\n") yamlFile.write(" version: '" + version + "'\n") yamlFile.write(" url: " + str(url) + "\n") yamlFile.write(" license: UNKNOWN\n") if sourceName is not None: yamlFile.write(" vmwsource-distribution: " + str(sourceName) + "\n") if modified: yamlFile.write(" modified: true\n") yamlFile.write("\n") if not singleFile: yamlFile.close() if singleFile: yamlFile.close() logger.debug("Generated source yaml files for all packages")
def _constructBuildAndRunTimeDependencyGraph(self, basePackage): addBuildTimeGraph = True addRunTimeGraph = True if basePackage in self.__buildDependencyGraph: addBuildTimeGraph = False if basePackage in self.__runTimeDependencyGraph: addRunTimeGraph = False nextPackagesToConstructGraph = set() if addBuildTimeGraph: dependentRpmPackages = SPECS.getData().getBuildRequiresForPackage( basePackage) dependentPackages = set() for rpmPkg in dependentRpmPackages: basePkg = SPECS.getData().getSpecName(rpmPkg.package) dependentPackages.add(basePkg) self.__buildDependencyGraph[basePackage] = dependentPackages nextPackagesToConstructGraph.update(dependentPackages) if addRunTimeGraph: rpmPackages = SPECS.getData().getPackages(basePackage) dependentPackages = set() dependentRpmPackagesNames = set() for rpmPkg in rpmPackages: dependentRpmPackages = SPECS.getData( ).getRequiresAllForPackage(rpmPkg) for pkgName in dependentRpmPackages: dependentRpmPackagesNames.add(pkgName.package) self.__runTimeDependencyGraph[rpmPkg] = copy.copy( dependentRpmPackagesNames) for pkg in dependentRpmPackagesNames: dependentPackages.add(SPECS.getData().getSpecName(pkg)) nextPackagesToConstructGraph.update(dependentPackages) for pkg in nextPackagesToConstructGraph: self._constructBuildAndRunTimeDependencyGraph(pkg)
def buildPackagesForAllSpecs(listBuildOptionPackages, pkgBuildOptionFile, logger, buildThreads, pkgInfoJsonFile, pkgBuildType): listPackages = SPECS.getData().getListPackages() logger.info("List of packages to build:") logger.info(listPackages) if constants.rpmCheck: constants.setTestForceRPMS(listPackages[:]) pkgManager = PackageManager() pkgManager.buildPackages(listPackages, listBuildOptionPackages, pkgBuildOptionFile, buildThreads, pkgBuildType) #Generating package info file which is required by installer logger.info("Writing Package info to the file:"+pkgInfoJsonFile) pkgInfo = PackageInfo() pkgInfo.loadPackagesData() pkgInfo.writePkgListToFile(pkgInfoJsonFile)
def process(self, inputType, inputValue, displayOption, outputFile=None): whoNeedsList = [] inputPackages = [] whatNeedsBuild = [] mapDependencies = {} parent = {} if inputType == "pkg" or inputType == "json": if inputType == "pkg": inputPackages.append(inputValue) else: inputPackages = self.getAllPackageNames(inputValue) self.calculateSpecDependency(inputPackages, mapDependencies, parent) if outputFile is not None: return self.displayDependencies(displayOption, inputType, outputFile, mapDependencies, parent) else: return self.displayDependencies(displayOption, inputType, inputValue, mapDependencies, parent) elif inputType == "get-upward-deps": depList = [] for specFile in inputValue.split(":"): if specFile in SPECS.getData().mapSpecFileNameToSpecObj: specObj = SPECS.getData().mapSpecFileNameToSpecObj[specFile] whoNeedsList.append(specObj.name+"-"+specObj.version) depList.append(specObj.name+"-"+specObj.version) self.findTotalWhoNeeds(depList, whoNeedsList) return whoNeedsList elif inputType == "who-needs": for depPackage in SPECS.getData().mapPackageToSpec: pkg=inputValue+"-"+SPECS.getData().getHighestVersion(inputValue) for version in SPECS.getData().getVersions(depPackage): depPkg = depPackage+"-"+version self.logger.info(depPkg) if pkg in SPECS.getData().getRequiresForPkg(depPkg): whoNeedsList.append(depPkg) self.logger.info(whoNeedsList) return whoNeedsList elif inputType == "is-toolchain-pkg": for specFile in inputValue.split(":"): if specFile in SPECS.getData().mapSpecFileNameToSpecObj: specObj = SPECS.getData().mapSpecFileNameToSpecObj[specFile] if (specObj.name in constants.listCoreToolChainPackages) \ or (specObj.name in constants.listToolChainPackages): return True return False
def adjustGCCSpecs(self, sandbox, package, version): opt = " " + SPECS.getData().getSecurityHardeningOption(package, version) sandbox.put(self.adjustGCCSpecScript, "/tmp") cmd = "/tmp/" + self.adjustGCCSpecScript + opt returnVal = sandbox.run(cmd, logfn=self.logger.debug) if returnVal == 0: return # in debugging ... sandbox.run("ls -la /tmp/" + self.adjustGCCSpecScript, logfn=self.logger.debug) sandbox.run("lsof /tmp/" + self.adjustGCCSpecScript, logfn=self.logger.debug) sandbox.run("ps ax", logfn=self.logger.debug) self.logger.error("Failed while adjusting gcc specs") raise Exception("Failed while adjusting gcc specs")
def buildRPMSForGivenPackage(self, sandbox, package, version, destLogPath): self.logger.info("Building package : " + package) listSourcesFiles = SPECS.getData().getSources(package, version) listPatchFiles = SPECS.getData().getPatches(package, version) specFile = SPECS.getData().getSpecFile(package, version) specName = SPECS.getData().getSpecName(package) + ".spec" sourcePath = constants.topDirPath + "/SOURCES/" specPath = constants.topDirPath + "/SPECS/" if (constants.rpmCheck and package in constants.testForceRPMS and SPECS.getData().isCheckAvailable(package)): logFilePath = destLogPath + "/" + package + "-test.log" else: logFilePath = destLogPath + "/" + package + ".log" sandbox.put(specFile, specPath + specName) sources_urls, macros = self._getAdditionalBuildOptions(package) self.logger.debug("Extra macros for " + package + ": " + str(macros)) self.logger.debug("Extra source URLs for " + package + ": " + str(sources_urls)) constants.setExtraSourcesURLs(package, sources_urls) self._copySources(sandbox, listSourcesFiles, package, version, sourcePath) self._copySources(sandbox, listPatchFiles, package, version, sourcePath) #Adding rpm macros listRPMMacros = constants.userDefinedMacros for macroName, value in listRPMMacros.items(): macros.append(macroName + " " + value) listRPMFiles = [] listSRPMFiles = [] try: listRPMFiles, listSRPMFiles = self._buildRPM( sandbox, specPath + specName, logFilePath, package, macros) logmsg = package + " build done - RPMs : [ " for f in listRPMFiles: logmsg += (os.path.basename(f) + " ") logmsg += "]\n" self.logger.info(logmsg) except Exception as e: self.logger.error("Failed while building rpm:" + package) raise e finally: if (constants.rpmCheck and package in constants.testForceRPMS and SPECS.getData().isCheckAvailable(package)): cmd = ( "sed -i '/^Executing(%check):/,/^Processing files:/{//!b};d' " + logFilePath) CommandUtils().runCommandInShell(cmd, logfn=self.logger.debug) self.logger.debug("RPM build is successful")
def adjustGCCSpecs(self, sandbox, package, version): # TODO: need to harden cross compiller also opt = " " + SPECS.getData().getSecurityHardeningOption(package, version) sandbox.put(os.path.join(os.path.dirname(__file__), self.adjustGCCSpecScript), "/tmp") cmd = "/tmp/" + self.adjustGCCSpecScript + opt returnVal = sandbox.run(cmd, logfn=self.logger.debug) if returnVal == 0: return # in debugging ... sandbox.run("ls -la /tmp/" + self.adjustGCCSpecScript, logfn=self.logger.debug) sandbox.run("lsof /tmp/" + self.adjustGCCSpecScript, logfn=self.logger.debug) sandbox.run("ps ax", logfn=self.logger.debug) self.logger.error("Failed while adjusting gcc specs") raise Exception("Failed while adjusting gcc specs")
def adjustGCCSpecsInContainer(self, package, containerID, logPath, index=0): opt = " " + SPECS.getData().getSecurityHardeningOption(package, index) adjustCmd = "/" + self.adjustGCCSpecScript + opt adjustCmd = "/bin/bash -l -c '" + adjustCmd + "'" logFile = logPath + "/adjustGCCSpecScript.log" #TODO: Error code from exec_run scriptLog = containerID.exec_run(adjustCmd) if scriptLog: with open(logFile, 'w') as logfile: logfile.write(scriptLog.decode()) return self.logger.debug(containerID.exec_run("ls -la /tmp/" + self.adjustGCCSpecScript)) self.logger.debug(containerID.exec_run("lsof /tmp/" + self.adjustGCCSpecScript)) self.logger.debug(containerID.exec_run("ps ax")) self.logger.error("Failed while adjusting gcc specs") raise Exception("Failed while adjusting gcc specs")
def buildRPMSForGivenPackage(self, sandbox, package, version, destLogPath): self.logger.info("Building package : " + package) listSourcesFiles = SPECS.getData().getSources(package, version) listPatchFiles = SPECS.getData().getPatches(package, version) specFile = SPECS.getData().getSpecFile(package, version) specName = SPECS.getData().getSpecName(package) + ".spec" sourcePath = constants.topDirPath + "/SOURCES/" specPath = constants.topDirPath + "/SPECS/" if (constants.rpmCheck and package in constants.testForceRPMS and SPECS.getData().isCheckAvailable(package, version)): logFilePath = destLogPath + "/" + package + "-test.log" else: logFilePath = destLogPath + "/" + package + ".log" sandbox.put(specFile, specPath + specName) sources_urls, macros = self._getAdditionalBuildOptions(package) self.logger.debug("Extra macros for " + package + ": " + str(macros)) self.logger.debug("Extra source URLs for " + package + ": " + str(sources_urls)) constants.setExtraSourcesURLs(package, sources_urls) self._copySources(sandbox, listSourcesFiles, package, version, sourcePath) self._copySources(sandbox, listPatchFiles, package, version, sourcePath) #Adding rpm macros listRPMMacros = constants.userDefinedMacros for macroName, value in listRPMMacros.items(): macros.append(macroName + " " + value) listRPMFiles = [] listSRPMFiles = [] try: listRPMFiles, listSRPMFiles = self._buildRPM(sandbox, specPath + specName, logFilePath, package, version, macros) logmsg = package + " build done - RPMs : [ " for f in listRPMFiles: logmsg += (os.path.basename(f) + " ") logmsg += "]\n" self.logger.info(logmsg) except Exception as e: self.logger.error("Failed while building rpm:" + package) raise e finally: if (constants.rpmCheck and package in constants.testForceRPMS and SPECS.getData().isCheckAvailable(package, version)): cmd = ("sed -i '/^Executing(%check):/,/^Processing files:/{//!b};d' " + logFilePath) CommandUtils().runCommandInShell(cmd, logfn=self.logger.debug) self.logger.debug("RPM build is successful")
def _findDependentPackagesAndInstalledRPM(self, sandbox): listInstalledPackages, listInstalledRPMs = self._findInstalledPackages(sandbox) self.logger.debug(listInstalledPackages) listDependentPackages = self._findBuildTimeRequiredPackages() listTestPackages=[] if constants.rpmCheck and self.package in constants.testForceRPMS: # One time optimization if constants.listMakeCheckRPMPkgWithVersionstoInstall is None: constants.listMakeCheckRPMPkgWithVersionstoInstall=[] for package in constants.listMakeCheckRPMPkgtoInstall: version = SPECS.getData().getHighestVersion(package) constants.listMakeCheckRPMPkgWithVersionstoInstall.append(package+"-"+version) listDependentPackages.extend(self._findBuildTimeCheckRequiredPackages()) testPackages = (set(constants.listMakeCheckRPMPkgWithVersionstoInstall) - set(listInstalledPackages) - set([self.package+"-"+self.version])) listTestPackages=list(set(testPackages)) listDependentPackages = list(set(listDependentPackages)) return listDependentPackages, listTestPackages, listInstalledPackages, listInstalledRPMs
def findTotalRequires(self, mapDependencies, depQue, parent): while not depQue.empty(): specPkg = depQue.get() try: listRequiredPackages = SPECS.getData().getRequiresForPkg(specPkg) except Exception as e: self.logger.info("Caught Exception:"+str(e)) self.logger.info(specPkg + " is missing") raise e for depPkg in listRequiredPackages: if depPkg in mapDependencies: if mapDependencies[depPkg] < mapDependencies[specPkg] + 1: mapDependencies[depPkg] = mapDependencies[specPkg] + 1 parent[depPkg] = specPkg self.updateLevels(mapDependencies, depPkg, parent, mapDependencies[depPkg]) else: mapDependencies[depPkg] = mapDependencies[specPkg] + 1 parent[depPkg] = specPkg depQue.put(depPkg)
def adjustGCCSpecs(self, package, chrootID, logPath, index=0): opt = " " + SPECS.getData().getSecurityHardeningOption(package, index) cmdUtils = CommandUtils() cpcmd = ("cp " + self.adjustGCCSpecScript + " " + chrootID + "/tmp/" + self.adjustGCCSpecScript) cmd = "/tmp/" + self.adjustGCCSpecScript + opt logFile = logPath + "/adjustGCCSpecScript.log" chrootCmd = self.runInChrootCommand + " " + chrootID returnVal = cmdUtils.runCommandInShell(cpcmd, logFile) if not returnVal: self.logger.error("Error during copying the file adjust gcc spec") raise Exception("Failed while copying adjust gcc spec file") returnVal = cmdUtils.runCommandInShell(cmd, logFile, chrootCmd) if returnVal: return self.logger.debug(cmdUtils.runCommandInShell2("ls -la " + chrootID + "/tmp/" + self.adjustGCCSpecScript)) self.logger.debug(cmdUtils.runCommandInShell2("lsof " + chrootID + "/tmp/" + self.adjustGCCSpecScript)) self.logger.debug(cmdUtils.runCommandInShell2("ps ax")) self.logger.error("Failed while adjusting gcc specs") raise Exception("Failed while adjusting gcc specs")
def buildPackagesList(csvFilename): with open(csvFilename, "w") as csvFile: csvFile.write("Package,Version,License,URL,Sources,Patches\n") listPackages = SPECS.getData().getListPackages() listPackages.sort() for package in listPackages: name = package for version in SPECS.getData().getVersions(package): packagelicense = SPECS.getData().getLicense(package, version) listPatches = SPECS.getData().getPatches(package, version) url = SPECS.getData().getURL(package, version) listSourceNames = SPECS.getData().getSources(package, version) sources = "" patches = "" if listPatches is not None: patches = " ".join(listPatches) if listSourceNames is not None: sources = " ".join(listSourceNames) csvFile.write(name + "," + version + "," + packagelicense + "," + url + "," + sources + "," + patches + "\n")
def buildPackagesForAllSpecs(buildThreads, pkgBuildType, pkgInfoJsonFile, logger): listPackages = SPECS.getData().getListPackages() buildSpecifiedPackages(listPackages, buildThreads, pkgBuildType, pkgInfoJsonFile, logger)
def _buildRPMinContainer(self, specFile, rpmLogFile, destLogFile, containerID, package, macros): rpmBuildCmd = self.rpmbuildBinary + " " + self.rpmbuildBuildallOption if constants.rpmCheck and package in constants.testForceRPMS: self.logger.info("#" * (68 + 2 * len(package))) if not SPECS.getData().isCheckAvailable(package, index): self.logger.info("####### " + package + " MakeCheck is not available. Skipping MakeCheck TEST for " + package + " #######") rpmBuildCmd = self.rpmbuildBinary + " --clean" else: self.logger.info("####### " + package + " MakeCheck is available. Running MakeCheck TEST for " + package + " #######") rpmBuildCmd = self.rpmbuildBinary + " " + self.rpmbuildCheckOption self.logger.info("#" * (68 + 2 * len(package))) else: rpmBuildCmd += " "+self.rpmbuildNocheckOption for macro in macros: rpmBuildCmd += ' --define \"%s\"' % macro rpmBuildCmd += " " + specFile rpmBuildCmd = "/bin/bash -l -c '" + rpmBuildCmd + " > " + rpmLogFile + " 2>&1'" rpmBuildCmd = "docker exec -t " + str(containerID.short_id) + " " + rpmBuildCmd cmdUtils = CommandUtils() self.logger.info("Building rpm for package: " + package) #TODO: Show running log of rpmbuildcmd #TODO: Get exit status of rpmBuildCmd #containerID.exec_run(rpmBuildCmd) returnVal = cmdUtils.runCommandInShell(rpmBuildCmd) if not os.path.isfile(destLogFile): self.logger.error("RPM build not file not found. Building rpm failed for: " + specFile) raise Exception("RPM Build failed") if constants.rpmCheck and package in constants.testForceRPMS: if not SPECS.getData().isCheckAvailable(package, index): constants.testLogger.info(package + " : N/A") elif returnVal: constants.testLogger.info(package + " : PASS") else: constants.testLogger.error(package + " : FAIL") if constants.rpmCheck: if not returnVal and constants.rpmCheckStopOnError: self.logger.error("Checking rpm is failed " + specFile) raise Exception("RPM check failed") else: if not returnVal: self.logger.error("Building rpm is failed " + specFile) raise Exception("RPM build failed") #Extracting rpms created from log file listRPMFiles = [] listSRPMFiles = [] with open(destLogFile, 'r') as logfile: rpmBuildLogLines = logfile.readlines() for i in range(0, len(rpmBuildLogLines)): if re.search("^Wrote:", rpmBuildLogLines[i]): listcontents = rpmBuildLogLines[i].split() if ((len(listcontents) == 2) and listcontents[1].strip().endswith(".rpm") and "/RPMS/" in listcontents[1]): listRPMFiles.append(listcontents[1]) if ((len(listcontents) == 2) and listcontents[1].strip().endswith(".src.rpm") and "/SRPMS/" in listcontents[1]): listSRPMFiles.append(listcontents[1]) #if not listRPMFiles: # self.logger.error("Building rpm failed for " + specFile) # raise Exception("RPM Build failed") return listRPMFiles, listSRPMFiles
def _buildRPM(self, specFile, logFile, chrootCmd, package, macros): rpmBuildcmd = self.rpmbuildBinary + " " + self.rpmbuildBuildallOption if constants.rpmCheck and package in constants.testForceRPMS: self.logger.info("#" * (68 + 2 * len(package))) if not SPECS.getData().isCheckAvailable(package): self.logger.info("####### " + package + " MakeCheck is not available. Skipping MakeCheck TEST for " + package + " #######") rpmBuildcmd = self.rpmbuildBinary + " --clean" else: self.logger.info("####### " + package + " MakeCheck is available. Running MakeCheck TEST for " + package + " #######") rpmBuildcmd = self.rpmbuildBinary + " " + self.rpmbuildCheckOption self.logger.info("#" * (68 + 2 * len(package))) else: rpmBuildcmd += " " + self.rpmbuildNocheckOption for macro in macros: rpmBuildcmd += ' --define \\\"%s\\\"' % macro rpmBuildcmd += " " + specFile cmdUtils = CommandUtils() self.logger.info("Building rpm....") self.logger.info(rpmBuildcmd) returnVal = cmdUtils.runCommandInShell(rpmBuildcmd, logFile, chrootCmd) if constants.rpmCheck and package in constants.testForceRPMS: if not SPECS.getData().isCheckAvailable(package): constants.testLogger.info(package + " : N/A") elif returnVal: constants.testLogger.info(package + " : PASS") else: constants.testLogger.error(package + " : FAIL") if constants.rpmCheck: if not returnVal and constants.rpmCheckStopOnError: self.logger.error("Checking rpm is failed " + specFile) raise Exception("RPM check failed") else: if not returnVal: self.logger.error("Building rpm is failed " + specFile) raise Exception("RPM build failed") #Extracting rpms created from log file listRPMFiles = [] listSRPMFiles = [] with open(logFile, 'r') as logfile: fileContents = logfile.readlines() for i in range(0, len(fileContents)): if re.search("^Wrote:", fileContents[i]): listcontents = fileContents[i].split() if ((len(listcontents) == 2) and listcontents[1].strip().endswith(".rpm") and "/RPMS/" in listcontents[1]): listRPMFiles.append(listcontents[1]) if ((len(listcontents) == 2) and listcontents[1].strip().endswith(".src.rpm") and "/SRPMS/" in listcontents[1]): listSRPMFiles.append(listcontents[1]) return listRPMFiles, listSRPMFiles
def buildRPMSForGivenPackageInContainer(self, package, containerID, destLogPath=None, index=0): self.logger.info("Building rpm's for package " + package + " in container " + containerID.short_id) listSourcesFiles = SPECS.getData().getSources(package, index) listPatchFiles = SPECS.getData().getPatches(package, index) specFile = SPECS.getData().getSpecFile(package, index) specName = SPECS.getData().getSpecName(package) + ".spec" sourcePath = constants.topDirPath + "/SOURCES/" specPath = constants.topDirPath + "/SPECS/" rpmLogFile = constants.topDirPath + "/LOGS/" + package + ".log" destLogFile = destLogPath + "/" + package + ".log" cmdUtils = CommandUtils() #TODO: mount it in, don't copy cpSpecCmd = "docker cp " + specFile + " " + containerID.short_id \ + ":" + specPath + specName returnVal = cmdUtils.runCommandInShell(cpSpecCmd) if not returnVal: self.logger.error("Error copying source SPEC file to container") raise Exception("Failed copying source SPEC to container") #FIXME: some sources are located in SPECS/.. how to mount? # if os.geteuid()==0: #TODO: mount it in, don't copy macros = [] self._copySourcesToContainer(listSourcesFiles, package, containerID, sourcePath, index) #TODO: mount it in, don't copy self._copySourcesToContainer(listPatchFiles, package, containerID, sourcePath, index) listAdditionalFiles, macros = self._getAdditionalBuildFiles(package) self._copyAdditionalBuildFilesToContainer(listAdditionalFiles, containerID) # Add rpm macros listRPMMacros = constants.userDefinedMacros for macroName, value in listRPMMacros.items(): macros.append(macroName + " " + value) # Build RPMs listRPMFiles = [] listSRPMFiles = [] try: listRPMFiles, listSRPMFiles = self._buildRPMinContainer( specPath + specName, rpmLogFile, destLogFile, containerID, package, macros) self.logger.info("Successfully built rpm:" + package) except Exception as e: self.logger.error("Failed while building rpm: " + package) raise e finally: if destLogPath is not None: rpmLog = destLogPath + "/" + package + ".log" if (constants.rpmCheck and package in constants.testForceRPMS and SPECS.getData().isCheckAvailable(package, index)): cmd = "sed -i '/^Executing(%check):/,/^Processing files:/{//!b};d' " + rpmLog logFile = destLogPath + "/adjustTestFile.log" returnVal = CommandUtils().runCommandInShell(cmd, logFile) testLogFile = destLogPath + "/" + package + "-test.log" shutil.copyfile(rpmLog, testLogFile) self.logger.info("RPM build is successful") # Verify RPM and SRPM files exist as success criteria for rpmFile in listRPMFiles: rpmName = os.path.basename(rpmFile) rpmDestDir = self._getRPMDestDir(rpmName, constants.rpmPath) rpmDestFile = rpmDestDir + "/" + rpmName if not os.path.isfile(rpmDestFile): self.logger.error("Could not find RPM file: " + rpmDestFile) raise Exception("Built RPM file not found.") for srpmFile in listSRPMFiles: srpmName = os.path.basename(srpmFile) srpmDestDir = self._getRPMDestDir(os.path.basename(srpmFile), constants.sourceRpmPath) srpmDestFile = srpmDestDir + "/" + srpmName if not os.path.isfile(srpmDestFile): self.logger.error("Could not find RPM file: " + srpmDestFile) raise Exception("Built SRPM file not found.")