def installRPMSInAOneShot(self,chrootID,destLogPath): chrootCmd=self.runInChrootCommand+" "+chrootID rpmInstallcmd=self.rpmBinary+" "+ self.installRPMPackageOptions if self.noDepsRPMFilesToInstallInAOneShot != "": self.logger.info("Installing nodeps rpms: " + self.noDepsPackagesToInstallInAOneShot) logFile=chrootID+constants.topDirPath+"/LOGS/install_rpms_nodeps.log" cmdUtils = CommandUtils() cmd = rpmInstallcmd+" "+self.nodepsRPMPackageOptions + " " + self.noDepsRPMFilesToInstallInAOneShot returnVal = cmdUtils.runCommandInShell(cmd, logFile, chrootCmd) if destLogPath is not None: shutil.copy2(logFile, destLogPath) if not returnVal: self.logger.error("Unable to install rpms") raise Exception("RPM installation failed") if self.rpmFilesToInstallInAOneShot != "": self.logger.info("Installing rpms: " + self.packagesToInstallInAOneShot) logFile=chrootID+constants.topDirPath+"/LOGS/install_rpms.log" cmdUtils = CommandUtils() cmd=rpmInstallcmd+" "+self.rpmFilesToInstallInAOneShot returnVal = cmdUtils.runCommandInShell(cmd, logFile, chrootCmd) if destLogPath is not None: shutil.copy2(logFile, destLogPath) if not returnVal: self.logger.error("Unable to install rpms") raise Exception("RPM installation failed")
def installRPM(self, package, chrootID, noDeps=False, destLogPath=None): self.logger.info("Installing rpm for package:" + package) self.logger.debug("No deps:" + str(noDeps)) rpmfile = self.findRPMFileForGivenPackage(package) if rpmfile is None: self.logger.error("No rpm file found for package:" + package) raise Exception("Missing rpm file") rpmDestFile = self.copyRPM(rpmfile, chrootID + constants.topDirPath + "/RPMS") rpmFile = rpmDestFile.replace(chrootID, "") chrootCmd = self.runInChrootCommand + " " + chrootID logFile = chrootID + constants.topDirPath + "/LOGS" + "/" + package + ".completed" rpmInstallcmd = self.rpmBinary + " " + self.installRPMPackageOptions if noDeps: rpmInstallcmd += " " + self.nodepsRPMPackageOptions rpmInstallcmd += " " + rpmFile cmdUtils = CommandUtils() returnVal = cmdUtils.runCommandInShell(rpmInstallcmd, logFile, chrootCmd) if destLogPath is not None: shutil.copy2(logFile, destLogPath) if not returnVal: self.logger.error("Unable to install rpm:" + rpmFile) raise Exception("RPM installation failed")
def loadPackagesData(self): listPackages = SPECS.getData().getListPackages() listPackages.sort() cmdUtils = CommandUtils() for package in listPackages: for version in SPECS.getData().getVersions(package): release = SPECS.getData().getRelease(package, version) listRPMPackages = SPECS.getData().getRPMPackages(package, version) srpmFileName = package + "-" + version + "-" + release + ".src.rpm" srpmFiles = cmdUtils.findFile(srpmFileName, constants.sourceRpmPath) srpmFile = None if len(srpmFiles) == 1: srpmFile = srpmFiles[0] debugrpmFileName = package + "-debuginfo-" + version + "-" + release + "*" debugrpmFiles = cmdUtils.findFile(debugrpmFileName, constants.rpmPath) debugrpmFile = None if len(debugrpmFiles) == 1: debugrpmFile = debugrpmFiles[0] pkgUtils = PackageUtils(self.logName, self.logPath) for rpmPkg in listRPMPackages: rpmFile = pkgUtils.findRPMFile(rpmPkg, version) if rpmFile is not None: listPkgAttributes = {"sourcerpm":srpmFile, "rpm":rpmFile, "debugrpm":debugrpmFile} self.pkgList[rpmPkg+"-"+version] = listPkgAttributes self.logger.debug("Added " + rpmPkg + "-" + version + " to the package info json") else: self.logger.debug("Missing rpm file for package:" + rpmPkg)
def buildRPM(self,specFile,logFile,chrootCmd,package,macros): rpmBuildcmd= self.rpmbuildBinary+" "+self.rpmbuildBuildallOption+" "+self.rpmbuildDistOption if not constants.rpmCheck: rpmBuildcmd+=" "+self.rpmbuildNocheckOption for macro in macros: rpmBuildcmd+=' --define \\\"%s\\\"' % macro rpmBuildcmd+=" "+self.rpmbuildBuildNum+" "+self.rpmbuildReleaseVer rpmBuildcmd+=" "+specFile cmdUtils = CommandUtils() self.logger.info("Building rpm....") self.logger.info(rpmBuildcmd) returnVal = cmdUtils.runCommandInShell(rpmBuildcmd, logFile, chrootCmd) if not returnVal: self.logger.error("Building rpm is failed "+specFile) raise Exception("RPM Build failed") #Extracting rpms created from log file logfile=open(logFile,'r') fileContents=logfile.readlines() logfile.close() listRPMFiles=[] listSRPMFiles=[] for i in range(0,len(fileContents)): if re.search("^Wrote:",fileContents[i]): listcontents=fileContents[i].split() if (len(listcontents) == 2) and listcontents[1].strip()[-4:] == ".rpm" and listcontents[1].find("/RPMS/") != -1: listRPMFiles.append(listcontents[1]) if (len(listcontents) == 2) and listcontents[1].strip()[-8:] == ".src.rpm" and listcontents[1].find("/SRPMS/") != -1: listSRPMFiles.append(listcontents[1]) return listRPMFiles,listSRPMFiles
def copySourcesTobuildroot(self,listSourceFiles,package,destDir): cmdUtils = CommandUtils() for source in listSourceFiles: # Fetch/verify sources if sha1 not None. sha1 = constants.specData.getSHA1(package, source) if sha1 is not None: PullSources.get(source, sha1, constants.sourcePath, constants.pullsourcesConfig) sourcePath = cmdUtils.findFile(source,constants.sourcePath) if sourcePath is None or len(sourcePath) == 0: sourcePath = cmdUtils.findFile(source,constants.specPath) if sourcePath is None or len(sourcePath) == 0: if sha1 is None: self.logger.error("No sha1 found or missing source for "+source) raise Exception("No sha1 found or missing source") else: self.logger.error("Missing source: "+source+". Cannot find sources for package: "+package) raise Exception("Missing source") else: if sha1 is None: self.logger.error("No sha1 found for "+source) raise Exception("No sha1 found") if len(sourcePath) > 1: self.logger.error("Multiple sources found for source:"+source+"\n"+ ",".join(sourcePath) +"\nUnable to determine one.") raise Exception("Multiple sources found") self.logger.info("Copying... Source path :" + source + " Source filename: " + sourcePath[0]) shutil.copy2(sourcePath[0], destDir)
def buildPackage(self,package): #should initialize a logger based on package name chrUtils = ChrootUtils(self.logName,self.logPath) chrootName="build-"+package chrootID=None isToolChainPackage=False if package in constants.listToolChainPackages: isToolChainPackage=True try: chrootID = self.prepareBuildRoot(chrootName,isToolChainPackage) destLogPath=constants.logPath+"/build-"+package if not os.path.isdir(destLogPath): cmdUtils = CommandUtils() cmdUtils.runCommandInShell("mkdir -p "+destLogPath) listInstalledPackages=self.findInstalledPackages(chrootID) self.logger.info("List of installed packages") self.logger.info(listInstalledPackages) listDependentPackages=self.findBuildTimeRequiredPackages(package) if len(listDependentPackages) != 0: self.logger.info("Installing the build time dependent packages......") for pkg in listDependentPackages: self.installPackage(pkg,chrootID,destLogPath,listInstalledPackages) self.logger.info("Finished installing the build time dependent packages......") self.adjustGCCSpecs(package, chrootID, destLogPath) pkgUtils = PackageUtils(self.logName,self.logPath) pkgUtils.buildRPMSForGivenPackage(package,chrootID,destLogPath) self.logger.info("Successfully built the package:"+package) except Exception as e: self.logger.error("Failed while building package:" + package) self.logger.debug("Chroot with ID: " + chrootID + " not deleted for debugging.") raise e if chrootID is not None: chrUtils.destroyChroot(chrootID)
def _copySourcesToContainer(self, listSourceFiles, package, containerID, destDir, index=0): cmdUtils = CommandUtils() for source in listSourceFiles: sourcePath = self._verifyShaAndGetSourcePath(source, package, index) self.logger.info("Copying source file: " + sourcePath[0]) copyCmd = "docker cp " + sourcePath[0] + " " + containerID.short_id + ":" + destDir cmdUtils.runCommandInShell(copyCmd)
def _verifyShaAndGetSourcePath(self, source, package, index=0): cmdUtils = CommandUtils() # Fetch/verify sources if sha1 not None. sha1 = SPECS.getData().getSHA1(package, source, index) if sha1 is not None: PullSources.get(package, source, sha1, constants.sourcePath, constants.pullsourcesConfig, self.logger) sourcePath = cmdUtils.findFile(source, constants.sourcePath) if not sourcePath: sourcePath = cmdUtils.findFile(source, constants.specPath) if not sourcePath: if sha1 is None: self.logger.error("No sha1 found or missing source for " + source) raise Exception("No sha1 found or missing source for " + source) else: self.logger.error("Missing source: " + source + ". Cannot find sources for package: " + package) raise Exception("Missing source") else: if sha1 is None: self.logger.error("No sha1 found for "+source) raise Exception("No sha1 found") if len(sourcePath) > 1: self.logger.error("Multiple sources found for source:" + source + "\n" + ",".join(sourcePath) +"\nUnable to determine one.") raise Exception("Multiple sources found") return sourcePath
def adjustGCCSpecs(self, package, chrootID, logPath): opt = "" # TODO: reading of hardening flag from spec files if package == "linux" or package == "glibc": opt = " clean" elif package.startswith("xf86-") or package.startswith("xorg-server") : opt = " nonow" shutil.copy2(self.adjustGCCSpecScript, chrootID+"/tmp/"+self.adjustGCCSpecScript) cmdUtils=CommandUtils() cmd = "/tmp/"+self.adjustGCCSpecScript+opt logFile = logPath+"/adjustGCCSpecScript.log" chrootCmd=self.runInChrootCommand+" "+chrootID retryCount=10 returnVal=False while retryCount > 0: returnVal = cmdUtils.runCommandInShell(cmd, logFile, chrootCmd) if returnVal: return self.logger.error("Failed while adjusting gcc specs") self.logger.error("Retrying again .....") retryCount = retryCount - 1 sleep(5) if not returnVal: self.logger.error("Failed while adjusting gcc specs") raise "Failed while adjusting gcc specs"
def buildCoreToolChainPackages(self): self.logger.info("Building core tool chain packages.....") chrootID=None try: pkgUtils=PackageUtils(self.logName,self.logPath) for package in constants.listCoreToolChainRPMPackages: rpmPkg=pkgUtils.findRPMFileForGivenPackage(package) if rpmPkg is not None: continue chrUtils = ChrootUtils(self.logName,self.logPath) chrootName="build-core-toolchain" destLogPath=constants.logPath+"/build-"+package if not os.path.isdir(destLogPath): cmdUtils = CommandUtils() cmdUtils.runCommandInShell("mkdir -p "+destLogPath) returnVal,chrootID = chrUtils.createChroot(chrootName) if not returnVal: self.logger.error("Creating chroot failed") raise Exception("creating chroot failed") self.installToolChainRPMS(chrootID) pkgUtils.adjustGCCSpecs(package, chrootID, destLogPath) pkgUtils.buildRPMSForGivenPackage(package, chrootID,destLogPath) chrUtils.destroyChroot(chrootID) chrootID=None self.logger.info("Successfully built toolchain") if chrootID is not None: chrUtils.destroyChroot(chrootID) except Exception as e: self.logger.error("Unable to build tool chain.") # print stacktrace traceback.print_exc() raise e
def get(package, source, sha1, sourcesPath, URLs, logger): cmdUtils = CommandUtils() sourcePath = cmdUtils.findFile(source, sourcesPath) if sourcePath is not None and len(sourcePath) > 0: if len(sourcePath) > 1: raise Exception("Multiple sources found for source:" + source + "\n" + ",".join(sourcePath) +"\nUnable to determine one.") if sha1 == getFileHash(sourcePath[0]): # Use file from sourcesPath return else: logger.info("sha1 of " + sourcePath[0] + " does not match. " + sha1 + " vs " + getFileHash(sourcePath[0])) for baseurl in URLs: #form url: https://dl.bintray.com/vmware/photon_sources/1.0/<filename>. url = '%s/%s' % (baseurl, source) destfile = os.path.join(sourcesPath, source) logger.debug("Downloading: " + url) try: downloadFile(url, destfile) if sha1 != getFileHash(destfile): raise Exception('Invalid sha1 for package %s file %s' % package, source) return except requests.exceptions.HTTPError as e: logger.exception(e) # on any HTTP errors - try next config continue except Exception as e: logger.exception(e) raise Exception("Missing source: " + source)
def installCustomToolChainRPMS(self, chrootID, listOfToolChainPkgs, packageName): self.logger.info("Installing package specific tool chain RPMs for " + packageName + ".......") rpmFiles = "" packages = "" cmdUtils = CommandUtils() for package in listOfToolChainPkgs: pkgUtils = PackageUtils(self.logName, self.logPath) print("DEBUG:" + package) if "openjre8" in packageName or "openjdk8" in packageName: # x86_64 has openjdk/jre as a published rpms but aarch64 has openjdk8/jre8 # Remove this condition after publishxrpms for x86_^4 got updated if ((package == "openjdk" or package == "openjre") and platform.machine() == "aarch64"): package = package + "8" rpmFile = self.findRPMFileInGivenLocation(package, constants.prevPublishXRPMRepo) else: rpmFile = self.findRPMFileInGivenLocation(package, constants.prevPublishRPMRepo) if rpmFile is None: self.logger.error("Unable to find rpm "+ package + " in current and previous versions") raise Exception("Input Error") rpmFiles += " " + rpmFile packages += " " + package self.logger.debug("Installing custom rpms:" + packages) cmd = (self.rpmCommand + " -i -v --nodeps --noorder --force --root " + chrootID + " --define \'_dbpath /var/lib/rpm\' " + rpmFiles) retVal = cmdUtils.runCommandInShell(cmd, self.logPath + "/install_custom_toolchain_rpms.log") if not retVal: self.logger.debug("Command Executed:" + cmd) self.logger.error("Installing tool chain failed") raise Exception("RPM installation failed") self.logger.info("Successfully installed all Tool Chain X RPMS")
def loadPackagesData(self): listPackages = constants.specData.getListPackages() listPackages.sort() listRPMFiles = [] cmdUtils = CommandUtils() for package in listPackages: release = constants.specData.getRelease(package) version = constants.specData.getVersion(package) listRPMPackages = constants.specData.getRPMPackages(package) srpmFileName = package+"-"+version+"-"+release+".src.rpm" srpmFiles = cmdUtils.findFile(srpmFileName, constants.sourceRpmPath) srpmFile = None if len(srpmFiles) == 1: srpmFile = srpmFiles[0] debugrpmFileName = package+"-debuginfo-"+version+"-"+release+"*" debugrpmFiles = cmdUtils.findFile(debugrpmFileName, constants.rpmPath) debugrpmFile = None if len(debugrpmFiles) == 1: debugrpmFile = debugrpmFiles[0] pkgUtils = PackageUtils(self.logName,self.logPath) for rpmPkg in listRPMPackages: rpmFile = pkgUtils.findRPMFileForGivenPackage(rpmPkg) if rpmFile is not None: listRPMFiles.append(rpmFile) listPkgAttributes = {"sourcerpm":srpmFile, "rpm":rpmFile, "debugrpm":debugrpmFile} self.pkgList[rpmPkg] = listPkgAttributes self.logger.debug("Added "+rpmPkg +" rpm package to the list") else: self.logger.error("Missing rpm file for package:"+rpmPkg)
def buildRPM(self, specFile, logFile, chrootCmd): rpmBuildcmd = self.rpmbuildBinary + " " + self.rpmbuildBuildallOption + " " + self.rpmbuildNocheckOption rpmBuildcmd += " " + specFile cmdUtils = CommandUtils() returnVal = cmdUtils.runCommandInShell(rpmBuildcmd, logFile, chrootCmd) if not returnVal: self.logger.error("Building rpm is failed " + specFile) raise Exception("RPM Build failed") # Extracting rpms created from log file logfile = open(logFile, "r") fileContents = logfile.readlines() logfile.close() listRPMFiles = [] for i in range(0, len(fileContents)): if re.search("^Wrote:", fileContents[i]): listcontents = fileContents[i].split() if ( (len(listcontents) == 2) and listcontents[1].strip()[-4:] == ".rpm" and listcontents[1].find("/RPMS/") != -1 ): listRPMFiles.append(listcontents[1]) return listRPMFiles
def findRPMFileForGivenPackage(self, package,version="*", index=0): cmdUtils = CommandUtils() release = "*" if version == "*": version = SPECS.getData().getVersion(package, index) release = SPECS.getData().getRelease(package, index) listFoundRPMFiles = sum([cmdUtils.findFile(package + "-" + version + "-" + release + "." + platform.machine()+".rpm", constants.rpmPath), cmdUtils.findFile(package + "-" + version + "-" + release + ".noarch.rpm", constants.rpmPath)], []) if constants.inputRPMSPath is not None: listFoundRPMFiles = sum([cmdUtils.findFile(package + "-" + version + "-" + release + "." + platform.machine()+".rpm", constants.inputRPMSPath), cmdUtils.findFile(package + "-" + version + "-" + release + ".noarch.rpm", constants.inputRPMSPath)], listFoundRPMFiles) if len(listFoundRPMFiles) == 1: return listFoundRPMFiles[0] if len(listFoundRPMFiles) == 0: return None if len(listFoundRPMFiles) > 1: self.logger.error("Found multiple rpm files for given package in rpm directory." + "Unable to determine the rpm file for package:" + package) raise Exception("Multiple rpm files found")
def main(): usage = "Usage: %prog [options]" parser = ArgumentParser(usage) parser.add_argument("-i", "--input-type", dest="input_type", default=DEFAULT_INPUT_TYPE) parser.add_argument("-p", "--pkg", dest="pkg") parser.add_argument("-f", "--file", dest="json_file", default="packages_minimal.json") parser.add_argument("-d", "--disp", dest="display_option", default=DEFAULT_DISPLAY_OPTION) parser.add_argument("-s", "--spec-dir", dest="spec_dir", default=SPEC_FILE_DIR) parser.add_argument("-l", "--log-dir", dest="log_dir", default=LOG_FILE_DIR) parser.add_argument("-t", "--stage-dir", dest="stage_dir", default="../../stage") parser.add_argument("-a", "--input-data-dir", dest="input_data_dir", default="../../common/data/") parser.add_argument("-o", "--output-dir", dest="output_dir", default="../../stage/common/data") options = parser.parse_args() constants.setSpecPath(options.spec_dir) constants.setLogPath(options.log_dir) constants.initialize() cmdUtils = CommandUtils() if not os.path.isdir(options.output_dir): cmdUtils.runCommandInShell2("mkdir -p "+options.output_dir) if not options.input_data_dir.endswith('/'): options.input_data_dir += '/' try: specDeps = SpecDependencyGenerator() # To display/print package dependencies on console if (options.input_type == "pkg" or options.input_type == "who-needs" or options.input_type == "who-needs-build"): specDeps.process(options.input_type, options.pkg, options.display_option) elif options.input_type == "json": list_json_files = options.json_file.split("\n") # Generate the expanded package dependencies json file based on package_list_file for json_file in list_json_files: shutil.copy2(json_file, options.output_dir) json_wrapper_option_list = JsonWrapper(json_file) option_list_json = json_wrapper_option_list.read() options_sorted = option_list_json.items() for install_option in options_sorted: output_file = None input_value = os.path.join(os.path.dirname(json_file), install_option[1]["file"]) if options.display_option == "tree" and install_option[1]["title"] == "ISO Packages": continue if options.display_option == "json": output_file = os.path.join(options.output_dir, install_option[1]["file"]) print ("Generating the install time dependency list for " + json_file) specDeps.process(options.input_type, input_value, options.display_option, output_file) except Exception as e: traceback.print_exc() sys.stderr.write(str(e)) sys.stderr.write("Failed to generate dependency lists from spec files\n") sys.exit(1) sys.stderr.write("Successfully generated dependency lists from spec files\n") sys.exit(0)
def findInstalledRPMPackages(self, chrootID): cmd = self.rpmBinary + " " + self.queryRpmPackageOptions chrootCmd = self.runInChrootCommand + " " + chrootID cmdUtils = CommandUtils() result = cmdUtils.runCommandInShell2(cmd, chrootCmd) if result is not None: return result.decode().split() return result
def _buildPackagePrepareFunction(self, package): self.package = package self.logName = "build-" + package self.logPath = constants.logPath + "/build-" + package if not os.path.isdir(self.logPath): cmdUtils = CommandUtils() cmdUtils.runCommandInShell("mkdir -p " + self.logPath) self.logger = Logger.getLogger(self.logName, self.logPath)
def writePkgListToFile(self, fileName): self.logger.debug("Writing package list to the json file") cmdUtils = CommandUtils() dirPath = os.path.basename(fileName) if not os.path.isdir(dirPath): cmdUtils.runCommandInShell("mkdir -p " + dirPath) with open(fileName, 'w+') as pkgInfoFile: json.dump(self.pkgList, pkgInfoFile, indent=4)
def writePkgListToFile(fileName): SourcePackageInfo.logger.info("Writing source package list to the json file") cmdUtils = CommandUtils() dirPath = os.path.basename(fileName) if not os.path.isdir(dirPath): cmdUtils.runCommandInShell("mkdir -p " + dirPath) pkgInfoFile = open(fileName, "w+") json.dump(SourcePackageInfo.sourcePkgList, pkgInfoFile, indent=4) pkgInfoFile.close()
def copyRPM(self, rpmFile, destDir): cmdUtils = CommandUtils() rpmName = os.path.basename(rpmFile) rpmDestDir = self.getRPMDestDir(rpmName, destDir) if not os.path.isdir(rpmDestDir): cmdUtils.runCommandInShell("mkdir -p " + rpmDestDir) rpmDestPath = rpmDestDir + "/" + rpmName shutil.copyfile(rpmFile, rpmDestPath) return rpmDestPath
def buildRPM(self,specFile,logFile,chrootCmd,package,macros): rpmBuildcmd=self.rpmbuildBinary+" "+self.rpmbuildBuildallOption+" "+self.rpmbuildDistOption if constants.rpmCheck and package in constants.testForceRPMS: self.logger.info("#"*(68+2*len(package))) if not constants.specData.isCheckAvailable(package): self.logger.info("####### "+package+" MakeCheck is not available. Skipping MakeCheck TEST for "+package+ " #######") rpmBuildcmd=self.rpmbuildBinary+" --clean" else: self.logger.info("####### "+package+" MakeCheck is available. Running MakeCheck TEST for "+package+ " #######") rpmBuildcmd=self.rpmbuildBinary+" "+self.rpmbuildCheckOption self.logger.info("#"*(68+2*len(package))) else: rpmBuildcmd+=" "+self.rpmbuildNocheckOption for macro in macros: rpmBuildcmd+=' --define \\\"%s\\\"' % macro rpmBuildcmd+=" "+specFile cmdUtils = CommandUtils() self.logger.info("Building rpm....") self.logger.info(rpmBuildcmd) returnVal = cmdUtils.runCommandInShell(rpmBuildcmd, logFile, chrootCmd) if constants.rpmCheck and package in constants.testForceRPMS: if not constants.specData.isCheckAvailable(package): constants.testLogger.info(package+" : N/A") elif returnVal: constants.testLogger.info(package+" : PASS") else: constants.testLogger.error(package+" : FAIL" ) if constants.rpmCheck: if not returnVal and constants.rpmCheckStopOnError: self.logger.error("Checking rpm is failed "+specFile) raise Exception("RPM check failed") else: if not returnVal: self.logger.error("Building rpm is failed "+specFile) raise Exception("RPM build failed") #Extracting rpms created from log file logfile=open(logFile,'r') fileContents=logfile.readlines() logfile.close() listRPMFiles=[] listSRPMFiles=[] for i in range(0,len(fileContents)): if re.search("^Wrote:",fileContents[i]): listcontents=fileContents[i].split() if (len(listcontents) == 2) and listcontents[1].strip()[-4:] == ".rpm" and listcontents[1].find("/RPMS/") != -1: listRPMFiles.append(listcontents[1]) if (len(listcontents) == 2) and listcontents[1].strip()[-8:] == ".src.rpm" and listcontents[1].find("/SRPMS/") != -1: listSRPMFiles.append(listcontents[1]) return listRPMFiles,listSRPMFiles
def installToolChainRPMS(self, chrootID, packageName, logPath=None): if logPath is None: logPath = self.logPath cmdUtils = CommandUtils() self.prepareBuildRoot(chrootID) self.logger.info("Installing Tool Chain RPMS.......") rpmFiles = "" packages = "" for package in constants.listToolChainRPMsToInstall: pkgUtils = PackageUtils(self.logName, self.logPath) rpmFile = None if constants.rpmCheck: rpmFile = pkgUtils.findRPMFileForGivenPackage(package) else: if (packageName not in constants.listToolChainRPMsToInstall or constants.listToolChainRPMsToInstall.index(packageName) > constants.listToolChainRPMsToInstall.index(package)): rpmFile = pkgUtils.findRPMFileForGivenPackage(package) if rpmFile is None: # sqlite-autoconf package was renamed, but it still published as sqlite-autoconf if (package == "sqlite") and (platform.machine() == "x86_64"): package = "sqlite-autoconf" rpmFile = self.findRPMFileInGivenLocation(package, constants.prevPublishRPMRepo) if rpmFile is None: if package in constants.listOfRPMsProvidedAfterBuild: self.logger.info("No old version of " + package + " exists, skip until the new version is built") continue self.logger.error("Unable to find rpm " + package + " in current and previous versions") raise Exception("Input Error") rpmFiles += " " + rpmFile packages += " " + package self.logger.debug("Installing toolchain rpms:" + packages) cmd = (self.rpmCommand + " -i -v --nodeps --noorder --force --root " + chrootID +" --define \'_dbpath /var/lib/rpm\' "+ rpmFiles) retVal = cmdUtils.runCommandInShell(cmd, logPath + "/install_toolchain_rpms.log") if not retVal: self.logger.debug("Command Executed:" + cmd) self.logger.error("Installing tool chain failed") raise Exception("RPM installation failed") self.logger.info("Successfully installed default Tool Chain RPMS in Chroot:" + chrootID) print("Building Package: ".format(packageName)) print(constants.perPackageToolChain) if packageName in constants.perPackageToolChain: print(constants.perPackageToolChain[packageName]) self.installCustomToolChainRPMS(chrootID, constants.perPackageToolChain[packageName], packageName) #convert rpm db to lmdb (if required) convertLogFile = self.logPath+"/rpmdbconvert.log" pkgUtilsConvert=PackageUtils(self.logName,self.logPath) pkgUtilsConvert.ensureLMDBForRPMDB(chrootID, convertLogFile)
def findRPMFileForGivenPackage(self,package): cmdUtils = CommandUtils() version = constants.specData.getVersion(package) release = constants.specData.getRelease(package) listFoundRPMFiles = cmdUtils.findFile(package+"-"+version+"-"+release+"*.rpm",constants.rpmPath) if len(listFoundRPMFiles) == 1 : return listFoundRPMFiles[0] if len(listFoundRPMFiles) == 0 : return None if len(listFoundRPMFiles) > 1 : self.logger.error("Found multiple rpm files for given package in rpm directory.Unable to determine the rpm file for package:"+package) raise Exception("Multiple rpm files found")
def buildPackage(self,package): #do not build if RPM is already built #test only if the package is in the testForceRPMS with rpmCheck #build only if the package is not in the testForceRPMS with rpmCheck if self.checkIfPackageIsAlreadyBuilt(package): if not constants.rpmCheck: self.logger.info("Skipping building the package:"+package) return elif constants.rpmCheck and package not in constants.testForceRPMS: self.logger.info("Skipping testing the package:"+package) return #should initialize a logger based on package name chrUtils = ChrootUtils(self.logName,self.logPath) chrootName="build-"+package chrootID=None isToolChainPackage=False if package in constants.listToolChainPackages: isToolChainPackage=True try: chrootID = self.prepareBuildRoot(chrootName,isToolChainPackage) destLogPath=constants.logPath+"/build-"+package if not os.path.isdir(destLogPath): cmdUtils = CommandUtils() cmdUtils.runCommandInShell("mkdir -p "+destLogPath) listInstalledPackages=self.findInstalledPackages(chrootID) listDependentPackages=self.findBuildTimeRequiredPackages(package) if constants.rpmCheck and package in constants.testForceRPMS: testPackages=set(constants.listMakeCheckRPMPkgtoInstall)-set(listInstalledPackages)-set([package]) listDependentPackages.extend(testPackages) pkgUtils = PackageUtils(self.logName,self.logPath) if len(listDependentPackages) != 0: self.logger.info("Installing the build time dependent packages......") for pkg in listDependentPackages: self.installPackage(pkgUtils, pkg,chrootID,destLogPath,listInstalledPackages) pkgUtils.installRPMSInAOneShot(chrootID,destLogPath) self.logger.info("Finished installing the build time dependent packages......") pkgUtils.adjustGCCSpecs(package, chrootID, destLogPath) pkgUtils.buildRPMSForGivenPackage(package,chrootID,self.listBuildOptionPackages,self.pkgBuildOptionFile,destLogPath) self.logger.info("Successfully built the package:"+package) except Exception as e: self.logger.error("Failed while building package:" + package) self.logger.debug("Chroot with ID: " + chrootID + " not deleted for debugging.") logFileName = os.path.join(destLogPath, package + ".log") fileLog = os.popen('tail -n 100 ' + logFileName).read() self.logger.debug(fileLog) raise e if chrootID is not None: chrUtils.destroyChroot(chrootID)
def buildSourcesList(yamlDir, blackListPkgs, logger, singleFile=True): cmdUtils = CommandUtils() yamlSourceDir = os.path.join(yamlDir, "yaml_sources") if not os.path.isdir(yamlSourceDir): cmdUtils.runCommandInShell("mkdir -p " + yamlSourceDir) if singleFile: yamlFile = open(yamlSourceDir + "/sources_list.yaml", "w") listPackages = SPECS.getData().getListPackages() listPackages.sort() import PullSources for package in listPackages: if package in blackListPkgs: continue ossname = package for version in SPECS.getData().getVersions(package): modified = False listPatches = SPECS.getData().getPatches(package, version) if listPatches: modified = True url = SPECS.getData().getSourceURL(package, version) if url is None: url = SPECS.getData().getURL(package, version) sourceName = None listSourceNames = SPECS.getData().getSources(package, version) if listSourceNames: sourceName = listSourceNames[0] sha1 = SPECS.getData().getSHA1(package, version, sourceName) if sha1 is not None: PullSources.get(package, sourceName, sha1, yamlSourceDir, constants.getPullSourcesURLs(package), logger) if not singleFile: yamlFile = open(yamlSourceDir + "/" + ossname + "-" + version + ".yaml", "w") yamlFile.write("vmwsource:" + ossname + ":" + version + ":\n") yamlFile.write(" repository: VMWsource\n") yamlFile.write(" name: '" + ossname + "'\n") yamlFile.write(" version: '" + version + "'\n") yamlFile.write(" url: " + str(url) + "\n") yamlFile.write(" license: UNKNOWN\n") if sourceName is not None: yamlFile.write(" vmwsource-distribution: " + str(sourceName) + "\n") if modified: yamlFile.write(" modified: true\n") yamlFile.write("\n") if not singleFile: yamlFile.close() if singleFile: yamlFile.close() logger.debug("Generated source yaml files for all packages")
def createChroot(self,chrootName): chrootID=constants.buildRootPath+"/"+chrootName if os.path.isdir(chrootID): if not self.destroyChroot(chrootID): self.logger.error("Given chroot "+chrootID+" is already exists. unable to destroy it ") return False,None # need to add timeout for this step # http://stackoverflow.com/questions/1191374/subprocess-with-timeout cmdUtils=CommandUtils() returnVal=cmdUtils.runCommandInShell("mkdir -p "+chrootID) if not returnVal: self.logger.error("Unable to create chroot:"+ chrootID +".Unknown error.") return False,None return True,chrootID
def buildSRPMList(srpmPath, yamlDir, blackListPkgs, dist_tag, logger, singleFile=True): cmdUtils = CommandUtils() yamlSrpmDir = os.path.join(yamlDir, "yaml_srpms") if not os.path.isdir(yamlSrpmDir): cmdUtils.runCommandInShell("mkdir -p " + yamlSrpmDir) if singleFile: yamlFile = open(yamlSrpmDir + "/srpm_list.yaml", "w") listPackages = SPECS.getData().getListPackages() listPackages.sort() for package in listPackages: if package in blackListPkgs: continue ossname = package for ossversion in SPECS.getData().getVersions(package): ossrelease = SPECS.getData().getRelease(package, ossversion) srpm_file_name = "%s-%s-%s%s.src.rpm" % (ossname, ossversion, ossrelease, dist_tag) logger.info("srpm name is %s" % (srpm_file_name)) listFoundSRPMFiles = cmdUtils.findFile(srpm_file_name, srpmPath) srpmName = None if len(listFoundSRPMFiles) == 1: srpmFullPath = listFoundSRPMFiles[0] srpmName = os.path.basename(srpmFullPath) cpcmd = "cp " + srpmFullPath + " " + yamlSrpmDir + "/" returnVal = cmdUtils.runCommandInShell(cpcmd) if returnVal != 0: logger.error("Copy SRPM File is failed for package:" + ossname) else: logger.error("SRPM file is not found:" + ossname) if not singleFile: yamlFile = open(yamlSrpmDir + "/" + ossname + "-" + ossversion + "-" + ossrelease + ".yaml", "w") yamlFile.write("baseos:" + ossname + ":" + ossversion + "-" + ossrelease + dist_tag + ":\n") yamlFile.write(" repository: BaseOS\n") yamlFile.write(" name: '" + ossname + "'\n") yamlFile.write(" version: '" + ossversion + "-" + ossrelease + dist_tag +"'\n") yamlFile.write(" url: 'http://www.vmware.com'\n") yamlFile.write(" baseos-style: rpm\n") yamlFile.write(" baseos-source: '" + str(srpmName) + "'\n") yamlFile.write(" baseos-osname: 'photon'\n") yamlFile.write("\n") if not singleFile: yamlFile.close() if singleFile: yamlFile.close() logger.debug("Generated SRPM yaml files for all packages")
def _copyRPM(self, rpmFile, destDir): cmdUtils = CommandUtils() rpmName = os.path.basename(rpmFile) rpmDestDir = self._getRPMDestDir(rpmName, destDir) # shutil is not atomic. copy & move to ensure atomicity. rpmDestPath = rpmDestDir + "/" + rpmName rpmDestPathTemp = (rpmDestDir + "/." + ''.join([random.choice(string.ascii_letters + string.digits) for n in range(10)])) if os.geteuid() == 0: if not os.path.isdir(rpmDestDir): cmdUtils.runCommandInShell("mkdir -p " + rpmDestDir) shutil.copyfile(rpmFile, rpmDestPathTemp) shutil.move(rpmDestPathTemp, rpmDestPath) return rpmDestPath
def destroyChroot(self,chrootID): # need to add timeout for this step # http://stackoverflow.com/questions/1191374/subprocess-with-timeout cmdUtils=CommandUtils() returnVal=cmdUtils.runCommandInShell("./cleanup-build-root.sh "+chrootID) if not returnVal: self.logger.error("Unable to destroy chroot:"+ chrootID +".Unknown error.") return False returnVal=cmdUtils.runCommandInShell("rm -rf "+chrootID) if not returnVal: self.logger.error("Unable to destroy chroot:"+ chrootID +".Unknown error.") return False self.logger.info("Successfully destroyed chroot:"+chrootID) return True
def loadPackagesData(self): listPackages = SPECS.getData().getListPackages() listPackages.sort() listRPMFiles = [] cmdUtils = CommandUtils() for package in listPackages: for version in SPECS.getData().getVersions(package): release = SPECS.getData().getRelease(package, version) listRPMPackages = SPECS.getData().getRPMPackages( package, version) srpmFileName = package + "-" + version + "-" + release + ".src.rpm" srpmFiles = cmdUtils.findFile(srpmFileName, constants.sourceRpmPath) srpmFile = None if len(srpmFiles) == 1: srpmFile = srpmFiles[0] debugrpmFileName = package + "-debuginfo-" + version + "-" + release + "*" debugrpmFiles = cmdUtils.findFile(debugrpmFileName, constants.rpmPath) debugrpmFile = None if len(debugrpmFiles) == 1: debugrpmFile = debugrpmFiles[0] pkgUtils = PackageUtils(self.logName, self.logPath) for rpmPkg in listRPMPackages: rpmFile = pkgUtils.findRPMFileForGivenPackage(rpmPkg) if rpmFile is not None: listRPMFiles.append(rpmFile) listPkgAttributes = { "sourcerpm": srpmFile, "rpm": rpmFile, "debugrpm": debugrpmFile } self.pkgList[rpmPkg] = listPkgAttributes self.logger.debug("Added " + rpmPkg + " rpm package to the list") else: self.logger.debug("Missing rpm file for package:" + rpmPkg)
def installCustomToolChainRPMS(self, chrootID, listOfToolChainPkgs, packageName): self.logger.debug("Installing package specific tool chain RPMs for " + packageName + ".......") rpmFiles = "" packages = "" cmdUtils = CommandUtils() for package in listOfToolChainPkgs: pkgUtils = PackageUtils(self.logName, self.logPath) if re.match("openjre*", packageName) is not None or re.match( "openjdk*", packageName): # x86_64 has openjdk/jre as a published rpms but aarch64 has openjdk8/jre8 # Remove this condition after publishxrpms for x86_^4 got updated if ((package == "openjdk" or package == "openjre") and platform.machine() == "aarch64"): package = package + "8" rpmFile = self.findRPMFileInGivenLocation( package, constants.prevPublishXRPMRepo) else: rpmFile = self.findRPMFileInGivenLocation( package, constants.prevPublishRPMRepo) if rpmFile is None: self.logger.error("Unable to find rpm " + package + " in current and previous versions") raise Exception("Input Error") rpmFiles += " " + rpmFile packages += " " + package self.logger.debug("Installing custom rpms:" + packages) cmd = (self.rpmCommand + " -i -v --nodeps --noorder --force --root " + chrootID + " --define \'_dbpath /var/lib/rpm\' " + rpmFiles) retVal = cmdUtils.runCommandInShell( cmd, self.logPath + "/install_custom_toolchain_rpms.log") if not retVal: self.logger.debug("Command Executed:" + cmd) self.logger.error("Installing tool chain failed") raise Exception("RPM installation failed") self.logger.debug("Successfully installed all Tool Chain X RPMS")
def buildRPMSForGivenPackage(self, sandbox, package, version, destLogPath): self.logger.info("Building package : " + package) listSourcesFiles = SPECS.getData().getSources(package, version) listPatchFiles = SPECS.getData().getPatches(package, version) specFile = SPECS.getData().getSpecFile(package, version) specName = SPECS.getData().getSpecName(package) + ".spec" sourcePath = constants.topDirPath + "/SOURCES/" specPath = constants.topDirPath + "/SPECS/" if (constants.rpmCheck and package in constants.testForceRPMS and SPECS.getData().isCheckAvailable(package, version)): logFilePath = destLogPath + "/" + package + "-test.log" else: logFilePath = destLogPath + "/" + package + ".log" sandbox.put(specFile, specPath + specName) sources_urls, macros = self._getAdditionalBuildOptions(package) self.logger.debug("Extra macros for " + package + ": " + str(macros)) self.logger.debug("Extra source URLs for " + package + ": " + str(sources_urls)) constants.setExtraSourcesURLs(package, sources_urls) self._copySources(sandbox, listSourcesFiles, package, version, sourcePath) self._copySources(sandbox, listPatchFiles, package, version, sourcePath) #Adding rpm macros listRPMMacros = constants.userDefinedMacros for macroName, value in listRPMMacros.items(): macros.append(macroName + " " + value) listRPMFiles = [] listSRPMFiles = [] try: listRPMFiles, listSRPMFiles = self._buildRPM(sandbox, specPath + specName, logFilePath, package, version, macros) logmsg = package + " build done - RPMs : [ " for f in listRPMFiles: logmsg += (os.path.basename(f) + " ") logmsg += "]\n" self.logger.info(logmsg) except Exception as e: self.logger.error("Failed while building rpm:" + package) raise e finally: if (constants.rpmCheck and package in constants.testForceRPMS and SPECS.getData().isCheckAvailable(package, version)): cmd = ("sed -i '/^Executing(%check):/,/^Processing files:/{//!b};d' " + logFilePath) CommandUtils().runCommandInShell(cmd, logfn=self.logger.debug) self.logger.debug("RPM build is successful")
def buildRPM(self,specFile,logFile,chrootCmd): rpmBuildcmd= self.rpmbuildBinary+" "+self.rpmbuildBuildallOption+" "+self.rpmbuildNocheckOption +" "+self.rpmbuildDistOption rpmBuildcmd+=" "+specFile cmdUtils = CommandUtils() returnVal = cmdUtils.runCommandInShell(rpmBuildcmd, logFile, chrootCmd) if not returnVal: self.logger.error("Building rpm is failed "+specFile) raise Exception("RPM Build failed") #Extracting rpms created from log file logfile=open(logFile,'r') fileContents=logfile.readlines() logfile.close() listRPMFiles=[] for i in range(0,len(fileContents)): if re.search("^Wrote:",fileContents[i]): listcontents=fileContents[i].split() if (len(listcontents) == 2) and listcontents[1].strip()[-4:] == ".rpm" and listcontents[1].find("/RPMS/") != -1: listRPMFiles.append(listcontents[1]) return listRPMFiles
def buildCoreToolChainPackages(self): self.logger.info("Building core toolchain packages.....") chrootID = None pkgCount = 0 try: pkgUtils = PackageUtils(self.logName, self.logPath) for package in constants.listCoreToolChainPackages: rpmPkg = pkgUtils.findRPMFileForGivenPackage(package) if rpmPkg is not None: continue self.logger.info("Building core toolchain package: " + package) chrUtils = ChrootUtils(self.logName, self.logPath) chrootName = "build-" + package destLogPath = constants.logPath + "/build-" + package if not os.path.isdir(destLogPath): cmdUtils = CommandUtils() cmdUtils.runCommandInShell("mkdir -p " + destLogPath) returnVal, chrootID = chrUtils.createChroot(chrootName) if not returnVal: self.logger.error("Creating chroot failed") raise Exception("creating chroot failed") self.installToolChainRPMS(chrootID, package, destLogPath) pkgUtils.adjustGCCSpecs(package, chrootID, destLogPath) pkgUtils.buildRPMSForGivenPackage(package, chrootID, destLogPath) pkgCount += 1 chrUtils.destroyChroot(chrootID) chrootID = None self.logger.info("Successfully built toolchain") if chrootID is not None: chrUtils.destroyChroot(chrootID) except Exception as e: self.logger.error("Unable to build tool chain.") # print stacktrace traceback.print_exc() raise e return pkgCount
def _copyAdditionalBuildFilesToContainer(self, listAdditionalFiles, containerID): cmdUtils = CommandUtils() #self.logger.debug("VDBG-PU-copyAdditionalBuildFilesToContainer id: " + # containerID.short_id) #self.logger.debug(listAdditionalFiles) for additionalFile in listAdditionalFiles: source = additionalFile["src"] destDir = additionalFile["dst"] destPath = containerID.short_id + ":" + destDir #TODO: exit status of exec_run containerID.exec_run("mkdir -p " + destDir) if os.path.exists(source): copyCmd = "docker cp " + source if os.path.isfile(source): self.logger.info("Copying additional source file: " + source) copyCmd += " " + destPath else: self.logger.info("Copying additional source file tree: " + source) copyCmd += "/. " + destPath #TODO: cmd error code cmdUtils.runCommandInShell(copyCmd)
def get(source, sha1, sourcesPath, configs, logger): cmdUtils = CommandUtils() sourcePath = cmdUtils.findFile(source, sourcesPath) if sourcePath is not None and len(sourcePath) > 0: if len(sourcePath) > 1: raise Exception("Multiple sources found for source:" + source + "\n" + ",".join(sourcePath) + "\nUnable to determine one.") if sha1 == getFileHash(sourcePath[0]): # Use file from sourcesPath return else: logger.info("sha1 of " + sourcePath[0] + " does not match. " + sha1 + " vs " + getFileHash(sourcePath[0])) configFiles = configs.split(":") for config in configFiles: p = pullSources(config, logger) package_path = os.path.join(sourcesPath, source) try: p.downloadFileHelper(source, package_path, sha1) return except Exception as e: logger.exception(e) raise Exception("Missing source: " + source)
def installRPMSInAOneShot(self, chrootID, destLogPath): chrootCmd = self.runInChrootCommand + " " + chrootID rpmInstallcmd = self.rpmBinary + " " + self.installRPMPackageOptions cmdUtils = CommandUtils() if self.noDepsRPMFilesToInstallInAOneShot != "": self.logger.info("Installing nodeps rpms: " + self.noDepsPackagesToInstallInAOneShot) logFile = destLogPath + "/install_rpms_nodeps.log" cmd = (rpmInstallcmd+" "+self.nodepsRPMPackageOptions + " " + self.noDepsRPMFilesToInstallInAOneShot) returnVal = cmdUtils.runCommandInShell(cmd, logFile, chrootCmd) if not returnVal: self.logger.debug("Command Executed:" + cmd) self.logger.error("Unable to install rpms") raise Exception("RPM installation failed") if self.rpmFilesToInstallInAOneShot != "": self.logger.info("Installing rpms: " + self.packagesToInstallInAOneShot) logFile = destLogPath+"/install_rpms.log" cmd = rpmInstallcmd+" "+self.rpmFilesToInstallInAOneShot returnVal = cmdUtils.runCommandInShell(cmd, logFile, chrootCmd) if not returnVal: self.logger.debug("Command Executed:" + cmd) self.logger.error("Unable to install rpms") raise Exception("RPM installation failed")
def __init__(self, logName=None, logPath=None, pkgBuildType="chroot"): if logName is None: logName = "PackageManager" if logPath is None: logPath = constants.logPath self.logName = logName self.logPath = logPath self.logLevel = constants.logLevel self.logger = Logger.getLogger(logName, logPath, constants.logLevel) self.mapCyclesToPackageList = {} self.mapPackageToCycle = {} self.sortedPackageList = [] self.listOfPackagesAlreadyBuilt = set() self.pkgBuildType = pkgBuildType if self.pkgBuildType == "container": import docker self.dockerClient = docker.from_env(version="auto") cmdUtils = CommandUtils() # if rpm doesnt have zstd support if cmdUtils.runCommandInShell( 'rpm --showrc | grep -i "rpmlib(PayloadIsZstd)"', logfn=self.logger.debug): constants.hostRpmIsNotUsable = True self.createZstdBuilderImage()
def _createBuildContainer(self, usePublishedRPMs): self.logger.debug("Generating photon build container..") try: #TODO image name constants.buildContainerImageName self.dockerClient.images.remove(constants.buildContainerImage, force=True) except Exception as e: #TODO - better handling self.logger.debug("Photon build container image not found.") # Create toolchain chroot and install toolchain RPMs chroot = None try: #TODO: constants.tcrootname chroot = Chroot(self.logger) chroot.create("toolchain-chroot") tcUtils = ToolChainUtils("toolchain-chroot", self.logPath) tcUtils.installToolchainRPMS(chroot, usePublishedRPMS=usePublishedRPMs) except Exception as e: if chroot: chroot.destroy() raise e self.logger.debug("createBuildContainer: " + chroot.getID()) # Create photon build container using toolchain chroot chroot.unmountAll() #TODO: Coalesce logging cmdUtils = CommandUtils() cmd = "cd " + chroot.getID() + " && tar -czf ../tcroot.tar.gz ." cmdUtils.runCommandInShell(cmd, logfn=self.logger.debug) cmd = "mv " + chroot.getID() + "/../tcroot.tar.gz ." cmdUtils.runCommandInShell(cmd, logfn=self.logger.debug) #TODO: Container name, docker file name from constants. self.dockerClient.images.build( tag=constants.buildContainerImage, path=".", rm=True, dockerfile="Dockerfile.photon_build_container") # Cleanup cmd = "rm -f ./tcroot.tar.gz" cmdUtils.runCommandInShell(cmd, logfn=self.logger.debug) chroot.destroy() self.logger.debug("Photon build container successfully created.")
def _findPublishedRPM(self, package, rpmdirPath): listFoundRPMFiles = CommandUtils.findFile(package + "-*.rpm", rpmdirPath) listFilterRPMFiles = [] for f in listFoundRPMFiles: rpmFileName = os.path.basename(f) checkRPMName = rpmFileName.replace(package, "") rpmNameSplit = checkRPMName.split("-") if len(rpmNameSplit) == 3: listFilterRPMFiles.append(f) if len(listFilterRPMFiles) == 1: return listFilterRPMFiles[0] if len(listFilterRPMFiles) == 0: return None if len(listFilterRPMFiles) > 1: self.logger.error("Found multiple rpm files for given package in rpm directory." + "Unable to determine the rpm file for package:" + package) return None
def installToolChainRPMS(self, chrootID, packageName): cmdUtils = CommandUtils() self.prepareBuildRoot(chrootID) self.logger.info("Installing Tool Chain RPMS.......") rpmFiles = "" packages = "" for package in constants.listToolChainRPMsToInstall: pkgUtils = PackageUtils(self.logName, self.logPath) rpmFile = pkgUtils.findRPMFileForGivenPackage(package) if rpmFile is None: # sqlite-autoconf package was renamed, but it still published as sqlite-autoconf if package == "sqlite": package = "sqlite-autoconf" rpmFile = self.findRPMFileInGivenLocation( package, constants.prevPublishRPMRepo) if rpmFile is None: if package in constants.listOfRPMsProvidedAfterBuild: self.logger.info( "No old version of " + package + " exists, skip until the new version is built") continue self.logger.error("Unable to find rpm " + package + " in current and previous versions") raise Exception("Input Error") rpmFiles += " " + rpmFile packages += " " + package self.logger.debug("Installing rpms:" + packages) cmd = self.rpmCommand + " -i --nodeps --force --root " + chrootID + " --define \'_dbpath /var/lib/rpm\' " + rpmFiles process = subprocess.Popen("%s" % cmd, shell=True, stdout=subprocess.PIPE) retval = process.wait() if retval != 0: self.logger.error("Installing tool chain failed") raise Exception("RPM installation failed") self.logger.info( "Successfully installed default Tool Chain RPMS in Chroot:" + chrootID) print "Building Package:" + packageName print constants.perPackageToolChain if packageName in constants.perPackageToolChain: print constants.perPackageToolChain[packageName] self.installCustomToolChainRPMS( chrootID, constants.perPackageToolChain[packageName], packageName)
def buildSRPMList(srpmPath, yamlDir, blackListPkgs, logger, singleFile=True): cmdUtils = CommandUtils() yamlSrpmDir = os.path.join(yamlDir, "yaml_srpms") if not os.path.isdir(yamlSrpmDir): cmdUtils.runCommandInShell("mkdir -p " + yamlSrpmDir) if singleFile: yamlFile = open(yamlSrpmDir + "/srpm_list.yaml", "w") listPackages = SPECS.getData().getListPackages() listPackages.sort() for package in listPackages: if package in blackListPkgs: continue ossname = package for ossversion in SPECS.getData().getVersions(package): ossrelease = SPECS.getData().getRelease(package, ossversion) listFoundSRPMFiles = cmdUtils.findFile( ossname + "-" + ossversion + "-" + ossrelease + ".src.rpm", srpmPath) srpmName = None if len(listFoundSRPMFiles) == 1: srpmFullPath = listFoundSRPMFiles[0] srpmName = os.path.basename(srpmFullPath) cpcmd = "cp " + srpmFullPath + " " + yamlSrpmDir + "/" returnVal = cmdUtils.runCommandInShell(cpcmd) if not returnVal: logger.error("Copy SRPM File is failed for package:" + ossname) else: logger.error("SRPM file is not found:" + ossname) if not singleFile: yamlFile = open( yamlSrpmDir + "/" + ossname + "-" + ossversion + "-" + ossrelease + ".yaml", "w") yamlFile.write("baseos:" + ossname + ":" + ossversion + "-" + ossrelease + ":\n") yamlFile.write(" repository: BaseOS\n") yamlFile.write(" name: '" + ossname + "'\n") yamlFile.write(" version: '" + ossversion + "-" + ossrelease + "'\n") yamlFile.write(" url: 'http://www.vmware.com'\n") yamlFile.write(" baseos-style: rpm\n") yamlFile.write(" baseos-source: '" + str(srpmName) + "'\n") yamlFile.write(" baseos-osname: 'photon'\n") yamlFile.write("\n") if not singleFile: yamlFile.close() if singleFile: yamlFile.close() logger.info("Generated SRPM yaml files for all packages")
def findRPMFile(self, package, version="*"): cmdUtils = CommandUtils() if version == "*": version = SPECS.getData().getHighestVersion(package) release = SPECS.getData().getRelease(package, version) buildarch = SPECS.getData().getBuildArch(package, version) filename = package + "-" + version + "-" + release + "." + buildarch + ".rpm" fullpath = constants.rpmPath + "/" + buildarch + "/" + filename if os.path.isfile(fullpath): return fullpath if constants.inputRPMSPath is not None: fullpath = constants.inputRPMSPath + "/" + buildarch + "/" + filename if os.path.isfile(fullpath): return fullpath return None
def installToolChain(self, chrootID): self.logger.info("Installing toolchain.....") self.prepareBuildRoot(chrootID) cmdUtils = CommandUtils() rpmFiles = "" packages = "" for package in constants.listToolChainRPMPkgsToInstall: pkgUtils = PackageUtils(self.logName, self.logPath) rpmFile = pkgUtils.findRPMFileForGivenPackage(package) if rpmFile is None: rpmFile = self.findRPMFileInGivenLocation( package, constants.prevPublishRPMRepo) if rpmFile is None: if package == "util-linux-devel": self.logger.info( "No old version of util-linux-devel exists, skip until the new version is built" ) continue if package == "flex-devel": self.logger.info( "No old version of flex-devel exists, skip until the new version is built" ) continue self.logger.error("Unable to find rpm " + package + " in current and previous versions") raise Exception("Input Error") rpmFiles += " " + rpmFile packages += " " + package self.logger.debug("Installing toolchain rpms:" + packages) cmd = self.rpmCommand + " -i --nodeps --force --root " + chrootID + " --define \'_dbpath /var/lib/rpm\' " + rpmFiles process = subprocess.Popen("%s" % cmd, shell=True, stdout=subprocess.PIPE) retval = process.wait() if retval != 0: self.logger.error("Installing toolchain rpms failed") raise Exception("RPM installation failed") self.logger.info("Installed toolchain successfully on chroot:" + chrootID)
def installToolChainRPMS(self,chrootID): cmdUtils = CommandUtils() self.prepareBuildRoot(chrootID) self.logger.info("Installing Tool Chain RPMS.......") for package in constants.listToolChainRPMPkgsToBuild: pkgUtils=PackageUtils(self.logName,self.logPath) rpmFile=pkgUtils.findRPMFileForGivenPackage(package) if rpmFile is None: rpmFile=self.findRPMFileInGivenLocation(package, constants.prevPublishRPMRepo) if rpmFile is None: self.logger.error("Unable to find rpm "+ package +" in current and previous versions") raise "Input Error" self.logger.debug("Installing rpm:"+rpmFile) cmd="rpm -i --nodeps --force --root "+chrootID+" --define \'_dbpath /var/lib/rpm\' "+ rpmFile process = subprocess.Popen("%s" %cmd,shell=True,stdout=subprocess.PIPE) retval = process.wait() if retval != 0: self.logger.error("Installing tool chain package "+package+" failed") raise "RPM installation failed" self.logger.info("Successfully installed all Tool Chain RPMS in Chroot:"+chrootID)
def findRPMFileForGivenPackage(self,package): cmdUtils = CommandUtils() version = constants.specData.getVersion(package) release = constants.specData.getRelease(package) listFoundRPMFiles = sum([cmdUtils.findFile(package+"-"+version+"-"+release+".x86_64.rpm",constants.rpmPath), cmdUtils.findFile(package+"-"+version+"-"+release+".noarch.rpm",constants.rpmPath)], []) if constants.inputRPMSPath is not None: listFoundRPMFiles = sum([cmdUtils.findFile(package+"-"+version+"-"+release+".x86_64.rpm",constants.inputRPMSPath), cmdUtils.findFile(package+"-"+version+"-"+release+".noarch.rpm",constants.inputRPMSPath)], listFoundRPMFiles) if len(listFoundRPMFiles) == 1 : return listFoundRPMFiles[0] if len(listFoundRPMFiles) == 0 : return None if len(listFoundRPMFiles) > 1 : self.logger.error("Found multiple rpm files for given package in rpm directory.Unable to determine the rpm file for package:"+package) raise Exception("Multiple rpm files found")
def adjustGCCSpecs(self, package, chrootID, logPath): opt = " " + constants.specData.getSecurityHardeningOption(package) cmdUtils=CommandUtils() cpcmd="cp "+ self.adjustGCCSpecScript+" "+chrootID+"/tmp/"+self.adjustGCCSpecScript cmd = "/tmp/"+self.adjustGCCSpecScript+opt logFile = logPath+"/adjustGCCSpecScript.log" chrootCmd=self.runInChrootCommand+" "+chrootID returnVal = cmdUtils.runCommandInShell(cpcmd, logFile) if not returnVal: self.logger.error("Error during copying the file adjust gcc spec") raise Exception("Failed while copying adjust gcc spec file") returnVal = cmdUtils.runCommandInShell(cmd, logFile, chrootCmd) if returnVal: return self.logger.debug(cmdUtils.runCommandInShell2("ls -la " + chrootID + "/tmp/" + self.adjustGCCSpecScript)) self.logger.debug(cmdUtils.runCommandInShell2("lsof " + chrootID + "/tmp/" + self.adjustGCCSpecScript)) self.logger.debug(cmdUtils.runCommandInShell2("ps ax")) self.logger.error("Failed while adjusting gcc specs") raise Exception("Failed while adjusting gcc specs")
def findRPMFile(self, package,version="*",arch=None, throw=False): if not arch: arch=constants.currentArch cmdUtils = CommandUtils() if version == "*": version = SPECS.getData(arch).getHighestVersion(package) release = SPECS.getData(arch).getRelease(package, version) buildarch=SPECS.getData(arch).getBuildArch(package, version) filename= package + "-" + version + "-" + release + "." + buildarch+".rpm" fullpath = constants.rpmPath + "/" + buildarch + "/" + filename if os.path.isfile(fullpath): return fullpath if constants.inputRPMSPath is not None: fullpath = constants.inputRPMSPath + "/" + buildarch + "/" + filename if os.path.isfile(fullpath): return fullpath if throw: raise Exception("RPM %s not found" % (filename)) return None
def adjustGCCSpecs(self, package, chrootID, logPath): opt = " " + constants.specData.getSecurityHardeningOption(package) shutil.copy2(self.adjustGCCSpecScript, chrootID + "/tmp/" + self.adjustGCCSpecScript) cmdUtils = CommandUtils() cmd = "/tmp/" + self.adjustGCCSpecScript + opt logFile = logPath + "/adjustGCCSpecScript.log" chrootCmd = self.runInChrootCommand + " " + chrootID returnVal = cmdUtils.runCommandInShell(cmd, logFile, chrootCmd) if returnVal: return self.logger.debug( cmdUtils.runCommandInShell2("ls -la " + chrootID + "/tmp/" + self.adjustGCCSpecScript)) self.logger.debug( cmdUtils.runCommandInShell2("lsof " + chrootID + "/tmp/" + self.adjustGCCSpecScript)) self.logger.debug(cmdUtils.runCommandInShell2("ps ax")) self.logger.error("Failed while adjusting gcc specs") raise "Failed while adjusting gcc specs"
def buildRPMSForGivenPackage(self, package, version, chrootID, destLogPath=None): self.logger.info("Building rpm's for package:" + package) listSourcesFiles = SPECS.getData().getSources(package, version) listPatchFiles = SPECS.getData().getPatches(package, version) specFile = SPECS.getData().getSpecFile(package, version) specName = SPECS.getData().getSpecName(package) + ".spec" chrootSourcePath = chrootID + constants.topDirPath + "/SOURCES/" chrootSpecPath = constants.topDirPath + "/SPECS/" chrootLogsFilePath = chrootID + constants.topDirPath + "/LOGS/" + package + ".log" chrootCmd = self.runInChrootCommand + " " + chrootID shutil.copyfile(specFile, chrootID + chrootSpecPath + specName) # FIXME: some sources are located in SPECS/.. how to mount? # if os.geteuid()==0: self._copySourcesTobuildroot(listSourcesFiles, package, version, chrootSourcePath) self._copySourcesTobuildroot(listPatchFiles, package, version, chrootSourcePath) macros = [] listAdditionalFiles, macros = self._getAdditionalBuildFiles(package) self._copyAdditionalBuildFiles(listAdditionalFiles, chrootID) #Adding rpm macros listRPMMacros = constants.userDefinedMacros for macroName, value in listRPMMacros.items(): macros.append(macroName + " " + value) listRPMFiles = [] listSRPMFiles = [] try: listRPMFiles, listSRPMFiles = self._buildRPM( chrootSpecPath + specName, chrootLogsFilePath, chrootCmd, package, macros) self.logger.info("Successfully built rpm:" + package) except Exception as e: self.logger.error("Failed while building rpm:" + package) raise e finally: if destLogPath is not None: if (constants.rpmCheck and package in constants.testForceRPMS and SPECS.getData().isCheckAvailable(package)): cmd = ( "sed -i '/^Executing(%check):/,/^Processing files:/{//!b};d' " + chrootLogsFilePath) logFile = destLogPath + "/adjustTestFile.log" returnVal = CommandUtils().runCommandInShell(cmd, logFile) testLogFile = destLogPath + "/" + package + "-test.log" shutil.copyfile(chrootLogsFilePath, testLogFile) else: shutil.copy2(chrootLogsFilePath, destLogPath) self.logger.info("RPM build is successful") for rpmFile in listRPMFiles: self._copyRPM(chrootID + "/" + rpmFile, constants.rpmPath) for srpmFile in listSRPMFiles: srpmDestFile = self._copyRPM(chrootID + "/" + srpmFile, constants.sourceRpmPath)
def prepareBuildContainer(self, containerTaskName, packageName, isToolChainPackage=False): # Prepare an empty chroot environment to let docker use the BUILD folder. # This avoids docker using overlayFS which will cause make check failure. chrootName = "build-" + packageName chrUtils = ChrootUtils(self.base.logName, self.base.logPath) returnVal, chrootID = chrUtils.createChroot(chrootName) if not returnVal: raise Exception("Unable to prepare build root") cmdUtils = CommandUtils() cmdUtils.runCommandInShell("mkdir -p " + chrootID + constants.topDirPath) cmdUtils.runCommandInShell("mkdir -p " + chrootID + constants.topDirPath + "/BUILD") containerID = None mountVols = { constants.prevPublishRPMRepo: { 'bind': '/publishrpms', 'mode': 'ro' }, constants.prevPublishXRPMRepo: { 'bind': '/publishxrpms', 'mode': 'ro' }, constants.tmpDirPath: { 'bind': '/tmp', 'mode': 'rw' }, constants.rpmPath: { 'bind': constants.topDirPath + "/RPMS", 'mode': 'rw' }, constants.sourceRpmPath: { 'bind': constants.topDirPath + "/SRPMS", 'mode': 'rw' }, constants.logPath + "/" + self.base.logName: { 'bind': constants.topDirPath + "/LOGS", 'mode': 'rw' }, chrootID + constants.topDirPath + "/BUILD": { 'bind': constants.topDirPath + "/BUILD", 'mode': 'rw' }, constants.dockerUnixSocket: { 'bind': constants.dockerUnixSocket, 'mode': 'rw' } } containerName = containerTaskName containerName = containerName.replace("+", "p") try: oldContainerID = self.dockerClient.containers.get(containerName) if oldContainerID is not None: oldContainerID.remove(force=True) except docker.errors.NotFound: sys.exc_clear() try: self.base.logger.info( "BuildContainer-prepareBuildContainer: Starting build container: " + containerName) #TODO: Is init=True equivalent of --sig-proxy? privilegedDocker = False cap_list = ['SYS_PTRACE'] if packageName in constants.listReqPrivilegedDockerForTest: privilegedDocker = True containerID = self.dockerClient.containers.run( self.buildContainerImage, detach=True, cap_add=cap_list, privileged=privilegedDocker, name=containerName, network_mode="host", volumes=mountVols, command="/bin/bash -l -c /wait.sh") self.base.logger.debug("Started Photon build container for task " + containerTaskName + " ID: " + containerID.short_id) if not containerID: raise Exception( "Unable to start Photon build container for task " + containerTaskName) except Exception as e: self.base.logger.debug( "Unable to start Photon build container for task " + containerTaskName) raise e return containerID, chrootID
def main(): usage = "Usage: %prog [options] <package name>" parser = ArgumentParser(usage) parser.add_argument("-s", "--spec-path", dest="specPath", default="../../SPECS") parser.add_argument("-l", "--log-path", dest="logPath", default="../../stage/LOGS") parser.add_argument("-a", "--source-rpm-path", dest="sourceRpmPath", default="../../stage/SRPMS") parser.add_argument("-j", "--output-dir", dest="outputDirPath", default="../../stage/") parser.add_argument("-z", "--log-level", dest="logLevel", default="info") parser.add_argument("-c", "--pullsources-config", dest="pullsourcesConfig", default="pullsources.conf") parser.add_argument("-f", "--pkg-blacklist-file", dest="pkgBlacklistFile", default=None) parser.add_argument("-p", "--generate-pkg-list", dest="generatePkgList", default=False, action="store_true") parser.add_argument("-y", "--generate-yaml-files", dest="generateYamlFiles", default=False, action="store_true") parser.add_argument("-d", "--dist-tag", dest="dist", default="") options = parser.parse_args() errorFlag = False cmdUtils = CommandUtils() try: logName = "GenerateYamlFiles" logger = Logger.getLogger(logName, options.logPath, options.logLevel) if options.generateYamlFiles: if (options.pkgBlacklistFile is not None and options.pkgBlacklistFile != "" and not os.path.isfile(options.pkgBlacklistFile)): logger.error("Given package blacklist file is not valid:" + options.pkgBlacklistFile) errorFlag = True if not os.path.isdir(options.specPath): logger.error("Given Specs Path is not a directory:" + options.specPath) errorFlag = True if not os.path.isdir(options.sourceRpmPath): logger.error("Given SRPM Path is not a directory:" + options.sourceRpmPath) errorFlag = True if options.generateYamlFiles and not os.path.isfile(options.pullsourcesConfig): logger.error("Given Source config file is not a valid file:" + options.pullsourcesConfig) errorFlag = True if options.dist: dist_tag = options.dist logger.info("release tag is %s" % (dist_tag)) if errorFlag: logger.error("Found some errors. Please fix input options and re-run it.") sys.exit(1) if options.generateYamlFiles: if not os.path.isdir(options.outputDirPath): cmdUtils.runCommandInShell("mkdir -p "+options.outputDirPath) constants.setSpecPath(options.specPath) constants.setSourceRpmPath(options.sourceRpmPath) constants.setLogPath(options.logPath) constants.setLogLevel(options.logLevel) constants.setPullSourcesURL(get_baseurl(options.pullsourcesConfig)) constants.initialize() # parse SPECS folder SPECS() if options.generatePkgList: buildPackagesList(options.outputDirPath + "/packages_list.csv") elif options.generateYamlFiles: blackListPkgs = readBlackListPackages(options.pkgBlacklistFile) buildSourcesList(options.outputDirPath, blackListPkgs, logger) buildSRPMList(options.sourceRpmPath, options.outputDirPath, blackListPkgs, dist_tag, logger) except Exception as e: print("Caught Exception: " + str(e)) traceback.print_exc() sys.exit(1) sys.exit(0)
def prepareBuildRoot(self, chrootID): self.logger.info("Preparing build environment") cmdUtils = CommandUtils() cmdUtils.runCommandInShell("mkdir -p " + chrootID + "/dev") cmdUtils.runCommandInShell("mkdir -p " + chrootID + "/etc") cmdUtils.runCommandInShell("mkdir -p " + chrootID + "/proc") cmdUtils.runCommandInShell("mkdir -p " + chrootID + "/run") cmdUtils.runCommandInShell("mkdir -p " + chrootID + "/sys") cmdUtils.runCommandInShell("mkdir -p " + chrootID + "/tmp") cmdUtils.runCommandInShell("mkdir -p " + chrootID + constants.topDirPath) cmdUtils.runCommandInShell("mkdir -p " + chrootID + constants.topDirPath + "/RPMS/x86_64") cmdUtils.runCommandInShell("mkdir -p " + chrootID + constants.topDirPath + "/RPMS/noarch") cmdUtils.runCommandInShell("mkdir -p " + chrootID + constants.topDirPath + "/SOURCES") cmdUtils.runCommandInShell("mkdir -p " + chrootID + constants.topDirPath + "/SPECS") cmdUtils.runCommandInShell("mkdir -p " + chrootID + constants.topDirPath + "/LOGS") cmdUtils.runCommandInShell("mkdir -p " + chrootID + constants.topDirPath + "/BUILD") cmdUtils.runCommandInShell("mkdir -p " + chrootID + constants.topDirPath + "/BUILDROOT") prepareChrootCmd = self.prepareBuildRootCmd + " " + chrootID logFile = constants.logPath + "/prepareBuildRoot.log" returnVal = cmdUtils.runCommandInShell(prepareChrootCmd, logFile) if not returnVal: self.logger.error( "Prepare build root script failed.Unable to prepare chroot.") raise Exception("Prepare build root script failed") self.logger.info("Successfully prepared chroot:" + chrootID)
def main(): usage = "Usage: %prog [options] <package name>" parser = ArgumentParser(usage) parser.add_argument("-s", "--spec-path", dest="specPath", default="../../SPECS") parser.add_argument("-x", "--source-path", dest="sourcePath", default="../../stage/SOURCES") parser.add_argument("-r", "--rpm-path", dest="rpmPath", default="../../stage/RPMS") parser.add_argument("-i", "--install-package", dest="installPackage", default=False, action ="store_true") parser.add_argument("-p", "--publish-RPMS-path", dest="publishRPMSPath", default="../../stage/PUBLISHRPMS") parser.add_argument("-e", "--publish-XRPMS-path", dest="publishXRPMSPath", default="../../stage/PUBLISHXRPMS") parser.add_argument("-l", "--log-path", dest="logPath", default="../../stage/LOGS") parser.add_argument("-z", "--top-dir-path", dest="topDirPath", default="/usr/src/photon") parser.add_argument("-b", "--build-root-path", dest="buildRootPath", default="/mnt") parser.add_argument("-t", "--threads", dest="buildThreads", default=1, type=int, help="Number of working threads") parser.add_argument("-m", "--tool-chain-stage", dest="toolChainStage", default="None") parser.add_argument("-c", "--pullsources-config", dest="pullsourcesConfig", default="pullsources.conf") parser.add_argument("-d", "--dist", dest="dist", default="") parser.add_argument("-k", "--input-RPMS-path", dest="inputRPMSPath", default=None) parser.add_argument("-n", "--build-number", dest="buildNumber", default="0000000") parser.add_argument("-v", "--release-version", dest="releaseVersion", default="NNNnNNN") parser.add_argument("-u", "--enable-rpmcheck", dest="rpmCheck", default=False, action ="store_true") parser.add_argument("-a", "--source-rpm-path", dest="sourceRpmPath", default="../../stage/SRPMS") parser.add_argument("-w", "--pkginfo-file", dest="pkgInfoFile", default="../../stage/pkg_info.json") parser.add_argument("-g", "--pkg-build-option-file", dest="pkgBuildOptionFile", default="../../common/data/pkg_build_options.json") parser.add_argument("-q", "--rpmcheck-stop-on-error", dest="rpmCheckStopOnError", default=False, action ="store_true") parser.add_argument("PackageName", nargs='?') options = parser.parse_args() cmdUtils=CommandUtils() if not os.path.isdir(options.logPath): cmdUtils.runCommandInShell("mkdir -p "+options.logPath) logger=Logger.getLogger(options.logPath+"/Main") errorFlag=False package = None pkgInfoJsonFile = options.pkgInfoFile if not os.path.isdir(options.sourcePath): logger.error("Given Sources Path is not a directory:"+options.sourcePath) errorFlag = True if not os.path.isdir(options.specPath): logger.error("Given Specs Path is not a directory:"+options.specPath) errorFlag = True if not os.path.isdir(options.publishRPMSPath): logger.error("Given RPMS Path is not a directory:"+options.publishRPMSPath) errorFlag = True if not os.path.isdir(options.publishXRPMSPath): logger.error("Given X RPMS Path is not a directory:"+options.publishXRPMSPath) errorFlag = True if not os.path.isdir(options.publishRPMSPath+"/x86_64"): logger.error("Given RPMS Path is missing x86_64 sub-directory:"+options.publishRPMSPath) errorFlag = True if not os.path.isdir(options.publishXRPMSPath+"/x86_64"): logger.error("Given X RPMS Path is missing x86_64 sub-directory:"+options.publishXRPMSPath) errorFlag = True if not os.path.isdir(options.publishRPMSPath+"/noarch"): logger.error("Given RPMS Path is missing noarch sub-directory:"+options.publishRPMSPath) errorFlag = True if not os.path.isdir(options.publishXRPMSPath+"/noarch"): logger.error("Given X RPMS Path is missing noarch sub-directory:"+options.publishXRPMSPath) errorFlag = True if not os.path.isfile(options.pkgBuildOptionFile): logger.warning("Given JSON File is not a file:"+options.pkgBuildOptionFile) if options.inputRPMSPath is not None and not os.path.isdir(options.inputRPMSPath): logger.error("Given input RPMS Path is not a directory:"+options.inputRPMSPath) errorFlag = True if options.installPackage : if not options.PackageName: logger.error("Please provide package name") errorFlag = True else: package=options.PackageName if errorFlag: logger.error("Found some errors. Please fix input options and re-run it.") return False if not os.path.isdir(options.rpmPath): cmdUtils.runCommandInShell("mkdir -p "+options.rpmPath+"/x86_64") cmdUtils.runCommandInShell("mkdir -p "+options.rpmPath+"/noarch") if not os.path.isdir(options.sourceRpmPath): cmdUtils.runCommandInShell("mkdir -p "+options.sourceRpmPath) if not os.path.isdir(options.buildRootPath): cmdUtils.runCommandInShell("mkdir -p "+options.buildRootPath) logger.info("Source Path :"+options.sourcePath) logger.info("Spec Path :" + options.specPath) logger.info("Rpm Path :" + options.rpmPath) logger.info("Log Path :" + options.logPath) logger.info("Top Dir Path :" + options.topDirPath) logger.info("Publish RPMS Path :" + options.publishRPMSPath) logger.info("Publish X RPMS Path :" + options.publishXRPMSPath) if options.installPackage: logger.info("Package to build:"+package) listBuildOptionPackages = get_packages_with_build_options(options.pkgBuildOptionFile) try: constants.initialize(options) if package == "packages_list": buildPackagesList(options.buildRootPath+"/../packages_list.csv") elif package == "sources_list": if not os.path.isdir("../../stage/yaml_sources"): cmdUtils.runCommandInShell("mkdir -p ../../stage/yaml_sources") buildSourcesList('../../stage/yaml_sources',logger) elif package == "srpms_list": if not os.path.isdir("../../stage/yaml_srpms"): cmdUtils.runCommandInShell("mkdir -p ../../stage/yaml_srpms") buildSRPMList(options.sourceRpmPath,"../../stage/yaml_srpms",logger) elif options.toolChainStage == "stage1": pkgManager = PackageManager() pkgManager.buildToolChain() elif options.toolChainStage == "stage2": pkgManager = PackageManager() pkgManager.buildToolChainPackages(options.buildThreads) elif options.installPackage: buildAPackage(package, listBuildOptionPackages, options.pkgBuildOptionFile, options.buildThreads) else: buildPackagesForAllSpecs(listBuildOptionPackages, options.pkgBuildOptionFile, logger, options.buildThreads, pkgInfoJsonFile) except Exception as e: logger.error("Caught an exception") logger.error(str(e)) # print stacktrace traceback.print_exc() sys.exit(1) sys.exit(0)
def main(): usage = "Usage: %prog [options] <package name>" parser = ArgumentParser(usage) parser.add_argument("-s", "--spec-path", dest="specPath", default="../../SPECS") parser.add_argument("-x", "--source-path", dest="sourcePath", default="../../stage/SOURCES") parser.add_argument("-r", "--rpm-path", dest="rpmPath", default="../../stage/RPMS") parser.add_argument("-i", "--install-package", dest="installPackage", default=False, action="store_true") parser.add_argument("-p", "--publish-RPMS-path", dest="publishRPMSPath", default="../../stage/PUBLISHRPMS") parser.add_argument("-e", "--publish-XRPMS-path", dest="publishXRPMSPath", default="../../stage/PUBLISHXRPMS") parser.add_argument("-l", "--log-path", dest="logPath", default="../../stage/LOGS") parser.add_argument("-y", "--log-level", dest="logLevel", default="error") parser.add_argument("-z", "--top-dir-path", dest="topDirPath", default="/usr/src/photon") parser.add_argument("-b", "--build-root-path", dest="buildRootPath", default="/mnt") parser.add_argument("-t", "--threads", dest="buildThreads", default=1, type=int, help="Number of working threads") parser.add_argument("-m", "--tool-chain-stage", dest="toolChainStage", default="None") parser.add_argument("-c", "--pullsources-config", dest="pullsourcesConfig", default="pullsources.conf") parser.add_argument("-d", "--dist-tag", dest="dist", default="") parser.add_argument("-k", "--input-RPMS-path", dest="inputRPMSPath", default=None) parser.add_argument("-n", "--build-number", dest="buildNumber", default="0000000") parser.add_argument("-v", "--release-version", dest="releaseVersion", default="NNNnNNN") parser.add_argument("-u", "--enable-rpmcheck", dest="rpmCheck", default=False, action="store_true") parser.add_argument("-a", "--source-rpm-path", dest="sourceRpmPath", default="../../stage/SRPMS") parser.add_argument("-w", "--pkginfo-file", dest="pkgInfoFile", default="../../stage/pkg_info.json") parser.add_argument("-g", "--pkg-build-option-file", dest="pkgBuildOptionFile", default="../../common/data/pkg_build_options.json") parser.add_argument("-q", "--rpmcheck-stop-on-error", dest="rpmCheckStopOnError", default=False, action="store_true") parser.add_argument("-bd", "--publish-build-dependencies", dest="publishBuildDependencies", default=False) parser.add_argument("-pw", "--package-weights-path", dest="packageWeightsPath", default=None) parser.add_argument("-bt", "--build-type", dest="pkgBuildType", default="chroot") parser.add_argument("-F", "--kat-build", dest="katBuild", default=None) parser.add_argument("-pj", "--packages-json-input", dest="pkgJsonInput", default=None) parser.add_argument("PackageName", nargs='?') options = parser.parse_args() cmdUtils = CommandUtils() if not os.path.isdir(options.logPath): cmdUtils.runCommandInShell("mkdir -p " + options.logPath) logger = Logger.getLogger("Main", options.logPath, options.logLevel) errorFlag = False package = None pkgInfoJsonFile = options.pkgInfoFile if not os.path.isdir(options.sourcePath): logger.error("Given Sources Path is not a directory:" + options.sourcePath) errorFlag = True if not os.path.isdir(options.specPath): logger.error("Given Specs Path is not a directory:" + options.specPath) errorFlag = True if not os.path.isdir(options.publishRPMSPath): logger.error("Given RPMS Path is not a directory:" + options.publishRPMSPath) errorFlag = True if not os.path.isdir(options.publishXRPMSPath): logger.error("Given X RPMS Path is not a directory:" + options.publishXRPMSPath) errorFlag = True if not os.path.isdir(options.publishRPMSPath + "/" + platform.machine()): logger.error("Given RPMS Path is missing " + platform.machine() + " sub-directory:" + options.publishRPMSPath) errorFlag = True if not os.path.isdir(options.publishXRPMSPath + "/" + platform.machine()): logger.error("Given X RPMS Path is missing " + platform.machine() + " sub-directory:" + options.publishXRPMSPath) errorFlag = True if not os.path.isdir(options.publishRPMSPath + "/noarch"): logger.error("Given RPMS Path is missing noarch sub-directory:" + options.publishRPMSPath) errorFlag = True if not os.path.isdir(options.publishXRPMSPath + "/noarch"): logger.error("Given X RPMS Path is missing noarch sub-directory:" + options.publishXRPMSPath) errorFlag = True if not os.path.isfile(options.pkgBuildOptionFile): logger.warning("Given JSON File is not a file:" + options.pkgBuildOptionFile) if options.inputRPMSPath is not None and not os.path.isdir( options.inputRPMSPath): logger.error("Given input RPMS Path is not a directory:" + options.inputRPMSPath) errorFlag = True if options.packageWeightsPath is not None and not os.path.isfile( options.packageWeightsPath): logger.error("Given input Weights file is not a file:" + options.packageWeightsPath) errorFlag = True if options.pkgJsonInput is not None and not os.path.isfile( options.pkgJsonInput): logger.error("Given input packages file is not a file:" + options.pkgJsonInput) errorFlag = True if options.installPackage: if not options.PackageName: logger.error("Please provide package name") errorFlag = True else: package = options.PackageName if errorFlag: logger.error( "Found some errors. Please fix input options and re-run it.") return False if not os.path.isdir(options.rpmPath): cmdUtils.runCommandInShell("mkdir -p " + options.rpmPath + "/" + platform.machine()) cmdUtils.runCommandInShell("mkdir -p " + options.rpmPath + "/noarch") if not os.path.isdir(options.sourceRpmPath): cmdUtils.runCommandInShell("mkdir -p " + options.sourceRpmPath) if not os.path.isdir(options.buildRootPath): cmdUtils.runCommandInShell("mkdir -p " + options.buildRootPath) logger.debug("Source Path :" + options.sourcePath) logger.debug("Spec Path :" + options.specPath) logger.debug("Rpm Path :" + options.rpmPath) logger.debug("Log Path :" + options.logPath) logger.debug("Log Level :" + options.logLevel) logger.debug("Top Dir Path :" + options.topDirPath) logger.debug("Publish RPMS Path :" + options.publishRPMSPath) logger.debug("Publish X RPMS Path :" + options.publishXRPMSPath) if options.installPackage: logger.debug("Package to build:" + package) get_packages_with_build_options(options.pkgBuildOptionFile) try: constants.setSpecPath(options.specPath) constants.setSourcePath(options.sourcePath) constants.setRpmPath(options.rpmPath) constants.setSourceRpmPath(options.sourceRpmPath) constants.setTopDirPath(options.topDirPath) constants.setLogPath(options.logPath) constants.setLogLevel(options.logLevel) constants.setDist(options.dist) constants.setBuildNumber(options.buildNumber) constants.setReleaseVersion(options.releaseVersion) constants.setPrevPublishRPMRepo(options.publishRPMSPath) constants.setPrevPublishXRPMRepo(options.publishXRPMSPath) constants.setBuildRootPath(options.buildRootPath) constants.setPullSourcesURL(get_baseurl(options.pullsourcesConfig)) constants.setInputRPMSPath(options.inputRPMSPath) constants.setRPMCheck(options.rpmCheck) constants.setRpmCheckStopOnError(options.rpmCheckStopOnError) constants.setPublishBuildDependencies(options.publishBuildDependencies) constants.setPackageWeightsPath(options.packageWeightsPath) constants.setKatBuild(options.katBuild) constants.initialize() # parse SPECS folder SPECS() if options.toolChainStage == "stage1": pkgManager = PackageManager() pkgManager.buildToolChain() elif options.toolChainStage == "stage2": pkgManager = PackageManager() pkgManager.buildToolChainPackages(options.buildThreads) elif options.installPackage: buildSpecifiedPackages([package], options.buildThreads, options.pkgBuildType) elif options.pkgJsonInput: buildPackagesInJson(options.pkgJsonInput, options.buildThreads, options.pkgBuildType, pkgInfoJsonFile, logger) else: buildPackagesForAllSpecs(options.buildThreads, options.pkgBuildType, pkgInfoJsonFile, logger) except Exception as e: logger.error("Caught an exception") logger.error(str(e)) # print stacktrace traceback.print_exc() sys.exit(1) sys.exit(0)
def _buildRPMinContainer(self, specFile, rpmLogFile, destLogFile, containerID, package, macros): rpmBuildCmd = self.rpmbuildBinary + " " + self.rpmbuildBuildallOption if constants.rpmCheck and package in constants.testForceRPMS: self.logger.info("#" * (68 + 2 * len(package))) if not SPECS.getData().isCheckAvailable(package, index): self.logger.info("####### " + package + " MakeCheck is not available. Skipping MakeCheck TEST for " + package + " #######") rpmBuildCmd = self.rpmbuildBinary + " --clean" else: self.logger.info("####### " + package + " MakeCheck is available. Running MakeCheck TEST for " + package + " #######") rpmBuildCmd = self.rpmbuildBinary + " " + self.rpmbuildCheckOption self.logger.info("#" * (68 + 2 * len(package))) else: rpmBuildCmd += " "+self.rpmbuildNocheckOption for macro in macros: rpmBuildCmd += ' --define \"%s\"' % macro rpmBuildCmd += " " + specFile rpmBuildCmd = "/bin/bash -l -c '" + rpmBuildCmd + " > " + rpmLogFile + " 2>&1'" rpmBuildCmd = "docker exec -t " + str(containerID.short_id) + " " + rpmBuildCmd cmdUtils = CommandUtils() self.logger.info("Building rpm for package: " + package) #TODO: Show running log of rpmbuildcmd #TODO: Get exit status of rpmBuildCmd #containerID.exec_run(rpmBuildCmd) returnVal = cmdUtils.runCommandInShell(rpmBuildCmd) if not os.path.isfile(destLogFile): self.logger.error("RPM build not file not found. Building rpm failed for: " + specFile) raise Exception("RPM Build failed") if constants.rpmCheck and package in constants.testForceRPMS: if not SPECS.getData().isCheckAvailable(package, index): constants.testLogger.info(package + " : N/A") elif returnVal: constants.testLogger.info(package + " : PASS") else: constants.testLogger.error(package + " : FAIL") if constants.rpmCheck: if not returnVal and constants.rpmCheckStopOnError: self.logger.error("Checking rpm is failed " + specFile) raise Exception("RPM check failed") else: if not returnVal: self.logger.error("Building rpm is failed " + specFile) raise Exception("RPM build failed") #Extracting rpms created from log file listRPMFiles = [] listSRPMFiles = [] with open(destLogFile, 'r') as logfile: rpmBuildLogLines = logfile.readlines() for i in range(0, len(rpmBuildLogLines)): if re.search("^Wrote:", rpmBuildLogLines[i]): listcontents = rpmBuildLogLines[i].split() if ((len(listcontents) == 2) and listcontents[1].strip().endswith(".rpm") and "/RPMS/" in listcontents[1]): listRPMFiles.append(listcontents[1]) if ((len(listcontents) == 2) and listcontents[1].strip().endswith(".src.rpm") and "/SRPMS/" in listcontents[1]): listSRPMFiles.append(listcontents[1]) #if not listRPMFiles: # self.logger.error("Building rpm failed for " + specFile) # raise Exception("RPM Build failed") return listRPMFiles, listSRPMFiles
def _buildRPM(self, specFile, logFile, chrootCmd, package, macros): rpmBuildcmd = self.rpmbuildBinary + " " + self.rpmbuildBuildallOption if constants.rpmCheck and package in constants.testForceRPMS: self.logger.info("#" * (68 + 2 * len(package))) if not SPECS.getData().isCheckAvailable(package): self.logger.info("####### " + package + " MakeCheck is not available. Skipping MakeCheck TEST for " + package + " #######") rpmBuildcmd = self.rpmbuildBinary + " --clean" else: self.logger.info("####### " + package + " MakeCheck is available. Running MakeCheck TEST for " + package + " #######") rpmBuildcmd = self.rpmbuildBinary + " " + self.rpmbuildCheckOption self.logger.info("#" * (68 + 2 * len(package))) else: rpmBuildcmd += " " + self.rpmbuildNocheckOption for macro in macros: rpmBuildcmd += ' --define \\\"%s\\\"' % macro rpmBuildcmd += " " + specFile cmdUtils = CommandUtils() self.logger.info("Building rpm....") self.logger.info(rpmBuildcmd) returnVal = cmdUtils.runCommandInShell(rpmBuildcmd, logFile, chrootCmd) if constants.rpmCheck and package in constants.testForceRPMS: if not SPECS.getData().isCheckAvailable(package): constants.testLogger.info(package + " : N/A") elif returnVal: constants.testLogger.info(package + " : PASS") else: constants.testLogger.error(package + " : FAIL") if constants.rpmCheck: if not returnVal and constants.rpmCheckStopOnError: self.logger.error("Checking rpm is failed " + specFile) raise Exception("RPM check failed") else: if not returnVal: self.logger.error("Building rpm is failed " + specFile) raise Exception("RPM build failed") #Extracting rpms created from log file listRPMFiles = [] listSRPMFiles = [] with open(logFile, 'r') as logfile: fileContents = logfile.readlines() for i in range(0, len(fileContents)): if re.search("^Wrote:", fileContents[i]): listcontents = fileContents[i].split() if ((len(listcontents) == 2) and listcontents[1].strip().endswith(".rpm") and "/RPMS/" in listcontents[1]): listRPMFiles.append(listcontents[1]) if ((len(listcontents) == 2) and listcontents[1].strip().endswith(".src.rpm") and "/SRPMS/" in listcontents[1]): listSRPMFiles.append(listcontents[1]) return listRPMFiles, listSRPMFiles
def main(): usage = "Usage: %prog [options] <package name>" parser = OptionParser(usage) parser.add_option("-s", "--spec-path", dest="specPath", default="../../SPECS") parser.add_option("-x", "--source-path", dest="sourcePath", default="../../stage/SOURCES") parser.add_option("-r", "--rpm-path", dest="rpmPath", default="../../stage/RPMS") parser.add_option("-i", "--install-package", dest="installPackage", default=False, action="store_true") parser.add_option("-p", "--publish-RPMS-path", dest="publishRPMSPath", default="../../stage/PUBLISHRPMS") parser.add_option("-l", "--log-path", dest="logPath", default="../../stage/LOGS") parser.add_option("-o", "--build-option", dest="buildOption", default="full") parser.add_option("-z", "--top-dir-path", dest="topDirPath", default="/usr/src/photon") parser.add_option( "-j", "--json-file", dest="inputJSONFile", default="../../common/data/build_install_options_all.json") parser.add_option("-b", "--build-root-path", dest="buildRootPath", default="/mnt") parser.add_option("-t", "--threads", dest="buildThreads", default=1, type="int", help="Numbeer of working threads") parser.add_option("-m", "--tool-chain-stage", dest="toolChainStage", default="None") parser.add_option("-c", "--pullsources-config", dest="pullsourcesConfig", default="pullsources.conf") parser.add_option("-d", "--dist", dest="dist", default="") parser.add_option("-k", "--input-RPMS-path", dest="inputRPMSPath", default=None) parser.add_option("-n", "--build-number", dest="buildNumber", default="0000000") parser.add_option("-v", "--release-version", dest="releaseVersion", default="NNNnNNN") parser.add_option("-u", "--enable-rpmcheck", dest="rpmCheck", default=False, action="store_true") parser.add_option("-a", "--source-rpm-path", dest="sourceRpmPath", default="../../stage/SRPMS") (options, args) = parser.parse_args() cmdUtils = CommandUtils() if not os.path.isdir(options.logPath): cmdUtils.runCommandInShell("mkdir -p " + options.logPath) logger = Logger.getLogger(options.logPath + "/Main") errorFlag = False package = None if not os.path.isdir(options.sourcePath): logger.error("Given Sources Path is not a directory:" + options.sourcePath) errorFlag = True if not os.path.isdir(options.specPath): logger.error("Given Specs Path is not a directory:" + options.specPath) errorFlag = True if not os.path.isdir(options.publishRPMSPath): logger.error("Given RPMS Path is not a directory:" + options.publishRPMSPath) errorFlag = True if not os.path.isdir(options.publishRPMSPath + "/x86_64"): logger.error("Given RPMS Path is missing x86_64 sub-directory:" + options.publishRPMSPath) errorFlag = True if not os.path.isdir(options.publishRPMSPath + "/noarch"): logger.error("Given RPMS Path is missing noarch sub-directory:" + options.publishRPMSPath) errorFlag = True if not os.path.isfile( options.inputJSONFile) and not options.installPackage: logger.error("Given JSON File is not a file:" + options.inputJSONFile) errorFlag = True if options.inputRPMSPath is not None and not os.path.isdir( options.inputRPMSPath): logger.error("Given input RPMS Path is not a directory:" + options.publishRPMSPath) errorFlag = True if options.installPackage: if len(args) != 1: logger.error("Please provide package name") errorFlag = True else: package = args[0] if errorFlag: logger.error( "Found some errors. Please fix input options and re-run it.") return False if not os.path.isdir(options.rpmPath): cmdUtils.runCommandInShell("mkdir -p " + options.rpmPath + "/x86_64") cmdUtils.runCommandInShell("mkdir -p " + options.rpmPath + "/noarch") if not os.path.isdir(options.sourceRpmPath): cmdUtils.runCommandInShell("mkdir -p " + options.sourceRpmPath) if not os.path.isdir(options.buildRootPath): cmdUtils.runCommandInShell("mkdir -p " + options.buildRootPath) logger.info("Source Path :" + options.sourcePath) logger.info("Spec Path :" + options.specPath) logger.info("Rpm Path :" + options.rpmPath) logger.info("Log Path :" + options.logPath) logger.info("Top Dir Path :" + options.topDirPath) logger.info("Publish RPMS Path :" + options.publishRPMSPath) if not options.installPackage: logger.info("JSON File :" + options.inputJSONFile) else: logger.info("Package to build:" + package) ''' listPackages=["acl","attr","autoconf","automake","bash","bc","bindutils","binutils","bison","boost","btrfs-progs","bzip2","ca-certificates","cdrkit","check", "cloud-init","cmake","coreutils","cpio","cracklib","createrepo","curl","cyrus-sasl","db","dbus","deltarpm","diffutils","docbook-xml","docbook-xsl", "docker","dracut","e2fsprogs","elfutils","etcd","expat","file","filesystem","findutils","flex","gawk","gcc","gdb","gdbm","gettext","git", "glib","glibc","glibmm","gmp","go","gobject-introspection","google-daemon","google-startup-scripts","gperf","gpgme","gptfdisk","grep","groff", "grub","gtk-doc","gzip","haveged","hawkey","iana-etc","inetutils","intltool","iproute2","iptables","itstool","json-glib","kbd","kmod","krb5", "kubernetes","less","libaio","libassuan","libcap","libdnet","libffi","libgpg-error","libgsystem","libhif","libmspack","libpcap","libpipeline", "librepo","libselinux","libsepol","libsigc++","libsolv","libtool","libxml2","libxslt","libyaml","linux","linux-api-headers","Linux-PAM","lua", "lvm2","lzo","m4","make","man-db","man-pages","mercurial","mpc","mpfr","nano","ncurses","nspr","nss","ntp","openldap","openssh","openssl", "open-vm-tools","ostree","parted","patch","pcre","perl","perl-common-sense","perl-Config-IniFiles","perl-DBD-SQLite","perl-DBI","perl-DBIx-Simple", "perl-Exporter-Tiny","perl-JSON-XS","perl-libintl","perl-List-MoreUtils","perl-Module-Install","perl-Module-ScanDeps","perl-Types-Serialiser", "perl-WWW-Curl","perl-YAML","perl-YAML-Tiny","photon-release","pkg-config","popt","procps-ng","psmisc","pycurl","pygobject","python2", "python-configobj","python-iniparse","python-jsonpatch","python-jsonpointer","python-prettytable","python-requests","python-setuptools", "python-six","PyYAML","readline","rocket","rpm","rpm-ostree","rpm-ostree-toolbox","ruby","sed","shadow","sqlite-autoconf","strace","sudo", "swig","systemd","tar","tcpdump","tcsh","tdnf","texinfo","thin-provisioning-tools","tzdata","unzip","urlgrabber","util-linux","vim","wget", "which","xerces-c","XML-Parser","xml-security-c","xz","yum","yum-metadata-parser","zlib"] ''' try: constants.initialize(options) if package == "packages_list": buildPackagesList(options.specPath, options.buildRootPath + "/../packages_list.csv") elif package == "sources_list": buildSourcesList(options.specPath, options.buildRootPath + "/../") elif options.toolChainStage == "stage1": pkgManager = PackageManager() pkgManager.buildToolChain() elif options.toolChainStage == "stage2": pkgManager = PackageManager() pkgManager.buildToolChainPackages(options.buildThreads) elif options.installPackage: buildAPackage(package, options.buildThreads) else: buildPackagesFromGivenJSONFile(options.inputJSONFile, options.buildOption, logger, options.buildThreads) except Exception as e: logger.error("Caught an exception") logger.error(str(e)) # print stacktrace traceback.print_exc() sys.exit(1) sys.exit(0)
def buildRPMSForGivenPackageInContainer(self, package, containerID, destLogPath=None, index=0): self.logger.info("Building rpm's for package " + package + " in container " + containerID.short_id) listSourcesFiles = SPECS.getData().getSources(package, index) listPatchFiles = SPECS.getData().getPatches(package, index) specFile = SPECS.getData().getSpecFile(package, index) specName = SPECS.getData().getSpecName(package) + ".spec" sourcePath = constants.topDirPath + "/SOURCES/" specPath = constants.topDirPath + "/SPECS/" rpmLogFile = constants.topDirPath + "/LOGS/" + package + ".log" destLogFile = destLogPath + "/" + package + ".log" cmdUtils = CommandUtils() #TODO: mount it in, don't copy cpSpecCmd = "docker cp " + specFile + " " + containerID.short_id \ + ":" + specPath + specName returnVal = cmdUtils.runCommandInShell(cpSpecCmd) if not returnVal: self.logger.error("Error copying source SPEC file to container") raise Exception("Failed copying source SPEC to container") #FIXME: some sources are located in SPECS/.. how to mount? # if os.geteuid()==0: #TODO: mount it in, don't copy macros = [] self._copySourcesToContainer(listSourcesFiles, package, containerID, sourcePath, index) #TODO: mount it in, don't copy self._copySourcesToContainer(listPatchFiles, package, containerID, sourcePath, index) listAdditionalFiles, macros = self._getAdditionalBuildFiles(package) self._copyAdditionalBuildFilesToContainer(listAdditionalFiles, containerID) # Add rpm macros listRPMMacros = constants.userDefinedMacros for macroName, value in listRPMMacros.items(): macros.append(macroName + " " + value) # Build RPMs listRPMFiles = [] listSRPMFiles = [] try: listRPMFiles, listSRPMFiles = self._buildRPMinContainer( specPath + specName, rpmLogFile, destLogFile, containerID, package, macros) self.logger.info("Successfully built rpm:" + package) except Exception as e: self.logger.error("Failed while building rpm: " + package) raise e finally: if destLogPath is not None: rpmLog = destLogPath + "/" + package + ".log" if (constants.rpmCheck and package in constants.testForceRPMS and SPECS.getData().isCheckAvailable(package, index)): cmd = "sed -i '/^Executing(%check):/,/^Processing files:/{//!b};d' " + rpmLog logFile = destLogPath + "/adjustTestFile.log" returnVal = CommandUtils().runCommandInShell(cmd, logFile) testLogFile = destLogPath + "/" + package + "-test.log" shutil.copyfile(rpmLog, testLogFile) self.logger.info("RPM build is successful") # Verify RPM and SRPM files exist as success criteria for rpmFile in listRPMFiles: rpmName = os.path.basename(rpmFile) rpmDestDir = self._getRPMDestDir(rpmName, constants.rpmPath) rpmDestFile = rpmDestDir + "/" + rpmName if not os.path.isfile(rpmDestFile): self.logger.error("Could not find RPM file: " + rpmDestFile) raise Exception("Built RPM file not found.") for srpmFile in listSRPMFiles: srpmName = os.path.basename(srpmFile) srpmDestDir = self._getRPMDestDir(os.path.basename(srpmFile), constants.sourceRpmPath) srpmDestFile = srpmDestDir + "/" + srpmName if not os.path.isfile(srpmDestFile): self.logger.error("Could not find RPM file: " + srpmDestFile) raise Exception("Built SRPM file not found.")