def buildRPMSForGivenPackage(self,package,chrootID,listBuildOptionPackages,pkgBuildOptionFile,destLogPath=None): self.logger.info("Building rpm's for package:"+package) listSourcesFiles = constants.specData.getSources(package) listPatchFiles = constants.specData.getPatches(package) specFile = constants.specData.getSpecFile(package) specName = constants.specData.getSpecName(package) + ".spec" chrootSourcePath=chrootID+constants.topDirPath+"/SOURCES/" chrootSpecPath=constants.topDirPath+"/SPECS/" chrootLogsFilePath=chrootID+constants.topDirPath+"/LOGS/"+package+".log" chrootCmd=self.runInChrootCommand+" "+chrootID shutil.copyfile(specFile, chrootID+chrootSpecPath+specName ) # FIXME: some sources are located in SPECS/.. how to mount? # if os.geteuid()==0: self.copySourcesTobuildroot(listSourcesFiles,package,chrootSourcePath) self.copySourcesTobuildroot(listPatchFiles,package,chrootSourcePath) listAdditionalFiles = [] macros = [] if package in listBuildOptionPackages: jsonData = open(pkgBuildOptionFile) pkg_build_option_json = json.load(jsonData, object_pairs_hook=collections.OrderedDict) jsonData.close() pkgs_sorted = pkg_build_option_json.items() for pkg in pkgs_sorted: p = str(pkg[0].encode('utf-8')) if p == package: filelist = pkg[1]["files"] for f in filelist: listAdditionalFiles.append(str(f.encode('utf-8'))) macrolist = pkg[1]["macros"] for macro in macrolist: macros.append(str(macro.encode('utf-8'))) chrootAdditionalPath=chrootID+constants.topDirPath+"/ADDITIONAL" self.copyAdditionalBuildFiles(listAdditionalFiles,chrootAdditionalPath) listRPMFiles=[] listSRPMFiles=[] try: listRPMFiles,listSRPMFiles = self.buildRPM(chrootSpecPath+"/"+specName,chrootLogsFilePath,chrootCmd,package,macros) except Exception as e: self.logger.error("Failed while building rpm:"+package) raise e finally: if destLogPath is not None: shutil.copy2(chrootLogsFilePath, destLogPath) self.logger.info("RPM build is successful") arch = self.getRPMArch(listRPMFiles[0]) for rpmFile in listRPMFiles: self.copyRPM(chrootID+"/"+rpmFile, constants.rpmPath) for srpmFile in listSRPMFiles: self.copyRPM(chrootID+"/"+srpmFile, constants.sourceRpmPath) srpmName = os.path.basename(srpmFile) package,version,release = self.findPackageInfoFromSourceRPMFile(srpmFile) SourcePackageInfo.addSRPMData(package,version,release,arch,srpmName)
def buildRPMSForGivenPackage(self,package, chrootID,destLogPath=None): self.logger.info("Building rpm's for package:"+package) listSourcesFiles = constants.specData.getSources(package) listPatchFiles = constants.specData.getPatches(package) specFile = constants.specData.getSpecFile(package) specName = constants.specData.getSpecName(package) + ".spec" chrootSourcePath=chrootID+constants.topDirPath+"/SOURCES/" chrootSpecPath=constants.topDirPath+"/SPECS/" chrootLogsFilePath=chrootID+constants.topDirPath+"/LOGS/"+package+".log" chrootCmd=self.runInChrootCommand+" "+chrootID shutil.copyfile(specFile, chrootID+chrootSpecPath+specName ) # FIXME: some sources are located in SPECS/.. how to mount? # if os.geteuid()==0: self.copySourcesTobuildroot(listSourcesFiles,package,chrootSourcePath) self.copySourcesTobuildroot(listPatchFiles,package,chrootSourcePath) listRPMFiles=[] listSRPMFiles=[] try: listRPMFiles,listSRPMFiles = self.buildRPM(chrootSpecPath+"/"+specName,chrootLogsFilePath, chrootCmd) except Exception as e: self.logger.error("Failed while building rpm:"+package) raise e finally: if destLogPath is not None: shutil.copy2(chrootLogsFilePath, destLogPath) self.logger.info("RPM build is successful") arch = self.getRPMArch(listRPMFiles[0]) for rpmFile in listRPMFiles: self.copyRPM(chrootID+"/"+rpmFile, constants.rpmPath) for srpmFile in listSRPMFiles: self.copyRPM(chrootID+"/"+srpmFile, constants.sourceRpmPath) srpmName = os.path.basename(srpmFile) package,version,release = self.findPackageInfoFromSourceRPMFile(srpmFile) SourcePackageInfo.addSRPMData(package,version,release,arch,srpmName)
def main(): usage = "Usage: %prog [options] <package name>" parser = OptionParser(usage) parser.add_option("-s", "--spec-path", dest="specPath", default="../../SPECS") parser.add_option("-x", "--source-path", dest="sourcePath", default="../../stage/SOURCES") parser.add_option("-r", "--rpm-path", dest="rpmPath", default="../../stage/RPMS") parser.add_option("-i", "--install-package", dest="installPackage", default=False, action ="store_true") parser.add_option("-p", "--publish-RPMS-path", dest="publishRPMSPath", default="../../stage/PUBLISHRPMS") parser.add_option("-l", "--log-path", dest="logPath", default="../../stage/LOGS") parser.add_option("-o", "--build-option", dest="buildOption", default="full") parser.add_option("-z", "--top-dir-path", dest="topDirPath", default="/usr/src/photon") parser.add_option("-j", "--json-file", dest="inputJSONFile", default="../../common/data/build_install_options_all.json") parser.add_option("-b", "--build-root-path", dest="buildRootPath", default="/mnt") parser.add_option("-t", "--threads", dest="buildThreads", default=1, type="int", help="Number of working threads") parser.add_option("-m", "--tool-chain-stage", dest="toolChainStage", default="None") parser.add_option("-c", "--pullsources-config", dest="pullsourcesConfig", default="pullsources.conf") parser.add_option("-d", "--dist", dest="dist", default="") parser.add_option("-k", "--input-RPMS-path", dest="inputRPMSPath", default=None) parser.add_option("-n", "--build-number", dest="buildNumber", default="0000000") parser.add_option("-v", "--release-version", dest="releaseVersion", default="NNNnNNN") parser.add_option("-u", "--enable-rpmcheck", dest="rpmCheck", default=False, action ="store_true") parser.add_option("-a", "--source-rpm-path", dest="sourceRpmPath", default="../../stage/SRPMS") parser.add_option("-w", "--pkginfo-file", dest="pkgInfoFile", default="../../common/data/pkg_info.json") parser.add_option("-g", "--pkg-build-option-file", dest="pkgBuildOptionFile", default="../../common/data/pkg_build_options.json") parser.add_option("-q", "--rpmcheck-stop-on-error", dest="rpmCheckStopOnError", default=False, action ="store_true") (options, args) = parser.parse_args() cmdUtils=CommandUtils() if not os.path.isdir(options.logPath): cmdUtils.runCommandInShell("mkdir -p "+options.logPath) logger=Logger.getLogger(options.logPath+"/Main") errorFlag=False package = None pkgInfoJsonFile=options.pkgInfoFile if not os.path.isdir(options.sourcePath): logger.error("Given Sources Path is not a directory:"+options.sourcePath) errorFlag = True if not os.path.isdir(options.specPath): logger.error("Given Specs Path is not a directory:"+options.specPath) errorFlag = True if not os.path.isdir(options.publishRPMSPath): logger.error("Given RPMS Path is not a directory:"+options.publishRPMSPath) errorFlag = True if not os.path.isdir(options.publishRPMSPath+"/x86_64"): logger.error("Given RPMS Path is missing x86_64 sub-directory:"+options.publishRPMSPath) errorFlag = True if not os.path.isdir(options.publishRPMSPath+"/noarch"): logger.error("Given RPMS Path is missing noarch sub-directory:"+options.publishRPMSPath) errorFlag = True if not os.path.isfile(options.inputJSONFile) and not options.installPackage: logger.error("Given JSON File is not a file:"+options.inputJSONFile) errorFlag = True if not os.path.isfile(options.pkgBuildOptionFile): logger.warning("Given JSON File is not a file:"+options.pkgBuildOptionFile) if options.inputRPMSPath is not None and not os.path.isdir(options.inputRPMSPath): logger.error("Given input RPMS Path is not a directory:"+options.publishRPMSPath) errorFlag = True if options.installPackage : if len(args) != 1: logger.error("Please provide package name") errorFlag = True else: package=args[0] if errorFlag: logger.error("Found some errors. Please fix input options and re-run it.") return False if not os.path.isdir(options.rpmPath): cmdUtils.runCommandInShell("mkdir -p "+options.rpmPath+"/x86_64") cmdUtils.runCommandInShell("mkdir -p "+options.rpmPath+"/noarch") if not os.path.isdir(options.sourceRpmPath): cmdUtils.runCommandInShell("mkdir -p "+options.sourceRpmPath) if not os.path.isdir(options.buildRootPath): cmdUtils.runCommandInShell("mkdir -p "+options.buildRootPath) logger.info("Source Path :"+options.sourcePath) logger.info("Spec Path :" + options.specPath) logger.info("Rpm Path :" + options.rpmPath) logger.info("Log Path :" + options.logPath) logger.info("Top Dir Path :" + options.topDirPath) logger.info("Publish RPMS Path :" + options.publishRPMSPath) if not options.installPackage: logger.info("JSON File :" + options.inputJSONFile) else: logger.info("Package to build:"+package) listBuildOptionPackages = get_packages_with_build_options(options.pkgBuildOptionFile) try: constants.initialize(options) SourcePackageInfo.setLogging() SourcePackageInfo.loadPkgInfoFromFile(pkgInfoJsonFile) if package == "packages_list": buildPackagesList(options.specPath, options.buildRootPath+"/../packages_list.csv") elif package == "sources_list": if not os.path.isdir("../../stage/yaml_sources"): cmdUtils.runCommandInShell("mkdir -p ../../stage/yaml_sources") buildSourcesList(options.specPath,"../../stage/yaml_sources", options.buildRootPath+"/../",logger) elif package == "srpms_list": if not os.path.isdir("../../stage/yaml_srpms"): cmdUtils.runCommandInShell("mkdir -p ../../stage/yaml_srpms") buildSRPMList(options.specPath,options.sourceRpmPath,"../../stage/yaml_srpms",logger) elif options.toolChainStage == "stage1": pkgManager = PackageManager() pkgManager.buildToolChain() elif options.toolChainStage == "stage2": pkgManager = PackageManager() pkgManager.buildToolChainPackages(options.buildThreads) elif options.installPackage: buildAPackage(package, listBuildOptionPackages, options.pkgBuildOptionFile, options.buildThreads) else: buildPackagesFromGivenJSONFile(options.inputJSONFile, options.buildOption, listBuildOptionPackages, options.pkgBuildOptionFile, logger, options.buildThreads) except Exception as e: logger.error("Caught an exception") logger.error(str(e)) # print stacktrace traceback.print_exc() sys.exit(1) logger.info("Writing Package info to the file:"+pkgInfoJsonFile) SourcePackageInfo.writePkgListToFile(pkgInfoJsonFile) sys.exit(0)
def main(): usage = "Usage: %prog [options] <package name>" parser = OptionParser(usage) parser.add_option("-s", "--spec-path", dest="specPath", default="../../SPECS") parser.add_option("-x", "--source-path", dest="sourcePath", default="../../stage/SOURCES") parser.add_option("-r", "--rpm-path", dest="rpmPath", default="../../stage/RPMS") parser.add_option("-i", "--install-package", dest="installPackage", default=False, action ="store_true") parser.add_option("-p", "--publish-RPMS-path", dest="publishRPMSPath", default="../../stage/PUBLISHRPMS") parser.add_option("-l", "--log-path", dest="logPath", default="../../stage/LOGS") parser.add_option("-o", "--build-option", dest="buildOption", default="full") parser.add_option("-z", "--top-dir-path", dest="topDirPath", default="/usr/src/photon") parser.add_option("-j", "--json-file", dest="inputJSONFile", default="../../common/data/build_install_options_all.json") parser.add_option("-b", "--build-root-path", dest="buildRootPath", default="/mnt") parser.add_option("-t", "--threads", dest="buildThreads", default=1, type="int", help="Number of working threads") parser.add_option("-m", "--tool-chain-stage", dest="toolChainStage", default="None") parser.add_option("-c", "--pullsources-config", dest="pullsourcesConfig", default="pullsources.conf") parser.add_option("-d", "--dist", dest="dist", default="") parser.add_option("-k", "--input-RPMS-path", dest="inputRPMSPath", default=None) parser.add_option("-n", "--build-number", dest="buildNumber", default="0000000") parser.add_option("-v", "--release-version", dest="releaseVersion", default="NNNnNNN") parser.add_option("-u", "--enable-rpmcheck", dest="rpmCheck", default=False, action ="store_true") parser.add_option("-a", "--source-rpm-path", dest="sourceRpmPath", default="../../stage/SRPMS") parser.add_option("-w", "--pkginfo-file", dest="pkgInfoFile", default="../../common/data/pkg_info.json") parser.add_option("-g", "--pkg-build-option-file", dest="pkgBuildOptionFile", default="../../common/data/pkg_build_options.json") parser.add_option("-q", "--rpmcheck-stop-on-error", dest="rpmCheckStopOnError", default=False, action ="store_true") (options, args) = parser.parse_args() cmdUtils=CommandUtils() if not os.path.isdir(options.logPath): cmdUtils.runCommandInShell("mkdir -p "+options.logPath) logger=Logger.getLogger(options.logPath+"/Main") errorFlag=False package = None pkgInfoJsonFile=options.pkgInfoFile if not os.path.isdir(options.sourcePath): logger.error("Given Sources Path is not a directory:"+options.sourcePath) errorFlag = True if not os.path.isdir(options.specPath): logger.error("Given Specs Path is not a directory:"+options.specPath) errorFlag = True if not os.path.isdir(options.publishRPMSPath): logger.error("Given RPMS Path is not a directory:"+options.publishRPMSPath) errorFlag = True if not os.path.isdir(options.publishRPMSPath+"/x86_64"): logger.error("Given RPMS Path is missing x86_64 sub-directory:"+options.publishRPMSPath) errorFlag = True if not os.path.isdir(options.publishRPMSPath+"/noarch"): logger.error("Given RPMS Path is missing noarch sub-directory:"+options.publishRPMSPath) errorFlag = True if not os.path.isfile(options.inputJSONFile) and not options.installPackage: logger.error("Given JSON File is not a file:"+options.inputJSONFile) errorFlag = True if not os.path.isfile(options.pkgBuildOptionFile): logger.warning("Given JSON File is not a file:"+options.pkgBuildOptionFile) if options.inputRPMSPath is not None and not os.path.isdir(options.inputRPMSPath): logger.error("Given input RPMS Path is not a directory:"+options.publishRPMSPath) errorFlag = True if options.installPackage : if len(args) != 1: logger.error("Please provide package name") errorFlag = True else: package=args[0] if errorFlag: logger.error("Found some errors. Please fix input options and re-run it.") return False if not os.path.isdir(options.rpmPath): cmdUtils.runCommandInShell("mkdir -p "+options.rpmPath+"/x86_64") cmdUtils.runCommandInShell("mkdir -p "+options.rpmPath+"/noarch") if not os.path.isdir(options.sourceRpmPath): cmdUtils.runCommandInShell("mkdir -p "+options.sourceRpmPath) if not os.path.isdir(options.buildRootPath): cmdUtils.runCommandInShell("mkdir -p "+options.buildRootPath) logger.info("Source Path :"+options.sourcePath) logger.info("Spec Path :" + options.specPath) logger.info("Rpm Path :" + options.rpmPath) logger.info("Log Path :" + options.logPath) logger.info("Top Dir Path :" + options.topDirPath) logger.info("Publish RPMS Path :" + options.publishRPMSPath) if not options.installPackage: logger.info("JSON File :" + options.inputJSONFile) else: logger.info("Package to build:"+package) listBuildOptionPackages = get_packages_with_build_options(options.pkgBuildOptionFile) try: constants.initialize(options) SourcePackageInfo.setLogging() SourcePackageInfo.loadPkgInfoFromFile(pkgInfoJsonFile) if package == "packages_list": buildPackagesList(options.specPath, options.buildRootPath+"/../packages_list.csv") elif package == "sources_list": if not os.path.isdir("../../stage/yaml_sources"): cmdUtils.runCommandInShell("mkdir -p ../../stage/yaml_sources") buildSourcesList('../../stage/yaml_sources',logger) elif package == "srpms_list": if not os.path.isdir("../../stage/yaml_srpms"): cmdUtils.runCommandInShell("mkdir -p ../../stage/yaml_srpms") buildSRPMList(options.sourceRpmPath,"../../stage/yaml_srpms",logger) elif options.toolChainStage == "stage1": pkgManager = PackageManager() pkgManager.buildToolChain() elif options.toolChainStage == "stage2": pkgManager = PackageManager() pkgManager.buildToolChainPackages(options.buildThreads) elif options.installPackage: buildAPackage(package, listBuildOptionPackages, options.pkgBuildOptionFile, options.buildThreads) else: buildPackagesFromGivenJSONFile(options.inputJSONFile, options.buildOption, listBuildOptionPackages, options.pkgBuildOptionFile, logger, options.buildThreads) except Exception as e: logger.error("Caught an exception") logger.error(str(e)) # print stacktrace traceback.print_exc() sys.exit(1) logger.info("Writing Package info to the file:"+pkgInfoJsonFile) SourcePackageInfo.writePkgListToFile(pkgInfoJsonFile) sys.exit(0)
def buildRPMSForGivenPackage(self, package, chrootID, listBuildOptionPackages, pkgBuildOptionFile, destLogPath=None): self.logger.info("Building rpm's for package:" + package) listSourcesFiles = constants.specData.getSources(package) listPatchFiles = constants.specData.getPatches(package) specFile = constants.specData.getSpecFile(package) specName = constants.specData.getSpecName(package) + ".spec" chrootSourcePath = chrootID + constants.topDirPath + "/SOURCES/" chrootSpecPath = constants.topDirPath + "/SPECS/" chrootLogsFilePath = chrootID + constants.topDirPath + "/LOGS/" + package + ".log" chrootCmd = self.runInChrootCommand + " " + chrootID shutil.copyfile(specFile, chrootID + chrootSpecPath + specName) # FIXME: some sources are located in SPECS/.. how to mount? # if os.geteuid()==0: self.copySourcesTobuildroot(listSourcesFiles, package, chrootSourcePath) self.copySourcesTobuildroot(listPatchFiles, package, chrootSourcePath) listAdditionalFiles = [] macros = [] if package in listBuildOptionPackages: jsonData = open(pkgBuildOptionFile) pkg_build_option_json = json.load( jsonData, object_pairs_hook=collections.OrderedDict) jsonData.close() pkgs_sorted = pkg_build_option_json.items() for pkg in pkgs_sorted: p = str(pkg[0].encode('utf-8')) if p == package: filelist = pkg[1]["files"] for f in filelist: listAdditionalFiles.append(f) macrolist = pkg[1]["macros"] for macro in macrolist: macros.append(str(macro.encode('utf-8'))) self.copyAdditionalBuildFiles(listAdditionalFiles, chrootID) #Adding rpm macros listRPMMacros = constants.specData.getRPMMacros() for macroName in listRPMMacros.keys(): macros.append(macroName + " " + listRPMMacros[macroName]) listRPMFiles = [] listSRPMFiles = [] try: listRPMFiles, listSRPMFiles = self.buildRPM( chrootSpecPath + "/" + specName, chrootLogsFilePath, chrootCmd, package, macros) except Exception as e: self.logger.error("Failed while building rpm:" + package) raise e finally: if destLogPath is not None: if constants.rpmCheck and package in constants.testForceRPMS and constants.specData.isCheckAvailable( package): cmd = "sed -i '/^Executing(%check):/,/^Processing files:/{//!b};d' " + chrootLogsFilePath logFile = destLogPath + "/adjustTestFile.log" returnVal = CommandUtils().runCommandInShell(cmd, logFile) testLogFile = destLogPath + "/" + package + "-test.log" shutil.copyfile(chrootLogsFilePath, testLogFile) else: shutil.copy2(chrootLogsFilePath, destLogPath) self.logger.info("RPM build is successful") for rpmFile in listRPMFiles: self.copyRPM(chrootID + "/" + rpmFile, constants.rpmPath) for srpmFile in listSRPMFiles: self.copyRPM(chrootID + "/" + srpmFile, constants.sourceRpmPath) srpmName = os.path.basename(srpmFile) package, version, release = self.findPackageInfoFromSourceRPMFile( srpmFile) arch = self.getRPMArch(listRPMFiles[0]) SourcePackageInfo.addSRPMData(package, version, release, arch, srpmName)
def buildRPMSForGivenPackage(self,package,chrootID,listBuildOptionPackages,pkgBuildOptionFile,destLogPath=None): self.logger.info("Building rpm's for package:"+package) listSourcesFiles = constants.specData.getSources(package) listPatchFiles = constants.specData.getPatches(package) specFile = constants.specData.getSpecFile(package) specName = constants.specData.getSpecName(package) + ".spec" chrootSourcePath=chrootID+constants.topDirPath+"/SOURCES/" chrootSpecPath=constants.topDirPath+"/SPECS/" chrootLogsFilePath=chrootID+constants.topDirPath+"/LOGS/"+package+".log" chrootCmd=self.runInChrootCommand+" "+chrootID shutil.copyfile(specFile, chrootID+chrootSpecPath+specName ) # FIXME: some sources are located in SPECS/.. how to mount? # if os.geteuid()==0: self.copySourcesTobuildroot(listSourcesFiles,package,chrootSourcePath) self.copySourcesTobuildroot(listPatchFiles,package,chrootSourcePath) listAdditionalFiles = [] macros = [] if package in listBuildOptionPackages: jsonData = open(pkgBuildOptionFile) pkg_build_option_json = json.load(jsonData, object_pairs_hook=collections.OrderedDict) jsonData.close() pkgs_sorted = pkg_build_option_json.items() for pkg in pkgs_sorted: p = str(pkg[0].encode('utf-8')) if p == package: filelist = pkg[1]["files"] for f in filelist: listAdditionalFiles.append(f) macrolist = pkg[1]["macros"] for macro in macrolist: macros.append(str(macro.encode('utf-8'))) self.copyAdditionalBuildFiles(listAdditionalFiles,chrootID) #Adding rpm macros listRPMMacros = constants.specData.getRPMMacros() for macroName in listRPMMacros.keys(): macros.append(macroName+" "+listRPMMacros[macroName]) listRPMFiles=[] listSRPMFiles=[] try: listRPMFiles,listSRPMFiles = self.buildRPM(chrootSpecPath+"/"+specName,chrootLogsFilePath,chrootCmd,package,macros) except Exception as e: self.logger.error("Failed while building rpm:"+package) raise e finally: if destLogPath is not None: if constants.rpmCheck and package in constants.testForceRPMS and constants.specData.isCheckAvailable(package): cmd="sed -i '/^Executing(%check):/,/^Processing files:/{//!b};d' "+ chrootLogsFilePath logFile = destLogPath+"/adjustTestFile.log" returnVal = CommandUtils().runCommandInShell(cmd, logFile) testLogFile = destLogPath+"/"+package+"-test.log" shutil.copyfile(chrootLogsFilePath, testLogFile) else: shutil.copy2(chrootLogsFilePath, destLogPath) self.logger.info("RPM build is successful") for rpmFile in listRPMFiles: self.copyRPM(chrootID+"/"+rpmFile, constants.rpmPath) for srpmFile in listSRPMFiles: self.copyRPM(chrootID+"/"+srpmFile, constants.sourceRpmPath) srpmName = os.path.basename(srpmFile) package,version,release = self.findPackageInfoFromSourceRPMFile(srpmFile) arch = self.getRPMArch(listRPMFiles[0]) SourcePackageInfo.addSRPMData(package,version,release,arch,srpmName)