def installRPMSInAOneShot(self,chrootID,destLogPath):
chrootCmd=self.runInChrootCommand+" "+chrootID
rpmInstallcmd=self.rpmBinary+" "+ self.installRPMPackageOptions
if self.noDepsRPMFilesToInstallInAOneShot != "":
self.logger.info("Installing nodeps rpms: " + self.noDepsPackagesToInstallInAOneShot)
logFile=chrootID+constants.topDirPath+"/LOGS/install_rpms_nodeps.log"
cmdUtils = CommandUtils()
cmd = rpmInstallcmd+" "+self.nodepsRPMPackageOptions + " " + self.noDepsRPMFilesToInstallInAOneShot
returnVal = cmdUtils.runCommandInShell(cmd, logFile, chrootCmd)
if destLogPath is not None:
shutil.copy2(logFile, destLogPath)
if not returnVal:
self.logger.error("Unable to install rpms")
raise Exception("RPM installation failed")
if self.rpmFilesToInstallInAOneShot != "":
self.logger.info("Installing rpms: " + self.packagesToInstallInAOneShot)
logFile=chrootID+constants.topDirPath+"/LOGS/install_rpms.log"
cmdUtils = CommandUtils()
cmd=rpmInstallcmd+" "+self.rpmFilesToInstallInAOneShot
returnVal = cmdUtils.runCommandInShell(cmd, logFile, chrootCmd)
if destLogPath is not None:
shutil.copy2(logFile, destLogPath)
if not returnVal:
self.logger.error("Unable to install rpms")
raise Exception("RPM installation failed")
def installRPM(self, package, chrootID, noDeps=False, destLogPath=None):
self.logger.info("Installing rpm for package:" + package)
self.logger.debug("No deps:" + str(noDeps))
rpmfile = self.findRPMFileForGivenPackage(package)
if rpmfile is None:
self.logger.error("No rpm file found for package:" + package)
raise Exception("Missing rpm file")
rpmDestFile = self.copyRPM(rpmfile, chrootID + constants.topDirPath + "/RPMS")
rpmFile = rpmDestFile.replace(chrootID, "")
chrootCmd = self.runInChrootCommand + " " + chrootID
logFile = chrootID + constants.topDirPath + "/LOGS" + "/" + package + ".completed"
rpmInstallcmd = self.rpmBinary + " " + self.installRPMPackageOptions
if noDeps:
rpmInstallcmd += " " + self.nodepsRPMPackageOptions
rpmInstallcmd += " " + rpmFile
cmdUtils = CommandUtils()
returnVal = cmdUtils.runCommandInShell(rpmInstallcmd, logFile, chrootCmd)
if destLogPath is not None:
shutil.copy2(logFile, destLogPath)
if not returnVal:
self.logger.error("Unable to install rpm:" + rpmFile)
raise Exception("RPM installation failed")
def loadPackagesData(self):
listPackages = SPECS.getData().getListPackages()
listPackages.sort()
cmdUtils = CommandUtils()
for package in listPackages:
for version in SPECS.getData().getVersions(package):
release = SPECS.getData().getRelease(package, version)
listRPMPackages = SPECS.getData().getRPMPackages(package, version)
srpmFileName = package + "-" + version + "-" + release + ".src.rpm"
srpmFiles = cmdUtils.findFile(srpmFileName, constants.sourceRpmPath)
srpmFile = None
if len(srpmFiles) == 1:
srpmFile = srpmFiles[0]
debugrpmFileName = package + "-debuginfo-" + version + "-" + release + "*"
debugrpmFiles = cmdUtils.findFile(debugrpmFileName, constants.rpmPath)
debugrpmFile = None
if len(debugrpmFiles) == 1:
debugrpmFile = debugrpmFiles[0]
pkgUtils = PackageUtils(self.logName, self.logPath)
for rpmPkg in listRPMPackages:
rpmFile = pkgUtils.findRPMFile(rpmPkg, version)
if rpmFile is not None:
listPkgAttributes = {"sourcerpm":srpmFile, "rpm":rpmFile,
"debugrpm":debugrpmFile}
self.pkgList[rpmPkg+"-"+version] = listPkgAttributes
self.logger.debug("Added " + rpmPkg + "-" + version + " to the package info json")
else:
self.logger.debug("Missing rpm file for package:" + rpmPkg)
def buildRPM(self,specFile,logFile,chrootCmd,package,macros):
rpmBuildcmd= self.rpmbuildBinary+" "+self.rpmbuildBuildallOption+" "+self.rpmbuildDistOption
if not constants.rpmCheck:
rpmBuildcmd+=" "+self.rpmbuildNocheckOption
for macro in macros:
rpmBuildcmd+=' --define \\\"%s\\\"' % macro
rpmBuildcmd+=" "+self.rpmbuildBuildNum+" "+self.rpmbuildReleaseVer
rpmBuildcmd+=" "+specFile
cmdUtils = CommandUtils()
self.logger.info("Building rpm....")
self.logger.info(rpmBuildcmd)
returnVal = cmdUtils.runCommandInShell(rpmBuildcmd, logFile, chrootCmd)
if not returnVal:
self.logger.error("Building rpm is failed "+specFile)
raise Exception("RPM Build failed")
#Extracting rpms created from log file
logfile=open(logFile,'r')
fileContents=logfile.readlines()
logfile.close()
listRPMFiles=[]
listSRPMFiles=[]
for i in range(0,len(fileContents)):
if re.search("^Wrote:",fileContents[i]):
listcontents=fileContents[i].split()
if (len(listcontents) == 2) and listcontents[1].strip()[-4:] == ".rpm" and listcontents[1].find("/RPMS/") != -1:
listRPMFiles.append(listcontents[1])
if (len(listcontents) == 2) and listcontents[1].strip()[-8:] == ".src.rpm" and listcontents[1].find("/SRPMS/") != -1:
listSRPMFiles.append(listcontents[1])
return listRPMFiles,listSRPMFiles
def copySourcesTobuildroot(self,listSourceFiles,package,destDir):
cmdUtils = CommandUtils()
for source in listSourceFiles:
# Fetch/verify sources if sha1 not None.
sha1 = constants.specData.getSHA1(package, source)
if sha1 is not None:
PullSources.get(source, sha1, constants.sourcePath, constants.pullsourcesConfig)
sourcePath = cmdUtils.findFile(source,constants.sourcePath)
if sourcePath is None or len(sourcePath) == 0:
sourcePath = cmdUtils.findFile(source,constants.specPath)
if sourcePath is None or len(sourcePath) == 0:
if sha1 is None:
self.logger.error("No sha1 found or missing source for "+source)
raise Exception("No sha1 found or missing source")
else:
self.logger.error("Missing source: "+source+". Cannot find sources for package: "+package)
raise Exception("Missing source")
else:
if sha1 is None:
self.logger.error("No sha1 found for "+source)
raise Exception("No sha1 found")
if len(sourcePath) > 1:
self.logger.error("Multiple sources found for source:"+source+"\n"+ ",".join(sourcePath) +"\nUnable to determine one.")
raise Exception("Multiple sources found")
self.logger.info("Copying... Source path :" + source + " Source filename: " + sourcePath[0])
shutil.copy2(sourcePath[0], destDir)
def buildPackage(self,package):
#should initialize a logger based on package name
chrUtils = ChrootUtils(self.logName,self.logPath)
chrootName="build-"+package
chrootID=None
isToolChainPackage=False
if package in constants.listToolChainPackages:
isToolChainPackage=True
try:
chrootID = self.prepareBuildRoot(chrootName,isToolChainPackage)
destLogPath=constants.logPath+"/build-"+package
if not os.path.isdir(destLogPath):
cmdUtils = CommandUtils()
cmdUtils.runCommandInShell("mkdir -p "+destLogPath)
listInstalledPackages=self.findInstalledPackages(chrootID)
self.logger.info("List of installed packages")
self.logger.info(listInstalledPackages)
listDependentPackages=self.findBuildTimeRequiredPackages(package)
if len(listDependentPackages) != 0:
self.logger.info("Installing the build time dependent packages......")
for pkg in listDependentPackages:
self.installPackage(pkg,chrootID,destLogPath,listInstalledPackages)
self.logger.info("Finished installing the build time dependent packages......")
self.adjustGCCSpecs(package, chrootID, destLogPath)
pkgUtils = PackageUtils(self.logName,self.logPath)
pkgUtils.buildRPMSForGivenPackage(package,chrootID,destLogPath)
self.logger.info("Successfully built the package:"+package)
except Exception as e:
self.logger.error("Failed while building package:" + package)
self.logger.debug("Chroot with ID: " + chrootID + " not deleted for debugging.")
raise e
if chrootID is not None:
chrUtils.destroyChroot(chrootID)
def _copySourcesToContainer(self, listSourceFiles, package, containerID, destDir, index=0):
cmdUtils = CommandUtils()
for source in listSourceFiles:
sourcePath = self._verifyShaAndGetSourcePath(source, package, index)
self.logger.info("Copying source file: " + sourcePath[0])
copyCmd = "docker cp " + sourcePath[0] + " " + containerID.short_id + ":" + destDir
cmdUtils.runCommandInShell(copyCmd)
def _verifyShaAndGetSourcePath(self, source, package, index=0):
cmdUtils = CommandUtils()
# Fetch/verify sources if sha1 not None.
sha1 = SPECS.getData().getSHA1(package, source, index)
if sha1 is not None:
PullSources.get(package, source, sha1, constants.sourcePath,
constants.pullsourcesConfig, self.logger)
sourcePath = cmdUtils.findFile(source, constants.sourcePath)
if not sourcePath:
sourcePath = cmdUtils.findFile(source, constants.specPath)
if not sourcePath:
if sha1 is None:
self.logger.error("No sha1 found or missing source for " + source)
raise Exception("No sha1 found or missing source for " + source)
else:
self.logger.error("Missing source: " + source +
". Cannot find sources for package: " + package)
raise Exception("Missing source")
else:
if sha1 is None:
self.logger.error("No sha1 found for "+source)
raise Exception("No sha1 found")
if len(sourcePath) > 1:
self.logger.error("Multiple sources found for source:" + source + "\n" +
",".join(sourcePath) +"\nUnable to determine one.")
raise Exception("Multiple sources found")
return sourcePath
def adjustGCCSpecs(self, package, chrootID, logPath):
opt = ""
# TODO: reading of hardening flag from spec files
if package == "linux" or package == "glibc":
opt = " clean"
elif package.startswith("xf86-") or package.startswith("xorg-server") :
opt = " nonow"
shutil.copy2(self.adjustGCCSpecScript, chrootID+"/tmp/"+self.adjustGCCSpecScript)
cmdUtils=CommandUtils()
cmd = "/tmp/"+self.adjustGCCSpecScript+opt
logFile = logPath+"/adjustGCCSpecScript.log"
chrootCmd=self.runInChrootCommand+" "+chrootID
retryCount=10
returnVal=False
while retryCount > 0:
returnVal = cmdUtils.runCommandInShell(cmd, logFile, chrootCmd)
if returnVal:
return
self.logger.error("Failed while adjusting gcc specs")
self.logger.error("Retrying again .....")
retryCount = retryCount - 1
sleep(5)
if not returnVal:
self.logger.error("Failed while adjusting gcc specs")
raise "Failed while adjusting gcc specs"
def buildCoreToolChainPackages(self):
self.logger.info("Building core tool chain packages.....")
chrootID=None
try:
pkgUtils=PackageUtils(self.logName,self.logPath)
for package in constants.listCoreToolChainRPMPackages:
rpmPkg=pkgUtils.findRPMFileForGivenPackage(package)
if rpmPkg is not None:
continue
chrUtils = ChrootUtils(self.logName,self.logPath)
chrootName="build-core-toolchain"
destLogPath=constants.logPath+"/build-"+package
if not os.path.isdir(destLogPath):
cmdUtils = CommandUtils()
cmdUtils.runCommandInShell("mkdir -p "+destLogPath)
returnVal,chrootID = chrUtils.createChroot(chrootName)
if not returnVal:
self.logger.error("Creating chroot failed")
raise Exception("creating chroot failed")
self.installToolChainRPMS(chrootID)
pkgUtils.adjustGCCSpecs(package, chrootID, destLogPath)
pkgUtils.buildRPMSForGivenPackage(package, chrootID,destLogPath)
chrUtils.destroyChroot(chrootID)
chrootID=None
self.logger.info("Successfully built toolchain")
if chrootID is not None:
chrUtils.destroyChroot(chrootID)
except Exception as e:
self.logger.error("Unable to build tool chain.")
# print stacktrace
traceback.print_exc()
raise e
def get(package, source, sha1, sourcesPath, URLs, logger):
cmdUtils = CommandUtils()
sourcePath = cmdUtils.findFile(source, sourcesPath)
if sourcePath is not None and len(sourcePath) > 0:
if len(sourcePath) > 1:
raise Exception("Multiple sources found for source:" + source + "\n" +
",".join(sourcePath) +"\nUnable to determine one.")
if sha1 == getFileHash(sourcePath[0]):
# Use file from sourcesPath
return
else:
logger.info("sha1 of " + sourcePath[0] + " does not match. " + sha1 +
" vs " + getFileHash(sourcePath[0]))
for baseurl in URLs:
#form url: https://dl.bintray.com/vmware/photon_sources/1.0/<filename>.
url = '%s/%s' % (baseurl, source)
destfile = os.path.join(sourcesPath, source)
logger.debug("Downloading: " + url)
try:
downloadFile(url, destfile)
if sha1 != getFileHash(destfile):
raise Exception('Invalid sha1 for package %s file %s' % package, source)
return
except requests.exceptions.HTTPError as e:
logger.exception(e)
# on any HTTP errors - try next config
continue
except Exception as e:
logger.exception(e)
raise Exception("Missing source: " + source)
def installCustomToolChainRPMS(self, chrootID, listOfToolChainPkgs, packageName):
self.logger.info("Installing package specific tool chain RPMs for " + packageName +
".......")
rpmFiles = ""
packages = ""
cmdUtils = CommandUtils()
for package in listOfToolChainPkgs:
pkgUtils = PackageUtils(self.logName, self.logPath)
print("DEBUG:" + package)
if "openjre8" in packageName or "openjdk8" in packageName:
# x86_64 has openjdk/jre as a published rpms but aarch64 has openjdk8/jre8
# Remove this condition after publishxrpms for x86_^4 got updated
if ((package == "openjdk" or package == "openjre") and
platform.machine() == "aarch64"):
package = package + "8"
rpmFile = self.findRPMFileInGivenLocation(package, constants.prevPublishXRPMRepo)
else:
rpmFile = self.findRPMFileInGivenLocation(package, constants.prevPublishRPMRepo)
if rpmFile is None:
self.logger.error("Unable to find rpm "+ package +
" in current and previous versions")
raise Exception("Input Error")
rpmFiles += " " + rpmFile
packages += " " + package
self.logger.debug("Installing custom rpms:" + packages)
cmd = (self.rpmCommand + " -i -v --nodeps --noorder --force --root " +
chrootID + " --define \'_dbpath /var/lib/rpm\' " + rpmFiles)
retVal = cmdUtils.runCommandInShell(cmd, self.logPath +
"/install_custom_toolchain_rpms.log")
if not retVal:
self.logger.debug("Command Executed:" + cmd)
self.logger.error("Installing tool chain failed")
raise Exception("RPM installation failed")
self.logger.info("Successfully installed all Tool Chain X RPMS")
def loadPackagesData(self):
listPackages = constants.specData.getListPackages()
listPackages.sort()
listRPMFiles = []
cmdUtils = CommandUtils()
for package in listPackages:
release = constants.specData.getRelease(package)
version = constants.specData.getVersion(package)
listRPMPackages = constants.specData.getRPMPackages(package)
srpmFileName = package+"-"+version+"-"+release+".src.rpm"
srpmFiles = cmdUtils.findFile(srpmFileName, constants.sourceRpmPath)
srpmFile = None
if len(srpmFiles) == 1:
srpmFile = srpmFiles[0]
debugrpmFileName = package+"-debuginfo-"+version+"-"+release+"*"
debugrpmFiles = cmdUtils.findFile(debugrpmFileName, constants.rpmPath)
debugrpmFile = None
if len(debugrpmFiles) == 1:
debugrpmFile = debugrpmFiles[0]
pkgUtils = PackageUtils(self.logName,self.logPath)
for rpmPkg in listRPMPackages:
rpmFile = pkgUtils.findRPMFileForGivenPackage(rpmPkg)
if rpmFile is not None:
listRPMFiles.append(rpmFile)
listPkgAttributes = {"sourcerpm":srpmFile, "rpm":rpmFile, "debugrpm":debugrpmFile}
self.pkgList[rpmPkg] = listPkgAttributes
self.logger.debug("Added "+rpmPkg +" rpm package to the list")
else:
self.logger.error("Missing rpm file for package:"+rpmPkg)
def buildRPM(self, specFile, logFile, chrootCmd):
rpmBuildcmd = self.rpmbuildBinary + " " + self.rpmbuildBuildallOption + " " + self.rpmbuildNocheckOption
rpmBuildcmd += " " + specFile
cmdUtils = CommandUtils()
returnVal = cmdUtils.runCommandInShell(rpmBuildcmd, logFile, chrootCmd)
if not returnVal:
self.logger.error("Building rpm is failed " + specFile)
raise Exception("RPM Build failed")
# Extracting rpms created from log file
logfile = open(logFile, "r")
fileContents = logfile.readlines()
logfile.close()
listRPMFiles = []
for i in range(0, len(fileContents)):
if re.search("^Wrote:", fileContents[i]):
listcontents = fileContents[i].split()
if (
(len(listcontents) == 2)
and listcontents[1].strip()[-4:] == ".rpm"
and listcontents[1].find("/RPMS/") != -1
):
listRPMFiles.append(listcontents[1])
return listRPMFiles
def findRPMFileForGivenPackage(self, package,version="*", index=0):
cmdUtils = CommandUtils()
release = "*"
if version == "*":
version = SPECS.getData().getVersion(package, index)
release = SPECS.getData().getRelease(package, index)
listFoundRPMFiles = sum([cmdUtils.findFile(package + "-" + version + "-" + release + "." +
platform.machine()+".rpm",
constants.rpmPath),
cmdUtils.findFile(package + "-" + version + "-" + release +
".noarch.rpm",
constants.rpmPath)], [])
if constants.inputRPMSPath is not None:
listFoundRPMFiles = sum([cmdUtils.findFile(package + "-" + version + "-" + release +
"." + platform.machine()+".rpm",
constants.inputRPMSPath),
cmdUtils.findFile(package + "-" + version + "-" + release +
".noarch.rpm", constants.inputRPMSPath)],
listFoundRPMFiles)
if len(listFoundRPMFiles) == 1:
return listFoundRPMFiles[0]
if len(listFoundRPMFiles) == 0:
return None
if len(listFoundRPMFiles) > 1:
self.logger.error("Found multiple rpm files for given package in rpm directory." +
"Unable to determine the rpm file for package:" + package)
raise Exception("Multiple rpm files found")
def main():
usage = "Usage: %prog [options]"
parser = ArgumentParser(usage)
parser.add_argument("-i", "--input-type", dest="input_type", default=DEFAULT_INPUT_TYPE)
parser.add_argument("-p", "--pkg", dest="pkg")
parser.add_argument("-f", "--file", dest="json_file", default="packages_minimal.json")
parser.add_argument("-d", "--disp", dest="display_option", default=DEFAULT_DISPLAY_OPTION)
parser.add_argument("-s", "--spec-dir", dest="spec_dir", default=SPEC_FILE_DIR)
parser.add_argument("-l", "--log-dir", dest="log_dir", default=LOG_FILE_DIR)
parser.add_argument("-t", "--stage-dir", dest="stage_dir", default="../../stage")
parser.add_argument("-a", "--input-data-dir", dest="input_data_dir", default="../../common/data/")
parser.add_argument("-o", "--output-dir", dest="output_dir", default="../../stage/common/data")
options = parser.parse_args()
constants.setSpecPath(options.spec_dir)
constants.setLogPath(options.log_dir)
constants.initialize()
cmdUtils = CommandUtils()
if not os.path.isdir(options.output_dir):
cmdUtils.runCommandInShell2("mkdir -p "+options.output_dir)
if not options.input_data_dir.endswith('/'):
options.input_data_dir += '/'
try:
specDeps = SpecDependencyGenerator()
# To display/print package dependencies on console
if (options.input_type == "pkg" or
options.input_type == "who-needs" or
options.input_type == "who-needs-build"):
specDeps.process(options.input_type, options.pkg, options.display_option)
elif options.input_type == "json":
list_json_files = options.json_file.split("\n")
# Generate the expanded package dependencies json file based on package_list_file
for json_file in list_json_files:
shutil.copy2(json_file, options.output_dir)
json_wrapper_option_list = JsonWrapper(json_file)
option_list_json = json_wrapper_option_list.read()
options_sorted = option_list_json.items()
for install_option in options_sorted:
output_file = None
input_value = os.path.join(os.path.dirname(json_file), install_option[1]["file"])
if options.display_option == "tree" and install_option[1]["title"] == "ISO Packages":
continue
if options.display_option == "json":
output_file = os.path.join(options.output_dir, install_option[1]["file"])
print ("Generating the install time dependency list for " + json_file)
specDeps.process(options.input_type, input_value, options.display_option, output_file)
except Exception as e:
traceback.print_exc()
sys.stderr.write(str(e))
sys.stderr.write("Failed to generate dependency lists from spec files\n")
sys.exit(1)
sys.stderr.write("Successfully generated dependency lists from spec files\n")
sys.exit(0)
def findInstalledRPMPackages(self, chrootID):
cmd = self.rpmBinary + " " + self.queryRpmPackageOptions
chrootCmd = self.runInChrootCommand + " " + chrootID
cmdUtils = CommandUtils()
result = cmdUtils.runCommandInShell2(cmd, chrootCmd)
if result is not None:
return result.decode().split()
return result
def _buildPackagePrepareFunction(self, package):
self.package = package
self.logName = "build-" + package
self.logPath = constants.logPath + "/build-" + package
if not os.path.isdir(self.logPath):
cmdUtils = CommandUtils()
cmdUtils.runCommandInShell("mkdir -p " + self.logPath)
self.logger = Logger.getLogger(self.logName, self.logPath)
def writePkgListToFile(self, fileName):
self.logger.debug("Writing package list to the json file")
cmdUtils = CommandUtils()
dirPath = os.path.basename(fileName)
if not os.path.isdir(dirPath):
cmdUtils.runCommandInShell("mkdir -p " + dirPath)
with open(fileName, 'w+') as pkgInfoFile:
json.dump(self.pkgList, pkgInfoFile, indent=4)
def writePkgListToFile(fileName):
SourcePackageInfo.logger.info("Writing source package list to the json file")
cmdUtils = CommandUtils()
dirPath = os.path.basename(fileName)
if not os.path.isdir(dirPath):
cmdUtils.runCommandInShell("mkdir -p " + dirPath)
pkgInfoFile = open(fileName, "w+")
json.dump(SourcePackageInfo.sourcePkgList, pkgInfoFile, indent=4)
pkgInfoFile.close()
def copyRPM(self, rpmFile, destDir):
cmdUtils = CommandUtils()
rpmName = os.path.basename(rpmFile)
rpmDestDir = self.getRPMDestDir(rpmName, destDir)
if not os.path.isdir(rpmDestDir):
cmdUtils.runCommandInShell("mkdir -p " + rpmDestDir)
rpmDestPath = rpmDestDir + "/" + rpmName
shutil.copyfile(rpmFile, rpmDestPath)
return rpmDestPath
def buildRPM(self,specFile,logFile,chrootCmd,package,macros):
rpmBuildcmd=self.rpmbuildBinary+" "+self.rpmbuildBuildallOption+" "+self.rpmbuildDistOption
if constants.rpmCheck and package in constants.testForceRPMS:
self.logger.info("#"*(68+2*len(package)))
if not constants.specData.isCheckAvailable(package):
self.logger.info("####### "+package+" MakeCheck is not available. Skipping MakeCheck TEST for "+package+ " #######")
rpmBuildcmd=self.rpmbuildBinary+" --clean"
else:
self.logger.info("####### "+package+" MakeCheck is available. Running MakeCheck TEST for "+package+ " #######")
rpmBuildcmd=self.rpmbuildBinary+" "+self.rpmbuildCheckOption
self.logger.info("#"*(68+2*len(package)))
else:
rpmBuildcmd+=" "+self.rpmbuildNocheckOption
for macro in macros:
rpmBuildcmd+=' --define \\\"%s\\\"' % macro
rpmBuildcmd+=" "+specFile
cmdUtils = CommandUtils()
self.logger.info("Building rpm....")
self.logger.info(rpmBuildcmd)
returnVal = cmdUtils.runCommandInShell(rpmBuildcmd, logFile, chrootCmd)
if constants.rpmCheck and package in constants.testForceRPMS:
if not constants.specData.isCheckAvailable(package):
constants.testLogger.info(package+" : N/A")
elif returnVal:
constants.testLogger.info(package+" : PASS")
else:
constants.testLogger.error(package+" : FAIL" )
if constants.rpmCheck:
if not returnVal and constants.rpmCheckStopOnError:
self.logger.error("Checking rpm is failed "+specFile)
raise Exception("RPM check failed")
else:
if not returnVal:
self.logger.error("Building rpm is failed "+specFile)
raise Exception("RPM build failed")
#Extracting rpms created from log file
logfile=open(logFile,'r')
fileContents=logfile.readlines()
logfile.close()
listRPMFiles=[]
listSRPMFiles=[]
for i in range(0,len(fileContents)):
if re.search("^Wrote:",fileContents[i]):
listcontents=fileContents[i].split()
if (len(listcontents) == 2) and listcontents[1].strip()[-4:] == ".rpm" and listcontents[1].find("/RPMS/") != -1:
listRPMFiles.append(listcontents[1])
if (len(listcontents) == 2) and listcontents[1].strip()[-8:] == ".src.rpm" and listcontents[1].find("/SRPMS/") != -1:
listSRPMFiles.append(listcontents[1])
return listRPMFiles,listSRPMFiles
def installToolChainRPMS(self, chrootID, packageName, logPath=None):
if logPath is None:
logPath = self.logPath
cmdUtils = CommandUtils()
self.prepareBuildRoot(chrootID)
self.logger.info("Installing Tool Chain RPMS.......")
rpmFiles = ""
packages = ""
for package in constants.listToolChainRPMsToInstall:
pkgUtils = PackageUtils(self.logName, self.logPath)
rpmFile = None
if constants.rpmCheck:
rpmFile = pkgUtils.findRPMFileForGivenPackage(package)
else:
if (packageName not in constants.listToolChainRPMsToInstall or
constants.listToolChainRPMsToInstall.index(packageName) >
constants.listToolChainRPMsToInstall.index(package)):
rpmFile = pkgUtils.findRPMFileForGivenPackage(package)
if rpmFile is None:
# sqlite-autoconf package was renamed, but it still published as sqlite-autoconf
if (package == "sqlite") and (platform.machine() == "x86_64"):
package = "sqlite-autoconf"
rpmFile = self.findRPMFileInGivenLocation(package, constants.prevPublishRPMRepo)
if rpmFile is None:
if package in constants.listOfRPMsProvidedAfterBuild:
self.logger.info("No old version of " + package +
" exists, skip until the new version is built")
continue
self.logger.error("Unable to find rpm " + package +
" in current and previous versions")
raise Exception("Input Error")
rpmFiles += " " + rpmFile
packages += " " + package
self.logger.debug("Installing toolchain rpms:" + packages)
cmd = (self.rpmCommand + " -i -v --nodeps --noorder --force --root " +
chrootID +" --define \'_dbpath /var/lib/rpm\' "+ rpmFiles)
retVal = cmdUtils.runCommandInShell(cmd, logPath + "/install_toolchain_rpms.log")
if not retVal:
self.logger.debug("Command Executed:" + cmd)
self.logger.error("Installing tool chain failed")
raise Exception("RPM installation failed")
self.logger.info("Successfully installed default Tool Chain RPMS in Chroot:" + chrootID)
print("Building Package: ".format(packageName))
print(constants.perPackageToolChain)
if packageName in constants.perPackageToolChain:
print(constants.perPackageToolChain[packageName])
self.installCustomToolChainRPMS(chrootID, constants.perPackageToolChain[packageName],
packageName)
#convert rpm db to lmdb (if required)
convertLogFile = self.logPath+"/rpmdbconvert.log"
pkgUtilsConvert=PackageUtils(self.logName,self.logPath)
pkgUtilsConvert.ensureLMDBForRPMDB(chrootID, convertLogFile)
def findRPMFileForGivenPackage(self,package):
cmdUtils = CommandUtils()
version = constants.specData.getVersion(package)
release = constants.specData.getRelease(package)
listFoundRPMFiles = cmdUtils.findFile(package+"-"+version+"-"+release+"*.rpm",constants.rpmPath)
if len(listFoundRPMFiles) == 1 :
return listFoundRPMFiles[0]
if len(listFoundRPMFiles) == 0 :
return None
if len(listFoundRPMFiles) > 1 :
self.logger.error("Found multiple rpm files for given package in rpm directory.Unable to determine the rpm file for package:"+package)
raise Exception("Multiple rpm files found")
def buildPackage(self,package):
#do not build if RPM is already built
#test only if the package is in the testForceRPMS with rpmCheck
#build only if the package is not in the testForceRPMS with rpmCheck
if self.checkIfPackageIsAlreadyBuilt(package):
if not constants.rpmCheck:
self.logger.info("Skipping building the package:"+package)
return
elif constants.rpmCheck and package not in constants.testForceRPMS:
self.logger.info("Skipping testing the package:"+package)
return
#should initialize a logger based on package name
chrUtils = ChrootUtils(self.logName,self.logPath)
chrootName="build-"+package
chrootID=None
isToolChainPackage=False
if package in constants.listToolChainPackages:
isToolChainPackage=True
try:
chrootID = self.prepareBuildRoot(chrootName,isToolChainPackage)
destLogPath=constants.logPath+"/build-"+package
if not os.path.isdir(destLogPath):
cmdUtils = CommandUtils()
cmdUtils.runCommandInShell("mkdir -p "+destLogPath)
listInstalledPackages=self.findInstalledPackages(chrootID)
listDependentPackages=self.findBuildTimeRequiredPackages(package)
if constants.rpmCheck and package in constants.testForceRPMS:
testPackages=set(constants.listMakeCheckRPMPkgtoInstall)-set(listInstalledPackages)-set([package])
listDependentPackages.extend(testPackages)
pkgUtils = PackageUtils(self.logName,self.logPath)
if len(listDependentPackages) != 0:
self.logger.info("Installing the build time dependent packages......")
for pkg in listDependentPackages:
self.installPackage(pkgUtils, pkg,chrootID,destLogPath,listInstalledPackages)
pkgUtils.installRPMSInAOneShot(chrootID,destLogPath)
self.logger.info("Finished installing the build time dependent packages......")
pkgUtils.adjustGCCSpecs(package, chrootID, destLogPath)
pkgUtils.buildRPMSForGivenPackage(package,chrootID,self.listBuildOptionPackages,self.pkgBuildOptionFile,destLogPath)
self.logger.info("Successfully built the package:"+package)
except Exception as e:
self.logger.error("Failed while building package:" + package)
self.logger.debug("Chroot with ID: " + chrootID + " not deleted for debugging.")
logFileName = os.path.join(destLogPath, package + ".log")
fileLog = os.popen('tail -n 100 ' + logFileName).read()
self.logger.debug(fileLog)
raise e
if chrootID is not None:
chrUtils.destroyChroot(chrootID)
def buildSourcesList(yamlDir, blackListPkgs, logger, singleFile=True):
cmdUtils = CommandUtils()
yamlSourceDir = os.path.join(yamlDir, "yaml_sources")
if not os.path.isdir(yamlSourceDir):
cmdUtils.runCommandInShell("mkdir -p " + yamlSourceDir)
if singleFile:
yamlFile = open(yamlSourceDir + "/sources_list.yaml", "w")
listPackages = SPECS.getData().getListPackages()
listPackages.sort()
import PullSources
for package in listPackages:
if package in blackListPkgs:
continue
ossname = package
for version in SPECS.getData().getVersions(package):
modified = False
listPatches = SPECS.getData().getPatches(package, version)
if listPatches:
modified = True
url = SPECS.getData().getSourceURL(package, version)
if url is None:
url = SPECS.getData().getURL(package, version)
sourceName = None
listSourceNames = SPECS.getData().getSources(package, version)
if listSourceNames:
sourceName = listSourceNames[0]
sha1 = SPECS.getData().getSHA1(package, version, sourceName)
if sha1 is not None:
PullSources.get(package, sourceName, sha1, yamlSourceDir,
constants.getPullSourcesURLs(package),
logger)
if not singleFile:
yamlFile = open(yamlSourceDir + "/" + ossname + "-" + version + ".yaml", "w")
yamlFile.write("vmwsource:" + ossname + ":" + version + ":\n")
yamlFile.write(" repository: VMWsource\n")
yamlFile.write(" name: '" + ossname + "'\n")
yamlFile.write(" version: '" + version + "'\n")
yamlFile.write(" url: " + str(url) + "\n")
yamlFile.write(" license: UNKNOWN\n")
if sourceName is not None:
yamlFile.write(" vmwsource-distribution: " + str(sourceName) + "\n")
if modified:
yamlFile.write(" modified: true\n")
yamlFile.write("\n")
if not singleFile:
yamlFile.close()
if singleFile:
yamlFile.close()
logger.debug("Generated source yaml files for all packages")
def createChroot(self,chrootName):
chrootID=constants.buildRootPath+"/"+chrootName
if os.path.isdir(chrootID):
if not self.destroyChroot(chrootID):
self.logger.error("Given chroot "+chrootID+" is already exists. unable to destroy it ")
return False,None
# need to add timeout for this step
# http://stackoverflow.com/questions/1191374/subprocess-with-timeout
cmdUtils=CommandUtils()
returnVal=cmdUtils.runCommandInShell("mkdir -p "+chrootID)
if not returnVal:
self.logger.error("Unable to create chroot:"+ chrootID +".Unknown error.")
return False,None
return True,chrootID
def buildSRPMList(srpmPath, yamlDir, blackListPkgs, dist_tag, logger, singleFile=True):
cmdUtils = CommandUtils()
yamlSrpmDir = os.path.join(yamlDir, "yaml_srpms")
if not os.path.isdir(yamlSrpmDir):
cmdUtils.runCommandInShell("mkdir -p " + yamlSrpmDir)
if singleFile:
yamlFile = open(yamlSrpmDir + "/srpm_list.yaml", "w")
listPackages = SPECS.getData().getListPackages()
listPackages.sort()
for package in listPackages:
if package in blackListPkgs:
continue
ossname = package
for ossversion in SPECS.getData().getVersions(package):
ossrelease = SPECS.getData().getRelease(package, ossversion)
srpm_file_name = "%s-%s-%s%s.src.rpm" % (ossname, ossversion, ossrelease, dist_tag)
logger.info("srpm name is %s" % (srpm_file_name))
listFoundSRPMFiles = cmdUtils.findFile(srpm_file_name, srpmPath)
srpmName = None
if len(listFoundSRPMFiles) == 1:
srpmFullPath = listFoundSRPMFiles[0]
srpmName = os.path.basename(srpmFullPath)
cpcmd = "cp " + srpmFullPath + " " + yamlSrpmDir + "/"
returnVal = cmdUtils.runCommandInShell(cpcmd)
if returnVal != 0:
logger.error("Copy SRPM File is failed for package:" + ossname)
else:
logger.error("SRPM file is not found:" + ossname)
if not singleFile:
yamlFile = open(yamlSrpmDir + "/" + ossname + "-" + ossversion + "-"
+ ossrelease + ".yaml", "w")
yamlFile.write("baseos:" + ossname + ":" + ossversion + "-" + ossrelease + dist_tag + ":\n")
yamlFile.write(" repository: BaseOS\n")
yamlFile.write(" name: '" + ossname + "'\n")
yamlFile.write(" version: '" + ossversion + "-" + ossrelease + dist_tag +"'\n")
yamlFile.write(" url: 'http://www.vmware.com'\n")
yamlFile.write(" baseos-style: rpm\n")
yamlFile.write(" baseos-source: '" + str(srpmName) + "'\n")
yamlFile.write(" baseos-osname: 'photon'\n")
yamlFile.write("\n")
if not singleFile:
yamlFile.close()
if singleFile:
yamlFile.close()
logger.debug("Generated SRPM yaml files for all packages")
def _copyRPM(self, rpmFile, destDir):
cmdUtils = CommandUtils()
rpmName = os.path.basename(rpmFile)
rpmDestDir = self._getRPMDestDir(rpmName, destDir)
# shutil is not atomic. copy & move to ensure atomicity.
rpmDestPath = rpmDestDir + "/" + rpmName
rpmDestPathTemp = (rpmDestDir + "/." +
''.join([random.choice(string.ascii_letters +
string.digits) for n in range(10)]))
if os.geteuid() == 0:
if not os.path.isdir(rpmDestDir):
cmdUtils.runCommandInShell("mkdir -p " + rpmDestDir)
shutil.copyfile(rpmFile, rpmDestPathTemp)
shutil.move(rpmDestPathTemp, rpmDestPath)
return rpmDestPath
def destroyChroot(self,chrootID):
# need to add timeout for this step
# http://stackoverflow.com/questions/1191374/subprocess-with-timeout
cmdUtils=CommandUtils()
returnVal=cmdUtils.runCommandInShell("./cleanup-build-root.sh "+chrootID)
if not returnVal:
self.logger.error("Unable to destroy chroot:"+ chrootID +".Unknown error.")
return False
returnVal=cmdUtils.runCommandInShell("rm -rf "+chrootID)
if not returnVal:
self.logger.error("Unable to destroy chroot:"+ chrootID +".Unknown error.")
return False
self.logger.info("Successfully destroyed chroot:"+chrootID)
return True
def loadPackagesData(self):
listPackages = SPECS.getData().getListPackages()
listPackages.sort()
listRPMFiles = []
cmdUtils = CommandUtils()
for package in listPackages:
for version in SPECS.getData().getVersions(package):
release = SPECS.getData().getRelease(package, version)
listRPMPackages = SPECS.getData().getRPMPackages(
package, version)
srpmFileName = package + "-" + version + "-" + release + ".src.rpm"
srpmFiles = cmdUtils.findFile(srpmFileName,
constants.sourceRpmPath)
srpmFile = None
if len(srpmFiles) == 1:
srpmFile = srpmFiles[0]
debugrpmFileName = package + "-debuginfo-" + version + "-" + release + "*"
debugrpmFiles = cmdUtils.findFile(debugrpmFileName,
constants.rpmPath)
debugrpmFile = None
if len(debugrpmFiles) == 1:
debugrpmFile = debugrpmFiles[0]
pkgUtils = PackageUtils(self.logName, self.logPath)
for rpmPkg in listRPMPackages:
rpmFile = pkgUtils.findRPMFileForGivenPackage(rpmPkg)
if rpmFile is not None:
listRPMFiles.append(rpmFile)
listPkgAttributes = {
"sourcerpm": srpmFile,
"rpm": rpmFile,
"debugrpm": debugrpmFile
}
self.pkgList[rpmPkg] = listPkgAttributes
self.logger.debug("Added " + rpmPkg +
" rpm package to the list")
else:
self.logger.debug("Missing rpm file for package:" +
rpmPkg)
def installCustomToolChainRPMS(self, chrootID, listOfToolChainPkgs,
packageName):
self.logger.debug("Installing package specific tool chain RPMs for " +
packageName + ".......")
rpmFiles = ""
packages = ""
cmdUtils = CommandUtils()
for package in listOfToolChainPkgs:
pkgUtils = PackageUtils(self.logName, self.logPath)
if re.match("openjre*", packageName) is not None or re.match(
"openjdk*", packageName):
# x86_64 has openjdk/jre as a published rpms but aarch64 has openjdk8/jre8
# Remove this condition after publishxrpms for x86_^4 got updated
if ((package == "openjdk" or package == "openjre")
and platform.machine() == "aarch64"):
package = package + "8"
rpmFile = self.findRPMFileInGivenLocation(
package, constants.prevPublishXRPMRepo)
else:
rpmFile = self.findRPMFileInGivenLocation(
package, constants.prevPublishRPMRepo)
if rpmFile is None:
self.logger.error("Unable to find rpm " + package +
" in current and previous versions")
raise Exception("Input Error")
rpmFiles += " " + rpmFile
packages += " " + package
self.logger.debug("Installing custom rpms:" + packages)
cmd = (self.rpmCommand + " -i -v --nodeps --noorder --force --root " +
chrootID + " --define \'_dbpath /var/lib/rpm\' " + rpmFiles)
retVal = cmdUtils.runCommandInShell(
cmd, self.logPath + "/install_custom_toolchain_rpms.log")
if not retVal:
self.logger.debug("Command Executed:" + cmd)
self.logger.error("Installing tool chain failed")
raise Exception("RPM installation failed")
self.logger.debug("Successfully installed all Tool Chain X RPMS")
def buildRPMSForGivenPackage(self, sandbox, package, version, destLogPath):
self.logger.info("Building package : " + package)
listSourcesFiles = SPECS.getData().getSources(package, version)
listPatchFiles = SPECS.getData().getPatches(package, version)
specFile = SPECS.getData().getSpecFile(package, version)
specName = SPECS.getData().getSpecName(package) + ".spec"
sourcePath = constants.topDirPath + "/SOURCES/"
specPath = constants.topDirPath + "/SPECS/"
if (constants.rpmCheck and
package in constants.testForceRPMS and
SPECS.getData().isCheckAvailable(package, version)):
logFilePath = destLogPath + "/" + package + "-test.log"
else:
logFilePath = destLogPath + "/" + package + ".log"
sandbox.put(specFile, specPath + specName)
sources_urls, macros = self._getAdditionalBuildOptions(package)
self.logger.debug("Extra macros for " + package + ": " + str(macros))
self.logger.debug("Extra source URLs for " + package + ": " + str(sources_urls))
constants.setExtraSourcesURLs(package, sources_urls)
self._copySources(sandbox, listSourcesFiles, package, version, sourcePath)
self._copySources(sandbox, listPatchFiles, package, version, sourcePath)
#Adding rpm macros
listRPMMacros = constants.userDefinedMacros
for macroName, value in listRPMMacros.items():
macros.append(macroName + " " + value)
listRPMFiles = []
listSRPMFiles = []
try:
listRPMFiles, listSRPMFiles = self._buildRPM(sandbox, specPath + specName,
logFilePath, package, version, macros)
logmsg = package + " build done - RPMs : [ "
for f in listRPMFiles:
logmsg += (os.path.basename(f) + " ")
logmsg += "]\n"
self.logger.info(logmsg)
except Exception as e:
self.logger.error("Failed while building rpm:" + package)
raise e
finally:
if (constants.rpmCheck and
package in constants.testForceRPMS and
SPECS.getData().isCheckAvailable(package, version)):
cmd = ("sed -i '/^Executing(%check):/,/^Processing files:/{//!b};d' " + logFilePath)
CommandUtils().runCommandInShell(cmd, logfn=self.logger.debug)
self.logger.debug("RPM build is successful")
def buildRPM(self,specFile,logFile,chrootCmd):
rpmBuildcmd= self.rpmbuildBinary+" "+self.rpmbuildBuildallOption+" "+self.rpmbuildNocheckOption +" "+self.rpmbuildDistOption
rpmBuildcmd+=" "+specFile
cmdUtils = CommandUtils()
returnVal = cmdUtils.runCommandInShell(rpmBuildcmd, logFile, chrootCmd)
if not returnVal:
self.logger.error("Building rpm is failed "+specFile)
raise Exception("RPM Build failed")
#Extracting rpms created from log file
logfile=open(logFile,'r')
fileContents=logfile.readlines()
logfile.close()
listRPMFiles=[]
for i in range(0,len(fileContents)):
if re.search("^Wrote:",fileContents[i]):
listcontents=fileContents[i].split()
if (len(listcontents) == 2) and listcontents[1].strip()[-4:] == ".rpm" and listcontents[1].find("/RPMS/") != -1:
listRPMFiles.append(listcontents[1])
return listRPMFiles
def buildCoreToolChainPackages(self):
self.logger.info("Building core toolchain packages.....")
chrootID = None
pkgCount = 0
try:
pkgUtils = PackageUtils(self.logName, self.logPath)
for package in constants.listCoreToolChainPackages:
rpmPkg = pkgUtils.findRPMFileForGivenPackage(package)
if rpmPkg is not None:
continue
self.logger.info("Building core toolchain package: " + package)
chrUtils = ChrootUtils(self.logName, self.logPath)
chrootName = "build-" + package
destLogPath = constants.logPath + "/build-" + package
if not os.path.isdir(destLogPath):
cmdUtils = CommandUtils()
cmdUtils.runCommandInShell("mkdir -p " + destLogPath)
returnVal, chrootID = chrUtils.createChroot(chrootName)
if not returnVal:
self.logger.error("Creating chroot failed")
raise Exception("creating chroot failed")
self.installToolChainRPMS(chrootID, package, destLogPath)
pkgUtils.adjustGCCSpecs(package, chrootID, destLogPath)
pkgUtils.buildRPMSForGivenPackage(package, chrootID,
destLogPath)
pkgCount += 1
chrUtils.destroyChroot(chrootID)
chrootID = None
self.logger.info("Successfully built toolchain")
if chrootID is not None:
chrUtils.destroyChroot(chrootID)
except Exception as e:
self.logger.error("Unable to build tool chain.")
# print stacktrace
traceback.print_exc()
raise e
return pkgCount
def _copyAdditionalBuildFilesToContainer(self, listAdditionalFiles,
containerID):
cmdUtils = CommandUtils()
#self.logger.debug("VDBG-PU-copyAdditionalBuildFilesToContainer id: " +
# containerID.short_id)
#self.logger.debug(listAdditionalFiles)
for additionalFile in listAdditionalFiles:
source = additionalFile["src"]
destDir = additionalFile["dst"]
destPath = containerID.short_id + ":" + destDir
#TODO: exit status of exec_run
containerID.exec_run("mkdir -p " + destDir)
if os.path.exists(source):
copyCmd = "docker cp " + source
if os.path.isfile(source):
self.logger.info("Copying additional source file: " +
source)
copyCmd += " " + destPath
else:
self.logger.info("Copying additional source file tree: " +
source)
copyCmd += "/. " + destPath
#TODO: cmd error code
cmdUtils.runCommandInShell(copyCmd)
def get(source, sha1, sourcesPath, configs, logger):
cmdUtils = CommandUtils()
sourcePath = cmdUtils.findFile(source, sourcesPath)
if sourcePath is not None and len(sourcePath) > 0:
if len(sourcePath) > 1:
raise Exception("Multiple sources found for source:" + source +
"\n" + ",".join(sourcePath) +
"\nUnable to determine one.")
if sha1 == getFileHash(sourcePath[0]):
# Use file from sourcesPath
return
else:
logger.info("sha1 of " + sourcePath[0] + " does not match. " +
sha1 + " vs " + getFileHash(sourcePath[0]))
configFiles = configs.split(":")
for config in configFiles:
p = pullSources(config, logger)
package_path = os.path.join(sourcesPath, source)
try:
p.downloadFileHelper(source, package_path, sha1)
return
except Exception as e:
logger.exception(e)
raise Exception("Missing source: " + source)
def installRPMSInAOneShot(self, chrootID, destLogPath):
chrootCmd = self.runInChrootCommand + " " + chrootID
rpmInstallcmd = self.rpmBinary + " " + self.installRPMPackageOptions
cmdUtils = CommandUtils()
if self.noDepsRPMFilesToInstallInAOneShot != "":
self.logger.info("Installing nodeps rpms: " +
self.noDepsPackagesToInstallInAOneShot)
logFile = destLogPath + "/install_rpms_nodeps.log"
cmd = (rpmInstallcmd+" "+self.nodepsRPMPackageOptions + " " +
self.noDepsRPMFilesToInstallInAOneShot)
returnVal = cmdUtils.runCommandInShell(cmd, logFile, chrootCmd)
if not returnVal:
self.logger.debug("Command Executed:" + cmd)
self.logger.error("Unable to install rpms")
raise Exception("RPM installation failed")
if self.rpmFilesToInstallInAOneShot != "":
self.logger.info("Installing rpms: " + self.packagesToInstallInAOneShot)
logFile = destLogPath+"/install_rpms.log"
cmd = rpmInstallcmd+" "+self.rpmFilesToInstallInAOneShot
returnVal = cmdUtils.runCommandInShell(cmd, logFile, chrootCmd)
if not returnVal:
self.logger.debug("Command Executed:" + cmd)
self.logger.error("Unable to install rpms")
raise Exception("RPM installation failed")
def __init__(self, logName=None, logPath=None, pkgBuildType="chroot"):
if logName is None:
logName = "PackageManager"
if logPath is None:
logPath = constants.logPath
self.logName = logName
self.logPath = logPath
self.logLevel = constants.logLevel
self.logger = Logger.getLogger(logName, logPath, constants.logLevel)
self.mapCyclesToPackageList = {}
self.mapPackageToCycle = {}
self.sortedPackageList = []
self.listOfPackagesAlreadyBuilt = set()
self.pkgBuildType = pkgBuildType
if self.pkgBuildType == "container":
import docker
self.dockerClient = docker.from_env(version="auto")
cmdUtils = CommandUtils()
# if rpm doesnt have zstd support
if cmdUtils.runCommandInShell(
'rpm --showrc | grep -i "rpmlib(PayloadIsZstd)"',
logfn=self.logger.debug):
constants.hostRpmIsNotUsable = True
self.createZstdBuilderImage()
def _createBuildContainer(self, usePublishedRPMs):
self.logger.debug("Generating photon build container..")
try:
#TODO image name constants.buildContainerImageName
self.dockerClient.images.remove(constants.buildContainerImage,
force=True)
except Exception as e:
#TODO - better handling
self.logger.debug("Photon build container image not found.")
# Create toolchain chroot and install toolchain RPMs
chroot = None
try:
#TODO: constants.tcrootname
chroot = Chroot(self.logger)
chroot.create("toolchain-chroot")
tcUtils = ToolChainUtils("toolchain-chroot", self.logPath)
tcUtils.installToolchainRPMS(chroot,
usePublishedRPMS=usePublishedRPMs)
except Exception as e:
if chroot:
chroot.destroy()
raise e
self.logger.debug("createBuildContainer: " + chroot.getID())
# Create photon build container using toolchain chroot
chroot.unmountAll()
#TODO: Coalesce logging
cmdUtils = CommandUtils()
cmd = "cd " + chroot.getID() + " && tar -czf ../tcroot.tar.gz ."
cmdUtils.runCommandInShell(cmd, logfn=self.logger.debug)
cmd = "mv " + chroot.getID() + "/../tcroot.tar.gz ."
cmdUtils.runCommandInShell(cmd, logfn=self.logger.debug)
#TODO: Container name, docker file name from constants.
self.dockerClient.images.build(
tag=constants.buildContainerImage,
path=".",
rm=True,
dockerfile="Dockerfile.photon_build_container")
# Cleanup
cmd = "rm -f ./tcroot.tar.gz"
cmdUtils.runCommandInShell(cmd, logfn=self.logger.debug)
chroot.destroy()
self.logger.debug("Photon build container successfully created.")
def _findPublishedRPM(self, package, rpmdirPath):
listFoundRPMFiles = CommandUtils.findFile(package + "-*.rpm", rpmdirPath)
listFilterRPMFiles = []
for f in listFoundRPMFiles:
rpmFileName = os.path.basename(f)
checkRPMName = rpmFileName.replace(package, "")
rpmNameSplit = checkRPMName.split("-")
if len(rpmNameSplit) == 3:
listFilterRPMFiles.append(f)
if len(listFilterRPMFiles) == 1:
return listFilterRPMFiles[0]
if len(listFilterRPMFiles) == 0:
return None
if len(listFilterRPMFiles) > 1:
self.logger.error("Found multiple rpm files for given package in rpm directory." +
"Unable to determine the rpm file for package:" + package)
return None
def installToolChainRPMS(self, chrootID, packageName):
cmdUtils = CommandUtils()
self.prepareBuildRoot(chrootID)
self.logger.info("Installing Tool Chain RPMS.......")
rpmFiles = ""
packages = ""
for package in constants.listToolChainRPMsToInstall:
pkgUtils = PackageUtils(self.logName, self.logPath)
rpmFile = pkgUtils.findRPMFileForGivenPackage(package)
if rpmFile is None:
# sqlite-autoconf package was renamed, but it still published as sqlite-autoconf
if package == "sqlite":
package = "sqlite-autoconf"
rpmFile = self.findRPMFileInGivenLocation(
package, constants.prevPublishRPMRepo)
if rpmFile is None:
if package in constants.listOfRPMsProvidedAfterBuild:
self.logger.info(
"No old version of " + package +
" exists, skip until the new version is built")
continue
self.logger.error("Unable to find rpm " + package +
" in current and previous versions")
raise Exception("Input Error")
rpmFiles += " " + rpmFile
packages += " " + package
self.logger.debug("Installing rpms:" + packages)
cmd = self.rpmCommand + " -i --nodeps --force --root " + chrootID + " --define \'_dbpath /var/lib/rpm\' " + rpmFiles
process = subprocess.Popen("%s" % cmd,
shell=True,
stdout=subprocess.PIPE)
retval = process.wait()
if retval != 0:
self.logger.error("Installing tool chain failed")
raise Exception("RPM installation failed")
self.logger.info(
"Successfully installed default Tool Chain RPMS in Chroot:" +
chrootID)
print "Building Package:" + packageName
print constants.perPackageToolChain
if packageName in constants.perPackageToolChain:
print constants.perPackageToolChain[packageName]
self.installCustomToolChainRPMS(
chrootID, constants.perPackageToolChain[packageName],
packageName)
def buildSRPMList(srpmPath, yamlDir, blackListPkgs, logger, singleFile=True):
cmdUtils = CommandUtils()
yamlSrpmDir = os.path.join(yamlDir, "yaml_srpms")
if not os.path.isdir(yamlSrpmDir):
cmdUtils.runCommandInShell("mkdir -p " + yamlSrpmDir)
if singleFile:
yamlFile = open(yamlSrpmDir + "/srpm_list.yaml", "w")
listPackages = SPECS.getData().getListPackages()
listPackages.sort()
for package in listPackages:
if package in blackListPkgs:
continue
ossname = package
for ossversion in SPECS.getData().getVersions(package):
ossrelease = SPECS.getData().getRelease(package, ossversion)
listFoundSRPMFiles = cmdUtils.findFile(
ossname + "-" + ossversion + "-" + ossrelease + ".src.rpm",
srpmPath)
srpmName = None
if len(listFoundSRPMFiles) == 1:
srpmFullPath = listFoundSRPMFiles[0]
srpmName = os.path.basename(srpmFullPath)
cpcmd = "cp " + srpmFullPath + " " + yamlSrpmDir + "/"
returnVal = cmdUtils.runCommandInShell(cpcmd)
if not returnVal:
logger.error("Copy SRPM File is failed for package:" +
ossname)
else:
logger.error("SRPM file is not found:" + ossname)
if not singleFile:
yamlFile = open(
yamlSrpmDir + "/" + ossname + "-" + ossversion + "-" +
ossrelease + ".yaml", "w")
yamlFile.write("baseos:" + ossname + ":" + ossversion + "-" +
ossrelease + ":\n")
yamlFile.write(" repository: BaseOS\n")
yamlFile.write(" name: '" + ossname + "'\n")
yamlFile.write(" version: '" + ossversion + "-" + ossrelease +
"'\n")
yamlFile.write(" url: 'http://www.vmware.com'\n")
yamlFile.write(" baseos-style: rpm\n")
yamlFile.write(" baseos-source: '" + str(srpmName) + "'\n")
yamlFile.write(" baseos-osname: 'photon'\n")
yamlFile.write("\n")
if not singleFile:
yamlFile.close()
if singleFile:
yamlFile.close()
logger.info("Generated SRPM yaml files for all packages")
def findRPMFile(self, package, version="*"):
cmdUtils = CommandUtils()
if version == "*":
version = SPECS.getData().getHighestVersion(package)
release = SPECS.getData().getRelease(package, version)
buildarch = SPECS.getData().getBuildArch(package, version)
filename = package + "-" + version + "-" + release + "." + buildarch + ".rpm"
fullpath = constants.rpmPath + "/" + buildarch + "/" + filename
if os.path.isfile(fullpath):
return fullpath
if constants.inputRPMSPath is not None:
fullpath = constants.inputRPMSPath + "/" + buildarch + "/" + filename
if os.path.isfile(fullpath):
return fullpath
return None
def installToolChain(self, chrootID):
self.logger.info("Installing toolchain.....")
self.prepareBuildRoot(chrootID)
cmdUtils = CommandUtils()
rpmFiles = ""
packages = ""
for package in constants.listToolChainRPMPkgsToInstall:
pkgUtils = PackageUtils(self.logName, self.logPath)
rpmFile = pkgUtils.findRPMFileForGivenPackage(package)
if rpmFile is None:
rpmFile = self.findRPMFileInGivenLocation(
package, constants.prevPublishRPMRepo)
if rpmFile is None:
if package == "util-linux-devel":
self.logger.info(
"No old version of util-linux-devel exists, skip until the new version is built"
)
continue
if package == "flex-devel":
self.logger.info(
"No old version of flex-devel exists, skip until the new version is built"
)
continue
self.logger.error("Unable to find rpm " + package +
" in current and previous versions")
raise Exception("Input Error")
rpmFiles += " " + rpmFile
packages += " " + package
self.logger.debug("Installing toolchain rpms:" + packages)
cmd = self.rpmCommand + " -i --nodeps --force --root " + chrootID + " --define \'_dbpath /var/lib/rpm\' " + rpmFiles
process = subprocess.Popen("%s" % cmd,
shell=True,
stdout=subprocess.PIPE)
retval = process.wait()
if retval != 0:
self.logger.error("Installing toolchain rpms failed")
raise Exception("RPM installation failed")
self.logger.info("Installed toolchain successfully on chroot:" +
chrootID)
def installToolChainRPMS(self,chrootID):
cmdUtils = CommandUtils()
self.prepareBuildRoot(chrootID)
self.logger.info("Installing Tool Chain RPMS.......")
for package in constants.listToolChainRPMPkgsToBuild:
pkgUtils=PackageUtils(self.logName,self.logPath)
rpmFile=pkgUtils.findRPMFileForGivenPackage(package)
if rpmFile is None:
rpmFile=self.findRPMFileInGivenLocation(package, constants.prevPublishRPMRepo)
if rpmFile is None:
self.logger.error("Unable to find rpm "+ package +" in current and previous versions")
raise "Input Error"
self.logger.debug("Installing rpm:"+rpmFile)
cmd="rpm -i --nodeps --force --root "+chrootID+" --define \'_dbpath /var/lib/rpm\' "+ rpmFile
process = subprocess.Popen("%s" %cmd,shell=True,stdout=subprocess.PIPE)
retval = process.wait()
if retval != 0:
self.logger.error("Installing tool chain package "+package+" failed")
raise "RPM installation failed"
self.logger.info("Successfully installed all Tool Chain RPMS in Chroot:"+chrootID)
def findRPMFileForGivenPackage(self,package):
cmdUtils = CommandUtils()
version = constants.specData.getVersion(package)
release = constants.specData.getRelease(package)
listFoundRPMFiles = sum([cmdUtils.findFile(package+"-"+version+"-"+release+".x86_64.rpm",constants.rpmPath),
cmdUtils.findFile(package+"-"+version+"-"+release+".noarch.rpm",constants.rpmPath)], [])
if constants.inputRPMSPath is not None:
listFoundRPMFiles = sum([cmdUtils.findFile(package+"-"+version+"-"+release+".x86_64.rpm",constants.inputRPMSPath),
cmdUtils.findFile(package+"-"+version+"-"+release+".noarch.rpm",constants.inputRPMSPath)], listFoundRPMFiles)
if len(listFoundRPMFiles) == 1 :
return listFoundRPMFiles[0]
if len(listFoundRPMFiles) == 0 :
return None
if len(listFoundRPMFiles) > 1 :
self.logger.error("Found multiple rpm files for given package in rpm directory.Unable to determine the rpm file for package:"+package)
raise Exception("Multiple rpm files found")
def adjustGCCSpecs(self, package, chrootID, logPath):
opt = " " + constants.specData.getSecurityHardeningOption(package)
cmdUtils=CommandUtils()
cpcmd="cp "+ self.adjustGCCSpecScript+" "+chrootID+"/tmp/"+self.adjustGCCSpecScript
cmd = "/tmp/"+self.adjustGCCSpecScript+opt
logFile = logPath+"/adjustGCCSpecScript.log"
chrootCmd=self.runInChrootCommand+" "+chrootID
returnVal = cmdUtils.runCommandInShell(cpcmd, logFile)
if not returnVal:
self.logger.error("Error during copying the file adjust gcc spec")
raise Exception("Failed while copying adjust gcc spec file")
returnVal = cmdUtils.runCommandInShell(cmd, logFile, chrootCmd)
if returnVal:
return
self.logger.debug(cmdUtils.runCommandInShell2("ls -la " + chrootID + "/tmp/" + self.adjustGCCSpecScript))
self.logger.debug(cmdUtils.runCommandInShell2("lsof " + chrootID + "/tmp/" + self.adjustGCCSpecScript))
self.logger.debug(cmdUtils.runCommandInShell2("ps ax"))
self.logger.error("Failed while adjusting gcc specs")
raise Exception("Failed while adjusting gcc specs")
def findRPMFile(self, package,version="*",arch=None, throw=False):
if not arch:
arch=constants.currentArch
cmdUtils = CommandUtils()
if version == "*":
version = SPECS.getData(arch).getHighestVersion(package)
release = SPECS.getData(arch).getRelease(package, version)
buildarch=SPECS.getData(arch).getBuildArch(package, version)
filename= package + "-" + version + "-" + release + "." + buildarch+".rpm"
fullpath = constants.rpmPath + "/" + buildarch + "/" + filename
if os.path.isfile(fullpath):
return fullpath
if constants.inputRPMSPath is not None:
fullpath = constants.inputRPMSPath + "/" + buildarch + "/" + filename
if os.path.isfile(fullpath):
return fullpath
if throw:
raise Exception("RPM %s not found" % (filename))
return None
def adjustGCCSpecs(self, package, chrootID, logPath):
opt = " " + constants.specData.getSecurityHardeningOption(package)
shutil.copy2(self.adjustGCCSpecScript,
chrootID + "/tmp/" + self.adjustGCCSpecScript)
cmdUtils = CommandUtils()
cmd = "/tmp/" + self.adjustGCCSpecScript + opt
logFile = logPath + "/adjustGCCSpecScript.log"
chrootCmd = self.runInChrootCommand + " " + chrootID
returnVal = cmdUtils.runCommandInShell(cmd, logFile, chrootCmd)
if returnVal:
return
self.logger.debug(
cmdUtils.runCommandInShell2("ls -la " + chrootID + "/tmp/" +
self.adjustGCCSpecScript))
self.logger.debug(
cmdUtils.runCommandInShell2("lsof " + chrootID + "/tmp/" +
self.adjustGCCSpecScript))
self.logger.debug(cmdUtils.runCommandInShell2("ps ax"))
self.logger.error("Failed while adjusting gcc specs")
raise "Failed while adjusting gcc specs"
def buildRPMSForGivenPackage(self,
package,
version,
chrootID,
destLogPath=None):
self.logger.info("Building rpm's for package:" + package)
listSourcesFiles = SPECS.getData().getSources(package, version)
listPatchFiles = SPECS.getData().getPatches(package, version)
specFile = SPECS.getData().getSpecFile(package, version)
specName = SPECS.getData().getSpecName(package) + ".spec"
chrootSourcePath = chrootID + constants.topDirPath + "/SOURCES/"
chrootSpecPath = constants.topDirPath + "/SPECS/"
chrootLogsFilePath = chrootID + constants.topDirPath + "/LOGS/" + package + ".log"
chrootCmd = self.runInChrootCommand + " " + chrootID
shutil.copyfile(specFile, chrootID + chrootSpecPath + specName)
# FIXME: some sources are located in SPECS/.. how to mount?
# if os.geteuid()==0:
self._copySourcesTobuildroot(listSourcesFiles, package, version,
chrootSourcePath)
self._copySourcesTobuildroot(listPatchFiles, package, version,
chrootSourcePath)
macros = []
listAdditionalFiles, macros = self._getAdditionalBuildFiles(package)
self._copyAdditionalBuildFiles(listAdditionalFiles, chrootID)
#Adding rpm macros
listRPMMacros = constants.userDefinedMacros
for macroName, value in listRPMMacros.items():
macros.append(macroName + " " + value)
listRPMFiles = []
listSRPMFiles = []
try:
listRPMFiles, listSRPMFiles = self._buildRPM(
chrootSpecPath + specName, chrootLogsFilePath, chrootCmd,
package, macros)
self.logger.info("Successfully built rpm:" + package)
except Exception as e:
self.logger.error("Failed while building rpm:" + package)
raise e
finally:
if destLogPath is not None:
if (constants.rpmCheck and package in constants.testForceRPMS
and SPECS.getData().isCheckAvailable(package)):
cmd = (
"sed -i '/^Executing(%check):/,/^Processing files:/{//!b};d' "
+ chrootLogsFilePath)
logFile = destLogPath + "/adjustTestFile.log"
returnVal = CommandUtils().runCommandInShell(cmd, logFile)
testLogFile = destLogPath + "/" + package + "-test.log"
shutil.copyfile(chrootLogsFilePath, testLogFile)
else:
shutil.copy2(chrootLogsFilePath, destLogPath)
self.logger.info("RPM build is successful")
for rpmFile in listRPMFiles:
self._copyRPM(chrootID + "/" + rpmFile, constants.rpmPath)
for srpmFile in listSRPMFiles:
srpmDestFile = self._copyRPM(chrootID + "/" + srpmFile,
constants.sourceRpmPath)
def prepareBuildContainer(self,
containerTaskName,
packageName,
isToolChainPackage=False):
# Prepare an empty chroot environment to let docker use the BUILD folder.
# This avoids docker using overlayFS which will cause make check failure.
chrootName = "build-" + packageName
chrUtils = ChrootUtils(self.base.logName, self.base.logPath)
returnVal, chrootID = chrUtils.createChroot(chrootName)
if not returnVal:
raise Exception("Unable to prepare build root")
cmdUtils = CommandUtils()
cmdUtils.runCommandInShell("mkdir -p " + chrootID +
constants.topDirPath)
cmdUtils.runCommandInShell("mkdir -p " + chrootID +
constants.topDirPath + "/BUILD")
containerID = None
mountVols = {
constants.prevPublishRPMRepo: {
'bind': '/publishrpms',
'mode': 'ro'
},
constants.prevPublishXRPMRepo: {
'bind': '/publishxrpms',
'mode': 'ro'
},
constants.tmpDirPath: {
'bind': '/tmp',
'mode': 'rw'
},
constants.rpmPath: {
'bind': constants.topDirPath + "/RPMS",
'mode': 'rw'
},
constants.sourceRpmPath: {
'bind': constants.topDirPath + "/SRPMS",
'mode': 'rw'
},
constants.logPath + "/" + self.base.logName: {
'bind': constants.topDirPath + "/LOGS",
'mode': 'rw'
},
chrootID + constants.topDirPath + "/BUILD": {
'bind': constants.topDirPath + "/BUILD",
'mode': 'rw'
},
constants.dockerUnixSocket: {
'bind': constants.dockerUnixSocket,
'mode': 'rw'
}
}
containerName = containerTaskName
containerName = containerName.replace("+", "p")
try:
oldContainerID = self.dockerClient.containers.get(containerName)
if oldContainerID is not None:
oldContainerID.remove(force=True)
except docker.errors.NotFound:
sys.exc_clear()
try:
self.base.logger.info(
"BuildContainer-prepareBuildContainer: Starting build container: "
+ containerName)
#TODO: Is init=True equivalent of --sig-proxy?
privilegedDocker = False
cap_list = ['SYS_PTRACE']
if packageName in constants.listReqPrivilegedDockerForTest:
privilegedDocker = True
containerID = self.dockerClient.containers.run(
self.buildContainerImage,
detach=True,
cap_add=cap_list,
privileged=privilegedDocker,
name=containerName,
network_mode="host",
volumes=mountVols,
command="/bin/bash -l -c /wait.sh")
self.base.logger.debug("Started Photon build container for task " +
containerTaskName + " ID: " +
containerID.short_id)
if not containerID:
raise Exception(
"Unable to start Photon build container for task " +
containerTaskName)
except Exception as e:
self.base.logger.debug(
"Unable to start Photon build container for task " +
containerTaskName)
raise e
return containerID, chrootID
def main():
usage = "Usage: %prog [options] <package name>"
parser = ArgumentParser(usage)
parser.add_argument("-s", "--spec-path", dest="specPath",
default="../../SPECS")
parser.add_argument("-l", "--log-path", dest="logPath",
default="../../stage/LOGS")
parser.add_argument("-a", "--source-rpm-path", dest="sourceRpmPath",
default="../../stage/SRPMS")
parser.add_argument("-j", "--output-dir", dest="outputDirPath",
default="../../stage/")
parser.add_argument("-z", "--log-level", dest="logLevel",
default="info")
parser.add_argument("-c", "--pullsources-config", dest="pullsourcesConfig",
default="pullsources.conf")
parser.add_argument("-f", "--pkg-blacklist-file", dest="pkgBlacklistFile",
default=None)
parser.add_argument("-p", "--generate-pkg-list", dest="generatePkgList",
default=False, action="store_true")
parser.add_argument("-y", "--generate-yaml-files", dest="generateYamlFiles",
default=False, action="store_true")
parser.add_argument("-d", "--dist-tag", dest="dist",
default="")
options = parser.parse_args()
errorFlag = False
cmdUtils = CommandUtils()
try:
logName = "GenerateYamlFiles"
logger = Logger.getLogger(logName, options.logPath, options.logLevel)
if options.generateYamlFiles:
if (options.pkgBlacklistFile is not None and
options.pkgBlacklistFile != "" and
not os.path.isfile(options.pkgBlacklistFile)):
logger.error("Given package blacklist file is not valid:"
+ options.pkgBlacklistFile)
errorFlag = True
if not os.path.isdir(options.specPath):
logger.error("Given Specs Path is not a directory:" + options.specPath)
errorFlag = True
if not os.path.isdir(options.sourceRpmPath):
logger.error("Given SRPM Path is not a directory:" + options.sourceRpmPath)
errorFlag = True
if options.generateYamlFiles and not os.path.isfile(options.pullsourcesConfig):
logger.error("Given Source config file is not a valid file:"
+ options.pullsourcesConfig)
errorFlag = True
if options.dist:
dist_tag = options.dist
logger.info("release tag is %s" % (dist_tag))
if errorFlag:
logger.error("Found some errors. Please fix input options and re-run it.")
sys.exit(1)
if options.generateYamlFiles:
if not os.path.isdir(options.outputDirPath):
cmdUtils.runCommandInShell("mkdir -p "+options.outputDirPath)
constants.setSpecPath(options.specPath)
constants.setSourceRpmPath(options.sourceRpmPath)
constants.setLogPath(options.logPath)
constants.setLogLevel(options.logLevel)
constants.setPullSourcesURL(get_baseurl(options.pullsourcesConfig))
constants.initialize()
# parse SPECS folder
SPECS()
if options.generatePkgList:
buildPackagesList(options.outputDirPath + "/packages_list.csv")
elif options.generateYamlFiles:
blackListPkgs = readBlackListPackages(options.pkgBlacklistFile)
buildSourcesList(options.outputDirPath, blackListPkgs, logger)
buildSRPMList(options.sourceRpmPath, options.outputDirPath, blackListPkgs, dist_tag, logger)
except Exception as e:
print("Caught Exception: " + str(e))
traceback.print_exc()
sys.exit(1)
sys.exit(0)
def prepareBuildRoot(self, chrootID):
self.logger.info("Preparing build environment")
cmdUtils = CommandUtils()
cmdUtils.runCommandInShell("mkdir -p " + chrootID + "/dev")
cmdUtils.runCommandInShell("mkdir -p " + chrootID + "/etc")
cmdUtils.runCommandInShell("mkdir -p " + chrootID + "/proc")
cmdUtils.runCommandInShell("mkdir -p " + chrootID + "/run")
cmdUtils.runCommandInShell("mkdir -p " + chrootID + "/sys")
cmdUtils.runCommandInShell("mkdir -p " + chrootID + "/tmp")
cmdUtils.runCommandInShell("mkdir -p " + chrootID +
constants.topDirPath)
cmdUtils.runCommandInShell("mkdir -p " + chrootID +
constants.topDirPath + "/RPMS/x86_64")
cmdUtils.runCommandInShell("mkdir -p " + chrootID +
constants.topDirPath + "/RPMS/noarch")
cmdUtils.runCommandInShell("mkdir -p " + chrootID +
constants.topDirPath + "/SOURCES")
cmdUtils.runCommandInShell("mkdir -p " + chrootID +
constants.topDirPath + "/SPECS")
cmdUtils.runCommandInShell("mkdir -p " + chrootID +
constants.topDirPath + "/LOGS")
cmdUtils.runCommandInShell("mkdir -p " + chrootID +
constants.topDirPath + "/BUILD")
cmdUtils.runCommandInShell("mkdir -p " + chrootID +
constants.topDirPath + "/BUILDROOT")
prepareChrootCmd = self.prepareBuildRootCmd + " " + chrootID
logFile = constants.logPath + "/prepareBuildRoot.log"
returnVal = cmdUtils.runCommandInShell(prepareChrootCmd, logFile)
if not returnVal:
self.logger.error(
"Prepare build root script failed.Unable to prepare chroot.")
raise Exception("Prepare build root script failed")
self.logger.info("Successfully prepared chroot:" + chrootID)
def main():
usage = "Usage: %prog [options] <package name>"
parser = ArgumentParser(usage)
parser.add_argument("-s", "--spec-path", dest="specPath", default="../../SPECS")
parser.add_argument("-x", "--source-path", dest="sourcePath", default="../../stage/SOURCES")
parser.add_argument("-r", "--rpm-path", dest="rpmPath", default="../../stage/RPMS")
parser.add_argument("-i", "--install-package", dest="installPackage", default=False, action ="store_true")
parser.add_argument("-p", "--publish-RPMS-path", dest="publishRPMSPath", default="../../stage/PUBLISHRPMS")
parser.add_argument("-e", "--publish-XRPMS-path", dest="publishXRPMSPath", default="../../stage/PUBLISHXRPMS")
parser.add_argument("-l", "--log-path", dest="logPath", default="../../stage/LOGS")
parser.add_argument("-z", "--top-dir-path", dest="topDirPath", default="/usr/src/photon")
parser.add_argument("-b", "--build-root-path", dest="buildRootPath", default="/mnt")
parser.add_argument("-t", "--threads", dest="buildThreads", default=1, type=int, help="Number of working threads")
parser.add_argument("-m", "--tool-chain-stage", dest="toolChainStage", default="None")
parser.add_argument("-c", "--pullsources-config", dest="pullsourcesConfig", default="pullsources.conf")
parser.add_argument("-d", "--dist", dest="dist", default="")
parser.add_argument("-k", "--input-RPMS-path", dest="inputRPMSPath", default=None)
parser.add_argument("-n", "--build-number", dest="buildNumber", default="0000000")
parser.add_argument("-v", "--release-version", dest="releaseVersion", default="NNNnNNN")
parser.add_argument("-u", "--enable-rpmcheck", dest="rpmCheck", default=False, action ="store_true")
parser.add_argument("-a", "--source-rpm-path", dest="sourceRpmPath", default="../../stage/SRPMS")
parser.add_argument("-w", "--pkginfo-file", dest="pkgInfoFile", default="../../stage/pkg_info.json")
parser.add_argument("-g", "--pkg-build-option-file", dest="pkgBuildOptionFile", default="../../common/data/pkg_build_options.json")
parser.add_argument("-q", "--rpmcheck-stop-on-error", dest="rpmCheckStopOnError", default=False, action ="store_true")
parser.add_argument("PackageName", nargs='?')
options = parser.parse_args()
cmdUtils=CommandUtils()
if not os.path.isdir(options.logPath):
cmdUtils.runCommandInShell("mkdir -p "+options.logPath)
logger=Logger.getLogger(options.logPath+"/Main")
errorFlag=False
package = None
pkgInfoJsonFile = options.pkgInfoFile
if not os.path.isdir(options.sourcePath):
logger.error("Given Sources Path is not a directory:"+options.sourcePath)
errorFlag = True
if not os.path.isdir(options.specPath):
logger.error("Given Specs Path is not a directory:"+options.specPath)
errorFlag = True
if not os.path.isdir(options.publishRPMSPath):
logger.error("Given RPMS Path is not a directory:"+options.publishRPMSPath)
errorFlag = True
if not os.path.isdir(options.publishXRPMSPath):
logger.error("Given X RPMS Path is not a directory:"+options.publishXRPMSPath)
errorFlag = True
if not os.path.isdir(options.publishRPMSPath+"/x86_64"):
logger.error("Given RPMS Path is missing x86_64 sub-directory:"+options.publishRPMSPath)
errorFlag = True
if not os.path.isdir(options.publishXRPMSPath+"/x86_64"):
logger.error("Given X RPMS Path is missing x86_64 sub-directory:"+options.publishXRPMSPath)
errorFlag = True
if not os.path.isdir(options.publishRPMSPath+"/noarch"):
logger.error("Given RPMS Path is missing noarch sub-directory:"+options.publishRPMSPath)
errorFlag = True
if not os.path.isdir(options.publishXRPMSPath+"/noarch"):
logger.error("Given X RPMS Path is missing noarch sub-directory:"+options.publishXRPMSPath)
errorFlag = True
if not os.path.isfile(options.pkgBuildOptionFile):
logger.warning("Given JSON File is not a file:"+options.pkgBuildOptionFile)
if options.inputRPMSPath is not None and not os.path.isdir(options.inputRPMSPath):
logger.error("Given input RPMS Path is not a directory:"+options.inputRPMSPath)
errorFlag = True
if options.installPackage :
if not options.PackageName:
logger.error("Please provide package name")
errorFlag = True
else:
package=options.PackageName
if errorFlag:
logger.error("Found some errors. Please fix input options and re-run it.")
return False
if not os.path.isdir(options.rpmPath):
cmdUtils.runCommandInShell("mkdir -p "+options.rpmPath+"/x86_64")
cmdUtils.runCommandInShell("mkdir -p "+options.rpmPath+"/noarch")
if not os.path.isdir(options.sourceRpmPath):
cmdUtils.runCommandInShell("mkdir -p "+options.sourceRpmPath)
if not os.path.isdir(options.buildRootPath):
cmdUtils.runCommandInShell("mkdir -p "+options.buildRootPath)
logger.info("Source Path :"+options.sourcePath)
logger.info("Spec Path :" + options.specPath)
logger.info("Rpm Path :" + options.rpmPath)
logger.info("Log Path :" + options.logPath)
logger.info("Top Dir Path :" + options.topDirPath)
logger.info("Publish RPMS Path :" + options.publishRPMSPath)
logger.info("Publish X RPMS Path :" + options.publishXRPMSPath)
if options.installPackage:
logger.info("Package to build:"+package)
listBuildOptionPackages = get_packages_with_build_options(options.pkgBuildOptionFile)
try:
constants.initialize(options)
if package == "packages_list":
buildPackagesList(options.buildRootPath+"/../packages_list.csv")
elif package == "sources_list":
if not os.path.isdir("../../stage/yaml_sources"):
cmdUtils.runCommandInShell("mkdir -p ../../stage/yaml_sources")
buildSourcesList('../../stage/yaml_sources',logger)
elif package == "srpms_list":
if not os.path.isdir("../../stage/yaml_srpms"):
cmdUtils.runCommandInShell("mkdir -p ../../stage/yaml_srpms")
buildSRPMList(options.sourceRpmPath,"../../stage/yaml_srpms",logger)
elif options.toolChainStage == "stage1":
pkgManager = PackageManager()
pkgManager.buildToolChain()
elif options.toolChainStage == "stage2":
pkgManager = PackageManager()
pkgManager.buildToolChainPackages(options.buildThreads)
elif options.installPackage:
buildAPackage(package, listBuildOptionPackages, options.pkgBuildOptionFile, options.buildThreads)
else:
buildPackagesForAllSpecs(listBuildOptionPackages, options.pkgBuildOptionFile, logger, options.buildThreads, pkgInfoJsonFile)
except Exception as e:
logger.error("Caught an exception")
logger.error(str(e))
# print stacktrace
traceback.print_exc()
sys.exit(1)
sys.exit(0)
def main():
usage = "Usage: %prog [options] <package name>"
parser = ArgumentParser(usage)
parser.add_argument("-s",
"--spec-path",
dest="specPath",
default="../../SPECS")
parser.add_argument("-x",
"--source-path",
dest="sourcePath",
default="../../stage/SOURCES")
parser.add_argument("-r",
"--rpm-path",
dest="rpmPath",
default="../../stage/RPMS")
parser.add_argument("-i",
"--install-package",
dest="installPackage",
default=False,
action="store_true")
parser.add_argument("-p",
"--publish-RPMS-path",
dest="publishRPMSPath",
default="../../stage/PUBLISHRPMS")
parser.add_argument("-e",
"--publish-XRPMS-path",
dest="publishXRPMSPath",
default="../../stage/PUBLISHXRPMS")
parser.add_argument("-l",
"--log-path",
dest="logPath",
default="../../stage/LOGS")
parser.add_argument("-y", "--log-level", dest="logLevel", default="error")
parser.add_argument("-z",
"--top-dir-path",
dest="topDirPath",
default="/usr/src/photon")
parser.add_argument("-b",
"--build-root-path",
dest="buildRootPath",
default="/mnt")
parser.add_argument("-t",
"--threads",
dest="buildThreads",
default=1,
type=int,
help="Number of working threads")
parser.add_argument("-m",
"--tool-chain-stage",
dest="toolChainStage",
default="None")
parser.add_argument("-c",
"--pullsources-config",
dest="pullsourcesConfig",
default="pullsources.conf")
parser.add_argument("-d", "--dist-tag", dest="dist", default="")
parser.add_argument("-k",
"--input-RPMS-path",
dest="inputRPMSPath",
default=None)
parser.add_argument("-n",
"--build-number",
dest="buildNumber",
default="0000000")
parser.add_argument("-v",
"--release-version",
dest="releaseVersion",
default="NNNnNNN")
parser.add_argument("-u",
"--enable-rpmcheck",
dest="rpmCheck",
default=False,
action="store_true")
parser.add_argument("-a",
"--source-rpm-path",
dest="sourceRpmPath",
default="../../stage/SRPMS")
parser.add_argument("-w",
"--pkginfo-file",
dest="pkgInfoFile",
default="../../stage/pkg_info.json")
parser.add_argument("-g",
"--pkg-build-option-file",
dest="pkgBuildOptionFile",
default="../../common/data/pkg_build_options.json")
parser.add_argument("-q",
"--rpmcheck-stop-on-error",
dest="rpmCheckStopOnError",
default=False,
action="store_true")
parser.add_argument("-bd",
"--publish-build-dependencies",
dest="publishBuildDependencies",
default=False)
parser.add_argument("-pw",
"--package-weights-path",
dest="packageWeightsPath",
default=None)
parser.add_argument("-bt",
"--build-type",
dest="pkgBuildType",
default="chroot")
parser.add_argument("-F", "--kat-build", dest="katBuild", default=None)
parser.add_argument("-pj",
"--packages-json-input",
dest="pkgJsonInput",
default=None)
parser.add_argument("PackageName", nargs='?')
options = parser.parse_args()
cmdUtils = CommandUtils()
if not os.path.isdir(options.logPath):
cmdUtils.runCommandInShell("mkdir -p " + options.logPath)
logger = Logger.getLogger("Main", options.logPath, options.logLevel)
errorFlag = False
package = None
pkgInfoJsonFile = options.pkgInfoFile
if not os.path.isdir(options.sourcePath):
logger.error("Given Sources Path is not a directory:" +
options.sourcePath)
errorFlag = True
if not os.path.isdir(options.specPath):
logger.error("Given Specs Path is not a directory:" + options.specPath)
errorFlag = True
if not os.path.isdir(options.publishRPMSPath):
logger.error("Given RPMS Path is not a directory:" +
options.publishRPMSPath)
errorFlag = True
if not os.path.isdir(options.publishXRPMSPath):
logger.error("Given X RPMS Path is not a directory:" +
options.publishXRPMSPath)
errorFlag = True
if not os.path.isdir(options.publishRPMSPath + "/" + platform.machine()):
logger.error("Given RPMS Path is missing " + platform.machine() +
" sub-directory:" + options.publishRPMSPath)
errorFlag = True
if not os.path.isdir(options.publishXRPMSPath + "/" + platform.machine()):
logger.error("Given X RPMS Path is missing " + platform.machine() +
" sub-directory:" + options.publishXRPMSPath)
errorFlag = True
if not os.path.isdir(options.publishRPMSPath + "/noarch"):
logger.error("Given RPMS Path is missing noarch sub-directory:" +
options.publishRPMSPath)
errorFlag = True
if not os.path.isdir(options.publishXRPMSPath + "/noarch"):
logger.error("Given X RPMS Path is missing noarch sub-directory:" +
options.publishXRPMSPath)
errorFlag = True
if not os.path.isfile(options.pkgBuildOptionFile):
logger.warning("Given JSON File is not a file:" +
options.pkgBuildOptionFile)
if options.inputRPMSPath is not None and not os.path.isdir(
options.inputRPMSPath):
logger.error("Given input RPMS Path is not a directory:" +
options.inputRPMSPath)
errorFlag = True
if options.packageWeightsPath is not None and not os.path.isfile(
options.packageWeightsPath):
logger.error("Given input Weights file is not a file:" +
options.packageWeightsPath)
errorFlag = True
if options.pkgJsonInput is not None and not os.path.isfile(
options.pkgJsonInput):
logger.error("Given input packages file is not a file:" +
options.pkgJsonInput)
errorFlag = True
if options.installPackage:
if not options.PackageName:
logger.error("Please provide package name")
errorFlag = True
else:
package = options.PackageName
if errorFlag:
logger.error(
"Found some errors. Please fix input options and re-run it.")
return False
if not os.path.isdir(options.rpmPath):
cmdUtils.runCommandInShell("mkdir -p " + options.rpmPath + "/" +
platform.machine())
cmdUtils.runCommandInShell("mkdir -p " + options.rpmPath + "/noarch")
if not os.path.isdir(options.sourceRpmPath):
cmdUtils.runCommandInShell("mkdir -p " + options.sourceRpmPath)
if not os.path.isdir(options.buildRootPath):
cmdUtils.runCommandInShell("mkdir -p " + options.buildRootPath)
logger.debug("Source Path :" + options.sourcePath)
logger.debug("Spec Path :" + options.specPath)
logger.debug("Rpm Path :" + options.rpmPath)
logger.debug("Log Path :" + options.logPath)
logger.debug("Log Level :" + options.logLevel)
logger.debug("Top Dir Path :" + options.topDirPath)
logger.debug("Publish RPMS Path :" + options.publishRPMSPath)
logger.debug("Publish X RPMS Path :" + options.publishXRPMSPath)
if options.installPackage:
logger.debug("Package to build:" + package)
get_packages_with_build_options(options.pkgBuildOptionFile)
try:
constants.setSpecPath(options.specPath)
constants.setSourcePath(options.sourcePath)
constants.setRpmPath(options.rpmPath)
constants.setSourceRpmPath(options.sourceRpmPath)
constants.setTopDirPath(options.topDirPath)
constants.setLogPath(options.logPath)
constants.setLogLevel(options.logLevel)
constants.setDist(options.dist)
constants.setBuildNumber(options.buildNumber)
constants.setReleaseVersion(options.releaseVersion)
constants.setPrevPublishRPMRepo(options.publishRPMSPath)
constants.setPrevPublishXRPMRepo(options.publishXRPMSPath)
constants.setBuildRootPath(options.buildRootPath)
constants.setPullSourcesURL(get_baseurl(options.pullsourcesConfig))
constants.setInputRPMSPath(options.inputRPMSPath)
constants.setRPMCheck(options.rpmCheck)
constants.setRpmCheckStopOnError(options.rpmCheckStopOnError)
constants.setPublishBuildDependencies(options.publishBuildDependencies)
constants.setPackageWeightsPath(options.packageWeightsPath)
constants.setKatBuild(options.katBuild)
constants.initialize()
# parse SPECS folder
SPECS()
if options.toolChainStage == "stage1":
pkgManager = PackageManager()
pkgManager.buildToolChain()
elif options.toolChainStage == "stage2":
pkgManager = PackageManager()
pkgManager.buildToolChainPackages(options.buildThreads)
elif options.installPackage:
buildSpecifiedPackages([package], options.buildThreads,
options.pkgBuildType)
elif options.pkgJsonInput:
buildPackagesInJson(options.pkgJsonInput, options.buildThreads,
options.pkgBuildType, pkgInfoJsonFile, logger)
else:
buildPackagesForAllSpecs(options.buildThreads,
options.pkgBuildType, pkgInfoJsonFile,
logger)
except Exception as e:
logger.error("Caught an exception")
logger.error(str(e))
# print stacktrace
traceback.print_exc()
sys.exit(1)
sys.exit(0)
def _buildRPMinContainer(self, specFile, rpmLogFile, destLogFile, containerID, package, macros):
rpmBuildCmd = self.rpmbuildBinary + " " + self.rpmbuildBuildallOption
if constants.rpmCheck and package in constants.testForceRPMS:
self.logger.info("#" * (68 + 2 * len(package)))
if not SPECS.getData().isCheckAvailable(package, index):
self.logger.info("####### " + package +
" MakeCheck is not available. Skipping MakeCheck TEST for " +
package + " #######")
rpmBuildCmd = self.rpmbuildBinary + " --clean"
else:
self.logger.info("####### " + package +
" MakeCheck is available. Running MakeCheck TEST for " +
package + " #######")
rpmBuildCmd = self.rpmbuildBinary + " " + self.rpmbuildCheckOption
self.logger.info("#" * (68 + 2 * len(package)))
else:
rpmBuildCmd += " "+self.rpmbuildNocheckOption
for macro in macros:
rpmBuildCmd += ' --define \"%s\"' % macro
rpmBuildCmd += " " + specFile
rpmBuildCmd = "/bin/bash -l -c '" + rpmBuildCmd + " > " + rpmLogFile + " 2>&1'"
rpmBuildCmd = "docker exec -t " + str(containerID.short_id) + " " + rpmBuildCmd
cmdUtils = CommandUtils()
self.logger.info("Building rpm for package: " + package)
#TODO: Show running log of rpmbuildcmd
#TODO: Get exit status of rpmBuildCmd
#containerID.exec_run(rpmBuildCmd)
returnVal = cmdUtils.runCommandInShell(rpmBuildCmd)
if not os.path.isfile(destLogFile):
self.logger.error("RPM build not file not found. Building rpm failed for: " + specFile)
raise Exception("RPM Build failed")
if constants.rpmCheck and package in constants.testForceRPMS:
if not SPECS.getData().isCheckAvailable(package, index):
constants.testLogger.info(package + " : N/A")
elif returnVal:
constants.testLogger.info(package + " : PASS")
else:
constants.testLogger.error(package + " : FAIL")
if constants.rpmCheck:
if not returnVal and constants.rpmCheckStopOnError:
self.logger.error("Checking rpm is failed " + specFile)
raise Exception("RPM check failed")
else:
if not returnVal:
self.logger.error("Building rpm is failed " + specFile)
raise Exception("RPM build failed")
#Extracting rpms created from log file
listRPMFiles = []
listSRPMFiles = []
with open(destLogFile, 'r') as logfile:
rpmBuildLogLines = logfile.readlines()
for i in range(0, len(rpmBuildLogLines)):
if re.search("^Wrote:", rpmBuildLogLines[i]):
listcontents = rpmBuildLogLines[i].split()
if ((len(listcontents) == 2) and
listcontents[1].strip().endswith(".rpm") and
"/RPMS/" in listcontents[1]):
listRPMFiles.append(listcontents[1])
if ((len(listcontents) == 2) and
listcontents[1].strip().endswith(".src.rpm") and
"/SRPMS/" in listcontents[1]):
listSRPMFiles.append(listcontents[1])
#if not listRPMFiles:
# self.logger.error("Building rpm failed for " + specFile)
# raise Exception("RPM Build failed")
return listRPMFiles, listSRPMFiles
def _buildRPM(self, specFile, logFile, chrootCmd, package, macros):
rpmBuildcmd = self.rpmbuildBinary + " " + self.rpmbuildBuildallOption
if constants.rpmCheck and package in constants.testForceRPMS:
self.logger.info("#" * (68 + 2 * len(package)))
if not SPECS.getData().isCheckAvailable(package):
self.logger.info("####### " + package +
" MakeCheck is not available. Skipping MakeCheck TEST for " +
package + " #######")
rpmBuildcmd = self.rpmbuildBinary + " --clean"
else:
self.logger.info("####### " + package +
" MakeCheck is available. Running MakeCheck TEST for " +
package + " #######")
rpmBuildcmd = self.rpmbuildBinary + " " + self.rpmbuildCheckOption
self.logger.info("#" * (68 + 2 * len(package)))
else:
rpmBuildcmd += " " + self.rpmbuildNocheckOption
for macro in macros:
rpmBuildcmd += ' --define \\\"%s\\\"' % macro
rpmBuildcmd += " " + specFile
cmdUtils = CommandUtils()
self.logger.info("Building rpm....")
self.logger.info(rpmBuildcmd)
returnVal = cmdUtils.runCommandInShell(rpmBuildcmd, logFile, chrootCmd)
if constants.rpmCheck and package in constants.testForceRPMS:
if not SPECS.getData().isCheckAvailable(package):
constants.testLogger.info(package + " : N/A")
elif returnVal:
constants.testLogger.info(package + " : PASS")
else:
constants.testLogger.error(package + " : FAIL")
if constants.rpmCheck:
if not returnVal and constants.rpmCheckStopOnError:
self.logger.error("Checking rpm is failed " + specFile)
raise Exception("RPM check failed")
else:
if not returnVal:
self.logger.error("Building rpm is failed " + specFile)
raise Exception("RPM build failed")
#Extracting rpms created from log file
listRPMFiles = []
listSRPMFiles = []
with open(logFile, 'r') as logfile:
fileContents = logfile.readlines()
for i in range(0, len(fileContents)):
if re.search("^Wrote:", fileContents[i]):
listcontents = fileContents[i].split()
if ((len(listcontents) == 2) and
listcontents[1].strip().endswith(".rpm") and
"/RPMS/" in listcontents[1]):
listRPMFiles.append(listcontents[1])
if ((len(listcontents) == 2) and
listcontents[1].strip().endswith(".src.rpm") and
"/SRPMS/" in listcontents[1]):
listSRPMFiles.append(listcontents[1])
return listRPMFiles, listSRPMFiles
def main():
usage = "Usage: %prog [options] <package name>"
parser = OptionParser(usage)
parser.add_option("-s",
"--spec-path",
dest="specPath",
default="../../SPECS")
parser.add_option("-x",
"--source-path",
dest="sourcePath",
default="../../stage/SOURCES")
parser.add_option("-r",
"--rpm-path",
dest="rpmPath",
default="../../stage/RPMS")
parser.add_option("-i",
"--install-package",
dest="installPackage",
default=False,
action="store_true")
parser.add_option("-p",
"--publish-RPMS-path",
dest="publishRPMSPath",
default="../../stage/PUBLISHRPMS")
parser.add_option("-l",
"--log-path",
dest="logPath",
default="../../stage/LOGS")
parser.add_option("-o",
"--build-option",
dest="buildOption",
default="full")
parser.add_option("-z",
"--top-dir-path",
dest="topDirPath",
default="/usr/src/photon")
parser.add_option(
"-j",
"--json-file",
dest="inputJSONFile",
default="../../common/data/build_install_options_all.json")
parser.add_option("-b",
"--build-root-path",
dest="buildRootPath",
default="/mnt")
parser.add_option("-t",
"--threads",
dest="buildThreads",
default=1,
type="int",
help="Numbeer of working threads")
parser.add_option("-m",
"--tool-chain-stage",
dest="toolChainStage",
default="None")
parser.add_option("-c",
"--pullsources-config",
dest="pullsourcesConfig",
default="pullsources.conf")
parser.add_option("-d", "--dist", dest="dist", default="")
parser.add_option("-k",
"--input-RPMS-path",
dest="inputRPMSPath",
default=None)
parser.add_option("-n",
"--build-number",
dest="buildNumber",
default="0000000")
parser.add_option("-v",
"--release-version",
dest="releaseVersion",
default="NNNnNNN")
parser.add_option("-u",
"--enable-rpmcheck",
dest="rpmCheck",
default=False,
action="store_true")
parser.add_option("-a",
"--source-rpm-path",
dest="sourceRpmPath",
default="../../stage/SRPMS")
(options, args) = parser.parse_args()
cmdUtils = CommandUtils()
if not os.path.isdir(options.logPath):
cmdUtils.runCommandInShell("mkdir -p " + options.logPath)
logger = Logger.getLogger(options.logPath + "/Main")
errorFlag = False
package = None
if not os.path.isdir(options.sourcePath):
logger.error("Given Sources Path is not a directory:" +
options.sourcePath)
errorFlag = True
if not os.path.isdir(options.specPath):
logger.error("Given Specs Path is not a directory:" + options.specPath)
errorFlag = True
if not os.path.isdir(options.publishRPMSPath):
logger.error("Given RPMS Path is not a directory:" +
options.publishRPMSPath)
errorFlag = True
if not os.path.isdir(options.publishRPMSPath + "/x86_64"):
logger.error("Given RPMS Path is missing x86_64 sub-directory:" +
options.publishRPMSPath)
errorFlag = True
if not os.path.isdir(options.publishRPMSPath + "/noarch"):
logger.error("Given RPMS Path is missing noarch sub-directory:" +
options.publishRPMSPath)
errorFlag = True
if not os.path.isfile(
options.inputJSONFile) and not options.installPackage:
logger.error("Given JSON File is not a file:" + options.inputJSONFile)
errorFlag = True
if options.inputRPMSPath is not None and not os.path.isdir(
options.inputRPMSPath):
logger.error("Given input RPMS Path is not a directory:" +
options.publishRPMSPath)
errorFlag = True
if options.installPackage:
if len(args) != 1:
logger.error("Please provide package name")
errorFlag = True
else:
package = args[0]
if errorFlag:
logger.error(
"Found some errors. Please fix input options and re-run it.")
return False
if not os.path.isdir(options.rpmPath):
cmdUtils.runCommandInShell("mkdir -p " + options.rpmPath + "/x86_64")
cmdUtils.runCommandInShell("mkdir -p " + options.rpmPath + "/noarch")
if not os.path.isdir(options.sourceRpmPath):
cmdUtils.runCommandInShell("mkdir -p " + options.sourceRpmPath)
if not os.path.isdir(options.buildRootPath):
cmdUtils.runCommandInShell("mkdir -p " + options.buildRootPath)
logger.info("Source Path :" + options.sourcePath)
logger.info("Spec Path :" + options.specPath)
logger.info("Rpm Path :" + options.rpmPath)
logger.info("Log Path :" + options.logPath)
logger.info("Top Dir Path :" + options.topDirPath)
logger.info("Publish RPMS Path :" + options.publishRPMSPath)
if not options.installPackage:
logger.info("JSON File :" + options.inputJSONFile)
else:
logger.info("Package to build:" + package)
'''
listPackages=["acl","attr","autoconf","automake","bash","bc","bindutils","binutils","bison","boost","btrfs-progs","bzip2","ca-certificates","cdrkit","check",
"cloud-init","cmake","coreutils","cpio","cracklib","createrepo","curl","cyrus-sasl","db","dbus","deltarpm","diffutils","docbook-xml","docbook-xsl",
"docker","dracut","e2fsprogs","elfutils","etcd","expat","file","filesystem","findutils","flex","gawk","gcc","gdb","gdbm","gettext","git",
"glib","glibc","glibmm","gmp","go","gobject-introspection","google-daemon","google-startup-scripts","gperf","gpgme","gptfdisk","grep","groff",
"grub","gtk-doc","gzip","haveged","hawkey","iana-etc","inetutils","intltool","iproute2","iptables","itstool","json-glib","kbd","kmod","krb5",
"kubernetes","less","libaio","libassuan","libcap","libdnet","libffi","libgpg-error","libgsystem","libhif","libmspack","libpcap","libpipeline",
"librepo","libselinux","libsepol","libsigc++","libsolv","libtool","libxml2","libxslt","libyaml","linux","linux-api-headers","Linux-PAM","lua",
"lvm2","lzo","m4","make","man-db","man-pages","mercurial","mpc","mpfr","nano","ncurses","nspr","nss","ntp","openldap","openssh","openssl",
"open-vm-tools","ostree","parted","patch","pcre","perl","perl-common-sense","perl-Config-IniFiles","perl-DBD-SQLite","perl-DBI","perl-DBIx-Simple",
"perl-Exporter-Tiny","perl-JSON-XS","perl-libintl","perl-List-MoreUtils","perl-Module-Install","perl-Module-ScanDeps","perl-Types-Serialiser",
"perl-WWW-Curl","perl-YAML","perl-YAML-Tiny","photon-release","pkg-config","popt","procps-ng","psmisc","pycurl","pygobject","python2",
"python-configobj","python-iniparse","python-jsonpatch","python-jsonpointer","python-prettytable","python-requests","python-setuptools",
"python-six","PyYAML","readline","rocket","rpm","rpm-ostree","rpm-ostree-toolbox","ruby","sed","shadow","sqlite-autoconf","strace","sudo",
"swig","systemd","tar","tcpdump","tcsh","tdnf","texinfo","thin-provisioning-tools","tzdata","unzip","urlgrabber","util-linux","vim","wget",
"which","xerces-c","XML-Parser","xml-security-c","xz","yum","yum-metadata-parser","zlib"]
'''
try:
constants.initialize(options)
if package == "packages_list":
buildPackagesList(options.specPath,
options.buildRootPath + "/../packages_list.csv")
elif package == "sources_list":
buildSourcesList(options.specPath, options.buildRootPath + "/../")
elif options.toolChainStage == "stage1":
pkgManager = PackageManager()
pkgManager.buildToolChain()
elif options.toolChainStage == "stage2":
pkgManager = PackageManager()
pkgManager.buildToolChainPackages(options.buildThreads)
elif options.installPackage:
buildAPackage(package, options.buildThreads)
else:
buildPackagesFromGivenJSONFile(options.inputJSONFile,
options.buildOption, logger,
options.buildThreads)
except Exception as e:
logger.error("Caught an exception")
logger.error(str(e))
# print stacktrace
traceback.print_exc()
sys.exit(1)
sys.exit(0)
def buildRPMSForGivenPackageInContainer(self, package, containerID, destLogPath=None, index=0):
self.logger.info("Building rpm's for package " + package + " in container " +
containerID.short_id)
listSourcesFiles = SPECS.getData().getSources(package, index)
listPatchFiles = SPECS.getData().getPatches(package, index)
specFile = SPECS.getData().getSpecFile(package, index)
specName = SPECS.getData().getSpecName(package) + ".spec"
sourcePath = constants.topDirPath + "/SOURCES/"
specPath = constants.topDirPath + "/SPECS/"
rpmLogFile = constants.topDirPath + "/LOGS/" + package + ".log"
destLogFile = destLogPath + "/" + package + ".log"
cmdUtils = CommandUtils()
#TODO: mount it in, don't copy
cpSpecCmd = "docker cp " + specFile + " " + containerID.short_id \
+ ":" + specPath + specName
returnVal = cmdUtils.runCommandInShell(cpSpecCmd)
if not returnVal:
self.logger.error("Error copying source SPEC file to container")
raise Exception("Failed copying source SPEC to container")
#FIXME: some sources are located in SPECS/.. how to mount?
# if os.geteuid()==0:
#TODO: mount it in, don't copy
macros = []
self._copySourcesToContainer(listSourcesFiles, package, containerID, sourcePath, index)
#TODO: mount it in, don't copy
self._copySourcesToContainer(listPatchFiles, package, containerID, sourcePath, index)
listAdditionalFiles, macros = self._getAdditionalBuildFiles(package)
self._copyAdditionalBuildFilesToContainer(listAdditionalFiles, containerID)
# Add rpm macros
listRPMMacros = constants.userDefinedMacros
for macroName, value in listRPMMacros.items():
macros.append(macroName + " " + value)
# Build RPMs
listRPMFiles = []
listSRPMFiles = []
try:
listRPMFiles, listSRPMFiles = self._buildRPMinContainer(
specPath + specName,
rpmLogFile,
destLogFile,
containerID,
package,
macros)
self.logger.info("Successfully built rpm:" + package)
except Exception as e:
self.logger.error("Failed while building rpm: " + package)
raise e
finally:
if destLogPath is not None:
rpmLog = destLogPath + "/" + package + ".log"
if (constants.rpmCheck and
package in constants.testForceRPMS and
SPECS.getData().isCheckAvailable(package, index)):
cmd = "sed -i '/^Executing(%check):/,/^Processing files:/{//!b};d' " + rpmLog
logFile = destLogPath + "/adjustTestFile.log"
returnVal = CommandUtils().runCommandInShell(cmd, logFile)
testLogFile = destLogPath + "/" + package + "-test.log"
shutil.copyfile(rpmLog, testLogFile)
self.logger.info("RPM build is successful")
# Verify RPM and SRPM files exist as success criteria
for rpmFile in listRPMFiles:
rpmName = os.path.basename(rpmFile)
rpmDestDir = self._getRPMDestDir(rpmName, constants.rpmPath)
rpmDestFile = rpmDestDir + "/" + rpmName
if not os.path.isfile(rpmDestFile):
self.logger.error("Could not find RPM file: " + rpmDestFile)
raise Exception("Built RPM file not found.")
for srpmFile in listSRPMFiles:
srpmName = os.path.basename(srpmFile)
srpmDestDir = self._getRPMDestDir(os.path.basename(srpmFile), constants.sourceRpmPath)
srpmDestFile = srpmDestDir + "/" + srpmName
if not os.path.isfile(srpmDestFile):
self.logger.error("Could not find RPM file: " + srpmDestFile)
raise Exception("Built SRPM file not found.")
