def installToolChainRPMS(self, chrootID, packageName, logPath=None):
if logPath is None:
logPath = self.logPath
cmdUtils = CommandUtils()
self.prepareBuildRoot(chrootID)
self.logger.info("Installing Tool Chain RPMS.......")
rpmFiles = ""
packages = ""
for package in constants.listToolChainRPMsToInstall:
pkgUtils = PackageUtils(self.logName, self.logPath)
rpmFile = None
if constants.rpmCheck:
rpmFile = pkgUtils.findRPMFileForGivenPackage(package)
else:
if (packageName not in constants.listToolChainRPMsToInstall or
constants.listToolChainRPMsToInstall.index(packageName)
> constants.listToolChainRPMsToInstall.index(package)):
rpmFile = pkgUtils.findRPMFileForGivenPackage(package)
if rpmFile is None:
# sqlite-autoconf package was renamed, but it still published as sqlite-autoconf
if (package == "sqlite") and (platform.machine() == "x86_64"):
package = "sqlite-autoconf"
rpmFile = self.findRPMFileInGivenLocation(
package, constants.prevPublishRPMRepo)
if rpmFile is None:
if package in constants.listOfRPMsProvidedAfterBuild:
self.logger.info(
"No old version of " + package +
" exists, skip until the new version is built")
continue
self.logger.error("Unable to find rpm " + package +
" in current and previous versions")
raise Exception("Input Error")
rpmFiles += " " + rpmFile
packages += " " + package
self.logger.debug("Installing toolchain rpms:" + packages)
cmd = (self.rpmCommand + " -i -v --nodeps --noorder --force --root " +
chrootID + " --define \'_dbpath /var/lib/rpm\' " + rpmFiles)
retVal = cmdUtils.runCommandInShell(
cmd, logPath + "/install_toolchain_rpms.log")
if not retVal:
self.logger.debug("Command Executed:" + cmd)
self.logger.error("Installing tool chain failed")
raise Exception("RPM installation failed")
self.logger.info(
"Successfully installed default Tool Chain RPMS in Chroot:" +
chrootID)
print("Building Package: ".format(packageName))
print(constants.perPackageToolChain)
if packageName in constants.perPackageToolChain:
print(constants.perPackageToolChain[packageName])
self.installCustomToolChainRPMS(
chrootID, constants.perPackageToolChain[packageName],
packageName)
#convert rpm db to lmdb (if required)
convertLogFile = self.logPath + "/rpmdbconvert.log"
pkgUtilsConvert = PackageUtils(self.logName, self.logPath)
pkgUtilsConvert.ensureLMDBForRPMDB(chrootID, convertLogFile)
def installToolChainRPMS(self, chrootID, packageName, logPath=None):
if logPath is None:
logPath = self.logPath
cmdUtils = CommandUtils()
self.prepareBuildRoot(chrootID)
self.logger.info("Installing Tool Chain RPMS.......")
rpmFiles = ""
packages = ""
for package in constants.listToolChainRPMsToInstall:
pkgUtils = PackageUtils(self.logName, self.logPath)
rpmFile = None
if constants.rpmCheck:
rpmFile = pkgUtils.findRPMFileForGivenPackage(package)
else:
if (packageName not in constants.listToolChainRPMsToInstall or
constants.listToolChainRPMsToInstall.index(packageName) >
constants.listToolChainRPMsToInstall.index(package)):
rpmFile = pkgUtils.findRPMFileForGivenPackage(package)
if rpmFile is None:
# sqlite-autoconf package was renamed, but it still published as sqlite-autoconf
if (package == "sqlite") and (platform.machine() == "x86_64"):
package = "sqlite-autoconf"
rpmFile = self.findRPMFileInGivenLocation(package, constants.prevPublishRPMRepo)
if rpmFile is None:
if package in constants.listOfRPMsProvidedAfterBuild:
self.logger.info("No old version of " + package +
" exists, skip until the new version is built")
continue
self.logger.error("Unable to find rpm " + package +
" in current and previous versions")
raise Exception("Input Error")
rpmFiles += " " + rpmFile
packages += " " + package
self.logger.debug("Installing toolchain rpms:" + packages)
cmd = (self.rpmCommand + " -i -v --nodeps --noorder --force --root " +
chrootID +" --define \'_dbpath /var/lib/rpm\' "+ rpmFiles)
retVal = cmdUtils.runCommandInShell(cmd, logPath + "/install_toolchain_rpms.log")
if not retVal:
self.logger.debug("Command Executed:" + cmd)
self.logger.error("Installing tool chain failed")
raise Exception("RPM installation failed")
self.logger.info("Successfully installed default Tool Chain RPMS in Chroot:" + chrootID)
print("Building Package: ".format(packageName))
print(constants.perPackageToolChain)
if packageName in constants.perPackageToolChain:
print(constants.perPackageToolChain[packageName])
self.installCustomToolChainRPMS(chrootID, constants.perPackageToolChain[packageName],
packageName)
#convert rpm db to lmdb (if required)
convertLogFile = self.logPath+"/rpmdbconvert.log"
pkgUtilsConvert=PackageUtils(self.logName,self.logPath)
pkgUtilsConvert.ensureLMDBForRPMDB(chrootID, convertLogFile)
def installToolChainRPMS(self, chrootID, packageName):
cmdUtils = CommandUtils()
self.prepareBuildRoot(chrootID)
self.logger.info("Installing Tool Chain RPMS.......")
rpmFiles = ""
packages = ""
for package in constants.listToolChainRPMsToInstall:
pkgUtils = PackageUtils(self.logName, self.logPath)
rpmFile = None
if constants.rpmCheck:
rpmFile = pkgUtils.findRPMFileForGivenPackage(package)
else:
if (packageName not in constants.listToolChainRPMsToInstall or
constants.listToolChainRPMsToInstall.index(packageName)
> constants.listToolChainRPMsToInstall.index(package)):
rpmFile = pkgUtils.findRPMFileForGivenPackage(package)
if rpmFile is None:
# sqlite-autoconf package was renamed, but it still published as sqlite-autoconf
if package == "sqlite":
package = "sqlite-autoconf"
rpmFile = self.findRPMFileInGivenLocation(
package, constants.prevPublishRPMRepo)
if rpmFile is None:
if package in constants.listOfRPMsProvidedAfterBuild:
self.logger.info(
"No old version of " + package +
" exists, skip until the new version is built")
continue
self.logger.error("Unable to find rpm " + package +
" in current and previous versions")
raise Exception("Input Error")
rpmFiles += " " + rpmFile
packages += " " + package
self.logger.debug("Installing rpms:" + packages)
cmd = self.rpmCommand + " -i --nodeps --force --root " + chrootID + " --define \'_dbpath /var/lib/rpm\' " + rpmFiles
process = subprocess.Popen("%s" % cmd,
shell=True,
stdout=subprocess.PIPE)
retval = process.wait()
if retval != 0:
self.logger.error("Installing tool chain failed")
raise Exception("RPM installation failed")
self.logger.info(
"Successfully installed default Tool Chain RPMS in Chroot:" +
chrootID)
print "Building Package:" + packageName
print constants.perPackageToolChain
if packageName in constants.perPackageToolChain:
print constants.perPackageToolChain[packageName]
self.installCustomToolChainRPMS(
chrootID, constants.perPackageToolChain[packageName],
packageName)
def prepareBuildRoot(self,chrootID):
self.logger.info("Preparing build environment")
cmdUtils = CommandUtils()
cmdUtils.runCommandInShell("mkdir -p "+chrootID+"/tmp")
cmdUtils.runCommandInShell("mkdir -p "+chrootID+constants.topDirPath)
cmdUtils.runCommandInShell("mkdir -p "+chrootID+constants.topDirPath+"/RPMS/x86_64")
cmdUtils.runCommandInShell("mkdir -p "+chrootID+constants.topDirPath+"/RPMS/noarch")
cmdUtils.runCommandInShell("mkdir -p "+chrootID+constants.topDirPath+"/SOURCES")
cmdUtils.runCommandInShell("mkdir -p "+chrootID+constants.topDirPath+"/SPECS")
cmdUtils.runCommandInShell("mkdir -p "+chrootID+constants.topDirPath+"/LOGS")
cmdUtils.runCommandInShell("mkdir -p "+chrootID+constants.topDirPath+"/BUILD")
cmdUtils.runCommandInShell("mkdir -p "+chrootID+constants.topDirPath+"/BUILDROOT")
package="filesystem"
pkgUtils=PackageUtils(self.logName,self.logPath)
rpmFile=pkgUtils.findRPMFileForGivenPackage(package)
if rpmFile is None:
specFile=constants.specData.getSpecFile(package)
cmd=self.rpmbuildCommand+" -ba --nocheck --define \'_topdir "+chrootID+constants.topDirPath+"\' --define \'_dbpath "+chrootID+"/var/lib/rpm\' --define \'dist "+constants.dist+"\' --define \'photon_build_number "+constants.buildNumber+"\' --define \'photon_release_version "+constants.releaseVersion+"\' "+specFile
self.logger.info(cmd)
cmdUtils.runCommandInShell(cmd,self.logPath+"/filesystem.log")
filesystemrpmFile = cmdUtils.findFile(package+"-[0-9]*.rpm", chrootID+constants.topDirPath+"/RPMS")
filesystemsrpmFile = cmdUtils.findFile(package+"-[0-9]*.src.rpm", chrootID+constants.topDirPath+"/SRPMS")
if len(filesystemrpmFile) > 0:
shutil.copy2(filesystemrpmFile[0],constants.rpmPath+"/x86_64/")
if len(filesystemsrpmFile) > 0:
shutil.copy2(filesystemsrpmFile[0],constants.sourceRpmPath+"/")
rpmFile=pkgUtils.findRPMFileForGivenPackage(package)
if rpmFile is None:
self.logger.error("Cannot find filesystem rpm")
raise Exception("Cannot find filesystem rpm")
self.logger.debug("Installing filesystem rpms:" + package)
if os.geteuid()==0:
cmd=self.rpmCommand + " -i --nodeps --root "+chrootID+" --define '_dbpath /var/lib/rpm' "+ rpmFile
else:
cmd=self.rpmCommand + " -i --nodeps --badreloc --relocate /="+chrootID+" --define '_dbpath "+chrootID+"/var/lib/rpm' "+ rpmFile
process = subprocess.Popen("%s" %cmd,shell=True,stdout=subprocess.PIPE)
retval = process.wait()
if retval != 0:
self.logger.error("Installing filesystem rpm failed")
raise Exception("RPM installation failed")
prepareChrootCmd=self.prepareBuildRootCmd+" "+chrootID
logFile=constants.logPath+"/prepareBuildRoot.log"
returnVal=cmdUtils.runCommandInShell(prepareChrootCmd,logFile)
if not returnVal:
self.logger.error("Prepare build root script failed.Unable to prepare chroot.")
raise Exception("Prepare build root script failed")
self.logger.info("Successfully prepared chroot:"+chrootID)
def prepareBuildRoot(self,chrootID):
self.logger.info("Preparing build environment")
cmdUtils = CommandUtils()
cmdUtils.runCommandInShell("mkdir -p "+chrootID+"/tmp")
cmdUtils.runCommandInShell("mkdir -p "+chrootID+constants.topDirPath)
cmdUtils.runCommandInShell("mkdir -p "+chrootID+constants.topDirPath+"/RPMS/x86_64")
cmdUtils.runCommandInShell("mkdir -p "+chrootID+constants.topDirPath+"/RPMS/noarch")
cmdUtils.runCommandInShell("mkdir -p "+chrootID+constants.topDirPath+"/SOURCES")
cmdUtils.runCommandInShell("mkdir -p "+chrootID+constants.topDirPath+"/SPECS")
cmdUtils.runCommandInShell("mkdir -p "+chrootID+constants.topDirPath+"/LOGS")
cmdUtils.runCommandInShell("mkdir -p "+chrootID+constants.topDirPath+"/BUILD")
cmdUtils.runCommandInShell("mkdir -p "+chrootID+constants.topDirPath+"/BUILDROOT")
package="filesystem"
pkgUtils=PackageUtils(self.logName,self.logPath)
rpmFile=pkgUtils.findRPMFileForGivenPackage(package)
if rpmFile is None:
specFile=constants.specData.getSpecFile(package)
cmd=self.rpmbuildCommand+" -ba --nocheck --define \'_topdir "+chrootID+constants.topDirPath+"\' --define \'_dbpath "+chrootID+"/var/lib/rpm\' --define \'dist "+constants.dist+"\' "+specFile
self.logger.info(cmd)
cmdUtils.runCommandInShell(cmd,self.logPath+"/filesystem.log")
filesystemrpmFile = cmdUtils.findFile(package+"-[0-9]*.rpm", chrootID+constants.topDirPath+"/RPMS")
filesystemsrpmFile = cmdUtils.findFile(package+"-[0-9]*.src.rpm", chrootID+constants.topDirPath+"/SRPMS")
if len(filesystemrpmFile) > 0:
shutil.copy2(filesystemrpmFile[0],constants.rpmPath+"/x86_64/")
if len(filesystemsrpmFile) > 0:
shutil.copy2(filesystemsrpmFile[0],constants.sourceRpmPath+"/")
rpmFile=pkgUtils.findRPMFileForGivenPackage(package)
if rpmFile is None:
self.logger.error("Cannot find filesystem rpm")
raise Exception("Cannot find filesystem rpm")
self.logger.debug("Installing filesystem rpms:" + package)
if os.geteuid()==0:
cmd=self.rpmCommand + " -i --nodeps --root "+chrootID+" --define '_dbpath /var/lib/rpm' "+ rpmFile
else:
cmd=self.rpmCommand + " -i --nodeps --badreloc --relocate /="+chrootID+" --define '_dbpath "+chrootID+"/var/lib/rpm' "+ rpmFile
process = subprocess.Popen("%s" %cmd,shell=True,stdout=subprocess.PIPE)
retval = process.wait()
if retval != 0:
self.logger.error("Installing filesystem rpm failed")
raise Exception("RPM installation failed")
prepareChrootCmd=self.prepareBuildRootCmd+" "+chrootID
logFile=constants.logPath+"/prepareBuildRoot.log"
returnVal=cmdUtils.runCommandInShell(prepareChrootCmd,logFile)
if not returnVal:
self.logger.error("Prepare build root script failed.Unable to prepare chroot.")
raise Exception("Prepare build root script failed")
self.logger.info("Successfully prepared chroot:"+chrootID)
def installToolChainRPMS(self,chrootID):
cmdUtils = CommandUtils()
self.prepareBuildRoot(chrootID)
self.logger.info("Installing Tool Chain RPMS.......")
rpmFiles = ""
packages = ""
for package in constants.listToolChainRPMPkgsToBuild:
pkgUtils=PackageUtils(self.logName,self.logPath)
rpmFile=pkgUtils.findRPMFileForGivenPackage(package)
if rpmFile is None:
rpmFile=self.findRPMFileInGivenLocation(package, constants.prevPublishRPMRepo)
if rpmFile is None:
self.logger.error("Unable to find rpm "+ package +" in current and previous versions")
raise "Input Error"
rpmFiles += " " + rpmFile
packages += " " + package
self.logger.debug("Installing rpms:"+packages)
cmd=self.rpmCommand + " -i --nodeps --force --root "+chrootID+" --define \'_dbpath /var/lib/rpm\' "+ rpmFiles
process = subprocess.Popen("%s" %cmd,shell=True,stdout=subprocess.PIPE)
retval = process.wait()
if retval != 0:
self.logger.error("Installing tool chain failed")
raise "RPM installation failed"
self.logger.info("Successfully installed all Tool Chain RPMS in Chroot:"+chrootID)
def installToolChainRPMS(self,chrootID):
cmdUtils = CommandUtils()
self.prepareBuildRoot(chrootID)
self.logger.info("Installing Tool Chain RPMS.......")
rpmFiles = ""
packages = ""
for package in constants.listToolChainRPMPkgsToBuild:
pkgUtils=PackageUtils(self.logName,self.logPath)
rpmFile=pkgUtils.findRPMFileForGivenPackage(package)
if rpmFile is None:
rpmFile=self.findRPMFileInGivenLocation(package, constants.prevPublishRPMRepo)
if rpmFile is None:
if package == "util-linux-devel":
self.logger.info("No old version of util-linux-devel exists, skip until the new version is built")
continue
if package == "flex-devel":
self.logger.info("No old version of flex-devel exists, skip until the new version is built")
continue
self.logger.error("Unable to find rpm "+ package +" in current and previous versions")
raise Exception("Input Error")
rpmFiles += " " + rpmFile
packages += " " + package
self.logger.debug("Installing rpms:"+packages)
cmd=self.rpmCommand + " -i --nodeps --force --root "+chrootID+" --define \'_dbpath /var/lib/rpm\' "+ rpmFiles
process = subprocess.Popen("%s" %cmd,shell=True,stdout=subprocess.PIPE)
retval = process.wait()
if retval != 0:
self.logger.error("Installing tool chain failed")
raise Exception("RPM installation failed")
self.logger.info("Successfully installed all Tool Chain RPMS in Chroot:"+chrootID)
def buildCoreToolChainPackages(self):
self.logger.info("Building core tool chain packages.....")
chrootID=None
try:
pkgUtils=PackageUtils(self.logName,self.logPath)
for package in constants.listCoreToolChainRPMPackages:
rpmPkg=pkgUtils.findRPMFileForGivenPackage(package)
if rpmPkg is not None:
continue
chrUtils = ChrootUtils(self.logName,self.logPath)
chrootName="build-core-toolchain"
destLogPath=constants.logPath+"/build-"+package
if not os.path.isdir(destLogPath):
cmdUtils = CommandUtils()
cmdUtils.runCommandInShell("mkdir -p "+destLogPath)
returnVal,chrootID = chrUtils.createChroot(chrootName)
if not returnVal:
self.logger.error("Creating chroot failed")
raise Exception("creating chroot failed")
self.installToolChainRPMS(chrootID)
pkgUtils.adjustGCCSpecs(package, chrootID, destLogPath)
pkgUtils.buildRPMSForGivenPackage(package, chrootID,destLogPath)
chrUtils.destroyChroot(chrootID)
chrootID=None
self.logger.info("Successfully built toolchain")
if chrootID is not None:
chrUtils.destroyChroot(chrootID)
except Exception as e:
self.logger.error("Unable to build tool chain.")
# print stacktrace
traceback.print_exc()
raise e
def buildCoreToolChainPackages(self):
self.logger.info("Building core tool chain packages.....")
chrootID = None
try:
pkgUtils = PackageUtils(self.logName, self.logPath)
for package in constants.listCoreToolChainRPMPackages:
rpmPkg = pkgUtils.findRPMFileForGivenPackage(package)
if rpmPkg is not None:
continue
chrUtils = ChrootUtils(self.logName, self.logPath)
chrootName = "build-core-toolchain"
destLogPath = constants.logPath + "/build-" + package
if not os.path.isdir(destLogPath):
cmdUtils = CommandUtils()
cmdUtils.runCommandInShell("mkdir -p " + destLogPath)
returnVal, chrootID = chrUtils.createChroot(chrootName)
if not returnVal:
self.logger.error("Creating chroot failed")
raise Exception("creating chroot failed")
self.installToolChainRPMS(chrootID)
pkgUtils.buildRPMSForGivenPackage(package, chrootID,
destLogPath)
chrUtils.destroyChroot(chrootID)
chrootID = None
self.logger.info("Successfully built toolchain")
except Exception as e:
self.logger.error("Unable to build tool chain.")
raise e
finally:
if chrootID is not None:
chrUtils.destroyChroot(chrootID)
def loadPackagesData(self):
listPackages = constants.specData.getListPackages()
listPackages.sort()
listRPMFiles = []
cmdUtils = CommandUtils()
for package in listPackages:
release = constants.specData.getRelease(package)
version = constants.specData.getVersion(package)
listRPMPackages = constants.specData.getRPMPackages(package)
srpmFileName = package+"-"+version+"-"+release+".src.rpm"
srpmFiles = cmdUtils.findFile(srpmFileName, constants.sourceRpmPath)
srpmFile = None
if len(srpmFiles) == 1:
srpmFile = srpmFiles[0]
debugrpmFileName = package+"-debuginfo-"+version+"-"+release+"*"
debugrpmFiles = cmdUtils.findFile(debugrpmFileName, constants.rpmPath)
debugrpmFile = None
if len(debugrpmFiles) == 1:
debugrpmFile = debugrpmFiles[0]
pkgUtils = PackageUtils(self.logName,self.logPath)
for rpmPkg in listRPMPackages:
rpmFile = pkgUtils.findRPMFileForGivenPackage(rpmPkg)
if rpmFile is not None:
listRPMFiles.append(rpmFile)
listPkgAttributes = {"sourcerpm":srpmFile, "rpm":rpmFile, "debugrpm":debugrpmFile}
self.pkgList[rpmPkg] = listPkgAttributes
self.logger.debug("Added "+rpmPkg +" rpm package to the list")
else:
self.logger.error("Missing rpm file for package:"+rpmPkg)
def loadPackagesData(self):
listPackages = SPECS.getData().getListPackages()
listPackages.sort()
listRPMFiles = []
cmdUtils = CommandUtils()
for package in listPackages:
release = SPECS.getData().getRelease(package)
version = SPECS.getData().getVersion(package)
listRPMPackages = SPECS.getData().getRPMPackages(package)
srpmFileName = package + "-" + version + "-" + release + ".src.rpm"
srpmFiles = cmdUtils.findFile(srpmFileName, constants.sourceRpmPath)
srpmFile = None
if len(srpmFiles) == 1:
srpmFile = srpmFiles[0]
debugrpmFileName = package + "-debuginfo-" + version + "-" + release + "*"
debugrpmFiles = cmdUtils.findFile(debugrpmFileName, constants.rpmPath)
debugrpmFile = None
if len(debugrpmFiles) == 1:
debugrpmFile = debugrpmFiles[0]
pkgUtils = PackageUtils(self.logName, self.logPath)
for rpmPkg in listRPMPackages:
rpmFile = pkgUtils.findRPMFileForGivenPackage(rpmPkg)
if rpmFile is not None:
listRPMFiles.append(rpmFile)
listPkgAttributes = {"sourcerpm":srpmFile, "rpm":rpmFile,
"debugrpm":debugrpmFile}
self.pkgList[rpmPkg] = listPkgAttributes
self.logger.debug("Added " + rpmPkg + " rpm package to the list")
else:
self.logger.error("Missing rpm file for package:" + rpmPkg)
def installCoreToolChainPackages(self,chrootID):
self.logger.info("Installing toolchain.....")
cmdUtils = CommandUtils()
self.prepareBuildRoot(chrootID)
rpmFiles = ""
packages = ""
for package in constants.listToolChainRPMPkgsToBuild:
pkgUtils=PackageUtils(self.logName,self.logPath)
rpmFile = None
if package in constants.listCoreToolChainRPMPackages:
rpmFile=pkgUtils.findRPMFileForGivenPackage(package)
else:
rpmFile=self.findRPMFileInGivenLocation(package, constants.prevPublishRPMRepo)
if rpmFile is None:
self.logger.error("Unable to find rpm "+ package)
raise Exception("Input Error")
rpmFiles += " " + rpmFile
packages += " " + package
self.logger.debug("Installing core toolchain rpms:" + packages)
cmd=self.rpmCommand + " -i --nodeps --force --root "+chrootID+" --define \'_dbpath /var/lib/rpm\' "+ rpmFiles
process = subprocess.Popen("%s" %cmd,shell=True,stdout=subprocess.PIPE)
retval = process.wait()
if retval != 0:
self.logger.error("Installing toolchain rpms failed")
raise Exception("RPM installation failed")
self.logger.info("Installed core tool chain packages successfully on chroot:"+chrootID)
def installToolChain(self,chrootID):
self.logger.info("Installing toolchain.....")
self.prepareBuildRoot(chrootID)
cmdUtils = CommandUtils()
rpmFiles = ""
packages = ""
for package in constants.listToolChainRPMPkgsToInstall:
pkgUtils=PackageUtils(self.logName,self.logPath)
rpmFile=pkgUtils.findRPMFileForGivenPackage(package)
if rpmFile is None:
rpmFile=self.findRPMFileInGivenLocation(package, constants.prevPublishRPMRepo)
if rpmFile is None:
if package == "util-linux-devel":
self.logger.info("No old verion of util-linux-devel exists, skip until the new version is built")
continue
if package == "flex-devel":
self.logger.info("No old verion of flex-devel exists, skip until the new version is built")
continue
self.logger.error("Unable to find rpm "+ package +" in current and previous versions")
raise Exception("Input Error")
rpmFiles += " " + rpmFile
packages += " " + package
self.logger.debug("Installing toolchain rpms:" + packages)
cmd=self.rpmCommand + " -i --nodeps --force --root "+chrootID+" --define \'_dbpath /var/lib/rpm\' "+ rpmFiles
process = subprocess.Popen("%s" %cmd,shell=True,stdout=subprocess.PIPE)
retval = process.wait()
if retval != 0:
self.logger.error("Installing toolchain rpms failed")
raise Exception("RPM installation failed")
self.logger.info("Installed toolchain successfully on chroot:"+chrootID)
def installCoreToolChainPackages(self, chrootID):
self.logger.info("Installing toolchain.....")
cmdUtils = CommandUtils()
self.prepareBuildRoot(chrootID)
for package in constants.listToolChainRPMPkgsToBuild:
pkgUtils = PackageUtils(self.logName, self.logPath)
rpmFile = None
if package in constants.listCoreToolChainRPMPackages:
rpmFile = pkgUtils.findRPMFileForGivenPackage(package)
else:
rpmFile = self.findRPMFileInGivenLocation(
package, constants.prevPublishRPMRepo)
if rpmFile is None:
self.logger.error("Unable to find rpm " + package)
raise "Input Error"
self.logger.debug("Installing rpm:" + rpmFile)
cmd = "rpm -i --nodeps --force --root " + chrootID + " --define \'_dbpath /var/lib/rpm\' " + rpmFile
process = subprocess.Popen("%s" % cmd,
shell=True,
stdout=subprocess.PIPE)
retval = process.wait()
if retval != 0:
self.logger.error("Installing tool chain package " + package +
" failed")
raise "RPM installation failed"
self.logger.info(
"Installed core tool chain packages successfully on chroot:" +
chrootID)
def installToolChain(self,chrootID):
self.logger.info("Installing toolchain.....")
self.prepareBuildRoot(chrootID)
cmdUtils = CommandUtils()
rpmFiles = ""
packages = ""
for package in constants.listToolChainRPMPkgsToInstall:
pkgUtils=PackageUtils(self.logName,self.logPath)
rpmFile=pkgUtils.findRPMFileForGivenPackage(package)
if rpmFile is None:
rpmFile=self.findRPMFileInGivenLocation(package, constants.prevPublishRPMRepo)
if rpmFile is None:
if package == "rpm-build" or package == "rpm-devel":
continue
self.logger.error("Unable to find rpm "+ package +" in current and previous versions")
raise "Input Error"
rpmFiles += " " + rpmFile
packages += " " + package
self.logger.debug("Installing toolchain rpms:" + packages)
cmd=self.rpmCommand + " -i --nodeps --force --root "+chrootID+" --define \'_dbpath /var/lib/rpm\' "+ rpmFiles
process = subprocess.Popen("%s" %cmd,shell=True,stdout=subprocess.PIPE)
retval = process.wait()
if retval != 0:
self.logger.error("Installing toolchain rpms failed")
raise "RPM installation failed"
self.logger.info("Installed toolchain successfully on chroot:"+chrootID)
def buildCoreToolChainPackages(self):
self.logger.info("Step 1 : Building the core toolchain packages.....")
self.logger.info(constants.listCoreToolChainPackages)
self.logger.info("")
chrootID = None
pkgCount = 0
try:
pkgUtils = PackageUtils(self.logName, self.logPath)
coreToolChainYetToBuild = []
for package in constants.listCoreToolChainPackages:
version = SPECS.getData().getHighestVersion(package)
rpmPkg = pkgUtils.findRPMFileForGivenPackage(package, version)
if rpmPkg is not None:
continue
else:
coreToolChainYetToBuild.append(package)
if coreToolChainYetToBuild:
self.logger.info(
"The following core toolchain packages need to be built :")
self.logger.info(coreToolChainYetToBuild)
else:
self.logger.info(
"Core toolchain packages are already available")
for package in coreToolChainYetToBuild:
version = SPECS.getData().getHighestVersion(package)
self.logger.debug("Building core toolchain package : " +
package)
chrUtils = ChrootUtils(self.logName, self.logPath)
chrootName = "build-" + package
destLogPath = constants.logPath + "/build-" + package
if not os.path.isdir(destLogPath):
cmdUtils = CommandUtils()
cmdUtils.runCommandInShell("mkdir -p " + destLogPath)
returnVal, chrootID = chrUtils.createChroot(chrootName)
if not returnVal:
self.logger.error("Creating chroot failed")
raise Exception("creating chroot failed")
self.installToolChainRPMS(package, version, chrootID,
destLogPath)
pkgUtils.adjustGCCSpecs(package, version, chrootID,
destLogPath)
pkgUtils.buildRPMSForGivenPackage(package, version, chrootID,
destLogPath)
pkgCount += 1
chrUtils.destroyChroot(chrootID)
chrootID = None
self.logger.debug("Successfully built toolchain")
self.logger.info("-" * 45 + "\n")
if chrootID is not None:
chrUtils.destroyChroot(chrootID)
except Exception as e:
self.logger.error("Unable to build tool chain.")
# print stacktrace
traceback.print_exc()
raise e
return pkgCount
def installToolChain2(self,chrootID,packagesInToolChain):
self.logger.info("Installing toolchain.....")
self.prepareBuildRoot(chrootID)
cmdUtils = CommandUtils()
rpmFiles = ""
packages = ""
for package in packagesInToolChain:
pkgUtils=PackageUtils(self.logName,self.logPath)
currentRpmFile=pkgUtils.findRPMFileForGivenPackage(package)
prevPublishedRpmFile=None
if currentRpmFile is None or ("-devel" not in package and "-lib" not in package and "perl" not in package):
prevPublishedRpmFile=self.findRPMFileInGivenLocation(package, constants.prevPublishRPMRepo)
if prevPublishedRpmFile is None:
if package == "util-linux-devel":
self.logger.info("No old verion of util-linux-devel exists, skip until the new version is built")
continue
if package == "flex-devel":
self.logger.info("No old verion of flex-devel exists, skip until the new version is built")
continue
self.logger.error("Unable to find rpm "+ package +" in current and previous versions")
raise Exception("Input Error")
if (currentRpmFile is not None) and (prevPublishedRpmFile is not None):
temp = currentRpmFile[len(package)+ 1:]
currentRpmVersion = temp[:temp.find('-')]
temp = prevPublishedRpmFile[len(package)+1:]
prevPublishedRpmVersion = temp[:temp.find('-')]
if(currentRpmVersion != prevPublishedRpmVersion):
rpmFiles += " " + prevPublishedRpmFile
packages += " " + package
rpmFiles += " " + currentRpmFile
packages += " " + package
else:
if prevPublishedRpmFile is not None:
rpmFiles += " " + prevPublishedRpmFile
packages += " " + package
if currentRpmFile is not None:
rpmFiles += " " + currentRpmFile
packages += " " + package
self.logger.debug("Installing toolchain rpms:" + packages)
self.logger.debug("Installing toolchain rpms:" + rpmFiles)
cmd=self.rpmCommand + " -i --nodeps --force --root "+chrootID+" --define \'_dbpath /var/lib/rpm\' "+ rpmFiles
process = subprocess.Popen("%s" %cmd,shell=True,stdout=subprocess.PIPE)
retval = process.wait()
if retval != 0:
self.logger.error("Installing toolchain rpms failed")
raise Exception("RPM installation failed")
self.logger.info("Installed toolchain successfully on chroot:"+chrootID)
def checkIfPackageIsAlreadyBuilt(self, package):
basePkg=constants.specData.getSpecName(package)
listRPMPackages=constants.specData.getRPMPackages(basePkg)
packageIsAlreadyBuilt=True
pkgUtils = PackageUtils(self.logName,self.logPath)
for pkg in listRPMPackages:
if pkgUtils.findRPMFileForGivenPackage(pkg) is None:
packageIsAlreadyBuilt=False
break
return packageIsAlreadyBuilt
def _checkIfPackageIsAlreadyBuilt(self):
basePkg = SPECS.getData().getSpecName(self.package)
listRPMPackages = SPECS.getData().getRPMPackages(basePkg)
packageIsAlreadyBuilt = True
pkgUtils = PackageUtils(self.logName, self.logPath)
for pkg in listRPMPackages:
if pkgUtils.findRPMFileForGivenPackage(pkg) is None:
packageIsAlreadyBuilt = False
break
return packageIsAlreadyBuilt
def _checkIfPackageIsAlreadyBuilt(self, package, version):
basePkg = SPECS.getData().getSpecName(package)
listRPMPackages = SPECS.getData().getRPMPackages(basePkg, version)
packageIsAlreadyBuilt = True
pkgUtils = PackageUtils()
for pkg in listRPMPackages:
if pkgUtils.findRPMFileForGivenPackage(pkg, version) is None:
packageIsAlreadyBuilt = False
break
return packageIsAlreadyBuilt
def checkIfPackageIsAlreadyBuilt(self):
basePkg = SPECS.getData().getSpecName(self.package)
listRPMPackages = SPECS.getData().getRPMPackages(basePkg)
packageIsAlreadyBuilt = True
pkgUtils = PackageUtils(self.logName, self.logPath)
for pkg in listRPMPackages:
if pkgUtils.findRPMFileForGivenPackage(pkg) is None:
packageIsAlreadyBuilt = False
break
return packageIsAlreadyBuilt
def installToolChainRPMSinContainer(self, containerID):
self.logger.info("Installing tool-chain RPMS in container: " +
containerID.short_id)
rpmFiles = ""
packages = ""
pkgUtils = PackageUtils(self.logName, self.logPath)
for package in constants.listToolChainRPMPkgsToInstall:
version = SPECS.getData().getHighestVersion(package)
rpmFile = pkgUtils.findRPMFileForGivenPackage(package, version)
if rpmFile is None:
# sqlite-autoconf package was renamed, but it still published as sqlite-autoconf
# if (package == "sqlite") and (platform.machine() == "x86_64"):
# package = "sqlite-autoconf"
rpmFile = self.findRPMFileInGivenLocation(
package, constants.prevPublishRPMRepo)
if rpmFile is None:
if package in constants.listOfRPMsProvidedAfterBuild:
self.logger.info(
"No old version of " + package +
" exists, skip until the new version is built")
continue
self.logger.error("Unable to find rpm " + package +
" in current and previous versions")
raise Exception("Input Error")
if rpmFile.find("stage/PUBLISHRPMS"):
rpmFile = rpmFile.replace(constants.prevPublishRPMRepo,
"/publishrpms")
if rpmFile.find("stage/PUBLISHXRPMS"):
rpmFile = rpmFile.replace(constants.prevPublishXRPMRepo,
"/publishxrpms")
if rpmFile.find("stage/RPMS"):
rpmFile = rpmFile.replace(constants.rpmPath,
constants.topDirPath + "/RPMS")
rpmFiles += " " + rpmFile
packages += " " + package
self.logger.debug("Installing tool-chain rpms: " + packages)
cmd = "/usr/bin/bash -l -c '/usr/bin/rpm -Uvh --force --nodeps " + rpmFiles + "'"
self.logger.info(
"VDBG-TCU-installToolChainRPMSinContainer: Installing rpms cmd: " +
cmd)
tcInstallLog = containerID.exec_run(cmd)
# TODO: Find a way to collect exit status of the command that was run.
if not tcInstallLog:
self.logger.error("Installing tool chain in container failed")
raise Exception("RPM installation in container failed")
self.logger.info(tcInstallLog)
self.logger.info(
"Successfully installed default tool-chain RPMS in container: " +
containerID.short_id)
def _readAlreadyAvailablePackages(self):
listAvailablePackages = set()
pkgUtils = PackageUtils(self.logName, self.logPath)
listPackages = SPECS.getData().getListPackages()
for package in listPackages:
for version in SPECS.getData().getVersions(package):
# Mark package available only if all subpackages are available
packageIsAlreadyBuilt = True
listRPMPackages = SPECS.getData().getRPMPackages(
package, version)
for rpmPkg in listRPMPackages:
if pkgUtils.findRPMFileForGivenPackage(rpmPkg) is None:
packageIsAlreadyBuilt = False
break
if packageIsAlreadyBuilt:
listAvailablePackages.add(package + "-" + version)
self.logger.debug("List of Already built packages")
self.logger.debug(listAvailablePackages)
return listAvailablePackages
def installToolChainRPMSinContainer(self, containerID):
self.logger.info("Installing tool-chain RPMS in container: " + containerID.short_id)
rpmFiles = ""
packages = ""
pkgUtils = PackageUtils(self.logName, self.logPath)
for package in constants.listToolChainRPMPkgsToInstall:
rpmFile = pkgUtils.findRPMFileForGivenPackage(package)
if rpmFile is None:
# sqlite-autoconf package was renamed, but it still published as sqlite-autoconf
# if (package == "sqlite") and (platform.machine() == "x86_64"):
# package = "sqlite-autoconf"
rpmFile = self.findRPMFileInGivenLocation(package, constants.prevPublishRPMRepo)
if rpmFile is None:
if package in constants.listOfRPMsProvidedAfterBuild:
self.logger.info("No old version of " + package +
" exists, skip until the new version is built")
continue
self.logger.error("Unable to find rpm " + package +
" in current and previous versions")
raise Exception("Input Error")
if rpmFile.find("stage/PUBLISHRPMS"):
rpmFile = rpmFile.replace(constants.prevPublishRPMRepo, "/publishrpms")
if rpmFile.find("stage/PUBLISHXRPMS"):
rpmFile = rpmFile.replace(constants.prevPublishXRPMRepo, "/publishxrpms")
if rpmFile.find("stage/RPMS"):
rpmFile = rpmFile.replace(constants.rpmPath, constants.topDirPath + "/RPMS")
rpmFiles += " " + rpmFile
packages += " " + package
self.logger.debug("Installing tool-chain rpms: " + packages)
cmd = "/usr/bin/bash -l -c '/usr/bin/rpm -Uvh --force --nodeps " + rpmFiles + "'"
self.logger.info("VDBG-TCU-installToolChainRPMSinContainer: Installing rpms cmd: " + cmd)
tcInstallLog = containerID.exec_run(cmd)
# TODO: Find a way to collect exit status of the command that was run.
if not tcInstallLog:
self.logger.error("Installing tool chain in container failed")
raise Exception("RPM installation in container failed")
self.logger.info(tcInstallLog)
self.logger.info("Successfully installed default tool-chain RPMS in container: " +
containerID.short_id)
def buildCoreToolChainPackages(self, listBuildOptionPackages,
pkgBuildOptionFile):
self.logger.info("Building core toolchain packages.....")
chrootID = None
pkgCount = 0
try:
pkgUtils = PackageUtils(self.logName, self.logPath)
for package in constants.listCoreToolChainPackages:
rpmPkg = pkgUtils.findRPMFileForGivenPackage(package)
if rpmPkg is not None:
continue
self.logger.info("Building core toolchain package: " + package)
chrUtils = ChrootUtils(self.logName, self.logPath)
chrootName = "build-" + package
destLogPath = constants.logPath + "/build-" + package
if not os.path.isdir(destLogPath):
cmdUtils = CommandUtils()
cmdUtils.runCommandInShell("mkdir -p " + destLogPath)
returnVal, chrootID = chrUtils.createChroot(chrootName)
if not returnVal:
self.logger.error("Creating chroot failed")
raise Exception("creating chroot failed")
self.installToolChainRPMS(chrootID, package, destLogPath)
pkgUtils.adjustGCCSpecs(package, chrootID, destLogPath)
pkgUtils.buildRPMSForGivenPackage(package, chrootID,
listBuildOptionPackages,
pkgBuildOptionFile,
destLogPath)
pkgCount += 1
chrUtils.destroyChroot(chrootID)
chrootID = None
self.logger.info("Successfully built toolchain")
if chrootID is not None:
chrUtils.destroyChroot(chrootID)
except Exception as e:
self.logger.error("Unable to build tool chain.")
# print stacktrace
traceback.print_exc()
raise e
return pkgCount
def installToolChainRPMS(self,
chroot,
packageName=None,
packageVersion=None,
logPath=None):
if logPath is None:
logPath = self.logPath
self.logger.debug("Installing Tool Chain RPMS.......")
rpmFiles = ""
packages = ""
listBuildRequiresPackages = []
if packageName:
listBuildRequiresPackages = self.getListDependentPackages(
packageName, packageVersion)
for package in constants.listToolChainRPMsToInstall:
pkgUtils = PackageUtils(self.logName, self.logPath)
rpmFile = None
version = "*"
for depPkg in listBuildRequiresPackages:
depPkgName, depPkgVersion = StringUtils.splitPackageNameAndVersion(
depPkg)
if depPkgName == package:
version = depPkgVersion
if constants.rpmCheck:
rpmFile = pkgUtils.findRPMFileForGivenPackage(package, version)
elif (packageName is None
or packageName not in constants.listToolChainRPMsToInstall
or constants.listToolChainRPMsToInstall.index(packageName) >
constants.listToolChainRPMsToInstall.index(package)):
rpmFile = pkgUtils.findRPMFileForGivenPackage(package, version)
if rpmFile is None:
# sqlite-autoconf package was renamed, but it still published as sqlite-autoconf
if (package == "sqlite") and (platform.machine() == "x86_64"):
package = "sqlite-autoconf"
rpmFile = self.findRPMFileInGivenLocation(
package, constants.prevPublishRPMRepo)
if rpmFile is None:
if package in constants.listOfRPMsProvidedAfterBuild:
self.logger.debug(
"No old version of " + package +
" exists, skip until the new version is built")
continue
self.logger.error("Unable to find rpm " + package +
" in current and previous versions")
raise Exception("Input Error")
rpmFiles += " " + rpmFile
packages += " " + package + "-" + version
self.logger.debug(packages)
cmd = (self.rpmCommand + " -i -v --nodeps --noorder --force --root " +
chroot.getPath() + " --define \'_dbpath /var/lib/rpm\' " +
rpmFiles)
retVal = CommandUtils.runCommandInShell(cmd, logfn=self.logger.debug)
if retVal != 0:
self.logger.debug("Command Executed:" + cmd)
self.logger.error("Installing tool chain failed")
raise Exception("RPM installation failed")
self.logger.debug(
"Successfully installed default Tool Chain RPMS in Chroot:" +
chroot.getPath())
if packageName:
self.installCustomToolChainRPMS(chroot, packageName,
packageVersion)
