コード例 #1
0
    def __constructBuildAndRunTimeDependencyGraph(self, package):
        basePackage = SPECS.getData().getSpecName(package)

        addBuildTimeGraph = True
        addRunTimeGraph = True
        if basePackage in self.__buildDependencyGraph:
            addBuildTimeGraph = False
        if basePackage in self.__runTimeDependencyGraph:
            addRunTimeGraph = False

        nextPackagesToConstructGraph = []
        if addBuildTimeGraph:
            listDependentRpmPackages = SPECS.getData(
            ).getBuildRequiresForPackage(basePackage)
            listDependentPackages = []
            for rpmPkg in listDependentRpmPackages:
                basePkg = SPECS.getData().getSpecName(rpmPkg)
                if basePkg not in listDependentPackages:
                    listDependentPackages.append(basePkg)
            self.__buildDependencyGraph[basePackage] = listDependentPackages
            nextPackagesToConstructGraph.extend(listDependentPackages)

        if addRunTimeGraph:
            listRpmPackages = SPECS.getData().getPackages(basePackage)
            for rpmPkg in listRpmPackages:
                listDependentRpmPackages = SPECS.getData(
                ).getRequiresAllForPackage(rpmPkg)
                self.__runTimeDependencyGraph[
                    rpmPkg] = listDependentRpmPackages[:]
                nextPackagesToConstructGraph.extend(listDependentRpmPackages)

        for pkg in nextPackagesToConstructGraph:
            self.__constructBuildAndRunTimeDependencyGraph(pkg)
コード例 #2
0
ファイル: PackageUtils.py プロジェクト: quinndiggity/photon
    def buildRPMSForGivenPackage(self, package, chrootID, destLogPath=None):
        self.logger.info("Building rpm's for package:" + package)

        listSourcesFiles = SPECS.getData().getSources(package)
        listPatchFiles = SPECS.getData().getPatches(package)
        specFile = SPECS.getData().getSpecFile(package)
        specName = SPECS.getData().getSpecName(package) + ".spec"

        chrootSourcePath = chrootID + constants.topDirPath + "/SOURCES/"
        chrootSpecPath = constants.topDirPath + "/SPECS/"
        chrootLogsFilePath = chrootID + constants.topDirPath + "/LOGS/" + package + ".log"
        chrootCmd = self.runInChrootCommand + " " + chrootID
        shutil.copyfile(specFile, chrootID + chrootSpecPath + specName)

        # FIXME: some sources are located in SPECS/.. how to mount?
        #        if os.geteuid()==0:
        self._copySourcesTobuildroot(listSourcesFiles, package,
                                     chrootSourcePath)
        self._copySourcesTobuildroot(listPatchFiles, package, chrootSourcePath)
        macros = []

        listAdditionalFiles, macros = self._getAdditionalBuildFiles(package)
        self._copyAdditionalBuildFiles(listAdditionalFiles, chrootID)

        #Adding rpm macros
        listRPMMacros = constants.userDefinedMacros
        for macroName, value in listRPMMacros.items():
            macros.append(macroName + " " + value)

        listRPMFiles = []
        listSRPMFiles = []
        try:
            listRPMFiles, listSRPMFiles = self._buildRPM(
                chrootSpecPath + specName, chrootLogsFilePath, chrootCmd,
                package, macros)
            self.logger.info("Successfully built rpm:" + package)
        except Exception as e:
            self.logger.error("Failed while building rpm:" + package)
            raise e
        finally:
            if destLogPath is not None:
                if (constants.rpmCheck and package in constants.testForceRPMS
                        and SPECS.getData().isCheckAvailable(package)):
                    cmd = (
                        "sed -i '/^Executing(%check):/,/^Processing files:/{//!b};d' "
                        + chrootLogsFilePath)
                    logFile = destLogPath + "/adjustTestFile.log"
                    returnVal = CommandUtils().runCommandInShell(cmd, logFile)
                    testLogFile = destLogPath + "/" + package + "-test.log"
                    shutil.copyfile(chrootLogsFilePath, testLogFile)
                else:
                    shutil.copy2(chrootLogsFilePath, destLogPath)
        self.logger.info("RPM build is successful")

        for rpmFile in listRPMFiles:
            self._copyRPM(chrootID + "/" + rpmFile, constants.rpmPath)

        for srpmFile in listSRPMFiles:
            srpmDestFile = self._copyRPM(chrootID + "/" + srpmFile,
                                         constants.sourceRpmPath)
コード例 #3
0
ファイル: Scheduler.py プロジェクト: jadeiceman/photon
    def _getListNextPackagesReadyToBuild():
        for pkg in Scheduler.listOfPackagesToBuild:
            if pkg in Scheduler.listOfPackagesCurrentlyBuilding:
                continue
            listRequiredSubPackages = list(set(SPECS.getData().getBuildRequiresForPkg(pkg) + \
                                   SPECS.getData().getRequiresAllForPkg(pkg)))

            # extend to full Requires tree
            for p in listRequiredSubPackages:
                reqs = SPECS.getData().getRequiresAllForPkg(p)
                for r in reqs:
                    if r not in listRequiredSubPackages:
                        listRequiredSubPackages.append(r)

            # convert subpackages to basepkg
            listRequiredPackages = set()
            for p in listRequiredSubPackages:
                listRequiredPackages.add(SPECS.getData().getBasePkg(p))

            canBuild = True
            for reqPkg in listRequiredPackages:
                if reqPkg not in Scheduler.listOfAlreadyBuiltPackages:
                    canBuild = False
                    break

            #if canBuild and constants.currentArch == "arm":
            #    canBuild = pkg in constants.customPackageList
            #    if not canBuild:
            #        Scheduler.logger.debug(">>>> %s is not in custom package list" % pkg)

            if canBuild:
                Scheduler.listOfPackagesNextToBuild.put((-Scheduler._getPriority(pkg), pkg))
                Scheduler.logger.debug("Adding " + pkg + " to the schedule list")
コード例 #4
0
ファイル: PackageInfo.py プロジェクト: frapposelli/photon
 def loadPackagesData(self):
     listPackages = SPECS.getData().getListPackages()
     listPackages.sort()
     cmdUtils = CommandUtils()
     for package in listPackages:
         for version in SPECS.getData().getVersions(package):
             release = SPECS.getData().getRelease(package, version)
             listRPMPackages = SPECS.getData().getRPMPackages(package, version)
             srpmFileName = package + "-" + version + "-" + release + ".src.rpm"
             srpmFiles = cmdUtils.findFile(srpmFileName, constants.sourceRpmPath)
             srpmFile = None
             if len(srpmFiles) == 1:
                 srpmFile = srpmFiles[0]
             debugrpmFileName = package + "-debuginfo-" + version + "-" + release + "*"
             debugrpmFiles = cmdUtils.findFile(debugrpmFileName, constants.rpmPath)
             debugrpmFile = None
             if len(debugrpmFiles) == 1:
                 debugrpmFile = debugrpmFiles[0]
             pkgUtils = PackageUtils(self.logName, self.logPath)
             for rpmPkg in listRPMPackages:
                 rpmFile = pkgUtils.findRPMFile(rpmPkg, version)
                 if rpmFile is not None:
                     listPkgAttributes = {"sourcerpm":srpmFile, "rpm":rpmFile,
                                          "debugrpm":debugrpmFile}
                     self.pkgList[rpmPkg+"-"+version] = listPkgAttributes
                     self.logger.debug("Added " + rpmPkg + "-" + version + " to the package info json")
                 else:
                     self.logger.debug("Missing rpm file for package:" + rpmPkg)
コード例 #5
0
ファイル: ToolChainUtils.py プロジェクト: stephenrob/photon
    def installTargetToolchain(self, chroot, stopAtPackage=None):
        self.logger.debug("Installing target toolchain RPMS.......")
        pkgUtils = PackageUtils(self.logName, self.logPath)
        rpmFiles = ""
        packages = ""
        for package in constants.listCoreToolChainPackages:
            if stopAtPackage and package == stopAtPackage:
                break
            version = SPECS.getData().getHighestVersion(package)
            basePkg = SPECS.getData().getSpecName(package)
            # install all subpackages of given package
            # for instance: for 'glibc' we want glibc-devel, glibc-tools,
            #               glibc-i18n, etc also to be installed
            subpackages = SPECS.getData().getRPMPackages(basePkg, version)
            for p in subpackages:
                rpmFile = pkgUtils.findRPMFile(p, version, constants.targetArch)
                rpmFiles += " " + rpmFile
                packages += " " + package+"-"+version

        self.logger.debug(packages)

        cmd = "mkdir -p " + chroot.getID() +"/target-"+ constants.targetArch+"/var/lib/rpm"
        CommandUtils.runCommandInShell(cmd, logfn=self.logger.debug)

        if rpmFiles != "":
            cmd = (self.rpmCommand+" -Uvh --nodeps --ignorearch --noscripts --root "+
                   chroot.getID() +"/target-"+ constants.targetArch+
                   " --define \'_dbpath /var/lib/rpm\' "+rpmFiles)
            retVal = CommandUtils.runCommandInShell(cmd, logfn=self.logger.debug)
            if retVal != 0:
                self.logger.debug("Command Executed:" + cmd)
                self.logger.error("Installing toolchain failed")
                raise Exception("RPM installation failed")
        self.logger.debug("Successfully installed target toolchain RPMS in chroot:" + chroot.getID())
コード例 #6
0
ファイル: Scheduler.py プロジェクト: frapposelli/photon
    def _getListNextPackagesReadyToBuild():
        for pkg in Scheduler.listOfPackagesToBuild:
            if pkg in Scheduler.listOfPackagesCurrentlyBuilding:
                continue
            listRequiredSubPackages = list(set(SPECS.getData().getBuildRequiresForPkg(pkg) + \
                                   SPECS.getData().getRequiresAllForPkg(pkg)))

            # extend to full Requires tree
            for p in listRequiredSubPackages:
                reqs = SPECS.getData().getRequiresAllForPkg(p)
                for r in reqs:
                    if r not in listRequiredSubPackages:
                        listRequiredSubPackages.append(r)

            # convert subpackages to basepkg
            listRequiredPackages = set()
            for p in listRequiredSubPackages:
                listRequiredPackages.add(SPECS.getData().getBasePkg(p))

            canBuild = True
            for reqPkg in listRequiredPackages:
                if reqPkg not in Scheduler.listOfAlreadyBuiltPackages:
                    canBuild = False
                    break
            if canBuild:
                Scheduler.listOfPackagesNextToBuild.put((-Scheduler._getPriority(pkg), pkg))
                Scheduler.logger.debug("Adding " + pkg + " to the schedule list")
コード例 #7
0
 def getListDependentPackages(self, package, version):
     listBuildRequiresPkg = SPECS.getData(
         constants.buildArch).getBuildRequiresForPackage(package, version)
     listBuildRequiresPkg.extend(
         SPECS.getData(constants.buildArch).getCheckBuildRequiresForPackage(
             package, version))
     return listBuildRequiresPkg
コード例 #8
0
ファイル: PackageInfo.py プロジェクト: cybernetics/photon-1
 def loadPackagesData(self):
     listPackages = SPECS.getData().getListPackages()
     listPackages.sort()
     pkgUtils = PackageUtils(self.logName, self.logPath)
     for package in listPackages:
         for version in SPECS.getData().getVersions(package):
             srpmFile = pkgUtils.findSourceRPMFile(package, version)
             debugrpmFile = pkgUtils.findDebugRPMFile(package, version)
             listRPMPackages = SPECS.getData().getRPMPackages(
                 package, version)
             for rpmPkg in listRPMPackages:
                 rpmFile = pkgUtils.findRPMFile(rpmPkg, version)
                 if rpmFile is not None:
                     listPkgAttributes = {
                         "sourcerpm": srpmFile,
                         "rpm": rpmFile,
                         "debugrpm": debugrpmFile
                     }
                     self.pkgList[rpmPkg + "-" +
                                  version] = listPkgAttributes
                     self.logger.debug("Added " + rpmPkg + "-" + version +
                                       " to the package info json")
                 else:
                     self.logger.debug("Missing rpm file for package:" +
                                       rpmPkg)
コード例 #9
0
 def findRPMFileForGivenPackage(self, package, version="*"):
     cmdUtils = CommandUtils()
     if version == "*":
         version = SPECS.getData().getHighestVersion(package)
     release = SPECS.getData().getRelease(package, version)
     buildarch = SPECS.getData().getBuildArch(package, version)
     listFoundRPMFiles = sum([
         cmdUtils.findFile(
             package + "-" + version + "-" + release + "." + buildarch +
             ".rpm", constants.rpmPath)
     ], [])
     if constants.inputRPMSPath is not None:
         listFoundRPMFiles = sum([
             cmdUtils.findFile(
                 package + "-" + version + "-" + release + "." + buildarch +
                 ".rpm", constants.inputRPMSPath)
         ], listFoundRPMFiles)
     if len(listFoundRPMFiles) == 1:
         return listFoundRPMFiles[0]
     if len(listFoundRPMFiles) == 0:
         return None
     if len(listFoundRPMFiles) > 1:
         self.logger.error(
             "Found multiple rpm files for given package in rpm directory."
             + "Unable to determine the rpm file for package:" + package +
             " : " + str(listFoundRPMFiles))
         raise Exception("Multiple rpm files found")
コード例 #10
0
    def _verifyShaAndGetSourcePath(self, source, package, version):
        cmdUtils = CommandUtils()
        # Fetch/verify sources if sha1 not None.
        sha1 = SPECS.getData().getSHA1(package, version, source)
        if sha1 is not None:
            PullSources.get(package, source, sha1, constants.sourcePath,
                            constants.getPullSourcesURLs(package), self.logger)

        sourcePath = cmdUtils.findFile(source, constants.sourcePath)
        if not sourcePath:
            sourcePath = cmdUtils.findFile(source, os.path.dirname(SPECS.getData().getSpecFile(package, version)))
            if not sourcePath:
                if sha1 is None:
                    self.logger.error("No sha1 found or missing source for " + source)
                    raise Exception("No sha1 found or missing source for " + source)
                else:
                    self.logger.error("Missing source: " + source +
                                      ". Cannot find sources for package: " + package)
                    raise Exception("Missing source")
        else:
            if sha1 is None:
                self.logger.error("No sha1 found for "+source)
                raise Exception("No sha1 found")
        if len(sourcePath) > 1:
            self.logger.error("Multiple sources found for source:" + source + "\n" +
                              ",".join(sourcePath) +"\nUnable to determine one.")
            raise Exception("Multiple sources found")
        return sourcePath
コード例 #11
0
    def _constructBuildAndRunTimeDependencyGraph(self, basePackage):
        addBuildTimeGraph = True
        addRunTimeGraph = True
        if basePackage in self.__buildDependencyGraph:
            addBuildTimeGraph = False
        if basePackage in self.__runTimeDependencyGraph:
            addRunTimeGraph = False

        nextPackagesToConstructGraph = set()
        if addBuildTimeGraph:
            dependentRpmPackages = SPECS.getData().getBuildRequiresForPkg(
                basePackage)
            dependentPackages = set()
            for dependentPkg in dependentRpmPackages:
                dependentPackages.add(SPECS.getData().getBasePkg(dependentPkg))
            self.__buildDependencyGraph[basePackage] = dependentPackages
            nextPackagesToConstructGraph.update(dependentPackages)

        if addRunTimeGraph:
            dependentPackages = set()
            for rpmPkg in SPECS.getData().getPackagesForPkg(basePackage):
                dependentRpmPackages = SPECS.getData().getRequiresAllForPkg(
                    rpmPkg)
                self.__runTimeDependencyGraph[rpmPkg] = copy.copy(
                    set(dependentRpmPackages))
                for pkg in dependentRpmPackages:
                    dependentPackages.add(SPECS.getData().getBasePkg(pkg))
            nextPackagesToConstructGraph.update(dependentPackages)

        for pkg in nextPackagesToConstructGraph:
            self._constructBuildAndRunTimeDependencyGraph(pkg)
コード例 #12
0
ファイル: PackageInfo.py プロジェクト: tonyganga/photon
 def loadPackagesData(self):
     listPackages = SPECS.getData().getListPackages()
     listPackages.sort()
     listRPMFiles = []
     cmdUtils = CommandUtils()
     for package in listPackages:
         release = SPECS.getData().getRelease(package)
         version = SPECS.getData().getVersion(package)
         listRPMPackages = SPECS.getData().getRPMPackages(package)
         srpmFileName = package + "-" + version + "-" + release + ".src.rpm"
         srpmFiles = cmdUtils.findFile(srpmFileName, constants.sourceRpmPath)
         srpmFile = None
         if len(srpmFiles) == 1:
             srpmFile = srpmFiles[0]
         debugrpmFileName = package + "-debuginfo-" + version + "-" + release + "*"
         debugrpmFiles = cmdUtils.findFile(debugrpmFileName, constants.rpmPath)
         debugrpmFile = None
         if len(debugrpmFiles) == 1:
             debugrpmFile = debugrpmFiles[0]
         pkgUtils = PackageUtils(self.logName, self.logPath)
         for rpmPkg in listRPMPackages:
             rpmFile = pkgUtils.findRPMFileForGivenPackage(rpmPkg)
             if rpmFile is not None:
                 listRPMFiles.append(rpmFile)
                 listPkgAttributes = {"sourcerpm":srpmFile, "rpm":rpmFile,
                                      "debugrpm":debugrpmFile}
                 self.pkgList[rpmPkg] = listPkgAttributes
                 self.logger.debug("Added " + rpmPkg + " rpm package to the list")
             else:
                 self.logger.error("Missing rpm file for package:" + rpmPkg)
コード例 #13
0
 def findRPMFileForGivenPackage(self, package,version="*", index=0):
     cmdUtils = CommandUtils()
     release = "*"
     if version == "*":
             version = SPECS.getData().getVersion(package, index)
             release = SPECS.getData().getRelease(package, index)
     listFoundRPMFiles = sum([cmdUtils.findFile(package + "-" + version + "-" + release + "." +
                                                platform.machine()+".rpm",
                                                constants.rpmPath),
                              cmdUtils.findFile(package + "-" + version + "-" + release +
                                                ".noarch.rpm",
                                                constants.rpmPath)], [])
     if constants.inputRPMSPath is not None:
         listFoundRPMFiles = sum([cmdUtils.findFile(package + "-" + version + "-" + release +
                                                    "." + platform.machine()+".rpm",
                                                    constants.inputRPMSPath),
                                  cmdUtils.findFile(package + "-" + version + "-" + release +
                                                    ".noarch.rpm", constants.inputRPMSPath)],
                                 listFoundRPMFiles)
     if len(listFoundRPMFiles) == 1:
         return listFoundRPMFiles[0]
     if len(listFoundRPMFiles) == 0:
         return None
     if len(listFoundRPMFiles) > 1:
         self.logger.error("Found multiple rpm files for given package in rpm directory." +
                           "Unable to determine the rpm file for package:" + package)
         raise Exception("Multiple rpm files found")
コード例 #14
0
ファイル: PackageUtils.py プロジェクト: frapposelli/photon
    def _verifyShaAndGetSourcePath(self, source, package, version):
        cmdUtils = CommandUtils()
        # Fetch/verify sources if sha1 not None.
        sha1 = SPECS.getData().getSHA1(package, version, source)
        if sha1 is not None:
            PullSources.get(package, source, sha1, constants.sourcePath,
                            constants.getPullSourcesURLs(package), self.logger)

        sourcePath = cmdUtils.findFile(source, constants.sourcePath)
        if not sourcePath:
            sourcePath = cmdUtils.findFile(source, os.path.dirname(SPECS.getData().getSpecFile(package, version)))
            if not sourcePath:
                if sha1 is None:
                    self.logger.error("No sha1 found or missing source for " + source)
                    raise Exception("No sha1 found or missing source for " + source)
                else:
                    self.logger.error("Missing source: " + source +
                                      ". Cannot find sources for package: " + package)
                    raise Exception("Missing source")
        else:
            if sha1 is None:
                self.logger.error("No sha1 found for "+source)
                raise Exception("No sha1 found")
        if len(sourcePath) > 1:
            self.logger.error("Multiple sources found for source:" + source + "\n" +
                              ",".join(sourcePath) +"\nUnable to determine one.")
            raise Exception("Multiple sources found")
        return sourcePath
コード例 #15
0
ファイル: GenerateOSSFiles.py プロジェクト: zhikun0704/photon
def buildSRPMList(srpmPath,
                  yamlDir,
                  blackListPkgs,
                  dist_tag,
                  logger,
                  singleFile=True):
    cmdUtils = CommandUtils()
    yamlSrpmDir = os.path.join(yamlDir, "yaml_srpms")
    if not os.path.isdir(yamlSrpmDir):
        cmdUtils.runCommandInShell("mkdir -p " + yamlSrpmDir)
    if singleFile:
        yamlFile = open(yamlSrpmDir + "/srpm_list.yaml", "w")
    listPackages = SPECS.getData().getListPackages()
    listPackages.sort()
    for package in listPackages:
        if package in blackListPkgs:
            continue
        ossname = package
        for ossversion in SPECS.getData().getVersions(package):
            ossrelease = SPECS.getData().getRelease(package, ossversion)
            srpm_file_name = "%s-%s-%s%s.src.rpm" % (ossname, ossversion,
                                                     ossrelease, dist_tag)
            logger.info("srpm name is %s" % (srpm_file_name))
            listFoundSRPMFiles = cmdUtils.findFile(srpm_file_name, srpmPath)

            srpmName = None
            if len(listFoundSRPMFiles) == 1:
                srpmFullPath = listFoundSRPMFiles[0]
                srpmName = os.path.basename(srpmFullPath)
                cpcmd = "cp " + srpmFullPath + " " + yamlSrpmDir + "/"
                returnVal = cmdUtils.runCommandInShell(cpcmd)
                if returnVal != 0:
                    logger.error("Copy SRPM File is failed for package:" +
                                 ossname)
            else:
                logger.error("SRPM file is not found:" + ossname)

            if not singleFile:
                yamlFile = open(
                    yamlSrpmDir + "/" + ossname + "-" + ossversion + "-" +
                    ossrelease + ".yaml", "w")

            yamlFile.write("baseos:" + ossname + ":" + ossversion + "-" +
                           ossrelease + ":\n")
            yamlFile.write("  repository: BaseOS\n")
            yamlFile.write("  name: '" + ossname + "'\n")
            yamlFile.write("  version: '" + ossversion + "-" + ossrelease +
                           "'\n")
            yamlFile.write("  url: 'http://www.vmware.com'\n")
            yamlFile.write("  baseos-style: rpm\n")
            yamlFile.write("  baseos-source: '" + str(srpmName) + "'\n")
            yamlFile.write("  baseos-osname: 'photon'\n")
            yamlFile.write("\n")
            if not singleFile:
                yamlFile.close()

    if singleFile:
        yamlFile.close()
    logger.debug("Generated SRPM yaml files for all packages")
コード例 #16
0
    def _buildGivenPackages(self, listPackages, buildThreads):
        # Extend listPackages from ["name1", "name2",..] to ["name1-vers1", "name2-vers2",..]
        listPackageNamesAndVersions = []
        for pkg in listPackages:
            for version in SPECS.getData().getVersions(pkg):
                listPackageNamesAndVersions.append(pkg + "-" + version)

        if constants.rpmCheck:
            listMakeCheckPackages = set()
            for pkg in listPackages:
                version = SPECS.getData().getHighestVersion(pkg)
                listMakeCheckPackages.add(pkg + "-" + version)
            alreadyBuiltRPMS = self._readAlreadyAvailablePackages()
            listPackageNamesAndVersions = (list(
                set(listPackageNamesAndVersions)
                | (listMakeCheckPackages - alreadyBuiltRPMS)))

        returnVal = self._calculateParams(listPackageNamesAndVersions)
        if not returnVal:
            self.logger.error(
                "Unable to set paramaters. Terminating the package manager.")
            raise Exception("Unable to set paramaters")

        statusEvent = threading.Event()
        self._initializeScheduler(statusEvent)
        self._initializeThreadPool(statusEvent)

        for i in range(0, buildThreads):
            workerName = "WorkerThread" + str(i)
            ThreadPool.addWorkerThread(workerName)
            ThreadPool.startWorkerThread(workerName)

        statusEvent.wait()
        Scheduler.stopScheduling = True
        self.logger.info("Waiting for all remaining worker threads")
        ThreadPool.join_all()

        setFailFlag = False
        allPackagesBuilt = False
        if Scheduler.isAnyPackagesFailedToBuild():
            setFailFlag = True

        if Scheduler.isAllPackagesBuilt():
            allPackagesBuilt = True

        if setFailFlag:
            self.logger.error("Some of the packages failed:")
            self.logger.error(Scheduler.listOfFailedPackages)
            raise Exception("Failed during building package")

        if not setFailFlag:
            if allPackagesBuilt:
                self.logger.info("All packages built successfully")
            else:
                self.logger.error("Build stopped unexpectedly.Unknown error.")
                raise Exception("Unknown error")

        self.logger.info("Terminated")
コード例 #17
0
ファイル: PackageUtils.py プロジェクト: TiejunChina/photon
    def buildRPMSForGivenPackage(self, package, chrootID, destLogPath=None, index=0):
        self.logger.info("Building rpm's for package:" + package)

        listSourcesFiles = SPECS.getData().getSources(package, index)
        listPatchFiles = SPECS.getData().getPatches(package, index)
        specFile = SPECS.getData().getSpecFile(package, index)
        specName = SPECS.getData().getSpecName(package) + ".spec"

        chrootSourcePath = chrootID + constants.topDirPath + "/SOURCES/"
        chrootSpecPath = constants.topDirPath + "/SPECS/"
        chrootLogsFilePath = chrootID + constants.topDirPath + "/LOGS/" + package + ".log"
        chrootCmd = self.runInChrootCommand + " " + chrootID
        shutil.copyfile(specFile, chrootID + chrootSpecPath + specName)

        # FIXME: some sources are located in SPECS/.. how to mount?
        #        if os.geteuid()==0:
        self._copySourcesTobuildroot(listSourcesFiles, package, chrootSourcePath, index)
        self._copySourcesTobuildroot(listPatchFiles, package, chrootSourcePath, index)
        macros = []

        listAdditionalFiles, macros = self._getAdditionalBuildFiles(package)
        self._copyAdditionalBuildFiles(listAdditionalFiles, chrootID)

        #Adding rpm macros
        listRPMMacros = constants.userDefinedMacros
        for macroName, value in listRPMMacros.items():
            macros.append(macroName + " " + value)

        listRPMFiles = []
        listSRPMFiles = []
        try:
            listRPMFiles, listSRPMFiles = self._buildRPM(chrootSpecPath + specName,
                                                         chrootLogsFilePath, chrootCmd,
                                                         package, macros)
            self.logger.info("Successfully built rpm:" + package)
        except Exception as e:
            self.logger.error("Failed while building rpm:" + package)
            raise e
        finally:
            if destLogPath is not None:
                if (constants.rpmCheck and
                        package in constants.testForceRPMS and
                        SPECS.getData().isCheckAvailable(package)):
                    cmd = ("sed -i '/^Executing(%check):/,/^Processing files:/{//!b};d' " +
                           chrootLogsFilePath)
                    logFile = destLogPath + "/adjustTestFile.log"
                    returnVal = CommandUtils().runCommandInShell(cmd, logFile)
                    testLogFile = destLogPath + "/" + package + "-test.log"
                    shutil.copyfile(chrootLogsFilePath, testLogFile)
                else:
                    shutil.copy2(chrootLogsFilePath, destLogPath)
        self.logger.info("RPM build is successful")

        for rpmFile in listRPMFiles:
            self._copyRPM(chrootID + "/" + rpmFile, constants.rpmPath)

        for srpmFile in listSRPMFiles:
            srpmDestFile = self._copyRPM(chrootID + "/" + srpmFile, constants.sourceRpmPath)
コード例 #18
0
ファイル: SpecDeps.py プロジェクト: frapposelli/photon
 def updateLevels(self, mapDependencies, inPkg, parent, level):
     listPackages = SPECS.getData().getPackagesForPkg(inPkg)
     for depPkg in SPECS.getData().getRequiresForPkg(inPkg):
         if depPkg in listPackages:
             continue
         if depPkg in mapDependencies and mapDependencies[depPkg] < level + 1:
             mapDependencies[depPkg] = level + 1
             parent[depPkg] = inPkg
             self.updateLevels(mapDependencies, depPkg, parent, mapDependencies[depPkg])
コード例 #19
0
ファイル: SpecDeps.py プロジェクト: frapposelli/photon
 def getBasePackagesRequired(self, pkg):
     listBasePackagesRequired=[]
     listPackagesRequired = SPECS.getData().getBuildRequiresForPkg(pkg)
     listPackagesRequired.extend(SPECS.getData().getRequiresAllForPkg(pkg))
     for p in listPackagesRequired:
         basePkg = SPECS.getData().getBasePkg(p)
         if basePkg not in listBasePackagesRequired:
             listBasePackagesRequired.append(basePkg)
     return listBasePackagesRequired
コード例 #20
0
    def buildCoreToolChainPackages(self):
        self.logger.info("Step 1 : Building the core toolchain packages.....")
        self.logger.info(constants.listCoreToolChainPackages)
        self.logger.info("")
        chrootID = None
        pkgCount = 0
        try:
            pkgUtils = PackageUtils(self.logName, self.logPath)
            coreToolChainYetToBuild = []
            for package in constants.listCoreToolChainPackages:
                version = SPECS.getData().getHighestVersion(package)
                rpmPkg = pkgUtils.findRPMFileForGivenPackage(package, version)
                if rpmPkg is not None:
                    continue
                else:
                    coreToolChainYetToBuild.append(package)
            if coreToolChainYetToBuild:
                self.logger.info(
                    "The following core toolchain packages need to be built :")
                self.logger.info(coreToolChainYetToBuild)
            else:
                self.logger.info(
                    "Core toolchain packages are already available")

            for package in coreToolChainYetToBuild:
                version = SPECS.getData().getHighestVersion(package)
                self.logger.debug("Building core toolchain package : " +
                                  package)
                chrUtils = ChrootUtils(self.logName, self.logPath)
                chrootName = "build-" + package
                destLogPath = constants.logPath + "/build-" + package
                if not os.path.isdir(destLogPath):
                    cmdUtils = CommandUtils()
                    cmdUtils.runCommandInShell("mkdir -p " + destLogPath)
                returnVal, chrootID = chrUtils.createChroot(chrootName)
                if not returnVal:
                    self.logger.error("Creating chroot failed")
                    raise Exception("creating chroot failed")
                self.installToolChainRPMS(package, version, chrootID,
                                          destLogPath)
                pkgUtils.adjustGCCSpecs(package, version, chrootID,
                                        destLogPath)
                pkgUtils.buildRPMSForGivenPackage(package, version, chrootID,
                                                  destLogPath)
                pkgCount += 1
                chrUtils.destroyChroot(chrootID)
                chrootID = None
            self.logger.debug("Successfully built toolchain")
            self.logger.info("-" * 45 + "\n")
            if chrootID is not None:
                chrUtils.destroyChroot(chrootID)
        except Exception as e:
            self.logger.error("Unable to build tool chain.")
            # print stacktrace
            traceback.print_exc()
            raise e
        return pkgCount
コード例 #21
0
 def getListDependentPackageLineContent(self, index):
     listBuildRequiresPkgLineContent = SPECS.getData(
     ).getBuildRequiresForPackage(self.package, index)
     listBuildRequiresPkgLineContent.extend(
         SPECS.getData().getCheckBuildRequiresForPackage(
             self.package, index))
     listBuildRequiresPkgLineContent = list(
         set(listBuildRequiresPkgLineContent))
     return listBuildRequiresPkgLineContent
コード例 #22
0
ファイル: SpecDeps.py プロジェクト: transcendtron/photon
 def updateLevels(self, mapDependencies, inPkg, parent, level):
     listPackages = SPECS.getData().getPackagesForPkg(inPkg)
     for depPkg in SPECS.getData().getRequiresForPkg(inPkg):
         if depPkg in listPackages:
             continue
         if depPkg in mapDependencies and mapDependencies[depPkg] < level + 1:
             mapDependencies[depPkg] = level + 1
             parent[depPkg] = inPkg
             self.updateLevels(mapDependencies, depPkg, parent, mapDependencies[depPkg])
コード例 #23
0
ファイル: SpecDeps.py プロジェクト: transcendtron/photon
 def getBasePackagesRequired(self, pkg):
     listBasePackagesRequired=[]
     listPackagesRequired = SPECS.getData().getBuildRequiresForPkg(pkg)
     listPackagesRequired.extend(SPECS.getData().getRequiresAllForPkg(pkg))
     for p in listPackagesRequired:
         basePkg = SPECS.getData().getBasePkg(p)
         if basePkg not in listBasePackagesRequired:
             listBasePackagesRequired.append(basePkg)
     return listBasePackagesRequired
コード例 #24
0
 def checkIfPackageIsAlreadyBuilt(self):
     basePkg = SPECS.getData().getSpecName(self.package)
     listRPMPackages = SPECS.getData().getRPMPackages(basePkg)
     packageIsAlreadyBuilt = True
     pkgUtils = PackageUtils(self.logName, self.logPath)
     for pkg in listRPMPackages:
         if pkgUtils.findRPMFileForGivenPackage(pkg) is None:
             packageIsAlreadyBuilt = False
             break
     return packageIsAlreadyBuilt
コード例 #25
0
ファイル: PackageBuilder.py プロジェクト: vinaykul/photon
 def _checkIfPackageIsAlreadyBuilt(self):
     basePkg = SPECS.getData().getSpecName(self.package)
     listRPMPackages = SPECS.getData().getRPMPackages(basePkg)
     packageIsAlreadyBuilt = True
     pkgUtils = PackageUtils(self.logName, self.logPath)
     for pkg in listRPMPackages:
         if pkgUtils.findRPMFileForGivenPackage(pkg) is None:
             packageIsAlreadyBuilt = False
             break
     return packageIsAlreadyBuilt
コード例 #26
0
 def _checkIfPackageIsAlreadyBuilt(self, package, version):
     basePkg = SPECS.getData().getSpecName(package)
     listRPMPackages = SPECS.getData().getRPMPackages(basePkg, version)
     packageIsAlreadyBuilt = True
     pkgUtils = PackageUtils()
     for pkg in listRPMPackages:
         if pkgUtils.findRPMFileForGivenPackage(pkg, version) is None:
             packageIsAlreadyBuilt = False
             break
     return packageIsAlreadyBuilt
コード例 #27
0
ファイル: PackageUtils.py プロジェクト: cybernetics/photon-1
    def findSourceRPMFile(self, package,version="*"):
        if version == "*":
                version = SPECS.getData().getHighestVersion(package)
        release = SPECS.getData().getRelease(package, version)
        filename= package + "-" + version + "-" + release + "src.rpm"

        fullpath = constants.sourceRpmPath + "/" + filename
        if os.path.isfile(fullpath):
            return fullpath
        return None
コード例 #28
0
    def _findBuildTimeRequiredPackages(self, arch):
        deps = SPECS.getData(arch).getBuildRequiresForPackage(
            self.package, self.version)

        # Add BuildRequiresNative list
        if constants.crossCompiling and arch == constants.buildArch:
            deps.extend(
                SPECS.getData(arch).getBuildRequiresNativeForPackage(
                    self.package, self.version))

        return deps
コード例 #29
0
    def _getBuildRequiredPackages(package):
        listRequiredRPMPackages = []
        listRequiredRPMPackages.extend(SPECS.getData().getBuildRequiresForPackage(package))

        listRequiredPackages = []

        for pkg in listRequiredRPMPackages:
            basePkg = SPECS.getData().getSpecName(pkg.package)
            if basePkg not in listRequiredPackages:
                listRequiredPackages.append(basePkg)

        return listRequiredPackages
コード例 #30
0
ファイル: SpecDeps.py プロジェクト: transcendtron/photon
 def findTotalWhoNeeds(self, depList, whoNeeds):
     while depList:
         pkg = depList.pop(0)
         for depPackage in SPECS.getData().getListPackages():
             for version in SPECS.getData().getVersions(depPackage):
                 depBasePkg = depPackage+"-"+version
                 if depBasePkg in whoNeeds:
                     continue
                 if pkg in self.getBasePackagesRequired(depBasePkg):
                     whoNeeds.append(depBasePkg)
                     if depBasePkg not in depList:
                         depList.append(depBasePkg)
コード例 #31
0
ファイル: Scheduler.py プロジェクト: transcendtron/photon
    def _getBuildRequiredPackages(pkg):
        listRequiredRPMPackages = []
        listRequiredRPMPackages.extend(SPECS.getData().getBuildRequiresForPkg(pkg))

        listRequiredPackages = []

        for reqPkg in listRequiredRPMPackages:
            basePkg = SPECS.getData().getBasePkg(reqPkg)
            if basePkg not in listRequiredPackages:
                listRequiredPackages.append(basePkg)

        return listRequiredPackages
コード例 #32
0
ファイル: SpecDeps.py プロジェクト: frapposelli/photon
 def findTotalWhoNeeds(self, depList, whoNeeds):
     while depList:
         pkg = depList.pop(0)
         for depPackage in SPECS.getData().getListPackages():
             for version in SPECS.getData().getVersions(depPackage):
                 depBasePkg = depPackage+"-"+version
                 if depBasePkg in whoNeeds:
                     continue
                 if pkg in self.getBasePackagesRequired(depBasePkg):
                     whoNeeds.append(depBasePkg)
                     if depBasePkg not in depList:
                         depList.append(depBasePkg)
コード例 #33
0
    def process(self, inputType, inputValue, displayOption, outputFile=None):
        whoNeedsList = []
        inputPackages = []
        whatNeedsBuild = []
        mapDependencies = {}
        parent = {}
        if inputType == "pkg" or inputType == "json":
            if inputType == "pkg":
                inputPackages.append(inputValue)
            else:
                inputPackages = self.getAllPackageNames(inputValue)
            self.calculateSpecDependency(inputPackages, mapDependencies, parent)
            if outputFile is not None:
                return self.displayDependencies(displayOption, inputType, outputFile, mapDependencies, parent)
            else:
                return self.displayDependencies(displayOption, inputType, inputValue, mapDependencies, parent)
        elif inputType == "get-upward-deps":
            depList = []
            for specFile in inputValue.split(":"):
                if specFile in SPECS.getData().mapSpecFileNameToSpecObj:
                    specObj = SPECS.getData().mapSpecFileNameToSpecObj[specFile]
                    whoNeedsList.append(specObj.name+"-"+specObj.version)
                    depList.append(specObj.name+"-"+specObj.version)
            self.findTotalWhoNeeds(depList, whoNeedsList)
            return whoNeedsList

        elif inputType == "who-needs":
            for depPackage in SPECS.getData().mapPackageToSpec:
                pkg=inputValue+"-"+SPECS.getData().getHighestVersion(inputValue)
                for version in SPECS.getData().getVersions(depPackage):
                    depPkg = depPackage+"-"+version
                    self.logger.info(depPkg)
                    if pkg in SPECS.getData().getRequiresForPkg(depPkg):
                        whoNeedsList.append(depPkg)
            self.logger.info(whoNeedsList)
            return whoNeedsList

        elif inputType == "all-requires":
            pkg=inputValue+"-"+SPECS.getData().getHighestVersion(inputValue)
            requires = SPECS.getData().getRequiresTreeOfBasePkgsForPkg(pkg)
            requires.sort()
            self.logger.info(requires)
            return requires

        elif inputType == "is-toolchain-pkg":
            for specFile in inputValue.split(":"):
                if specFile in SPECS.getData().mapSpecFileNameToSpecObj:
                    specObj = SPECS.getData().mapSpecFileNameToSpecObj[specFile]
                    if (specObj.name in constants.listCoreToolChainPackages) \
                        or (specObj.name in constants.listToolChainPackages):
                        return True
            return False
コード例 #34
0
ファイル: PackageUtils.py プロジェクト: cybernetics/photon-1
    def findDebugRPMFile(self, package,version="*",arch=None):
        if not arch:
            arch=constants.currentArch

        if version == "*":
                version = SPECS.getData(arch).getHighestVersion(package)
        release = SPECS.getData(arch).getRelease(package, version)
        filename= package + "-debuginfo-" + version + "-" + release + "." + arch +".rpm"

        fullpath = constants.rpmPath + "/" + arch + "/" + filename
        if os.path.isfile(fullpath):
            return fullpath
        return None
コード例 #35
0
    def _buildGivenPackages(self, listPackages, buildThreads):
        # Extend listPackages from ["name1", "name2",..] to ["name1-vers1", "name2-vers2",..]
        listPackageNamesAndVersions=set()
        for pkg in listPackages:
            base = SPECS.getData().getSpecName(pkg)
            for version in SPECS.getData().getVersions(base):
                listPackageNamesAndVersions.add(base+"-"+version)

        returnVal = self._calculateParams(listPackageNamesAndVersions)
        if not returnVal:
            self.logger.error("Unable to set parameters. Terminating the package manager.")
            raise Exception("Unable to set parameters")
        self._buildPackages(buildThreads)
コード例 #36
0
ファイル: SpecDeps.py プロジェクト: frapposelli/photon
 def calculateSpecDependency(self, inputPackages, mapDependencies, parent):
     depQue = queue.Queue()
     for package in inputPackages:
         if SPECS.getData().isRPMPackage(package):
             version = SPECS.getData().getHighestVersion(package)
             pkg = package+"-"+version
             if pkg not in mapDependencies:
                 mapDependencies[pkg] = 0
                 parent[pkg] = ""
                 depQue.put(pkg)
                 self.findTotalRequires(mapDependencies, depQue, parent)
         else:
             self.logger.info("Could not find spec for " + package)
コード例 #37
0
ファイル: SpecDeps.py プロジェクト: transcendtron/photon
 def calculateSpecDependency(self, inputPackages, mapDependencies, parent):
     depQue = queue.Queue()
     for package in inputPackages:
         if SPECS.getData().isRPMPackage(package):
             for version in SPECS.getData().getVersions(package):
                 pkg = package+"-"+version
                 if pkg not in mapDependencies:
                     mapDependencies[pkg] = 0
                     parent[pkg] = ""
                     depQue.put(pkg)
                     self.findTotalRequires(mapDependencies, depQue, parent)
         else:
             self.logger.info("Could not find spec for " + package)
コード例 #38
0
ファイル: ToolChainUtils.py プロジェクト: jadeiceman/photon
    def buildCoreToolChainPackages(self):
        self.logger.info("Step 1 : Building the core toolchain packages for " + constants.currentArch)
        self.logger.info(constants.listCoreToolChainPackages)
        self.logger.info("")
        chroot = None
        pkgCount = 0
        try:
            pkgUtils = PackageUtils(self.logName, self.logPath)
            coreToolChainYetToBuild = []
            doneList = []
            #self.logger.debug(">>>> Looking for available packages...")
            for package in constants.listCoreToolChainPackages:
                version = SPECS.getData().getHighestVersion(package)
                rpmPkg = pkgUtils.findRPMFile(package, version)
                #rpmPkg = pkgUtils.findRPMFile(package, version, constants.buildArch)
                if rpmPkg is not None:
                    doneList.append(package+'-'+version)
                    #self.logger.debug(">>>>>> %s", package+'-'+version)
                    continue
                else:
                    coreToolChainYetToBuild.append(package)
            if coreToolChainYetToBuild:
                self.logger.info("The following core toolchain packages need to be built :")
                self.logger.info(coreToolChainYetToBuild)
            else:
                self.logger.info("Core toolchain packages are already available")

            for package in coreToolChainYetToBuild:
                self.logger.debug("Building core toolchain package : " + package)
                version = SPECS.getData().getHighestVersion(package)
                destLogPath = constants.logPath + "/" + package + "-" + version + "." + constants.currentArch
                if not os.path.isdir(destLogPath):
                    CommandUtils.runCommandInShell("mkdir -p " + destLogPath)
                chroot = Chroot(self.logger)
                chroot.create(package + "-" + version)
                #self.logger.debug(">>>>> Done List:")
                #self.logger.info(doneList)
                self.installToolchainRPMS(chroot, package, version, availablePackages=doneList)
                pkgUtils.adjustGCCSpecs(chroot, package, version)
                pkgUtils.buildRPMSForGivenPackage(chroot, package, version, destLogPath)
                pkgCount += 1
                chroot.destroy()
                doneList.append(package+'-'+version)
            self.logger.debug("Successfully built toolchain")
            self.logger.info("-" * 45 + "\n")
        except Exception as e:
            self.logger.error("Unable to build toolchain.")
            # print stacktrace
            traceback.print_exc()
            raise e
        return pkgCount
コード例 #39
0
ファイル: GenerateOSSFiles.py プロジェクト: vmware/photon
def buildSRPMList(srpmPath, yamlDir, blackListPkgs, dist_tag, logger, singleFile=True):
    cmdUtils = CommandUtils()
    yamlSrpmDir = os.path.join(yamlDir, "yaml_srpms")
    if not os.path.isdir(yamlSrpmDir):
        cmdUtils.runCommandInShell("mkdir -p " + yamlSrpmDir)
    if singleFile:
        yamlFile = open(yamlSrpmDir + "/srpm_list.yaml", "w")
    listPackages = SPECS.getData().getListPackages()
    listPackages.sort()
    for package in listPackages:
        if package in blackListPkgs:
            continue
        ossname = package
        for ossversion in SPECS.getData().getVersions(package):
            ossrelease = SPECS.getData().getRelease(package, ossversion)
            srpm_file_name = "%s-%s-%s%s.src.rpm" % (ossname, ossversion, ossrelease, dist_tag)
            logger.info("srpm name is %s" % (srpm_file_name))
            listFoundSRPMFiles = cmdUtils.findFile(srpm_file_name, srpmPath)

            srpmName = None
            if len(listFoundSRPMFiles) == 1:
                srpmFullPath = listFoundSRPMFiles[0]
                srpmName = os.path.basename(srpmFullPath)
                cpcmd = "cp " + srpmFullPath + " " + yamlSrpmDir + "/"
                returnVal = cmdUtils.runCommandInShell(cpcmd)
                if returnVal != 0:
                    logger.error("Copy SRPM File is failed for package:" + ossname)
            else:
                logger.error("SRPM file is not found:" + ossname)

            if not singleFile:
                yamlFile = open(yamlSrpmDir + "/" + ossname + "-" + ossversion + "-"
                                + ossrelease + ".yaml", "w")

            yamlFile.write("baseos:" + ossname + ":" + ossversion + "-" + ossrelease + dist_tag +  ":\n")
            yamlFile.write("  repository: BaseOS\n")
            yamlFile.write("  name: '" + ossname + "'\n")
            yamlFile.write("  version: '" + ossversion + "-" + ossrelease + dist_tag +"'\n")
            yamlFile.write("  url: 'http://www.vmware.com'\n")
            yamlFile.write("  baseos-style: rpm\n")
            yamlFile.write("  baseos-source: '" + str(srpmName) + "'\n")
            yamlFile.write("  baseos-osname: 'photon'\n")
            yamlFile.write("\n")
            if not singleFile:
                yamlFile.close()

    if singleFile:
        yamlFile.close()
    logger.debug("Generated SRPM yaml files for all packages")
コード例 #40
0
ファイル: PackageManager.py プロジェクト: TiejunChina/photon
    def _calculateParams(self, listPackages):
        self.mapCyclesToPackageList.clear()
        self.mapPackageToCycle.clear()
        self.sortedPackageList = []

        self.listOfPackagesAlreadyBuilt = self._readAlreadyAvailablePackages()

        updateBuiltRPMSList = False
        while not updateBuiltRPMSList:
            updateBuiltRPMSList = True
            listOfPackagesAlreadyBuilt = list(self.listOfPackagesAlreadyBuilt)
            for pkg in listOfPackagesAlreadyBuilt:
                listDependentRpmPackages = SPECS.getData().getRequiresAllForPackage(pkg)
                needToRebuild = False
                for dependentPkg in listDependentRpmPackages:
                    if dependentPkg not in self.listOfPackagesAlreadyBuilt:
                        needToRebuild = True
                        updateBuiltRPMSList = False
                if needToRebuild:
                    self.listOfPackagesAlreadyBuilt.remove(pkg)

        listPackagesToBuild = copy.copy(listPackages)
        for pkg in listPackages:
            if (pkg in self.listOfPackagesAlreadyBuilt and
                    not constants.rpmCheck):
                listPackagesToBuild.remove(pkg)

        if not self._readPackageBuildData(listPackagesToBuild):
            return False
        return True
コード例 #41
0
    def adjustGCCSpecs(self, package, version, chrootID, logPath):
        opt = " " + SPECS.getData().getSecurityHardeningOption(
            package, version)
        cmdUtils = CommandUtils()
        cpcmd = ("cp " + self.adjustGCCSpecScript + " " + chrootID + "/tmp/" +
                 self.adjustGCCSpecScript)
        cmd = "/tmp/" + self.adjustGCCSpecScript + opt
        logFile = logPath + "/adjustGCCSpecScript.log"
        chrootCmd = self.runInChrootCommand + " " + chrootID
        returnVal = cmdUtils.runCommandInShell(cpcmd, logFile)
        if not returnVal:
            self.logger.error("Error during copying the file adjust gcc spec")
            raise Exception("Failed while copying adjust gcc spec file")
        returnVal = cmdUtils.runCommandInShell(cmd, logFile, chrootCmd)
        if returnVal:
            return

        self.logger.debug(
            cmdUtils.runCommandInShell2("ls -la " + chrootID + "/tmp/" +
                                        self.adjustGCCSpecScript))
        self.logger.debug(
            cmdUtils.runCommandInShell2("lsof " + chrootID + "/tmp/" +
                                        self.adjustGCCSpecScript))
        self.logger.debug(cmdUtils.runCommandInShell2("ps ax"))

        self.logger.error("Failed while adjusting gcc specs")
        raise Exception("Failed while adjusting gcc specs")
コード例 #42
0
ファイル: PackageManager.py プロジェクト: frapposelli/photon
    def _buildGivenPackages(self, listPackages, buildThreads):
        # Extend listPackages from ["name1", "name2",..] to ["name1-vers1", "name2-vers2",..]
        listPackageNamesAndVersions=set()
        for pkg in listPackages:
            base = SPECS.getData().getSpecName(pkg)
            for version in SPECS.getData().getVersions(base):
                listPackageNamesAndVersions.add(base+"-"+version)

        returnVal = self._calculateParams(listPackageNamesAndVersions)
        if not returnVal:
            self.logger.error("Unable to set parameters. Terminating the package manager.")
            raise Exception("Unable to set parameters")

        statusEvent = threading.Event()
        self._initializeScheduler(statusEvent)
        self._initializeThreadPool(statusEvent)

        for i in range(0, buildThreads):
            workerName = "WorkerThread" + str(i)
            ThreadPool.addWorkerThread(workerName)
            ThreadPool.startWorkerThread(workerName)

        statusEvent.wait()
        Scheduler.stopScheduling = True
        self.logger.debug("Waiting for all remaining worker threads")
        ThreadPool.join_all()

        setFailFlag = False
        allPackagesBuilt = False
        if Scheduler.isAnyPackagesFailedToBuild():
            setFailFlag = True

        if Scheduler.isAllPackagesBuilt():
            allPackagesBuilt = True

        if setFailFlag:
            self.logger.error("Some of the packages failed:")
            self.logger.error(Scheduler.listOfFailedPackages)
            raise Exception("Failed during building package")

        if not setFailFlag:
            if allPackagesBuilt:
                self.logger.debug("All packages built successfully")
            else:
                self.logger.error("Build stopped unexpectedly.Unknown error.")
                raise Exception("Unknown error")
コード例 #43
0
ファイル: PackageManager.py プロジェクト: frapposelli/photon
    def _readAlreadyAvailablePackages(self):
        listAvailablePackages = set()
        pkgUtils = PackageUtils(self.logName, self.logPath)
        listPackages = SPECS.getData().getListPackages()
        for package in listPackages:
            for version in SPECS.getData().getVersions(package):
                # Mark package available only if all subpackages are available
                packageIsAlreadyBuilt=True
                listRPMPackages = SPECS.getData().getRPMPackages(package, version)
                for rpmPkg in listRPMPackages:
                    if pkgUtils.findRPMFile(rpmPkg, version) is None:
                        packageIsAlreadyBuilt=False
                        break;
                if packageIsAlreadyBuilt:
                    listAvailablePackages.add(package+"-"+version)

        return listAvailablePackages
コード例 #44
0
ファイル: Scheduler.py プロジェクト: frapposelli/photon
    def __getRequiredTypePackages(pkg, requiresType):
        listRPMPackages = []
        if requiresType == "build":
            listRPMPackages.extend(SPECS.getData().getBuildRequiresForPkg(pkg))
        elif requiresType == "install":
            listRPMPackages.extend(SPECS.getData().getRequiresAllForPkg(pkg))

        # Remove duplicates.
        listRPMPackages = list(set(listRPMPackages))

        listPackages = set()

        for reqPkg in listRPMPackages:
            basePkg = SPECS.getData().getBasePkg(reqPkg)
            listPackages.add(basePkg)

        return list(listPackages)
コード例 #45
0
ファイル: PackageUtils.py プロジェクト: frapposelli/photon
    def findRPMFile(self, package,version="*"):
        cmdUtils = CommandUtils()
        if version == "*":
                version = SPECS.getData().getHighestVersion(package)
        release = SPECS.getData().getRelease(package, version)
        buildarch=SPECS.getData().getBuildArch(package, version)
        filename= package + "-" + version + "-" + release + "." + buildarch+".rpm"

        fullpath = constants.rpmPath + "/" + buildarch + "/" + filename
        if os.path.isfile(fullpath):
            return fullpath

        if constants.inputRPMSPath is not None:
            fullpath = constants.inputRPMSPath + "/" + buildarch + "/" + filename
        if os.path.isfile(fullpath):
            return fullpath

        return None
コード例 #46
0
ファイル: PackageManager.py プロジェクト: TiejunChina/photon
    def _readAlreadyAvailablePackages(self):
        listAvailablePackages = set()
        listFoundRPMPackages = set()
        listRPMFiles = set()
        listDirectorys = set()
        listDirectorys.add(constants.rpmPath)
        if constants.inputRPMSPath is not None:
            listDirectorys.add(constants.inputRPMSPath)

        while listDirectorys:
            dirPath = listDirectorys.pop()
            for dirEntry in os.listdir(dirPath):
                dirEntryPath = os.path.join(dirPath, dirEntry)
                if os.path.isfile(dirEntryPath) and dirEntryPath.endswith(".rpm"):
                    listRPMFiles.add(dirEntryPath)
                elif os.path.isdir(dirEntryPath):
                    listDirectorys.add(dirEntryPath)
        pkgUtils = PackageUtils(self.logName, self.logPath)
        for rpmfile in listRPMFiles:
            package, version, release = pkgUtils.findPackageInfoFromRPMFile(rpmfile)
            if SPECS.getData().isRPMPackage(package):
                specVersion = SPECS.getData().getVersion(package)
                specRelease = SPECS.getData().getRelease(package)
                if version == specVersion and release == specRelease:
                    listFoundRPMPackages.add(package)
        #Mark package available only if all sub packages are available
        for package in listFoundRPMPackages:
            basePkg = SPECS.getData().getSpecName(package)
            if basePkg in listAvailablePackages:
                continue
            listRPMPackages = SPECS.getData().getRPMPackages(basePkg)
            packageIsAlreadyBuilt = True
            for rpmpkg in listRPMPackages:
                if rpmpkg not in listFoundRPMPackages:
                    packageIsAlreadyBuilt = False
            if packageIsAlreadyBuilt:
                listAvailablePackages.add(package)
        self.logger.info("List of Already built packages")
        self.logger.info(listAvailablePackages)
        return listAvailablePackages
コード例 #47
0
ファイル: GenerateOSSFiles.py プロジェクト: vmware/photon
def buildSourcesList(yamlDir, blackListPkgs, logger, singleFile=True):
    cmdUtils = CommandUtils()
    yamlSourceDir = os.path.join(yamlDir, "yaml_sources")
    if not os.path.isdir(yamlSourceDir):
        cmdUtils.runCommandInShell("mkdir -p " + yamlSourceDir)
    if singleFile:
        yamlFile = open(yamlSourceDir + "/sources_list.yaml", "w")
    listPackages = SPECS.getData().getListPackages()
    listPackages.sort()
    import PullSources
    for package in listPackages:
        if package in blackListPkgs:
            continue
        ossname = package
        for version in SPECS.getData().getVersions(package):
            modified = False
            listPatches = SPECS.getData().getPatches(package, version)
            if listPatches:
                modified = True
            url = SPECS.getData().getSourceURL(package, version)
            if url is None:
                url = SPECS.getData().getURL(package, version)

            sourceName = None
            listSourceNames = SPECS.getData().getSources(package, version)
            if listSourceNames:
                sourceName = listSourceNames[0]
                sha1 = SPECS.getData().getSHA1(package, version, sourceName)
                if sha1 is not None:
                    PullSources.get(package, sourceName, sha1, yamlSourceDir,
                                    constants.getPullSourcesURLs(package),
                                    logger)

            if not singleFile:
                yamlFile = open(yamlSourceDir + "/" + ossname + "-" + version + ".yaml", "w")
            yamlFile.write("vmwsource:" + ossname + ":" + version + ":\n")
            yamlFile.write("  repository: VMWsource\n")
            yamlFile.write("  name: '" + ossname + "'\n")
            yamlFile.write("  version: '" + version + "'\n")
            yamlFile.write("  url: " + str(url) + "\n")
            yamlFile.write("  license: UNKNOWN\n")
            if sourceName is not None:
                yamlFile.write("  vmwsource-distribution: " + str(sourceName) + "\n")
            if modified:
                yamlFile.write("  modified: true\n")
            yamlFile.write("\n")
            if not singleFile:
                yamlFile.close()

    if singleFile:
        yamlFile.close()
    logger.debug("Generated source yaml files for all packages")
コード例 #48
0
ファイル: SpecDeps.py プロジェクト: frapposelli/photon
    def process(self, inputType, inputValue, displayOption, outputFile=None):
        whoNeedsList = []
        inputPackages = []
        whatNeedsBuild = []
        mapDependencies = {}
        parent = {}
        if inputType == "pkg" or inputType == "json":
            if inputType == "pkg":
                inputPackages.append(inputValue)
            else:
                inputPackages = self.getAllPackageNames(inputValue)
            self.calculateSpecDependency(inputPackages, mapDependencies, parent)
            if outputFile is not None:
                return self.displayDependencies(displayOption, inputType, outputFile, mapDependencies, parent)
            else:
                return self.displayDependencies(displayOption, inputType, inputValue, mapDependencies, parent)
        elif inputType == "get-upward-deps":
            depList = []
            for specFile in inputValue.split(":"):
                if specFile in SPECS.getData().mapSpecFileNameToSpecObj:
                    specObj = SPECS.getData().mapSpecFileNameToSpecObj[specFile]
                    whoNeedsList.append(specObj.name+"-"+specObj.version)
                    depList.append(specObj.name+"-"+specObj.version)
            self.findTotalWhoNeeds(depList, whoNeedsList)
            return whoNeedsList

        elif inputType == "who-needs":
            for depPackage in SPECS.getData().mapPackageToSpec:
                pkg=inputValue+"-"+SPECS.getData().getHighestVersion(inputValue)
                for version in SPECS.getData().getVersions(depPackage):
                    depPkg = depPackage+"-"+version
                    self.logger.info(depPkg)
                    if pkg in SPECS.getData().getRequiresForPkg(depPkg):
                        whoNeedsList.append(depPkg)
            self.logger.info(whoNeedsList)
            return whoNeedsList

        elif inputType == "is-toolchain-pkg":
            for specFile in inputValue.split(":"):
                if specFile in SPECS.getData().mapSpecFileNameToSpecObj:
                    specObj = SPECS.getData().mapSpecFileNameToSpecObj[specFile]
                    if (specObj.name in constants.listCoreToolChainPackages) \
                        or (specObj.name in constants.listToolChainPackages):
                        return True
            return False
コード例 #49
0
ファイル: PackageUtils.py プロジェクト: frapposelli/photon
    def adjustGCCSpecs(self, sandbox, package, version):
        opt = " " + SPECS.getData().getSecurityHardeningOption(package, version)
        sandbox.put(self.adjustGCCSpecScript, "/tmp")
        cmd = "/tmp/" + self.adjustGCCSpecScript + opt
        returnVal = sandbox.run(cmd, logfn=self.logger.debug)
        if returnVal == 0:
            return

        # in debugging ...
        sandbox.run("ls -la /tmp/" + self.adjustGCCSpecScript,
                    logfn=self.logger.debug)
        sandbox.run("lsof /tmp/" + self.adjustGCCSpecScript,
                    logfn=self.logger.debug)
        sandbox.run("ps ax", logfn=self.logger.debug)

        self.logger.error("Failed while adjusting gcc specs")
        raise Exception("Failed while adjusting gcc specs")
コード例 #50
0
ファイル: PackageUtils.py プロジェクト: TiejunChina/photon
    def adjustGCCSpecsInContainer(self, package, containerID, logPath, index=0):
        opt = " " + SPECS.getData().getSecurityHardeningOption(package, index)
        adjustCmd = "/" + self.adjustGCCSpecScript + opt
        adjustCmd = "/bin/bash -l -c '" + adjustCmd + "'"
        logFile = logPath + "/adjustGCCSpecScript.log"

        #TODO: Error code from exec_run
        scriptLog = containerID.exec_run(adjustCmd)
        if scriptLog:
            with open(logFile, 'w') as logfile:
                logfile.write(scriptLog.decode())
            return

        self.logger.debug(containerID.exec_run("ls -la /tmp/" + self.adjustGCCSpecScript))
        self.logger.debug(containerID.exec_run("lsof /tmp/" + self.adjustGCCSpecScript))
        self.logger.debug(containerID.exec_run("ps ax"))
        self.logger.error("Failed while adjusting gcc specs")
        raise Exception("Failed while adjusting gcc specs")
コード例 #51
0
ファイル: PackageUtils.py プロジェクト: frapposelli/photon
    def buildRPMSForGivenPackage(self, sandbox, package, version, destLogPath):
        self.logger.info("Building package : " + package)

        listSourcesFiles = SPECS.getData().getSources(package, version)
        listPatchFiles = SPECS.getData().getPatches(package, version)
        specFile = SPECS.getData().getSpecFile(package, version)
        specName = SPECS.getData().getSpecName(package) + ".spec"
        sourcePath = constants.topDirPath + "/SOURCES/"
        specPath = constants.topDirPath + "/SPECS/"
        if (constants.rpmCheck and
                package in constants.testForceRPMS and
                SPECS.getData().isCheckAvailable(package, version)):
            logFilePath = destLogPath + "/" + package + "-test.log"
        else:
            logFilePath = destLogPath + "/" + package + ".log"
        sandbox.put(specFile, specPath + specName)

        sources_urls, macros = self._getAdditionalBuildOptions(package)
        self.logger.debug("Extra macros for " + package + ": " + str(macros))
        self.logger.debug("Extra source URLs for " + package + ": " + str(sources_urls))
        constants.setExtraSourcesURLs(package, sources_urls)

        self._copySources(sandbox, listSourcesFiles, package, version, sourcePath)
        self._copySources(sandbox, listPatchFiles, package, version, sourcePath)

        #Adding rpm macros
        listRPMMacros = constants.userDefinedMacros
        for macroName, value in listRPMMacros.items():
            macros.append(macroName + " " + value)

        listRPMFiles = []
        listSRPMFiles = []
        try:
            listRPMFiles, listSRPMFiles = self._buildRPM(sandbox, specPath + specName,
                                                         logFilePath, package, version, macros)
            logmsg = package + " build done - RPMs : [ "
            for f in listRPMFiles:
                logmsg += (os.path.basename(f) + " ")
            logmsg += "]\n"
            self.logger.info(logmsg)
        except Exception as e:
            self.logger.error("Failed while building rpm:" + package)
            raise e
        finally:
            if (constants.rpmCheck and
                    package in constants.testForceRPMS and
                    SPECS.getData().isCheckAvailable(package, version)):
                cmd = ("sed -i '/^Executing(%check):/,/^Processing files:/{//!b};d' " + logFilePath)
                CommandUtils().runCommandInShell(cmd, logfn=self.logger.debug)
        self.logger.debug("RPM build is successful")
コード例 #52
0
ファイル: PackageBuilder.py プロジェクト: frapposelli/photon
    def _findDependentPackagesAndInstalledRPM(self, sandbox):
        listInstalledPackages, listInstalledRPMs = self._findInstalledPackages(sandbox)
        self.logger.debug(listInstalledPackages)
        listDependentPackages = self._findBuildTimeRequiredPackages()
        listTestPackages=[]
        if constants.rpmCheck and self.package in constants.testForceRPMS:
            # One time optimization
            if constants.listMakeCheckRPMPkgWithVersionstoInstall is None:
                constants.listMakeCheckRPMPkgWithVersionstoInstall=[]
                for package in constants.listMakeCheckRPMPkgtoInstall:
                    version = SPECS.getData().getHighestVersion(package)
                    constants.listMakeCheckRPMPkgWithVersionstoInstall.append(package+"-"+version)

            listDependentPackages.extend(self._findBuildTimeCheckRequiredPackages())
            testPackages = (set(constants.listMakeCheckRPMPkgWithVersionstoInstall) -
                            set(listInstalledPackages) -
                            set([self.package+"-"+self.version]))
            listTestPackages=list(set(testPackages))
            listDependentPackages = list(set(listDependentPackages))
        return listDependentPackages, listTestPackages, listInstalledPackages, listInstalledRPMs
コード例 #53
0
ファイル: SpecDeps.py プロジェクト: frapposelli/photon
    def findTotalRequires(self, mapDependencies, depQue, parent):
        while not depQue.empty():
            specPkg = depQue.get()
            try:
                listRequiredPackages = SPECS.getData().getRequiresForPkg(specPkg)
            except Exception as e:
                self.logger.info("Caught Exception:"+str(e))
                self.logger.info(specPkg + " is missing")
                raise e

            for depPkg in listRequiredPackages:
                if depPkg in mapDependencies:
                    if mapDependencies[depPkg] < mapDependencies[specPkg] + 1:
                        mapDependencies[depPkg] = mapDependencies[specPkg] + 1
                        parent[depPkg] = specPkg
                        self.updateLevels(mapDependencies, depPkg, parent, mapDependencies[depPkg])
                else:
                    mapDependencies[depPkg] = mapDependencies[specPkg] + 1
                    parent[depPkg] = specPkg
                    depQue.put(depPkg)
コード例 #54
0
ファイル: PackageUtils.py プロジェクト: TiejunChina/photon
    def adjustGCCSpecs(self, package, chrootID, logPath, index=0):
        opt = " " + SPECS.getData().getSecurityHardeningOption(package, index)
        cmdUtils = CommandUtils()
        cpcmd = ("cp " + self.adjustGCCSpecScript + " " + chrootID +
                 "/tmp/" + self.adjustGCCSpecScript)
        cmd = "/tmp/" + self.adjustGCCSpecScript + opt
        logFile = logPath + "/adjustGCCSpecScript.log"
        chrootCmd = self.runInChrootCommand + " " + chrootID
        returnVal = cmdUtils.runCommandInShell(cpcmd, logFile)
        if not returnVal:
            self.logger.error("Error during copying the file adjust gcc spec")
            raise Exception("Failed while copying adjust gcc spec file")
        returnVal = cmdUtils.runCommandInShell(cmd, logFile, chrootCmd)
        if returnVal:
            return

        self.logger.debug(cmdUtils.runCommandInShell2("ls -la " + chrootID +
                                                      "/tmp/" + self.adjustGCCSpecScript))
        self.logger.debug(cmdUtils.runCommandInShell2("lsof " + chrootID + "/tmp/" +
                                                      self.adjustGCCSpecScript))
        self.logger.debug(cmdUtils.runCommandInShell2("ps ax"))

        self.logger.error("Failed while adjusting gcc specs")
        raise Exception("Failed while adjusting gcc specs")
コード例 #55
0
ファイル: GenerateOSSFiles.py プロジェクト: vmware/photon
def buildPackagesList(csvFilename):
    with open(csvFilename, "w") as csvFile:
        csvFile.write("Package,Version,License,URL,Sources,Patches\n")
        listPackages = SPECS.getData().getListPackages()
        listPackages.sort()
        for package in listPackages:
            name = package
            for version in SPECS.getData().getVersions(package):
                packagelicense = SPECS.getData().getLicense(package, version)
                listPatches = SPECS.getData().getPatches(package, version)
                url = SPECS.getData().getURL(package, version)
                listSourceNames = SPECS.getData().getSources(package, version)
                sources = ""
                patches = ""
                if listPatches is not None:
                    patches = " ".join(listPatches)
                if listSourceNames is not None:
                    sources = " ".join(listSourceNames)
                csvFile.write(name + "," + version + "," + packagelicense + "," + url + "," +
                            sources + "," + patches + "\n")
コード例 #56
0
ファイル: builder.py プロジェクト: frapposelli/photon
def buildPackagesForAllSpecs(buildThreads, pkgBuildType, pkgInfoJsonFile, logger):
    listPackages = SPECS.getData().getListPackages()
    buildSpecifiedPackages(listPackages, buildThreads, pkgBuildType, pkgInfoJsonFile, logger)
コード例 #57
0
ファイル: PackageUtils.py プロジェクト: TiejunChina/photon
    def _buildRPMinContainer(self, specFile, rpmLogFile, destLogFile, containerID, package, macros):

        rpmBuildCmd = self.rpmbuildBinary + " " + self.rpmbuildBuildallOption

        if constants.rpmCheck and package in constants.testForceRPMS:
            self.logger.info("#" * (68 + 2 * len(package)))
            if not SPECS.getData().isCheckAvailable(package, index):
                self.logger.info("####### " + package +
                                 " MakeCheck is not available. Skipping MakeCheck TEST for " +
                                 package + " #######")
                rpmBuildCmd = self.rpmbuildBinary + " --clean"
            else:
                self.logger.info("####### " + package +
                                 " MakeCheck is available. Running MakeCheck TEST for " +
                                 package + " #######")
                rpmBuildCmd = self.rpmbuildBinary + " " + self.rpmbuildCheckOption
            self.logger.info("#" * (68 + 2 * len(package)))
        else:
            rpmBuildCmd += " "+self.rpmbuildNocheckOption

        for macro in macros:
            rpmBuildCmd += ' --define \"%s\"' % macro
        rpmBuildCmd += " " + specFile
        rpmBuildCmd = "/bin/bash -l -c '" + rpmBuildCmd + " > " + rpmLogFile + " 2>&1'"
        rpmBuildCmd = "docker exec -t " + str(containerID.short_id) + " " + rpmBuildCmd

        cmdUtils = CommandUtils()
        self.logger.info("Building rpm for package: " + package)
        #TODO: Show running log of rpmbuildcmd
        #TODO: Get exit status of rpmBuildCmd
        #containerID.exec_run(rpmBuildCmd)
        returnVal = cmdUtils.runCommandInShell(rpmBuildCmd)

        if not os.path.isfile(destLogFile):
            self.logger.error("RPM build not file not found. Building rpm failed for: " + specFile)
            raise Exception("RPM Build failed")

        if constants.rpmCheck and package in constants.testForceRPMS:
            if not SPECS.getData().isCheckAvailable(package, index):
                constants.testLogger.info(package + " : N/A")
            elif returnVal:
                constants.testLogger.info(package + " : PASS")
            else:
                constants.testLogger.error(package + " : FAIL")

        if constants.rpmCheck:
            if not returnVal and constants.rpmCheckStopOnError:
                self.logger.error("Checking rpm is failed " + specFile)
                raise Exception("RPM check failed")
        else:
            if not returnVal:
                self.logger.error("Building rpm is failed " + specFile)
                raise Exception("RPM build failed")

        #Extracting rpms created from log file
        listRPMFiles = []
        listSRPMFiles = []
        with open(destLogFile, 'r') as logfile:
            rpmBuildLogLines = logfile.readlines()
            for i in range(0, len(rpmBuildLogLines)):
                if re.search("^Wrote:", rpmBuildLogLines[i]):
                    listcontents = rpmBuildLogLines[i].split()
                    if ((len(listcontents) == 2) and
                            listcontents[1].strip().endswith(".rpm") and
                            "/RPMS/" in listcontents[1]):
                        listRPMFiles.append(listcontents[1])
                    if ((len(listcontents) == 2) and
                            listcontents[1].strip().endswith(".src.rpm") and
                            "/SRPMS/" in listcontents[1]):
                        listSRPMFiles.append(listcontents[1])
        #if not listRPMFiles:
        #    self.logger.error("Building rpm failed for " + specFile)
        #    raise Exception("RPM Build failed")
        return listRPMFiles, listSRPMFiles
コード例 #58
0
ファイル: PackageUtils.py プロジェクト: TiejunChina/photon
    def _buildRPM(self, specFile, logFile, chrootCmd, package, macros):

        rpmBuildcmd = self.rpmbuildBinary + " " + self.rpmbuildBuildallOption

        if constants.rpmCheck and package in constants.testForceRPMS:
            self.logger.info("#" * (68 + 2 * len(package)))
            if not SPECS.getData().isCheckAvailable(package):
                self.logger.info("####### " + package +
                                 " MakeCheck is not available. Skipping MakeCheck TEST for " +
                                 package + " #######")
                rpmBuildcmd = self.rpmbuildBinary + " --clean"
            else:
                self.logger.info("####### " + package +
                                 " MakeCheck is available. Running MakeCheck TEST for " +
                                 package + " #######")
                rpmBuildcmd = self.rpmbuildBinary + " " + self.rpmbuildCheckOption
            self.logger.info("#" * (68 + 2 * len(package)))
        else:
            rpmBuildcmd += " " + self.rpmbuildNocheckOption

        for macro in macros:
            rpmBuildcmd += ' --define \\\"%s\\\"' % macro
        rpmBuildcmd += " " + specFile

        cmdUtils = CommandUtils()
        self.logger.info("Building rpm....")
        self.logger.info(rpmBuildcmd)
        returnVal = cmdUtils.runCommandInShell(rpmBuildcmd, logFile, chrootCmd)
        if constants.rpmCheck and package in constants.testForceRPMS:
            if not SPECS.getData().isCheckAvailable(package):
                constants.testLogger.info(package + " : N/A")
            elif returnVal:
                constants.testLogger.info(package + " : PASS")
            else:
                constants.testLogger.error(package + " : FAIL")

        if constants.rpmCheck:
            if not returnVal and constants.rpmCheckStopOnError:
                self.logger.error("Checking rpm is failed " + specFile)
                raise Exception("RPM check failed")
        else:
            if not returnVal:
                self.logger.error("Building rpm is failed " + specFile)
                raise Exception("RPM build failed")

        #Extracting rpms created from log file
        listRPMFiles = []
        listSRPMFiles = []
        with open(logFile, 'r') as logfile:
            fileContents = logfile.readlines()
            for i in range(0, len(fileContents)):
                if re.search("^Wrote:", fileContents[i]):
                    listcontents = fileContents[i].split()
                    if ((len(listcontents) == 2) and
                            listcontents[1].strip().endswith(".rpm") and
                            "/RPMS/" in listcontents[1]):
                        listRPMFiles.append(listcontents[1])
                    if ((len(listcontents) == 2) and
                            listcontents[1].strip().endswith(".src.rpm") and
                            "/SRPMS/" in listcontents[1]):
                        listSRPMFiles.append(listcontents[1])
        return listRPMFiles, listSRPMFiles
コード例 #59
0
ファイル: PackageUtils.py プロジェクト: TiejunChina/photon
    def buildRPMSForGivenPackageInContainer(self, package, containerID, destLogPath=None, index=0):
        self.logger.info("Building rpm's for package " + package + " in container " +
                         containerID.short_id)

        listSourcesFiles = SPECS.getData().getSources(package, index)
        listPatchFiles = SPECS.getData().getPatches(package, index)
        specFile = SPECS.getData().getSpecFile(package, index)
        specName = SPECS.getData().getSpecName(package) + ".spec"
        sourcePath = constants.topDirPath + "/SOURCES/"
        specPath = constants.topDirPath + "/SPECS/"
        rpmLogFile = constants.topDirPath + "/LOGS/" + package + ".log"
        destLogFile = destLogPath + "/" + package + ".log"
        cmdUtils = CommandUtils()

        #TODO: mount it in, don't copy
        cpSpecCmd = "docker cp " + specFile + " " + containerID.short_id \
                        + ":" + specPath + specName
        returnVal = cmdUtils.runCommandInShell(cpSpecCmd)
        if not returnVal:
            self.logger.error("Error copying source SPEC file to container")
            raise Exception("Failed copying source SPEC to container")

        #FIXME: some sources are located in SPECS/.. how to mount?
        #        if os.geteuid()==0:
        #TODO: mount it in, don't copy
        macros = []
        self._copySourcesToContainer(listSourcesFiles, package, containerID, sourcePath, index)
        #TODO: mount it in, don't copy
        self._copySourcesToContainer(listPatchFiles, package, containerID, sourcePath, index)
        listAdditionalFiles, macros = self._getAdditionalBuildFiles(package)
        self._copyAdditionalBuildFilesToContainer(listAdditionalFiles, containerID)

        # Add rpm macros
        listRPMMacros = constants.userDefinedMacros
        for macroName, value in listRPMMacros.items():
            macros.append(macroName + " " + value)

        # Build RPMs
        listRPMFiles = []
        listSRPMFiles = []
        try:
            listRPMFiles, listSRPMFiles = self._buildRPMinContainer(
                specPath + specName,
                rpmLogFile,
                destLogFile,
                containerID,
                package,
                macros)
            self.logger.info("Successfully built rpm:" + package)
        except Exception as e:
            self.logger.error("Failed while building rpm: " + package)
            raise e
        finally:
            if destLogPath is not None:
                rpmLog = destLogPath + "/" + package + ".log"
                if (constants.rpmCheck and
                        package in constants.testForceRPMS and
                        SPECS.getData().isCheckAvailable(package, index)):
                    cmd = "sed -i '/^Executing(%check):/,/^Processing files:/{//!b};d' " + rpmLog
                    logFile = destLogPath + "/adjustTestFile.log"
                    returnVal = CommandUtils().runCommandInShell(cmd, logFile)
                    testLogFile = destLogPath + "/" + package + "-test.log"
                    shutil.copyfile(rpmLog, testLogFile)
        self.logger.info("RPM build is successful")

        # Verify RPM and SRPM files exist as success criteria
        for rpmFile in listRPMFiles:
            rpmName = os.path.basename(rpmFile)
            rpmDestDir = self._getRPMDestDir(rpmName, constants.rpmPath)
            rpmDestFile = rpmDestDir + "/" + rpmName
            if not os.path.isfile(rpmDestFile):
                self.logger.error("Could not find RPM file: " + rpmDestFile)
                raise Exception("Built RPM file not found.")

        for srpmFile in listSRPMFiles:
            srpmName = os.path.basename(srpmFile)
            srpmDestDir = self._getRPMDestDir(os.path.basename(srpmFile), constants.sourceRpmPath)
            srpmDestFile = srpmDestDir + "/" + srpmName
            if not os.path.isfile(srpmDestFile):
                self.logger.error("Could not find RPM file: " + srpmDestFile)
                raise Exception("Built SRPM file not found.")