示例#1
0
    def build(self):
        self.job.jobStarted("Starting Build %s (pid %s)" % (self.job.jobId,
                            os.getpid()), pid=os.getpid())
        # main loop is here.
        if not self.initialized:
            if not self.initializeBuild():
                return False

        self.job.jobBuilding('Building troves')
        if self.dh.moreToDo():
            while self.dh.moreToDo():
                self.worker.handleRequestIfReady()
                if self.worker._checkForResults():
                    self.resolveIfReady()
                elif self.dh.hasBuildableTroves():
                    trv, (buildReqs, crossReqs, bootstrapReqs
                            ) = self.dh.popBuildableTrove()
                    self.buildTrove(trv, buildReqs, crossReqs, bootstrapReqs)
                elif self.dh.hasSpecialTroves():
                    self.actOnTrove(self.dh.popSpecialTrove())
                elif not self.resolveIfReady():
                    time.sleep(0.1)
            if self.dh.jobPassed():
                self.job.jobPassed("build job finished successfully")
                return True
            msg = ['Build job had failures:\n']
            for trove in sorted(self.job.iterPrimaryFailureTroves()):
                err = trove.getFailureReason().getShortError()
                msg.append('   * %s: %s\n' % (trove.getName(), err))
            self.job.jobFailed(''.join(msg))
        else:
            msg = ['Did not find any buildable troves:\n']
            for trove in sorted(self.job.iterPrimaryFailureTroves()):
                err = trove.getFailureReason().getShortError()
                msg.append('   * %s: %s\n' % (trove.getName(), err))
            self.job.jobFailed(''.join(msg))
        return False
示例#2
0
 def troveDuplicate(self, trove, troveList):
     package = trove.getName().split(':')[0]
     possibleMatches = self.depState.getTrovesByPackage(package)
     for match in possibleMatches:
         if match is trove:
             continue
         elif match.getBinaryTroves() == set(troveList):
             self.depState.troveDuplicate(trove, match)
             return
         elif set(match.getBinaryTroves()) & set(troveList):
             trove.troveFailed('Two versions of %s=%s[%s] were built at the same time but resulted in different components.  If these packages should have different flavors, then add flavor information to them.  Otherwise, try building only one of them.' % trove.getNameVersionFlavor())
             return
         elif not match.getBinaryTroves() and match.isBuilding():
             # it's possible that the two results just came back in the 
             # wrong order
             self._possibleDuplicates.setdefault(match, []).append((trove, 
                                                              troveList))
             return
     trove.troveFailed('Package was committed at the same time as the same package was built in another job.  Make sure no-one else is building the same packages as you, and that you didn\'t accidentally build the same package twice with the same flavor.')
示例#3
0
 def troveDuplicate(self, trove, troveList):
     package = trove.getName().split(':')[0]
     possibleMatches = self.depState.getTrovesByPackage(package)
     for match in possibleMatches:
         if match is trove:
             continue
         elif match.getBinaryTroves() == set(troveList):
             self.depState.troveDuplicate(trove, match)
             return
         elif set(match.getBinaryTroves()) & set(troveList):
             trove.troveFailed('Two versions of %s=%s[%s] were built at the same time but resulted in different components.  If these packages should have different flavors, then add flavor information to them.  Otherwise, try building only one of them.' % trove.getNameVersionFlavor())
             return
         elif not match.getBinaryTroves() and match.isBuilding():
             # it's possible that the two results just came back in the 
             # wrong order
             self._possibleDuplicates.setdefault(match, []).append((trove, 
                                                              troveList))
             return
     trove.troveFailed('Package was committed at the same time as the same package was built in another job.  Make sure no-one else is building the same packages as you, and that you didn\'t accidentally build the same package twice with the same flavor.')
示例#4
0
 def startTroveLogger(self, trove):
     key = ''.join([ chr(random.randrange(ord('a'), ord('z'))) 
                   for x in range(10) ])
     r = recorder.BuildLogRecorder(key)
     r.attach(trove)
     logHost = r.getHost()
     logPort = r.getPort()
     trove.logPath = r.getLogPath()
     pid = self.worker._fork('BuildLogger for %s{%s}' % (trove.getName(),
         trove.getContext() or ''))
     if not pid:
         try:
             r.closeOtherFds()
             r._installSignalHandlers()
             r.serve_forever()
         finally:
             os._exit(3)
     else:
         r.close()
         trove.logPid = pid
     return logHost, logPort, key
示例#5
0
文件: display.py 项目: pombr/conary
    def formatInfo(self, trove):
        """ returns iterator of format lines about this local trove """
        # TODO: it'd be nice if this were set up to do arbitrary
        # formats...

        n, v, f = trove.getName(), trove.getVersion(), trove.getFlavor()
        dcfg = self.dcfg
        troveSource = dcfg.getTroveSource()

        sourceName = trove.getSourceName()
        sourceTrove = None

        if sourceName:
            try:
                sourceVer = v.getSourceVersion()
                if sourceVer.isOnLocalHost():
                    sourceVer = sourceVer.parentVersion()

                sourceTrove = troveSource.getTrove(
                    sourceName, sourceVer, deps.Flavor(), withFiles = False)
                # FIXME: all trove sources should return TroveMissing
                # on failed getTrove calls
            except errors.TroveMissing:
                pass
            except errors.InsufficientPermission:
                pass

        elif n.endswith(':source'):
            sourceTrove = trove

        if trove.getBuildTime():
            buildTime = time.strftime("%c",
                                time.localtime(trove.getBuildTime()))
        else:
            buildTime = "(unknown)"

        if trove.getSize() is not None:
            size = "%s" % trove.getSize()
        else:
            size = "(unknown)"

        yield "%-30s %s" % \
            (("Name      : %s" % trove.getName(),
             ("Build time: %s" % buildTime)))

        if dcfg.fullVersions:
            yield "Version   : %s" %v
            yield "Label     : %s" % v.branch().label().asString()
        else:
            yield "%-30s %s" % \
                (("Version   : %s" %
                            v.trailingRevision().asString()),
                 ("Label     : %s" %
                            v.branch().label().asString()))

        yield '%-30s' % ("Size      : %s" % size)
        if hasattr(troveSource, 'trovesArePinned'):
            yield "Pinned    : %s" % troveSource.trovesArePinned(
                                                            [ (n, v, f) ])[0]

        commitTime = v.trailingRevision().timeStamp
        if commitTime:
            commitTime = time.strftime("%c", time.localtime(commitTime))
        else:
            commitTime = '(unknown)'
        if not trove.getBuildTime() or log.getVerbosity() <= log.DEBUG:
            yield "Committed : " + commitTime
        yield "%-30s" % ("Flavor    : %s" % deps.formatFlavor(f))
        if trove.getInstallTime():
            yield 'Installed : %s' % (time.strftime("%c",
                time.localtime(trove.getInstallTime())))

        imageGroup = trove.troveInfo.imageGroup()
        if imageGroup is not None:
            yield 'Image Group: %s' % bool(imageGroup)

        for ln in self.formatMetadata(trove):
            yield ln
        if sourceTrove:
            if not n.endswith(':source'):
                yield 'Source    : %s' % trove.getSourceName()
            cl = sourceTrove.getChangeLog()
            if cl:
                yield "Change log: %s (%s)" % (cl.getName(), cl.getContact())
                lines = cl.getMessage().split("\n")[:-1]
                for l in lines:
                    yield "    " + l
        if log.getVerbosity() <= log.DEBUG:
            yield "%-30s %s" % (("Incomp.   : %s" %
                                 bool(trove.troveInfo.incomplete())),
                                ("TroveVer  : %s" %
                                            trove.troveInfo.troveVersion()))
            yield "%-30s" % (("Clone of  : %s" % trove.troveInfo.clonedFrom()))
            subPackages = set(trove.troveInfo.subPackages())
            subPackages.discard(trove.getName())
            if subPackages:
                yield "%-30s" % (("Siblings  : %s" % (' '.join(sorted(subPackages)))))
            yield "%-30s" % (("Conary version : %s" % trove.troveInfo.conaryVersion()))
示例#6
0
 def isTroveTarget(trove):
     if targets is None:
         return True
     return trove.getName() in targets
示例#7
0
    def formatInfo(self, trove):
        """ returns iterator of format lines about this local trove """
        # TODO: it'd be nice if this were set up to do arbitrary
        # formats...

        n, v, f = trove.getName(), trove.getVersion(), trove.getFlavor()
        dcfg = self.dcfg
        troveSource = dcfg.getTroveSource()

        sourceName = trove.getSourceName()
        sourceTrove = None

        if sourceName:
            try:
                sourceVer = v.getSourceVersion()
                if sourceVer.isOnLocalHost():
                    sourceVer = sourceVer.parentVersion()

                sourceTrove = troveSource.getTrove(sourceName,
                                                   sourceVer,
                                                   deps.Flavor(),
                                                   withFiles=False)
                # FIXME: all trove sources should return TroveMissing
                # on failed getTrove calls
            except errors.TroveMissing:
                pass
            except errors.InsufficientPermission:
                pass

        elif n.endswith(':source'):
            sourceTrove = trove

        if trove.getBuildTime():
            buildTime = time.strftime("%c",
                                      time.localtime(trove.getBuildTime()))
        else:
            buildTime = "(unknown)"

        if trove.getSize() is not None:
            size = "%s" % trove.getSize()
        else:
            size = "(unknown)"

        yield "%-30s %s" % \
            (("Name      : %s" % trove.getName(),
             ("Build time: %s" % buildTime)))

        if dcfg.fullVersions:
            yield "Version   : %s" % v
            yield "Label     : %s" % v.branch().label().asString()
        else:
            yield "%-30s %s" % \
                (("Version   : %s" %
                            v.trailingRevision().asString()),
                 ("Label     : %s" %
                            v.branch().label().asString()))

        yield '%-30s' % ("Size      : %s" % size)
        if hasattr(troveSource, 'trovesArePinned'):
            yield "Pinned    : %s" % troveSource.trovesArePinned([(n, v, f)
                                                                  ])[0]

        yield "%-30s" % ("Flavor    : %s" % deps.formatFlavor(f))
        if trove.getInstallTime():
            yield 'Installed : %s' % (time.strftime(
                "%c", time.localtime(trove.getInstallTime())))

        imageGroup = trove.troveInfo.imageGroup()
        if imageGroup is not None:
            yield 'Image Group: %s' % bool(imageGroup)

        for ln in self.formatMetadata(trove):
            yield ln
        if sourceTrove:
            if not n.endswith(':source'):
                yield 'Source    : %s' % trove.getSourceName()
            cl = sourceTrove.getChangeLog()
            if cl:
                yield "Change log: %s (%s)" % (cl.getName(), cl.getContact())
                lines = cl.getMessage().split("\n")[:-1]
                for l in lines:
                    yield "    " + l

        if log.getVerbosity() <= log.DEBUG:
            yield "%-30s %s" % (
                ("Incomp.   : %s" % bool(trove.troveInfo.incomplete())),
                ("TroveVer  : %s" % trove.troveInfo.troveVersion()))
            yield "%-30s" % (("Clone of  : %s" % trove.troveInfo.clonedFrom()))
            yield "%-30s" % (
                ("Conary version : %s" % trove.troveInfo.conaryVersion()))
示例#8
0
    def mineLabel(self, labelText, jiraProject):

        print 'Looking at %s product...' %jiraProject

        sourceMap = {}
        sourceOwner = {}
        label = versions.Label(labelText)

        repoPkgs = frozenset([ x for x in self.repos.troveNames(label) if ':' not in x and not (x.startswith('cross-') or x.startswith('bootstrap-') or trove.troveIsGroup(x)) ])

        cu = self.db.cursor()
        cu.execute("""SELECT component.cname
                      FROM component, project
                      WHERE component.project = project.id
                        AND project.pname = %s""", jiraProject)
        jiraPkgs = frozenset([r[0] for r in cu.fetchall()])

        newPkgs = sorted(list(repoPkgs-jiraPkgs))

        troveVersions = self.repos.getTroveLeavesByLabel(
            dict.fromkeys(newPkgs, {label: None}))

        for troveName in newPkgs:
            self.log('checking binary package ' + troveName)
            # need latest version
            troveVersion = sorted(troveVersions[troveName].keys())[-1]
            # we only need one flavor, any flavor, to get the sourceName
            troveFlavor = troveVersions[troveName][troveVersion][0]
            trove = self.repos.getTrove(troveName, troveVersion, troveFlavor,
                                   withFiles=False)
            if trove.isRedirect():
                # We do not want to modify jira automatically when we
                # see a redirect, because the redirect may not apply to
                # all versions, and we might really want to keep existing
                # versions the same.
                self.log(' ...ignoring redirected trove ' + troveName)
                continue

            sourceName = trove.getSourceName()
            if not sourceName:
                # old package from before troveinfo
                continue
            sourceNick = sourceName.split(':')[0]
            if sourceNick in jiraPkgs:
                # database doesn't like double-adds
                self.log(' ...source trove %s already in jira' %sourceNick)
                continue
            if sourceNick in sourceMap:
                sourceMap[sourceNick][trove.getName()] = True
                # only investigate each source trove once
                self.log(' ...already checked source trove ' + sourceNick)
                continue
            sourceMap[sourceNick] = {trove.getName(): True}

            sourceVerList = self.repos.getTroveVersionsByLabel(
                {sourceName: {label : None} })
            sourceVerList = sorted(sourceVerList[sourceName].keys())
            l = []
            for sourceVer in sourceVerList:
                l.extend(((sourceName, sourceVer, deps.Flavor()),))
            sourceTroves = self.repos.getTroves(l)

            personMap = {}
            firstPerson = None
            for sourceTrove in sourceTroves:
                cl = sourceTrove.getChangeLog()
                person = self.getPerson(cl.getName(), labelText)
                if not firstPerson:
                    firstPerson = person
                if person in personMap:
                    personMap[person] += 1
                else:
                    personMap[person] = 1
            if firstPerson:
                # original committer is more likely to be the responsible party
                personMap[firstPerson] += 3

            candidate = sorted(personMap.items(), key=lambda x: x[1])[-1][0]
            if not candidate:
                print "No best owner recognized for %s" %sourceNick
                continue
            sourceOwner[sourceNick] = candidate
            print " Best owner for source %s is %s" %(
                    sourceNick, sourceOwner[sourceNick])

        self.sourceMap[jiraProject] = sourceMap
        self.sourceOwner[jiraProject] = sourceOwner
示例#9
0
文件: recipe.py 项目: sweptr/conary
 def isTroveTarget(trove):
     if targets is None:
         return True
     return trove.getName() in targets
示例#10
0
    def addSourceTrovesToGraph(self, sourceTroves):
        sourceTroves = [ x for x in sourceTroves if not x.isFailed() ]
        [ self.depGraph.addNode(trove) for trove in sourceTroves ]
        for trove in sourceTroves:
            if trove.isPrepOnly():
                continue
            for package in trove.getDerivedPackages():
                self.trovesByPackage.setdefault(package, []).append(trove)

        for trove in sourceTroves:
            try:
                if trove.isPrimaryTrove():
                    self.hasPrimaryTroves = True
                trove.addBuildRequirements(trove.cfg.defaultBuildReqs)
                if trove.isPrepOnly():
                    continue
                for buildReq in trove.getBuildRequirementSpecs():
                    self._addReq(trove, buildReq, False)

                for crossReq in trove.getCrossRequirementSpecs():
                    self._addReq(trove, crossReq, True)
            except Exception, err:
                errMsg =  'Error adding buildreqs to %s: %s: %s' % (trove.getName(), err.__class__.__name__, err)
                failureReason = failure.LoadFailed(errMsg,
                                                   traceback.format_exc())
                trove.troveFailed(failureReason)
                # We can't continue the build now
                raise errors.RmakeError, errMsg, sys.exc_info()[2]

            # if're loading something that we're also building
            # then we should make sure that we build the thing with the 
            # loadInstalled line secondly
            for loadSpec, sourceTup in trove.iterAllLoadedSpecs():
                name, label, flavor = loadSpec
                providingTroves = self.trovesByPackage.get(name, [])
                for provTrove in providingTroves:
                    if provTrove.getVersion() != sourceTup[1]:
                        continue
                    elif self._flavorsMatch(trove.getFlavor(),
                                            provTrove.getFlavor(),
                                            flavor, False):
                        # FIXME: we really shouldn't allow loadInstalled
                        # loops to occur.  It means that we're building
                        # two recipes that loadInstall each other which
                        # means that we can't even trust the buildReqs
                        # specified for each.
                        self.dependsOn(trove, provTrove, (False, loadSpec))
                    else:
                        self.rejectDep(trove, provTrove, False)

            # If we have multiple groups that are building w/ the same
            # name and version, we send them to be built together.
            if trove.isGroupRecipe():
                name, version = trove.getName(), trove.getVersion()
                if (name, version) not in self.groupsByNameVersion:
                    self.groupsByNameVersion[name, version] = []
                self.groupsByNameVersion[name, version].append(trove)

            # Troves like groups, redirects, etc, have requirements
            # that control when they can be built.
            for sourceTup in trove.getDelayedRequirements():
                name, version, flavor = sourceTup
                package = name.split(':')[0]
                providingTroves = self.trovesByPackage.get(package, [])
                for provTrove in providingTroves:
                    if provTrove.getVersion() != sourceTup[1]:
                        continue
                    if (flavor is None or
                        provTrove.getFlavor().toStrongFlavor().satisfies(
                                                flavor.toStrongFlavor())):
                        self.dependsOn(trove, provTrove,
                                        (False, (name, version, flavor)))
示例#11
0
    def mineLabel(self, labelText, jiraProject):

        print 'Looking at %s product...' % jiraProject

        sourceMap = {}
        sourceOwner = {}
        label = versions.Label(labelText)

        repoPkgs = frozenset([
            x for x in self.repos.troveNames(label)
            if ':' not in x and not (x.startswith('cross-') or x.startswith(
                'bootstrap-') or trove.troveIsGroup(x))
        ])

        cu = self.db.cursor()
        cu.execute(
            """SELECT component.cname
                      FROM component, project
                      WHERE component.project = project.id
                        AND project.pname = %s""", jiraProject)
        jiraPkgs = frozenset([r[0] for r in cu.fetchall()])

        newPkgs = sorted(list(repoPkgs - jiraPkgs))

        troveVersions = self.repos.getTroveLeavesByLabel(
            dict.fromkeys(newPkgs, {label: None}))

        for troveName in newPkgs:
            self.log('checking binary package ' + troveName)
            # need latest version
            troveVersion = sorted(troveVersions[troveName].keys())[-1]
            # we only need one flavor, any flavor, to get the sourceName
            troveFlavor = troveVersions[troveName][troveVersion][0]
            trove = self.repos.getTrove(troveName,
                                        troveVersion,
                                        troveFlavor,
                                        withFiles=False)
            if trove.isRedirect():
                # We do not want to modify jira automatically when we
                # see a redirect, because the redirect may not apply to
                # all versions, and we might really want to keep existing
                # versions the same.
                self.log(' ...ignoring redirected trove ' + troveName)
                continue

            sourceName = trove.getSourceName()
            if not sourceName:
                # old package from before troveinfo
                continue
            sourceNick = sourceName.split(':')[0]
            if sourceNick in jiraPkgs:
                # database doesn't like double-adds
                self.log(' ...source trove %s already in jira' % sourceNick)
                continue
            if sourceNick in sourceMap:
                sourceMap[sourceNick][trove.getName()] = True
                # only investigate each source trove once
                self.log(' ...already checked source trove ' + sourceNick)
                continue
            sourceMap[sourceNick] = {trove.getName(): True}

            sourceVerList = self.repos.getTroveVersionsByLabel(
                {sourceName: {
                    label: None
                }})
            sourceVerList = sorted(sourceVerList[sourceName].keys())
            l = []
            for sourceVer in sourceVerList:
                l.extend(((sourceName, sourceVer, deps.Flavor()), ))
            sourceTroves = self.repos.getTroves(l)

            personMap = {}
            firstPerson = None
            for sourceTrove in sourceTroves:
                cl = sourceTrove.getChangeLog()
                person = self.getPerson(cl.getName(), labelText)
                if not firstPerson:
                    firstPerson = person
                if person in personMap:
                    personMap[person] += 1
                else:
                    personMap[person] = 1
            if firstPerson:
                # original committer is more likely to be the responsible party
                personMap[firstPerson] += 3

            candidate = sorted(personMap.items(), key=lambda x: x[1])[-1][0]
            if not candidate:
                print "No best owner recognized for %s" % sourceNick
                continue
            sourceOwner[sourceNick] = candidate
            print " Best owner for source %s is %s" % (sourceNick,
                                                       sourceOwner[sourceNick])

        self.sourceMap[jiraProject] = sourceMap
        self.sourceOwner[jiraProject] = sourceOwner
示例#12
0
    def addSourceTrovesToGraph(self, sourceTroves):
        sourceTroves = [ x for x in sourceTroves if not x.isFailed() ]
        [ self.depGraph.addNode(trove) for trove in sourceTroves ]
        for trove in sourceTroves:
            if trove.isPrepOnly():
                continue
            for package in trove.getDerivedPackages():
                self.trovesByPackage.setdefault(package, []).append(trove)

        for trove in sourceTroves:
            try:
                if trove.isPrimaryTrove():
                    self.hasPrimaryTroves = True
                trove.addBuildRequirements(trove.cfg.defaultBuildReqs)
                if trove.isPrepOnly():
                    continue
                for buildReq in trove.getBuildRequirementSpecs():
                    self._addReq(trove, buildReq, False)

                for crossReq in trove.getCrossRequirementSpecs():
                    self._addReq(trove, crossReq, True)
            except Exception, err:
                errMsg =  'Error adding buildreqs to %s: %s: %s' % (trove.getName(), err.__class__.__name__, err)
                failureReason = failure.LoadFailed(errMsg,
                                                   traceback.format_exc())
                trove.troveFailed(failureReason)
                # We can't continue the build now
                raise errors.RmakeError, errMsg, sys.exc_info()[2]

            # if're loading something that we're also building
            # then we should make sure that we build the thing with the 
            # loadInstalled line secondly
            for loadSpec, sourceTup in trove.iterAllLoadedSpecs():
                name, label, flavor = loadSpec
                providingTroves = self.trovesByPackage.get(name, [])
                for provTrove in providingTroves:
                    if provTrove.getVersion() != sourceTup[1]:
                        continue
                    elif self._flavorsMatch(trove.getFlavor(),
                                            provTrove.getFlavor(),
                                            flavor, False):
                        # FIXME: we really shouldn't allow loadInstalled
                        # loops to occur.  It means that we're building
                        # two recipes that loadInstall each other which
                        # means that we can't even trust the buildReqs
                        # specified for each.
                        self.dependsOn(trove, provTrove, (False, loadSpec))
                    else:
                        self.rejectDep(trove, provTrove, False)

            # If we have multiple groups that are building w/ the same
            # name and version, we send them to be built together.
            if trove.isGroupRecipe():
                name, version = trove.getName(), trove.getVersion()
                if (name, version) not in self.groupsByNameVersion:
                    self.groupsByNameVersion[name, version] = []
                self.groupsByNameVersion[name, version].append(trove)

            # Troves like groups, redirects, etc, have requirements
            # that control when they can be built.
            for sourceTup in trove.getDelayedRequirements():
                name, version, flavor = sourceTup
                package = name.split(':')[0]
                providingTroves = self.trovesByPackage.get(package, [])
                for provTrove in providingTroves:
                    if provTrove.getVersion() != sourceTup[1]:
                        continue
                    if (flavor is None or
                        provTrove.getFlavor().toStrongFlavor().satisfies(
                                                flavor.toStrongFlavor())):
                        self.dependsOn(trove, provTrove,
                                        (False, (name, version, flavor)))