def build(self):
self.job.jobStarted("Starting Build %s (pid %s)" % (self.job.jobId,
os.getpid()), pid=os.getpid())
# main loop is here.
if not self.initialized:
if not self.initializeBuild():
return False
self.job.jobBuilding('Building troves')
if self.dh.moreToDo():
while self.dh.moreToDo():
self.worker.handleRequestIfReady()
if self.worker._checkForResults():
self.resolveIfReady()
elif self.dh.hasBuildableTroves():
trv, (buildReqs, crossReqs, bootstrapReqs
) = self.dh.popBuildableTrove()
self.buildTrove(trv, buildReqs, crossReqs, bootstrapReqs)
elif self.dh.hasSpecialTroves():
self.actOnTrove(self.dh.popSpecialTrove())
elif not self.resolveIfReady():
time.sleep(0.1)
if self.dh.jobPassed():
self.job.jobPassed("build job finished successfully")
return True
msg = ['Build job had failures:\n']
for trove in sorted(self.job.iterPrimaryFailureTroves()):
err = trove.getFailureReason().getShortError()
msg.append(' * %s: %s\n' % (trove.getName(), err))
self.job.jobFailed(''.join(msg))
else:
msg = ['Did not find any buildable troves:\n']
for trove in sorted(self.job.iterPrimaryFailureTroves()):
err = trove.getFailureReason().getShortError()
msg.append(' * %s: %s\n' % (trove.getName(), err))
self.job.jobFailed(''.join(msg))
return False
def troveDuplicate(self, trove, troveList):
package = trove.getName().split(':')[0]
possibleMatches = self.depState.getTrovesByPackage(package)
for match in possibleMatches:
if match is trove:
continue
elif match.getBinaryTroves() == set(troveList):
self.depState.troveDuplicate(trove, match)
return
elif set(match.getBinaryTroves()) & set(troveList):
trove.troveFailed('Two versions of %s=%s[%s] were built at the same time but resulted in different components. If these packages should have different flavors, then add flavor information to them. Otherwise, try building only one of them.' % trove.getNameVersionFlavor())
return
elif not match.getBinaryTroves() and match.isBuilding():
# it's possible that the two results just came back in the
# wrong order
self._possibleDuplicates.setdefault(match, []).append((trove,
troveList))
return
trove.troveFailed('Package was committed at the same time as the same package was built in another job. Make sure no-one else is building the same packages as you, and that you didn\'t accidentally build the same package twice with the same flavor.')
def troveDuplicate(self, trove, troveList):
package = trove.getName().split(':')[0]
possibleMatches = self.depState.getTrovesByPackage(package)
for match in possibleMatches:
if match is trove:
continue
elif match.getBinaryTroves() == set(troveList):
self.depState.troveDuplicate(trove, match)
return
elif set(match.getBinaryTroves()) & set(troveList):
trove.troveFailed('Two versions of %s=%s[%s] were built at the same time but resulted in different components. If these packages should have different flavors, then add flavor information to them. Otherwise, try building only one of them.' % trove.getNameVersionFlavor())
return
elif not match.getBinaryTroves() and match.isBuilding():
# it's possible that the two results just came back in the
# wrong order
self._possibleDuplicates.setdefault(match, []).append((trove,
troveList))
return
trove.troveFailed('Package was committed at the same time as the same package was built in another job. Make sure no-one else is building the same packages as you, and that you didn\'t accidentally build the same package twice with the same flavor.')
def startTroveLogger(self, trove):
key = ''.join([ chr(random.randrange(ord('a'), ord('z')))
for x in range(10) ])
r = recorder.BuildLogRecorder(key)
r.attach(trove)
logHost = r.getHost()
logPort = r.getPort()
trove.logPath = r.getLogPath()
pid = self.worker._fork('BuildLogger for %s{%s}' % (trove.getName(),
trove.getContext() or ''))
if not pid:
try:
r.closeOtherFds()
r._installSignalHandlers()
r.serve_forever()
finally:
os._exit(3)
else:
r.close()
trove.logPid = pid
return logHost, logPort, key
def formatInfo(self, trove):
""" returns iterator of format lines about this local trove """
# TODO: it'd be nice if this were set up to do arbitrary
# formats...
n, v, f = trove.getName(), trove.getVersion(), trove.getFlavor()
dcfg = self.dcfg
troveSource = dcfg.getTroveSource()
sourceName = trove.getSourceName()
sourceTrove = None
if sourceName:
try:
sourceVer = v.getSourceVersion()
if sourceVer.isOnLocalHost():
sourceVer = sourceVer.parentVersion()
sourceTrove = troveSource.getTrove(
sourceName, sourceVer, deps.Flavor(), withFiles = False)
# FIXME: all trove sources should return TroveMissing
# on failed getTrove calls
except errors.TroveMissing:
pass
except errors.InsufficientPermission:
pass
elif n.endswith(':source'):
sourceTrove = trove
if trove.getBuildTime():
buildTime = time.strftime("%c",
time.localtime(trove.getBuildTime()))
else:
buildTime = "(unknown)"
if trove.getSize() is not None:
size = "%s" % trove.getSize()
else:
size = "(unknown)"
yield "%-30s %s" % \
(("Name : %s" % trove.getName(),
("Build time: %s" % buildTime)))
if dcfg.fullVersions:
yield "Version : %s" %v
yield "Label : %s" % v.branch().label().asString()
else:
yield "%-30s %s" % \
(("Version : %s" %
v.trailingRevision().asString()),
("Label : %s" %
v.branch().label().asString()))
yield '%-30s' % ("Size : %s" % size)
if hasattr(troveSource, 'trovesArePinned'):
yield "Pinned : %s" % troveSource.trovesArePinned(
[ (n, v, f) ])[0]
commitTime = v.trailingRevision().timeStamp
if commitTime:
commitTime = time.strftime("%c", time.localtime(commitTime))
else:
commitTime = '(unknown)'
if not trove.getBuildTime() or log.getVerbosity() <= log.DEBUG:
yield "Committed : " + commitTime
yield "%-30s" % ("Flavor : %s" % deps.formatFlavor(f))
if trove.getInstallTime():
yield 'Installed : %s' % (time.strftime("%c",
time.localtime(trove.getInstallTime())))
imageGroup = trove.troveInfo.imageGroup()
if imageGroup is not None:
yield 'Image Group: %s' % bool(imageGroup)
for ln in self.formatMetadata(trove):
yield ln
if sourceTrove:
if not n.endswith(':source'):
yield 'Source : %s' % trove.getSourceName()
cl = sourceTrove.getChangeLog()
if cl:
yield "Change log: %s (%s)" % (cl.getName(), cl.getContact())
lines = cl.getMessage().split("\n")[:-1]
for l in lines:
yield " " + l
if log.getVerbosity() <= log.DEBUG:
yield "%-30s %s" % (("Incomp. : %s" %
bool(trove.troveInfo.incomplete())),
("TroveVer : %s" %
trove.troveInfo.troveVersion()))
yield "%-30s" % (("Clone of : %s" % trove.troveInfo.clonedFrom()))
subPackages = set(trove.troveInfo.subPackages())
subPackages.discard(trove.getName())
if subPackages:
yield "%-30s" % (("Siblings : %s" % (' '.join(sorted(subPackages)))))
yield "%-30s" % (("Conary version : %s" % trove.troveInfo.conaryVersion()))
def isTroveTarget(trove):
if targets is None:
return True
return trove.getName() in targets
def formatInfo(self, trove):
""" returns iterator of format lines about this local trove """
# TODO: it'd be nice if this were set up to do arbitrary
# formats...
n, v, f = trove.getName(), trove.getVersion(), trove.getFlavor()
dcfg = self.dcfg
troveSource = dcfg.getTroveSource()
sourceName = trove.getSourceName()
sourceTrove = None
if sourceName:
try:
sourceVer = v.getSourceVersion()
if sourceVer.isOnLocalHost():
sourceVer = sourceVer.parentVersion()
sourceTrove = troveSource.getTrove(sourceName,
sourceVer,
deps.Flavor(),
withFiles=False)
# FIXME: all trove sources should return TroveMissing
# on failed getTrove calls
except errors.TroveMissing:
pass
except errors.InsufficientPermission:
pass
elif n.endswith(':source'):
sourceTrove = trove
if trove.getBuildTime():
buildTime = time.strftime("%c",
time.localtime(trove.getBuildTime()))
else:
buildTime = "(unknown)"
if trove.getSize() is not None:
size = "%s" % trove.getSize()
else:
size = "(unknown)"
yield "%-30s %s" % \
(("Name : %s" % trove.getName(),
("Build time: %s" % buildTime)))
if dcfg.fullVersions:
yield "Version : %s" % v
yield "Label : %s" % v.branch().label().asString()
else:
yield "%-30s %s" % \
(("Version : %s" %
v.trailingRevision().asString()),
("Label : %s" %
v.branch().label().asString()))
yield '%-30s' % ("Size : %s" % size)
if hasattr(troveSource, 'trovesArePinned'):
yield "Pinned : %s" % troveSource.trovesArePinned([(n, v, f)
])[0]
yield "%-30s" % ("Flavor : %s" % deps.formatFlavor(f))
if trove.getInstallTime():
yield 'Installed : %s' % (time.strftime(
"%c", time.localtime(trove.getInstallTime())))
imageGroup = trove.troveInfo.imageGroup()
if imageGroup is not None:
yield 'Image Group: %s' % bool(imageGroup)
for ln in self.formatMetadata(trove):
yield ln
if sourceTrove:
if not n.endswith(':source'):
yield 'Source : %s' % trove.getSourceName()
cl = sourceTrove.getChangeLog()
if cl:
yield "Change log: %s (%s)" % (cl.getName(), cl.getContact())
lines = cl.getMessage().split("\n")[:-1]
for l in lines:
yield " " + l
if log.getVerbosity() <= log.DEBUG:
yield "%-30s %s" % (
("Incomp. : %s" % bool(trove.troveInfo.incomplete())),
("TroveVer : %s" % trove.troveInfo.troveVersion()))
yield "%-30s" % (("Clone of : %s" % trove.troveInfo.clonedFrom()))
yield "%-30s" % (
("Conary version : %s" % trove.troveInfo.conaryVersion()))
def mineLabel(self, labelText, jiraProject):
print 'Looking at %s product...' %jiraProject
sourceMap = {}
sourceOwner = {}
label = versions.Label(labelText)
repoPkgs = frozenset([ x for x in self.repos.troveNames(label) if ':' not in x and not (x.startswith('cross-') or x.startswith('bootstrap-') or trove.troveIsGroup(x)) ])
cu = self.db.cursor()
cu.execute("""SELECT component.cname
FROM component, project
WHERE component.project = project.id
AND project.pname = %s""", jiraProject)
jiraPkgs = frozenset([r[0] for r in cu.fetchall()])
newPkgs = sorted(list(repoPkgs-jiraPkgs))
troveVersions = self.repos.getTroveLeavesByLabel(
dict.fromkeys(newPkgs, {label: None}))
for troveName in newPkgs:
self.log('checking binary package ' + troveName)
# need latest version
troveVersion = sorted(troveVersions[troveName].keys())[-1]
# we only need one flavor, any flavor, to get the sourceName
troveFlavor = troveVersions[troveName][troveVersion][0]
trove = self.repos.getTrove(troveName, troveVersion, troveFlavor,
withFiles=False)
if trove.isRedirect():
# We do not want to modify jira automatically when we
# see a redirect, because the redirect may not apply to
# all versions, and we might really want to keep existing
# versions the same.
self.log(' ...ignoring redirected trove ' + troveName)
continue
sourceName = trove.getSourceName()
if not sourceName:
# old package from before troveinfo
continue
sourceNick = sourceName.split(':')[0]
if sourceNick in jiraPkgs:
# database doesn't like double-adds
self.log(' ...source trove %s already in jira' %sourceNick)
continue
if sourceNick in sourceMap:
sourceMap[sourceNick][trove.getName()] = True
# only investigate each source trove once
self.log(' ...already checked source trove ' + sourceNick)
continue
sourceMap[sourceNick] = {trove.getName(): True}
sourceVerList = self.repos.getTroveVersionsByLabel(
{sourceName: {label : None} })
sourceVerList = sorted(sourceVerList[sourceName].keys())
l = []
for sourceVer in sourceVerList:
l.extend(((sourceName, sourceVer, deps.Flavor()),))
sourceTroves = self.repos.getTroves(l)
personMap = {}
firstPerson = None
for sourceTrove in sourceTroves:
cl = sourceTrove.getChangeLog()
person = self.getPerson(cl.getName(), labelText)
if not firstPerson:
firstPerson = person
if person in personMap:
personMap[person] += 1
else:
personMap[person] = 1
if firstPerson:
# original committer is more likely to be the responsible party
personMap[firstPerson] += 3
candidate = sorted(personMap.items(), key=lambda x: x[1])[-1][0]
if not candidate:
print "No best owner recognized for %s" %sourceNick
continue
sourceOwner[sourceNick] = candidate
print " Best owner for source %s is %s" %(
sourceNick, sourceOwner[sourceNick])
self.sourceMap[jiraProject] = sourceMap
self.sourceOwner[jiraProject] = sourceOwner
def isTroveTarget(trove):
if targets is None:
return True
return trove.getName() in targets
def addSourceTrovesToGraph(self, sourceTroves):
sourceTroves = [ x for x in sourceTroves if not x.isFailed() ]
[ self.depGraph.addNode(trove) for trove in sourceTroves ]
for trove in sourceTroves:
if trove.isPrepOnly():
continue
for package in trove.getDerivedPackages():
self.trovesByPackage.setdefault(package, []).append(trove)
for trove in sourceTroves:
try:
if trove.isPrimaryTrove():
self.hasPrimaryTroves = True
trove.addBuildRequirements(trove.cfg.defaultBuildReqs)
if trove.isPrepOnly():
continue
for buildReq in trove.getBuildRequirementSpecs():
self._addReq(trove, buildReq, False)
for crossReq in trove.getCrossRequirementSpecs():
self._addReq(trove, crossReq, True)
except Exception, err:
errMsg = 'Error adding buildreqs to %s: %s: %s' % (trove.getName(), err.__class__.__name__, err)
failureReason = failure.LoadFailed(errMsg,
traceback.format_exc())
trove.troveFailed(failureReason)
# We can't continue the build now
raise errors.RmakeError, errMsg, sys.exc_info()[2]
# if're loading something that we're also building
# then we should make sure that we build the thing with the
# loadInstalled line secondly
for loadSpec, sourceTup in trove.iterAllLoadedSpecs():
name, label, flavor = loadSpec
providingTroves = self.trovesByPackage.get(name, [])
for provTrove in providingTroves:
if provTrove.getVersion() != sourceTup[1]:
continue
elif self._flavorsMatch(trove.getFlavor(),
provTrove.getFlavor(),
flavor, False):
# FIXME: we really shouldn't allow loadInstalled
# loops to occur. It means that we're building
# two recipes that loadInstall each other which
# means that we can't even trust the buildReqs
# specified for each.
self.dependsOn(trove, provTrove, (False, loadSpec))
else:
self.rejectDep(trove, provTrove, False)
# If we have multiple groups that are building w/ the same
# name and version, we send them to be built together.
if trove.isGroupRecipe():
name, version = trove.getName(), trove.getVersion()
if (name, version) not in self.groupsByNameVersion:
self.groupsByNameVersion[name, version] = []
self.groupsByNameVersion[name, version].append(trove)
# Troves like groups, redirects, etc, have requirements
# that control when they can be built.
for sourceTup in trove.getDelayedRequirements():
name, version, flavor = sourceTup
package = name.split(':')[0]
providingTroves = self.trovesByPackage.get(package, [])
for provTrove in providingTroves:
if provTrove.getVersion() != sourceTup[1]:
continue
if (flavor is None or
provTrove.getFlavor().toStrongFlavor().satisfies(
flavor.toStrongFlavor())):
self.dependsOn(trove, provTrove,
(False, (name, version, flavor)))
def mineLabel(self, labelText, jiraProject):
print 'Looking at %s product...' % jiraProject
sourceMap = {}
sourceOwner = {}
label = versions.Label(labelText)
repoPkgs = frozenset([
x for x in self.repos.troveNames(label)
if ':' not in x and not (x.startswith('cross-') or x.startswith(
'bootstrap-') or trove.troveIsGroup(x))
])
cu = self.db.cursor()
cu.execute(
"""SELECT component.cname
FROM component, project
WHERE component.project = project.id
AND project.pname = %s""", jiraProject)
jiraPkgs = frozenset([r[0] for r in cu.fetchall()])
newPkgs = sorted(list(repoPkgs - jiraPkgs))
troveVersions = self.repos.getTroveLeavesByLabel(
dict.fromkeys(newPkgs, {label: None}))
for troveName in newPkgs:
self.log('checking binary package ' + troveName)
# need latest version
troveVersion = sorted(troveVersions[troveName].keys())[-1]
# we only need one flavor, any flavor, to get the sourceName
troveFlavor = troveVersions[troveName][troveVersion][0]
trove = self.repos.getTrove(troveName,
troveVersion,
troveFlavor,
withFiles=False)
if trove.isRedirect():
# We do not want to modify jira automatically when we
# see a redirect, because the redirect may not apply to
# all versions, and we might really want to keep existing
# versions the same.
self.log(' ...ignoring redirected trove ' + troveName)
continue
sourceName = trove.getSourceName()
if not sourceName:
# old package from before troveinfo
continue
sourceNick = sourceName.split(':')[0]
if sourceNick in jiraPkgs:
# database doesn't like double-adds
self.log(' ...source trove %s already in jira' % sourceNick)
continue
if sourceNick in sourceMap:
sourceMap[sourceNick][trove.getName()] = True
# only investigate each source trove once
self.log(' ...already checked source trove ' + sourceNick)
continue
sourceMap[sourceNick] = {trove.getName(): True}
sourceVerList = self.repos.getTroveVersionsByLabel(
{sourceName: {
label: None
}})
sourceVerList = sorted(sourceVerList[sourceName].keys())
l = []
for sourceVer in sourceVerList:
l.extend(((sourceName, sourceVer, deps.Flavor()), ))
sourceTroves = self.repos.getTroves(l)
personMap = {}
firstPerson = None
for sourceTrove in sourceTroves:
cl = sourceTrove.getChangeLog()
person = self.getPerson(cl.getName(), labelText)
if not firstPerson:
firstPerson = person
if person in personMap:
personMap[person] += 1
else:
personMap[person] = 1
if firstPerson:
# original committer is more likely to be the responsible party
personMap[firstPerson] += 3
candidate = sorted(personMap.items(), key=lambda x: x[1])[-1][0]
if not candidate:
print "No best owner recognized for %s" % sourceNick
continue
sourceOwner[sourceNick] = candidate
print " Best owner for source %s is %s" % (sourceNick,
sourceOwner[sourceNick])
self.sourceMap[jiraProject] = sourceMap
self.sourceOwner[jiraProject] = sourceOwner
def addSourceTrovesToGraph(self, sourceTroves):
sourceTroves = [ x for x in sourceTroves if not x.isFailed() ]
[ self.depGraph.addNode(trove) for trove in sourceTroves ]
for trove in sourceTroves:
if trove.isPrepOnly():
continue
for package in trove.getDerivedPackages():
self.trovesByPackage.setdefault(package, []).append(trove)
for trove in sourceTroves:
try:
if trove.isPrimaryTrove():
self.hasPrimaryTroves = True
trove.addBuildRequirements(trove.cfg.defaultBuildReqs)
if trove.isPrepOnly():
continue
for buildReq in trove.getBuildRequirementSpecs():
self._addReq(trove, buildReq, False)
for crossReq in trove.getCrossRequirementSpecs():
self._addReq(trove, crossReq, True)
except Exception, err:
errMsg = 'Error adding buildreqs to %s: %s: %s' % (trove.getName(), err.__class__.__name__, err)
failureReason = failure.LoadFailed(errMsg,
traceback.format_exc())
trove.troveFailed(failureReason)
# We can't continue the build now
raise errors.RmakeError, errMsg, sys.exc_info()[2]
# if're loading something that we're also building
# then we should make sure that we build the thing with the
# loadInstalled line secondly
for loadSpec, sourceTup in trove.iterAllLoadedSpecs():
name, label, flavor = loadSpec
providingTroves = self.trovesByPackage.get(name, [])
for provTrove in providingTroves:
if provTrove.getVersion() != sourceTup[1]:
continue
elif self._flavorsMatch(trove.getFlavor(),
provTrove.getFlavor(),
flavor, False):
# FIXME: we really shouldn't allow loadInstalled
# loops to occur. It means that we're building
# two recipes that loadInstall each other which
# means that we can't even trust the buildReqs
# specified for each.
self.dependsOn(trove, provTrove, (False, loadSpec))
else:
self.rejectDep(trove, provTrove, False)
# If we have multiple groups that are building w/ the same
# name and version, we send them to be built together.
if trove.isGroupRecipe():
name, version = trove.getName(), trove.getVersion()
if (name, version) not in self.groupsByNameVersion:
self.groupsByNameVersion[name, version] = []
self.groupsByNameVersion[name, version].append(trove)
# Troves like groups, redirects, etc, have requirements
# that control when they can be built.
for sourceTup in trove.getDelayedRequirements():
name, version, flavor = sourceTup
package = name.split(':')[0]
providingTroves = self.trovesByPackage.get(package, [])
for provTrove in providingTroves:
if provTrove.getVersion() != sourceTup[1]:
continue
if (flavor is None or
provTrove.getFlavor().toStrongFlavor().satisfies(
flavor.toStrongFlavor())):
self.dependsOn(trove, provTrove,
(False, (name, version, flavor)))