def testJobStoreImage(self):
db = self.openRmakeDatabase()
job = buildjob.BuildJob()
trv = imagetrove.ImageTrove(None, *self.makeTroveTuple('group-foo'))
trv.setProductName('product')
trv.setImageType('imageType')
job.addBuildTrove(trv)
job.setMainConfig(self.buildCfg)
db.addJob(job)
db.subscribeToJob(job)
newJob = db.getJob(job.jobId)
newTrv, = newJob.troves.values()
assert (isinstance(newTrv, imagetrove.ImageTrove))
assert (newTrv.getProductName() == 'product')
assert (newTrv.getImageType() == 'imageType')
trv.setImageBuildId(31) # should automatically update
trv.troveBuilding()
newTrv = db.getTrove(job.jobId, *trv.getNameVersionFlavor())
assert (newTrv.getImageBuildId() == 31)
newJob = db.getJob(job.jobId)
newTrv, = newJob.troves.values()
assert (newTrv.getImageBuildId() == 31)
trv.setImageBuildId(32) # should automatically update
job.setBuildTroves(job.troves.values())
newTrv = db.getTrove(job.jobId, *trv.getNameVersionFlavor())
assert (newTrv.getImageBuildId() == 32)
def createImageJob(self, productName, imageList):
allTroveSpecs = {}
finalImageList = []
for image in imageList:
image = list(image)
if len(image) < 4:
image.append('')
# Make it easy to append more parameters extensibly later
image = image[0:4]
finalImageList.append(image)
for troveSpec, imageType, imageOptions, buildName in finalImageList:
if isinstance(troveSpec, str):
troveSpec = cmdline.parseTroveSpec(troveSpec)
allTroveSpecs.setdefault(troveSpec, []).append((imageType,
buildName,
imageOptions))
cfg = self.buildConfig
cfg.initializeFlavors()
repos = self.getRepos()
results = repos.findTroves(cfg.buildLabel, allTroveSpecs,
cfg.buildFlavor)
def getContextName(buildName):
return buildName.replace(' ', '_')
contextCache = set()
i = 1
job = buildjob.BuildJob()
for troveSpec, troveTupList in results.iteritems():
for imageType, buildName, imageOptions in allTroveSpecs[troveSpec]:
for name, version, flavor in troveTupList:
context = getContextName(buildName)
while not context or context in contextCache:
if buildName:
context = '%s_(%d)' %(context, i)
else:
context = 'Image_%d' %i
i += 1
contextCache.add(context)
imageTrove = imagetrove.ImageTrove(None,
name, version, flavor,
context=context)
imageTrove.setImageType(imageType)
imageTrove.setImageOptions(imageOptions)
imageTrove.setProductName(productName)
imageTrove.setBuildName(buildName)
job.setTroveConfig(imageTrove, cfg)
job.addTrove(name, version, flavor, context, imageTrove)
job.setMainConfig(cfg)
return job
def buildTroves(self, troveList, cfg):
"""
Request to build the given sources and build environment.
jobId of created job
@param job: buildJob containing the troves to build and their
configuration
@type job: buildJob
@rtype: int
@return: jobId of created job.
@raise: rMakeError: If server cannot add job.
"""
job = buildjob.BuildJob()
troveList = [buildtrove.BuildTrove(None, *x) for x in troveList]
for trove in troveList:
job.addTrove(buildTrove=trove, *trove.getNameVersionFlavor())
job.setMainConfig(cfg)
return self.buildJob(job)
def testLoadCommand(self):
job = buildjob.BuildJob(1)
trv1 = buildtrove.BuildTrove(1, *self.getNVF('foo:source'))
trv2 = buildtrove.BuildTrove(1, *self.getNVF('bar:source'))
job.addBuildTrove(trv1)
job.addBuildTrove(trv2)
result1 = buildtrove.LoadTroveResult()
result1.packages = set(['foo', 'baz'])
result2 = buildtrove.LoadTroveResult()
result2.buildRequirements = set(['initscripts:runtime'])
troves = [trv1, trv2]
tups = [x.getNameVersionFlavor(True) for x in troves]
results = [result1, result2]
resultSet = dict(zip(tups, results))
def getSourceTrovesFromJob(job_, troves_, repos, reposName):
self.assertEqual(job_, job)
self.assertEqual(troves_, troves)
self.assertEqual(reposName, self.rmakeCfg.reposName)
return resultSet
self.mock(recipeutil, 'getSourceTrovesFromJob', getSourceTrovesFromJob)
mock.mock(conaryclient, 'ConaryClient')
# call
cmd = command.LoadCommand(self.rmakeCfg, 'cmd', job.jobId, None, job,
tups, self.rmakeCfg.reposName)
cmd.publisher = mock.MockObject()
cmd.runAttachedCommand()
cmd.publisher.attach._mock.assertCalled(trv1)
cmd.publisher.attach._mock.assertCalled(trv2)
self.assertEqual(trv1.packages, set(['foo', 'baz']))
self.assertEqual(trv2.buildRequirements, set(['initscripts:runtime']))
def testRestart(self):
mock.mock(helper.rMakeHelper, 'createRestartJob')
helper.rMakeHelper.createRestartJob._mock.setReturn(
buildjob.BuildJob(), _mockAll=True)
mock.mock(helper.rMakeHelper, 'buildJob')
mock.mock(helper.rMakeHelper, 'displayJob')
self.checkRmake(
'restart 1 --no-watch --info --clear-build-list --clear-prebuilt-list',
'rmake.cmdline.helper.rMakeHelper.createRestartJob', [None, 1, []],
updateSpecs=[],
excludeSpecs=[],
updateConfigKeys=None,
clearBuildList=True,
clearPrebuiltList=True)
self.checkRmake('restart 1 tmpwatch --no-watch',
'rmake.cmdline.helper.rMakeHelper.createRestartJob',
[None, 1, ['tmpwatch']],
updateSpecs=[],
excludeSpecs=[],
updateConfigKeys=None,
clearBuildList=False,
clearPrebuiltList=False)
self.checkRmake(
'restart 1 --no-watch --update foo-* --update -foo-bar',
'rmake.cmdline.helper.rMakeHelper.createRestartJob', [None, 1, []],
updateSpecs=['foo-*', '-foo-bar'],
excludeSpecs=[],
updateConfigKeys=None,
clearBuildList=False,
clearPrebuiltList=False)
self.checkRmake('restart 1 --no-watch --no-update',
'rmake.cmdline.helper.rMakeHelper.createRestartJob',
[None, 1, []],
updateSpecs=['-*'],
excludeSpecs=[],
updateConfigKeys=None,
clearBuildList=False,
clearPrebuiltList=False)
self.checkRmake(
'restart 1 --no-watch --exclude=foo --update-config=installLabelPath',
'rmake.cmdline.helper.rMakeHelper.createRestartJob', [None, 1, []],
updateSpecs=[],
excludeSpecs=['foo'],
updateConfigKeys=['installLabelPath'],
clearBuildList=False,
clearPrebuiltList=False)
self.checkRmake(
'restart 1 --no-watch --exclude=foo'
' --update-config=all',
'rmake.cmdline.helper.rMakeHelper.createRestartJob', [None, 1, []],
updateSpecs=[],
excludeSpecs=['foo'],
updateConfigKeys=['all'],
clearBuildList=False,
clearPrebuiltList=False)
self.checkRmake(
'restart 1 --no-watch --to-file=foo'
' --update-config=all',
'rmake.build.buildjob.BuildJob.writeToFile', [None, 'foo'],
sanitize=True)
callback = checkBuildConfig(ignoreExternalRebuildDeps=True)
self.checkRmake(
'restart 1 --no-watch --ignore-external-rebuild-deps'
' --update-config=all',
'rmake.cmdline.helper.rMakeHelper.buildJob', [None, None],
quiet=False,
checkCallback=callback)
callback = checkBuildConfig(ignoreAllRebuildDeps=True)
self.checkRmake(
'restart 1 --no-watch --ignore-rebuild-deps'
' --update-config=all',
'rmake.cmdline.helper.rMakeHelper.buildJob', [None, None],
quiet=False,
checkCallback=callback)
def getJobs(self, jobIdList, withTroves=False, withConfigs=False):
if not jobIdList:
return []
cu = self.db.cursor()
jobIds = ','.join('%d' % x for x in jobIdList)
results = cu.execute("""
SELECT jobId, uuid, owner, state, status, start, finish,
failureReason, failureData, pid
FROM Jobs WHERE jobId IN (%s)
""" % (jobIds, ))
jobsById = {}
for (jobId, uuid, owner, state, status, start, finish, failureReason,
failureData, pid) in results:
if state is None:
# quick catch check for missing jobs
raise KeyError, jobId
failureReason = thaw('FailureReason',
(failureReason, cu.frombinary(failureData)))
job = buildjob.BuildJob(jobId,
status=status,
state=state,
start=float(start),
finish=float(finish),
failureReason=failureReason,
uuid=uuid,
pid=pid,
owner=owner)
jobsById[jobId] = job
if withTroves:
results = cu.execute("""
SELECT jobId, BuildTroves.troveId, troveName, version,
flavor, context, state, status, failureReason,
failureData, start, finish, logPath, pid, recipeType,
buildType, troveType, Chroots.path, Chroots.nodeName
FROM BuildTroves
LEFT JOIN Chroots USING(chrootId)
WHERE jobId IN (%s)
""" % (jobIds, ))
trovesById = {}
for (jobId, troveId, name, version, flavor, context, state, status,
failureReason, failureData, start, finish, logPath, pid,
recipeType, buildType, troveType, chrootPath,
chrootHost) in results:
if chrootPath is None:
chrootPath = ''
chrootHost = ''
version = versions.ThawVersion(version)
flavor = ThawFlavor(flavor)
failureReason = thaw(
'FailureReason',
(failureReason, cu.frombinary(failureData)))
troveClass = buildtrove.getClassForTroveType(troveType)
buildTrove = troveClass(jobId,
name,
version,
flavor,
state=state,
start=float(start),
finish=float(finish),
logPath=logPath,
status=status,
failureReason=failureReason,
pid=pid,
recipeType=recipeType,
chrootPath=chrootPath,
chrootHost=chrootHost,
buildType=buildType,
context=context)
trovesById[troveId] = buildTrove
jobsById[jobId].addTrove(name, version, flavor, context,
buildTrove)
results = cu.execute("""
SELECT BuildTroves.troveId, BinaryTroves.troveName,
BinaryTroves.version, BinaryTroves.flavor
FROM BuildTroves
JOIN BinaryTroves USING(troveId)
WHERE jobId IN (%s)
""" % (jobIds, ))
builtTroves = {}
for troveId, name, version, flavor in results:
builtTroves.setdefault(troveId, []).append(
(name, ThawVersion(version), ThawFlavor(flavor)))
for troveId, binTroves in builtTroves.iteritems():
trv = trovesById.get(troveId)
if trv is not None:
trv.setBuiltTroves(binTroves)
cu.execute("""
SELECT troveId, key, value
FROM TroveSettings
WHERE jobId IN (%s)
ORDER BY key, ord
""" % (jobIds, ))
troveSettings = {}
for troveId, key, value in cu:
d = troveSettings.setdefault(troveId, {})
d.setdefault(key, []).append(value)
for troveId, settings in troveSettings.items():
settingsClass = settings.pop('_class')[0]
trv = trovesById.get(troveId)
if trv is not None:
trv.settings = thaw('TroveSettings',
(settingsClass, settings))
else:
results = cu.execute("""
SELECT jobId, troveName, version, flavor, context
FROM BuildTroves
WHERE jobId IN (%s)
""" % (jobIds, ))
for (jobId, n, v, f, context) in results:
jobsById[jobId].addTrove(n, versions.ThawVersion(v),
ThawFlavor(f), context)
if withConfigs:
cu.execute("""
SELECT jobId, context, key, value
FROM JobConfig
WHERE jobId IN (%s)
ORDER BY key, ord
""" % (jobIds, ))
jobConfigD = {}
for jobId, context, key, value in cu:
configD = jobConfigD.setdefault(jobId, {})
d = configD.setdefault(context, {})
d.setdefault(key, []).append(value)
for jobId, configD in jobConfigD.items():
configD = dict((x[0], thaw('BuildConfiguration', x[1]))
for x in configD.iteritems())
jobsById[jobId].setConfigs(configD)
return [jobsById[jobId] for jobId in jobIdList]
def getBuildJob(buildConfig, conaryclient, troveSpecList,
message=None, recurseGroups=BUILD_RECURSE_GROUPS_NONE,
configDict=None, oldTroveDict=None, updateSpecs=None,
rebuild=False):
trovesByContext = {}
for troveSpec in list(troveSpecList):
if not isinstance(troveSpec, tuple):
troveSpec = cmdutil.parseTroveSpec(troveSpec)
if len(troveSpec) == 3:
context = ''
else:
context = troveSpec[3]
troveSpec = troveSpec[:3]
if troveSpec[2] is None:
troveSpec = (troveSpec[0], troveSpec[1], deps.parseFlavor(''))
trovesByContext.setdefault(context, []).append(troveSpec)
job = buildjob.BuildJob()
# don't store all the contexts with this job - they're useless past the
# initialization step.
if configDict:
mainConfig = configDict['']
job.setMainConfig(configDict[''])
else:
cfg = copy.deepcopy(buildConfig)
cfg.dropContexts()
mainConfig = cfg
mainConfig.recurseGroups = int(recurseGroups)
job.setMainConfig(mainConfig)
baseMatchRules = mainConfig.matchTroveRule
for contextStr, troveSpecList in trovesByContext.iteritems():
contextBaseMatchRules = baseMatchRules
if configDict and contextStr in configDict:
cfg = configDict[contextStr]
elif contextStr:
# making this a copy is critical
cfg = copy.deepcopy(buildConfig)
for context in contextStr.split(','):
cfg.setContext(context)
cfg.dropContexts()
else:
# don't bother with baseMatchRules in the base config.
contextBaseMatchRules = []
cfg = copy.deepcopy(buildConfig)
cfg.dropContexts()
contextStr = ''
job.setMainConfig(cfg)
cfg.initializeFlavors()
use.setBuildFlagsFromFlavor(None, cfg.buildFlavor, error=False)
if not cfg.buildLabel and cfg.installLabelPath:
cfg.buildLabel = cfg.installLabelPath[0]
troveSpecList = list(set(troveSpecList))
troveList = getTrovesToBuild(cfg, conaryclient, troveSpecList,
message=None,
recurseGroups=recurseGroups,
matchSpecs=contextBaseMatchRules + cfg.matchTroveRule,
reposName=mainConfig.reposName,
updateSpecs=updateSpecs)
if updateSpecs and oldTroveDict and contextStr in oldTroveDict:
troveList = _matchUpdateRestrictions(mainConfig.reposName,
oldTroveDict[contextStr],
troveList,
updateSpecs)
if rebuild:
prebuiltBinaries = _findLatestBinariesForTroves(conaryclient,
mainConfig.reposName,
troveList)
if not job.getMainConfig().prebuiltBinaries:
job.getMainConfig().prebuiltBinaries = prebuiltBinaries
else:
job.getMainConfig().prebuiltBinaries.extend(prebuiltBinaries)
if mainConfig.prepOnly:
buildType = buildtrove.TROVE_BUILD_TYPE_PREP
else:
buildType = buildtrove.TROVE_BUILD_TYPE_NORMAL
for name, version, flavor in troveList:
if flavor is None:
flavor = deps.parseFlavor('')
bt = buildtrove.BuildTrove(None, name, version, flavor,
context=contextStr,
buildType=buildType)
job.addTrove(name, version, flavor, contextStr, bt)
job.setTroveConfig(bt, cfg)
return job