def testCompatCheck(self):
cv = compat.ConaryVersion('1.0.29')
assert (not cv.supportsCloneCallback())
cv = compat.ConaryVersion('1.0.29_changeset')
# _changeset must not make conary assume latest (RMK-1077)
assert (not cv.supportsCloneCallback())
cv = compat.ConaryVersion('1.0.30')
assert (cv.supportsCloneCallback())
self.logFilter.add()
compat.ConaryVersion._warnedUser = False
cv = compat.ConaryVersion('foo')
self.logFilter.compare(
['warning: nonstandard conary version "foo". Assuming latest.'])
assert (cv.supportsCloneCallback())
cv = compat.ConaryVersion('1.1.18')
assert (cv.supportsCloneNoTracking())
cv = compat.ConaryVersion('1.1.16')
assert (not cv.supportsCloneNoTracking())
cv = compat.ConaryVersion('1.0.30')
assert (not cv.supportsCloneNoTracking())
cv = compat.ConaryVersion('2.0')
assert (not cv.signAfterPromote())
cv = compat.ConaryVersion('1.2.7')
assert (cv.signAfterPromote())
cv = compat.ConaryVersion('2.0.28')
assert (cv.supportsDefaultBuildReqs())
def _doCommit(recipePath, repos, cfg, message):
try:
kw = {}
if compat.ConaryVersion().supportsForceCommit():
kw.update(force=True)
rv = checkin.commit(repos, cfg, message, **kw)
except (conaryerrors.CvcError, conaryerrors.ConaryError), msg:
raise errors.RmakeError("Could not commit changes to build"
" recipe %s: %s" % (recipePath, msg))
def _getContext(self, buildConfig, conaryConfig, argSet):
context = conaryConfig.context
if buildConfig.context:
context = buildConfig.context
if os.path.exists('CONARY'):
conaryState = compat.ConaryVersion().ConaryStateFromFile(
'CONARY', parseSource=False)
if conaryState.hasContext():
context = conaryState.getContext()
context = os.environ.get('CONARY_CONTEXT', context)
context = argSet.pop('context', context)
return context
def runCommand(self, client, cfg, argSet, args):
if self.verbose:
log.setVerbosity(log.DEBUG)
else:
log.setVerbosity(log.INFO)
command, troveSpecs = self.requireParameters(args,
'troveSpec',
appendExtra=True)
if command == 'buildgroup':
log.warning(
'"buildgroup" is deprecated and will be removed in a future release - use "build --recurse" instead'
)
rebuild = (command == 'rebuild')
flavorSpec = argSet.pop('flavor', None)
if flavorSpec:
flavor = deps.parseFlavor(flavorSpec)
if flavor is None:
raise errors.ParseError("Invalid flavor: '%s'" % flavorSpec)
newFlavor = deps.overrideFlavor(client.buildConfig.buildFlavor,
flavor)
client.buildConfig.buildFlavor = newFlavor
newFlavors = []
for oldFlavor in client.buildConfig.flavor:
newFlavors.append(deps.overrideFlavor(oldFlavor, flavor))
client.buildConfig.flavor = newFlavors
matchSpecs = argSet.pop('match', [])
hosts = argSet.pop('host', [])
labels = argSet.pop('label', [])
recurseGroups = argSet.pop('recurse', False) or command == 'buildgroup'
if recurseGroups:
if argSet.pop('binary-search', False):
recurseGroups = client.BUILD_RECURSE_GROUPS_BINARY
elif not compat.ConaryVersion().supportsFindGroupSources():
log.warning('Your conary does not support recursing a group'
' source component, defaulting to searching the'
' binary version')
recurseGroups = client.BUILD_RECURSE_GROUPS_BINARY
else:
recurseGroups = client.BUILD_RECURSE_GROUPS_SOURCE
self._prep(client, argSet)
job = client.createBuildJob(troveSpecs,
limitToHosts=hosts,
limitToLabels=labels,
recurseGroups=recurseGroups,
matchSpecs=matchSpecs,
rebuild=rebuild)
return self._build(client, job, argSet)
class RmakeBuildContext(cfg.ConfigSection):
_cfg_bases = [conarycfg.ConaryContext]
bootstrapTroves = (
CfgList(CfgTroveSpec), [],
"INTERNAL USE ONLY: Troves to be installed before the remaining "
"chroot contents.")
copyInConary = (CfgBool, False)
copyInConfig = (CfgBool, False) # RMK-1052
rbuilderUrl = (cfgtypes.CfgString, 'https://localhost/')
rmakeUser = (CfgUser, None)
if not compat.ConaryVersion().supportsDefaultBuildReqs():
defaultBuildReqs = (CfgList(CfgString), [
'bash:runtime',
'coreutils:runtime',
'filesystem',
'conary:runtime',
'conary-build:runtime',
'dev:runtime',
'grep:runtime',
'sed:runtime',
'findutils:runtime',
'gawk:runtime',
])
else:
defaultBuildReqs = (CfgList(CfgString), [])
resolveTroves = (CfgList(CfgQuotedLineList(CfgTroveSpec)),
[[('group-dist', None, None)]])
matchTroveRule = (CfgList(CfgString), [])
resolveTrovesOnly = (CfgBool, False)
reuseRoots = (CfgBool, False)
rpmRequirements = (
CfgList(CfgDependency), [],
"INTERNAL USE ONLY: Dep provided by the RPM to be used for "
"installation of the chroot.")
strictMode = (CfgBool, False)
targetLabel = (CfgLabel, versions.Label('NONE@local:NONE'))
uuid = (CfgUUID, '')
# DEPRECATED
subscribe = CfgDict(CfgString)
def __init__(self, parent, doc=None):
cfg.ConfigSection.__init__(self, parent, doc=None)
if hasattr(self, 'addConfigOption'):
# Conary < 2.5
for info in conarycfg.ConaryContext._getConfigOptions():
if info[0] not in self:
self.addConfigOption(*info)
def testGroupRecipe(self):
if compat.ConaryVersion().conaryVersion[0:2] == [1,2]:
raise testhelp.SkipTestException('test fails on 1.2')
repos = self.openRmakeRepository()
db = self.openRmakeDatabase()
self.buildCfg.shortenGroupFlavors = True
self.buildCfg.setSection('foo') # add context foo
self.buildCfg.configLine('buildFlavor desktop is:x86')
self.buildCfg.setSection('bar')
self.buildCfg.configLine('buildFlavor !desktop is:x86')
simple = self.addComponent('simple:runtime', '1.0-1-1', '')
other1 = self.addComponent('other:runtime', '1.0-1-1', '!desktop',
filePrimer=1)
other2 = self.addComponent('other:runtime', '1.0-1-1', 'desktop',
filePrimer=1)
# Prevent the desktop flag from being pre-filtered out
recipe = groupRecipe + '\n if Use.desktop: pass\n'
trv = self.addComponent('group-foo:source',
'/localhost@rpl:linux//rmakehost@local:linux/1:1.0-1',
[('group-foo.recipe', recipe)])
troveList = [
trv.getNameVersionFlavor() + ('foo',),
trv.getNameVersionFlavor() + ('bar',),
]
job = self.newJob(*troveList)
b = builder.Builder(self.rmakeCfg, job)
logFile = logfile.LogFile(self.workDir + '/buildlog')
logFile.redirectOutput()
b.build()
logFile.restoreOutput()
assert b.job.isBuilt(), str(b.job.getFailureReason())
assert len(b.job.getBuiltTroveList()) == 2, b.job.getBuiltTroveList()
for tup in b.job.getBuiltTroveList():
groupTrove = repos.getTrove(*tup)
# this is just a very basic test of builder -> group build.
# tests of the group cook code's ability to include the right
# version in particular cases should be in cooktest.py
if '!desktop' in str(groupTrove.getFlavor()):
other = other1
else:
other = other2
self.assertEqual(sorted(groupTrove.iterTroveList(strongRefs=True)),
[other.getNameVersionFlavor(),
simple.getNameVersionFlavor()])
def checkSanity(self):
if self.copyInConary:
# we're just overriding the version of conary used
# as long as that't the only sanity check we can return
# immediately
return
for job in self.jobList:
if job[0] == 'conary:python':
version = job[2][0].trailingRevision().getVersion()
try:
compat.ConaryVersion(version).checkRequiredVersion()
except errors.RmakeError, error:
errorMsg = str(error) + (
' - tried to install version %s in chroot' % version)
raise error.__class__(errorMsg)
def main(argv):
log.setVerbosity(log.WARNING)
try:
argv = list(argv)
debugAll = '--debug-all' in argv
if debugAll:
debuggerException = Exception
argv.remove('--debug-all')
else:
debuggerException = errors.RmakeInternalError
sys.excepthook = errors.genExcepthook(debug=debugAll,
debugCtrlC=debugAll)
compat.ConaryVersion().checkRequiredVersion()
return RmakeMain().main(argv, debuggerException=debuggerException)
except debuggerException, err:
raise
def _old_commit(helper, job):
'''
Commit a job to the target repository.
@param job: rMake job
'''
log.info('Starting commit')
_start_time = time.time()
okay, changeset, nbf_map = clone_job(helper, job)
# Sanity check the resulting changeset and produce a mapping of
# committed versions
mapping = {job.jobId: {}}
if okay:
for trove in iter_new_troves(changeset, helper):
# Make sure there are no references to the internal repos.
for _, child_version, _ in trove.iterTroveList(
strongRefs=True, weakRefs=True):
assert child_version.getHost() \
!= helper.cfg.reposName, \
"Trove %s references repository" % trove
#assert not child_name.endswith(':testinfo'), \
# "Trove %s references :testinfo component" % trove
trove_name, trove_version, trove_flavor = \
trove.getNameVersionFlavor()
trove_branch = trove_version.branch()
trove, _ = nbf_map[(trove_name, trove_branch, trove_flavor)]
trove_nvfc = trove.getNameVersionFlavor(withContext=True)
# map jobId -> trove -> binaries
mapping[trove.jobId].setdefault(trove_nvfc, []).append(
(trove_name, trove_version, trove_flavor))
else:
raise RuntimeError('failed to clone finished build')
if compat.ConaryVersion().signAfterPromote():
changeset = cook.signAbsoluteChangeset(changeset)
helper.getRepos().commitChangeSet(changeset)
_finish_time = time.time()
log.info('Commit took %.03f seconds', _finish_time - _start_time)
return mapping
def updateRecipes(repos, cfg, recipeList, committedSources):
committedSourcesByNB = {}
for name, version, flavor in committedSources:
committedSourcesByNB[name, version.branch()] = version
for recipe in recipeList:
recipeDir = os.path.dirname(recipe)
stateFilePath = recipeDir + '/CONARY'
if not os.path.exists(stateFilePath):
continue
conaryStateFile = state.ConaryStateFromFile(stateFilePath)
if not conaryStateFile.hasSourceState():
continue
context = conaryStateFile.getContext()
stateFile = conaryStateFile.getSourceState()
troveName = stateFile.getName()
branch = stateFile.getBranch()
if (troveName, branch) not in committedSourcesByNB:
continue
stateVersion = stateFile.getVersion()
newVersion = committedSourcesByNB[troveName, branch]
if stateVersion != versions.NewVersion():
log.info('Updating %s after commit' % recipeDir)
if compat.ConaryVersion().updateSrcTakesMultipleVersions():
try:
# Added in CNY-3035
checkin.nologUpdateSrc(repos, [recipeDir])
except checkin.builderrors.UpToDate:
pass # Don't mention if the source is already up to date
except checkin.builderrors.CheckinError, e:
e.logError()
except AttributeError:
checkin.updateSrc(repos, [recipeDir])
else:
curDir = os.getcwd()
try:
os.chdir(recipeDir)
checkin.updateSrc(repos)
finally:
os.chdir(curDir)
def _shadowAndCommit(conaryclient, cfg, recipeDir, stateFile, message):
repos = conaryclient.getRepos()
conaryCompat = compat.ConaryVersion()
oldSourceVersion = stateFile.getVersion()
targetLabel = cfg.getTargetLabel(oldSourceVersion)
if not targetLabel:
raise errors.RmakeError(
'Cannot cook local recipes unless a target label is set')
skipped, cs = conaryclient.createShadowChangeSet(str(targetLabel),
[stateFile.getNameVersionFlavor()])
recipePath = recipeDir + '/' + stateFile.getName().split(':')[0] + '.recipe'
recipeClass, pathList = _getPathList(repos, cfg, recipePath, relative=True)
troveName = stateFile.getName()
troveVersion = stateFile.getVersion()
if not skipped:
signAbsoluteChangeset(cs, None)
repos.commitChangeSet(cs)
log.info("Shadowing %s to internal repository..." % troveName)
shadowBranch = troveVersion.createShadow(targetLabel).branch()
shadowVersion = repos.findTrove(None,
(troveName, str(shadowBranch),
None), None)[0][1]
cwd = os.getcwd()
prefix = 'rmake-shadow-%s-' % troveName.split(':')[0]
shadowSourceDir = tempfile.mkdtemp(prefix=prefix)
try:
log.info("Committing local changes to %s to the"
" internal repository..." % troveName)
log.resetErrorOccurred()
checkin.checkout(repos, cfg, shadowSourceDir,
['%s=%s' % (troveName, shadowVersion)])
if compat.ConaryVersion().stateFileVersion() > 0:
kw = dict(repos=repos)
else:
kw = {}
# grab new and old state and make any modifications due to adding
# or deleting of files (we assume files that don't exist are
# autosource and can be ignored)
oldState = conaryCompat.ConaryStateFromFile(recipeDir + '/CONARY',
repos=repos).getSourceState()
newConaryState = conaryCompat.ConaryStateFromFile(
shadowSourceDir + '/CONARY',
repos=repos)
newState = newConaryState.getSourceState()
neededFiles = set(x[1] for x in oldState.iterFileList()
if os.path.exists(os.path.join(recipeDir, x[1])))
neededFiles.update(pathList)
autoSourceFiles = set(x[1] for x in oldState.iterFileList()
if oldState.fileIsAutoSource(x[0]))
existingFiles = set(x[1] for x in newState.iterFileList()
if os.path.exists(os.path.join(shadowSourceDir, x[1])))
toCopy = neededFiles & existingFiles
toDel = existingFiles - neededFiles
toAdd = neededFiles - existingFiles
for sourceFile in (toCopy | toAdd):
newPath = os.path.join(shadowSourceDir, sourceFile)
if os.path.dirname(sourceFile):
util.mkdirChain(os.path.dirname(newPath))
if os.path.isdir(sourceFile):
util.mkdirChain(newPath)
else:
shutil.copyfile(os.path.join(recipeDir, sourceFile), newPath)
os.chdir(shadowSourceDir)
if hasattr(cfg.sourceSearchDir, '_getUnexpanded'):
cfg.configKey('sourceSearchDir',
cfg.sourceSearchDir._getUnexpanded())
for f in toDel:
checkin.removeFile(f)
if toDel:
# toDel modifies the CONARY file on disk, so reload with the
# changes made there.
newState = conaryCompat.ConaryStateFromFile(
shadowSourceDir + '/CONARY',
repos=repos).getSourceState()
if conaryCompat.stateFileVersion() == 0:
checkin.addFiles(toAdd)
else:
oldPathIds = dict((x[1], x[0]) for x in oldState.iterFileList())
for path in toAdd:
if path in oldPathIds:
isConfig = oldState.fileIsConfig(oldPathIds[path])
else:
isConfig = _getConfigInfo(path)
checkin.addFiles([path], binary=not isConfig, text=isConfig)
if toAdd:
# get the new pathIDs for all the added troves,
# since we can't set the refresh setting without the
# needed pathIds
newState = conaryCompat.ConaryStateFromFile(
shadowSourceDir + '/CONARY',
repos=repos).getSourceState()
newPathIds = dict((x[1], x[0]) for x in newState.iterFileList())
for path in (toCopy | toAdd):
if path in oldPathIds:
isConfig = oldState.fileIsConfig(oldPathIds[path])
else:
isConfig = _getConfigInfo(path)
newState.fileIsConfig(newPathIds[path], isConfig)
for path in autoSourceFiles:
if path in newPathIds:
needsRefresh = oldState.fileNeedsRefresh(oldPathIds[path])
newState.fileNeedsRefresh(newPathIds[path], needsRefresh)
# if the factory changed, update it
if newState.getFactory() != oldState.getFactory():
newState.setFactory(oldState.getFactory())
# we may have modified the state file. Write it back out to
# disk so it will be picked up by the commit.
newConaryState.setSourceState(newState)
newConaryState.write(shadowSourceDir + '/CONARY')
if message is None and compat.ConaryVersion().supportsCloneCallback():
message = 'Automated rMake commit'
_doCommit('%s/%s' % (recipeDir, troveName), repos, cfg, message)
newState = state.ConaryStateFromFile(shadowSourceDir + '/CONARY', **kw)
return newState.getSourceState().getNameVersionFlavor()
finally:
os.chdir(cwd)
if hasattr(cfg.sourceSearchDir, '_getUnexpanded'):
cfg.configKey('sourceSearchDir',
cfg.sourceSearchDir._getUnexpanded())
shutil.rmtree(shadowSourceDir)
try:
os.chdir('/tmp') # make sure we're in a directory
# that we can write to. Although
# this _shouldn't_ be an issue,
# conary 1.0.{19,20} require it.
# finally actually cook the recipe!
groupOptions = cook.GroupCookOptions(
alwaysBumpCount=False,
shortenFlavors=cfg.shortenGroupFlavors,
errorOnFlavorChange=False)
m = macros.Macros()
m._override('buildlabel', str(buildLabel))
m._override('buildbranch', str(buildBranch))
m._override('binarybranch', str(binaryBranch))
toCook = compat.ConaryVersion().getObjectsToCook(
loaders, recipeClasses)
built = cook.cookObject(repos,
cfg,
toCook,
version,
prep=False,
macros=m,
targetLabel=targetLabel,
changeSetFile=csFile,
alwaysBumpCount=False,
ignoreDeps=False,
logBuild=True,
crossCompile=crossCompile,
requireCleanSources=False,
groupOptions=groupOptions)
if not built:
def _commitRecipe(conaryclient, cfg, recipePath, message, branch=None):
repos = conaryclient.getRepos()
conaryCompat = compat.ConaryVersion()
recipeClass, pathList = _getPathList(repos, cfg, recipePath)
sourceName = recipeClass.name + ':source'
log.info("Creating a copy of %s in the rMake internal repository..." % recipeClass.name)
cwd = os.getcwd()
recipeDir = tempfile.mkdtemp()
log.resetErrorOccurred()
try:
fileNames = []
# Create a source trove that matches the recipe we're trying to cook
if not branch:
branch = versions.Branch([cfg.buildLabel])
targetLabel = cfg.getTargetLabel(branch)
if compat.ConaryVersion().supportsNewPkgBranch():
buildBranch = branch.createShadow(targetLabel)
kw = dict(buildBranch=buildBranch)
else:
buildBranch = versions.Branch([targetLabel])
kw={}
cfg.buildLabel = targetLabel
if not repos.getTroveLeavesByBranch(
{ sourceName : { buildBranch : None } }).get(sourceName, None):
# we pass it None for repos to avoid the label-based check for
# existing packages.
checkin.newTrove(None, cfg, recipeClass.name, dir=recipeDir, **kw)
else:
# see if this package exists on our build branch
checkin.checkout(repos, cfg, recipeDir,
['%s=%s' % (sourceName, buildBranch)])
os.chdir(recipeDir)
sourceState = state.ConaryStateFromFile(recipeDir + '/CONARY').getSourceState()
fileNames = dict((os.path.basename(x), x) for x in pathList)
for (pathId, baseName, fileId, version) in list(sourceState.iterFileList()):
# update or remove any currently existing files
if baseName not in fileNames:
sourceState.removeFilePath(baseName)
else:
shutil.copyfile(fileNames[baseName],
os.path.join(recipeDir, baseName))
del fileNames[baseName]
for baseName, path in fileNames.iteritems():
shutil.copyfile(path, os.path.join(recipeDir, baseName))
if conaryCompat.stateFileVersion() > 0:
# mark all the files as binary - this this version can
# never be checked in, it doesn't really matter, but
# conary likes us to give a value.
for fileName in fileNames:
isConfig = _getConfigInfo(fileName)
checkin.addFiles([fileName], binary=not isConfig, text=isConfig)
else:
checkin.addFiles(fileNames)
_doCommit(recipePath, repos, cfg, 'Temporary recipe build for rmake')
newState = conaryCompat.ConaryStateFromFile(recipeDir + '/CONARY',
repos=repos)
return newState.getSourceState().getNameVersionFlavor()
finally:
os.chdir(cwd)
shutil.rmtree(recipeDir)
def startRepository(cfg, fork=True, logger=None):
global conaryDir
baseDir = cfg.serverDir
if logger is None:
logger = log
reposDir = '%s/repos' % baseDir
util.mkdirChain(reposDir)
if not cfg.reposUser:
passwordFile = reposDir + '/password'
if os.path.exists(passwordFile):
password = open(passwordFile).readline()[:-1]
else:
password = ''.join(
[chr(random.randrange(ord('a'), ord('z'))) for x in range(10)])
open(passwordFile, 'w').write(password + '\n')
os.chmod(reposDir + '/password', 0700)
cfg.reposUser.addServerGlob(cfg.reposName, 'rmake', password)
serverConfig = os.path.join(cfg.getReposDir(), 'serverrc')
if os.path.exists(serverConfig):
os.unlink(serverConfig)
serverCfg = server.ServerConfig(os.path.join(cfg.getReposDir(),
'serverrc'))
serverCfg.repositoryDB = ('sqlite', cfg.getReposDbPath())
serverCfg.contentsDir = cfg.getContentsPath()
serverCfg.port = cfg.getReposInfo()[1]
serverCfg.configKey('serverName', cfg.reposName) # this works with either
# 1.0.16 or 1.0.17+
serverCfg.logFile = cfg.getReposDir() + '/repos.log'
serverCfg.logFile = None
# Transfer SSL settings from rMake config object
if hasattr(server, 'SSL') and server.SSL:
# The server supports starting in SSL mode
serverCfg.useSSL = cfg.reposRequiresSsl()
serverCfg.sslCert = cfg.sslCertPath
serverCfg.sslKey = cfg.sslCertPath
elif cfg.reposRequiresSsl():
raise errors.RmakeError(
'Tried to start repository at %s, but missing ssl server library: Please install m2crypto'
% (cfg.getRepositoryUrl(), ))
(driver, database) = serverCfg.repositoryDB
db = dbstore.connect(database, driver)
# Note - this will automatically migrate this repository!
# Since this is a throwaway repos anyway, I think that's
# acceptable.
compat.ConaryVersion().loadServerSchema(db)
db.commit()
db.close()
user, password = cfg.reposUser.find(cfg.reposName)
addUser(serverCfg, user, password, write=True)
if not serverCfg.useSSL:
# allow anonymous access if we're not securing this repos
# by using SSL - no use securing access if it's all going to be
# viewable via tcpdump.
addUser(serverCfg, 'anonymous', 'anonymous')
if fork:
pid = os.fork()
if pid:
try:
pingServer(cfg)
except:
killServer(pid)
raise
logger.info('Started repository "%s" on port %s (pid %s)' %
(cfg.reposName, serverCfg.port, pid))
return pid
elif hasattr(logger, 'close'):
logger.close()
try:
os.chdir(cfg.getReposDir())
serverrc = open(cfg.getReposConfigPath(), 'w')
serverCfg.store(serverrc, includeDocs=False)
util.mkdirChain(os.path.dirname(cfg.getReposLogPath()))
logFile = logfile.LogFile(cfg.getReposLogPath())
logFile.redirectOutput(close=True)
serverrc.close()
os.execv('%s/server/server.py' % conaryDir, [
'%s/server/server.py' % conaryDir, '--config-file',
cfg.getReposConfigPath()
])
except Exception, err:
print >> sys.stderr, 'Could not start repository server: %s' % err
os._exit(1)
def getRecipeObj(repos,
name,
version,
flavor,
trv,
loadInstalledSource=None,
installLabelPath=None,
loadRecipeSpecs=None,
buildLabel=None,
groupRecipeSource=None,
cfg=None):
if cfg:
cfg = copy.deepcopy(cfg)
else:
cfg = conarycfg.ConaryConfiguration(False)
cfg.initializeFlavors()
branch = version.branch()
if not buildLabel:
buildLabel = version.branch().label()
if not installLabelPath:
cfg.installLabelPath = [buildLabel]
else:
cfg.installLabelPath = installLabelPath
cfg.buildFlavor = flavor
cfg.defaultBasePackages = []
name = name.split(':')[0]
use.LocalFlags._clear()
assert (flavorutil.getArch(flavor))
use.setBuildFlagsFromFlavor(name, flavor, error=False)
use.resetUsed()
use.track(True)
ignoreInstalled = not loadInstalledSource
macros = {
'buildlabel': buildLabel.asString(),
'buildbranch': version.branch().asString()
}
cfg.lookaside = tempfile.mkdtemp()
try:
loader = RecipeLoaderFromSourceTrove(trv,
repos,
cfg,
name + ':source',
branch,
ignoreInstalled=ignoreInstalled,
db=loadInstalledSource,
buildFlavor=flavor)
recipeClass = loader.getRecipe()
recipeClass._trove = trv
if recipe.isGroupRecipe(recipeClass):
recipeObj = recipeClass(repos,
cfg,
buildLabel,
flavor,
None,
extraMacros=macros)
recipeObj.sourceVersion = version
recipeObj.loadPolicy()
recipeObj.setup()
elif (recipe.isPackageRecipe(recipeClass)
or recipe.isFactoryRecipe(recipeClass)
or recipe.isCapsuleRecipe(recipeClass)):
if recipe.isFactoryRecipe(recipeClass):
#This requires a specific capability in conary
compat.ConaryVersion().requireFactoryRecipeGeneration()
#Load the FactoryRecipe
factoryClass = recipeClass
loaded = cook.loadFactoryRecipe(factoryClass, cfg, repos,
flavor)
recipeClass = loaded.getRecipe()
lcache = lookaside.RepositoryCache(repos)
recipeObj = recipeClass(cfg,
lcache, [],
macros,
lightInstance=True)
recipeObj.sourceVersion = version
recipeObj.populateLcache()
if not recipeObj.needsCrossFlags():
recipeObj.crossRequires = []
recipeObj.loadPolicy()
recipeObj.setup()
elif recipe.isInfoRecipe(recipeClass):
recipeObj = recipeClass(cfg, None, None, macros)
recipeObj.sourceVersion = version
recipeObj.setup()
elif recipe.isRedirectRecipe(recipeClass):
binaryBranch = version.getBinaryVersion().branch()
recipeObj = recipeClass(repos, cfg, binaryBranch, flavor)
recipeObj.sourceVersion = version
recipeObj.setup()
elif recipe.isFileSetRecipe(recipeClass):
recipeObj = recipeClass(repos,
cfg,
buildLabel,
flavor,
extraMacros=macros)
recipeObj.sourceVersion = version
recipeObj.setup()
else:
raise RuntimeError, 'Unknown class type %s for recipe %s' % (
recipeClass, name)
finally:
util.rmtree(cfg.lookaside)
return recipeObj, loader
def commitJobs(conaryclient,
jobList,
reposName,
message=None,
commitOutdatedSources=False,
sourceOnly=False,
excludeSpecs=None,
writeToFile=None):
jobsToCommit = {}
alreadyCommitted = []
finalCs = changeset.ReadOnlyChangeSet()
mapping = {}
for job in jobList:
if job.isCommitted():
alreadyCommitted.append(job)
else:
jobsToCommit[job.jobId] = job
jobsToCommit = jobsToCommit.values() # dedup job list
if not jobsToCommit:
err = 'Job(s) already committed'
return False, err
allTroves = []
trovesByBranch = {}
alreadyCommitted = False
for job in jobsToCommit:
mapping[job.jobId] = {}
for trove in job.iterTroves():
allTroves.append(trove)
troveVersion = trove.getVersion()
if troveVersion.getHost() == reposName:
if not troveVersion.branch().hasParentBranch():
message = ('Cannot commit filesystem cook %s - '
' nowhere to commit to!' % trove.getName())
return False, message
assert (allTroves)
source = trovesource.SimpleTroveSource()
if excludeSpecs:
excludeSpecsWithContext = {}
troveMap = {}
for excludeSpec in excludeSpecs:
if len(excludeSpec) == 4:
context = excludeSpec[3]
else:
context = None
excludeSpecsWithContext.setdefault(excludeSpec[:3],
[]).append(context)
excludeSpecs = [x[:3] for x in excludeSpecs]
for trove in allTroves:
troveTup = (trove.getName().split(':')[0], trove.getVersion(),
trove.getFlavor())
source.addTrove(*troveTup)
troveMap.setdefault(troveTup, []).append(trove)
source.searchAsDatabase()
matches = source.findTroves(None,
excludeSpecs,
None,
allowMissing=True)
trvMatches = []
for excludeSpec, matchList in matches.iteritems():
contexts = excludeSpecsWithContext[excludeSpec]
for match in matchList:
for trv in troveMap[match]:
if trv.context in contexts or None in contexts:
trvMatches.append(trv)
allTroves = [x for x in allTroves if x not in trvMatches]
if not allTroves:
message = ('All troves excluded - not committing')
return False, message
repos = conaryclient.getRepos()
trovesByNBF = {}
sourcesToCheck = []
branchMap = {}
trovesToClone = []
for trove in allTroves:
builtTroves = list(trove.iterBuiltTroves())
if not builtTroves:
continue
if builtTroves[0][1].getHost() != reposName:
alreadyCommitted = True
for n, v, f in builtTroves:
trovesByNBF[n, v.branch(), f] = (trove, v)
continue
troveVersion = trove.getVersion()
if troveVersion.getHost() == reposName:
sourceTup = (trove.getName(), troveVersion, Flavor())
targetBranch = troveVersion.branch().parentBranch()
branchMap[troveVersion.branch()] = targetBranch
nbf = trove.getName(), targetBranch, Flavor()
if nbf in trovesByNBF:
if trovesByNBF[nbf][1] != troveVersion:
badVersion = trovesByNBF[nbf][1]
return False, (
"Cannot commit two different versions of source component %s:"
" %s and %s" %
(trove.getName(), troveVersion, badVersion))
trovesByNBF[nbf] = trove, troveVersion
sourcesToCheck.append(sourceTup)
if sourceOnly:
continue
for troveTup in builtTroves:
branch = troveTup[1].branch()
targetBranch = branch.parentBranch()
# add mapping so that when the cloning is done
# we can tell what commit resulted in what binaries.
nbf = (troveTup[0], targetBranch, troveTup[2])
if nbf in trovesByNBF:
otherBinary = trovesByNBF[nbf][0].getBinaryTroves()[0]
if otherBinary[1].branch() == targetBranch:
# this one's already committed.
break
# discard the later of the two commits.
if trovesByNBF[nbf][0].getVersion() > trove.getVersion():
# we're the earlier one
badTrove, badVersion = trovesByNBF[nbf]
newTrove = trove
newVersion = troveTup[1]
else:
badTrove = trove
badVersion = troveTup[1]
newTrove, newVersion = trovesByNBF[nbf]
name = nbf[0]
flavor = nbf[2]
skipped = []
for badTroveTup in badTrove.iterBuiltTroves():
badNbf = (badTroveTup[0], targetBranch, badTroveTup[2])
if not ':' in badTroveTup[0]:
skipped.append(badTroveTup[0])
if badNbf in trovesByNBF and badTrove is trovesByNBF[
badNbf][0]:
del trovesByNBF[badNbf]
skipped = '%s' % (', '.join(skipped))
log.warning("Not committing %s on %s[%s]%s - overridden by"
" %s[%s]%s" %
(skipped, badTroveTup[1], badTroveTup[2],
badTrove.getContextStr(), newVersion, flavor,
newTrove.getContextStr()))
if trove is badTrove:
break
trovesByNBF[nbf] = trove, troveTup[1]
branchMap[branch] = targetBranch
for nbf, (trove, tupVersion) in trovesByNBF.items():
if tupVersion.branch() != nbf[1]:
trovesToClone.append((nbf[0], tupVersion, nbf[2]))
if not trovesToClone:
if sourceOnly:
err = 'Could not find sources to commit'
elif alreadyCommitted:
log.warning('All built troves have already been committed')
return True, {}
else:
err = 'Can only commit built troves, none found'
return False, err
if sourcesToCheck and not commitOutdatedSources:
outdated = _checkOutdatedSources(repos, sourcesToCheck)
if outdated:
outdated = ( '%s=%s (replaced by newer %s)' \
% (name, builtVer, newVer.trailingRevision())
for (name, builtVer, newVer) in outdated)
err = ('The following source troves are out of date:\n%s\n\n'
'Use --commit-outdated-sources to commit anyway' %
'\n'.join(outdated))
return False, err
# only update build info if we'll be okay if some buildreqs are not
# updated
updateBuildInfo = compat.ConaryVersion().acceptsPartialBuildReqCloning()
callback = callbacks.CloneCallback(conaryclient.cfg, message)
passed, cs = conaryclient.createTargetedCloneChangeSet(
branchMap,
trovesToClone,
updateBuildInfo=updateBuildInfo,
cloneSources=False,
trackClone=False,
callback=callback,
fullRecurse=False)
if passed:
oldTroves = []
for troveCs in cs.iterNewTroveList():
if troveCs.getOldVersion():
oldTroves.append(troveCs.getOldNameVersionFlavor())
if oldTroves:
oldDict = {}
for oldTrove in repos.getTroves(oldTroves):
oldDict.setdefault(oldTrove.getNameVersionFlavor(),
[]).append(oldTrove)
for troveCs in cs.iterNewTroveList():
if troveCs.getOldVersion():
trv = oldDict[troveCs.getOldNameVersionFlavor()].pop()
trv.applyChangeSet(troveCs)
else:
trv = Trove(troveCs)
for _, childVersion, _ in trv.iterTroveList(strongRefs=True,
weakRefs=True):
# make sure there are not any references to the internal
# rmake repository - that would be a bad bug - easy to
# do with the way we do cooking of groups.
onRepos = childVersion.getHost() == reposName
assert not onRepos, "Trove %s references repository" % trv
n, v, f = troveCs.getNewNameVersionFlavor()
trove, troveVersion = trovesByNBF[n, v.branch(), f]
troveNVFC = trove.getNameVersionFlavor(withContext=True)
# map jobId -> trove -> binaries
mapping[trove.jobId].setdefault(troveNVFC, []).append((n, v, f))
else:
return False, 'Creating clone failed'
signatureKey = conaryclient.cfg.signatureKey
if signatureKey and compat.ConaryVersion().signAfterPromote():
finalCs = signAbsoluteChangeset(cs, signatureKey)
if writeToFile:
cs.writeToFile(writeToFile)
else:
repos.commitChangeSet(cs, callback=callback)
return True, mapping
def getTrovesToBuild(cfg, conaryclient, troveSpecList, message=None,
recurseGroups=BUILD_RECURSE_GROUPS_NONE, matchSpecs=None,
reposName=None, updateSpecs=None):
toBuild = []
toFind = {}
groupsToFind = []
if not matchSpecs:
matchSpecs = []
if reposName is None:
reposName = cfg.reposName
repos = conaryclient.getRepos()
cfg.resolveTroveTups = _getResolveTroveTups(cfg, repos)
cfg.recurseGroups = int(recurseGroups)
cfg.buildTroveSpecs = []
newTroveSpecs = []
recipesToCook = []
for troveSpec in list(troveSpecList):
if not isinstance(troveSpec, tuple):
troveSpec = cmdutil.parseTroveSpec(troveSpec)
if len(troveSpec) == 3:
context = ''
else:
context = troveSpec[3]
troveSpec = troveSpec[:3]
if (troveSpec[0].startswith('group-') and not recurseGroups
and not compat.ConaryVersion().supportsCloneNonRecursive()):
log.warning('You will not be able to commit this group build'
' without upgrading conary.')
if troveSpec[2] is None:
troveSpec = (troveSpec[0], troveSpec[1], deps.parseFlavor(''))
if (not troveSpec[1] and not os.path.isdir(troveSpec[0])
and os.access(troveSpec[0], os.R_OK)
and troveSpec[0].endswith('.recipe')):
# don't rely on cwd, but do allow for symlinks to change
# when restarting. Is that sane? Or should I just do realpath?
troveSpec = (os.path.abspath(troveSpec[0]),) + troveSpec[1:]
cfg.buildTroveSpecs.append(troveSpec)
recipesToCook.append((os.path.realpath(troveSpec[0]), troveSpec[2]))
continue
cfg.buildTroveSpecs.append(troveSpec)
if troveSpec[0].startswith('group-') and recurseGroups:
groupsToFind.append(troveSpec)
if recurseGroups == BUILD_RECURSE_GROUPS_SOURCE:
newTroveSpecs.append(troveSpec)
else:
newTroveSpecs.append(troveSpec)
localTroves = [(_getLocalCook(conaryclient, cfg, x[0], message), x[1])
for x in recipesToCook ]
localTroves = [(x[0][0], x[0][1], x[1]) for x in localTroves]
if recurseGroups == BUILD_RECURSE_GROUPS_SOURCE:
compat.ConaryVersion().requireFindGroupSources()
localGroupTroves = [ x for x in localTroves
if x[0].startswith('group-') ]
toBuild.extend(_findSourcesForSourceGroup(repos, reposName, cfg,
groupsToFind,
localGroupTroves,
updateSpecs))
elif recurseGroups == BUILD_RECURSE_GROUPS_BINARY:
newTroveSpecs.extend(_findSpecsForBinaryGroup(repos, reposName, cfg,
groupsToFind,
updateSpecs))
for troveSpec in newTroveSpecs:
sourceName = troveSpec[0].split(':')[0] + ':source'
s = toFind.setdefault((sourceName, troveSpec[1], None), [])
if troveSpec[2] not in s:
s.append(troveSpec[2])
results = repos.findTroves(cfg.buildLabel, toFind, None)
for troveSpec, troveTups in results.iteritems():
flavorList = toFind[troveSpec]
for troveTup in troveTups:
for flavor in flavorList:
toBuild.append((troveTup[0], troveTup[1], flavor))
toBuild.extend(localTroves)
if matchSpecs:
toBuild = _filterListByMatchSpecs(reposName, matchSpecs, toBuild)
return toBuild
def clone_job(helper, job):
'''
Create a changeset that will clone all built troves into the target
label.
'''
branch_map = {} # source_branch -> target_branch
# nbf_map := name, target_branch, flavor -> trove, source_version
# This maps a given name and built flavor back to the BuildTrove that
# created it, allowing us to find duplicate builds and build a clone
# job.
nbf_map = {}
for trove in job.iterTroves():
source_name, source_version, _ = trove.getNameVersionFlavor()
#assert source_version.getHost() == helper.cfg.reposName
# Determine which branch this will be committed to
source_branch = origin_branch = source_version.branch()
target_branch = origin_branch.createShadow(
helper.plan.getTargetLabel())
# Mark said branch for final promote
branch_map[source_branch] = target_branch
# Check for different versions of the same trove
nbf = source_name, target_branch, deps.Flavor()
if nbf in nbf_map and nbf_map[nbf][1] != source_version:
bad_version = nbf_map[nbf][1]
raise RuntimeError("Cannot commit two different versions of "
"source component %s: %s and %s" % (source_name,
source_version, bad_version))
nbf_map[nbf] = trove, source_version
# Add binary troves to mapping
for bin_name, bin_version, bin_flavor in trove.iterBuiltTroves():
# Don't commit test info
#if bin_name.endswith(':testinfo'):
# continue
nbf = bin_name, target_branch, bin_flavor
if nbf in nbf_map:
# Eliminate duplicate commits of the same NBF by keeping
# only the newer package
other_version = nbf_map[nbf][0].getBinaryTroves()[0][1]
if other_version < bin_version:
bad_trove, new_trove = nbf_map[nbf][0], trove
new_version = bin_version
else:
new_trove, bad_trove = nbf_map[nbf][0], trove
new_version = other_version
new_name = new_trove.getName().split(':')[0]
# Delete commit maps for the entire rejected package
for bad_name, bad_version, bad_flavor \
in bad_trove.iterBuiltTroves():
bad_nbf = (bad_name, target_branch, bad_flavor)
if not ':' in bad_name:
log.warning('Not committing %s=%s[%s] - overridden '
'by %s=%s', bad_name, bad_version, bad_flavor,
new_name, new_version)
if bad_nbf in nbf_map \
and bad_trove is nbf_map[bad_nbf][0]:
log.debug('Purging %s=%s[%s]' % (bad_name, bad_version,
bad_flavor))
del nbf_map[bad_nbf]
# If this trove is the bad trove, stop processing it
if trove is bad_trove:
break
nbf_map[nbf] = trove, bin_version
# Determine what to clone
troves_to_clone = []
for (trv_name, _, trv_flavor), (trove, trv_version) \
in nbf_map.iteritems():
troves_to_clone.append((trv_name, trv_version, trv_flavor))
# Do the clone
update_build_info = compat.ConaryVersion()\
.acceptsPartialBuildReqCloning()
callback = callbacks.CloneCallback(helper.cfg,
helper.plan.commitMessage)
okay, changeset = helper.getClient().createTargetedCloneChangeSet(
branch_map, troves_to_clone,
updateBuildInfo=update_build_info,
cloneSources=False,
trackClone=False,
callback=callback,
#cloneOnlyByDefaultTroves=True,
fullRecurse=False)
return okay, changeset, nbf_map
