def runCommand(self, client, cfg, argSet, args):
if self.verbose:
log.setVerbosity(log.DEBUG)
else:
log.setVerbosity(log.INFO)
command, troveSpecs = self.requireParameters(args,
'troveSpec',
appendExtra=True)
if command == 'buildgroup':
log.warning(
'"buildgroup" is deprecated and will be removed in a future release - use "build --recurse" instead'
)
rebuild = (command == 'rebuild')
flavorSpec = argSet.pop('flavor', None)
if flavorSpec:
flavor = deps.parseFlavor(flavorSpec)
if flavor is None:
raise errors.ParseError("Invalid flavor: '%s'" % flavorSpec)
newFlavor = deps.overrideFlavor(client.buildConfig.buildFlavor,
flavor)
client.buildConfig.buildFlavor = newFlavor
newFlavors = []
for oldFlavor in client.buildConfig.flavor:
newFlavors.append(deps.overrideFlavor(oldFlavor, flavor))
client.buildConfig.flavor = newFlavors
matchSpecs = argSet.pop('match', [])
hosts = argSet.pop('host', [])
labels = argSet.pop('label', [])
recurseGroups = argSet.pop('recurse', False) or command == 'buildgroup'
if recurseGroups:
if argSet.pop('binary-search', False):
recurseGroups = client.BUILD_RECURSE_GROUPS_BINARY
elif not compat.ConaryVersion().supportsFindGroupSources():
log.warning('Your conary does not support recursing a group'
' source component, defaulting to searching the'
' binary version')
recurseGroups = client.BUILD_RECURSE_GROUPS_BINARY
else:
recurseGroups = client.BUILD_RECURSE_GROUPS_SOURCE
self._prep(client, argSet)
job = client.createBuildJob(troveSpecs,
limitToHosts=hosts,
limitToLabels=labels,
recurseGroups=recurseGroups,
matchSpecs=matchSpecs,
rebuild=rebuild)
return self._build(client, job, argSet)
def testCook(self):
repos = self.openRepository()
self.makeSourceTrove('test1', test1Recipe)
cookFlavor = deps.parseFlavor('readline,ssl,X')
troveTup = repos.findTrove(self.cfg.buildLabel,
('test1:source', None, None), None)[0]
troveTup = (troveTup[0], troveTup[1], cookFlavor)
targetLabel = versions.Label('localhost@LOCAL:linux')
newTargetLabel = versions.Label('localhost@LOCAL:linux')
# adding an unknown flavor shouldn't matter with latest conary.
self.cfg.buildFlavor = deps.overrideFlavor(self.cfg.buildFlavor,
deps.parseFlavor('foobar'))
logger_ = logger.Logger()
logPath, pid, buildInfo = self.discardOutput(cook.cookTrove,
self.cfg, repos, logger_,
targetLabel=targetLabel, *troveTup)
result = self._wait(buildInfo)
assert(result.isBuildSuccess())
repos.commitChangeSetFile(result.getChangeSetFile())
troveTup = repos.findTrove(newTargetLabel,
('test1', None, deps.parseFlavor('readline,ssl')),
None)[0]
assert(troveTup[1].branch().label() == newTargetLabel)
assert(str(troveTup[2]) == 'readline,ssl')
self.updatePkg('test1=%s[readline,ssl]' % newTargetLabel,
raiseError=True)
self.verifyFile(self.rootDir + '/foo/bar', str(self.cfg.buildLabel) + '\n')
def testChangedRecipeCook(self):
repos = self.openRepository()
self.openRmakeRepository()
trv = self.addComponent(
'test1:source=/localhost@rpl:linux//rmakehost@LOCAL:linux/1.2-0.1',
[('test1.recipe', test1Recipe)])
cookFlavor = deps.parseFlavor('readline,ssl,X')
troveTup = trv.getNameVersionFlavor()
troveTup = (troveTup[0], troveTup[1], cookFlavor)
targetLabel = versions.Label('rmakehost@LOCAL:linux')
newTargetLabel = versions.Label('rmakehost@LOCAL:linux')
# adding an unknown flavor shouldn't matter with latest conary.
self.cfg.buildFlavor = deps.overrideFlavor(self.cfg.buildFlavor,
deps.parseFlavor('foobar'))
logger_ = logger.Logger()
logPath, pid, buildInfo = self.discardOutput(cook.cookTrove,
self.cfg,
repos,
logger_,
targetLabel=targetLabel,
*troveTup)
result = self._wait(buildInfo)
assert (result.isBuildSuccess())
def testBasic(self):
trv = self.addComponent('nocross:source', '1.0-1', '',
[('nocross.recipe', nocrossRecipe)])
trv2 = self.addComponent('crosstool:source', '1.0-1', '',
[('crosstool.recipe', crosstoolRecipe)])
ccRoot = self.rmakeCfg.buildDir + '/chroots/crosscompiled'
trv3 = self.addComponent('crosscompiled:source', '1.0-1', '',
[('crosscompiled.recipe', crosscompiledRecipe % (ccRoot, ccRoot))])
self.openRmakeRepository()
troveList = [
(trv.getName(), trv.getVersion(),
deps.parseFlavor('!cross target: x86_64')),
(trv2.getName(), trv2.getVersion(),
deps.parseFlavor('cross target: x86_64')),
(trv3.getName(), trv3.getVersion(),
deps.parseFlavor('!cross target: x86_64')) ]
db = self.openRmakeDatabase()
self.buildCfg.flavor = [deps.overrideFlavor(self.buildCfg.flavor[0], deps.parseFlavor('~cross is:x86 target:x86_64'))]
job = self.newJob(*troveList)
db.subscribeToJob(job)
b = builder.Builder(self.rmakeCfg, job)
self.logFilter.add()
logFile = logfile.LogFile(self.workDir + '/buildlog')
logFile.redirectOutput()
try:
b.build()
except Exception:
b.worker.stopAllCommands()
raise
logFile.restoreOutput()
assert(set([x.getName() for x in b.dh.depState.getBuiltTroves()])
== set([trv.getName(), trv2.getName(), trv3.getName()]))
def testCook(self):
repos = self.openRepository()
self.makeSourceTrove('test1', test1Recipe)
cookFlavor = deps.parseFlavor('readline,ssl,X')
troveTup = repos.findTrove(self.cfg.buildLabel,
('test1:source', None, None), None)[0]
troveTup = (troveTup[0], troveTup[1], cookFlavor)
targetLabel = versions.Label('localhost@LOCAL:linux')
newTargetLabel = versions.Label('localhost@LOCAL:linux')
# adding an unknown flavor shouldn't matter with latest conary.
self.cfg.buildFlavor = deps.overrideFlavor(self.cfg.buildFlavor,
deps.parseFlavor('foobar'))
logger_ = logger.Logger()
logPath, pid, buildInfo = self.discardOutput(cook.cookTrove,
self.cfg,
repos,
logger_,
targetLabel=targetLabel,
*troveTup)
result = self._wait(buildInfo)
assert (result.isBuildSuccess())
repos.commitChangeSetFile(result.getChangeSetFile())
troveTup = repos.findTrove(
newTargetLabel, ('test1', None, deps.parseFlavor('readline,ssl')),
None)[0]
assert (troveTup[1].branch().label() == newTargetLabel)
assert (str(troveTup[2]) == 'readline,ssl')
self.updatePkg('test1=%s[readline,ssl]' % newTargetLabel,
raiseError=True)
self.verifyFile(self.rootDir + '/foo/bar',
str(self.cfg.buildLabel) + '\n')
def loadRecipe(repos, name, version, flavor, trv,
defaultFlavor=None, loadInstalledSource=None,
installLabelPath=None, buildLabel=None, groupRecipeSource=None,
cfg=None):
name = name.split(':')[0]
try:
if defaultFlavor is not None:
fullFlavor = deps.overrideFlavor(defaultFlavor, flavor)
else:
fullFlavor = flavor
# set up necessary flavors and track used flags before
# calling loadRecipe, since even loading the class
# may check some flags that may never be checked inside
# the recipe
recipeObj, loader = getRecipeObj(repos, name,
version, fullFlavor, trv,
loadInstalledSource=loadInstalledSource,
installLabelPath=installLabelPath,
buildLabel=buildLabel,
cfg=cfg)
relevantFlavor = use.usedFlagsToFlavor(recipeObj.name)
relevantFlavor = flavorutil.removeInstructionSetFlavor(relevantFlavor)
# always add in the entire arch flavor. We need to ensure the
# relevant flavor is unique per architecture, also, arch flavors
# can affect the macros used.
if defaultFlavor is not None:
relevantFlavor.union(flavor)
relevantFlavor.union(flavorutil.getArchFlags(fullFlavor))
relevantFlags = flavorutil.getFlavorUseFlags(relevantFlavor)
flags = flavorutil.getFlavorUseFlags(fullFlavor)
use.track(False)
for flagSet in ('Use',):
# allow local flags not to be used -- they are set to their default
if flagSet not in relevantFlags:
continue
for flag in relevantFlags[flagSet]:
if flag not in flags[flagSet]:
raise (RuntimeError,
"Recipe %s uses Flavor %s but value not known" %(name, flag))
if 'Arch' in relevantFlags:
for majarch in relevantFlags['Arch'].keys():
for subarch in relevantFlags['Arch'][majarch]:
if not use.Arch[majarch][subarch]:
#negative values for subarches are assumed
continue
if subarch not in flags['Arch'][majarch]:
log.error("arch %s.%s used but not specified" % (
majarch, subarch))
raise RuntimeError, (
"arch %s.%s used but not specified" % (
majarch, subarch))
use.resetUsed()
except:
log.error('Error Loading Recipe (%s, %s, %s):\n%s' %
(name, version, fullFlavor,
''.join(traceback.format_exc())))
raise
return loader, recipeObj, relevantFlavor
def _flavorsMatch(self, troveFlavor, provFlavor, reqFlavor, isCross):
if isCross:
troveFlavor = flavorutil.getSysRootFlavor(troveFlavor)
archFlavor = flavorutil.getBuiltFlavor(flavorutil.getArchFlags(
troveFlavor, getTarget=False,
withFlags=False))
if reqFlavor is None:
reqFlavor = archFlavor
else:
reqFlavor = deps.overrideFlavor(archFlavor, reqFlavor)
if flavorutil.getArchFlags(provFlavor).isEmpty():
provFlavor = deps.overrideFlavor(archFlavor, provFlavor)
if flavorutil.getBuiltFlavor(provFlavor).toStrongFlavor().satisfies(
reqFlavor.toStrongFlavor()):
return True
return False
def _flavorsMatch(self, troveFlavor, provFlavor, reqFlavor, isCross):
if isCross:
troveFlavor = flavorutil.getSysRootFlavor(troveFlavor)
archFlavor = flavorutil.getBuiltFlavor(flavorutil.getArchFlags(
troveFlavor, getTarget=False,
withFlags=False))
if reqFlavor is None:
reqFlavor = archFlavor
else:
reqFlavor = deps.overrideFlavor(archFlavor, reqFlavor)
if flavorutil.getArchFlags(provFlavor).isEmpty():
provFlavor = deps.overrideFlavor(archFlavor, provFlavor)
if flavorutil.getBuiltFlavor(provFlavor).toStrongFlavor().satisfies(
reqFlavor.toStrongFlavor()):
return True
return False
def makeBuildFlavorMap(prd):
baseFlavor = prd.getBaseFlavor() or prd.getPlatformBaseFlavor() or ""
baseFlavor = deps.parseFlavor(baseFlavor)
flavorSets = prd.getFlavorSets()
architectures = prd.getArchitectures()
if prd.platform:
flavorSets += prd.platform.getFlavorSets()
architectures = prd.platform.getArchitectures()
res = {}
for flavorSet in flavorSets:
for architecture in architectures:
flv = deps.parseFlavor(flavorSet.flavor)
arch = deps.parseFlavor(architecture.flavor)
flavor = deps.overrideFlavor(baseFlavor, flv)
flavor = deps.overrideFlavor(flavor, arch)
res[str(flavor)] = "%s %s" % (flavorSet.displayName, architecture.displayName)
return res
def runCommand(self, client, cfg, argSet, args):
if self.verbose:
log.setVerbosity(log.DEBUG)
else:
log.setVerbosity(log.INFO)
command, troveSpecs = self.requireParameters(args, 'troveSpec',
appendExtra=True)
if command == 'buildgroup':
log.warning('"buildgroup" is deprecated and will be removed in a future release - use "build --recurse" instead')
rebuild = (command == 'rebuild')
flavorSpec = argSet.pop('flavor', None)
if flavorSpec:
flavor = deps.parseFlavor(flavorSpec)
if flavor is None:
raise errors.ParseError("Invalid flavor: '%s'" % flavorSpec)
newFlavor = deps.overrideFlavor(client.buildConfig.buildFlavor,
flavor)
client.buildConfig.buildFlavor = newFlavor
newFlavors = []
for oldFlavor in client.buildConfig.flavor:
newFlavors.append(deps.overrideFlavor(oldFlavor, flavor))
client.buildConfig.flavor = newFlavors
matchSpecs = argSet.pop('match', [])
hosts = argSet.pop('host', [])
labels = argSet.pop('label', [])
recurseGroups = argSet.pop('recurse', False) or command == 'buildgroup'
if recurseGroups:
if argSet.pop('binary-search', False):
recurseGroups = client.BUILD_RECURSE_GROUPS_BINARY
elif not compat.ConaryVersion().supportsFindGroupSources():
log.warning('Your conary does not support recursing a group'
' source component, defaulting to searching the'
' binary version')
recurseGroups = client.BUILD_RECURSE_GROUPS_BINARY
else:
recurseGroups = client.BUILD_RECURSE_GROUPS_SOURCE
self._prep(client, argSet)
job = client.createBuildJob(troveSpecs, limitToHosts=hosts,
limitToLabels=labels,
recurseGroups=recurseGroups,
matchSpecs=matchSpecs,
rebuild=rebuild)
return self._build(client, job, argSet)
def getSources(self, resolveJob, cross=False):
cfg = resolveJob.getConfig()
if cross:
buildFlavor = deps.overrideFlavor(resolveJob.buildCfg.buildFlavor,
resolveJob.getTrove().getFlavor())
buildFlavor = deps.overrideFlavor(buildFlavor,
deps.parseFlavor('!cross'))
builtTroveTups = (resolveJob.getCrossTroves()
+ resolveJob.getBuiltTroves())
cfg = copy.deepcopy(cfg)
cfg.flavor = [ flavorutil.setISFromTargetFlavor(buildFlavor) ]
else:
builtTroveTups = resolveJob.getBuiltTroves()
if cfg.isolateTroves:
builtTroves = []
else:
builtTroves = self.repos.getTroves(builtTroveTups, withFiles=False)
builtTroveSource = resolvesource.BuiltTroveSource(builtTroves,
self.repos)
if builtTroves:
# this makes sure that if someone searches for a buildreq on
# :branch, and the only thing we have is on :branch/rmakehost,
# the trove will be found.
rMakeHost = builtTroves[0].getVersion().trailingLabel().getHost()
builtTroveSource = recipeutil.RemoveHostSource(builtTroveSource,
rMakeHost)
if cfg.resolveTroveTups:
searchSource, resolveSource = self.getSourcesWithResolveTroves(cfg,
cfg.resolveTroveTups,
builtTroveSource)
else:
searchSource = resolvesource.DepHandlerSource(builtTroveSource,
[], self.repos,
expandLabelQueries=True)
resolveSource = resolvesource.rMakeResolveSource(cfg,
builtTroveSource, [],
None,
self.repos)
if cross:
resolveSource.removeFileDependencies = True
return searchSource, resolveSource
def getBuiltFlavor(flavor):
if not hasTarget(flavor):
majorArch = getTargetArch(flavor)[1]
if majorArch:
f = deps.Flavor()
f.addDep(deps.InstructionSetDependency, deps.Dependency(majorArch))
flavor = deps.overrideFlavor(flavor, f)
return flavor
if flavor.stronglySatisfies(crossFlavor):
return flavor
return setISFromTargetFlavor(flavor)
def getBuiltFlavor(flavor):
if not hasTarget(flavor):
majorArch = getTargetArch(flavor)[1]
if majorArch:
f = deps.Flavor()
f.addDep(deps.InstructionSetDependency, deps.Dependency(majorArch))
flavor = deps.overrideFlavor(flavor, f)
return flavor
if flavor.stronglySatisfies(crossFlavor):
return flavor
return setISFromTargetFlavor(flavor)
def getSources(self, resolveJob, cross=False):
cfg = resolveJob.getConfig()
if cross:
buildFlavor = deps.overrideFlavor(
resolveJob.buildCfg.buildFlavor,
resolveJob.getTrove().getFlavor())
buildFlavor = deps.overrideFlavor(buildFlavor,
deps.parseFlavor('!cross'))
builtTroveTups = (resolveJob.getCrossTroves() +
resolveJob.getBuiltTroves())
cfg = copy.deepcopy(cfg)
cfg.flavor = [flavorutil.setISFromTargetFlavor(buildFlavor)]
else:
builtTroveTups = resolveJob.getBuiltTroves()
if cfg.isolateTroves:
builtTroves = []
else:
builtTroves = self.repos.getTroves(builtTroveTups, withFiles=False)
builtTroveSource = resolvesource.BuiltTroveSource(
builtTroves, self.repos)
if builtTroves:
# this makes sure that if someone searches for a buildreq on
# :branch, and the only thing we have is on :branch/rmakehost,
# the trove will be found.
rMakeHost = builtTroves[0].getVersion().trailingLabel().getHost()
builtTroveSource = recipeutil.RemoveHostSource(
builtTroveSource, rMakeHost)
if cfg.resolveTroveTups:
searchSource, resolveSource = self.getSourcesWithResolveTroves(
cfg, cfg.resolveTroveTups, builtTroveSource)
else:
searchSource = resolvesource.DepHandlerSource(
builtTroveSource, [], self.repos, expandLabelQueries=True)
resolveSource = resolvesource.rMakeResolveSource(
cfg, builtTroveSource, [], None, self.repos)
if cross:
resolveSource.removeFileDependencies = True
return searchSource, resolveSource
def testOverrideFromConaryConfig(self):
cfg = buildcfg.BuildConfiguration(readConfigFiles=False)
cfg.strictMode = False
cfg.copyInConfig = True
cfg.configLine('buildLabel foo.rpath.org@rpl:devel')
cfg.configLine('flavor foo')
cfg.useConaryConfig(self.cfg)
assert (cfg.installLabelPath == self.cfg.installLabelPath)
assert (str(cfg.buildLabel) == 'foo.rpath.org@rpl:devel')
assert (str(cfg.flavor[0]) == 'foo')
cfg.initializeFlavors()
assert (cfg.flavor[0] == deps.overrideFlavor(self.cfg.flavor[0],
deps.parseFlavor('foo')))
def testOverrideFromConaryConfig(self):
cfg = buildcfg.BuildConfiguration(readConfigFiles=False)
cfg.strictMode = False
cfg.copyInConfig = True
cfg.configLine('buildLabel foo.rpath.org@rpl:devel')
cfg.configLine('flavor foo')
cfg.useConaryConfig(self.cfg)
assert(cfg.installLabelPath == self.cfg.installLabelPath)
assert(str(cfg.buildLabel) == 'foo.rpath.org@rpl:devel')
assert(str(cfg.flavor[0]) == 'foo')
cfg.initializeFlavors()
assert(cfg.flavor[0] == deps.overrideFlavor(self.cfg.flavor[0],
deps.parseFlavor('foo')))
def expand_targets(cfg):
'''
Accept a target config section and return a list of build flavors.
'''
# If no configuration is available, build is: x86
if not cfg or (not cfg.flavor_set and not cfg.flavor):
return SETS['rPL 1']['x86']
# Ensure flavor_set and flavor aren't both set
# This might be supported later, by recombining flavors from each
if cfg.flavor_set and cfg.flavor:
raise ValueError('flavor_set and flavor cannot be used together')
if cfg.flavor_set:
if ':' in cfg.flavor_set:
distro, set_name = cfg.flavor_set.split(':', 1)
else:
distro, set_name = 'rPL 1', cfg.flavor_set
try:
return SETS[distro][set_name]
except KeyError:
raise RuntimeError('flavor set "%s" is not defined for '
'distro "%s"' % (distro, set_name))
else:
ret = []
for flavor in cfg.flavor:
if '%' in flavor:
# Handle "templates" in flavors, e.g.
# flavor %rPL 1:x86% xen,dom0,!domU,!vmware
match = FLAVOR_TEMPLATE_RE.search(flavor)
if not match:
raise RuntimeError('Malformed template in flavor')
stripped_flavor = FLAVOR_TEMPLATE_RE.sub('', flavor)
if '%' in stripped_flavor:
raise RuntimeError('Cannot have multiple templates '
'in flavor')
distro_name, arch_name = match.groups()
distro = _DISTROS[distro_name]
base = distro['base'].copy()
base.union(distro['arches'][arch_name][0])
suffix = deps.parseFlavor(stripped_flavor)
ret.append(deps.overrideFlavor(base, suffix))
else:
ret.append(deps.parseFlavor(flavor))
return ret
def expand_targets(cfg):
'''
Accept a target config section and return a list of build flavors.
'''
# If no configuration is available, build is: x86
if not cfg or (not cfg.flavor_set and not cfg.flavor):
return SETS['rPL 1']['x86']
# Ensure flavor_set and flavor aren't both set
# This might be supported later, by recombining flavors from each
if cfg.flavor_set and cfg.flavor:
raise ValueError('flavor_set and flavor cannot be used together')
if cfg.flavor_set:
if ':' in cfg.flavor_set:
distro, set_name = cfg.flavor_set.split(':', 1)
else:
distro, set_name = 'rPL 1', cfg.flavor_set
try:
return SETS[distro][set_name]
except KeyError:
raise RuntimeError('flavor set "%s" is not defined for '
'distro "%s"' % (distro, set_name))
else:
ret = []
for flavor in cfg.flavor:
if '%' in flavor:
# Handle "templates" in flavors, e.g.
# flavor %rPL 1:x86% xen,dom0,!domU,!vmware
match = FLAVOR_TEMPLATE_RE.search(flavor)
if not match:
raise RuntimeError('Malformed template in flavor')
stripped_flavor = FLAVOR_TEMPLATE_RE.sub('', flavor)
if '%' in stripped_flavor:
raise RuntimeError('Cannot have multiple templates '
'in flavor')
distro_name, arch_name = match.groups()
distro = _DISTROS[distro_name]
base = distro['base'].copy()
base.union(distro['arches'][arch_name][0])
suffix = deps.parseFlavor(stripped_flavor)
ret.append(deps.overrideFlavor(base, suffix))
else:
ret.append(deps.parseFlavor(flavor))
return ret
def mergeFlavors(self, queryOptions, flavor, defaultFlavorPath=None):
"""
Merges the given flavor with the flavorPath - if flavor
doesn't contain use flags, then include the defaultFlavor's
use flags. If flavor doesn't contain an instruction set, then
include the flavorpath's instruction set(s)
"""
if defaultFlavorPath is None:
defaultFlavorPath = queryOptions.defaultFlavorPath
if not defaultFlavorPath:
return [flavor]
if flavor is None:
return defaultFlavorPath
return [ deps.overrideFlavor(x, flavor) for x in defaultFlavorPath ]
def mergeFlavors(self, queryOptions, flavor, defaultFlavorPath=None):
"""
Merges the given flavor with the flavorPath - if flavor
doesn't contain use flags, then include the defaultFlavor's
use flags. If flavor doesn't contain an instruction set, then
include the flavorpath's instruction set(s)
"""
if defaultFlavorPath is None:
defaultFlavorPath = queryOptions.defaultFlavorPath
if not defaultFlavorPath:
return [flavor]
if flavor is None:
return defaultFlavorPath
return [deps.overrideFlavor(x, flavor) for x in defaultFlavorPath]
def overrideFlavors(self, queryOptions, flavor, defaultFlavorPath=None):
"""
override the flavors in the defaultFlavorPath with flavor,
replacing instruction set entirely if given.
"""
if defaultFlavorPath is None:
defaultFlavorPath = queryOptions.defaultFlavorPath
if flavor is None:
return defaultFlavorPath
if not defaultFlavorPath:
return [flavor]
flavors = []
for defaultFlavor in queryOptions.defaultFlavorPath:
flavors.append(deps.overrideFlavor(defaultFlavor, flavor,
mergeType = deps.DEP_MERGE_TYPE_PREFS))
return flavors
def _cookTrove(cfg, repos, name, version, flavorList, targetLabel,
loadSpecsList, csFile, buildReqs, crossReqs,
failureFd, logger):
baseFlavor = cfg.buildFlavor
db = database.Database(cfg.root, cfg.dbPath)
buildLabel = version.trailingLabel()
buildBranch = version.branch()
binaryBranch = version.getBinaryVersion().branch()
if targetLabel:
source = recipeutil.RemoveHostSource(db, targetLabel.getHost())
if version.trailingLabel() == targetLabel and version.depth() > 1:
buildBranch = version.branch().parentBranch()
buildLabel = buildBranch.label()
revision = versions.Revision('1-1')
binaryBranch = buildBranch.createVersion(revision)\
.getBinaryVersion().branch()
else:
source = db
loaders = []
recipeClasses = []
if not isinstance(flavorList, (tuple, list)):
flavorList = [flavorList]
if not isinstance(loadSpecsList, (tuple, list)):
loadSpecsList = [loadSpecsList] * len(flavorList)
for flavor, loadSpecs in itertools.izip(flavorList, loadSpecsList):
try:
logger.debug('Cooking %s=%s[%s] to %s (stored in %s)' % \
(name, version, flavor, targetLabel, csFile))
cfg.buildFlavor = deps.overrideFlavor(baseFlavor, flavor)
cfg.initializeFlavors()
(loader, recipeClass, localFlags, usedFlags) = \
recipeutil.loadRecipeClass(repos, name, version,
cfg.buildFlavor,
ignoreInstalled=False, root=cfg.root,
loadInstalledSource=source,
overrides=loadSpecs,
cfg=cfg)
loaders.append(loader)
recipeClasses.append(recipeClass)
recipeClass.buildRequirementsOverride = buildReqs
recipeClass.crossRequirementsOverride = crossReqs
except Exception, msg:
errMsg = 'Error loading recipe %s=%s[%s]: %s' % \
(name, version, flavor, str(msg))
_buildFailed(failureFd, errMsg, traceback.format_exc())
def testResolveCrossDependencies(self):
self.openRmakeRepository()
# resolving cross root dependencies have a couple of odd features about
# them:
# 1. the flavor is !cross even if the build flavor is cross
# 2. the flavor moves the target: flavor is to the is: spot
# 3. file: dependencies are ignored
# 4. it includes troves in crossTroves list.
# We test some of these here.
self.addComponent('foo:runtime', '1.0', 'cross is:x86_64')
self.addComponent('foo:runtime', '1.0', '!cross is:x86_64')
self.addComponent('foo:runtime', '1.0', 'cross is:x86')
self.addComponent('foo:runtime', '1.0', '!cross is:x86')
self.addComponent('bar:runtime', '1.0', 'cross is:x86_64',
requires='trove:foo:runtime file: /tmp/blah')
self.addComponent('bar:runtime', '1.0', '!cross is:x86_64',
requires='trove:foo:runtime file: /tmp/blah')
self.addComponent('bar:runtime', '1.0', 'cross is:x86',
requires='trove:foo:runtime file: /tmp/blah')
self.addComponent('bar:runtime', '1.0', '!cross is:x86',
requires='trove:foo:runtime file: /tmp/blah')
self.addComponent('blah:runtime', '1.0', 'cross is:x86',
provides='file: /tmp/blah')
trv = self.addComponent('bam:source')
bt = self.newBuildTrove(1, trv.getName(), trv.getVersion(),
parseFlavor('cross is:x86 target:x86_64'))
bt.setBuildRequirements(['bar:runtime'])
bt.setCrossRequirements(['bar:runtime'])
resolveJob = dephandler.ResolveJob(bt, self.buildCfg, [], [])
self.buildCfg.flavor = [overrideFlavor(self.buildCfg.buildFlavor,
parseFlavor('cross is:x86 target:x86_64'))]
res = resolver.DependencyResolver(log, self.openRepository())
self.logFilter.add()
result = res.resolve(resolveJob)
assert(result.success)
buildReqNames = set([ x[0] for x in result.getBuildReqs()])
buildReqFlavors = set([ str(x[2][1]) for x in result.getBuildReqs()])
assert(buildReqNames == set(['bar:runtime', 'foo:runtime',
'blah:runtime']))
crossReqNames = set([ x[0] for x in result.getCrossReqs()])
crossReqFlavors = set([ str(x[2][1]) for x in result.getCrossReqs()])
assert(crossReqNames == set(['bar:runtime', 'foo:runtime']))
assert(crossReqFlavors == set(['!cross is: x86_64']))
assert(buildReqFlavors == set(['cross is: x86']))
def _cookTrove(cfg, repos, name, version, flavorList, targetLabel,
loadSpecsList, csFile, buildReqs, crossReqs, failureFd, logger):
baseFlavor = cfg.buildFlavor
db = database.Database(cfg.root, cfg.dbPath)
buildLabel = version.trailingLabel()
buildBranch = version.branch()
binaryBranch = version.getBinaryVersion().branch()
if targetLabel:
source = recipeutil.RemoveHostSource(db, targetLabel.getHost())
if version.trailingLabel() == targetLabel and version.depth() > 1:
buildBranch = version.branch().parentBranch()
buildLabel = buildBranch.label()
revision = versions.Revision('1-1')
binaryBranch = buildBranch.createVersion(revision)\
.getBinaryVersion().branch()
else:
source = db
loaders = []
recipeClasses = []
if not isinstance(flavorList, (tuple, list)):
flavorList = [flavorList]
if not isinstance(loadSpecsList, (tuple, list)):
loadSpecsList = [loadSpecsList] * len(flavorList)
for flavor, loadSpecs in itertools.izip(flavorList, loadSpecsList):
try:
logger.debug('Cooking %s=%s[%s] to %s (stored in %s)' % \
(name, version, flavor, targetLabel, csFile))
cfg.buildFlavor = deps.overrideFlavor(baseFlavor, flavor)
cfg.initializeFlavors()
(loader, recipeClass, localFlags, usedFlags) = \
recipeutil.loadRecipeClass(repos, name, version,
cfg.buildFlavor,
ignoreInstalled=False, root=cfg.root,
loadInstalledSource=source,
overrides=loadSpecs,
cfg=cfg)
loaders.append(loader)
recipeClasses.append(recipeClass)
recipeClass.buildRequirementsOverride = buildReqs
recipeClass.crossRequirementsOverride = crossReqs
except Exception, msg:
errMsg = 'Error loading recipe %s=%s[%s]: %s' % \
(name, version, flavor, str(msg))
_buildFailed(failureFd, errMsg, traceback.format_exc())
def initializeFlavors(self):
"""
Initialize flavor preferences based on files typically
found in /etc/conary/arch (archDirs) and /etc/conary/use
@raises RuntimeError: Raised if use flags conflict in
a way which cannot be reconciled
(see L{deps.DependencyClass.MergeFlags})
"""
self.flavorConfig = flavorcfg.FlavorConfig(self.useDirs,
self.archDirs)
if self.flavor == []:
self.flavor = [deps.Flavor()]
self.flavor = self.flavorConfig.toDependency(override=self.flavor)
newFlavors = []
hasIns = False
# if any flavor has an instruction set, don't merge
for flavor in self.flavor:
if deps.DEP_CLASS_IS in flavor.getDepClasses():
hasIns = True
break
if not hasIns:
# use all the flavors for the main arch first
for depList in arch.currentArch:
for flavor in self.flavor:
insSet = deps.Flavor()
for dep in depList:
insSet.addDep(deps.InstructionSetDependency, dep)
newFlavor = flavor.copy()
newFlavor.union(insSet)
newFlavors.append(newFlavor)
self.flavor = newFlavors
# buildFlavor is installFlavor + overrides
self.buildFlavor = deps.overrideFlavor(self.flavor[0],
self.buildFlavor)
if self.isDefault('flavorPreferences'):
self.flavorPreferences = arch.getFlavorPreferencesFromFlavor(
self.flavor[0])
self.flavorConfig.populateBuildFlags()
def overrideFlavors(self, queryOptions, flavor, defaultFlavorPath=None):
"""
override the flavors in the defaultFlavorPath with flavor,
replacing instruction set entirely if given.
"""
if defaultFlavorPath is None:
defaultFlavorPath = queryOptions.defaultFlavorPath
if flavor is None:
return defaultFlavorPath
if not defaultFlavorPath:
return [flavor]
flavors = []
for defaultFlavor in queryOptions.defaultFlavorPath:
flavors.append(
deps.overrideFlavor(defaultFlavor,
flavor,
mergeType=deps.DEP_MERGE_TYPE_PREFS))
return flavors
def initializeFlavors(self):
"""
Initialize flavor preferences based on files typically
found in /etc/conary/arch (archDirs) and /etc/conary/use
@raises RuntimeError: Raised if use flags conflict in
a way which cannot be reconciled
(see L{deps.DependencyClass.MergeFlags})
"""
self.flavorConfig = flavorcfg.FlavorConfig(self.useDirs, self.archDirs)
if self.flavor == []:
self.flavor = [deps.Flavor()]
self.flavor = self.flavorConfig.toDependency(override=self.flavor)
newFlavors = []
hasIns = False
# if any flavor has an instruction set, don't merge
for flavor in self.flavor:
if deps.DEP_CLASS_IS in flavor.getDepClasses():
hasIns = True
break
if not hasIns:
# use all the flavors for the main arch first
for depList in arch.currentArch:
for flavor in self.flavor:
insSet = deps.Flavor()
for dep in depList:
insSet.addDep(deps.InstructionSetDependency, dep)
newFlavor = flavor.copy()
newFlavor.union(insSet)
newFlavors.append(newFlavor)
self.flavor = newFlavors
# buildFlavor is installFlavor + overrides
self.buildFlavor = deps.overrideFlavor(self.flavor[0],
self.buildFlavor)
if self.isDefault('flavorPreferences'):
self.flavorPreferences = arch.getFlavorPreferencesFromFlavor(
self.flavor[0])
self.flavorConfig.populateBuildFlags()
def testBasic(self):
trv = self.addComponent('nocross:source', '1.0-1', '',
[('nocross.recipe', nocrossRecipe)])
trv2 = self.addComponent('crosstool:source', '1.0-1', '',
[('crosstool.recipe', crosstoolRecipe)])
ccRoot = self.rmakeCfg.buildDir + '/chroots/crosscompiled'
trv3 = self.addComponent(
'crosscompiled:source', '1.0-1', '',
[('crosscompiled.recipe', crosscompiledRecipe % (ccRoot, ccRoot))])
self.openRmakeRepository()
troveList = [(trv.getName(), trv.getVersion(),
deps.parseFlavor('!cross target: x86_64')),
(trv2.getName(), trv2.getVersion(),
deps.parseFlavor('cross target: x86_64')),
(trv3.getName(), trv3.getVersion(),
deps.parseFlavor('!cross target: x86_64'))]
db = self.openRmakeDatabase()
self.buildCfg.flavor = [
deps.overrideFlavor(
self.buildCfg.flavor[0],
deps.parseFlavor('~cross is:x86 target:x86_64'))
]
job = self.newJob(*troveList)
db.subscribeToJob(job)
b = builder.Builder(self.rmakeCfg, job)
self.logFilter.add()
logFile = logfile.LogFile(self.workDir + '/buildlog')
logFile.redirectOutput()
try:
b.build()
except Exception:
b.worker.stopAllCommands()
raise
logFile.restoreOutput()
assert (set([x.getName() for x in b.dh.depState.getBuiltTroves()
]) == set([trv.getName(),
trv2.getName(),
trv3.getName()]))
def testMergeDropConflicts(self):
def override1(flavor1, flavor2):
return str(overrideFlavor(parseFlavor(flavor1),
parseFlavor(flavor2),
mergeType=DEP_MERGE_TYPE_DROP_CONFLICTS))
def override2(flavor1, flavor2):
return str(mergeFlavorList([parseFlavor(flavor1), parseFlavor(flavor2)],
mergeType=DEP_MERGE_TYPE_DROP_CONFLICTS))
for override in override1, override2:
assert(override('~foo', 'foo') == 'foo')
assert(override('~foo', '~foo') == '~foo')
assert(override('~foo', '~!foo') == '')
assert(override('~foo', '!foo') == '!foo')
assert(override('~!foo', 'foo') == 'foo')
assert(override('~!foo', '~foo') == '')
assert(override('~!foo', '~!foo') == '~!foo')
assert(override('~!foo', '!foo') == '!foo')
assert(override('foo', 'foo') == 'foo')
assert(override('foo', '~foo') == 'foo')
assert(override('foo', '~!foo') == 'foo')
assert(override('foo', '!foo') == '')
assert(override('!foo', 'foo') == '')
assert(override('!foo', '~foo') == '!foo')
assert(override('!foo', '~!foo') == '!foo')
assert(override('!foo', '!foo') == '!foo')
assert(override('!foo', 'foo') == '')
# check to make sure parsed flavor is actually empty and doesn't
# have an empty use flag
test1 = overrideFlavor(parseFlavor('!foo'), parseFlavor('foo'),
mergeType=DEP_MERGE_TYPE_DROP_CONFLICTS)
assert(test1 == parseFlavor(''))
self.buildRecipe(skipCheckRedirectRecipe, "testRedirect")
try:
self.checkUpdate('redirect', ['test[foo]', 'test:runtime[foo]'])
except Exception, err:
# FIXME: We tried to follow a redirect, but failed.
# should we give a better error in this case?
assert (
str(err) ==
'test was not found on path localhost@rpl:linux (Closest alternate flavors found: [~foo])'
)
else:
assert (0)
self.cfg.flavor[0] = deps.overrideFlavor(self.cfg.flavor[0],
deps.parseFlavor('foo'))
self.checkUpdate('redirect', ['test[foo]', 'test:runtime[foo]'])
@testhelp.context('redirect')
def testRedirectWithFewerComponents(self):
# we've installed a version of trove redirect with :runtime and :data
# a redirect has been built from the trove "redirect" to "test",
# but only the :runtime component is redirected, because the latest
# version of the "redirect" trove only has one component!
# therefore, currently, the redirect:data component gets left behind.k
self.addComponent('test:runtime', '1')
self.addCollection('test', '1', [':runtime'])
self.addComponent('redirect:runtime', '1')
self.addComponent('redirect:data', '1', filePrimer=1)
self.addCollection('redirect', '1', [':runtime', ':data'])
def testCrossFlavorDeps(self):
provideRec = recipes.bashRecipe + ' if Arch.x86:pass'
userRec = recipes.bashUserRecipe + ' if Arch.x86:pass'
# build provides (foo) as x86
self.overrideBuildFlavor('is:x86')
(built, d) = self.buildRecipe(provideRec, 'Bash')
v1, f1 = built[0][1:3]
# build requires (bar) as x86
(built, d) = self.buildRecipe(userRec, 'BashUser')
user1, flavor1 = built[0][1:3]
# set flavor to be x86_64
x86Flavor = deps.overrideFlavor(self.cfg.flavor[0],
deps.parseFlavor('is:x86'))
x86_64Flavor = deps.overrideFlavor(self.cfg.flavor[0],
deps.parseFlavor('is:x86_64'))
self.cfg.flavor = [x86_64Flavor]
# install w/ x86 flavor request
rc, str = self.captureOutput(self.updatePkg,
self.rootDir,
'bashuser:runtime',
user1,
tagScript='/dev/null',
resolve=True,
flavor='is:x86')
# XXX ideally, the x86 flavor of the trove we are installing should
# override the install flavor path here, and find the right trove
assert (str == 'The following dependencies could not be resolved:\n'
' bashuser:runtime=0-1-1:\n'
'\tfile: /bin/bash\n')
# flavorPath is now x86_64, x86
self.cfg.flavor = [x86_64Flavor, x86Flavor]
rc, str = self.captureOutput(self.updatePkg,
self.rootDir,
'bashuser:runtime',
user1,
tagScript='/dev/null',
resolve=True,
justDatabase=True,
flavor='')
assert (str == 'Including extra troves to resolve dependencies:\n'
' bash:runtime=0-1-1\n')
# these erases spew output because of the justDatabase flags used
# during the install. squelch.
self.captureOutput(self.erasePkg,
self.rootDir,
'bashuser:runtime',
justDatabase=True)
self.captureOutput(self.erasePkg,
self.rootDir,
'bash:runtime',
justDatabase=True)
self.cfg.buildFlavor = x86_64Flavor
(built, d) = self.buildRecipe(provideRec, 'Bash')
v1, f2 = built[0][1:3]
rc, str = self.captureOutput(self.updatePkg,
self.rootDir,
'bashuser:runtime',
user1,
tagScript='/dev/null',
resolve=True,
justDatabase=True,
flavor='')
assert (str == 'Including extra troves to resolve dependencies:\n'
' bash:runtime=0-1-1\n')
db = database.Database(self.rootDir, self.cfg.dbPath)
# make sure that we installed the x86_64 one -- the first one that
# matches on our flavor path
bashTroves = db.findTrove(None, ('bash:runtime', None, None))
assert (len(bashTroves) == 1)
assert (bashTroves[0][2].freeze() == '1#x86_64|5#use:ssl')
def loadRecipe(repos,
name,
version,
flavor,
trv,
defaultFlavor=None,
loadInstalledSource=None,
installLabelPath=None,
buildLabel=None,
groupRecipeSource=None,
cfg=None):
name = name.split(':')[0]
try:
if defaultFlavor is not None:
fullFlavor = deps.overrideFlavor(defaultFlavor, flavor)
else:
fullFlavor = flavor
# set up necessary flavors and track used flags before
# calling loadRecipe, since even loading the class
# may check some flags that may never be checked inside
# the recipe
recipeObj, loader = getRecipeObj(
repos,
name,
version,
fullFlavor,
trv,
loadInstalledSource=loadInstalledSource,
installLabelPath=installLabelPath,
buildLabel=buildLabel,
cfg=cfg)
relevantFlavor = use.usedFlagsToFlavor(recipeObj.name)
relevantFlavor = flavorutil.removeInstructionSetFlavor(relevantFlavor)
# always add in the entire arch flavor. We need to ensure the
# relevant flavor is unique per architecture, also, arch flavors
# can affect the macros used.
if defaultFlavor is not None:
relevantFlavor.union(flavor)
relevantFlavor.union(flavorutil.getArchFlags(fullFlavor))
relevantFlags = flavorutil.getFlavorUseFlags(relevantFlavor)
flags = flavorutil.getFlavorUseFlags(fullFlavor)
use.track(False)
for flagSet in ('Use', ):
# allow local flags not to be used -- they are set to their default
if flagSet not in relevantFlags:
continue
for flag in relevantFlags[flagSet]:
if flag not in flags[flagSet]:
raise (RuntimeError,
"Recipe %s uses Flavor %s but value not known" %
(name, flag))
if 'Arch' in relevantFlags:
for majarch in relevantFlags['Arch'].keys():
for subarch in relevantFlags['Arch'][majarch]:
if not use.Arch[majarch][subarch]:
#negative values for subarches are assumed
continue
if subarch not in flags['Arch'][majarch]:
log.error("arch %s.%s used but not specified" %
(majarch, subarch))
raise RuntimeError, (
"arch %s.%s used but not specified" %
(majarch, subarch))
use.resetUsed()
except:
log.error('Error Loading Recipe (%s, %s, %s):\n%s' %
(name, version, fullFlavor, ''.join(traceback.format_exc())))
raise
return loader, recipeObj, relevantFlavor
def override(flavor1, flavor2):
return overrideFlavor(parseFlavor(flavor1),
parseFlavor(flavor2)).freeze()
def getTupleStrings(self, n, v, f):
"""
returns potentially shortened display versions and flavors for
a trove tuple.
@param n: trove name
@type n: str
@param v: trove version
@type v: versions.Version
@param f: trove flavor
@type f: deps.deps.Flavor
@rtype: (vStr, fStr) where vStr is the version string to display
for this trove and fStr is the flavor string (may be empty)
"""
if self.dcfg.useFullVersions():
vStr = str(v)
elif self.dcfg.showLabels:
vStr = '%s/%s' % (v.branch().label(), v.trailingRevision())
elif len(self._vCache.get(n, [])) > 1:
# print the trailing revision unless there
# are two versions that are on different
# branches.
vers = self._vCache[n]
branch = v.branch()
vStr = None
for ver in vers:
if ver.branch() != branch:
vStr = str(v)
if not vStr:
vStr = str(v.trailingRevision())
else:
vStr = str(v.trailingRevision())
if self.dcfg.useFullFlavors():
fStr = str(f)
else:
# print only the flavor differences between
# the two troves.
# FIXME Document all this!
if self.dcfg.affinityDb:
installed = set()
#installed = set([x[2] for x in self.dcfg.affinityDb.trovesByName(n)])
installed.discard(f)
installed = list(installed)
else:
installed = []
flavors = list(self._fCache.get(n, []))
if installed:
for baseFlavor in self.dcfg.baseFlavors:
flavors += [
deps.overrideFlavor(baseFlavor, x) for x in installed
]
else:
flavors += self.dcfg.baseFlavors
if len(flavors) > 1:
fStr = deps.flavorDifferences(list(flavors) + installed + [f],
strict=False)[f]
ISD = deps.InstructionSetDependency
if f.hasDepClass(ISD) and fStr.hasDepClass(ISD):
allDeps = set(x.name for x in f.iterDepsByClass(ISD))
allDeps.difference_update(
x.name for x in fStr.iterDepsByClass(ISD))
for depName in allDeps:
fStr.addDep(ISD, deps.Dependency(depName))
fStr = str(fStr)
else:
fStr = ''
return vStr, fStr
def _overrideFlavors(self, baseFlavor, flavorList):
baseFlavor = self._getFlavor(baseFlavor)
return [
str(deps.overrideFlavor(baseFlavor, self._getFlavor(x)))
for x in flavorList
]
def getFullFlavor(self):
return deps.overrideFlavor(self.cfg.buildFlavor, self.flavor)
def selectResolutionTrove(self, requiredBy, dep, depClass, troveTups,
installFlavor, affFlavorDict):
"""
determine which of the given set of troveTups is the
best choice for installing on this system. Because the
repository didn't try to determine which flavors are best for
our system, we have to filter the troves locally.
"""
#NOTE: this method should be a match exactly for the one in
# conary.repository.resolvemethod for conary 1.2 and later.
# when we drop support for earlier conary's we can drop this method.
# we filter the troves in the following ways:
# 1. prefer troves that match affinity flavor + are on the affinity
# label. (And don't drop an arch)
# 2. fall back to troves that match the install flavor.
# If we don't match an affinity flavor + label, then use flavor
# preferences and flavor scoring to select the best flavor.
# We'll have to check
# Within these two categories:
# 1. filter via flavor preferences for each trove (this may result
# in an older version for some troves)
# 2. only leave the latest version for each trove
# 3. pick the best flavor out of the remaining
affinityMatches = []
affinityFlavors = []
otherMatches = []
otherFlavors = []
if installFlavor is not None and not installFlavor.isEmpty():
flavoredList = []
for troveTup in troveTups:
label = troveTup[1].trailingLabel()
affTroves = affFlavorDict[troveTup[0]]
found = False
if affTroves:
for affName, affVersion, affFlavor in affTroves:
if affVersion.trailingLabel() != label:
continue
newFlavor = deps.overrideFlavor(
installFlavor,
affFlavor,
mergeType=deps.DEP_MERGE_TYPE_PREFS)
# implement never drop an arch for dep resolution
currentArch = deps.getInstructionSetFlavor(affFlavor)
if not troveTup[2].stronglySatisfies(currentArch):
continue
if newFlavor.satisfies(troveTup[2]):
affinityMatches.append((newFlavor, troveTup))
affinityFlavors.append(troveTup[2])
found = True
if not found and not affinityMatches:
if installFlavor.satisfies(troveTup[2]):
otherMatches.append((installFlavor, troveTup))
otherFlavors.append(troveTup[2])
else:
otherMatches = [(None, x) for x in troveTups]
otherFlavors = [x[2] for x in troveTups]
if affinityMatches:
allFlavors = affinityFlavors
flavoredList = affinityMatches
else:
allFlavors = otherFlavors
flavoredList = otherMatches
# Now filter by flavor preferences.
newFlavors = []
if self.flavorPreferences:
for flavor in self.flavorPreferences:
for trvFlavor in allFlavors:
if trvFlavor.stronglySatisfies(flavor):
newFlavors.append(trvFlavor)
if newFlavors:
break
if newFlavors:
flavoredList = [x for x in flavoredList if x[1][2] in newFlavors]
return self._selectMatchingResolutionTrove(requiredBy, dep, depClass,
flavoredList)
def override1(flavor1, flavor2):
return str(overrideFlavor(parseFlavor(flavor1),
parseFlavor(flavor2),
mergeType=DEP_MERGE_TYPE_DROP_CONFLICTS))
def override1(flavor1, flavor2):
return str(overrideFlavor(parseFlavor(flavor1), parseFlavor(flavor2),
mergeType=DEP_MERGE_TYPE_NORMAL))
def getTrovesToDisplay(repos,
troveSpecs,
pathList,
whatProvidesList,
versionFilter,
flavorFilter,
labelPath,
defaultFlavor,
affinityDb,
troveTypes=trovesource.TROVE_QUERY_PRESENT):
""" Finds troves that match the given trove specifiers, using the
current configuration, and parameters
@param repos: a network repository client
@type repos: repository.netclient.NetworkRepositoryClient
@param troveSpecs: troves to search for
@type troveSpecs: list of troveSpecs (n[=v][[f]])
@param versionFilter: The VERSION_FILTER_* to use. See man
page for documentation for now.
@param flavorFilter: The FLAVOR_FILTER_* to use. See man
page for documentation for now.
@param labelPath: The labelPath to search
@type labelPath: list
@param defaultFlavor: The default flavor(s) to search with
@type defaultFlavor: list
@param affinityDb: The affinity database to search with.
@type affinityDb: bool
@rtype: troveTupleList (list of (name, version, flavor) tuples)
"""
def _merge(resultD, response):
for troveName, troveVersions in response.iteritems():
d = resultD.setdefault(troveName, {})
for version, flavors in troveVersions.iteritems():
d.setdefault(version, []).extend(flavors)
return resultD
troveTups = []
if troveSpecs or pathList or whatProvidesList:
if whatProvidesList:
tupList = []
for label in labelPath:
sols = repos.resolveDependencies(label, whatProvidesList)
for solListList in sols.itervalues():
# list of list of solutions to the depSet
tupList.extend(itertools.chain(*solListList))
source = trovesource.SimpleTroveSource(tupList)
source.searchAsRepository()
troveNames = set(x[0] for x in tupList)
results = getTrovesToDisplay(source,
troveNames, [], [],
versionFilter,
flavorFilter,
labelPath,
defaultFlavor,
affinityDb=None,
troveTypes=troveTypes)
troveTups.extend(results)
# Search for troves using findTroves. The options we
# specify to findTroves are determined by the version and
# flavor filter.
if pathList:
troveTups += getTrovesByPath(repos, pathList, versionFilter,
flavorFilter, labelPath,
defaultFlavor)
if not troveSpecs:
return sorted(troveTups, display._sortTroves)
# Search for troves using findTroves. The options we
# specify to findTroves are determined by the version and
# flavor filter.
troveSpecs = [ ((not isinstance(x, str) and x) or
cmdline.parseTroveSpec(x, allowEmptyName=False)) \
for x in troveSpecs ]
searchFlavor = defaultFlavor
acrossLabels = True
if versionFilter == VERSION_FILTER_ALL:
getLeaves = False
elif versionFilter == VERSION_FILTER_LATEST:
# we just want to limit all searches for the very latest
# version node. Find trove makes this difficult, we
# do the leaves search and then filter.
getLeaves = True
acrossLabels = False
elif versionFilter == VERSION_FILTER_LEAVES:
# This will return all versions that are 'leaves', that is,
# are the latest with a unique flavor string.
getLeaves = True
acrossLabels = False
else:
assert (0)
exactFlavors = False
if flavorFilter == FLAVOR_FILTER_ALL:
searchFlavor = None
bestFlavor = False
acrossFlavors = True # there are no flavors to go 'across'
newSpecs = []
origSpecs = {}
# We do extra processing here. We want FLAVOR_FILTER_ALL to work
# when you specify a flavor to limit the all to.
# But findTrove won't let us do that, since it expects that
# the flavors it gets passed are supersets of the trove flavors
# So we search with no flavor and search by hand afterwards.
for (n, vS, fS) in troveSpecs:
origSpecs.setdefault((n, vS), []).append(fS)
newSpecs.append((n, vS, None))
troveSpecs = newSpecs
affinityDb = None
elif flavorFilter == FLAVOR_FILTER_AVAIL:
# match install flavor + maybe affinity, could affect rq branch,
# return all flavors that match.
bestFlavor = False
acrossFlavors = True
if versionFilter != VERSION_FILTER_ALL:
getLeaves = True
elif flavorFilter == FLAVOR_FILTER_BEST:
# match install flavor + affinity, could affect rq branch,
# return best match.
bestFlavor = True
acrossFlavors = False
elif flavorFilter == FLAVOR_FILTER_EXACT:
exactFlavors = True
acrossFlavors = False
bestFlavor = False
if not affinityDb:
acrossLabels = True
results = repos.findTroves(labelPath,
troveSpecs,
searchFlavor,
affinityDatabase=affinityDb,
acrossLabels=acrossLabels,
acrossFlavors=acrossFlavors,
allowMissing=False,
bestFlavor=bestFlavor,
getLeaves=getLeaves,
troveTypes=troveTypes,
exactFlavors=exactFlavors)
# do post processing on the result if necessary
if (flavorFilter == FLAVOR_FILTER_ALL
or versionFilter == VERSION_FILTER_LATEST):
for (n, vS, fS), tups in results.iteritems():
if not tups:
continue
if versionFilter == VERSION_FILTER_LATEST:
# only look at latest leaves (1 per branch).
versionsByBranch = {}
for tup in tups:
versionsByBranch.setdefault(tup[1].branch(),
[]).append(tup[1])
maxVersions = set(
max(x) for x in versionsByBranch.values())
tups = [x for x in tups if x[1] in maxVersions]
for (_, v, f) in tups:
if flavorFilter == FLAVOR_FILTER_ALL:
# only look at latest leaf.
foundMatch = False
for fS in origSpecs[n, vS]:
# FIXME: switch to stronglySatisfies
# in order to implement primary flavor support
# here at least?
if (fS is None) or f.satisfies(fS):
foundMatch = True
break
if not foundMatch:
continue
troveTups.append((n, v, f))
else:
troveTups.extend(itertools.chain(*results.itervalues()))
else:
if not labelPath:
raise LabelPathNeeded("No search label path given and no label "
"specified - set the installLabelPath")
if flavorFilter == FLAVOR_FILTER_EXACT:
flavorFilter = FLAVOR_FILTER_BEST
# no troves specified, use generic fns with no names given.
if versionFilter == VERSION_FILTER_ALL:
queryFn = repos.getTroveVersionsByLabel
elif versionFilter == VERSION_FILTER_LATEST:
queryFn = repos.getTroveLatestByLabel
elif versionFilter == VERSION_FILTER_LEAVES:
queryFn = repos.getTroveLatestByLabel
if flavorFilter == FLAVOR_FILTER_ALL:
flavor = None
bestFlavor = False
affinityDb = None
elif flavorFilter == FLAVOR_FILTER_AVAIL:
# match affinity flavors
# must be done client side...
flavor = None
bestFlavor = False
affinityDb = None # for now turn off
elif flavorFilter == FLAVOR_FILTER_BEST:
# match affinity flavors
# must be done client side...
flavor = None
bestFlavor = False
affinityDb = None # XXX for now turn off.
resultsDict = {}
resultsDict = queryFn({'': {
labelPath[0]: flavor
}},
bestFlavor=bestFlavor,
troveTypes=troveTypes)
for label in labelPath[1:]:
d = queryFn({'': {
label: flavor
}},
bestFlavor=bestFlavor,
troveTypes=troveTypes)
_merge(resultsDict, d)
# do post processing for VERSION_FILTER_LATEST, FLAVOR_FILTER_BEST,
# and FLAVOR_FILTER_AVAIL
troveTups = []
for name, versionDict in resultsDict.iteritems():
if affinityDb:
localFlavors = [x[2] for x in affinityDb.trovesByName(name)]
else:
localFlavors = []
versionsByLabel = {}
for version, flavorList in versionDict.iteritems():
versionsByLabel.setdefault(version.trailingLabel(), []).append(
(version, flavorList))
for versionDict in versionsByLabel.itervalues():
for version, flavorList in sorted(versionDict, reverse=True):
if flavorFilter == FLAVOR_FILTER_BEST:
best = None
for systemFlavor in defaultFlavor:
matchScores = []
if localFlavors:
matchFlavors = [
deps.overrideFlavor(systemFlavor, x)
for x in localFlavors
]
else:
matchFlavors = [systemFlavor]
for f in flavorList:
scores = ((x.score(f), f)
for x in matchFlavors)
scores = [
x for x in scores if x[0] is not False
]
if scores:
matchScores.append(max(scores))
if matchScores:
best = max(matchScores)[1]
break
if best is not None:
flavorList = [best]
else:
continue
elif flavorFilter == FLAVOR_FILTER_AVAIL:
if localFlavors:
matchFlavors = []
for systemFlavor in defaultFlavor:
matchFlavors.extend(
deps.overrideFlavor(systemFlavor, x)
for x in localFlavors)
else:
matchFlavors = defaultFlavor
added = False
for flavor in flavorList:
if flavorFilter == FLAVOR_FILTER_AVAIL:
found = False
for matchFlavor in matchFlavors:
if matchFlavor.satisfies(flavor):
found = True
break
if not found:
continue
troveTups.append((name, version, flavor))
added = True
if added and versionFilter == VERSION_FILTER_LATEST:
break
return sorted(troveTups, display._sortTroves)
def selectResolutionTrove(self, requiredBy, dep, depClass, troveTups,
installFlavor, affFlavorDict):
""" determine which of the given set of troveTups is the
best choice for installing on this system. Because the
repository didn't try to determine which flavors are best for
our system, we have to filter the troves locally.
"""
# we filter the troves in the following ways:
# 1. prefer troves that match affinity flavor + are on the affinity
# label. (And don't drop an arch)
# 2. fall back to troves that match the install flavor.
# If we don't match an affinity flavor + label, then use flavor
# preferences and flavor scoring to select the best flavor.
# We'll have to check
# Within these two categories:
# 1. filter via flavor preferences for each trove (this may result
# in an older version for some troves)
# 2. only leave the latest version for each trove
# 3. pick the best flavor out of the remaining
affinityMatches = []
affinityFlavors = []
otherMatches = []
otherFlavors = []
troveNames = set([x[0] for x in troveTups])
allAffinityTroves = list(
itertools.chain(*[affFlavorDict[x] or [] for x in troveNames]))
db = trovesource.SimpleTroveSource(allAffinityTroves)
repos = trovesource.SimpleTroveSource(troveTups)
repos.searchWithFlavor()
repos.setFlavorPreferenceList(self.flavorPreferences)
if installFlavor is not None and installFlavor.isEmpty():
installFlavor = None
# search for resolutions that would update an installed package.
results = repos.findTroves(None, [(x, None, None) for x in troveNames],
installFlavor,
getLeaves=False,
bestFlavor=False,
affinityDatabase=db,
allowMissing=True)
if results:
flavoredList = []
troveTups = list(itertools.chain(*results.itervalues()))
trovesByName = {}
for troveTup in troveTups:
if troveTup in allAffinityTroves:
continue
trovesByName.setdefault(troveTup[0], []).append(troveTup)
for troveName, troveTups in trovesByName.items():
affTups = affFlavorDict[troveName]
if affTups:
for affTup in affTups:
affFlavor = deps.overrideFlavor(
installFlavor,
affTup[2],
mergeType=deps.DEP_MERGE_TYPE_PREFS)
allTups = [
x for x in troveTups if affFlavor.satisfies(x[2])
]
allTups = repos.filterTrovesByPreferences(allTups)
for troveTup in allTups:
flavoredList.append((affFlavor, troveTup))
else:
allTups = repos.filterTrovesByPreferences(troveTups)
for troveTup in allTups:
flavoredList.append((installFlavor, troveTup))
else:
# fall back to searching for things that could be installed
# side-by-side.
results = repos.findTroves(None,
[(x, None, None) for x in troveNames],
installFlavor,
getLeaves=True,
allowMissing=True)
troveTups = list(itertools.chain(*results.itervalues()))
allTups = repos.filterTrovesByPreferences(troveTups)
flavoredList = [(installFlavor, x) for x in allTups]
return self._selectMatchingResolutionTrove(requiredBy, dep, depClass,
flavoredList)
