def testTroveMultiFlavor(self):
# create a package with 3 components, each use a different use
# flag. add the package before the components. the goal is
# to make the trovestore create more than one flavor
flags = ('foo', 'bar', 'baz')
flavors = []
for flag in flags:
flavor = deps.Flavor()
flavor.addDep(
deps.UseDependency,
deps.Dependency('use', [(flag, deps.FLAG_SENSE_REQUIRED)]))
flavor.addDep(deps.InstructionSetDependency,
deps.Dependency('x86', []))
flavors.append(flavor)
v10 = ThawVersion("/conary.rpath.com@test:trunk/10:1.2-10")
store = self._connect()
# create components to add to the package
troves = []
for flag, flavor in zip(flags, flavors):
trv = trove.Trove('test:%s' % flag, v10, flavor, None)
trv.computeDigests()
troves.append(trv)
# add the package
union = deps.Flavor()
for flavor in flavors:
union.union(flavor)
trv2 = trove.Trove("test", v10, union, None)
for trv in troves:
trv2.addTrove(trv.getName(), v10, trv2.getFlavor())
trv2.computeDigests()
store.addTroveSetStart([], [], [])
troveInfo = store.addTrove(trv2, trv2.diff(None)[0])
store.addTroveDone(troveInfo)
store.addTroveSetDone()
# add the troves
store.addTroveSetStart([], [], [])
for trv in troves:
troveInfo = store.addTrove(trv, trv.diff(None)[0])
store.addTroveDone(troveInfo)
store.addTroveSetDone()
for trv in troves:
self.assertEqual(
trv,
store.getTrove(trv.getName(), trv.getVersion(),
trv.getFlavor()))
troveFlavors = store.getTroveFlavors({'test': [v10]})
self.assertEqual(troveFlavors['test'][v10], [union.freeze()])
def testDependencies(self):
depSet = deps.DependencySet()
depSet.addDep(deps.FileDependencies, deps.Dependency('/foo/bar', []))
d = DependenciesStream()
d.set(depSet)
assert (d() == depSet)
d2 = DependenciesStream(d.freeze())
assert (d == d2)
assert (str(d2.deps) == "file: /foo/bar")
# currently there should be no diff between d and d2
assert (d.diff(d2) is None)
# add another dep to d (by modifying its dependency set)
# now the diff should contain all of the dependencies in d
# (when the diff method is used, d2 would be the "old" dependency
# set)
depSet.addDep(deps.FileDependencies, deps.Dependency('/foo/baz', []))
d.set(depSet)
diff = d.diff(d2)
d3 = DependenciesStream(diff)
assert (d == d3)
# now change d's depset again, and do a three way merge back to
# the d3 state
depSet.addDep(deps.FileDependencies, deps.Dependency('/foo/fred', []))
base = deps.DependencySet()
d.twm(diff, base)
assert (d == d3)
# verify that going from an empty depSet to one with deps in it
# works
depSet = deps.DependencySet()
d = DependenciesStream()
d.set(depSet)
diff = d2.diff(d)
d.twm(diff, d)
assert (d == d2)
# verify that going from a depSet with deps in it to an empty one
# works
depSet = deps.DependencySet()
d = DependenciesStream()
d.set(depSet)
assert (d2.freeze() != '')
assert (d.freeze() == '')
diff = d.diff(d2)
assert (diff == '')
d2.twm(diff, d2)
assert (d == d2)
assert (d2.freeze() == '')
def resolveRepo(self, fileDeps, comp, addedTroveDeps, removedFileDeps):
if not fileDeps:
return
resolvedDeps = set()
for label in self.cfg.installLabelPath:
solMap = self.repos.resolveDependencies(label,
self.toDepSets(
fileDeps,
deps.FileDependencies),
leavesOnly=True)
for r in solMap:
solList = solMap[r]
for s in itertools.chain(*solList):
if s[2].satisfies(comp.flavor):
fDep = list(r.iterDeps())[0][1]
resolvedDeps.add(fDep)
break
unresolvedDeps = fileDeps - resolvedDeps
if not unresolvedDeps:
return
paths = [str(x) for x in unresolvedDeps]
trvMap = {}
presolvedDeps = set()
for label in self.cfg.installLabelPath:
pathDict = self.repos.getTroveLeavesByPath(paths, label)
for p in pathDict:
if p not in trvMap and pathDict[p]:
trvMap[p] = pathDict[p]
presolvedDeps.add(deps.Dependency(p))
if not presolvedDeps:
return
for fDep in presolvedDeps:
f = str(fDep)
for nvf in trvMap[f]:
if nvf[2].satisfies(comp.flavor):
trovName = nvf[0]
self.info("Replacing requirement on file %s with a "
"requirement on trove %s since that file is not "
"directly provided." % (f, trovName))
addedTroveDeps.append(deps.Dependency(trovName))
removedFileDeps.append(fDep)
fileDeps.remove(fDep)
break
def _addVersionedDep(self, tags, dep, flags, ver, depset):
# Ignore any dep without flags
if not flags:
return
# Make sure it is an equal version
if not flags & RPMSENSE_EQUAL:
return
# Make sure not >= or <=
if flags & RPMSENSE_LESS or flags & RPMSENSE_GREATER:
return
# If the version contains an epoch, make sure there is only one colon.
if ':' in ver:
ver = ':'.join([x for x in ver.split(':') if x])
# Add provides for versions without the release string since some rpms
# just require the version without the release.
vers = [
ver,
]
if [x for x in tags if x == PROVIDENAME]:
vver = ver.split('-', 1)[0]
vers.append(vver)
# Add a provides without epoch for anything with an epoch of 0.
if ':' in ver and ver.split(':')[0] == '0':
vers.append(ver.split(':')[1])
vers.append(vver.split(':')[1])
# Add epoch provides for anything with an epoch of None.
elif ':' not in ver:
vers.append('0:%s' % ver)
vers.append('0:%s' % vver)
# Add version deps for anything that specifies an exact
# version in addition to the unversioned dep.
for v in vers:
verdep = '%s-%s' % (dep, v)
depset.addDep(deps.RpmDependencies, deps.Dependency(verdep, []))
def testOne(dir, flagList, emptyPreferred=True):
files = os.listdir(dir)
for fileName in files:
os.unlink(os.path.join(dir, fileName))
for (flag, sense) in flagList:
f = open(os.path.join(dir, flag), "w")
if sense == deps.FLAG_SENSE_PREFERRED:
if not emptyPreferred:
f.write('sense preferred\n')
elif sense == deps.FLAG_SENSE_PREFERNOT:
f.write('sense prefernot\n')
elif sense == deps.FLAG_SENSE_DISALLOWED:
f.write('sense disallowed\n')
elif sense == deps.FLAG_SENSE_REQUIRED:
f.write('sense required\n')
else:
assert (0)
f.close()
use = flavorcfg.FlavorConfig(dir, '')
cmp = deps.Flavor()
cmp.addDep(deps.UseDependency, deps.Dependency('use', flagList))
assert (use.toDependency() == cmp)
def _toDependency(self, recipeName):
depFlags = [('.'.join((recipeName, self._name)),
self._getDepSense())]
set = deps.Flavor()
dep = deps.Dependency('use', depFlags)
set.addDep(deps.UseDependency, dep)
return set
def resolveLocal(self, fileDeps, comp, addedTroveDeps, removedFileDeps):
if not fileDeps:
return
locDepSets = set()
trvMap = {}
for fDep in fileDeps.copy():
f = str(fDep)
trv0 = None
for trv in self.db.iterTrovesByPath(f):
if not trv0:
trv0 = trv
if trv.provides().satisfies(
self.toDepSet(fDep, deps.FileDependencies)):
fileDeps.remove(fDep)
break
else:
if trv0:
trovName = trv0.getName()
self.info("Replacing requirement on file %s with a "
"requirement on trove %s since that file is not "
"directly provided." % (f, trovName))
addedTroveDeps.append(deps.Dependency(trovName))
removedFileDeps.append(fDep)
fileDeps.remove(fDep)
def _toDependency(self):
set = deps.Flavor()
sense = self._getDepSense()
depFlags = [ (self._name, sense) ]
dep = deps.Dependency('use', depFlags)
set.addDep(deps.UseDependency, dep)
return set
def getArchFlags(flavor, getTarget=True, withFlags=True):
archFlavor = Flavor()
flavorInsSet = flavor.getDepClasses().get(deps.DEP_CLASS_IS, None)
if flavorInsSet is not None:
for insSet in flavorInsSet.getDeps():
if not withFlags:
insSet = deps.Dependency(insSet.name)
archFlavor.addDep(deps.InstructionSetDependency, insSet)
if not getTarget:
return archFlavor
flavorInsSet = flavor.getDepClasses().get(deps.DEP_CLASS_TARGET_IS, None)
if flavorInsSet is not None:
for insSet in flavorInsSet.getDeps():
if not withFlags:
insSet = deps.Dependency(insSet.name)
archFlavor.addDep(deps.TargetInstructionSetDependency, insSet)
return archFlavor
def updateArgs(self, *args, **keywords):
if len(args) is 2:
name = args[1]
if ':' not in name:
name = name + ':rpm'
reMatch = self.requirementRe.match(args[0])
if not reMatch or len(reMatch.groups()) != 3:
return
depClass = reMatch.group(1).strip().lower()
if depClass != 'rpm' and depClass != 'rpmlib':
raise policy.PolicyError, "RPMRequires cannot be used to " \
"provide the non-rpm dependency: '%s'" % args[0]
dep = reMatch.group(2).strip()
flags = reMatch.group(3).strip().split()
flags = [(x, deps.FLAG_SENSE_REQUIRED) for x in flags if x]
if not self.requirements.get(name):
self.requirements[name] = deps.DependencySet()
self.requirements[name].addDep(
deps.dependencyClassesByName[depClass],
deps.Dependency(dep, flags))
allowUnusedFilters = keywords.pop('allowUnusedFilters', False) or \
self.allowUnusedFilters
exceptions = keywords.pop('exceptions', None)
if exceptions:
if type(exceptions) is str:
self.excepts.add(exceptions)
if not allowUnusedFilters:
self.unusedFilters['exceptions'].add(exceptions)
elif type(exceptions) in (tuple, list):
self.excepts.update(exceptions)
if not allowUnusedFilters:
self.unusedFilters['exceptions'].update(exceptions)
exceptDeps = keywords.pop('exceptDeps', None)
if exceptDeps:
if type(exceptDeps) is str:
exceptDeps = ('.*', exceptDeps)
assert (type(exceptDeps) == tuple)
if type(exceptDeps[0]) is tuple:
self.exceptDeps.extend(exceptDeps)
else:
self.exceptDeps.append(exceptDeps)
# CNY-3518: set the default for whether to merge modules
if 'mergeKmodSymbols' in keywords:
self.mergeKmodSymbols = keywords.pop('mergeKmodSymbols')
self.recipe.RPMProvides(_mergeKmodSymbols=self.mergeKmodSymbols)
policy.Policy.updateArgs(self, **keywords)
def x86flags(archTag, baseArch, extraFlags, ofInterest):
try:
lines = open("/proc/cpuinfo").read().split("\n")
except IOError:
lines=[]
rc = [ (x, deps.FLAG_SENSE_PREFERRED) for x in extraFlags ]
for line in lines:
if not line.startswith("flags"): continue
fields = line.split()
if fields[0] != "flags": continue
for flag in fields[2:]:
if ofInterest.has_key(flag):
rc.append((flag, deps.FLAG_SENSE_PREFERRED))
return deps.Dependency(archTag, rc)
return deps.Dependency(archTag)
def getBuiltFlavor(flavor):
if not hasTarget(flavor):
majorArch = getTargetArch(flavor)[1]
if majorArch:
f = deps.Flavor()
f.addDep(deps.InstructionSetDependency, deps.Dependency(majorArch))
flavor = deps.overrideFlavor(flavor, f)
return flavor
if flavor.stronglySatisfies(crossFlavor):
return flavor
return setISFromTargetFlavor(flavor)
def __init__(self, name, recipe):
self.name = name
self.requires = deps.DependencySet()
self.provides = deps.DependencySet()
self.provides.addDep(deps.TroveDependencies, deps.Dependency(name))
self.flavor = _getUseFlavor(recipe)
self.linkGroups = {}
self.requiresMap = {}
self.providesMap = {}
self.hardlinkMap = {}
self.badhardlinks = []
self.recipe = recipe
dict.__init__(self)
def _addPhantomTrove(self, changeSet, rpmlibHeader, callback, num, total):
header = rpmhelper.headerFromBlob(rpmlibHeader.unload())
callback.capsuleSyncCreate(self.kind, str(header.getNevra()), num,
total)
name, version, flavor = self._getPhantomNVF(header)
# Fake trove
trv = trove.Trove(name, version, flavor)
provides = header.getProvides()
provides.addDep(deps.TroveDependencies, deps.Dependency(name))
trv.setProvides(provides)
trv.setRequires(header.getRequires(enableRPMVersionDeps=False))
# Fake capsule file
path = str(header.getNevra()) + '.rpm'
fileHelper = filetypes.RegularFile(contents='')
fileStream = fileHelper.get(pathId=trove.CAPSULE_PATHID)
trv.addRpmCapsule(path, version, fileStream.fileId(), header)
changeSet.addFile(None, fileStream.fileId(), fileStream.freeze())
# Fake encapsulated files
self._addPhantomContents(changeSet, trv, header)
trv.computeDigests()
changeSet.newTrove(trv.diff(None)[0])
# Make a fake package to contain the fake component
pkgName = name.split(':')[0]
if self.db.hasTrove(pkgName, version, flavor):
# It's possible to erase just the component and leave the package,
# so don't try to create it again.
return
pkg = trove.Trove(pkgName, version, flavor)
provides = deps.DependencySet()
provides.addDep(deps.TroveDependencies, deps.Dependency(pkgName))
pkg.setProvides(provides)
pkg.setIsCollection(True)
pkg.addTrove(name, version, flavor, byDefault=True)
pkg.computeDigests()
changeSet.newTrove(pkg.diff(None)[0])
def _toDependency(self, depType=deps.InstructionSetDependency):
""" Creates a Flavor dep set with the subarch in it.
Also includes any subsumed subarches if the
value of this subarch is true
(better comment about why we do that here)
"""
set = deps.Flavor()
sense = self._getDepSense()
depFlags = [ (self._name, sense) ]
parent = self._parent
if self._get():
depFlags.extend((parent[x]._name, sense) \
for x in self._subsumes)
dep = deps.Dependency(parent._name, depFlags)
set.addDep(depType, dep)
return set
def toDependency(self, override=None):
useFlags = deps.Flavor()
flags = [x.toDepFlag() for x in self.flags.values()]
useFlags.addDep(deps.UseDependency, deps.Dependency("use", flags))
if override:
if isinstance(override, list):
newUseFlags = []
for flavor in override:
useFlagsCopy = useFlags.copy()
useFlagsCopy.union(flavor,
mergeType=deps.DEP_MERGE_TYPE_OVERRIDE)
newUseFlags.append(useFlagsCopy)
return newUseFlags
else:
useFlags.union(override,
mergeType=deps.DEP_MERGE_TYPE_OVERRIDE)
return useFlags
def updateArgs(self, *args, **keywords):
if len(args) is 2:
name = args[1]
if ':' not in name:
name = name + ':rpm'
if not self.provisions.get(name):
self.provisions[name] = deps.DependencySet()
reMatch = self.provisionRe.match(args[0])
if not reMatch or len(reMatch.groups()) != 3:
return
depClass = reMatch.group(1).strip().lower()
if depClass != 'rpm' and depClass != 'rpmlib':
raise policy.PolicyError, "RPMProvides cannot be used to " \
"provide the non-rpm dependency: '%s'" % args[0]
dep = reMatch.group(2).strip()
flags = reMatch.group(3).strip().split()
flags = [(x, deps.FLAG_SENSE_REQUIRED) for x in flags if x]
if not self.provisions.get(name):
self.provisions[name] = deps.DependencySet()
self.provisions[name].addDep(
deps.dependencyClassesByName[depClass],
deps.Dependency(dep, flags))
exceptDeps = keywords.pop('exceptDeps', None)
if exceptDeps:
if type(exceptDeps) is str:
exceptDeps = ('.*', exceptDeps)
assert (type(exceptDeps) == tuple)
if type(exceptDeps[0]) is tuple:
self.exceptDeps.extend(exceptDeps)
else:
self.exceptDeps.append(exceptDeps)
# CNY-3518: set the default for whether to merge modules --
# this should be passed in only from RPMRequires
if '_mergeKmodSymbols' in keywords:
self.mergeKmodSymbols = keywords.pop('_mergeKmodSymbols')
policy.Policy.updateArgs(self, **keywords)
def fragment_flavor(package, flavor):
'''
Select instruction set and package flags from a flavor and return just
those parts.
'''
new_flavor = deps.Flavor()
for dep in flavor.iterDepsByClass(deps.InstructionSetDependency):
# Instruction set (e.g. arch)
new_flavor.addDep(deps.InstructionSetDependency, dep)
for dep in flavor.iterDepsByClass(deps.UseDependency):
new_flags = {}
for flag, sense in dep.flags.iteritems():
if flag.startswith(package + '.'):
new_flags[flag] = sense
new_flavor.addDep(deps.UseDependency,
deps.Dependency('use', new_flags))
return new_flavor
def __init__(self, readConfigFiles=False, ignoreErrors=False):
self.setIgnoreErrors(ignoreErrors)
servercfg.rMakeBuilderConfiguration.__init__(self)
if readConfigFiles:
self.readFiles()
if not self.hostName:
self.hostName = socket.getfqdn()
if not self.name:
self.name = self.hostName.split('.')[0]
if not self.buildFlavors:
insSet = deps.DependencySet()
for depList in arch.currentArch:
for dep in depList:
flags = dep.getFlags()[0]
# don't include "prefers" flags.
flags = [(x[0], x[1]) for x in flags
if x[1] in (deps.FLAG_SENSE_REQUIRED,
deps.FLAG_SENSE_DISALLOWED)]
newDep = deps.Dependency(dep.name, flags)
insSet.addDep(deps.InstructionSetDependency, newDep)
self.buildFlavors.add(insSet)
def _getBuildFlavor(cls, flavor=None):
"""
Converts a B{opaque} flavor object into an B{opaque} build
flavor object, striping any biarch flavor to just x86_64.
@param flavor: conary flavor
@type flavor: string or B{opaque} conary.deps.deps.Flavor
@return: B{opaque} Conary flavor object
@rtype: conary.deps.deps.Flavor
"""
flavor = cls._getFlavor(flavor=flavor)
# Remove any x86 dependencies that part of the flavor.
biarch = deps.parseFlavor('is: x86 x86_64')
if flavor.stronglySatisfies(biarch):
# Get a new flavor before modifying it.
flavor = flavor.copy()
# Remove the x86 deps.
flavor.removeDeps(deps.InstructionSetDependency,
[deps.Dependency('x86'), ])
return flavor
def mask_flavor(baseFlavor, maskFlavor):
'''
Remove flags from I{baseFlavor} not present in I{maskFlavor}. Sense of
flags in I{maskFlavor} is ignored.
'''
new_flavor = deps.Flavor()
for cls, mask_dep in maskFlavor.iterDeps():
base_class = baseFlavor.members.get(cls.tag, None)
if base_class is None:
continue
base_dep = base_class.members.get(mask_dep.name, None)
if base_dep is None:
continue
new_flags = {}
for flag, _ in mask_dep.flags.iteritems():
if flag in base_dep.flags:
new_flags[flag] = base_dep.flags[flag]
new_flavor.addDep(cls, deps.Dependency(mask_dep.name, new_flags))
return new_flavor
def do(self):
"""
Remove files from the Conary package manifest that match any specified
filters.
"""
# Compile the set of filters.
filters = {}
for name, regex in self.extraFilters:
name %= self.macros
filters.setdefault(name, []).append(
filter.Filter(regex, self.macros, name=name))
# Get a mapping of component name to component object
components = dict(
(x.name, x) for x in self.recipe.autopkg.getComponents())
for name, fltrs in filters.iteritems():
if name not in components:
self.error("Component %s does not exist" % name)
continue
provides = components[name].provides
# make a copy of the files list since it will be modified in place.
files = components[name].keys()
for fn in files:
for fltr in fltrs:
if fltr.match(fn):
self.recipe.autopkg.delFile(fn)
self.recipe._capsulePathMap.pop(fn)
self.recipe._capsuleDataMap.pop(fn)
provides.removeDeps(deps.FileDependencies,
[deps.Dependency(fn)],
missingOkay=True)
break
def getTupleStrings(self, n, v, f):
"""
returns potentially shortened display versions and flavors for
a trove tuple.
@param n: trove name
@type n: str
@param v: trove version
@type v: versions.Version
@param f: trove flavor
@type f: deps.deps.Flavor
@rtype: (vStr, fStr) where vStr is the version string to display
for this trove and fStr is the flavor string (may be empty)
"""
if self.dcfg.useFullVersions():
vStr = str(v)
elif self.dcfg.showLabels:
vStr = '%s/%s' % (v.branch().label(), v.trailingRevision())
elif len(self._vCache.get(n, [])) > 1:
# print the trailing revision unless there
# are two versions that are on different
# branches.
vers = self._vCache[n]
branch = v.branch()
vStr = None
for ver in vers:
if ver.branch() != branch:
vStr = str(v)
if not vStr:
vStr = str(v.trailingRevision())
else:
vStr = str(v.trailingRevision())
if self.dcfg.useFullFlavors():
fStr = str(f)
else:
# print only the flavor differences between
# the two troves.
# FIXME Document all this!
if self.dcfg.affinityDb:
installed = set()
#installed = set([x[2] for x in self.dcfg.affinityDb.trovesByName(n)])
installed.discard(f)
installed = list(installed)
else:
installed = []
flavors = list(self._fCache.get(n, []))
if installed:
for baseFlavor in self.dcfg.baseFlavors:
flavors += [
deps.overrideFlavor(baseFlavor, x) for x in installed
]
else:
flavors += self.dcfg.baseFlavors
if len(flavors) > 1:
fStr = deps.flavorDifferences(list(flavors) + installed + [f],
strict=False)[f]
ISD = deps.InstructionSetDependency
if f.hasDepClass(ISD) and fStr.hasDepClass(ISD):
allDeps = set(x.name for x in f.iterDepsByClass(ISD))
allDeps.difference_update(
x.name for x in fStr.iterDepsByClass(ISD))
for depName in allDeps:
fStr.addDep(ISD, deps.Dependency(depName))
fStr = str(fStr)
else:
fStr = ''
return vStr, fStr
def doFile(self, path):
if hasattr(self.recipe, '_getCapsulePathsForFile'):
if self.recipe._getCapsulePathsForFile(path):
return
d = self.macros.destdir
f = util.joinPaths(d, path)
if not os.path.islink(f):
return
recipe = self.recipe
contents = os.readlink(f)
if contents[0] == '/':
self.warn(
'Absolute symlink %s points to %s,'
' should probably be relative', path, contents)
return
abscontents = util.joinPaths(os.path.dirname(path), contents)
# now resolve any intermediate symlinks
dl = len(os.path.realpath(d))
abscontents = os.path.realpath(d + abscontents)[dl:]
ap = recipe.autopkg
if abscontents in ap.pathMap:
if ap.findComponent(abscontents) != ap.findComponent(path) and \
not path.endswith('.so') and \
not ap.findComponent(path).getName().endswith(':test'):
# warn about suspicious cross-component symlink
fromPkg = ap.findComponent(path)
targetPkg = ap.findComponent(abscontents)
found = False
for depClass, dep in fromPkg.requires.iterDeps():
d = deps.DependencySet()
d.addDep(depClass, dep)
if targetPkg.provides.satisfies(d):
found = True
break
if not found:
self.warn('symlink %s points from package %s to %s', path,
ap.findComponent(path).getName(),
ap.findComponent(abscontents).getName())
else:
for targetFilter, requirement in self.targetFilters:
if targetFilter.match(abscontents):
# contents are an exception
self.info('allowing special dangling symlink %s -> %s',
path, contents)
if requirement:
self.info(
'automatically adding requirement'
' %s for symlink %s', requirement, path)
# Requires has already run, touch this up
pkg = ap.findComponent(path)
if path not in pkg.requiresMap:
pkg.requiresMap[path] = deps.DependencySet()
pkg.requiresMap[path].addDep(
deps.TroveDependencies,
deps.Dependency(requirement, []))
f = pkg.getFile(path)
f.requires.set(pkg.requiresMap[path])
pkg.requires.union(f.requires())
return
for pathName in recipe.autopkg.pathMap:
if pathName.startswith(abscontents):
# a link to a subdirectory of a file that is
# packaged is still OK; this test is expensive
# and almost never needed, so put off till last
return
self.error("Dangling symlink: %s points to non-existant %s (%s)" %
(path, contents, abscontents))
# now that an error has been logged, we need to get rid of the file
# so the rest of policy won't barf trying to access a file which
# doesn't *really* exist (CNP-59)
os.unlink(self.recipe.macros.destdir + path)
def _toDependency(self, depType=deps.InstructionSetDependency):
set = deps.Flavor()
sense = self._getDepSense()
dep = deps.Dependency(self._name, [])
set.addDep(depType, dep)
return set
def testTroveFlavor(self):
flavor = deps.Flavor()
flavor.addDep(
deps.UseDependency,
deps.Dependency('use', [('foo', deps.FLAG_SENSE_REQUIRED)]))
self.testTroves(flavor=flavor)
def testDatabase1(self):
db = sqldb.Database(':memory:')
f1 = files.FileFromFilesystem("/etc/passwd", self.id1)
f2 = files.FileFromFilesystem("/etc/services", self.id2)
f3 = files.FileFromFilesystem("/etc/group", self.id3)
trv = trove.Trove("testcomp", self.v10, self.emptyFlavor, None)
trv.addFile(self.id1, "/bin/1", self.v10, f1.fileId())
trv.addFile(self.id2, "/bin/2", self.v10, f2.fileId())
trv.addFile(self.id3, "/bin/3", self.v10, f3.fileId())
trv.troveInfo.size.set(1234)
trv.troveInfo.sourceName.set('thesource')
req = deps.DependencySet()
req.addDep(deps.FileDependencies, deps.Dependency("/bin/bash"))
req.addDep(deps.TroveDependencies, deps.Dependency("foo:runtime"))
req.addDep(deps.SonameDependencies, deps.Dependency("libtest.so.1"))
trv.setRequires(req)
trvInfo = db.addTrove(trv)
db.addFile(trvInfo,
f1.pathId(),
"/bin/1",
f1.fileId(),
self.v10,
fileStream=f1.freeze())
db.addFile(trvInfo,
f2.pathId(),
"/bin/2",
f2.fileId(),
self.v10,
fileStream=f2.freeze())
db.addFile(trvInfo,
f3.pathId(),
"/bin/3",
f3.fileId(),
self.v10,
fileStream=f3.freeze())
db.addTroveDone(trvInfo)
dbTrv = db.getTroves([("testcomp", self.v10, self.emptyFlavor)])[0]
assert (dbTrv == trv)
assert (dbTrv.__class__ == trove.Trove)
assert (db.trovesArePinned([("testcomp", self.v10, self.emptyFlavor)
]) == [False])
dbTrv = db.getTroves([("testcomp", self.v10, self.emptyFlavor)],
withFileObjects=True)[0]
assert (dbTrv == trv)
assert (dbTrv.__class__ == trove.TroveWithFileObjects)
for f in (f1, f2, f3):
assert (dbTrv.getFileObject(f.fileId()) == f)
trv2 = trove.Trove("testpkg", self.v10, self.emptyFlavor, None)
ti = trv2.addTrove(trv.getName(), self.v10, trv.getFlavor())
trv2.addTrove("weakref", self.v10, trv.getFlavor(), weakRef=True)
ti = db.addTrove(trv2, pin=True)
db.addTroveDone(ti)
assert (db.trovesArePinned([("testpkg", self.v10, self.emptyFlavor)
]) == [True])
assert (db.getTroves([("testpkg", self.v10, self.emptyFlavor)
])[0] == trv2)
assert (db.getTroves([
("testpkg", self.v10, self.emptyFlavor)
])[0].getVersion().timeStamps() == trv2.getVersion().timeStamps())
assert (db.getTroves([("testpkg", self.v10, self.emptyFlavor),
("testcomp", self.v10, self.emptyFlavor),
("testitem", self.v10, self.emptyFlavor)],
True) == [trv2, trv, None])
assert (db.getTroves([("testpkg", self.v10, self.emptyFlavor),
("testcomp", self.v10, req)],
True) == [trv2, None])
assert (db.findTroveContainers(["testpkg",
"testcomp"]) == [[],
[("testpkg", self.v10,
self.emptyFlavor)]])
assert (db.getTroveContainers([
("testpkg", self.v10, self.emptyFlavor),
("testcomp", self.v10, self.emptyFlavor)
]) == [[], [("testpkg", self.v10, self.emptyFlavor)]])
res = db.findTroveReferences(["testpkg", "testcomp"])
assert (db.findTroveReferences(["testpkg", "testcomp"
]) == [[],
[("testcomp", self.v10,
self.emptyFlavor)]])
v10new = VersionFromString("/conary.rpath.com@test:trunk/1.2-10")
assert (db.getTroves([("testpkg", v10new, self.emptyFlavor)
])[0] == trv2)
assert (db.getTroves([
("testpkg", v10new, self.emptyFlavor)
])[0].getVersion().timeStamps() == trv2.getVersion().timeStamps())
assert (set(db.findByNames(['testpkg', 'testcomp'])) == set([
("testpkg", self.v10, self.emptyFlavor),
("testcomp", self.v10, self.emptyFlavor)
]))
db.eraseTrove("testcomp", self.v10, None)
assert (db.getTroves([("testpkg", self.v10, self.emptyFlavor)
])[0] == trv2)
trv.computePathHashes()
trvInfo = db.addTrove(trv)
db.addFile(trvInfo,
f1.pathId(),
"/bin/1",
f1.fileId(),
self.v10,
fileStream=f1.freeze())
db.addFile(trvInfo,
f2.pathId(),
"/bin/2",
f2.fileId(),
self.v10,
fileStream=f1.freeze())
db.addFile(trvInfo,
f3.pathId(),
"/bin/3",
f3.fileId(),
self.v10,
fileStream=f1.freeze())
db.addTroveDone(trvInfo)
assert (db.getTroves([("testcomp", self.v10, self.emptyFlavor)
])[0] == trv)
db.removeFileFromTrove(trv, "/bin/1")
changedTrv = db.getTroves([trv.getNameVersionFlavor()],
pristine=False)[0]
otherChangedTrv = db.getTroves([trv.getNameVersionFlavor()],
withFiles=False,
pristine=False)[0]
assert (len(changedTrv.idMap) + 1 == len(trv.idMap))
assert (len(changedTrv.troveInfo.pathHashes) + 1 == len(
trv.troveInfo.pathHashes))
assert (changedTrv.troveInfo.pathHashes ==
otherChangedTrv.troveInfo.pathHashes)
assert (len(otherChangedTrv.idMap) == 0)
changedTrv.addFile(self.id1, "/bin/1", self.v10, f1.fileId())
assert (changedTrv.idMap == trv.idMap)
changedTrv.computePathHashes()
assert (changedTrv.troveInfo.pathHashes == trv.troveInfo.pathHashes)
assert (db.getTroves([("testcomp", self.v10, self.emptyFlavor)],
pristine=True)[0] == trv)
db.eraseTrove("testpkg", self.v10, None)
assert (db.getTroves([("testpkg", self.v10, self.emptyFlavor)
]) == [None])
self.assertRaises(KeyError, db.instances.getVersion, 100)
db.eraseTrove("testcomp", self.v10, None)
db.commit()
cu = db.db.cursor()
# make sure the versions got removed; the None entry is still there
cu.execute("SELECT count(*) FROM Versions")
assert (cu.next()[0] == 1)
# make sure the dependency table got cleaned up
cu.execute("SELECT count(*) FROM Dependencies")
assert (cu.next()[0] == 0)
# make sure the instances table got cleaned up
cu.execute("SELECT count(*) FROM Instances")
assert (cu.next()[0] == 0)
# make sure the troveInfo table got cleaned up
cu.execute("SELECT count(*) FROM TroveInfo")
assert (cu.next()[0] == 0)
def testTroves(self, flavor=None):
if flavor is None:
flavor = deps.Flavor()
store = self._connect()
dirSet = set(['/etc', '/bin'])
baseSet = set(
['passwd', 'services', 'group', '1', '2', '3', 'distributed'])
v10 = ThawVersion("/conary.rpath.com@test:trunk/10:1.2-10")
branch = v10.branch()
store.createTroveBranch("testtrove", branch)
f1 = files.FileFromFilesystem("/etc/passwd", self.id1)
f2 = files.FileFromFilesystem("/etc/services", self.id2)
f3 = files.FileFromFilesystem("/etc/group", self.id3)
# make a really huge dependency, thus a very large file stream
req = deps.DependencySet()
for x in xrange(10000):
req.addDep(deps.SonameDependencies,
deps.Dependency("libtest.so.%d" % x))
f3.requires.set(req)
# make sure it's way too big for a blob in mysql
assert (len(f3.freeze()) >= 50000)
cl = changelog.ChangeLog(
"test", "*****@*****.**", """\
Some changes are good.
Some changes are bad.
Some changes just are.
""")
trv = trove.Trove('testcomp', v10, flavor, cl)
trv.addFile(f1.pathId(), "/bin/1", v10, f1.fileId())
trv.addFile(f2.pathId(), "/bin/2", v10, f2.fileId())
trv.addFile(f3.pathId(), "/bin/3", v10, f3.fileId())
trv.addFile(self.id4, "/bin/distributed", v10, self.fid4)
trv.troveInfo.size.set(1234)
trv.troveInfo.sourceName.set('somesource')
req = deps.DependencySet()
req.addDep(deps.FileDependencies, deps.Dependency("/bin/bash"))
req.addDep(deps.TroveDependencies, deps.Dependency("foo:runtime"))
req.addDep(deps.SonameDependencies, deps.Dependency("libtest.so.1"))
trv.setRequires(req)
# this also lets us peek at the database to make sure libtest.so.1
# is only in the dep table once
prv = deps.DependencySet()
prv.addDep(deps.SonameDependencies, deps.Dependency("libtest.so.1"))
trv.setProvides(prv)
trv.computeDigests()
store.db.transaction()
store.addTroveSetStart([], dirSet, baseSet)
troveInfo = store.addTrove(trv, trv.diff(None)[0])
troveInfo.addFile(f1.pathId(),
"/bin/1",
f1.fileId(),
v10,
fileStream=f1.freeze())
troveInfo.addFile(f2.pathId(),
"/bin/2",
f2.fileId(),
v10,
fileStream=f2.freeze())
troveInfo.addFile(f3.pathId(),
"/bin/3",
f3.fileId(),
v10,
fileStream=f3.freeze())
troveInfo.addFile(self.id4, "/bin/distributed", self.fid4, v10)
store.addTroveDone(troveInfo)
store.addTroveSetDone()
store.db.commit()
cu = store.db.cursor()
cu.execute("SELECT count(*) FROM Dependencies WHERE "
"name = 'libtest.so.1'")
self.assertEqual(cu.next(), (1, ))
# make sure the sha1s were stored
cu.execute("""
SELECT dirname, basename, sha1
FROM TroveFiles
JOIN FileStreams USING (streamId)
JOIN FilePaths ON TroveFiles.filePathId = FilePaths.filePathId
JOIN Dirnames ON FilePaths.dirnameId = Dirnames.dirnameId
JOIN Basenames ON FilePaths.basenameId = Basenames.basenameId
ORDER BY dirname,basename""")
items = [(os.path.join(cu.frombinary(x[0]),
cu.frombinary(x[1])), cu.frombinary(x[2]))
for x in cu.fetchall()]
self.assertEqual(items, [("/bin/1", f1.contents.sha1()),
("/bin/2", f2.contents.sha1()),
("/bin/3", f3.contents.sha1()),
("/bin/distributed", None)])
cl = changelog.ChangeLog("test", "*****@*****.**", "another log\n")
fromRepos = store.getTrove("testcomp", v10, flavor, cl)
self.assertEqual(fromRepos, trv)
self.assertEqual(fromRepos.getVersion().timeStamps(),
trv.getVersion().timeStamps())
self.assertEqual(fromRepos.getChangeLog(), trv.getChangeLog())
self.assertEqual([
x for x in store.getTrove("testcomp", v10, flavor,
withFiles=False).iterFileList()
], [])
l = store.iterFilesInTrove("testcomp", v10, flavor, sortByPath=True)
l = [x for x in l]
self.assertEqual(l, [(f1.pathId(), "/bin/1", f1.fileId(), v10),
(f2.pathId(), "/bin/2", f2.fileId(), v10),
(f3.pathId(), "/bin/3", f3.fileId(), v10),
(self.id4, "/bin/distributed", self.fid4, v10)])
cl = changelog.ChangeLog("test", "*****@*****.**", "log for testpkg\n")
trv2 = trove.Trove("testpkg", v10, flavor, cl)
trv2.addTrove(trv.getName(), v10, flavor)
trv2.addTrove("weakref", v10, flavor, weakRef=True)
trv2.computeDigests()
store.addTroveSetStart([], dirSet, baseSet)
troveInfo = store.addTrove(trv2, trv2.diff(None)[0])
store.addTroveDone(troveInfo)
store.addTroveSetDone()
self.assertEqual(store.getTrove("testpkg", v10, flavor), trv2)
self.assertEqual([
x for x in store.iterTroves([("testcomp", v10,
flavor), ("testpkg", v10, flavor)])
], [trv, trv2])
self.assertEqual([
x for x in store.iterTroves([("testpkg", v10,
flavor), ("testcomp", v10, flavor)])
], [trv2, trv])
self.assertEqual([
x for x in store.iterTroves([("testpkg", v10,
flavor), ("testpkg", v10, flavor)])
], [trv2, trv2])
self.assertEqual([
x for x in store.iterTroves([("testpkg", v10,
flavor), ("blah", v10, flavor)])
], [trv2, None])
self.assertEqual([
x for x in store.iterTroves([("blah", v10,
flavor), ("testpkg", v10, flavor)])
], [None, trv2])
self.assertEqual(
[x for x in store.iterTroves([("blah", v10, flavor)])], [None])
self.assertEqual([
x for x in store.iterTroves([(
"testcomp", v10, flavor), ("blah", v10,
flavor), ("testpkg", v10, flavor)])
], [trv, None, trv2])
# erasing doesn't work
#store.eraseTrove("testcomp", v10, None)
#store.commit()
self.assertEqual(store.getTrove("testpkg", v10, flavor), trv2)
map = {'testpkg': [v10]}
flavors = store.getTroveFlavors(map)
if flavor is not None:
flavorStr = flavor.freeze()
else:
flavorStr = ''
self.assertEqual(flavors, {'testpkg': {v10: [flavorStr]}})
map = {'testpkg3': [v10]}
flavors = store.getTroveFlavors(map)
self.assertEqual(flavors, {'testpkg3': {v10: []}})
# test getFiles
fileObjs = store.getFiles([(f1.pathId(), f1.fileId()),
(f2.pathId(), f2.fileId())])
self.assertEqual(fileObjs[(f1.pathId(), f1.fileId())], f1)
self.assertEqual(fileObjs[(f2.pathId(), f2.fileId())], f2)
# test that asking for an invalid fileid/pathid pair results
# in no entry for the (pathid, fileid) in the returned dict
invalidPathId = md5FromString('9' * 32)
invalidFileId = sha1FromString('9' * 40)
fileObjs = store.getFiles([(invalidPathId, invalidFileId)])
# make sure fileObjs is empty
assert (not fileObjs)
# test that asking for contents that have to come from
# a different repository works - we should get None
# back
fileObjs = store.getFiles([(self.id4, self.fid4)])
self.assertEqual(fileObjs, {(self.id4, self.fid4): None})
def _getDepsetFromHeader(self, tags, mergeKmodSymbols=False):
if isinstance(tags, tuple):
assert len(tags) == 2
rpmdeps = self.get(tags[0], [])
rpmvers = self.get(tags[1], [])
if len(rpmdeps) != len(rpmvers):
rpmvers = itertools.repeat(None, len(rpmdeps))
else:
rpmdeps = self.get(tags, [])
rpmvers = itertools.repeat(None, len(rpmdeps))
depset = deps.DependencySet()
for dep, ver in itertools.izip(rpmdeps, rpmvers):
if dep.startswith('/'):
depset.addDep(deps.FileDependencies, deps.Dependency(dep))
elif dep.startswith('rpmlib'):
# this is of the form rpmlib(Something). We just want the
# Something
depset.addDep(deps.RpmLibDependencies,
deps.Dependency(dep.split('(')[1].split(')')[0]))
elif '(' in dep:
if '.so' in dep.split('(')[0] and not (
dep.startswith('perl(') or dep.startswith('config(')):
# assume it is a shlib or package name;
# convert anything inside () to a flag
flags = self.flagre.findall(dep)
if flags:
# the dependency name is everything until the first (
dep = self.depnamere.match(dep).group(1)
if len(flags) == 2:
# if we have (flags)(64bit), we need to pop
# the 64bit marking off the end and namespace the
# dependency name.
dep += '[%s]' % flags.pop()
flags = [(x, deps.FLAG_SENSE_REQUIRED) for x in flags
if x]
else:
flags = []
depset.addDep(deps.RpmDependencies,
deps.Dependency(dep, flags))
elif self.localere.match(dep):
# locale RPM flags get translated to conary dep flags
m = self.localere.match(dep)
nf = m.group(1).split(':')
if len(nf) == 1:
name = ''
flags = nf[0].split(';')
else:
name = ':' + ':'.join(nf[0:-1])
flags = nf[-1].split(';')
flags = [(x, deps.FLAG_SENSE_REQUIRED) for x in flags if x]
depset.addDep(deps.RpmDependencies,
deps.Dependency('locale%s' % name, flags))
elif self.kmodre.match(dep):
m = self.kmodre.match(dep)
modname = m.group(2)
# add the version if it is a hex string with at least
# 8 chars
l = None
if ver and len(ver) >= 8:
try:
l = long(ver, 16)
except ValueError:
pass
if l:
modname = "%s:%s" % (modname, ver)
else:
log.warning("dependency '%s' is expected to have "
"a hexadecimal hash >= 8 characters "
"for a version. Instead it has a "
"version of '%s' which will be "
"ignored." % (dep, ver))
if mergeKmodSymbols:
flags = [
(modname, deps.FLAG_SENSE_REQUIRED),
]
depset.addDep(deps.RpmDependencies,
deps.Dependency(m.group(1), flags))
else:
modname = '%s[%s]' % (m.group(1), modname)
flags = []
depset.addDep(deps.RpmDependencies,
deps.Dependency(modname, flags))
else:
# replace any () with [] because () are special to Conary
dep = dep.replace('(', '[').replace(')', ']')
depset.addDep(deps.RpmDependencies,
deps.Dependency(dep, []))
else:
depset.addDep(deps.RpmDependencies, deps.Dependency(dep, []))
return depset
def do(self):
for comp in self.recipe.autopkg.components.items():
capsule = self.recipe._getCapsule(comp[0])
if capsule and capsule[0] == 'rpm':
if not self.filters:
self.filters = [(x, filter.Filter(x, self.macros))
for x in self.excepts]
path = capsule[1]
matchFound = False
for regexp, f in self.filters:
if f.match(path):
self.unusedFilters['exceptions'].discard(regexp)
matchFound = True
if matchFound:
continue
h = rpmhelper.readHeader(file(path))
rReqs, rProv = h.getDeps(
mergeKmodSymbols=self.mergeKmodSymbols)
# integrate user specified requirements
if self.requirements:
userReqs = self.requirements.get(comp[0])
if userReqs:
rReqs.union(userReqs)
# remove rpm provisions from the requirements
rReqs = rReqs.difference(rProv)
# cull duplicate rpm reqs that have a standard conary
# representations
# currently we only handle perl and sonames
culledReqs = deps.DependencySet()
cnyReqs = comp[1].requires
cnyProv = comp[1].provides
if rReqs.hasDepClass(deps.RpmDependencies):
soDeps = deps.DependencySet()
soDeps.addDeps(deps.SonameDependencies, \
list(cnyReqs.iterDepsByClass(deps.SonameDependencies))+\
list(cnyProv.iterDepsByClass(deps.SonameDependencies)))
for r in list(rReqs.iterDepsByClass(deps.RpmDependencies)):
reMatch = self.rpmStringRe.match(r.name)
if reMatch and reMatch.groups():
rpmFile = reMatch.group(1)
rpmFlags = reMatch.group(2).strip()
else:
rpmFile = r.name
rpmFlags = ''
if rpmFile == 'perl' and rpmFlags:
ds = deps.DependencySet()
dep = deps.Dependency(rpmFlags)
dep.flags = r.flags
ds.addDep(deps.PerlDependencies, dep)
if cnyReqs.satisfies(ds) or \
cnyProv.satisfies(ds):
culledReqs.addDep(deps.RpmDependencies, r)
elif '.so' in rpmFile:
ds = deps.DependencySet()
if rpmFlags == '64bit':
elfPrefix = 'ELF64/'
else:
elfPrefix = 'ELF32/'
dep = deps.Dependency(elfPrefix + rpmFile)
dep.flags = r.flags
ds.addDep(deps.SonameDependencies, dep)
if soDeps.satisfies(ds):
culledReqs.addDep(deps.RpmDependencies, r)
rReqs = rReqs.difference(culledReqs)
# remove any excepted deps
for filt, exceptRe in self.exceptDeps:
if filt.match(path):
for depClass, dep in list(rReqs.iterDeps()):
matchName = '%s: %s' % (depClass.tagName, str(dep))
if exceptRe.match(matchName):
rReqs.removeDeps(depClass, [dep])
cnyReqs.union(rReqs)
