def typeFilter(l, filterSet):
if not filterSet:
return l
filters = []
if 'group' in filterSet:
filters.append(trove.troveIsGroup)
if 'package' in filterSet:
filters.append(trove.troveIsPackage)
if 'component' in filterSet:
filters.append(trove.troveIsComponent)
if 'fileset' in filterSet:
filters.append(trove.troveIsFileSet)
if 'collection' in filterSet:
filters.append(trove.troveIsCollection)
if 'source' in filterSet:
filters.append(trove.troveIsSourceComponent)
if 'binarycomponent' in filterSet:
filters.append(lambda x: trove.troveIsComponent(x) and
not trove.troveIsSourceComponent(x))
filters.append(None)
filteredL = []
for item in l:
if filters:
for f in filters:
if f and f(item[0]): break
if f is None:
continue
filteredL.append(item)
return filteredL
def typeFilter(l, filterSet):
if not filterSet:
return l
filters = []
if 'group' in filterSet:
filters.append(trove.troveIsGroup)
if 'package' in filterSet:
filters.append(trove.troveIsPackage)
if 'component' in filterSet:
filters.append(trove.troveIsComponent)
if 'fileset' in filterSet:
filters.append(trove.troveIsFileSet)
if 'collection' in filterSet:
filters.append(trove.troveIsCollection)
if 'source' in filterSet:
filters.append(trove.troveIsSourceComponent)
if 'binarycomponent' in filterSet:
filters.append(lambda x: trove.troveIsComponent(x) and
not trove.troveIsSourceComponent(x))
filters.append(None)
filteredL = []
for item in l:
if filters:
for f in filters:
if f and f(item[0]): break
if f is None:
continue
filteredL.append(item)
return filteredL
def _cached(self, troveTupList, troveList):
for tup, trv in itertools.izip(troveTupList, troveList):
for name, version, flavor in trv.iterTroveList(strongRefs = True,
weakRefs = True):
if not trove.troveIsComponent(name):
continue
pkgName = name.split(':')[0]
tup = (pkgName, version, flavor)
l = self.componentMap.get(tup)
if l is None:
l = []
self.componentMap[tup] = l
l.append(name)
def includeAction(self, data):
if self.resultSet:
self.outSet._setInstall(self.resultSet._getInstallSet())
self.outSet._setOptional(self.resultSet._getOptionalSet())
return True
assert(not self.includeSet._getOptionalSet())
assert(not self.includeSet._getInstallSet() == 1)
nvf = list(self.includeSet._getInstallSet())[0]
if not trove.troveIsComponent(nvf[0]):
assert(trove.troveIsPackage(nvf[0]))
# getTrove is sometimes disabled to prevent one at a time calls
# can't be helped here
trv = data.troveCache.getTroves([ nvf], withFiles=False)[0]
found = None
for subNVF in trv.iterTroveList(strongRefs = True):
if subNVF[0].endswith(':cml'):
found = subNVF
break
if not found:
raise IncludeException('Package %s=%s[%s] does not contain a '
'cml component for inclusion' % nvf)
nvf = found
elif nvf[0].split(':')[1] not in [ 'cml', 'source' ]:
raise IncludeException('Include only supports source and cml '
'components')
if nvf in self.outSet.g.included:
raise IncludeException('Include loop detected involving %s=%s[%s]'
% nvf)
self.outSet.g.included.add(nvf)
cmlFileLines = self.getCML(data.troveCache, nvf)
model = cml.CML(None, context = nvf[0])
model.parse(fileData = cmlFileLines)
self.resultSet = self.compiler.augment(model, self.searchSet,
self.primaryTroveSet)
self.outSet.g.addEdge(self.resultSet, self.outSet)
self.outSet.finalSearchSet.setTroveSetList(
self.outSet.finalSearchSet.fetch(
[ self.resultSet.searchPath ]) )
return False
def _cached(self, troveTupList, troveList):
for tup, trv in itertools.izip(troveTupList, troveList):
for name, version, flavor in trv.iterTroveList(strongRefs=True,
weakRefs=True):
if not trove.troveIsComponent(name):
continue
pkgName = name.split(':')[0]
tup = (pkgName, version, flavor)
l = self.componentMap.get(tup)
if l is None:
l = []
self.componentMap[tup] = l
l.append(name)
def includeAction(self, data):
if self.resultSet:
self.outSet._setInstall(self.resultSet._getInstallSet())
self.outSet._setOptional(self.resultSet._getOptionalSet())
return True
assert (not self.includeSet._getOptionalSet())
assert (not self.includeSet._getInstallSet() == 1)
nvf = list(self.includeSet._getInstallSet())[0]
if not trove.troveIsComponent(nvf[0]):
assert (trove.troveIsPackage(nvf[0]))
# getTrove is sometimes disabled to prevent one at a time calls
# can't be helped here
trv = data.troveCache.getTroves([nvf], withFiles=False)[0]
found = None
for subNVF in trv.iterTroveList(strongRefs=True):
if subNVF[0].endswith(':cml'):
found = subNVF
break
if not found:
raise IncludeException('Package %s=%s[%s] does not contain a '
'cml component for inclusion' % nvf)
nvf = found
elif nvf[0].split(':')[1] not in ['cml', 'source']:
raise IncludeException('Include only supports source and cml '
'components')
if nvf in self.outSet.g.included:
raise IncludeException(
'Include loop detected involving %s=%s[%s]' % nvf)
self.outSet.g.included.add(nvf)
cmlFileLines = self.getCML(data.troveCache, nvf)
model = cml.CML(None, context=nvf[0])
model.parse(fileData=cmlFileLines)
self.resultSet = self.compiler.augment(model, self.searchSet,
self.primaryTroveSet)
self.outSet.g.addEdge(self.resultSet, self.outSet)
self.outSet.finalSearchSet.setTroveSetList(
self.outSet.finalSearchSet.fetch([self.resultSet.searchPath]))
return False
def _fetch(actionList, data):
troveTuples = set()
for action in actionList:
newTuples = [ x[0] for x in
action.primaryTroveSet._walk(data.troveCache,
newGroups = False) ]
if not action.fetchAll:
newTuples = [ x for x in newTuples
if not trove.troveIsComponent(x[0]) ]
troveTuples.update(newTuples)
data.troveCache.getTroves(troveTuples, withFiles = False)
def _closePackages(self, cache, trv, newTroves = None):
packagesAdded = set()
if newTroves is None:
newTroves = list(trv.iterTroveList(strongRefs = True))
for n, v, f in newTroves:
if trove.troveIsComponent(n):
packageN = n.split(':')[0]
if not trv.hasTrove(packageN, v, f):
log.info("adding package %s for component %s",
packageN, (n, v, f))
trv.addTrove(packageN, v, f)
packagesAdded.add( (packageN, v, f) )
cache.cacheComponentMap(packagesAdded)
return packagesAdded
def _closePackages(self, cache, trv, newTroves=None):
packagesAdded = set()
if newTroves is None:
newTroves = list(trv.iterTroveList(strongRefs=True))
for n, v, f in newTroves:
if trove.troveIsComponent(n):
packageN = n.split(':')[0]
if not trv.hasTrove(packageN, v, f):
log.info("adding package %s for component %s", packageN,
(n, v, f))
trv.addTrove(packageN, v, f)
packagesAdded.add((packageN, v, f))
cache.cacheComponentMap(packagesAdded)
return packagesAdded
def _fetch(actionList, data):
troveTuples = set()
for action in actionList:
newTuples = [
x[0] for x in action.primaryTroveSet._walk(data.troveCache,
newGroups=False)
]
if not action.fetchAll:
newTuples = [
x for x in newTuples if not trove.troveIsComponent(x[0])
]
troveTuples.update(newTuples)
data.troveCache.getTroves(troveTuples, withFiles=False)
def __init__(self, troveName, version = None):
"""
Initializes a TroveMissing exception.
@param troveName: trove which could not be found
@type troveName: str
@param version: version of the trove which does not exist
@type version: versions.Version, VFS string or [versions.Version]
"""
self.troveName = troveName
self.version = version
if trove.troveIsGroup(troveName):
self.type = 'group'
elif trove.troveIsFileSet(troveName):
self.type = 'fileset'
elif trove.troveIsComponent(troveName):
self.type = 'component'
else:
self.type = 'package'
def __init__(self, troveName, version=None):
"""
Initializes a TroveMissing exception.
@param troveName: trove which could not be found
@type troveName: str
@param version: version of the trove which does not exist
@type version: versions.Version, VFS string or [versions.Version]
"""
self.troveName = troveName
self.version = version
if trove.troveIsGroup(troveName):
self.type = 'group'
elif trove.troveIsFileSet(troveName):
self.type = 'fileset'
elif trove.troveIsComponent(troveName):
self.type = 'component'
else:
self.type = 'package'
def iterTroveListInfo(self, troveTup):
if trove.troveIsComponent(troveTup[0]): return []
return (self.cache[troveTup].iterTroveListInfo())
def iterTroveList(troveSource, troveTups, recurseAll=False,
recurseOne=False, recursePackages=False,
needTroves=False, getPristine=True,
showNotByDefault=False, showWeakRefs=False,
checkExists=False, showNotExists=False,
showFlags=False, showBuildLog=False,
filesToShow = [], primaryTroves=[]):
"""
Given a troveTup list, iterate over those troves and their child troves
as specified by parameters
@param troveSource: place to retrieve the trove instances matching troveTups
@type troveSource: display.DisplayConfig
@param recurseAll: if true, recursively descend through the listed troves
@type recurseAll: bool
@param recurseOne: if True, include just the first level of troves below
the listed troves (but do not recurse)
@param needTroves: if True, return trove objects. Otherwise, return None
as each trove object
@type needTroves: bool
@param getPristine: if True, get pristine trove objects
@type getPristine: bool
@param showNotByDefault: if True, yield not bydefault troves
@type showNotByDefault: bool
@param showWeakRefs: if True, yield troves that are weak references
@type showWeakRefs: bool
@param checkExists: if True, add flag MISSING for troves that do not
exist (but are referenced) in this troveSource
@type checkExists: bool
@param showNotExists: if True, show troves that do not exist (but are
referenced) in this troveSource
@type showNotExists: bool
@rtype: yields (troveTup, troveObj, flags, indent) tuples
"""
if not getPristine:
kw = {'pristine' : False}
else:
kw = {}
if recurseOne: # when we recurse one level deep, always recurse packages
# otherwise you might try conary q tmpwatch --troves and
# have that give no result.
recursePackages = True
if needTroves or showFlags:
troves = troveSource.getTroves(troveTups, withFiles=False, **kw)
troveCache = dict(itertools.izip(troveTups, troves))
elif recurseAll or recurseOne or recursePackages:
if recursePackages:
if recurseOne or recurseAll:
colls = [ x for x in troveTups if trove.troveIsCollection(x[0]) ]
else:
colls = [ x for x in troveTups if trove.troveIsPackage(x[0])]
else:
colls = [ x for x in troveTups if trove.troveIsCollection(x[0])
and not trove.troveIsPackage(x[0])]
troves = troveSource.getTroves(colls, withFiles=False, **kw)
troveCache = dict(itertools.izip(colls, troves))
else:
troves = [None] * len(troveTups)
troveCache = {}
hasTrovesCache = {}
if recurseAll or recurseOne or recursePackages:
# we're recursing, we can cache a lot of information -
# troves we'll need, hasTroves info we'll need.
# If we cache this now, we cut down significantly on the
# number of function calls we need.
childTups = list(itertools.chain(*( x.iterTroveList(strongRefs=True)
for x in troves if x)))
if recurseAll:
if recursePackages:
_check = lambda x: trove.troveIsCollection(x[0])
else:
_check = lambda x: (trove.troveIsCollection(x[0])
and not trove.troveIsPackage(x[0]))
colls = set(x for x in troveTups if _check(x))
childColls = [ x for x in childTups if _check(x)]
troves = troveSource.getTroves(childColls, withFiles=False, **kw)
troveCache.update(itertools.izip(childColls, troves))
allTups = troveTups + childTups
if checkExists:
hasTroves = troveSource.hasTroves(allTups)
hasTrovesCache = dict(itertools.izip(allTups, hasTroves))
troves = [ troveCache.get(x, None) for x in troveTups ]
seen = set() # cached info about what troves we've called hasTrove on.
#import epdb; epdb.st()
for troveTup, trv in itertools.izip(troveTups, troves):
if recurseAll:
# recurse all troves, depth first.
topTrove = trv
troves = [(trv, (troveTup, trv,
TROVE_BYDEFAULT | TROVE_STRONGREF | TROVE_HASTROVE, 0))]
while troves:
topTrove, info = troves.pop(0)
yield info
troveTup, trv, flags, depth = info
if not flags & TROVE_HASTROVE:
# we can't recurse this trove, it doesn't exist.
continue
if not trove.troveIsCollection(troveTup[0]):
# this could have been one of the troves we specified
# initially, in which case trying to recurse it will
# not work.
continue
if trove.troveIsPackage(troveTup[0]) and not recursePackages:
continue
newTroveTups = trv.iterTroveList(strongRefs=True,
weakRefs=showWeakRefs)
newTroveTups = sorted(newTroveTups)
if needTroves or trv.isRedirect():
# might as well grab all the troves, we're supposed
# to yield them all.
neededTroveTups = [ x for x in newTroveTups \
if x not in troveCache ]
newTroves = troveSource.getTroves(neededTroveTups,
withFiles=False)
troveCache.update(x for x \
in itertools.izip(neededTroveTups, newTroves) if x[1])
seen.update(neededTroveTups)
else:
newColls = [ x for x in trv.iterTroveList(weakRefs=True,
strongRefs=True)
if trove.troveIsCollection(x[0])
and x not in troveCache ]
newTroves = troveSource.getTroves(newColls,
withFiles=False)
troveCache.update(x for x in itertools.izip(newColls, newTroves))
seen.update(newColls)
if checkExists:
toCheck = set(x for x in trv.iterTroveList(True, True))
alsoToCheck = {}
for newTrove in newTroves:
if newTrove is None:
continue
alsoToCheck.update(dict((x, newTrove) for x in \
newTrove.iterTroveList(True, True)
if x not in toCheck and x not in seen))
if not showNotByDefault:
newToCheck = []
for tup in toCheck:
if topTrove.hasTrove(*tup):
if topTrove.includeTroveByDefault(*tup):
newToCheck.append(tup)
elif trv.includeTroveByDefault(*tup):
newToCheck.append(tup)
for tup, parent in alsoToCheck.iteritems():
if topTrove.hasTrove(*tup):
if topTrove.includeTroveByDefault(*tup):
newToCheck.append(tup)
elif trv.hasTrove(*tup):
if trv.includeTroveByDefault(*tup):
newToCheck.append(tup)
elif parent.includeTroveByDefault(*tup):
newToCheck.append(tup)
toCheck = newToCheck
else:
toCheck = list(toCheck)
if toCheck:
seen.update(toCheck)
toCheck = [ x for x in toCheck if x not in hasTrovesCache ]
hasTroves = troveSource.hasTroves(toCheck)
hasTrovesCache.update(x for x in itertools.izip(toCheck, hasTroves))
trovesToAdd = []
depth += 1
for troveTup in newTroveTups:
if not topTrove.hasTrove(*troveTup):
topTrove = trv
if not recursePackages and trove.troveIsComponent(troveTup[0]):
continue
installByDefault = topTrove.includeTroveByDefault(*troveTup)
if not installByDefault and not showNotByDefault:
continue
flags = TROVE_STRONGREF
if installByDefault:
flags |= TROVE_BYDEFAULT
if not checkExists or hasTrovesCache[troveTup]:
flags |= TROVE_HASTROVE
elif not showNotExists:
continue
newTrove = troveCache.get(troveTup, None)
if trove.troveIsCollection(troveTup[0]):
trovesToAdd.append((topTrove,
(troveTup, newTrove, flags, depth)))
else:
yield (troveTup, newTrove, flags, depth)
troves = trovesToAdd + troves
else:
# recurse one level or recurse no levels.
yield troveTup, trv, TROVE_STRONGREF | TROVE_BYDEFAULT | TROVE_HASTROVE, 0
if (trv and
(recurseOne
or recursePackages and (trove.troveIsPackage(trv.getName())))):
newTroveTups = trv.iterTroveListInfo()
if not showWeakRefs:
newTroveTups = (x for x in newTroveTups if x[2])
if not showNotByDefault:
newTroveTups = (x for x in newTroveTups if x[1])
newTroveTups = sorted(newTroveTups)
if needTroves or trv.isRedirect():
newTroves = troveSource.getTroves(
[x[0] for x in newTroveTups],
withFiles=False)
else:
newTroves = [None] * len(newTroveTups)
if checkExists:
toAdd = [ x[0] for x in newTroveTups if x[0] not in hasTrovesCache ]
hasTroves = troveSource.hasTroves(toAdd)
hasTrovesCache.update(itertools.izip(toAdd, hasTroves))
hasTroves = [ hasTrovesCache[x[0]] for x in newTroveTups ]
else:
hasTroves = [True] * len(newTroveTups)
for (troveTup, byDefault, strongRef), trv, hasTrove \
in itertools.izip(newTroveTups, newTroves, hasTroves):
flags = 0
if strongRef:
flags |= TROVE_STRONGREF
if byDefault:
flags |= TROVE_BYDEFAULT
if hasTrove:
flags |= TROVE_HASTROVE
elif not showNotExists:
continue
yield troveTup, trv, flags, 1
def _walk(self, troveCache, newGroups = True, recurse = False,
installSetOverrides = {}):
"""
Return ((name, version, flavor), inInstallSet, explicit) tuples
for the troves referenced by this TroveSet. inInstallSet is True
if this trove is included in the installSet (byDefault True) for
any of the troves which include it. It is considered explicit
iff it is included directly by this TroveSet.
@param troveCache: TroveCache to use for iterating trove contents
@type troveCache: TroveSource
@param newGroups: Return newly created groups. Version will
be NewVersion().
@type newGroups: bool
@param recurse: Return full recursive closure. When possible, implicit
includes are used to generate this information.
@type recurse: bool
@rtype: ((str, versions.Version, deps.Flavor), isInstall, isExplicit)
"""
if not recurse:
result = []
for (troveTup) in self._getInstallSet():
inInstallSet = installSetOverrides.get(troveTup, True)
if (newGroups
or not isinstance(troveTup[1], versions.NewVersion)):
result.append( (troveTup, inInstallSet, True) )
for (troveTup) in self._getOptionalSet():
inInstallSet = installSetOverrides.get(troveTup, False)
if (newGroups
or not isinstance(troveTup[1], versions.NewVersion)):
result.append( (troveTup, inInstallSet, True) )
return result
if not installSetOverrides and self._walkCache is not None:
return self._walkCache
walkResult = []
usedPackages = set()
for troveTuple in itertools.chain(self.installSet, self.optionalSet):
if trove.troveIsComponent(troveTuple[0]):
usedPackages.add(troveTuple[0].split(":")[0])
collections = list()
newCollections = list()
for troveTuple in itertools.chain(self.installSet, self.optionalSet):
if (isinstance(troveTuple[1], versions.NewVersion)):
newCollections.append(troveTuple)
elif (trove.troveIsGroup(troveTuple[0]) or
troveTuple[0] in usedPackages):
collections.append(troveTuple)
troveCache.cacheTroves(collections)
containedBy = dict ( (x, []) for x in
itertools.chain(self.installSet, self.optionalSet))
containsItems = dict ( (x, False) for x in
itertools.chain(self.installSet, self.optionalSet))
for troveTuple in itertools.chain(self.installSet, self.optionalSet):
for collection in itertools.chain(collections, newCollections):
if troveCache.troveReferencesTrove(collection, troveTuple):
containsItems[collection] = True
containedBy[troveTuple].append(collection)
# for each pair of troves determine the longest path between them; we
# do this through a simple tree walk
maxPathLength = {}
searchList = [ (x, x, 0) for x, y in containsItems.iteritems()
if not y ]
while searchList:
start, next, depth = searchList.pop(0)
knownDepth = maxPathLength.get( (start, next), -1 )
if depth > knownDepth:
maxPathLength[(start, next)] = depth
for container in containedBy[next]:
searchList.append( (start, container, depth + 2) )
searchList = sorted([ (x, x, 0) for x, y in containsItems.iteritems()
if not y ])
def handle(tt, dp, ii):
val = results.get(tt)
if val is None:
results[tt] = (dp, ii)
elif val[0] == dp:
results[tt] = (dp, ii or val[1])
elif val[0] > dp:
results[tt] = (dp, ii)
results = {}
seenDepths = {}
while searchList:
start, troveTup, depth = searchList.pop(0)
if depth < maxPathLength[(start, troveTup)]:
continue
assert(maxPathLength[(start, troveTup)] == depth)
seenAtDepth = seenDepths.get(troveTup)
if seenAtDepth is not None and seenAtDepth <= depth:
# we've walked this at a lower depth; there is no reason
# to do so again
continue
seenDepths[troveTup] = depth
inInstallSet = installSetOverrides.get(troveTup,
troveTup in self.installSet)
handle(troveTup, depth, inInstallSet)
for child in containedBy[troveTup]:
searchList.append( (start, child, depth + 2) )
if not recurse:
continue
if inInstallSet or not trove.troveIsPackage(troveTup[0]):
for subTroveTup, subIsInstall, subIsExplicit in \
troveCache.iterTroveListInfo(troveTup):
overridenSubIsInstall = installSetOverrides.get(
subTroveTup, subIsInstall)
handle(subTroveTup, depth + 1,
inInstallSet and overridenSubIsInstall)
else:
for componentName in troveCache.getPackageComponents(troveTup):
handle((componentName, troveTup[1], troveTup[2]),
depth + 1, False)
for (troveTup), (depth, isInstall) in results.iteritems():
if (newGroups
or not isinstance(troveTup[1], versions.NewVersion)):
walkResult.append(
(troveTup, isInstall,
(troveTup in self.installSet or
troveTup in self.optionalSet) ) )
if not installSetOverrides:
self._walkCache = walkResult
return walkResult
def _walk(self,
troveCache,
newGroups=True,
recurse=False,
installSetOverrides={}):
"""
Return ((name, version, flavor), inInstallSet, explicit) tuples
for the troves referenced by this TroveSet. inInstallSet is True
if this trove is included in the installSet (byDefault True) for
any of the troves which include it. It is considered explicit
iff it is included directly by this TroveSet.
@param troveCache: TroveCache to use for iterating trove contents
@type troveCache: TroveSource
@param newGroups: Return newly created groups. Version will
be NewVersion().
@type newGroups: bool
@param recurse: Return full recursive closure. When possible, implicit
includes are used to generate this information.
@type recurse: bool
@rtype: ((str, versions.Version, deps.Flavor), isInstall, isExplicit)
"""
if not recurse:
result = []
for (troveTup) in self._getInstallSet():
inInstallSet = installSetOverrides.get(troveTup, True)
if (newGroups
or not isinstance(troveTup[1], versions.NewVersion)):
result.append((troveTup, inInstallSet, True))
for (troveTup) in self._getOptionalSet():
inInstallSet = installSetOverrides.get(troveTup, False)
if (newGroups
or not isinstance(troveTup[1], versions.NewVersion)):
result.append((troveTup, inInstallSet, True))
return result
if not installSetOverrides and self._walkCache is not None:
return self._walkCache
walkResult = []
usedPackages = set()
for troveTuple in itertools.chain(self.installSet, self.optionalSet):
if trove.troveIsComponent(troveTuple[0]):
usedPackages.add(troveTuple[0].split(":")[0])
collections = list()
newCollections = list()
for troveTuple in itertools.chain(self.installSet, self.optionalSet):
if (isinstance(troveTuple[1], versions.NewVersion)):
newCollections.append(troveTuple)
elif (trove.troveIsGroup(troveTuple[0])
or troveTuple[0] in usedPackages):
collections.append(troveTuple)
troveCache.cacheTroves(collections)
containedBy = dict(
(x, [])
for x in itertools.chain(self.installSet, self.optionalSet))
containsItems = dict(
(x, False)
for x in itertools.chain(self.installSet, self.optionalSet))
for troveTuple in itertools.chain(self.installSet, self.optionalSet):
for collection in itertools.chain(collections, newCollections):
if troveCache.troveReferencesTrove(collection, troveTuple):
containsItems[collection] = True
containedBy[troveTuple].append(collection)
# for each pair of troves determine the longest path between them; we
# do this through a simple tree walk
maxPathLength = {}
searchList = [(x, x, 0) for x, y in containsItems.iteritems() if not y]
while searchList:
start, next, depth = searchList.pop(0)
knownDepth = maxPathLength.get((start, next), -1)
if depth > knownDepth:
maxPathLength[(start, next)] = depth
for container in containedBy[next]:
searchList.append((start, container, depth + 2))
searchList = sorted([(x, x, 0) for x, y in containsItems.iteritems()
if not y])
def handle(tt, dp, ii):
val = results.get(tt)
if val is None:
results[tt] = (dp, ii)
elif val[0] == dp:
results[tt] = (dp, ii or val[1])
elif val[0] > dp:
results[tt] = (dp, ii)
results = {}
seenDepths = {}
while searchList:
start, troveTup, depth = searchList.pop(0)
if depth < maxPathLength[(start, troveTup)]:
continue
assert (maxPathLength[(start, troveTup)] == depth)
seenAtDepth = seenDepths.get(troveTup)
if seenAtDepth is not None and seenAtDepth <= depth:
# we've walked this at a lower depth; there is no reason
# to do so again
continue
seenDepths[troveTup] = depth
inInstallSet = installSetOverrides.get(troveTup, troveTup
in self.installSet)
handle(troveTup, depth, inInstallSet)
for child in containedBy[troveTup]:
searchList.append((start, child, depth + 2))
if not recurse:
continue
if inInstallSet or not trove.troveIsPackage(troveTup[0]):
for subTroveTup, subIsInstall, subIsExplicit in \
troveCache.iterTroveListInfo(troveTup):
overridenSubIsInstall = installSetOverrides.get(
subTroveTup, subIsInstall)
handle(subTroveTup, depth + 1, inInstallSet
and overridenSubIsInstall)
else:
for componentName in troveCache.getPackageComponents(troveTup):
handle((componentName, troveTup[1], troveTup[2]),
depth + 1, False)
for (troveTup), (depth, isInstall) in results.iteritems():
if (newGroups or not isinstance(troveTup[1], versions.NewVersion)):
walkResult.append(
(troveTup, isInstall, (troveTup in self.installSet
or troveTup in self.optionalSet)))
if not installSetOverrides:
self._walkCache = walkResult
return walkResult
def iterTroveListInfo(self, troveTup):
if trove.troveIsComponent(troveTup[0]): return []
return(self.cache[troveTup].iterTroveListInfo())
def augment(self, model, totalSearchSet, finalTroveSet):
collections = set()
for op in model.modelOps:
if isinstance(op, model.SearchOperation):
continue
for troveTup in op:
name = troveTup[0]
if (isinstance(op, model.OfferTroveOperation) or
trove.troveIsComponent(name)):
collections.add(name.split(':')[0])
elif trove.troveIsGroup(name):
collections.add(name)
# this represents the path from "search" lines
newSearchPath = []
rebuildTotalSearchSet = False
# the "total search" searches the current troveset first, then the
# search path. we only reset this when an operation changed the
# working troveset in a way which would affect later operations,
# after searchTroveSet chagnes
# changed the current troveset in a way which a
# finalTroveSet is the current working set of what's been selected
# so far
for op in model.modelOps:
if isinstance(op, model.SearchOperation):
partialTup = op.item
if isinstance(partialTup, versions.Label):
newSearchTroveSet = troveset.SearchSourceTroveSet(
searchsource.NetworkSearchSource(self.repos,
[ partialTup ],
self.flavor),
graph = self.g)
newSearchSet = newSearchTroveSet
elif partialTup[0] is not None:
newSearchSet = self.reposTroveSet.find(partialTup)
else:
assert(0)
newSearchPath.insert(0, newSearchSet)
rebuildTotalSearchSet = True
continue
searchSpecs = []
localSpecs = []
for troveSpec in op:
if (troveSpec.version is not None and
troveSpec.version[0] == '/'):
try:
verObj = versions.VersionFromString(troveSpec.version)
if verObj.isInLocalNamespace():
localSpecs.append(troveSpec)
continue
except (errors.VersionStringError, errors.ParseError):
pass
searchSpecs.append(troveSpec)
if isinstance(op, model.EraseTroveOperation):
eraseMatches = self._splitFind(self.EraseFindAction,
finalTroveSet, searchSpecs, op)
finalTroveSet = finalTroveSet._action(eraseMatches,
ActionClass=self.RemoveAction,
index = op.getLocation())
continue
if isinstance(op, model.IncludeOperation):
# we need a complete total search set to pass into the sub
# ops, since they have their compilation deferred
rebuildTotalSearchSet = True
if rebuildTotalSearchSet:
totalSearchSet = self.SearchPathTroveSet( newSearchPath +
[ totalSearchSet ],
graph = self.g)
newSearchPath = []
rebuildTotalSearchSet = False
searchMatches = self._splitFind(self.FindAction, totalSearchSet,
searchSpecs, op)
localMatches = self._splitFind(self.FindAction, self.dbTroveSet,
localSpecs, op)
if searchMatches and localMatches:
matches = searchMatches._action(localMatches,
ActionClass = self.UnionAction,
index = op.getLocation())
elif searchMatches:
matches = searchMatches
else:
matches = localMatches
if isinstance(op, model.IncludeOperation):
assert(not localMatches)
finalTroveSet = finalTroveSet._action(
matches, totalSearchSet,
compiler = self,
ActionClass = self.IncludeAction,
SearchPathClass = self.SearchPathTroveSet)
totalSearchSet = finalTroveSet.finalSearchSet
continue
elif isinstance(op, model.InstallTroveOperation):
finalTroveSet = finalTroveSet._action(matches,
ActionClass = self.UnionAction,
index = op.getLocation())
elif isinstance(op, model.PatchTroveOperation):
finalTroveSet = finalTroveSet._action(matches,
ActionClass = self.PatchAction,
index = op.getLocation())
elif isinstance(op, model.UpdateTroveOperation):
finalTroveSet = finalTroveSet._action(matches,
ActionClass = self.UpdateAction,
index = op.getLocation())
elif isinstance(op, model.OfferTroveOperation):
finalTroveSet = finalTroveSet._action(matches,
ActionClass = self.OptionalAction,
index = op.getLocation())
else:
assert(0)
newSearchPath.insert(0, matches)
for troveSpec in op:
if troveSpec.name in collections:
rebuildTotalSearchSet = True
break
if newSearchPath:
totalSearchSet = self.SearchPathTroveSet( newSearchPath +
[ totalSearchSet ],
graph = self.g)
finalTroveSet.searchPath = totalSearchSet
return finalTroveSet
def augment(self, model, totalSearchSet, finalTroveSet):
collections = set()
for op in model.modelOps:
if isinstance(op, model.SearchOperation):
continue
for troveTup in op:
name = troveTup[0]
if (isinstance(op, model.OfferTroveOperation)
or trove.troveIsComponent(name)):
collections.add(name.split(':')[0])
elif trove.troveIsGroup(name):
collections.add(name)
# this represents the path from "search" lines
newSearchPath = []
rebuildTotalSearchSet = False
# the "total search" searches the current troveset first, then the
# search path. we only reset this when an operation changed the
# working troveset in a way which would affect later operations,
# after searchTroveSet chagnes
# changed the current troveset in a way which a
# finalTroveSet is the current working set of what's been selected
# so far
for op in model.modelOps:
if isinstance(op, model.SearchOperation):
partialTup = op.item
if isinstance(partialTup, versions.Label):
newSearchTroveSet = troveset.SearchSourceTroveSet(
searchsource.NetworkSearchSource(
self.repos, [partialTup], self.flavor),
graph=self.g)
newSearchSet = newSearchTroveSet
elif partialTup[0] is not None:
newSearchSet = self.reposTroveSet.find(partialTup)
else:
assert (0)
newSearchPath.insert(0, newSearchSet)
rebuildTotalSearchSet = True
continue
searchSpecs = []
localSpecs = []
for troveSpec in op:
if (troveSpec.version is not None
and troveSpec.version[0] == '/'):
try:
verObj = versions.VersionFromString(troveSpec.version)
if verObj.isInLocalNamespace():
localSpecs.append(troveSpec)
continue
except (errors.VersionStringError, errors.ParseError):
pass
searchSpecs.append(troveSpec)
if isinstance(op, model.EraseTroveOperation):
eraseMatches = self._splitFind(self.EraseFindAction,
finalTroveSet, searchSpecs, op)
finalTroveSet = finalTroveSet._action(
eraseMatches,
ActionClass=self.RemoveAction,
index=op.getLocation())
continue
if isinstance(op, model.IncludeOperation):
# we need a complete total search set to pass into the sub
# ops, since they have their compilation deferred
rebuildTotalSearchSet = True
if rebuildTotalSearchSet:
totalSearchSet = self.SearchPathTroveSet(newSearchPath +
[totalSearchSet],
graph=self.g)
newSearchPath = []
rebuildTotalSearchSet = False
searchMatches = self._splitFind(self.FindAction, totalSearchSet,
searchSpecs, op)
localMatches = self._splitFind(self.FindAction, self.dbTroveSet,
localSpecs, op)
if searchMatches and localMatches:
matches = searchMatches._action(localMatches,
ActionClass=self.UnionAction,
index=op.getLocation())
elif searchMatches:
matches = searchMatches
else:
matches = localMatches
if isinstance(op, model.IncludeOperation):
assert (not localMatches)
finalTroveSet = finalTroveSet._action(
matches,
totalSearchSet,
compiler=self,
ActionClass=self.IncludeAction,
SearchPathClass=self.SearchPathTroveSet)
totalSearchSet = finalTroveSet.finalSearchSet
continue
elif isinstance(op, model.InstallTroveOperation):
finalTroveSet = finalTroveSet._action(
matches,
ActionClass=self.UnionAction,
index=op.getLocation())
elif isinstance(op, model.PatchTroveOperation):
finalTroveSet = finalTroveSet._action(
matches,
ActionClass=self.PatchAction,
index=op.getLocation())
elif isinstance(op, model.UpdateTroveOperation):
finalTroveSet = finalTroveSet._action(
matches,
ActionClass=self.UpdateAction,
index=op.getLocation())
elif isinstance(op, model.OfferTroveOperation):
finalTroveSet = finalTroveSet._action(
matches,
ActionClass=self.OptionalAction,
index=op.getLocation())
else:
assert (0)
newSearchPath.insert(0, matches)
for troveSpec in op:
if troveSpec.name in collections:
rebuildTotalSearchSet = True
break
if newSearchPath:
totalSearchSet = self.SearchPathTroveSet(newSearchPath +
[totalSearchSet],
graph=self.g)
finalTroveSet.searchPath = totalSearchSet
return finalTroveSet
def iterTroveList(troveSource,
troveTups,
recurseAll=False,
recurseOne=False,
recursePackages=False,
needTroves=False,
getPristine=True,
showNotByDefault=False,
showWeakRefs=False,
checkExists=False,
showNotExists=False,
showFlags=False,
showBuildLog=False,
filesToShow=[],
primaryTroves=[]):
"""
Given a troveTup list, iterate over those troves and their child troves
as specified by parameters
@param troveSource: place to retrieve the trove instances matching troveTups
@type troveSource: display.DisplayConfig
@param recurseAll: if true, recursively descend through the listed troves
@type recurseAll: bool
@param recurseOne: if True, include just the first level of troves below
the listed troves (but do not recurse)
@param needTroves: if True, return trove objects. Otherwise, return None
as each trove object
@type needTroves: bool
@param getPristine: if True, get pristine trove objects
@type getPristine: bool
@param showNotByDefault: if True, yield not bydefault troves
@type showNotByDefault: bool
@param showWeakRefs: if True, yield troves that are weak references
@type showWeakRefs: bool
@param checkExists: if True, add flag MISSING for troves that do not
exist (but are referenced) in this troveSource
@type checkExists: bool
@param showNotExists: if True, show troves that do not exist (but are
referenced) in this troveSource
@type showNotExists: bool
@rtype: yields (troveTup, troveObj, flags, indent) tuples
"""
if not getPristine:
kw = {'pristine': False}
else:
kw = {}
if recurseOne: # when we recurse one level deep, always recurse packages
# otherwise you might try conary q tmpwatch --troves and
# have that give no result.
recursePackages = True
if needTroves or showFlags:
troves = troveSource.getTroves(troveTups, withFiles=False, **kw)
troveCache = dict(itertools.izip(troveTups, troves))
elif recurseAll or recurseOne or recursePackages:
if recursePackages:
if recurseOne or recurseAll:
colls = [x for x in troveTups if trove.troveIsCollection(x[0])]
else:
colls = [x for x in troveTups if trove.troveIsPackage(x[0])]
else:
colls = [
x for x in troveTups if trove.troveIsCollection(x[0])
and not trove.troveIsPackage(x[0])
]
troves = troveSource.getTroves(colls, withFiles=False, **kw)
troveCache = dict(itertools.izip(colls, troves))
else:
troves = [None] * len(troveTups)
troveCache = {}
hasTrovesCache = {}
if recurseAll or recurseOne or recursePackages:
# we're recursing, we can cache a lot of information -
# troves we'll need, hasTroves info we'll need.
# If we cache this now, we cut down significantly on the
# number of function calls we need.
childTups = list(
itertools.chain(*(x.iterTroveList(strongRefs=True) for x in troves
if x)))
if recurseAll:
if recursePackages:
_check = lambda x: trove.troveIsCollection(x[0])
else:
_check = lambda x: (trove.troveIsCollection(x[0]) and not trove
.troveIsPackage(x[0]))
colls = set(x for x in troveTups if _check(x))
childColls = [x for x in childTups if _check(x)]
troves = troveSource.getTroves(childColls, withFiles=False, **kw)
troveCache.update(itertools.izip(childColls, troves))
allTups = troveTups + childTups
if checkExists:
hasTroves = troveSource.hasTroves(allTups)
hasTrovesCache = dict(itertools.izip(allTups, hasTroves))
troves = [troveCache.get(x, None) for x in troveTups]
seen = set() # cached info about what troves we've called hasTrove on.
#import epdb; epdb.st()
for troveTup, trv in itertools.izip(troveTups, troves):
if recurseAll:
# recurse all troves, depth first.
topTrove = trv
troves = [
(trv, (troveTup, trv,
TROVE_BYDEFAULT | TROVE_STRONGREF | TROVE_HASTROVE, 0))
]
while troves:
topTrove, info = troves.pop(0)
yield info
troveTup, trv, flags, depth = info
if not flags & TROVE_HASTROVE:
# we can't recurse this trove, it doesn't exist.
continue
if not trove.troveIsCollection(troveTup[0]):
# this could have been one of the troves we specified
# initially, in which case trying to recurse it will
# not work.
continue
if trove.troveIsPackage(troveTup[0]) and not recursePackages:
continue
newTroveTups = trv.iterTroveList(strongRefs=True,
weakRefs=showWeakRefs)
newTroveTups = sorted(newTroveTups)
if needTroves or trv.isRedirect():
# might as well grab all the troves, we're supposed
# to yield them all.
neededTroveTups = [ x for x in newTroveTups \
if x not in troveCache ]
newTroves = troveSource.getTroves(neededTroveTups,
withFiles=False)
troveCache.update(x for x \
in itertools.izip(neededTroveTups, newTroves) if x[1])
seen.update(neededTroveTups)
else:
newColls = [
x for x in trv.iterTroveList(weakRefs=True,
strongRefs=True) if
trove.troveIsCollection(x[0]) and x not in troveCache
]
newTroves = troveSource.getTroves(newColls,
withFiles=False)
troveCache.update(
x for x in itertools.izip(newColls, newTroves))
seen.update(newColls)
if checkExists:
toCheck = set(x for x in trv.iterTroveList(True, True))
alsoToCheck = {}
for newTrove in newTroves:
if newTrove is None:
continue
alsoToCheck.update(dict((x, newTrove) for x in \
newTrove.iterTroveList(True, True)
if x not in toCheck and x not in seen))
if not showNotByDefault:
newToCheck = []
for tup in toCheck:
if topTrove.hasTrove(*tup):
if topTrove.includeTroveByDefault(*tup):
newToCheck.append(tup)
elif trv.includeTroveByDefault(*tup):
newToCheck.append(tup)
for tup, parent in alsoToCheck.iteritems():
if topTrove.hasTrove(*tup):
if topTrove.includeTroveByDefault(*tup):
newToCheck.append(tup)
elif trv.hasTrove(*tup):
if trv.includeTroveByDefault(*tup):
newToCheck.append(tup)
elif parent.includeTroveByDefault(*tup):
newToCheck.append(tup)
toCheck = newToCheck
else:
toCheck = list(toCheck)
if toCheck:
seen.update(toCheck)
toCheck = [
x for x in toCheck if x not in hasTrovesCache
]
hasTroves = troveSource.hasTroves(toCheck)
hasTrovesCache.update(
x for x in itertools.izip(toCheck, hasTroves))
trovesToAdd = []
depth += 1
for troveTup in newTroveTups:
if not topTrove.hasTrove(*troveTup):
topTrove = trv
if not recursePackages and trove.troveIsComponent(
troveTup[0]):
continue
installByDefault = topTrove.includeTroveByDefault(
*troveTup)
if not installByDefault and not showNotByDefault:
continue
flags = TROVE_STRONGREF
if installByDefault:
flags |= TROVE_BYDEFAULT
if not checkExists or hasTrovesCache[troveTup]:
flags |= TROVE_HASTROVE
elif not showNotExists:
continue
newTrove = troveCache.get(troveTup, None)
if trove.troveIsCollection(troveTup[0]):
trovesToAdd.append(
(topTrove, (troveTup, newTrove, flags, depth)))
else:
yield (troveTup, newTrove, flags, depth)
troves = trovesToAdd + troves
else:
# recurse one level or recurse no levels.
yield troveTup, trv, TROVE_STRONGREF | TROVE_BYDEFAULT | TROVE_HASTROVE, 0
if (trv and (recurseOne or recursePackages and
(trove.troveIsPackage(trv.getName())))):
newTroveTups = trv.iterTroveListInfo()
if not showWeakRefs:
newTroveTups = (x for x in newTroveTups if x[2])
if not showNotByDefault:
newTroveTups = (x for x in newTroveTups if x[1])
newTroveTups = sorted(newTroveTups)
if needTroves or trv.isRedirect():
newTroves = troveSource.getTroves(
[x[0] for x in newTroveTups], withFiles=False)
else:
newTroves = [None] * len(newTroveTups)
if checkExists:
toAdd = [
x[0] for x in newTroveTups
if x[0] not in hasTrovesCache
]
hasTroves = troveSource.hasTroves(toAdd)
hasTrovesCache.update(itertools.izip(toAdd, hasTroves))
hasTroves = [hasTrovesCache[x[0]] for x in newTroveTups]
else:
hasTroves = [True] * len(newTroveTups)
for (troveTup, byDefault, strongRef), trv, hasTrove \
in itertools.izip(newTroveTups, newTroves, hasTroves):
flags = 0
if strongRef:
flags |= TROVE_STRONGREF
if byDefault:
flags |= TROVE_BYDEFAULT
if hasTrove:
flags |= TROVE_HASTROVE
elif not showNotExists:
continue
yield troveTup, trv, flags, 1
