def search(self, request, hostname):
name = request.GET.get('name', None)
label = request.GET.get('label', None)
latest = request.GET.get('latest', False)
searchType = request.GET.get('type', None)
checkFnDict = {'group' : trove.troveIsGroup,
'source' : trove.troveIsSourceComponent,
'fileset' : trove.troveIsFileSet,
'package' : lambda x: (trove.troveIsCollection(x)
and not trove.troveIsGroup(x)),
None: None}
if searchType not in checkFnDict:
# XXX We probably want to use exceptions instead of direct maps to
# an error code
#raise errors.InvalidSearchType(searchType)
return response.Response('Invalid search type %s' % searchType,
status = 400)
checkFn = checkFnDict[searchType]
repos = self.getRepos()
if latest:
queryFn = repos.getTroveLatestByLabel
else:
queryFn = repos.getTroveVersionsByLabel
if not label:
return response.Response('Label not specified', status = 400)
try:
label = versions.Label(label)
except versions.ParseError, e:
return response.Response(
'Error parsing label %s: %s' % (label, e), status = 400)
def rdiffCommand(cfg, client, db, diffSpec, **kw):
troveSpec = cmdline.parseChangeList(diffSpec)[0]
if troveSpec[1][0] is None:
# Most likely, syntax did not specify <old>--<new>
return -1
kw.setdefault('recurse', None)
kw.setdefault('asDiff', False)
if kw['recurse'] is None:
kw['recurse'] = (trove.troveIsCollection(troveSpec[0][0])
and not trove.troveIsGroup(troveSpec[0][0]))
primaryCsList = cscmd.computeTroveList(client, [troveSpec])
if (primaryCsList[0][1] == primaryCsList[0][2]):
# Diffing against ourselves
print "Identical troves"
return 1
cs = client.createChangeSet(primaryCsList,
withFiles=True,
withFileContents=kw['asDiff'],
recurse=kw['recurse'])
showchangeset.displayChangeSet(db, cs, None, cfg, **kw)
def generateChangeSet(self, *args, **kwargs):
cs = DiffObject.generateChangeSet(self, *args, **kwargs)
if cs is not None:
# verify doesn't display changes in collections because those, by
# definition, match the database
for trvCs in list(cs.iterNewTroveList()):
if trove.troveIsCollection(trvCs.getName()):
cs.delNewTrove(*trvCs.getNewNameVersionFlavor())
return cs
def generateChangeSet(self, *args, **kwargs):
cs = DiffObject.generateChangeSet(self, *args, **kwargs)
if cs is not None:
# verify doesn't display changes in collections because those, by
# definition, match the database
for trvCs in list(cs.iterNewTroveList()):
if trove.troveIsCollection(trvCs.getName()):
cs.delNewTrove(*trvCs.getNewNameVersionFlavor())
return cs
def buildMap(self, db, troveInfoList):
map = set()
topTroves = db.getTroves([ x for x in troveInfoList
if trove.troveIsCollection(x[0]) ],
pristine = True)
for trv in topTroves:
for trvInfo, byDefault, isStrong in trv.iterTroveListInfo():
if byDefault:
map.add(trvInfo)
self.map = map
def getRecursiveBuildRequirements(self, db, cfg):
if self.buildRequirementsOverride is not None:
return self.buildRequirementsOverride
buildReqs = self.getBuildRequirementTroves(db)
buildReqs = set((x.getName(), x.getVersion(), x.getFlavor()) for x in buildReqs)
packageReqs = [x for x in self.buildReqMap.itervalues() if trove.troveIsCollection(x.getName())]
for package in packageReqs:
childPackages = [x for x in package.iterTroveList(strongRefs=True, weakRefs=True)]
hasTroves = db.hasTroves(childPackages)
buildReqs.update(x[0] for x in itertools.izip(childPackages, hasTroves) if x[1])
buildReqs = self._getRecursiveRequirements(db, buildReqs, cfg.flavor)
return buildReqs
def getRecursiveBuildRequirements(self, db, cfg):
if self.buildRequirementsOverride is not None:
return self.buildRequirementsOverride
buildReqs = self.getBuildRequirementTroves(db)
buildReqs = set(
(x.getName(), x.getVersion(), x.getFlavor()) for x in buildReqs)
packageReqs = [
x for x in self.buildReqMap.itervalues()
if trove.troveIsCollection(x.getName())
]
for package in packageReqs:
childPackages = [
x
for x in package.iterTroveList(strongRefs=True, weakRefs=True)
]
hasTroves = db.hasTroves(childPackages)
buildReqs.update(x[0]
for x in itertools.izip(childPackages, hasTroves)
if x[1])
buildReqs = self._getRecursiveRequirements(db, buildReqs, cfg.flavor)
return buildReqs
def rdiffCommand(cfg, client, db, diffSpec, **kw):
troveSpec = cmdline.parseChangeList(diffSpec)[0]
if troveSpec[1][0] is None:
# Most likely, syntax did not specify <old>--<new>
return -1
kw.setdefault('recurse', None)
kw.setdefault('asDiff', False)
if kw['recurse'] is None:
kw['recurse'] = (trove.troveIsCollection(troveSpec[0][0]) and
not trove.troveIsGroup(troveSpec[0][0]))
primaryCsList = cscmd.computeTroveList(client, [ troveSpec ])
if (primaryCsList[0][1] == primaryCsList[0][2]):
# Diffing against ourselves
print "Identical troves"
return 1
cs = client.createChangeSet(primaryCsList, withFiles=True,
withFileContents=kw['asDiff'],
recurse=kw['recurse'])
showchangeset.displayChangeSet(db, cs, None, cfg, **kw)
def _createChangeSetList(self, csList, recurse = True,
skipNotByDefault = False,
excludeList=cfgtypes.RegularExpressionList(),
callback = None):
primaryList = []
headerList = []
for (name, (oldVersion, oldFlavor),
(newVersion, newFlavor), abstract) in csList:
if newVersion:
primaryList.append((name, newVersion, newFlavor))
if oldVersion:
headerList.append( (name, (None, None),
(oldVersion, oldFlavor), True) )
headerList.append( (name, (None, None),
(newVersion, newFlavor), True) )
else:
primaryList.append((name, oldVersion, oldFlavor))
cs = self.repos.createChangeSet(headerList, recurse = recurse,
withFiles = False, callback = callback)
finalList = set()
jobList = csList[:]
while jobList:
job = jobList.pop(-1)
(name, (oldVersion, oldFlavor),
(newVersion, newFlavor), abstract) = job
skip = False
# troves explicitly listed should never be excluded
if (name, newVersion, newFlavor) not in primaryList:
if excludeList.match(name):
skip = True
if skip:
continue
finalList.add(job)
if not recurse or not trove.troveIsCollection(name):
continue
if job[2][1] is None:
continue
elif job[1][0] is None:
oldTrove = None
else:
oldTrove = trove.Trove(cs.getNewTroveVersion(name, oldVersion,
oldFlavor))
newTrove = trove.Trove(cs.getNewTroveVersion(name, newVersion,
newFlavor))
trvCs, filesNeeded, trovesNeeded = newTrove.diff(
oldTrove, (oldTrove == None))
for subJob in trovesNeeded:
if not subJob[2][0]:
jobList.append(subJob)
continue
if skipNotByDefault and not newTrove.includeTroveByDefault(
subJob[0], subJob[2][0], subJob[2][1]):
continue
jobList.append(subJob)
finalList = list(finalList)
# recreate primaryList without erase-only troves for the primary trove
# list
primaryList = [ (x[0], x[2][0], x[2][1]) for x in csList
if x[2][0] is not None ]
return (finalList, primaryList)
def has(self, troveTup, withFiles=False):
if not withFiles or trove.troveIsCollection(troveTup[0]):
return troveTup in self
return self.get(troveTup, (None, False))[1] is True
def iterTroveList(troveSource, troveTups, recurseAll=False,
recurseOne=False, recursePackages=False,
needTroves=False, getPristine=True,
showNotByDefault=False, showWeakRefs=False,
checkExists=False, showNotExists=False,
showFlags=False, showBuildLog=False,
filesToShow = [], primaryTroves=[]):
"""
Given a troveTup list, iterate over those troves and their child troves
as specified by parameters
@param troveSource: place to retrieve the trove instances matching troveTups
@type troveSource: display.DisplayConfig
@param recurseAll: if true, recursively descend through the listed troves
@type recurseAll: bool
@param recurseOne: if True, include just the first level of troves below
the listed troves (but do not recurse)
@param needTroves: if True, return trove objects. Otherwise, return None
as each trove object
@type needTroves: bool
@param getPristine: if True, get pristine trove objects
@type getPristine: bool
@param showNotByDefault: if True, yield not bydefault troves
@type showNotByDefault: bool
@param showWeakRefs: if True, yield troves that are weak references
@type showWeakRefs: bool
@param checkExists: if True, add flag MISSING for troves that do not
exist (but are referenced) in this troveSource
@type checkExists: bool
@param showNotExists: if True, show troves that do not exist (but are
referenced) in this troveSource
@type showNotExists: bool
@rtype: yields (troveTup, troveObj, flags, indent) tuples
"""
if not getPristine:
kw = {'pristine' : False}
else:
kw = {}
if recurseOne: # when we recurse one level deep, always recurse packages
# otherwise you might try conary q tmpwatch --troves and
# have that give no result.
recursePackages = True
if needTroves or showFlags:
troves = troveSource.getTroves(troveTups, withFiles=False, **kw)
troveCache = dict(itertools.izip(troveTups, troves))
elif recurseAll or recurseOne or recursePackages:
if recursePackages:
if recurseOne or recurseAll:
colls = [ x for x in troveTups if trove.troveIsCollection(x[0]) ]
else:
colls = [ x for x in troveTups if trove.troveIsPackage(x[0])]
else:
colls = [ x for x in troveTups if trove.troveIsCollection(x[0])
and not trove.troveIsPackage(x[0])]
troves = troveSource.getTroves(colls, withFiles=False, **kw)
troveCache = dict(itertools.izip(colls, troves))
else:
troves = [None] * len(troveTups)
troveCache = {}
hasTrovesCache = {}
if recurseAll or recurseOne or recursePackages:
# we're recursing, we can cache a lot of information -
# troves we'll need, hasTroves info we'll need.
# If we cache this now, we cut down significantly on the
# number of function calls we need.
childTups = list(itertools.chain(*( x.iterTroveList(strongRefs=True)
for x in troves if x)))
if recurseAll:
if recursePackages:
_check = lambda x: trove.troveIsCollection(x[0])
else:
_check = lambda x: (trove.troveIsCollection(x[0])
and not trove.troveIsPackage(x[0]))
colls = set(x for x in troveTups if _check(x))
childColls = [ x for x in childTups if _check(x)]
troves = troveSource.getTroves(childColls, withFiles=False, **kw)
troveCache.update(itertools.izip(childColls, troves))
allTups = troveTups + childTups
if checkExists:
hasTroves = troveSource.hasTroves(allTups)
hasTrovesCache = dict(itertools.izip(allTups, hasTroves))
troves = [ troveCache.get(x, None) for x in troveTups ]
seen = set() # cached info about what troves we've called hasTrove on.
#import epdb; epdb.st()
for troveTup, trv in itertools.izip(troveTups, troves):
if recurseAll:
# recurse all troves, depth first.
topTrove = trv
troves = [(trv, (troveTup, trv,
TROVE_BYDEFAULT | TROVE_STRONGREF | TROVE_HASTROVE, 0))]
while troves:
topTrove, info = troves.pop(0)
yield info
troveTup, trv, flags, depth = info
if not flags & TROVE_HASTROVE:
# we can't recurse this trove, it doesn't exist.
continue
if not trove.troveIsCollection(troveTup[0]):
# this could have been one of the troves we specified
# initially, in which case trying to recurse it will
# not work.
continue
if trove.troveIsPackage(troveTup[0]) and not recursePackages:
continue
newTroveTups = trv.iterTroveList(strongRefs=True,
weakRefs=showWeakRefs)
newTroveTups = sorted(newTroveTups)
if needTroves or trv.isRedirect():
# might as well grab all the troves, we're supposed
# to yield them all.
neededTroveTups = [ x for x in newTroveTups \
if x not in troveCache ]
newTroves = troveSource.getTroves(neededTroveTups,
withFiles=False)
troveCache.update(x for x \
in itertools.izip(neededTroveTups, newTroves) if x[1])
seen.update(neededTroveTups)
else:
newColls = [ x for x in trv.iterTroveList(weakRefs=True,
strongRefs=True)
if trove.troveIsCollection(x[0])
and x not in troveCache ]
newTroves = troveSource.getTroves(newColls,
withFiles=False)
troveCache.update(x for x in itertools.izip(newColls, newTroves))
seen.update(newColls)
if checkExists:
toCheck = set(x for x in trv.iterTroveList(True, True))
alsoToCheck = {}
for newTrove in newTroves:
if newTrove is None:
continue
alsoToCheck.update(dict((x, newTrove) for x in \
newTrove.iterTroveList(True, True)
if x not in toCheck and x not in seen))
if not showNotByDefault:
newToCheck = []
for tup in toCheck:
if topTrove.hasTrove(*tup):
if topTrove.includeTroveByDefault(*tup):
newToCheck.append(tup)
elif trv.includeTroveByDefault(*tup):
newToCheck.append(tup)
for tup, parent in alsoToCheck.iteritems():
if topTrove.hasTrove(*tup):
if topTrove.includeTroveByDefault(*tup):
newToCheck.append(tup)
elif trv.hasTrove(*tup):
if trv.includeTroveByDefault(*tup):
newToCheck.append(tup)
elif parent.includeTroveByDefault(*tup):
newToCheck.append(tup)
toCheck = newToCheck
else:
toCheck = list(toCheck)
if toCheck:
seen.update(toCheck)
toCheck = [ x for x in toCheck if x not in hasTrovesCache ]
hasTroves = troveSource.hasTroves(toCheck)
hasTrovesCache.update(x for x in itertools.izip(toCheck, hasTroves))
trovesToAdd = []
depth += 1
for troveTup in newTroveTups:
if not topTrove.hasTrove(*troveTup):
topTrove = trv
if not recursePackages and trove.troveIsComponent(troveTup[0]):
continue
installByDefault = topTrove.includeTroveByDefault(*troveTup)
if not installByDefault and not showNotByDefault:
continue
flags = TROVE_STRONGREF
if installByDefault:
flags |= TROVE_BYDEFAULT
if not checkExists or hasTrovesCache[troveTup]:
flags |= TROVE_HASTROVE
elif not showNotExists:
continue
newTrove = troveCache.get(troveTup, None)
if trove.troveIsCollection(troveTup[0]):
trovesToAdd.append((topTrove,
(troveTup, newTrove, flags, depth)))
else:
yield (troveTup, newTrove, flags, depth)
troves = trovesToAdd + troves
else:
# recurse one level or recurse no levels.
yield troveTup, trv, TROVE_STRONGREF | TROVE_BYDEFAULT | TROVE_HASTROVE, 0
if (trv and
(recurseOne
or recursePackages and (trove.troveIsPackage(trv.getName())))):
newTroveTups = trv.iterTroveListInfo()
if not showWeakRefs:
newTroveTups = (x for x in newTroveTups if x[2])
if not showNotByDefault:
newTroveTups = (x for x in newTroveTups if x[1])
newTroveTups = sorted(newTroveTups)
if needTroves or trv.isRedirect():
newTroves = troveSource.getTroves(
[x[0] for x in newTroveTups],
withFiles=False)
else:
newTroves = [None] * len(newTroveTups)
if checkExists:
toAdd = [ x[0] for x in newTroveTups if x[0] not in hasTrovesCache ]
hasTroves = troveSource.hasTroves(toAdd)
hasTrovesCache.update(itertools.izip(toAdd, hasTroves))
hasTroves = [ hasTrovesCache[x[0]] for x in newTroveTups ]
else:
hasTroves = [True] * len(newTroveTups)
for (troveTup, byDefault, strongRef), trv, hasTrove \
in itertools.izip(newTroveTups, newTroves, hasTroves):
flags = 0
if strongRef:
flags |= TROVE_STRONGREF
if byDefault:
flags |= TROVE_BYDEFAULT
if hasTrove:
flags |= TROVE_HASTROVE
elif not showNotExists:
continue
yield troveTup, trv, flags, 1
def has(self, troveTup, withFiles = False):
if not withFiles or trove.troveIsCollection(troveTup[0]):
return troveTup in self
return self.get(troveTup, (None, False))[1] is True
def _buildRedirect(self, trvCsDict, sourceFlavor, sourceVersion, rule,
target):
if target[0] is not None:
redirInfo = _RedirectInfo(target[0], target[1].branch(),
rule.targetFlavor)
else:
redirInfo = _RemoveRedirect()
self.redirections.add(rule.sourceName, sourceFlavor, redirInfo)
# Groups don't include any additional redirections, and
# neither do items which aren't collections
if (trove.troveIsGroup(rule.sourceName)
or not trove.troveIsCollection(rule.sourceName)):
return
if target[0] is not None:
targetTrove = self.repos.getTrove(withFiles=False, *target)
targetComponents = set([
x[0].split(':')[1]
for x in targetTrove.iterTroveList(strongRefs=True)
])
else:
targetComponents = set()
# we can't integrity check here because we got
# the trove w/o files
trvCs = trvCsDict[(rule.sourceName, sourceVersion, sourceFlavor)]
trv = trove.Trove(trvCs)
# assemble a set of all of the components included
# in this trove
currentComponents = set(
[x[0].split(':')[1] for x in trv.iterTroveList(strongRefs=True)])
# components shared between the current trove and
# the target should be redirected to the target
# components
for compName in currentComponents & targetComponents:
sourceCompName = rule.sourceName + ':' + compName
targetCompName = redirInfo.targetName + ':' + compName
self.redirections.add(
sourceCompName, sourceFlavor,
_RedirectInfo(targetCompName, redirInfo.targetBranch,
redirInfo.targetFlavor))
# now get all of the components which have been
# included in this trove anywhere on the branch; those
# components need to generate erase redirects
allVersions = self.repos.getTroveVersionsByBranch(
{trv.getName(): {
trv.getVersion().branch(): None
}})
l = []
for subVersion, subFlavorList in \
allVersions[trv.getName()].iteritems():
l += [(trv.getName(), subVersion, flavor)
for flavor in subFlavorList]
allTroves = self.repos.getTroves(l, withFiles=False)
allComponents = set()
for otherTrv in allTroves:
allComponents.update([
x[0].split(':')[1]
for x in otherTrv.iterTroveList(strongRefs=True)
])
# components which existed at any point for this
# trove but don't have a component in the redirect
# target need to be erased
for subName in allComponents - targetComponents:
newName = rule.sourceName + ':' + subName
self.redirections.add(newName, sourceFlavor, _RemoveRedirect())
# the package redirect includes references to the
# component redirects to let the update code know
# how to redirect the components; this tracks the
# components of this redirect
redirInfo.addComponents(
[rule.sourceName + ':' + x for x in allComponents])
def iterTroveList(troveSource,
troveTups,
recurseAll=False,
recurseOne=False,
recursePackages=False,
needTroves=False,
getPristine=True,
showNotByDefault=False,
showWeakRefs=False,
checkExists=False,
showNotExists=False,
showFlags=False,
showBuildLog=False,
filesToShow=[],
primaryTroves=[]):
"""
Given a troveTup list, iterate over those troves and their child troves
as specified by parameters
@param troveSource: place to retrieve the trove instances matching troveTups
@type troveSource: display.DisplayConfig
@param recurseAll: if true, recursively descend through the listed troves
@type recurseAll: bool
@param recurseOne: if True, include just the first level of troves below
the listed troves (but do not recurse)
@param needTroves: if True, return trove objects. Otherwise, return None
as each trove object
@type needTroves: bool
@param getPristine: if True, get pristine trove objects
@type getPristine: bool
@param showNotByDefault: if True, yield not bydefault troves
@type showNotByDefault: bool
@param showWeakRefs: if True, yield troves that are weak references
@type showWeakRefs: bool
@param checkExists: if True, add flag MISSING for troves that do not
exist (but are referenced) in this troveSource
@type checkExists: bool
@param showNotExists: if True, show troves that do not exist (but are
referenced) in this troveSource
@type showNotExists: bool
@rtype: yields (troveTup, troveObj, flags, indent) tuples
"""
if not getPristine:
kw = {'pristine': False}
else:
kw = {}
if recurseOne: # when we recurse one level deep, always recurse packages
# otherwise you might try conary q tmpwatch --troves and
# have that give no result.
recursePackages = True
if needTroves or showFlags:
troves = troveSource.getTroves(troveTups, withFiles=False, **kw)
troveCache = dict(itertools.izip(troveTups, troves))
elif recurseAll or recurseOne or recursePackages:
if recursePackages:
if recurseOne or recurseAll:
colls = [x for x in troveTups if trove.troveIsCollection(x[0])]
else:
colls = [x for x in troveTups if trove.troveIsPackage(x[0])]
else:
colls = [
x for x in troveTups if trove.troveIsCollection(x[0])
and not trove.troveIsPackage(x[0])
]
troves = troveSource.getTroves(colls, withFiles=False, **kw)
troveCache = dict(itertools.izip(colls, troves))
else:
troves = [None] * len(troveTups)
troveCache = {}
hasTrovesCache = {}
if recurseAll or recurseOne or recursePackages:
# we're recursing, we can cache a lot of information -
# troves we'll need, hasTroves info we'll need.
# If we cache this now, we cut down significantly on the
# number of function calls we need.
childTups = list(
itertools.chain(*(x.iterTroveList(strongRefs=True) for x in troves
if x)))
if recurseAll:
if recursePackages:
_check = lambda x: trove.troveIsCollection(x[0])
else:
_check = lambda x: (trove.troveIsCollection(x[0]) and not trove
.troveIsPackage(x[0]))
colls = set(x for x in troveTups if _check(x))
childColls = [x for x in childTups if _check(x)]
troves = troveSource.getTroves(childColls, withFiles=False, **kw)
troveCache.update(itertools.izip(childColls, troves))
allTups = troveTups + childTups
if checkExists:
hasTroves = troveSource.hasTroves(allTups)
hasTrovesCache = dict(itertools.izip(allTups, hasTroves))
troves = [troveCache.get(x, None) for x in troveTups]
seen = set() # cached info about what troves we've called hasTrove on.
#import epdb; epdb.st()
for troveTup, trv in itertools.izip(troveTups, troves):
if recurseAll:
# recurse all troves, depth first.
topTrove = trv
troves = [
(trv, (troveTup, trv,
TROVE_BYDEFAULT | TROVE_STRONGREF | TROVE_HASTROVE, 0))
]
while troves:
topTrove, info = troves.pop(0)
yield info
troveTup, trv, flags, depth = info
if not flags & TROVE_HASTROVE:
# we can't recurse this trove, it doesn't exist.
continue
if not trove.troveIsCollection(troveTup[0]):
# this could have been one of the troves we specified
# initially, in which case trying to recurse it will
# not work.
continue
if trove.troveIsPackage(troveTup[0]) and not recursePackages:
continue
newTroveTups = trv.iterTroveList(strongRefs=True,
weakRefs=showWeakRefs)
newTroveTups = sorted(newTroveTups)
if needTroves or trv.isRedirect():
# might as well grab all the troves, we're supposed
# to yield them all.
neededTroveTups = [ x for x in newTroveTups \
if x not in troveCache ]
newTroves = troveSource.getTroves(neededTroveTups,
withFiles=False)
troveCache.update(x for x \
in itertools.izip(neededTroveTups, newTroves) if x[1])
seen.update(neededTroveTups)
else:
newColls = [
x for x in trv.iterTroveList(weakRefs=True,
strongRefs=True) if
trove.troveIsCollection(x[0]) and x not in troveCache
]
newTroves = troveSource.getTroves(newColls,
withFiles=False)
troveCache.update(
x for x in itertools.izip(newColls, newTroves))
seen.update(newColls)
if checkExists:
toCheck = set(x for x in trv.iterTroveList(True, True))
alsoToCheck = {}
for newTrove in newTroves:
if newTrove is None:
continue
alsoToCheck.update(dict((x, newTrove) for x in \
newTrove.iterTroveList(True, True)
if x not in toCheck and x not in seen))
if not showNotByDefault:
newToCheck = []
for tup in toCheck:
if topTrove.hasTrove(*tup):
if topTrove.includeTroveByDefault(*tup):
newToCheck.append(tup)
elif trv.includeTroveByDefault(*tup):
newToCheck.append(tup)
for tup, parent in alsoToCheck.iteritems():
if topTrove.hasTrove(*tup):
if topTrove.includeTroveByDefault(*tup):
newToCheck.append(tup)
elif trv.hasTrove(*tup):
if trv.includeTroveByDefault(*tup):
newToCheck.append(tup)
elif parent.includeTroveByDefault(*tup):
newToCheck.append(tup)
toCheck = newToCheck
else:
toCheck = list(toCheck)
if toCheck:
seen.update(toCheck)
toCheck = [
x for x in toCheck if x not in hasTrovesCache
]
hasTroves = troveSource.hasTroves(toCheck)
hasTrovesCache.update(
x for x in itertools.izip(toCheck, hasTroves))
trovesToAdd = []
depth += 1
for troveTup in newTroveTups:
if not topTrove.hasTrove(*troveTup):
topTrove = trv
if not recursePackages and trove.troveIsComponent(
troveTup[0]):
continue
installByDefault = topTrove.includeTroveByDefault(
*troveTup)
if not installByDefault and not showNotByDefault:
continue
flags = TROVE_STRONGREF
if installByDefault:
flags |= TROVE_BYDEFAULT
if not checkExists or hasTrovesCache[troveTup]:
flags |= TROVE_HASTROVE
elif not showNotExists:
continue
newTrove = troveCache.get(troveTup, None)
if trove.troveIsCollection(troveTup[0]):
trovesToAdd.append(
(topTrove, (troveTup, newTrove, flags, depth)))
else:
yield (troveTup, newTrove, flags, depth)
troves = trovesToAdd + troves
else:
# recurse one level or recurse no levels.
yield troveTup, trv, TROVE_STRONGREF | TROVE_BYDEFAULT | TROVE_HASTROVE, 0
if (trv and (recurseOne or recursePackages and
(trove.troveIsPackage(trv.getName())))):
newTroveTups = trv.iterTroveListInfo()
if not showWeakRefs:
newTroveTups = (x for x in newTroveTups if x[2])
if not showNotByDefault:
newTroveTups = (x for x in newTroveTups if x[1])
newTroveTups = sorted(newTroveTups)
if needTroves or trv.isRedirect():
newTroves = troveSource.getTroves(
[x[0] for x in newTroveTups], withFiles=False)
else:
newTroves = [None] * len(newTroveTups)
if checkExists:
toAdd = [
x[0] for x in newTroveTups
if x[0] not in hasTrovesCache
]
hasTroves = troveSource.hasTroves(toAdd)
hasTrovesCache.update(itertools.izip(toAdd, hasTroves))
hasTroves = [hasTrovesCache[x[0]] for x in newTroveTups]
else:
hasTroves = [True] * len(newTroveTups)
for (troveTup, byDefault, strongRef), trv, hasTrove \
in itertools.izip(newTroveTups, newTroves, hasTroves):
flags = 0
if strongRef:
flags |= TROVE_STRONGREF
if byDefault:
flags |= TROVE_BYDEFAULT
if hasTrove:
flags |= TROVE_HASTROVE
elif not showNotExists:
continue
yield troveTup, trv, flags, 1
def _buildRedirect(self, trvCsDict, sourceFlavor, sourceVersion, rule, target):
if target[0] is not None:
redirInfo = _RedirectInfo(target[0], target[1].branch(), rule.targetFlavor)
else:
redirInfo = _RemoveRedirect()
self.redirections.add(rule.sourceName, sourceFlavor, redirInfo)
# Groups don't include any additional redirections, and
# neither do items which aren't collections
if trove.troveIsGroup(rule.sourceName) or not trove.troveIsCollection(rule.sourceName):
return
if target[0] is not None:
targetTrove = self.repos.getTrove(withFiles=False, *target)
targetComponents = set([x[0].split(":")[1] for x in targetTrove.iterTroveList(strongRefs=True)])
else:
targetComponents = set()
# we can't integrity check here because we got
# the trove w/o files
trvCs = trvCsDict[(rule.sourceName, sourceVersion, sourceFlavor)]
trv = trove.Trove(trvCs)
# assemble a set of all of the components included
# in this trove
currentComponents = set([x[0].split(":")[1] for x in trv.iterTroveList(strongRefs=True)])
# components shared between the current trove and
# the target should be redirected to the target
# components
for compName in currentComponents & targetComponents:
sourceCompName = rule.sourceName + ":" + compName
targetCompName = redirInfo.targetName + ":" + compName
self.redirections.add(
sourceCompName,
sourceFlavor,
_RedirectInfo(targetCompName, redirInfo.targetBranch, redirInfo.targetFlavor),
)
# now get all of the components which have been
# included in this trove anywhere on the branch; those
# components need to generate erase redirects
allVersions = self.repos.getTroveVersionsByBranch({trv.getName(): {trv.getVersion().branch(): None}})
l = []
for subVersion, subFlavorList in allVersions[trv.getName()].iteritems():
l += [(trv.getName(), subVersion, flavor) for flavor in subFlavorList]
allTroves = self.repos.getTroves(l, withFiles=False)
allComponents = set()
for otherTrv in allTroves:
allComponents.update([x[0].split(":")[1] for x in otherTrv.iterTroveList(strongRefs=True)])
# components which existed at any point for this
# trove but don't have a component in the redirect
# target need to be erased
for subName in allComponents - targetComponents:
newName = rule.sourceName + ":" + subName
self.redirections.add(newName, sourceFlavor, _RemoveRedirect())
# the package redirect includes references to the
# component redirects to let the update code know
# how to redirect the components; this tracks the
# components of this redirect
redirInfo.addComponents([rule.sourceName + ":" + x for x in allComponents])
def expandJobList(db, chgSetList, recurse):
"""
For each job in the list, find the set of troves which are recursively
included in it. The reutnr value is list parallel to chgSetList, each
item of which is a sorted list of those troves which are included in the
recursive changeset.
"""
# We mark old groups (ones without weak references) as uncachable
# because they're expensive to flatten (and so old that it
# hardly matters).
if not recurse:
return [ [ job ] for job in chgSetList ]
cu = db.cursor()
schema.resetTable(cu, "tmpNVF")
foundGroups = set()
foundWeak = set()
foundCollections = set()
insertList = []
for jobId, job in enumerate(chgSetList):
if trove.troveIsGroup(job[0]):
foundGroups.add(jobId)
insertList.append((jobId, job[0], job[2][0], job[2][1]))
db.bulkload("tmpNvf", insertList,
[ "idx", "name", "version", "flavor" ],
start_transaction = False)
db.analyze("tmpNVF")
newJobList = [ [ job ] for job in chgSetList ]
cu.execute("""SELECT
tmpNVF.idx, I_Items.item, I_Versions.version,
I_Flavors.flavor, TroveTroves.flags
FROM tmpNVF JOIN Items ON tmpNVF.name = Items.item
JOIN Versions ON (tmpNVF.version = Versions.version)
JOIN Flavors ON (tmpNVF.flavor = Flavors.flavor)
JOIN Instances ON
Items.itemId = Instances.itemId AND
Versions.versionId = Instances.versionId AND
Flavors.flavorId = Instances.flavorId
JOIN TroveTroves USING (instanceId)
JOIN Instances AS I_Instances ON
TroveTroves.includedId = I_Instances.instanceId
JOIN Items AS I_Items ON
I_Instances.itemId = I_Items.itemId
JOIN Versions AS I_Versions ON
I_Instances.versionId = I_Versions.versionId
JOIN Flavors AS I_Flavors ON
I_Instances.flavorId = I_Flavors.flavorId
WHERE
I_Instances.isPresent = 1
ORDER BY
I_Items.item, I_Versions.version, I_Flavors.flavor
""")
for (idx, name, version, flavor, flags) in cu:
newJobList[idx].append( (name, (None, None),
(version, flavor), True) )
if flags & schema.TROVE_TROVES_WEAKREF > 0:
foundWeak.add(idx)
if trove.troveIsCollection(name):
foundCollections.add(idx)
for idx in ((foundGroups & foundCollections) - foundWeak):
# groups which contain collections but no weak refs
# are uncachable
newJobList[idx] = None
return newJobList