def testStrToFrozen(self): assert (strToFrozen("/a/1.1", ["123"]) == '/a/123:1.1') assert (strToFrozen("/a/1.1//b", ["123"]) == '/a/123:1.1//b') assert (strToFrozen("/a/1.1//b//c/1.2/d", ["123", "456"]) == '/a/123:1.1//b//c/456:1.2/d') self.assertRaises(AssertionError, strToFrozen, "/a/1.1", ["123", "456"])
def versionStringToFrozen(self, verStr, timeStamps): if isinstance(timeStamps, basestring): timeStamps = [float(x) for x in timeStamps.split(':')] timeStamps = ['%.3f' % x for x in timeStamps] return versions.strToFrozen(verStr, timeStamps)
def searchNodes(cu, roleIds, label = None, mkUrl = None, filterSet = None, db = None, name = None, latest = 1): args = [] d = { 'labelCheck' : '', 'itemCheck' : '' } d['roleIds'] = ",".join( str(x) for x in roleIds) d['SOURCENAME'] = trove._TROVEINFO_TAG_SOURCENAME d['METADATA'] = trove._TROVEINFO_TAG_METADATA if label: d['labelCheck'] = "label = ? AND" args.append(label) if name: d['itemCheck'] = "item = ? AND" args.append(name) if latest: cu.execute(""" SELECT idTable.item, version, ts, finalTs, SourceNameTroveInfo.data, MetadataTroveInfo.data FROM (SELECT DISTINCT Items.item AS item, Nodes.versionId AS versionId, Nodes.timeStamps AS ts, Nodes.finalTimeStamp as finalTs, MIN(Instances.instanceId) AS instanceId FROM Labels JOIN LabelMap USING (labelId) JOIN LatestCache USING (itemId, branchId) JOIN Nodes USING (itemId, versionId) JOIN Instances USING (itemId, versionId) JOIN Items USING (itemId) WHERE %(labelCheck)s %(itemCheck)s LatestCache.latestType = 1 AND LatestCache.userGroupId in (%(roleIds)s) GROUP BY Items.item, Nodes.versionId, Nodes.timeStamps, Nodes.finalTimestamp) AS idTable JOIN Versions ON (idTable.versionId = Versions.versionId) LEFT OUTER JOIN TroveInfo AS SourceNameTroveInfo ON idTable.instanceId = SourceNameTroveInfo.instanceId AND SourceNameTroveInfo.infoType = %(SOURCENAME)d LEFT OUTER JOIN TroveInfo AS MetadataTroveInfo ON idTable.instanceId = MetadataTroveInfo.instanceId AND MetadataTroveInfo.infoType = %(METADATA)d """ % d, args) else: cu.execute(""" SELECT idTable.item, version, ts, finalTs, SourceNameTroveInfo.data, MetadataTroveInfo.data FROM (SELECT DISTINCT Items.item AS item, Nodes.versionId AS versionId, Nodes.timeStamps AS ts, Nodes.finalTimeStamp as finalTs, MIN(Instances.instanceId) AS instanceId FROM Labels JOIN LabelMap USING (labelId) JOIN Nodes USING (itemId, branchId) JOIN Instances USING (itemId, versionId) JOIN Items USING (itemId) JOIN usergroupinstancescache AS ugi ON Instances.instanceId = ugi.instanceId WHERE %(labelCheck)s %(itemCheck)s ugi.userGroupId in (%(roleIds)s) GROUP BY Items.item, Nodes.versionId, Nodes.timeStamps, Nodes.finalTimestamp) AS idTable JOIN Versions ON (idTable.versionId = Versions.versionId) LEFT OUTER JOIN TroveInfo AS SourceNameTroveInfo ON idTable.instanceId = SourceNameTroveInfo.instanceId AND SourceNameTroveInfo.infoType = %(SOURCENAME)d LEFT OUTER JOIN TroveInfo AS MetadataTroveInfo ON idTable.instanceId = MetadataTroveInfo.instanceId AND MetadataTroveInfo.infoType = %(METADATA)d """ % d, args) l = list(cu) filteredL = typeFilter(l, filterSet) # sort based on (name, version, desc(finalTimestamp)) def sortorder(x, y): c = cmp(x[0], y[0]) if c: return c return -(cmp(x[3], y[3])) filteredL.sort(sortorder) if latest: # keep the latest newL = [] last = None for item in filteredL: if last and last[0] == item[0]: continue newL.append(item) last = item filteredL = newL nodeList = datamodel.NamedNodeList(total = len(filteredL), start = 0) addList = [] for (name, version, ts, finalTs, sourceName, metadata) in filteredL: sourceName = cu.frombinary(sourceName) if sourceName is None and trove.troveIsSourceComponent(name): sourceName = name addList.append((sourceName, str(versions.VersionFromString(version).getSourceVersion()))) schema.resetTable(cu, 'tmpNVF') # This is painful, but it converts the source name from a blob to # a string db.bulkload("tmpNVF", [ (x[0],) + x[1] for x in enumerate(addList) ], ["idx", "name", "version"], start_transaction = False) cu.execute(""" SELECT ChangeLogs.name, ChangeLogs.message, tmpNVF.name FROM tmpNVF JOIN Items AS SourceItems ON tmpNVF.name = SourceItems.item LEFT OUTER JOIN Versions AS SourceVersion ON tmpNVF.version = SourceVersion.version LEFT OUTER JOIN Nodes ON SourceItems.itemId = Nodes.itemId AND SourceVersion.versionId = Nodes.versionId LEFT OUTER JOIN ChangeLogs USING (nodeId) ORDER BY tmpNVF.idx """) for ( (name, version, ts, finalTs, sourceName, metadata), (clName, clMessage, troveName) ) in itertools.izip(filteredL, cu): frzVer = versions.strToFrozen(version, [ x for x in ts.split(":") ]) ver = versions.ThawVersion(frzVer) shortdesc = None if metadata: metadata = cu.frombinary(metadata) md = trove.Metadata(metadata) shortdesc = md.get()['shortDesc'] if clName: cl = datamodel.ChangeLog(name = clName, message = clMessage) else: cl = None nodeList.append(name = name, version = ver, mkUrl = mkUrl, changeLog = cl, shortdesc = shortdesc) return nodeList
def getTrove(cu, roleIds, name, version, flavor, mkUrl = None, thisHost = None, displayFlavor = None, excludeCapsules = False): def buildTupleList(tuples, name, mkUrl = mkUrl): l = getattr(datamodel.SingleTrove, name)() for troveInfo in sorted(tuples.iter()): l.append(name = troveInfo.name(), version = troveInfo.version(), flavor = troveInfo.flavor(), mkUrl = mkUrl) return l def fileQuery(gfcu, filesInstanceId, dirName = None): # XXX restricing by dirName seems and obvious thing to do here, # but it actually slows things down?? # # the distinct here is unfortunate, but conary repositories had # a bug for about a year which caused it to store duplicate paths # if a path was committed for the first time duplicate times in # a single commit job gfcu.execute(""" SELECT DISTINCT dirName, basename, version, pathId, fileId FROM TroveFiles JOIN Versions USING (versionId) JOIN FileStreams ON (TroveFiles.streamId = FileStreams.streamId) JOIN FilePaths ON (TroveFiles.filePathId = FilePaths.filePathId) JOIN DirNames ON FilePaths.dirNameId = DirNames.dirNameId JOIN Basenames ON (FilePaths.baseNameId = Basenames.baseNameId) WHERE TroveFiles.instanceId = ? ORDER BY dirName, basename """, filesInstanceId) cu.execute(""" SELECT Instances.instanceId, Nodes.timeStamps FROM Instances JOIN Nodes USING (itemId, versionId) JOIN Items USING (itemId) JOIN Versions ON (Instances.versionId = Versions.versionId) JOIN Flavors ON (Instances.flavorId = Flavors.flavorId) JOIN UserGroupInstancesCache AS ugi ON (instances.instanceId = ugi.instanceId AND ugi.userGroupId in (%s)) WHERE item = ? AND version = ? AND flavor = ? """ % ",".join( str(x) for x in roleIds), name, version, deps.parseFlavor(flavor).freeze()) l = [ (x[0], x[1]) for x in cu ] if not l: return None instanceId, timeStamps = l[0] frzVer = versions.strToFrozen(version, timeStamps.split(":")) verobj = versions.ThawVersion(frzVer) tupleLists = [ ( trove._TROVEINFO_TAG_BUILDDEPS, 'builddeps' ), ( trove._TROVEINFO_TAG_POLICY_PROV, 'policyprovider' ), ( trove._TROVEINFO_TAG_LOADEDTROVES, 'loadedtroves' ), ( trove._TROVEINFO_TAG_COPIED_FROM, 'copiedfrom' ), ( trove._TROVEINFO_TAG_DERIVEDFROM, 'derivedfrom' ) ] cu.execute(""" SELECT infoType, data FROM TroveInfo WHERE instanceId = ? AND infoType IN (%s) """ % ",".join(str(x) for x in [ trove._TROVEINFO_TAG_SOURCENAME, trove._TROVEINFO_TAG_CLONEDFROM, trove._TROVEINFO_TAG_CLONEDFROMLIST, trove._TROVEINFO_TAG_BUILDTIME, trove._TROVEINFO_TAG_SIZE, trove._TROVEINFO_TAG_METADATA, trove._TROVEINFO_TAG_CAPSULE, ] + [ x[0] for x in tupleLists ] ), instanceId) troveInfo = {} for infoType, data in cu: data = cu.frombinary(data) infoClass = trove.TroveInfo.streamDict[infoType][1] troveInfo[infoType] = infoClass(data) kwargs = { 'name' : name, 'version' : verobj, 'flavor' : flavor } if displayFlavor is not None: kwargs['displayflavor'] = displayFlavor if trove._TROVEINFO_TAG_BUILDTIME in troveInfo: kwargs['buildtime'] = int(troveInfo[trove._TROVEINFO_TAG_BUILDTIME]()) if trove._TROVEINFO_TAG_SOURCENAME in troveInfo: kwargs['source'] = (troveInfo[trove._TROVEINFO_TAG_SOURCENAME](), verobj.getSourceVersion(), '') if trove._TROVEINFO_TAG_SIZE in troveInfo: kwargs['size'] = troveInfo[trove._TROVEINFO_TAG_SIZE]() if trove._TROVEINFO_TAG_METADATA in troveInfo: md = troveInfo[trove._TROVEINFO_TAG_METADATA].get() kwargs['shortdesc'] = md['shortDesc'] kwargs['longdesc'] = md['longDesc'] if md['licenses']: kwargs['license'] = [ x for x in md['licenses' ]] if md['crypto']: kwargs['crypto'] = [ x for x in md['crypto'] ] for (tag, tagName) in tupleLists: if tag in troveInfo: kwargs[tagName] = buildTupleList(troveInfo[tag], tagName, mkUrl = mkUrl) t = datamodel.SingleTrove(mkUrl = mkUrl, thisHost = thisHost, **kwargs) if trove._TROVEINFO_TAG_CLONEDFROMLIST in troveInfo: clonedFromList = troveInfo[trove._TROVEINFO_TAG_CLONEDFROMLIST] elif (trove._TROVEINFO_TAG_CLONEDFROM in troveInfo): clonedFromList = [ troveInfo[trove._TROVEINFO_TAG_CLONEDFROM]() ] else: clonedFromList = [] for ver in clonedFromList: t.addClonedFrom(name, ver, flavor, mkUrl = mkUrl) hasCapsule = False if trove._TROVEINFO_TAG_CAPSULE in troveInfo: if troveInfo[trove._TROVEINFO_TAG_CAPSULE].type(): hasCapsule = True fileQuery(cu, instanceId) for (dirName, baseName, fileVersion, pathId, fileId) in cu: dirName = cu.frombinary(dirName) baseName = cu.frombinary(baseName) if pathId == trove.CAPSULE_PATHID: isCapsule = 1 contentAvailable = not excludeCapsules else: isCapsule = None contentAvailable = not hasCapsule fileObj = datamodel.FileReference( path = os.path.join(dirName, baseName), version = fileVersion, pathId = md5ToString(cu.frombinary(pathId)), fileId = sha1ToString(cu.frombinary(fileId)), isCapsule = isCapsule, contentAvailable = contentAvailable, mkUrl = mkUrl, thisHost = thisHost) t.addFile(fileObj) cu.execute(""" SELECT item, version, flavor, TroveTroves.includedId, Nodes.timeStamps FROM TroveTroves JOIN Instances ON (Instances.instanceId = TroveTroves.includedId) JOIN Nodes USING (itemId, versionId) JOIN Items USING (itemId) JOIN Versions ON (Versions.versionId = Instances.versionId) JOIN Flavors ON (Flavors.flavorId = Instances.flavorId) WHERE TroveTroves.instanceId = ? AND (TroveTroves.flags & %d) = 0 ORDER BY item, version, flavor """ % schema.TROVE_TROVES_WEAKREF, instanceId) for (subName, subVersion, subFlavor, refInstanceId, subTS) in cu: subFlavor = str(deps.ThawFlavor(subFlavor)) frzVer = versions.strToFrozen(subVersion, [ x for x in subTS.split(":") ]) subV = versions.ThawVersion(frzVer) t.addReferencedTrove(subName, subV, subFlavor, mkUrl = mkUrl) # It would be far better to use file tags to identify these build # logs, but it's significantly slower as well because they're in # the file objects rather than the trove (and those file objects # could be stored on a different repository) if not subName.endswith(':debuginfo'): continue fileQuery(cu, refInstanceId, dirName = '/usr/src/debug/buildlogs') logHost = subV.getHost() for (dirName, baseName, fileVersion, pathId, fileId) in cu: if (dirName) != '/usr/src/debug/buildlogs': continue if baseName.endswith('-log.bz2'): t.setBuildLog(logHost, sha1ToString(fileId)) elif baseName.endswith('-xml.bz2'): t.setXMLBuildLog(logHost, sha1ToString(fileId)) return t
def searchTroves(cu, roleIds, label = None, filterSet = None, mkUrl = None, latest = True, start = 0, limit = None, name = None): d = { 'labelCheck' : '', 'nameCheck' : '' } args = [] regex = None if label: d['labelCheck'] = "label = ? AND" args.append(label) if name: if set('?.*[]\\()+') & set(name): # if only .* appears, replace them with '%' and use LIKE. this # code currently fails with \.* in the regex, but neither . # nor \* are valid trove names anyway likeName = name while '.*' in likeName: likeName = likeName.replace('.*', '%') if set('?.*[]\\()+') & set(name): regex = re.compile(name) else: d['nameCheck'] = "WHERE item LIKE ?" args.append(likeName) else: d['nameCheck' ] = "WHERE item = ?" args.append(name) d['roleIds'] = ",".join( str(x) for x in roleIds) if latest: cu.execute(""" SELECT item, version, flavor, ts FROM (SELECT DISTINCT Nodes.itemId AS itemId, Nodes.versionId AS versionId, flavorId, Nodes.timeStamps AS ts FROM Labels JOIN LabelMap USING (labelId) JOIN LatestCache USING (itemId, branchId) JOIN Nodes USING (itemId, versionId) WHERE %(labelCheck)s LatestCache.latestType = 1 AND LatestCache.userGroupId in (%(roleIds)s)) AS idTable JOIN Items USING (itemId) JOIN Versions ON (idTable.versionId = Versions.versionId) JOIN Flavors ON (idTable.flavorId = Flavors.flavorId) %(nameCheck)s ORDER BY item, version, flavor """ % d, *args) else: cu.execute(""" SELECT item, version, flavor, ts FROM (SELECT DISTINCT Instances.itemId AS itemId, Instances.versionId AS versionId, Instances.flavorId AS flavorId, Nodes.timeStamps AS ts FROM Labels JOIN LabelMap USING (labelId) JOIN Nodes USING (itemId, branchid) JOIN Instances USING (itemid, versionid) JOIN usergroupinstancescache AS ugi USING (instanceid) WHERE %(labelCheck)s ugi.userGroupId in (%(roleIds)s)) AS idTable JOIN Items USING (itemId) JOIN Versions ON (idTable.versionId = Versions.versionId) JOIN Flavors ON (idTable.flavorId = Flavors.flavorId) %(nameCheck)s ORDER BY item, version, flavor """ % d, *args) l = list(cu) filteredL = typeFilter(l, filterSet) if regex: filteredL = [ x for x in filteredL if regex.match(x[0]) ] if limit is None: limit = len(filteredL) - start troveList = datamodel.NamedTroveIdentList(total = len(filteredL), start = start) for (name, version, flavor, ts) in filteredL[start:start + limit]: flavor = str(deps.ThawFlavor(flavor)) frzVer = versions.strToFrozen(version, [ x for x in ts.split(":") ]) ver = versions.ThawVersion(frzVer) troveList.append(name = name, version = ver, flavor = flavor, mkUrl = mkUrl) return troveList
def getTrove(cu, roleIds, name, version, flavor, mkUrl = None, thisHost = None, displayFlavor = None): def buildTupleList(tuples, name, mkUrl = mkUrl): l = getattr(datamodel.SingleTrove, name)() for troveInfo in sorted(tuples.iter()): l.append(name = troveInfo.name(), version = troveInfo.version(), flavor = troveInfo.flavor(), mkUrl = mkUrl) return l def fileQuery(gfcu, filesInstanceId, dirName = None): # XXX restricing by dirName seems and obvious thing to do here, # but it actually slows things down?? # # the distinct here is unfortunate, but conary repositories had # a bug for about a year which caused it to store duplicate paths # if a path was committed for the first time duplicate times in # a single commit job gfcu.execute(""" SELECT DISTINCT dirName, basename, version, pathId, fileId FROM TroveFiles JOIN Versions USING (versionId) JOIN FileStreams ON (TroveFiles.streamId = FileStreams.streamId) JOIN FilePaths ON (TroveFiles.filePathId = FilePaths.filePathId) JOIN DirNames ON FilePaths.dirNameId = DirNames.dirNameId JOIN Basenames ON (FilePaths.baseNameId = Basenames.baseNameId) WHERE TroveFiles.instanceId = ? ORDER BY dirName, basename """, filesInstanceId) cu.execute(""" SELECT Instances.instanceId, Nodes.timeStamps FROM Instances JOIN Nodes USING (itemId, versionId) JOIN Items USING (itemId) JOIN Versions ON (Instances.versionId = Versions.versionId) JOIN Flavors ON (Instances.flavorId = Flavors.flavorId) JOIN UserGroupInstancesCache AS ugi ON (instances.instanceId = ugi.instanceId AND ugi.userGroupId in (%s)) WHERE item = ? AND version = ? AND flavor = ? """ % ",".join( str(x) for x in roleIds), name, version, deps.parseFlavor(flavor).freeze()) l = [ (x[0], x[1]) for x in cu ] if not l: return None instanceId, timeStamps = l[0] frzVer = versions.strToFrozen(version, timeStamps.split(":")) verobj = versions.ThawVersion(frzVer) tupleLists = [ ( trove._TROVEINFO_TAG_BUILDDEPS, 'builddeps' ), ( trove._TROVEINFO_TAG_POLICY_PROV, 'policyprovider' ), ( trove._TROVEINFO_TAG_LOADEDTROVES, 'loadedtroves' ), ( trove._TROVEINFO_TAG_COPIED_FROM, 'copiedfrom' ), ( trove._TROVEINFO_TAG_DERIVEDFROM, 'derivedfrom' ) ] cu.execute(""" SELECT infoType, data FROM TroveInfo WHERE instanceId = ? AND infoType IN (%s) """ % ",".join(str(x) for x in [ trove._TROVEINFO_TAG_SOURCENAME, trove._TROVEINFO_TAG_CLONEDFROM, trove._TROVEINFO_TAG_CLONEDFROMLIST, trove._TROVEINFO_TAG_BUILDTIME, trove._TROVEINFO_TAG_SIZE, trove._TROVEINFO_TAG_METADATA, trove._TROVEINFO_TAG_CAPSULE, ] + [ x[0] for x in tupleLists ] ), instanceId) troveInfo = {} for infoType, data in cu: data = cu.frombinary(data) infoClass = trove.TroveInfo.streamDict[infoType][1] troveInfo[infoType] = infoClass(data) kwargs = { 'name' : name, 'version' : verobj, 'flavor' : flavor } if displayFlavor is not None: kwargs['displayflavor'] = displayFlavor if trove._TROVEINFO_TAG_BUILDTIME in troveInfo: kwargs['buildtime'] = int(troveInfo[trove._TROVEINFO_TAG_BUILDTIME]()) if trove._TROVEINFO_TAG_SOURCENAME in troveInfo: kwargs['source'] = (troveInfo[trove._TROVEINFO_TAG_SOURCENAME](), verobj.getSourceVersion(), '') if trove._TROVEINFO_TAG_SIZE in troveInfo: kwargs['size'] = troveInfo[trove._TROVEINFO_TAG_SIZE]() if trove._TROVEINFO_TAG_METADATA in troveInfo: md = troveInfo[trove._TROVEINFO_TAG_METADATA].get() kwargs['shortdesc'] = md['shortDesc'] kwargs['longdesc'] = md['longDesc'] if md['licenses']: kwargs['license'] = [ x for x in md['licenses' ]] if md['crypto']: kwargs['crypto'] = [ x for x in md['crypto'] ] for (tag, tagName) in tupleLists: if tag in troveInfo: kwargs[tagName] = buildTupleList(troveInfo[tag], tagName, mkUrl = mkUrl) t = datamodel.SingleTrove(mkUrl = mkUrl, thisHost = thisHost, **kwargs) if trove._TROVEINFO_TAG_CLONEDFROMLIST in troveInfo: clonedFromList = troveInfo[trove._TROVEINFO_TAG_CLONEDFROMLIST] elif (trove._TROVEINFO_TAG_CLONEDFROM in troveInfo): clonedFromList = [ troveInfo[trove._TROVEINFO_TAG_CLONEDFROM]() ] else: clonedFromList = [] for ver in clonedFromList: t.addClonedFrom(name, ver, flavor, mkUrl = mkUrl) hasCapsule = False if trove._TROVEINFO_TAG_CAPSULE in troveInfo: if troveInfo[trove._TROVEINFO_TAG_CAPSULE].type(): hasCapsule = True fileQuery(cu, instanceId) for (dirName, baseName, fileVersion, pathId, fileId) in cu: dirName = cu.frombinary(dirName) baseName = cu.frombinary(baseName) if pathId == trove.CAPSULE_PATHID: isCapsule = 1 contentAvailable = True else: isCapsule = None contentAvailable = not hasCapsule fileObj = datamodel.FileReference( path = os.path.join(dirName, baseName), version = fileVersion, pathId = md5ToString(cu.frombinary(pathId)), fileId = sha1ToString(cu.frombinary(fileId)), isCapsule = isCapsule, contentAvailable = contentAvailable, mkUrl = mkUrl, thisHost = thisHost) t.addFile(fileObj) cu.execute(""" SELECT item, version, flavor, TroveTroves.includedId, Nodes.timeStamps FROM TroveTroves JOIN Instances ON (Instances.instanceId = TroveTroves.includedId) JOIN Nodes USING (itemId, versionId) JOIN Items USING (itemId) JOIN Versions ON (Versions.versionId = Instances.versionId) JOIN Flavors ON (Flavors.flavorId = Instances.flavorId) WHERE TroveTroves.instanceId = ? AND (TroveTroves.flags & %d) = 0 ORDER BY item, version, flavor """ % schema.TROVE_TROVES_WEAKREF, instanceId) for (subName, subVersion, subFlavor, refInstanceId, subTS) in cu: subFlavor = str(deps.ThawFlavor(subFlavor)) frzVer = versions.strToFrozen(subVersion, [ x for x in subTS.split(":") ]) subV = versions.ThawVersion(frzVer) t.addReferencedTrove(subName, subV, subFlavor, mkUrl = mkUrl) # It would be far better to use file tags to identify these build # logs, but it's significantly slower as well because they're in # the file objects rather than the trove (and those file objects # could be stored on a different repository) if not subName.endswith(':debuginfo'): continue fileQuery(cu, refInstanceId, dirName = '/usr/src/debug/buildlogs') logHost = subV.getHost() for (dirName, baseName, fileVersion, pathId, fileId) in cu: if (dirName) != '/usr/src/debug/buildlogs': continue if baseName.endswith('-log.bz2'): t.setBuildLog(logHost, sha1ToString(fileId)) elif baseName.endswith('-xml.bz2'): t.setXMLBuildLog(logHost, sha1ToString(fileId)) return t
def testStrToFrozen(self): assert strToFrozen("/a/1.1", ["123"]) == "/a/123:1.1" assert strToFrozen("/a/1.1//b", ["123"]) == "/a/123:1.1//b" assert strToFrozen("/a/1.1//b//c/1.2/d", ["123", "456"]) == "/a/123:1.1//b//c/456:1.2/d" self.assertRaises(AssertionError, strToFrozen, "/a/1.1", ["123", "456"])
def resolve(self, groupIds, label, depList, troveList=[], leavesOnly = False): """ Determine troves that provide the given dependencies, restricting by label and limiting to latest version for each (name, flavor) pair. """ cu = self.db.cursor() # need to make sure that depList does not contain duplicates # for efficiency reasons requires = {} for depStr in depList: depSet = deps.ThawDependencySet(depStr) requires[depSet] = depStr depSetList = requires.keys() depNums = self._setupDepSets(cu, depSetList) # 1. look up inmstances whose provides fully satisfy all the # flags of every depName within a depSet (flagCount check) # 2. out of those instances, only consider the ones that fully # satisfy all the depName deps within a depSet (depCount # check) # 3. filter only the instanceIds the user has access to query = """ select distinct tmpDepNum.idx as idx, tmpDepNum.depNum as depNum, Items.item, flavor, version, Nodes.timeStamps, Nodes.finalTimestamp as finalTimestamp from tmpDepNum join ( select tmpDeps.idx as idx, tmpDeps.depNum as depNum, Provides.instanceId as instanceid, count(*) as flagCount from tmpDeps join Dependencies using(class, name, flag) join Provides using(depId) group by tmpDeps.idx, tmpDeps.depNum, Provides.instanceId ) as DepSelect using(idx, depNum, flagCount) join Instances on Instances.instanceId = DepSelect.instanceId join Nodes using(itemId, versionId) """ where = ["ugi.userGroupId in (%s)" % ( ",".join("%d" % x for x in groupIds),)] args = [] if troveList: self._setupTroveList(cu, troveList) query += """ join tmpInstances as ti on ti.instanceId = Instances.instanceId join UserGroupInstancesCache as ugi on ugi.instanceId = ti.instanceId """ else: if leavesOnly: query += """ join LatestCache as ugi using (itemId, versionId, branchId) """ where.append("ugi.latestType = %d" % trovesource.TROVE_QUERY_NORMAL) where.append("ugi.flavorId = Instances.flavorId") else: query += """ join UserGroupInstancesCache as ugi on ugi.instanceId = Instances.instanceId """ # restrict by label if label: query += """ join LabelMap on Instances.itemId = LabelMap.itemId and Nodes.branchId = LabelMap.branchId join Labels using (labelId) """ where.append("Labels.label = ?") args.append(label) # final joins to allow us to extract the query results as strings query += """ join Items on Instances.itemId = Items.itemId join Versions on Instances.versionId = Versions.versionId join Flavors on Instances.flavorId = Flavors.flavorId where %s order by idx, depNum, finalTimestamp desc """ % ( " and ".join(where)) cu.execute(query, args) # sqlite version 3.2.2 have trouble sorting correctly the # results from the previous query. If we're running on sqlite, # we take the hit here and resort the results in Python... if self.db.driver == "sqlite": # order by idx, depNum, finalTimestamp desc retList = sorted(cu, key = lambda a: (a[0], a[1], -a[6])) else: retList = cu ret = {} for (depId, depNum, troveName, flavorStr, versionStr, timeStamps, ft) in retList: retd = ret.setdefault(depId, [{} for x in xrange(depNums[depId])]) # remember the first version of each (n,f) tuple for each query retd[depNum].setdefault((troveName, flavorStr), (versionStr, timeStamps)) ret2 = {} for depId, depDictList in ret.iteritems(): key = requires[depSetList[depId]] retList = ret2.setdefault(key, [ [] for x in xrange(len(depDictList)) ]) for i, depDict in enumerate(depDictList): retList[i] = [ (trv[0], versions.strToFrozen(ver[0], ver[1].split(":")), trv[1]) for trv, ver in depDict.iteritems() ] # fill in the result dictionary for the values we have not resolved deps for result = {} for depId, depSet in enumerate(depSetList): result.setdefault(requires[depSet], [ [] for x in xrange(depNums[depId])]) result.update(ret2) return result
def resolve(self, groupIds, label, depList, troveList=[], leavesOnly=False): """ Determine troves that provide the given dependencies, restricting by label and limiting to latest version for each (name, flavor) pair. """ cu = self.db.cursor() # need to make sure that depList does not contain duplicates # for efficiency reasons requires = {} for depStr in depList: depSet = deps.ThawDependencySet(depStr) requires[depSet] = depStr depSetList = requires.keys() depNums = self._setupDepSets(cu, depSetList) # 1. look up inmstances whose provides fully satisfy all the # flags of every depName within a depSet (flagCount check) # 2. out of those instances, only consider the ones that fully # satisfy all the depName deps within a depSet (depCount # check) # 3. filter only the instanceIds the user has access to query = """ select distinct tmpDepNum.idx as idx, tmpDepNum.depNum as depNum, Items.item, flavor, version, Nodes.timeStamps, Nodes.finalTimestamp as finalTimestamp from tmpDepNum join ( select tmpDeps.idx as idx, tmpDeps.depNum as depNum, Provides.instanceId as instanceid, count(*) as flagCount from tmpDeps join Dependencies using(class, name, flag) join Provides using(depId) group by tmpDeps.idx, tmpDeps.depNum, Provides.instanceId ) as DepSelect using(idx, depNum, flagCount) join Instances on Instances.instanceId = DepSelect.instanceId join Nodes using(itemId, versionId) """ where = [ "ugi.userGroupId in (%s)" % (",".join("%d" % x for x in groupIds), ) ] args = [] if troveList: self._setupTroveList(cu, troveList) query += """ join tmpInstances as ti on ti.instanceId = Instances.instanceId join UserGroupInstancesCache as ugi on ugi.instanceId = ti.instanceId """ else: if leavesOnly: query += """ join LatestCache as ugi using (itemId, versionId, branchId) """ where.append("ugi.latestType = %d" % trovesource.TROVE_QUERY_NORMAL) where.append("ugi.flavorId = Instances.flavorId") else: query += """ join UserGroupInstancesCache as ugi on ugi.instanceId = Instances.instanceId """ # restrict by label if label: query += """ join LabelMap on Instances.itemId = LabelMap.itemId and Nodes.branchId = LabelMap.branchId join Labels using (labelId) """ where.append("Labels.label = ?") args.append(label) # final joins to allow us to extract the query results as strings query += """ join Items on Instances.itemId = Items.itemId join Versions on Instances.versionId = Versions.versionId join Flavors on Instances.flavorId = Flavors.flavorId where %s order by idx, depNum, finalTimestamp desc """ % (" and ".join(where)) cu.execute(query, args) # sqlite version 3.2.2 have trouble sorting correctly the # results from the previous query. If we're running on sqlite, # we take the hit here and resort the results in Python... if self.db.driver == "sqlite": # order by idx, depNum, finalTimestamp desc retList = sorted(cu, key=lambda a: (a[0], a[1], -a[6])) else: retList = cu ret = {} for (depId, depNum, troveName, flavorStr, versionStr, timeStamps, ft) in retList: retd = ret.setdefault(depId, [{} for x in xrange(depNums[depId])]) # remember the first version of each (n,f) tuple for each query retd[depNum].setdefault((troveName, flavorStr), (versionStr, timeStamps)) ret2 = {} for depId, depDictList in ret.iteritems(): key = requires[depSetList[depId]] retList = ret2.setdefault(key, [[] for x in xrange(len(depDictList))]) for i, depDict in enumerate(depDictList): retList[i] = [(trv[0], versions.strToFrozen(ver[0], ver[1].split(":")), trv[1]) for trv, ver in depDict.iteritems()] # fill in the result dictionary for the values we have not resolved deps for result = {} for depId, depSet in enumerate(depSetList): result.setdefault(requires[depSet], [[] for x in xrange(depNums[depId])]) result.update(ret2) return result