def testStrToFrozen(self):
assert (strToFrozen("/a/1.1", ["123"]) == '/a/123:1.1')
assert (strToFrozen("/a/1.1//b", ["123"]) == '/a/123:1.1//b')
assert (strToFrozen("/a/1.1//b//c/1.2/d",
["123", "456"]) == '/a/123:1.1//b//c/456:1.2/d')
self.assertRaises(AssertionError, strToFrozen, "/a/1.1",
["123", "456"])
def versionStringToFrozen(self, verStr, timeStamps):
if isinstance(timeStamps, basestring):
timeStamps = [float(x) for x in timeStamps.split(':')]
timeStamps = ['%.3f' % x for x in timeStamps]
return versions.strToFrozen(verStr, timeStamps)
def searchNodes(cu, roleIds, label = None, mkUrl = None, filterSet = None,
db = None, name = None, latest = 1):
args = []
d = { 'labelCheck' : '', 'itemCheck' : '' }
d['roleIds'] = ",".join( str(x) for x in roleIds)
d['SOURCENAME'] = trove._TROVEINFO_TAG_SOURCENAME
d['METADATA'] = trove._TROVEINFO_TAG_METADATA
if label:
d['labelCheck'] = "label = ? AND"
args.append(label)
if name:
d['itemCheck'] = "item = ? AND"
args.append(name)
if latest:
cu.execute("""
SELECT idTable.item, version, ts, finalTs, SourceNameTroveInfo.data,
MetadataTroveInfo.data FROM
(SELECT DISTINCT Items.item AS item,
Nodes.versionId AS versionId,
Nodes.timeStamps AS ts,
Nodes.finalTimeStamp as finalTs,
MIN(Instances.instanceId) AS instanceId
FROM Labels
JOIN LabelMap USING (labelId)
JOIN LatestCache USING (itemId, branchId)
JOIN Nodes USING (itemId, versionId)
JOIN Instances USING (itemId, versionId)
JOIN Items USING (itemId)
WHERE %(labelCheck)s
%(itemCheck)s
LatestCache.latestType = 1 AND
LatestCache.userGroupId in (%(roleIds)s)
GROUP BY
Items.item, Nodes.versionId, Nodes.timeStamps,
Nodes.finalTimestamp)
AS idTable
JOIN Versions ON (idTable.versionId = Versions.versionId)
LEFT OUTER JOIN TroveInfo AS SourceNameTroveInfo ON
idTable.instanceId = SourceNameTroveInfo.instanceId AND
SourceNameTroveInfo.infoType = %(SOURCENAME)d
LEFT OUTER JOIN TroveInfo AS MetadataTroveInfo ON
idTable.instanceId = MetadataTroveInfo.instanceId AND
MetadataTroveInfo.infoType = %(METADATA)d
""" % d, args)
else:
cu.execute("""
SELECT idTable.item, version, ts, finalTs, SourceNameTroveInfo.data,
MetadataTroveInfo.data FROM
(SELECT DISTINCT Items.item AS item,
Nodes.versionId AS versionId,
Nodes.timeStamps AS ts,
Nodes.finalTimeStamp as finalTs,
MIN(Instances.instanceId) AS instanceId
FROM Labels
JOIN LabelMap USING (labelId)
JOIN Nodes USING (itemId, branchId)
JOIN Instances USING (itemId, versionId)
JOIN Items USING (itemId)
JOIN usergroupinstancescache AS ugi ON
Instances.instanceId = ugi.instanceId
WHERE %(labelCheck)s
%(itemCheck)s
ugi.userGroupId in (%(roleIds)s)
GROUP BY
Items.item, Nodes.versionId, Nodes.timeStamps,
Nodes.finalTimestamp)
AS idTable
JOIN Versions ON (idTable.versionId = Versions.versionId)
LEFT OUTER JOIN TroveInfo AS SourceNameTroveInfo ON
idTable.instanceId = SourceNameTroveInfo.instanceId AND
SourceNameTroveInfo.infoType = %(SOURCENAME)d
LEFT OUTER JOIN TroveInfo AS MetadataTroveInfo ON
idTable.instanceId = MetadataTroveInfo.instanceId AND
MetadataTroveInfo.infoType = %(METADATA)d
""" % d, args)
l = list(cu)
filteredL = typeFilter(l, filterSet)
# sort based on (name, version, desc(finalTimestamp))
def sortorder(x, y):
c = cmp(x[0], y[0])
if c:
return c
return -(cmp(x[3], y[3]))
filteredL.sort(sortorder)
if latest:
# keep the latest
newL = []
last = None
for item in filteredL:
if last and last[0] == item[0]:
continue
newL.append(item)
last = item
filteredL = newL
nodeList = datamodel.NamedNodeList(total = len(filteredL), start = 0)
addList = []
for (name, version, ts, finalTs, sourceName, metadata) in filteredL:
sourceName = cu.frombinary(sourceName)
if sourceName is None and trove.troveIsSourceComponent(name):
sourceName = name
addList.append((sourceName,
str(versions.VersionFromString(version).getSourceVersion())))
schema.resetTable(cu, 'tmpNVF')
# This is painful, but it converts the source name from a blob to
# a string
db.bulkload("tmpNVF", [ (x[0],) + x[1] for x in enumerate(addList) ],
["idx", "name", "version"],
start_transaction = False)
cu.execute("""
SELECT ChangeLogs.name, ChangeLogs.message, tmpNVF.name
FROM tmpNVF JOIN Items AS SourceItems ON
tmpNVF.name = SourceItems.item
LEFT OUTER JOIN Versions AS SourceVersion ON
tmpNVF.version = SourceVersion.version
LEFT OUTER JOIN Nodes ON
SourceItems.itemId = Nodes.itemId AND
SourceVersion.versionId = Nodes.versionId
LEFT OUTER JOIN ChangeLogs USING (nodeId)
ORDER BY tmpNVF.idx
""")
for ( (name, version, ts, finalTs, sourceName, metadata),
(clName, clMessage, troveName) ) in itertools.izip(filteredL, cu):
frzVer = versions.strToFrozen(version,
[ x for x in ts.split(":") ])
ver = versions.ThawVersion(frzVer)
shortdesc = None
if metadata:
metadata = cu.frombinary(metadata)
md = trove.Metadata(metadata)
shortdesc = md.get()['shortDesc']
if clName:
cl = datamodel.ChangeLog(name = clName, message = clMessage)
else:
cl = None
nodeList.append(name = name, version = ver, mkUrl = mkUrl,
changeLog = cl, shortdesc = shortdesc)
return nodeList
def getTrove(cu, roleIds, name, version, flavor, mkUrl = None,
thisHost = None, displayFlavor = None, excludeCapsules = False):
def buildTupleList(tuples, name, mkUrl = mkUrl):
l = getattr(datamodel.SingleTrove, name)()
for troveInfo in sorted(tuples.iter()):
l.append(name = troveInfo.name(), version = troveInfo.version(),
flavor = troveInfo.flavor(), mkUrl = mkUrl)
return l
def fileQuery(gfcu, filesInstanceId, dirName = None):
# XXX restricing by dirName seems and obvious thing to do here,
# but it actually slows things down??
#
# the distinct here is unfortunate, but conary repositories had
# a bug for about a year which caused it to store duplicate paths
# if a path was committed for the first time duplicate times in
# a single commit job
gfcu.execute("""
SELECT DISTINCT dirName, basename, version, pathId, fileId
FROM TroveFiles
JOIN Versions USING (versionId)
JOIN FileStreams ON (TroveFiles.streamId = FileStreams.streamId)
JOIN FilePaths ON (TroveFiles.filePathId = FilePaths.filePathId)
JOIN DirNames ON
FilePaths.dirNameId = DirNames.dirNameId
JOIN Basenames ON (FilePaths.baseNameId = Basenames.baseNameId)
WHERE TroveFiles.instanceId = ? ORDER BY dirName, basename
""", filesInstanceId)
cu.execute("""
SELECT Instances.instanceId, Nodes.timeStamps FROM Instances
JOIN Nodes USING (itemId, versionId)
JOIN Items USING (itemId)
JOIN Versions ON (Instances.versionId = Versions.versionId)
JOIN Flavors ON (Instances.flavorId = Flavors.flavorId)
JOIN UserGroupInstancesCache AS ugi
ON (instances.instanceId = ugi.instanceId AND
ugi.userGroupId in (%s))
WHERE
item = ? AND version = ? AND flavor = ?
""" % ",".join( str(x) for x in roleIds), name, version,
deps.parseFlavor(flavor).freeze())
l = [ (x[0], x[1]) for x in cu ]
if not l:
return None
instanceId, timeStamps = l[0]
frzVer = versions.strToFrozen(version, timeStamps.split(":"))
verobj = versions.ThawVersion(frzVer)
tupleLists = [ ( trove._TROVEINFO_TAG_BUILDDEPS, 'builddeps' ),
( trove._TROVEINFO_TAG_POLICY_PROV, 'policyprovider' ),
( trove._TROVEINFO_TAG_LOADEDTROVES, 'loadedtroves' ),
( trove._TROVEINFO_TAG_COPIED_FROM, 'copiedfrom' ),
( trove._TROVEINFO_TAG_DERIVEDFROM, 'derivedfrom' ) ]
cu.execute("""
SELECT infoType, data FROM TroveInfo WHERE instanceId = ? AND
infoType IN (%s)
""" % ",".join(str(x) for x in
[ trove._TROVEINFO_TAG_SOURCENAME,
trove._TROVEINFO_TAG_CLONEDFROM,
trove._TROVEINFO_TAG_CLONEDFROMLIST,
trove._TROVEINFO_TAG_BUILDTIME,
trove._TROVEINFO_TAG_SIZE,
trove._TROVEINFO_TAG_METADATA,
trove._TROVEINFO_TAG_CAPSULE,
] + [ x[0] for x in tupleLists ]
), instanceId)
troveInfo = {}
for infoType, data in cu:
data = cu.frombinary(data)
infoClass = trove.TroveInfo.streamDict[infoType][1]
troveInfo[infoType] = infoClass(data)
kwargs = { 'name' : name,
'version' : verobj,
'flavor' : flavor }
if displayFlavor is not None:
kwargs['displayflavor'] = displayFlavor
if trove._TROVEINFO_TAG_BUILDTIME in troveInfo:
kwargs['buildtime'] = int(troveInfo[trove._TROVEINFO_TAG_BUILDTIME]())
if trove._TROVEINFO_TAG_SOURCENAME in troveInfo:
kwargs['source'] = (troveInfo[trove._TROVEINFO_TAG_SOURCENAME](),
verobj.getSourceVersion(), '')
if trove._TROVEINFO_TAG_SIZE in troveInfo:
kwargs['size'] = troveInfo[trove._TROVEINFO_TAG_SIZE]()
if trove._TROVEINFO_TAG_METADATA in troveInfo:
md = troveInfo[trove._TROVEINFO_TAG_METADATA].get()
kwargs['shortdesc'] = md['shortDesc']
kwargs['longdesc'] = md['longDesc']
if md['licenses']:
kwargs['license'] = [ x for x in md['licenses' ]]
if md['crypto']:
kwargs['crypto'] = [ x for x in md['crypto'] ]
for (tag, tagName) in tupleLists:
if tag in troveInfo:
kwargs[tagName] = buildTupleList(troveInfo[tag], tagName,
mkUrl = mkUrl)
t = datamodel.SingleTrove(mkUrl = mkUrl, thisHost = thisHost, **kwargs)
if trove._TROVEINFO_TAG_CLONEDFROMLIST in troveInfo:
clonedFromList = troveInfo[trove._TROVEINFO_TAG_CLONEDFROMLIST]
elif (trove._TROVEINFO_TAG_CLONEDFROM in troveInfo):
clonedFromList = [ troveInfo[trove._TROVEINFO_TAG_CLONEDFROM]() ]
else:
clonedFromList = []
for ver in clonedFromList:
t.addClonedFrom(name, ver, flavor, mkUrl = mkUrl)
hasCapsule = False
if trove._TROVEINFO_TAG_CAPSULE in troveInfo:
if troveInfo[trove._TROVEINFO_TAG_CAPSULE].type():
hasCapsule = True
fileQuery(cu, instanceId)
for (dirName, baseName, fileVersion, pathId, fileId) in cu:
dirName = cu.frombinary(dirName)
baseName = cu.frombinary(baseName)
if pathId == trove.CAPSULE_PATHID:
isCapsule = 1
contentAvailable = not excludeCapsules
else:
isCapsule = None
contentAvailable = not hasCapsule
fileObj = datamodel.FileReference(
path = os.path.join(dirName, baseName),
version = fileVersion,
pathId = md5ToString(cu.frombinary(pathId)),
fileId = sha1ToString(cu.frombinary(fileId)),
isCapsule = isCapsule,
contentAvailable = contentAvailable,
mkUrl = mkUrl, thisHost = thisHost)
t.addFile(fileObj)
cu.execute("""
SELECT item, version, flavor, TroveTroves.includedId, Nodes.timeStamps
FROM TroveTroves
JOIN Instances ON (Instances.instanceId = TroveTroves.includedId)
JOIN Nodes USING (itemId, versionId)
JOIN Items USING (itemId)
JOIN Versions ON (Versions.versionId = Instances.versionId)
JOIN Flavors ON (Flavors.flavorId = Instances.flavorId)
WHERE
TroveTroves.instanceId = ? AND
(TroveTroves.flags & %d) = 0
ORDER BY item, version, flavor
""" % schema.TROVE_TROVES_WEAKREF, instanceId)
for (subName, subVersion, subFlavor, refInstanceId, subTS) in cu:
subFlavor = str(deps.ThawFlavor(subFlavor))
frzVer = versions.strToFrozen(subVersion,
[ x for x in subTS.split(":") ])
subV = versions.ThawVersion(frzVer)
t.addReferencedTrove(subName, subV, subFlavor, mkUrl = mkUrl)
# It would be far better to use file tags to identify these build
# logs, but it's significantly slower as well because they're in
# the file objects rather than the trove (and those file objects
# could be stored on a different repository)
if not subName.endswith(':debuginfo'):
continue
fileQuery(cu, refInstanceId, dirName = '/usr/src/debug/buildlogs')
logHost = subV.getHost()
for (dirName, baseName, fileVersion, pathId, fileId) in cu:
if (dirName) != '/usr/src/debug/buildlogs':
continue
if baseName.endswith('-log.bz2'):
t.setBuildLog(logHost, sha1ToString(fileId))
elif baseName.endswith('-xml.bz2'):
t.setXMLBuildLog(logHost, sha1ToString(fileId))
return t
def searchTroves(cu, roleIds, label = None, filterSet = None, mkUrl = None,
latest = True, start = 0, limit = None, name = None):
d = { 'labelCheck' : '', 'nameCheck' : '' }
args = []
regex = None
if label:
d['labelCheck'] = "label = ? AND"
args.append(label)
if name:
if set('?.*[]\\()+') & set(name):
# if only .* appears, replace them with '%' and use LIKE. this
# code currently fails with \.* in the regex, but neither .
# nor \* are valid trove names anyway
likeName = name
while '.*' in likeName:
likeName = likeName.replace('.*', '%')
if set('?.*[]\\()+') & set(name):
regex = re.compile(name)
else:
d['nameCheck'] = "WHERE item LIKE ?"
args.append(likeName)
else:
d['nameCheck' ] = "WHERE item = ?"
args.append(name)
d['roleIds'] = ",".join( str(x) for x in roleIds)
if latest:
cu.execute("""
SELECT item, version, flavor, ts FROM
(SELECT DISTINCT Nodes.itemId AS itemId,
Nodes.versionId AS versionId, flavorId,
Nodes.timeStamps AS ts FROM Labels
JOIN LabelMap USING (labelId)
JOIN LatestCache USING (itemId, branchId)
JOIN Nodes USING (itemId, versionId)
WHERE %(labelCheck)s
LatestCache.latestType = 1 AND
LatestCache.userGroupId in (%(roleIds)s))
AS idTable JOIN
Items USING (itemId) JOIN
Versions ON (idTable.versionId = Versions.versionId) JOIN
Flavors ON (idTable.flavorId = Flavors.flavorId)
%(nameCheck)s
ORDER BY item, version, flavor
""" % d, *args)
else:
cu.execute("""
SELECT item, version, flavor, ts FROM
(SELECT DISTINCT Instances.itemId AS itemId,
Instances.versionId AS versionId,
Instances.flavorId AS flavorId,
Nodes.timeStamps AS ts
FROM Labels
JOIN LabelMap USING (labelId)
JOIN Nodes USING (itemId, branchid)
JOIN Instances USING (itemid, versionid)
JOIN usergroupinstancescache AS ugi USING (instanceid)
WHERE %(labelCheck)s
ugi.userGroupId in (%(roleIds)s))
AS idTable JOIN
Items USING (itemId) JOIN
Versions ON (idTable.versionId = Versions.versionId) JOIN
Flavors ON (idTable.flavorId = Flavors.flavorId)
%(nameCheck)s
ORDER BY item, version, flavor
""" % d, *args)
l = list(cu)
filteredL = typeFilter(l, filterSet)
if regex:
filteredL = [ x for x in filteredL if regex.match(x[0]) ]
if limit is None:
limit = len(filteredL) - start
troveList = datamodel.NamedTroveIdentList(total = len(filteredL),
start = start)
for (name, version, flavor, ts) in filteredL[start:start + limit]:
flavor = str(deps.ThawFlavor(flavor))
frzVer = versions.strToFrozen(version,
[ x for x in ts.split(":") ])
ver = versions.ThawVersion(frzVer)
troveList.append(name = name, version = ver, flavor = flavor,
mkUrl = mkUrl)
return troveList
def versionStringToFrozen(self, verStr, timeStamps):
if isinstance(timeStamps, basestring):
timeStamps = [float(x) for x in timeStamps.split(':')]
timeStamps = ['%.3f' % x for x in timeStamps]
return versions.strToFrozen(verStr, timeStamps)
def searchNodes(cu, roleIds, label = None, mkUrl = None, filterSet = None,
db = None, name = None, latest = 1):
args = []
d = { 'labelCheck' : '', 'itemCheck' : '' }
d['roleIds'] = ",".join( str(x) for x in roleIds)
d['SOURCENAME'] = trove._TROVEINFO_TAG_SOURCENAME
d['METADATA'] = trove._TROVEINFO_TAG_METADATA
if label:
d['labelCheck'] = "label = ? AND"
args.append(label)
if name:
d['itemCheck'] = "item = ? AND"
args.append(name)
if latest:
cu.execute("""
SELECT idTable.item, version, ts, finalTs, SourceNameTroveInfo.data,
MetadataTroveInfo.data FROM
(SELECT DISTINCT Items.item AS item,
Nodes.versionId AS versionId,
Nodes.timeStamps AS ts,
Nodes.finalTimeStamp as finalTs,
MIN(Instances.instanceId) AS instanceId
FROM Labels
JOIN LabelMap USING (labelId)
JOIN LatestCache USING (itemId, branchId)
JOIN Nodes USING (itemId, versionId)
JOIN Instances USING (itemId, versionId)
JOIN Items USING (itemId)
WHERE %(labelCheck)s
%(itemCheck)s
LatestCache.latestType = 1 AND
LatestCache.userGroupId in (%(roleIds)s)
GROUP BY
Items.item, Nodes.versionId, Nodes.timeStamps,
Nodes.finalTimestamp)
AS idTable
JOIN Versions ON (idTable.versionId = Versions.versionId)
LEFT OUTER JOIN TroveInfo AS SourceNameTroveInfo ON
idTable.instanceId = SourceNameTroveInfo.instanceId AND
SourceNameTroveInfo.infoType = %(SOURCENAME)d
LEFT OUTER JOIN TroveInfo AS MetadataTroveInfo ON
idTable.instanceId = MetadataTroveInfo.instanceId AND
MetadataTroveInfo.infoType = %(METADATA)d
""" % d, args)
else:
cu.execute("""
SELECT idTable.item, version, ts, finalTs, SourceNameTroveInfo.data,
MetadataTroveInfo.data FROM
(SELECT DISTINCT Items.item AS item,
Nodes.versionId AS versionId,
Nodes.timeStamps AS ts,
Nodes.finalTimeStamp as finalTs,
MIN(Instances.instanceId) AS instanceId
FROM Labels
JOIN LabelMap USING (labelId)
JOIN Nodes USING (itemId, branchId)
JOIN Instances USING (itemId, versionId)
JOIN Items USING (itemId)
JOIN usergroupinstancescache AS ugi ON
Instances.instanceId = ugi.instanceId
WHERE %(labelCheck)s
%(itemCheck)s
ugi.userGroupId in (%(roleIds)s)
GROUP BY
Items.item, Nodes.versionId, Nodes.timeStamps,
Nodes.finalTimestamp)
AS idTable
JOIN Versions ON (idTable.versionId = Versions.versionId)
LEFT OUTER JOIN TroveInfo AS SourceNameTroveInfo ON
idTable.instanceId = SourceNameTroveInfo.instanceId AND
SourceNameTroveInfo.infoType = %(SOURCENAME)d
LEFT OUTER JOIN TroveInfo AS MetadataTroveInfo ON
idTable.instanceId = MetadataTroveInfo.instanceId AND
MetadataTroveInfo.infoType = %(METADATA)d
""" % d, args)
l = list(cu)
filteredL = typeFilter(l, filterSet)
# sort based on (name, version, desc(finalTimestamp))
def sortorder(x, y):
c = cmp(x[0], y[0])
if c:
return c
return -(cmp(x[3], y[3]))
filteredL.sort(sortorder)
if latest:
# keep the latest
newL = []
last = None
for item in filteredL:
if last and last[0] == item[0]:
continue
newL.append(item)
last = item
filteredL = newL
nodeList = datamodel.NamedNodeList(total = len(filteredL), start = 0)
addList = []
for (name, version, ts, finalTs, sourceName, metadata) in filteredL:
sourceName = cu.frombinary(sourceName)
if sourceName is None and trove.troveIsSourceComponent(name):
sourceName = name
addList.append((sourceName,
str(versions.VersionFromString(version).getSourceVersion())))
schema.resetTable(cu, 'tmpNVF')
# This is painful, but it converts the source name from a blob to
# a string
db.bulkload("tmpNVF", [ (x[0],) + x[1] for x in enumerate(addList) ],
["idx", "name", "version"],
start_transaction = False)
cu.execute("""
SELECT ChangeLogs.name, ChangeLogs.message, tmpNVF.name
FROM tmpNVF JOIN Items AS SourceItems ON
tmpNVF.name = SourceItems.item
LEFT OUTER JOIN Versions AS SourceVersion ON
tmpNVF.version = SourceVersion.version
LEFT OUTER JOIN Nodes ON
SourceItems.itemId = Nodes.itemId AND
SourceVersion.versionId = Nodes.versionId
LEFT OUTER JOIN ChangeLogs USING (nodeId)
ORDER BY tmpNVF.idx
""")
for ( (name, version, ts, finalTs, sourceName, metadata),
(clName, clMessage, troveName) ) in itertools.izip(filteredL, cu):
frzVer = versions.strToFrozen(version,
[ x for x in ts.split(":") ])
ver = versions.ThawVersion(frzVer)
shortdesc = None
if metadata:
metadata = cu.frombinary(metadata)
md = trove.Metadata(metadata)
shortdesc = md.get()['shortDesc']
if clName:
cl = datamodel.ChangeLog(name = clName, message = clMessage)
else:
cl = None
nodeList.append(name = name, version = ver, mkUrl = mkUrl,
changeLog = cl, shortdesc = shortdesc)
return nodeList
def getTrove(cu, roleIds, name, version, flavor, mkUrl = None,
thisHost = None, displayFlavor = None):
def buildTupleList(tuples, name, mkUrl = mkUrl):
l = getattr(datamodel.SingleTrove, name)()
for troveInfo in sorted(tuples.iter()):
l.append(name = troveInfo.name(), version = troveInfo.version(),
flavor = troveInfo.flavor(), mkUrl = mkUrl)
return l
def fileQuery(gfcu, filesInstanceId, dirName = None):
# XXX restricing by dirName seems and obvious thing to do here,
# but it actually slows things down??
#
# the distinct here is unfortunate, but conary repositories had
# a bug for about a year which caused it to store duplicate paths
# if a path was committed for the first time duplicate times in
# a single commit job
gfcu.execute("""
SELECT DISTINCT dirName, basename, version, pathId, fileId
FROM TroveFiles
JOIN Versions USING (versionId)
JOIN FileStreams ON (TroveFiles.streamId = FileStreams.streamId)
JOIN FilePaths ON (TroveFiles.filePathId = FilePaths.filePathId)
JOIN DirNames ON
FilePaths.dirNameId = DirNames.dirNameId
JOIN Basenames ON (FilePaths.baseNameId = Basenames.baseNameId)
WHERE TroveFiles.instanceId = ? ORDER BY dirName, basename
""", filesInstanceId)
cu.execute("""
SELECT Instances.instanceId, Nodes.timeStamps FROM Instances
JOIN Nodes USING (itemId, versionId)
JOIN Items USING (itemId)
JOIN Versions ON (Instances.versionId = Versions.versionId)
JOIN Flavors ON (Instances.flavorId = Flavors.flavorId)
JOIN UserGroupInstancesCache AS ugi
ON (instances.instanceId = ugi.instanceId AND
ugi.userGroupId in (%s))
WHERE
item = ? AND version = ? AND flavor = ?
""" % ",".join( str(x) for x in roleIds), name, version,
deps.parseFlavor(flavor).freeze())
l = [ (x[0], x[1]) for x in cu ]
if not l:
return None
instanceId, timeStamps = l[0]
frzVer = versions.strToFrozen(version, timeStamps.split(":"))
verobj = versions.ThawVersion(frzVer)
tupleLists = [ ( trove._TROVEINFO_TAG_BUILDDEPS, 'builddeps' ),
( trove._TROVEINFO_TAG_POLICY_PROV, 'policyprovider' ),
( trove._TROVEINFO_TAG_LOADEDTROVES, 'loadedtroves' ),
( trove._TROVEINFO_TAG_COPIED_FROM, 'copiedfrom' ),
( trove._TROVEINFO_TAG_DERIVEDFROM, 'derivedfrom' ) ]
cu.execute("""
SELECT infoType, data FROM TroveInfo WHERE instanceId = ? AND
infoType IN (%s)
""" % ",".join(str(x) for x in
[ trove._TROVEINFO_TAG_SOURCENAME,
trove._TROVEINFO_TAG_CLONEDFROM,
trove._TROVEINFO_TAG_CLONEDFROMLIST,
trove._TROVEINFO_TAG_BUILDTIME,
trove._TROVEINFO_TAG_SIZE,
trove._TROVEINFO_TAG_METADATA,
trove._TROVEINFO_TAG_CAPSULE,
] + [ x[0] for x in tupleLists ]
), instanceId)
troveInfo = {}
for infoType, data in cu:
data = cu.frombinary(data)
infoClass = trove.TroveInfo.streamDict[infoType][1]
troveInfo[infoType] = infoClass(data)
kwargs = { 'name' : name,
'version' : verobj,
'flavor' : flavor }
if displayFlavor is not None:
kwargs['displayflavor'] = displayFlavor
if trove._TROVEINFO_TAG_BUILDTIME in troveInfo:
kwargs['buildtime'] = int(troveInfo[trove._TROVEINFO_TAG_BUILDTIME]())
if trove._TROVEINFO_TAG_SOURCENAME in troveInfo:
kwargs['source'] = (troveInfo[trove._TROVEINFO_TAG_SOURCENAME](),
verobj.getSourceVersion(), '')
if trove._TROVEINFO_TAG_SIZE in troveInfo:
kwargs['size'] = troveInfo[trove._TROVEINFO_TAG_SIZE]()
if trove._TROVEINFO_TAG_METADATA in troveInfo:
md = troveInfo[trove._TROVEINFO_TAG_METADATA].get()
kwargs['shortdesc'] = md['shortDesc']
kwargs['longdesc'] = md['longDesc']
if md['licenses']:
kwargs['license'] = [ x for x in md['licenses' ]]
if md['crypto']:
kwargs['crypto'] = [ x for x in md['crypto'] ]
for (tag, tagName) in tupleLists:
if tag in troveInfo:
kwargs[tagName] = buildTupleList(troveInfo[tag], tagName,
mkUrl = mkUrl)
t = datamodel.SingleTrove(mkUrl = mkUrl, thisHost = thisHost, **kwargs)
if trove._TROVEINFO_TAG_CLONEDFROMLIST in troveInfo:
clonedFromList = troveInfo[trove._TROVEINFO_TAG_CLONEDFROMLIST]
elif (trove._TROVEINFO_TAG_CLONEDFROM in troveInfo):
clonedFromList = [ troveInfo[trove._TROVEINFO_TAG_CLONEDFROM]() ]
else:
clonedFromList = []
for ver in clonedFromList:
t.addClonedFrom(name, ver, flavor, mkUrl = mkUrl)
hasCapsule = False
if trove._TROVEINFO_TAG_CAPSULE in troveInfo:
if troveInfo[trove._TROVEINFO_TAG_CAPSULE].type():
hasCapsule = True
fileQuery(cu, instanceId)
for (dirName, baseName, fileVersion, pathId, fileId) in cu:
dirName = cu.frombinary(dirName)
baseName = cu.frombinary(baseName)
if pathId == trove.CAPSULE_PATHID:
isCapsule = 1
contentAvailable = True
else:
isCapsule = None
contentAvailable = not hasCapsule
fileObj = datamodel.FileReference(
path = os.path.join(dirName, baseName),
version = fileVersion,
pathId = md5ToString(cu.frombinary(pathId)),
fileId = sha1ToString(cu.frombinary(fileId)),
isCapsule = isCapsule,
contentAvailable = contentAvailable,
mkUrl = mkUrl, thisHost = thisHost)
t.addFile(fileObj)
cu.execute("""
SELECT item, version, flavor, TroveTroves.includedId, Nodes.timeStamps
FROM TroveTroves
JOIN Instances ON (Instances.instanceId = TroveTroves.includedId)
JOIN Nodes USING (itemId, versionId)
JOIN Items USING (itemId)
JOIN Versions ON (Versions.versionId = Instances.versionId)
JOIN Flavors ON (Flavors.flavorId = Instances.flavorId)
WHERE
TroveTroves.instanceId = ? AND
(TroveTroves.flags & %d) = 0
ORDER BY item, version, flavor
""" % schema.TROVE_TROVES_WEAKREF, instanceId)
for (subName, subVersion, subFlavor, refInstanceId, subTS) in cu:
subFlavor = str(deps.ThawFlavor(subFlavor))
frzVer = versions.strToFrozen(subVersion,
[ x for x in subTS.split(":") ])
subV = versions.ThawVersion(frzVer)
t.addReferencedTrove(subName, subV, subFlavor, mkUrl = mkUrl)
# It would be far better to use file tags to identify these build
# logs, but it's significantly slower as well because they're in
# the file objects rather than the trove (and those file objects
# could be stored on a different repository)
if not subName.endswith(':debuginfo'):
continue
fileQuery(cu, refInstanceId, dirName = '/usr/src/debug/buildlogs')
logHost = subV.getHost()
for (dirName, baseName, fileVersion, pathId, fileId) in cu:
if (dirName) != '/usr/src/debug/buildlogs':
continue
if baseName.endswith('-log.bz2'):
t.setBuildLog(logHost, sha1ToString(fileId))
elif baseName.endswith('-xml.bz2'):
t.setXMLBuildLog(logHost, sha1ToString(fileId))
return t
def searchTroves(cu, roleIds, label = None, filterSet = None, mkUrl = None,
latest = True, start = 0, limit = None, name = None):
d = { 'labelCheck' : '', 'nameCheck' : '' }
args = []
regex = None
if label:
d['labelCheck'] = "label = ? AND"
args.append(label)
if name:
if set('?.*[]\\()+') & set(name):
# if only .* appears, replace them with '%' and use LIKE. this
# code currently fails with \.* in the regex, but neither .
# nor \* are valid trove names anyway
likeName = name
while '.*' in likeName:
likeName = likeName.replace('.*', '%')
if set('?.*[]\\()+') & set(name):
regex = re.compile(name)
else:
d['nameCheck'] = "WHERE item LIKE ?"
args.append(likeName)
else:
d['nameCheck' ] = "WHERE item = ?"
args.append(name)
d['roleIds'] = ",".join( str(x) for x in roleIds)
if latest:
cu.execute("""
SELECT item, version, flavor, ts FROM
(SELECT DISTINCT Nodes.itemId AS itemId,
Nodes.versionId AS versionId, flavorId,
Nodes.timeStamps AS ts FROM Labels
JOIN LabelMap USING (labelId)
JOIN LatestCache USING (itemId, branchId)
JOIN Nodes USING (itemId, versionId)
WHERE %(labelCheck)s
LatestCache.latestType = 1 AND
LatestCache.userGroupId in (%(roleIds)s))
AS idTable JOIN
Items USING (itemId) JOIN
Versions ON (idTable.versionId = Versions.versionId) JOIN
Flavors ON (idTable.flavorId = Flavors.flavorId)
%(nameCheck)s
ORDER BY item, version, flavor
""" % d, *args)
else:
cu.execute("""
SELECT item, version, flavor, ts FROM
(SELECT DISTINCT Instances.itemId AS itemId,
Instances.versionId AS versionId,
Instances.flavorId AS flavorId,
Nodes.timeStamps AS ts
FROM Labels
JOIN LabelMap USING (labelId)
JOIN Nodes USING (itemId, branchid)
JOIN Instances USING (itemid, versionid)
JOIN usergroupinstancescache AS ugi USING (instanceid)
WHERE %(labelCheck)s
ugi.userGroupId in (%(roleIds)s))
AS idTable JOIN
Items USING (itemId) JOIN
Versions ON (idTable.versionId = Versions.versionId) JOIN
Flavors ON (idTable.flavorId = Flavors.flavorId)
%(nameCheck)s
ORDER BY item, version, flavor
""" % d, *args)
l = list(cu)
filteredL = typeFilter(l, filterSet)
if regex:
filteredL = [ x for x in filteredL if regex.match(x[0]) ]
if limit is None:
limit = len(filteredL) - start
troveList = datamodel.NamedTroveIdentList(total = len(filteredL),
start = start)
for (name, version, flavor, ts) in filteredL[start:start + limit]:
flavor = str(deps.ThawFlavor(flavor))
frzVer = versions.strToFrozen(version,
[ x for x in ts.split(":") ])
ver = versions.ThawVersion(frzVer)
troveList.append(name = name, version = ver, flavor = flavor,
mkUrl = mkUrl)
return troveList
def testStrToFrozen(self):
assert strToFrozen("/a/1.1", ["123"]) == "/a/123:1.1"
assert strToFrozen("/a/1.1//b", ["123"]) == "/a/123:1.1//b"
assert strToFrozen("/a/1.1//b//c/1.2/d", ["123", "456"]) == "/a/123:1.1//b//c/456:1.2/d"
self.assertRaises(AssertionError, strToFrozen, "/a/1.1", ["123", "456"])
def resolve(self, groupIds, label, depList, troveList=[], leavesOnly = False):
""" Determine troves that provide the given dependencies,
restricting by label and limiting to latest version for
each (name, flavor) pair.
"""
cu = self.db.cursor()
# need to make sure that depList does not contain duplicates
# for efficiency reasons
requires = {}
for depStr in depList:
depSet = deps.ThawDependencySet(depStr)
requires[depSet] = depStr
depSetList = requires.keys()
depNums = self._setupDepSets(cu, depSetList)
# 1. look up inmstances whose provides fully satisfy all the
# flags of every depName within a depSet (flagCount check)
# 2. out of those instances, only consider the ones that fully
# satisfy all the depName deps within a depSet (depCount
# check)
# 3. filter only the instanceIds the user has access to
query = """
select distinct
tmpDepNum.idx as idx, tmpDepNum.depNum as depNum,
Items.item, flavor, version, Nodes.timeStamps,
Nodes.finalTimestamp as finalTimestamp
from tmpDepNum
join (
select
tmpDeps.idx as idx,
tmpDeps.depNum as depNum,
Provides.instanceId as instanceid,
count(*) as flagCount
from tmpDeps
join Dependencies using(class, name, flag)
join Provides using(depId)
group by tmpDeps.idx, tmpDeps.depNum, Provides.instanceId
) as DepSelect using(idx, depNum, flagCount)
join Instances on
Instances.instanceId = DepSelect.instanceId
join Nodes using(itemId, versionId) """
where = ["ugi.userGroupId in (%s)" % (
",".join("%d" % x for x in groupIds),)]
args = []
if troveList:
self._setupTroveList(cu, troveList)
query += """
join tmpInstances as ti on ti.instanceId = Instances.instanceId
join UserGroupInstancesCache as ugi on
ugi.instanceId = ti.instanceId """
else:
if leavesOnly:
query += """
join LatestCache as ugi using (itemId, versionId, branchId) """
where.append("ugi.latestType = %d" % trovesource.TROVE_QUERY_NORMAL)
where.append("ugi.flavorId = Instances.flavorId")
else:
query += """
join UserGroupInstancesCache as ugi on
ugi.instanceId = Instances.instanceId """
# restrict by label
if label:
query += """
join LabelMap on
Instances.itemId = LabelMap.itemId and
Nodes.branchId = LabelMap.branchId
join Labels using (labelId) """
where.append("Labels.label = ?")
args.append(label)
# final joins to allow us to extract the query results as strings
query += """
join Items on Instances.itemId = Items.itemId
join Versions on Instances.versionId = Versions.versionId
join Flavors on Instances.flavorId = Flavors.flavorId
where %s
order by idx, depNum, finalTimestamp desc """ % (
" and ".join(where))
cu.execute(query, args)
# sqlite version 3.2.2 have trouble sorting correctly the
# results from the previous query. If we're running on sqlite,
# we take the hit here and resort the results in Python...
if self.db.driver == "sqlite":
# order by idx, depNum, finalTimestamp desc
retList = sorted(cu, key = lambda a: (a[0], a[1], -a[6]))
else:
retList = cu
ret = {}
for (depId, depNum, troveName, flavorStr, versionStr, timeStamps, ft) in retList:
retd = ret.setdefault(depId, [{} for x in xrange(depNums[depId])])
# remember the first version of each (n,f) tuple for each query
retd[depNum].setdefault((troveName, flavorStr), (versionStr, timeStamps))
ret2 = {}
for depId, depDictList in ret.iteritems():
key = requires[depSetList[depId]]
retList = ret2.setdefault(key, [ [] for x in xrange(len(depDictList)) ])
for i, depDict in enumerate(depDictList):
retList[i] = [ (trv[0], versions.strToFrozen(ver[0], ver[1].split(":")),
trv[1]) for trv, ver in depDict.iteritems() ]
# fill in the result dictionary for the values we have not resolved deps for
result = {}
for depId, depSet in enumerate(depSetList):
result.setdefault(requires[depSet],
[ [] for x in xrange(depNums[depId])])
result.update(ret2)
return result
def resolve(self,
groupIds,
label,
depList,
troveList=[],
leavesOnly=False):
""" Determine troves that provide the given dependencies,
restricting by label and limiting to latest version for
each (name, flavor) pair.
"""
cu = self.db.cursor()
# need to make sure that depList does not contain duplicates
# for efficiency reasons
requires = {}
for depStr in depList:
depSet = deps.ThawDependencySet(depStr)
requires[depSet] = depStr
depSetList = requires.keys()
depNums = self._setupDepSets(cu, depSetList)
# 1. look up inmstances whose provides fully satisfy all the
# flags of every depName within a depSet (flagCount check)
# 2. out of those instances, only consider the ones that fully
# satisfy all the depName deps within a depSet (depCount
# check)
# 3. filter only the instanceIds the user has access to
query = """
select distinct
tmpDepNum.idx as idx, tmpDepNum.depNum as depNum,
Items.item, flavor, version, Nodes.timeStamps,
Nodes.finalTimestamp as finalTimestamp
from tmpDepNum
join (
select
tmpDeps.idx as idx,
tmpDeps.depNum as depNum,
Provides.instanceId as instanceid,
count(*) as flagCount
from tmpDeps
join Dependencies using(class, name, flag)
join Provides using(depId)
group by tmpDeps.idx, tmpDeps.depNum, Provides.instanceId
) as DepSelect using(idx, depNum, flagCount)
join Instances on
Instances.instanceId = DepSelect.instanceId
join Nodes using(itemId, versionId) """
where = [
"ugi.userGroupId in (%s)" % (",".join("%d" % x
for x in groupIds), )
]
args = []
if troveList:
self._setupTroveList(cu, troveList)
query += """
join tmpInstances as ti on ti.instanceId = Instances.instanceId
join UserGroupInstancesCache as ugi on
ugi.instanceId = ti.instanceId """
else:
if leavesOnly:
query += """
join LatestCache as ugi using (itemId, versionId, branchId) """
where.append("ugi.latestType = %d" %
trovesource.TROVE_QUERY_NORMAL)
where.append("ugi.flavorId = Instances.flavorId")
else:
query += """
join UserGroupInstancesCache as ugi on
ugi.instanceId = Instances.instanceId """
# restrict by label
if label:
query += """
join LabelMap on
Instances.itemId = LabelMap.itemId and
Nodes.branchId = LabelMap.branchId
join Labels using (labelId) """
where.append("Labels.label = ?")
args.append(label)
# final joins to allow us to extract the query results as strings
query += """
join Items on Instances.itemId = Items.itemId
join Versions on Instances.versionId = Versions.versionId
join Flavors on Instances.flavorId = Flavors.flavorId
where %s
order by idx, depNum, finalTimestamp desc """ % (" and ".join(where))
cu.execute(query, args)
# sqlite version 3.2.2 have trouble sorting correctly the
# results from the previous query. If we're running on sqlite,
# we take the hit here and resort the results in Python...
if self.db.driver == "sqlite":
# order by idx, depNum, finalTimestamp desc
retList = sorted(cu, key=lambda a: (a[0], a[1], -a[6]))
else:
retList = cu
ret = {}
for (depId, depNum, troveName, flavorStr, versionStr, timeStamps,
ft) in retList:
retd = ret.setdefault(depId, [{} for x in xrange(depNums[depId])])
# remember the first version of each (n,f) tuple for each query
retd[depNum].setdefault((troveName, flavorStr),
(versionStr, timeStamps))
ret2 = {}
for depId, depDictList in ret.iteritems():
key = requires[depSetList[depId]]
retList = ret2.setdefault(key,
[[] for x in xrange(len(depDictList))])
for i, depDict in enumerate(depDictList):
retList[i] = [(trv[0],
versions.strToFrozen(ver[0],
ver[1].split(":")), trv[1])
for trv, ver in depDict.iteritems()]
# fill in the result dictionary for the values we have not resolved deps for
result = {}
for depId, depSet in enumerate(depSetList):
result.setdefault(requires[depSet],
[[] for x in xrange(depNums[depId])])
result.update(ret2)
return result
