def _setupDepSets(self, cu, depSetList, reset=True):
if reset:
schema.resetTable(cu, "tmpDeps")
schema.resetTable(cu, "tmpDepNum")
# count how many dep classes are in each depSet
depNums = []
for i, depSet in enumerate(depSetList):
depNum = 0
for depClass, dep in depSet.iterDeps(sort=True):
# need to get these in sorted order as the depNums
# we get here are important...
classId = depClass.tag
depName = dep.getName()
flags = dep.getFlags()
for (depName, flags) in zip(dep.getName(), dep.getFlags()):
cu.execute("""
insert into tmpDeps(idx, depNum, class, name, flag)
values (?, ?, ?, ?, ?)""",
(i, depNum, classId, depName, NO_FLAG_MAGIC))
if flags:
for flag, sense in flags:
# assert sense is required
cu.execute("""
insert into tmpDeps(idx, depNum, class, name, flag)
values (?, ?, ?, ?, ?)""",
(i, depNum, classId, depName, flag))
cu.execute("""insert into tmpDepNum(idx, depNum, flagCount)
values (?, ?, ?)""", (i, depNum, len(flags)+1))
depNum += 1
depNums.append(depNum)
self.db.analyze("tmpDeps")
self.db.analyze("tmpDepNum")
return depNums
def _setupTroveList(self, cu, troveList):
if not troveList:
return
schema.resetTable(cu, "tmpInstances")
schema.resetTable(cu, "tmpId")
cu.executemany("""
insert into tmpInstances(instanceId)
select instanceId
from Instances
join Items on Instances.itemId = Items.itemId
JOIN versions on Instances.versionId = Versions.versionId
join Flavors on Instances.flavorId = Flavors.flavorId
where Items.item = ? and Versions.version = ? and Flavors.flavor = ?
""", troveList, start_transaction=False )
self.db.analyze("tmpInstances")
# now grab the instanceIds of their included troves, avoiding duplicates
cu.execute("""
insert into tmpId(id)
select distinct tt.includedId
from tmpInstances as ti
join TroveTroves as tt using(instanceId)
""", start_transaction=False)
# drop the ones we already have
cu.execute("delete from tmpId where id in "
"(select instanceId from tmpInstances)",
start_transaction=False)
# append the remaining instanceIds
cu.execute("insert into tmpInstances(instanceId) select id from tmpId",
start_transaction=False)
self.db.analyze("tmpInstances")
def deletePermissionId(self, permissionId, roleId):
cu = self.db.cursor()
# compute the list of troves for which no other RAP/RAT access exists
schema.resetTable(cu, "tmpInstances")
cu.execute("""
insert into tmpInstances(instanceId)
select ugi.instanceId from UserGroupInstancesCache as ugi
where ugi.userGroupId = ?
and not exists (
select 1 from UserGroupAllPermissions as ugap
where ugap.userGroupId = ?
and ugap.instanceId = ugi.instanceId
and ugap.permissionId != ? )
and not exists (
select 1 from UserGroupAllTroves as ugat
where ugat.instanceId = ugi.instanceId
and ugat.userGroupId = ? )""",
(roleId, roleId, permissionId, roleId),
start_transaction=False)
# clean up the flattened table
cu.execute(
"delete from UserGroupAllPermissions where permissionId = ?",
permissionId)
# now we have only the troves which need to be erased out of UGIC
self.db.analyze("tmpInstances")
cu.execute(
"""
delete from UserGroupInstancesCache
where userGroupId = ?
and instanceId in (select instanceId from tmpInstances)""", roleId)
# update Latest
self.latest.updateRoleId(cu, roleId, tmpInstances=True)
def _setupTroveList(self, cu, troveList):
if not troveList:
return
schema.resetTable(cu, "tmpInstances")
schema.resetTable(cu, "tmpId")
cu.executemany("""
insert into tmpInstances(instanceId)
select instanceId
from Instances
join Items on Instances.itemId = Items.itemId
JOIN versions on Instances.versionId = Versions.versionId
join Flavors on Instances.flavorId = Flavors.flavorId
where Items.item = ? and Versions.version = ? and Flavors.flavor = ?
""",
troveList,
start_transaction=False)
self.db.analyze("tmpInstances")
# now grab the instanceIds of their included troves, avoiding duplicates
cu.execute("""
insert into tmpId(id)
select distinct tt.includedId
from tmpInstances as ti
join TroveTroves as tt using(instanceId)
""",
start_transaction=False)
# drop the ones we already have
cu.execute(
"delete from tmpId where id in "
"(select instanceId from tmpInstances)",
start_transaction=False)
# append the remaining instanceIds
cu.execute("insert into tmpInstances(instanceId) select id from tmpId",
start_transaction=False)
self.db.analyze("tmpInstances")
def addTroveAccess(self, role, troveList, recursive=True):
roleId = self._getRoleId(role)
rtList = self.rt.add(roleId, troveList, recursive)
# we now know the ids of the new acls added. They're useful in
# updating the UGIC table
cu = self.db.cursor()
# grab the list of instanceIds we are adding to the UGIC table;
# we need those for a faster recomputation of the LatestCache table
schema.resetTable(cu, "tmpInstances")
cu.execute("""
insert into tmpInstances(instanceId)
select distinct ugat.instanceId
from UserGroupAllTroves as ugat
where ugat.ugtId in (%s)
and ugat.userGroupId = ?
and not exists (
select 1 from UserGroupInstancesCache as ugi
where ugi.userGroupId = ?
and ugi.instanceId = ugat.instanceId )
""" % (",".join("%d" % x for x in rtList),),
(roleId, roleId), start_transaction=False)
# insert into UGIC and recompute the latest table
cu.execute("""
insert into UserGroupInstancesCache (userGroupId, instanceId)
select %d, instanceId from tmpInstances """ %(roleId,))
# tmpInstances has instanceIds for which Latest needs to be recomputed
self.db.analyze("tmpInstances")
self.latest.updateRoleId(cu, roleId, tmpInstances=True)
def addPermissionId(self, permissionId, roleId):
cu = self.db.cursor()
self.rp.addId(cu, permissionId=permissionId)
# figure out newly accessible troves. We keep track separately
# to speed up the Latest update
schema.resetTable(cu, "tmpInstances")
cu.execute("""
insert into tmpInstances(instanceId)
select instanceId from UserGroupAllPermissions as ugap
where permissionId = ?
and not exists (
select instanceId from UserGroupInstancesCache as ugi
where ugi.userGroupId = ?
and ugi.instanceId = ugap.instanceId ) """,
(permissionId, roleId),
start_transaction=False)
# update UsergroupInstancesCache
cu.execute(
"""
insert into UserGroupInstancesCache (userGroupId, instanceId, canWrite)
select userGroupId, instanceId,
case when sum(canWrite) = 0 then 0 else 1 end as canWrite
from UserGroupAllPermissions
where permissionId = ?
and instanceId in (select instanceId from tmpInstances)
group by userGroupId, instanceId
""", permissionId)
# update Latest
self.latest.updateRoleId(cu, roleId, tmpInstances=True)
def addTroveAccess(self, role, troveList, recursive=True):
roleId = self._getRoleId(role)
rtList = self.rt.add(roleId, troveList, recursive)
# we now know the ids of the new acls added. They're useful in
# updating the UGIC table
cu = self.db.cursor()
# grab the list of instanceIds we are adding to the UGIC table;
# we need those for a faster recomputation of the LatestCache table
schema.resetTable(cu, "tmpInstances")
cu.execute("""
insert into tmpInstances(instanceId)
select distinct ugat.instanceId
from UserGroupAllTroves as ugat
where ugat.ugtId in (%s)
and ugat.userGroupId = ?
and not exists (
select 1 from UserGroupInstancesCache as ugi
where ugi.userGroupId = ?
and ugi.instanceId = ugat.instanceId )
""" % (",".join("%d" % x for x in rtList), ), (roleId, roleId),
start_transaction=False)
# insert into UGIC and recompute the latest table
cu.execute("""
insert into UserGroupInstancesCache (userGroupId, instanceId)
select %d, instanceId from tmpInstances """ % (roleId, ))
# tmpInstances has instanceIds for which Latest needs to be recomputed
self.db.analyze("tmpInstances")
self.latest.updateRoleId(cu, roleId, tmpInstances=True)
def addPermissionId(self, permissionId, roleId):
cu = self.db.cursor()
self.rp.addId(cu, permissionId = permissionId)
# figure out newly accessible troves. We keep track separately
# to speed up the Latest update
schema.resetTable(cu, "tmpInstances")
cu.execute("""
insert into tmpInstances(instanceId)
select instanceId from UserGroupAllPermissions as ugap
where permissionId = ?
and not exists (
select instanceId from UserGroupInstancesCache as ugi
where ugi.userGroupId = ?
and ugi.instanceId = ugap.instanceId ) """,
(permissionId, roleId),
start_transaction = False)
# update UsergroupInstancesCache
cu.execute("""
insert into UserGroupInstancesCache (userGroupId, instanceId, canWrite)
select userGroupId, instanceId,
case when sum(canWrite) = 0 then 0 else 1 end as canWrite
from UserGroupAllPermissions
where permissionId = ?
and instanceId in (select instanceId from tmpInstances)
group by userGroupId, instanceId
""", permissionId)
# update Latest
self.latest.updateRoleId(cu, roleId, tmpInstances=True)
def delete(self, roleId, troveList):
"""remove group access to troves passed in the (n,v,f) troveList"""
self._findInstanceIds(troveList, checkMissing=False)
cu = self.db.cursor()
schema.resetTable(cu, "tmpId")
cu.execute("""
insert into tmpId(id)
select ugtId from UserGroupTroves
where userGroupId = ?
and instanceId in (select instanceId from tmpInstanceId)
""", roleId, start_transaction=False)
self.db.analyze("tmpId")
# save what instanceIds will be affected by this delete
schema.resetTable(cu, "tmpInstances")
cu.execute("""
insert into tmpInstances(instanceId)
select distinct ugat.instanceId
from UserGroupAllTroves as ugat
where ugat.userGroupId = ?
and ugat.ugtId in (select id from tmpId)
""", roleId, start_transaction = False)
cu.execute("delete from UserGroupAllTroves where ugtId in (select id from tmpId)")
cu.execute("delete from UserGroupTroves where ugtId in (select id from tmpId)")
# filter out the ones that are still allowed based on other permissions
cu.execute("""
delete from tmpInstances
where exists (
select 1 from UserGroupAllTroves as ugat
where userGroupId = ?
and ugat.instanceId = tmpInstances.instanceId )
""", roleId, start_transaction=False)
return True
def deletePermissionId(self, permissionId, roleId):
cu = self.db.cursor()
# compute the list of troves for which no other RAP/RAT access exists
schema.resetTable(cu, "tmpInstances")
cu.execute("""
insert into tmpInstances(instanceId)
select ugi.instanceId from UserGroupInstancesCache as ugi
where ugi.userGroupId = ?
and not exists (
select 1 from UserGroupAllPermissions as ugap
where ugap.userGroupId = ?
and ugap.instanceId = ugi.instanceId
and ugap.permissionId != ? )
and not exists (
select 1 from UserGroupAllTroves as ugat
where ugat.instanceId = ugi.instanceId
and ugat.userGroupId = ? )""",
(roleId, roleId, permissionId, roleId),
start_transaction = False)
# clean up the flattened table
cu.execute("delete from UserGroupAllPermissions where permissionId = ?",
permissionId)
# now we have only the troves which need to be erased out of UGIC
self.db.analyze("tmpInstances")
cu.execute("""
delete from UserGroupInstancesCache
where userGroupId = ?
and instanceId in (select instanceId from tmpInstances)""", roleId)
# update Latest
self.latest.updateRoleId(cu, roleId, tmpInstances=True)
def updatePermissionId(self, permissionId, roleId):
cu = self.db.cursor()
schema.resetTable(cu, "tmpInstances")
# figure out how the access is changing
cu.execute("""
insert into tmpInstances(instanceId)
select instanceId from UserGroupAllPermissions
where permissionId = ? """,
permissionId,
start_transaction=False)
# re-add
self.rp.deleteId(cu, permissionId=permissionId)
self.rp.addId(cu, permissionId=permissionId)
# remove from consideration troves for which we still have access
cu.execute("""
delete from tmpInstances
where exists (
select 1 from UserGroupAllPermissions as ugap
where ugap.userGroupId = :roleId
and ugap.instanceId = tmpInstances.instanceId )
or exists (
select 1 from UserGroupAllTroves as ugat
where ugat.userGroupId = :roleId
and ugat.instanceId = tmpInstances.instanceId )
""",
roleId=roleId,
start_transaction=False)
self.db.analyze("tmpInstances")
# remove trove access from troves that are left
cu.execute("""
delete from UserGroupInstancesCache
where userGroupId = :roleId
and instanceId in (select instanceId from tmpInstances)
and not exists (
select 1 from UserGroupAllTroves as ugat
where ugat.userGroupId = :roleId
and ugat.instanceId = UserGroupInstancesCache.instanceId )
""",
roleId=roleId)
# add the new troves now
cu.execute(
"""
insert into UserGroupInstancesCache(userGroupId, instanceId, canWrite)
select userGroupId, instanceId,
case when sum(canWrite) = 0 then 0 else 1 end as canWrite
from UserGroupAllPermissions as ugap
where ugap.permissionId = ?
and not exists (
select 1 from UserGroupInstancesCache as ugi
where ugi.instanceId = ugap.instanceId
and ugi.userGroupId = ugap.userGroupId )
group by userGroupId, instanceId
""", permissionId)
self.latest.updateRoleId(cu, roleId)
return True
def fix(self):
db = self.getDB()
cu = db.cursor()
log.info("removing troveinfo records for non-prsent troves...")
schema.resetTable(cu, "tmpId")
cu.execute(""" insert into tmpId(id)
select distinct instanceId from Instances join TroveInfo using(instanceId)
where Instances.isPresent = ? """, instances.INSTANCE_PRESENT_MISSING)
cu.execute("delete from TroveInfo where instanceId in (select id from tmpId)")
self.commit()
return self.check()
def fix(self):
db = self.getDB()
cu = db.cursor()
log.info("removing troveinfo records for non-prsent troves...")
schema.resetTable(cu, "tmpId")
cu.execute(
""" insert into tmpId(id)
select distinct instanceId from Instances join TroveInfo using(instanceId)
where Instances.isPresent = ? """,
instances.INSTANCE_PRESENT_MISSING)
cu.execute(
"delete from TroveInfo where instanceId in (select id from tmpId)")
self.commit()
return self.check()
def batchCheck(self, authToken, troveList, write=False, cu=None):
""" checks access permissions for a set of *existing* troves in the repository """
# troveTupList is a list of (name, VFS) tuples
self.log(3, authToken[0],
"entitlements=%s write=%s" % (authToken[2], int(bool(write))),
troveList)
# process/check the troveList, which can be an iterator
checkList = []
for i, (n, v, f) in enumerate(troveList):
h = versions.VersionFromString(v).getHost()
if h not in self.serverNameList:
raise errors.RepositoryMismatch(self.serverNameList, h)
checkList.append((i, n, v, f))
# default to all failing
retlist = [False] * len(checkList)
if not authToken[0]:
return retlist
# check groupIds
if cu is None:
cu = self.db.cursor()
try:
groupIds = self.getAuthRoles(cu, authToken)
except errors.InsufficientPermission:
return retlist
if not len(groupIds):
return retlist
resetTable(cu, "tmpNVF")
self.db.bulkload("tmpNVF",
checkList, ["idx", "name", "version", "flavor"],
start_transaction=False)
self.db.analyze("tmpNVF")
writeCheck = ''
if write:
writeCheck = "and ugi.canWrite = 1"
cu.execute("""
select t.idx, i.instanceId
from tmpNVF as t
join Items on t.name = Items.item
join Versions on t.version = Versions.version
join Flavors on t.flavor = Flavors.flavor
join Instances as i on
i.itemId = Items.itemId and
i.versionId = Versions.versionId and
i.flavorId = Flavors.flavorId
join UserGroupInstancesCache as ugi on i.instanceId = ugi.instanceId
where ugi.userGroupId in (%s)
%s""" % (",".join("%d" % x for x in groupIds), writeCheck))
for i, instanceId in cu:
retlist[i] = True
return retlist
def updatePermissionId(self, permissionId, roleId):
cu = self.db.cursor()
schema.resetTable(cu, "tmpInstances")
# figure out how the access is changing
cu.execute("""
insert into tmpInstances(instanceId)
select instanceId from UserGroupAllPermissions
where permissionId = ? """, permissionId, start_transaction=False)
# re-add
self.rp.deleteId(cu, permissionId = permissionId)
self.rp.addId(cu, permissionId = permissionId)
# remove from consideration troves for which we still have access
cu.execute("""
delete from tmpInstances
where exists (
select 1 from UserGroupAllPermissions as ugap
where ugap.userGroupId = :roleId
and ugap.instanceId = tmpInstances.instanceId )
or exists (
select 1 from UserGroupAllTroves as ugat
where ugat.userGroupId = :roleId
and ugat.instanceId = tmpInstances.instanceId )
""", roleId=roleId, start_transaction=False)
self.db.analyze("tmpInstances")
# remove trove access from troves that are left
cu.execute("""
delete from UserGroupInstancesCache
where userGroupId = :roleId
and instanceId in (select instanceId from tmpInstances)
and not exists (
select 1 from UserGroupAllTroves as ugat
where ugat.userGroupId = :roleId
and ugat.instanceId = UserGroupInstancesCache.instanceId )
""", roleId=roleId)
# add the new troves now
cu.execute("""
insert into UserGroupInstancesCache(userGroupId, instanceId, canWrite)
select userGroupId, instanceId,
case when sum(canWrite) = 0 then 0 else 1 end as canWrite
from UserGroupAllPermissions as ugap
where ugap.permissionId = ?
and not exists (
select 1 from UserGroupInstancesCache as ugi
where ugi.instanceId = ugap.instanceId
and ugi.userGroupId = ugap.userGroupId )
group by userGroupId, instanceId
""", permissionId)
self.latest.updateRoleId(cu, roleId)
return True
def batchCheck(self, authToken, troveList, write = False, cu = None):
""" checks access permissions for a set of *existing* troves in the repository """
# troveTupList is a list of (name, VFS) tuples
self.log(3, authToken[0], "entitlements=%s write=%s" %(authToken[2], int(bool(write))),
troveList)
# process/check the troveList, which can be an iterator
checkList = []
for i, (n,v,f) in enumerate(troveList):
h = versions.VersionFromString(v).getHost()
if h not in self.serverNameList:
raise errors.RepositoryMismatch(self.serverNameList, h)
checkList.append((i,n,v,f))
# default to all failing
retlist = [ False ] * len(checkList)
if not authToken[0]:
return retlist
# check groupIds
if cu is None:
cu = self.db.cursor()
try:
groupIds = self.getAuthRoles(cu, authToken)
except errors.InsufficientPermission:
return retlist
if not len(groupIds):
return retlist
resetTable(cu, "tmpNVF")
self.db.bulkload("tmpNVF", checkList, ["idx","name","version", "flavor"],
start_transaction=False)
self.db.analyze("tmpNVF")
writeCheck = ''
if write:
writeCheck = "and ugi.canWrite = 1"
cu.execute("""
select t.idx, i.instanceId
from tmpNVF as t
join Items on t.name = Items.item
join Versions on t.version = Versions.version
join Flavors on t.flavor = Flavors.flavor
join Instances as i on
i.itemId = Items.itemId and
i.versionId = Versions.versionId and
i.flavorId = Flavors.flavorId
join UserGroupInstancesCache as ugi on i.instanceId = ugi.instanceId
where ugi.userGroupId in (%s)
%s""" % (",".join("%d" % x for x in groupIds), writeCheck) )
for i, instanceId in cu:
retlist[i] = True
return retlist
def delete(self, roleId, troveList):
"""remove group access to troves passed in the (n,v,f) troveList"""
self._findInstanceIds(troveList, checkMissing=False)
cu = self.db.cursor()
schema.resetTable(cu, "tmpId")
cu.execute("""
insert into tmpId(id)
select ugtId from UserGroupTroves
where userGroupId = ?
and instanceId in (select instanceId from tmpInstanceId)
""",
roleId,
start_transaction=False)
self.db.analyze("tmpId")
# save what instanceIds will be affected by this delete
schema.resetTable(cu, "tmpInstances")
cu.execute("""
insert into tmpInstances(instanceId)
select distinct ugat.instanceId
from UserGroupAllTroves as ugat
where ugat.userGroupId = ?
and ugat.ugtId in (select id from tmpId)
""",
roleId,
start_transaction=False)
cu.execute(
"delete from UserGroupAllTroves where ugtId in (select id from tmpId)"
)
cu.execute(
"delete from UserGroupTroves where ugtId in (select id from tmpId)"
)
# filter out the ones that are still allowed based on other permissions
cu.execute("""
delete from tmpInstances
where exists (
select 1 from UserGroupAllTroves as ugat
where userGroupId = ?
and ugat.instanceId = tmpInstances.instanceId )
""",
roleId,
start_transaction=False)
return True
def _findInstanceIds(self, troveList, checkMissing=True):
cu = self.db.cursor()
schema.resetTable(cu, "tmpNVF")
schema.resetTable(cu, "tmpInstanceId")
for (n, v, f) in troveList:
cu.execute(
"insert into tmpNVF (name, version, flavor) "
"values (?,?,?)", (n, v, f),
start_transaction=False)
self.db.analyze("tmpNVF")
cu.execute("""
insert into tmpInstanceId (idx, instanceId)
select tmpNVF.idx, Instances.instanceId
from tmpNVF
join Items on tmpNVF.name = Items.item
join Versions on tmpNVF.version = Versions.version
join Flavors on tmpNVF.flavor = Flavors.flavor
join Instances on
Instances.itemId = Items.itemId and
Instances.versionId = Versions.versionId and
Instances.flavorId = Flavors.flavorId
where
Instances.isPresent in (%d,%d)
""" % (instances.INSTANCE_PRESENT_NORMAL,
instances.INSTANCE_PRESENT_HIDDEN),
start_transaction=False)
self.db.analyze("tmpInstances")
# check if any troves specified are missing
cu.execute("""
select tmpNVF.idx, name, version, flavor
from tmpNVF
left join tmpInstanceId using(idx)
where tmpInstanceId.instanceId is NULL
""")
if checkMissing:
# granting permissions to a !present trove has a fuzzy meaning
for i, n, v, f in cu.fetchall():
raise errors.TroveMissing(n, v)
return True
def _findInstanceIds(self, troveList, checkMissing=True):
cu = self.db.cursor()
schema.resetTable(cu, "tmpNVF")
schema.resetTable(cu, "tmpInstanceId")
for (n,v,f) in troveList:
cu.execute("insert into tmpNVF (name, version, flavor) "
"values (?,?,?)", (n,v,f), start_transaction=False)
self.db.analyze("tmpNVF")
cu.execute("""
insert into tmpInstanceId (idx, instanceId)
select tmpNVF.idx, Instances.instanceId
from tmpNVF
join Items on tmpNVF.name = Items.item
join Versions on tmpNVF.version = Versions.version
join Flavors on tmpNVF.flavor = Flavors.flavor
join Instances on
Instances.itemId = Items.itemId and
Instances.versionId = Versions.versionId and
Instances.flavorId = Flavors.flavorId
where
Instances.isPresent in (%d,%d)
""" % (instances.INSTANCE_PRESENT_NORMAL,
instances.INSTANCE_PRESENT_HIDDEN),
start_transaction=False)
self.db.analyze("tmpInstances")
# check if any troves specified are missing
cu.execute("""
select tmpNVF.idx, name, version, flavor
from tmpNVF
left join tmpInstanceId using(idx)
where tmpInstanceId.instanceId is NULL
""")
if checkMissing:
# granting permissions to a !present trove has a fuzzy meaning
for i, n, v, f in cu.fetchall():
raise errors.TroveMissing(n,v)
return True
def _setupDepSets(self, cu, depSetList, reset=True):
if reset:
schema.resetTable(cu, "tmpDeps")
schema.resetTable(cu, "tmpDepNum")
# count how many dep classes are in each depSet
depNums = []
for i, depSet in enumerate(depSetList):
depNum = 0
for depClass, dep in depSet.iterDeps(sort=True):
# need to get these in sorted order as the depNums
# we get here are important...
classId = depClass.tag
depName = dep.getName()
flags = dep.getFlags()
for (depName, flags) in zip(dep.getName(), dep.getFlags()):
cu.execute(
"""
insert into tmpDeps(idx, depNum, class, name, flag)
values (?, ?, ?, ?, ?)""",
(i, depNum, classId, depName, NO_FLAG_MAGIC))
if flags:
for flag, sense in flags:
# assert sense is required
cu.execute(
"""
insert into tmpDeps(idx, depNum, class, name, flag)
values (?, ?, ?, ?, ?)""",
(i, depNum, classId, depName, flag))
cu.execute(
"""insert into tmpDepNum(idx, depNum, flagCount)
values (?, ?, ?)""", (i, depNum, len(flags) + 1))
depNum += 1
depNums.append(depNum)
self.db.analyze("tmpDeps")
self.db.analyze("tmpDepNum")
return depNums
def searchNodes(cu, roleIds, label = None, mkUrl = None, filterSet = None,
db = None, name = None, latest = 1):
args = []
d = { 'labelCheck' : '', 'itemCheck' : '' }
d['roleIds'] = ",".join( str(x) for x in roleIds)
d['SOURCENAME'] = trove._TROVEINFO_TAG_SOURCENAME
d['METADATA'] = trove._TROVEINFO_TAG_METADATA
if label:
d['labelCheck'] = "label = ? AND"
args.append(label)
if name:
d['itemCheck'] = "item = ? AND"
args.append(name)
if latest:
cu.execute("""
SELECT idTable.item, version, ts, finalTs, SourceNameTroveInfo.data,
MetadataTroveInfo.data FROM
(SELECT DISTINCT Items.item AS item,
Nodes.versionId AS versionId,
Nodes.timeStamps AS ts,
Nodes.finalTimeStamp as finalTs,
MIN(Instances.instanceId) AS instanceId
FROM Labels
JOIN LabelMap USING (labelId)
JOIN LatestCache USING (itemId, branchId)
JOIN Nodes USING (itemId, versionId)
JOIN Instances USING (itemId, versionId)
JOIN Items USING (itemId)
WHERE %(labelCheck)s
%(itemCheck)s
LatestCache.latestType = 1 AND
LatestCache.userGroupId in (%(roleIds)s)
GROUP BY
Items.item, Nodes.versionId, Nodes.timeStamps,
Nodes.finalTimestamp)
AS idTable
JOIN Versions ON (idTable.versionId = Versions.versionId)
LEFT OUTER JOIN TroveInfo AS SourceNameTroveInfo ON
idTable.instanceId = SourceNameTroveInfo.instanceId AND
SourceNameTroveInfo.infoType = %(SOURCENAME)d
LEFT OUTER JOIN TroveInfo AS MetadataTroveInfo ON
idTable.instanceId = MetadataTroveInfo.instanceId AND
MetadataTroveInfo.infoType = %(METADATA)d
""" % d, args)
else:
cu.execute("""
SELECT idTable.item, version, ts, finalTs, SourceNameTroveInfo.data,
MetadataTroveInfo.data FROM
(SELECT DISTINCT Items.item AS item,
Nodes.versionId AS versionId,
Nodes.timeStamps AS ts,
Nodes.finalTimeStamp as finalTs,
MIN(Instances.instanceId) AS instanceId
FROM Labels
JOIN LabelMap USING (labelId)
JOIN Nodes USING (itemId, branchId)
JOIN Instances USING (itemId, versionId)
JOIN Items USING (itemId)
JOIN usergroupinstancescache AS ugi ON
Instances.instanceId = ugi.instanceId
WHERE %(labelCheck)s
%(itemCheck)s
ugi.userGroupId in (%(roleIds)s)
GROUP BY
Items.item, Nodes.versionId, Nodes.timeStamps,
Nodes.finalTimestamp)
AS idTable
JOIN Versions ON (idTable.versionId = Versions.versionId)
LEFT OUTER JOIN TroveInfo AS SourceNameTroveInfo ON
idTable.instanceId = SourceNameTroveInfo.instanceId AND
SourceNameTroveInfo.infoType = %(SOURCENAME)d
LEFT OUTER JOIN TroveInfo AS MetadataTroveInfo ON
idTable.instanceId = MetadataTroveInfo.instanceId AND
MetadataTroveInfo.infoType = %(METADATA)d
""" % d, args)
l = list(cu)
filteredL = typeFilter(l, filterSet)
# sort based on (name, version, desc(finalTimestamp))
def sortorder(x, y):
c = cmp(x[0], y[0])
if c:
return c
return -(cmp(x[3], y[3]))
filteredL.sort(sortorder)
if latest:
# keep the latest
newL = []
last = None
for item in filteredL:
if last and last[0] == item[0]:
continue
newL.append(item)
last = item
filteredL = newL
nodeList = datamodel.NamedNodeList(total = len(filteredL), start = 0)
addList = []
for (name, version, ts, finalTs, sourceName, metadata) in filteredL:
sourceName = cu.frombinary(sourceName)
if sourceName is None and trove.troveIsSourceComponent(name):
sourceName = name
addList.append((sourceName,
str(versions.VersionFromString(version).getSourceVersion())))
schema.resetTable(cu, 'tmpNVF')
# This is painful, but it converts the source name from a blob to
# a string
db.bulkload("tmpNVF", [ (x[0],) + x[1] for x in enumerate(addList) ],
["idx", "name", "version"],
start_transaction = False)
cu.execute("""
SELECT ChangeLogs.name, ChangeLogs.message, tmpNVF.name
FROM tmpNVF JOIN Items AS SourceItems ON
tmpNVF.name = SourceItems.item
LEFT OUTER JOIN Versions AS SourceVersion ON
tmpNVF.version = SourceVersion.version
LEFT OUTER JOIN Nodes ON
SourceItems.itemId = Nodes.itemId AND
SourceVersion.versionId = Nodes.versionId
LEFT OUTER JOIN ChangeLogs USING (nodeId)
ORDER BY tmpNVF.idx
""")
for ( (name, version, ts, finalTs, sourceName, metadata),
(clName, clMessage, troveName) ) in itertools.izip(filteredL, cu):
frzVer = versions.strToFrozen(version,
[ x for x in ts.split(":") ])
ver = versions.ThawVersion(frzVer)
shortdesc = None
if metadata:
metadata = cu.frombinary(metadata)
md = trove.Metadata(metadata)
shortdesc = md.get()['shortDesc']
if clName:
cl = datamodel.ChangeLog(name = clName, message = clMessage)
else:
cl = None
nodeList.append(name = name, version = ver, mkUrl = mkUrl,
changeLog = cl, shortdesc = shortdesc)
return nodeList
def searchNodes(cu, roleIds, label = None, mkUrl = None, filterSet = None,
db = None, name = None, latest = 1):
args = []
d = { 'labelCheck' : '', 'itemCheck' : '' }
d['roleIds'] = ",".join( str(x) for x in roleIds)
d['SOURCENAME'] = trove._TROVEINFO_TAG_SOURCENAME
d['METADATA'] = trove._TROVEINFO_TAG_METADATA
if label:
d['labelCheck'] = "label = ? AND"
args.append(label)
if name:
d['itemCheck'] = "item = ? AND"
args.append(name)
if latest:
cu.execute("""
SELECT idTable.item, version, ts, finalTs, SourceNameTroveInfo.data,
MetadataTroveInfo.data FROM
(SELECT DISTINCT Items.item AS item,
Nodes.versionId AS versionId,
Nodes.timeStamps AS ts,
Nodes.finalTimeStamp as finalTs,
MIN(Instances.instanceId) AS instanceId
FROM Labels
JOIN LabelMap USING (labelId)
JOIN LatestCache USING (itemId, branchId)
JOIN Nodes USING (itemId, versionId)
JOIN Instances USING (itemId, versionId)
JOIN Items USING (itemId)
WHERE %(labelCheck)s
%(itemCheck)s
LatestCache.latestType = 1 AND
LatestCache.userGroupId in (%(roleIds)s)
GROUP BY
Items.item, Nodes.versionId, Nodes.timeStamps,
Nodes.finalTimestamp)
AS idTable
JOIN Versions ON (idTable.versionId = Versions.versionId)
LEFT OUTER JOIN TroveInfo AS SourceNameTroveInfo ON
idTable.instanceId = SourceNameTroveInfo.instanceId AND
SourceNameTroveInfo.infoType = %(SOURCENAME)d
LEFT OUTER JOIN TroveInfo AS MetadataTroveInfo ON
idTable.instanceId = MetadataTroveInfo.instanceId AND
MetadataTroveInfo.infoType = %(METADATA)d
""" % d, args)
else:
cu.execute("""
SELECT idTable.item, version, ts, finalTs, SourceNameTroveInfo.data,
MetadataTroveInfo.data FROM
(SELECT DISTINCT Items.item AS item,
Nodes.versionId AS versionId,
Nodes.timeStamps AS ts,
Nodes.finalTimeStamp as finalTs,
MIN(Instances.instanceId) AS instanceId
FROM Labels
JOIN LabelMap USING (labelId)
JOIN Nodes USING (itemId, branchId)
JOIN Instances USING (itemId, versionId)
JOIN Items USING (itemId)
JOIN usergroupinstancescache AS ugi ON
Instances.instanceId = ugi.instanceId
WHERE %(labelCheck)s
%(itemCheck)s
ugi.userGroupId in (%(roleIds)s)
GROUP BY
Items.item, Nodes.versionId, Nodes.timeStamps,
Nodes.finalTimestamp)
AS idTable
JOIN Versions ON (idTable.versionId = Versions.versionId)
LEFT OUTER JOIN TroveInfo AS SourceNameTroveInfo ON
idTable.instanceId = SourceNameTroveInfo.instanceId AND
SourceNameTroveInfo.infoType = %(SOURCENAME)d
LEFT OUTER JOIN TroveInfo AS MetadataTroveInfo ON
idTable.instanceId = MetadataTroveInfo.instanceId AND
MetadataTroveInfo.infoType = %(METADATA)d
""" % d, args)
l = list(cu)
filteredL = typeFilter(l, filterSet)
# sort based on (name, version, desc(finalTimestamp))
def sortorder(x, y):
c = cmp(x[0], y[0])
if c:
return c
return -(cmp(x[3], y[3]))
filteredL.sort(sortorder)
if latest:
# keep the latest
newL = []
last = None
for item in filteredL:
if last and last[0] == item[0]:
continue
newL.append(item)
last = item
filteredL = newL
nodeList = datamodel.NamedNodeList(total = len(filteredL), start = 0)
addList = []
for (name, version, ts, finalTs, sourceName, metadata) in filteredL:
sourceName = cu.frombinary(sourceName)
if sourceName is None and trove.troveIsSourceComponent(name):
sourceName = name
addList.append((sourceName,
str(versions.VersionFromString(version).getSourceVersion())))
schema.resetTable(cu, 'tmpNVF')
# This is painful, but it converts the source name from a blob to
# a string
db.bulkload("tmpNVF", [ (x[0],) + x[1] for x in enumerate(addList) ],
["idx", "name", "version"],
start_transaction = False)
cu.execute("""
SELECT ChangeLogs.name, ChangeLogs.message, tmpNVF.name
FROM tmpNVF JOIN Items AS SourceItems ON
tmpNVF.name = SourceItems.item
LEFT OUTER JOIN Versions AS SourceVersion ON
tmpNVF.version = SourceVersion.version
LEFT OUTER JOIN Nodes ON
SourceItems.itemId = Nodes.itemId AND
SourceVersion.versionId = Nodes.versionId
LEFT OUTER JOIN ChangeLogs USING (nodeId)
ORDER BY tmpNVF.idx
""")
for ( (name, version, ts, finalTs, sourceName, metadata),
(clName, clMessage, troveName) ) in itertools.izip(filteredL, cu):
frzVer = versions.strToFrozen(version,
[ x for x in ts.split(":") ])
ver = versions.ThawVersion(frzVer)
shortdesc = None
if metadata:
metadata = cu.frombinary(metadata)
md = trove.Metadata(metadata)
shortdesc = md.get()['shortDesc']
if clName:
cl = datamodel.ChangeLog(name = clName, message = clMessage)
else:
cl = None
nodeList.append(name = name, version = ver, mkUrl = mkUrl,
changeLog = cl, shortdesc = shortdesc)
return nodeList
def expandJobList(db, chgSetList, recurse):
"""
For each job in the list, find the set of troves which are recursively
included in it. The reutnr value is list parallel to chgSetList, each
item of which is a sorted list of those troves which are included in the
recursive changeset.
"""
# We mark old groups (ones without weak references) as uncachable
# because they're expensive to flatten (and so old that it
# hardly matters).
if not recurse:
return [ [ job ] for job in chgSetList ]
cu = db.cursor()
schema.resetTable(cu, "tmpNVF")
foundGroups = set()
foundWeak = set()
foundCollections = set()
insertList = []
for jobId, job in enumerate(chgSetList):
if trove.troveIsGroup(job[0]):
foundGroups.add(jobId)
insertList.append((jobId, job[0], job[2][0], job[2][1]))
db.bulkload("tmpNvf", insertList,
[ "idx", "name", "version", "flavor" ],
start_transaction = False)
db.analyze("tmpNVF")
newJobList = [ [ job ] for job in chgSetList ]
cu.execute("""SELECT
tmpNVF.idx, I_Items.item, I_Versions.version,
I_Flavors.flavor, TroveTroves.flags
FROM tmpNVF JOIN Items ON tmpNVF.name = Items.item
JOIN Versions ON (tmpNVF.version = Versions.version)
JOIN Flavors ON (tmpNVF.flavor = Flavors.flavor)
JOIN Instances ON
Items.itemId = Instances.itemId AND
Versions.versionId = Instances.versionId AND
Flavors.flavorId = Instances.flavorId
JOIN TroveTroves USING (instanceId)
JOIN Instances AS I_Instances ON
TroveTroves.includedId = I_Instances.instanceId
JOIN Items AS I_Items ON
I_Instances.itemId = I_Items.itemId
JOIN Versions AS I_Versions ON
I_Instances.versionId = I_Versions.versionId
JOIN Flavors AS I_Flavors ON
I_Instances.flavorId = I_Flavors.flavorId
WHERE
I_Instances.isPresent = 1
ORDER BY
I_Items.item, I_Versions.version, I_Flavors.flavor
""")
for (idx, name, version, flavor, flags) in cu:
newJobList[idx].append( (name, (None, None),
(version, flavor), True) )
if flags & schema.TROVE_TROVES_WEAKREF > 0:
foundWeak.add(idx)
if trove.troveIsCollection(name):
foundCollections.add(idx)
for idx in ((foundGroups & foundCollections) - foundWeak):
# groups which contain collections but no weak refs
# are uncachable
newJobList[idx] = None
return newJobList
