def _setupDepSets(self, cu, depSetList, reset=True): if reset: schema.resetTable(cu, "tmpDeps") schema.resetTable(cu, "tmpDepNum") # count how many dep classes are in each depSet depNums = [] for i, depSet in enumerate(depSetList): depNum = 0 for depClass, dep in depSet.iterDeps(sort=True): # need to get these in sorted order as the depNums # we get here are important... classId = depClass.tag depName = dep.getName() flags = dep.getFlags() for (depName, flags) in zip(dep.getName(), dep.getFlags()): cu.execute(""" insert into tmpDeps(idx, depNum, class, name, flag) values (?, ?, ?, ?, ?)""", (i, depNum, classId, depName, NO_FLAG_MAGIC)) if flags: for flag, sense in flags: # assert sense is required cu.execute(""" insert into tmpDeps(idx, depNum, class, name, flag) values (?, ?, ?, ?, ?)""", (i, depNum, classId, depName, flag)) cu.execute("""insert into tmpDepNum(idx, depNum, flagCount) values (?, ?, ?)""", (i, depNum, len(flags)+1)) depNum += 1 depNums.append(depNum) self.db.analyze("tmpDeps") self.db.analyze("tmpDepNum") return depNums
def _setupTroveList(self, cu, troveList): if not troveList: return schema.resetTable(cu, "tmpInstances") schema.resetTable(cu, "tmpId") cu.executemany(""" insert into tmpInstances(instanceId) select instanceId from Instances join Items on Instances.itemId = Items.itemId JOIN versions on Instances.versionId = Versions.versionId join Flavors on Instances.flavorId = Flavors.flavorId where Items.item = ? and Versions.version = ? and Flavors.flavor = ? """, troveList, start_transaction=False ) self.db.analyze("tmpInstances") # now grab the instanceIds of their included troves, avoiding duplicates cu.execute(""" insert into tmpId(id) select distinct tt.includedId from tmpInstances as ti join TroveTroves as tt using(instanceId) """, start_transaction=False) # drop the ones we already have cu.execute("delete from tmpId where id in " "(select instanceId from tmpInstances)", start_transaction=False) # append the remaining instanceIds cu.execute("insert into tmpInstances(instanceId) select id from tmpId", start_transaction=False) self.db.analyze("tmpInstances")
def deletePermissionId(self, permissionId, roleId): cu = self.db.cursor() # compute the list of troves for which no other RAP/RAT access exists schema.resetTable(cu, "tmpInstances") cu.execute(""" insert into tmpInstances(instanceId) select ugi.instanceId from UserGroupInstancesCache as ugi where ugi.userGroupId = ? and not exists ( select 1 from UserGroupAllPermissions as ugap where ugap.userGroupId = ? and ugap.instanceId = ugi.instanceId and ugap.permissionId != ? ) and not exists ( select 1 from UserGroupAllTroves as ugat where ugat.instanceId = ugi.instanceId and ugat.userGroupId = ? )""", (roleId, roleId, permissionId, roleId), start_transaction=False) # clean up the flattened table cu.execute( "delete from UserGroupAllPermissions where permissionId = ?", permissionId) # now we have only the troves which need to be erased out of UGIC self.db.analyze("tmpInstances") cu.execute( """ delete from UserGroupInstancesCache where userGroupId = ? and instanceId in (select instanceId from tmpInstances)""", roleId) # update Latest self.latest.updateRoleId(cu, roleId, tmpInstances=True)
def _setupTroveList(self, cu, troveList): if not troveList: return schema.resetTable(cu, "tmpInstances") schema.resetTable(cu, "tmpId") cu.executemany(""" insert into tmpInstances(instanceId) select instanceId from Instances join Items on Instances.itemId = Items.itemId JOIN versions on Instances.versionId = Versions.versionId join Flavors on Instances.flavorId = Flavors.flavorId where Items.item = ? and Versions.version = ? and Flavors.flavor = ? """, troveList, start_transaction=False) self.db.analyze("tmpInstances") # now grab the instanceIds of their included troves, avoiding duplicates cu.execute(""" insert into tmpId(id) select distinct tt.includedId from tmpInstances as ti join TroveTroves as tt using(instanceId) """, start_transaction=False) # drop the ones we already have cu.execute( "delete from tmpId where id in " "(select instanceId from tmpInstances)", start_transaction=False) # append the remaining instanceIds cu.execute("insert into tmpInstances(instanceId) select id from tmpId", start_transaction=False) self.db.analyze("tmpInstances")
def addTroveAccess(self, role, troveList, recursive=True): roleId = self._getRoleId(role) rtList = self.rt.add(roleId, troveList, recursive) # we now know the ids of the new acls added. They're useful in # updating the UGIC table cu = self.db.cursor() # grab the list of instanceIds we are adding to the UGIC table; # we need those for a faster recomputation of the LatestCache table schema.resetTable(cu, "tmpInstances") cu.execute(""" insert into tmpInstances(instanceId) select distinct ugat.instanceId from UserGroupAllTroves as ugat where ugat.ugtId in (%s) and ugat.userGroupId = ? and not exists ( select 1 from UserGroupInstancesCache as ugi where ugi.userGroupId = ? and ugi.instanceId = ugat.instanceId ) """ % (",".join("%d" % x for x in rtList),), (roleId, roleId), start_transaction=False) # insert into UGIC and recompute the latest table cu.execute(""" insert into UserGroupInstancesCache (userGroupId, instanceId) select %d, instanceId from tmpInstances """ %(roleId,)) # tmpInstances has instanceIds for which Latest needs to be recomputed self.db.analyze("tmpInstances") self.latest.updateRoleId(cu, roleId, tmpInstances=True)
def addPermissionId(self, permissionId, roleId): cu = self.db.cursor() self.rp.addId(cu, permissionId=permissionId) # figure out newly accessible troves. We keep track separately # to speed up the Latest update schema.resetTable(cu, "tmpInstances") cu.execute(""" insert into tmpInstances(instanceId) select instanceId from UserGroupAllPermissions as ugap where permissionId = ? and not exists ( select instanceId from UserGroupInstancesCache as ugi where ugi.userGroupId = ? and ugi.instanceId = ugap.instanceId ) """, (permissionId, roleId), start_transaction=False) # update UsergroupInstancesCache cu.execute( """ insert into UserGroupInstancesCache (userGroupId, instanceId, canWrite) select userGroupId, instanceId, case when sum(canWrite) = 0 then 0 else 1 end as canWrite from UserGroupAllPermissions where permissionId = ? and instanceId in (select instanceId from tmpInstances) group by userGroupId, instanceId """, permissionId) # update Latest self.latest.updateRoleId(cu, roleId, tmpInstances=True)
def addTroveAccess(self, role, troveList, recursive=True): roleId = self._getRoleId(role) rtList = self.rt.add(roleId, troveList, recursive) # we now know the ids of the new acls added. They're useful in # updating the UGIC table cu = self.db.cursor() # grab the list of instanceIds we are adding to the UGIC table; # we need those for a faster recomputation of the LatestCache table schema.resetTable(cu, "tmpInstances") cu.execute(""" insert into tmpInstances(instanceId) select distinct ugat.instanceId from UserGroupAllTroves as ugat where ugat.ugtId in (%s) and ugat.userGroupId = ? and not exists ( select 1 from UserGroupInstancesCache as ugi where ugi.userGroupId = ? and ugi.instanceId = ugat.instanceId ) """ % (",".join("%d" % x for x in rtList), ), (roleId, roleId), start_transaction=False) # insert into UGIC and recompute the latest table cu.execute(""" insert into UserGroupInstancesCache (userGroupId, instanceId) select %d, instanceId from tmpInstances """ % (roleId, )) # tmpInstances has instanceIds for which Latest needs to be recomputed self.db.analyze("tmpInstances") self.latest.updateRoleId(cu, roleId, tmpInstances=True)
def addPermissionId(self, permissionId, roleId): cu = self.db.cursor() self.rp.addId(cu, permissionId = permissionId) # figure out newly accessible troves. We keep track separately # to speed up the Latest update schema.resetTable(cu, "tmpInstances") cu.execute(""" insert into tmpInstances(instanceId) select instanceId from UserGroupAllPermissions as ugap where permissionId = ? and not exists ( select instanceId from UserGroupInstancesCache as ugi where ugi.userGroupId = ? and ugi.instanceId = ugap.instanceId ) """, (permissionId, roleId), start_transaction = False) # update UsergroupInstancesCache cu.execute(""" insert into UserGroupInstancesCache (userGroupId, instanceId, canWrite) select userGroupId, instanceId, case when sum(canWrite) = 0 then 0 else 1 end as canWrite from UserGroupAllPermissions where permissionId = ? and instanceId in (select instanceId from tmpInstances) group by userGroupId, instanceId """, permissionId) # update Latest self.latest.updateRoleId(cu, roleId, tmpInstances=True)
def delete(self, roleId, troveList): """remove group access to troves passed in the (n,v,f) troveList""" self._findInstanceIds(troveList, checkMissing=False) cu = self.db.cursor() schema.resetTable(cu, "tmpId") cu.execute(""" insert into tmpId(id) select ugtId from UserGroupTroves where userGroupId = ? and instanceId in (select instanceId from tmpInstanceId) """, roleId, start_transaction=False) self.db.analyze("tmpId") # save what instanceIds will be affected by this delete schema.resetTable(cu, "tmpInstances") cu.execute(""" insert into tmpInstances(instanceId) select distinct ugat.instanceId from UserGroupAllTroves as ugat where ugat.userGroupId = ? and ugat.ugtId in (select id from tmpId) """, roleId, start_transaction = False) cu.execute("delete from UserGroupAllTroves where ugtId in (select id from tmpId)") cu.execute("delete from UserGroupTroves where ugtId in (select id from tmpId)") # filter out the ones that are still allowed based on other permissions cu.execute(""" delete from tmpInstances where exists ( select 1 from UserGroupAllTroves as ugat where userGroupId = ? and ugat.instanceId = tmpInstances.instanceId ) """, roleId, start_transaction=False) return True
def deletePermissionId(self, permissionId, roleId): cu = self.db.cursor() # compute the list of troves for which no other RAP/RAT access exists schema.resetTable(cu, "tmpInstances") cu.execute(""" insert into tmpInstances(instanceId) select ugi.instanceId from UserGroupInstancesCache as ugi where ugi.userGroupId = ? and not exists ( select 1 from UserGroupAllPermissions as ugap where ugap.userGroupId = ? and ugap.instanceId = ugi.instanceId and ugap.permissionId != ? ) and not exists ( select 1 from UserGroupAllTroves as ugat where ugat.instanceId = ugi.instanceId and ugat.userGroupId = ? )""", (roleId, roleId, permissionId, roleId), start_transaction = False) # clean up the flattened table cu.execute("delete from UserGroupAllPermissions where permissionId = ?", permissionId) # now we have only the troves which need to be erased out of UGIC self.db.analyze("tmpInstances") cu.execute(""" delete from UserGroupInstancesCache where userGroupId = ? and instanceId in (select instanceId from tmpInstances)""", roleId) # update Latest self.latest.updateRoleId(cu, roleId, tmpInstances=True)
def updatePermissionId(self, permissionId, roleId): cu = self.db.cursor() schema.resetTable(cu, "tmpInstances") # figure out how the access is changing cu.execute(""" insert into tmpInstances(instanceId) select instanceId from UserGroupAllPermissions where permissionId = ? """, permissionId, start_transaction=False) # re-add self.rp.deleteId(cu, permissionId=permissionId) self.rp.addId(cu, permissionId=permissionId) # remove from consideration troves for which we still have access cu.execute(""" delete from tmpInstances where exists ( select 1 from UserGroupAllPermissions as ugap where ugap.userGroupId = :roleId and ugap.instanceId = tmpInstances.instanceId ) or exists ( select 1 from UserGroupAllTroves as ugat where ugat.userGroupId = :roleId and ugat.instanceId = tmpInstances.instanceId ) """, roleId=roleId, start_transaction=False) self.db.analyze("tmpInstances") # remove trove access from troves that are left cu.execute(""" delete from UserGroupInstancesCache where userGroupId = :roleId and instanceId in (select instanceId from tmpInstances) and not exists ( select 1 from UserGroupAllTroves as ugat where ugat.userGroupId = :roleId and ugat.instanceId = UserGroupInstancesCache.instanceId ) """, roleId=roleId) # add the new troves now cu.execute( """ insert into UserGroupInstancesCache(userGroupId, instanceId, canWrite) select userGroupId, instanceId, case when sum(canWrite) = 0 then 0 else 1 end as canWrite from UserGroupAllPermissions as ugap where ugap.permissionId = ? and not exists ( select 1 from UserGroupInstancesCache as ugi where ugi.instanceId = ugap.instanceId and ugi.userGroupId = ugap.userGroupId ) group by userGroupId, instanceId """, permissionId) self.latest.updateRoleId(cu, roleId) return True
def fix(self): db = self.getDB() cu = db.cursor() log.info("removing troveinfo records for non-prsent troves...") schema.resetTable(cu, "tmpId") cu.execute(""" insert into tmpId(id) select distinct instanceId from Instances join TroveInfo using(instanceId) where Instances.isPresent = ? """, instances.INSTANCE_PRESENT_MISSING) cu.execute("delete from TroveInfo where instanceId in (select id from tmpId)") self.commit() return self.check()
def fix(self): db = self.getDB() cu = db.cursor() log.info("removing troveinfo records for non-prsent troves...") schema.resetTable(cu, "tmpId") cu.execute( """ insert into tmpId(id) select distinct instanceId from Instances join TroveInfo using(instanceId) where Instances.isPresent = ? """, instances.INSTANCE_PRESENT_MISSING) cu.execute( "delete from TroveInfo where instanceId in (select id from tmpId)") self.commit() return self.check()
def batchCheck(self, authToken, troveList, write=False, cu=None): """ checks access permissions for a set of *existing* troves in the repository """ # troveTupList is a list of (name, VFS) tuples self.log(3, authToken[0], "entitlements=%s write=%s" % (authToken[2], int(bool(write))), troveList) # process/check the troveList, which can be an iterator checkList = [] for i, (n, v, f) in enumerate(troveList): h = versions.VersionFromString(v).getHost() if h not in self.serverNameList: raise errors.RepositoryMismatch(self.serverNameList, h) checkList.append((i, n, v, f)) # default to all failing retlist = [False] * len(checkList) if not authToken[0]: return retlist # check groupIds if cu is None: cu = self.db.cursor() try: groupIds = self.getAuthRoles(cu, authToken) except errors.InsufficientPermission: return retlist if not len(groupIds): return retlist resetTable(cu, "tmpNVF") self.db.bulkload("tmpNVF", checkList, ["idx", "name", "version", "flavor"], start_transaction=False) self.db.analyze("tmpNVF") writeCheck = '' if write: writeCheck = "and ugi.canWrite = 1" cu.execute(""" select t.idx, i.instanceId from tmpNVF as t join Items on t.name = Items.item join Versions on t.version = Versions.version join Flavors on t.flavor = Flavors.flavor join Instances as i on i.itemId = Items.itemId and i.versionId = Versions.versionId and i.flavorId = Flavors.flavorId join UserGroupInstancesCache as ugi on i.instanceId = ugi.instanceId where ugi.userGroupId in (%s) %s""" % (",".join("%d" % x for x in groupIds), writeCheck)) for i, instanceId in cu: retlist[i] = True return retlist
def updatePermissionId(self, permissionId, roleId): cu = self.db.cursor() schema.resetTable(cu, "tmpInstances") # figure out how the access is changing cu.execute(""" insert into tmpInstances(instanceId) select instanceId from UserGroupAllPermissions where permissionId = ? """, permissionId, start_transaction=False) # re-add self.rp.deleteId(cu, permissionId = permissionId) self.rp.addId(cu, permissionId = permissionId) # remove from consideration troves for which we still have access cu.execute(""" delete from tmpInstances where exists ( select 1 from UserGroupAllPermissions as ugap where ugap.userGroupId = :roleId and ugap.instanceId = tmpInstances.instanceId ) or exists ( select 1 from UserGroupAllTroves as ugat where ugat.userGroupId = :roleId and ugat.instanceId = tmpInstances.instanceId ) """, roleId=roleId, start_transaction=False) self.db.analyze("tmpInstances") # remove trove access from troves that are left cu.execute(""" delete from UserGroupInstancesCache where userGroupId = :roleId and instanceId in (select instanceId from tmpInstances) and not exists ( select 1 from UserGroupAllTroves as ugat where ugat.userGroupId = :roleId and ugat.instanceId = UserGroupInstancesCache.instanceId ) """, roleId=roleId) # add the new troves now cu.execute(""" insert into UserGroupInstancesCache(userGroupId, instanceId, canWrite) select userGroupId, instanceId, case when sum(canWrite) = 0 then 0 else 1 end as canWrite from UserGroupAllPermissions as ugap where ugap.permissionId = ? and not exists ( select 1 from UserGroupInstancesCache as ugi where ugi.instanceId = ugap.instanceId and ugi.userGroupId = ugap.userGroupId ) group by userGroupId, instanceId """, permissionId) self.latest.updateRoleId(cu, roleId) return True
def batchCheck(self, authToken, troveList, write = False, cu = None): """ checks access permissions for a set of *existing* troves in the repository """ # troveTupList is a list of (name, VFS) tuples self.log(3, authToken[0], "entitlements=%s write=%s" %(authToken[2], int(bool(write))), troveList) # process/check the troveList, which can be an iterator checkList = [] for i, (n,v,f) in enumerate(troveList): h = versions.VersionFromString(v).getHost() if h not in self.serverNameList: raise errors.RepositoryMismatch(self.serverNameList, h) checkList.append((i,n,v,f)) # default to all failing retlist = [ False ] * len(checkList) if not authToken[0]: return retlist # check groupIds if cu is None: cu = self.db.cursor() try: groupIds = self.getAuthRoles(cu, authToken) except errors.InsufficientPermission: return retlist if not len(groupIds): return retlist resetTable(cu, "tmpNVF") self.db.bulkload("tmpNVF", checkList, ["idx","name","version", "flavor"], start_transaction=False) self.db.analyze("tmpNVF") writeCheck = '' if write: writeCheck = "and ugi.canWrite = 1" cu.execute(""" select t.idx, i.instanceId from tmpNVF as t join Items on t.name = Items.item join Versions on t.version = Versions.version join Flavors on t.flavor = Flavors.flavor join Instances as i on i.itemId = Items.itemId and i.versionId = Versions.versionId and i.flavorId = Flavors.flavorId join UserGroupInstancesCache as ugi on i.instanceId = ugi.instanceId where ugi.userGroupId in (%s) %s""" % (",".join("%d" % x for x in groupIds), writeCheck) ) for i, instanceId in cu: retlist[i] = True return retlist
def delete(self, roleId, troveList): """remove group access to troves passed in the (n,v,f) troveList""" self._findInstanceIds(troveList, checkMissing=False) cu = self.db.cursor() schema.resetTable(cu, "tmpId") cu.execute(""" insert into tmpId(id) select ugtId from UserGroupTroves where userGroupId = ? and instanceId in (select instanceId from tmpInstanceId) """, roleId, start_transaction=False) self.db.analyze("tmpId") # save what instanceIds will be affected by this delete schema.resetTable(cu, "tmpInstances") cu.execute(""" insert into tmpInstances(instanceId) select distinct ugat.instanceId from UserGroupAllTroves as ugat where ugat.userGroupId = ? and ugat.ugtId in (select id from tmpId) """, roleId, start_transaction=False) cu.execute( "delete from UserGroupAllTroves where ugtId in (select id from tmpId)" ) cu.execute( "delete from UserGroupTroves where ugtId in (select id from tmpId)" ) # filter out the ones that are still allowed based on other permissions cu.execute(""" delete from tmpInstances where exists ( select 1 from UserGroupAllTroves as ugat where userGroupId = ? and ugat.instanceId = tmpInstances.instanceId ) """, roleId, start_transaction=False) return True
def _findInstanceIds(self, troveList, checkMissing=True): cu = self.db.cursor() schema.resetTable(cu, "tmpNVF") schema.resetTable(cu, "tmpInstanceId") for (n, v, f) in troveList: cu.execute( "insert into tmpNVF (name, version, flavor) " "values (?,?,?)", (n, v, f), start_transaction=False) self.db.analyze("tmpNVF") cu.execute(""" insert into tmpInstanceId (idx, instanceId) select tmpNVF.idx, Instances.instanceId from tmpNVF join Items on tmpNVF.name = Items.item join Versions on tmpNVF.version = Versions.version join Flavors on tmpNVF.flavor = Flavors.flavor join Instances on Instances.itemId = Items.itemId and Instances.versionId = Versions.versionId and Instances.flavorId = Flavors.flavorId where Instances.isPresent in (%d,%d) """ % (instances.INSTANCE_PRESENT_NORMAL, instances.INSTANCE_PRESENT_HIDDEN), start_transaction=False) self.db.analyze("tmpInstances") # check if any troves specified are missing cu.execute(""" select tmpNVF.idx, name, version, flavor from tmpNVF left join tmpInstanceId using(idx) where tmpInstanceId.instanceId is NULL """) if checkMissing: # granting permissions to a !present trove has a fuzzy meaning for i, n, v, f in cu.fetchall(): raise errors.TroveMissing(n, v) return True
def _findInstanceIds(self, troveList, checkMissing=True): cu = self.db.cursor() schema.resetTable(cu, "tmpNVF") schema.resetTable(cu, "tmpInstanceId") for (n,v,f) in troveList: cu.execute("insert into tmpNVF (name, version, flavor) " "values (?,?,?)", (n,v,f), start_transaction=False) self.db.analyze("tmpNVF") cu.execute(""" insert into tmpInstanceId (idx, instanceId) select tmpNVF.idx, Instances.instanceId from tmpNVF join Items on tmpNVF.name = Items.item join Versions on tmpNVF.version = Versions.version join Flavors on tmpNVF.flavor = Flavors.flavor join Instances on Instances.itemId = Items.itemId and Instances.versionId = Versions.versionId and Instances.flavorId = Flavors.flavorId where Instances.isPresent in (%d,%d) """ % (instances.INSTANCE_PRESENT_NORMAL, instances.INSTANCE_PRESENT_HIDDEN), start_transaction=False) self.db.analyze("tmpInstances") # check if any troves specified are missing cu.execute(""" select tmpNVF.idx, name, version, flavor from tmpNVF left join tmpInstanceId using(idx) where tmpInstanceId.instanceId is NULL """) if checkMissing: # granting permissions to a !present trove has a fuzzy meaning for i, n, v, f in cu.fetchall(): raise errors.TroveMissing(n,v) return True
def _setupDepSets(self, cu, depSetList, reset=True): if reset: schema.resetTable(cu, "tmpDeps") schema.resetTable(cu, "tmpDepNum") # count how many dep classes are in each depSet depNums = [] for i, depSet in enumerate(depSetList): depNum = 0 for depClass, dep in depSet.iterDeps(sort=True): # need to get these in sorted order as the depNums # we get here are important... classId = depClass.tag depName = dep.getName() flags = dep.getFlags() for (depName, flags) in zip(dep.getName(), dep.getFlags()): cu.execute( """ insert into tmpDeps(idx, depNum, class, name, flag) values (?, ?, ?, ?, ?)""", (i, depNum, classId, depName, NO_FLAG_MAGIC)) if flags: for flag, sense in flags: # assert sense is required cu.execute( """ insert into tmpDeps(idx, depNum, class, name, flag) values (?, ?, ?, ?, ?)""", (i, depNum, classId, depName, flag)) cu.execute( """insert into tmpDepNum(idx, depNum, flagCount) values (?, ?, ?)""", (i, depNum, len(flags) + 1)) depNum += 1 depNums.append(depNum) self.db.analyze("tmpDeps") self.db.analyze("tmpDepNum") return depNums
def searchNodes(cu, roleIds, label = None, mkUrl = None, filterSet = None, db = None, name = None, latest = 1): args = [] d = { 'labelCheck' : '', 'itemCheck' : '' } d['roleIds'] = ",".join( str(x) for x in roleIds) d['SOURCENAME'] = trove._TROVEINFO_TAG_SOURCENAME d['METADATA'] = trove._TROVEINFO_TAG_METADATA if label: d['labelCheck'] = "label = ? AND" args.append(label) if name: d['itemCheck'] = "item = ? AND" args.append(name) if latest: cu.execute(""" SELECT idTable.item, version, ts, finalTs, SourceNameTroveInfo.data, MetadataTroveInfo.data FROM (SELECT DISTINCT Items.item AS item, Nodes.versionId AS versionId, Nodes.timeStamps AS ts, Nodes.finalTimeStamp as finalTs, MIN(Instances.instanceId) AS instanceId FROM Labels JOIN LabelMap USING (labelId) JOIN LatestCache USING (itemId, branchId) JOIN Nodes USING (itemId, versionId) JOIN Instances USING (itemId, versionId) JOIN Items USING (itemId) WHERE %(labelCheck)s %(itemCheck)s LatestCache.latestType = 1 AND LatestCache.userGroupId in (%(roleIds)s) GROUP BY Items.item, Nodes.versionId, Nodes.timeStamps, Nodes.finalTimestamp) AS idTable JOIN Versions ON (idTable.versionId = Versions.versionId) LEFT OUTER JOIN TroveInfo AS SourceNameTroveInfo ON idTable.instanceId = SourceNameTroveInfo.instanceId AND SourceNameTroveInfo.infoType = %(SOURCENAME)d LEFT OUTER JOIN TroveInfo AS MetadataTroveInfo ON idTable.instanceId = MetadataTroveInfo.instanceId AND MetadataTroveInfo.infoType = %(METADATA)d """ % d, args) else: cu.execute(""" SELECT idTable.item, version, ts, finalTs, SourceNameTroveInfo.data, MetadataTroveInfo.data FROM (SELECT DISTINCT Items.item AS item, Nodes.versionId AS versionId, Nodes.timeStamps AS ts, Nodes.finalTimeStamp as finalTs, MIN(Instances.instanceId) AS instanceId FROM Labels JOIN LabelMap USING (labelId) JOIN Nodes USING (itemId, branchId) JOIN Instances USING (itemId, versionId) JOIN Items USING (itemId) JOIN usergroupinstancescache AS ugi ON Instances.instanceId = ugi.instanceId WHERE %(labelCheck)s %(itemCheck)s ugi.userGroupId in (%(roleIds)s) GROUP BY Items.item, Nodes.versionId, Nodes.timeStamps, Nodes.finalTimestamp) AS idTable JOIN Versions ON (idTable.versionId = Versions.versionId) LEFT OUTER JOIN TroveInfo AS SourceNameTroveInfo ON idTable.instanceId = SourceNameTroveInfo.instanceId AND SourceNameTroveInfo.infoType = %(SOURCENAME)d LEFT OUTER JOIN TroveInfo AS MetadataTroveInfo ON idTable.instanceId = MetadataTroveInfo.instanceId AND MetadataTroveInfo.infoType = %(METADATA)d """ % d, args) l = list(cu) filteredL = typeFilter(l, filterSet) # sort based on (name, version, desc(finalTimestamp)) def sortorder(x, y): c = cmp(x[0], y[0]) if c: return c return -(cmp(x[3], y[3])) filteredL.sort(sortorder) if latest: # keep the latest newL = [] last = None for item in filteredL: if last and last[0] == item[0]: continue newL.append(item) last = item filteredL = newL nodeList = datamodel.NamedNodeList(total = len(filteredL), start = 0) addList = [] for (name, version, ts, finalTs, sourceName, metadata) in filteredL: sourceName = cu.frombinary(sourceName) if sourceName is None and trove.troveIsSourceComponent(name): sourceName = name addList.append((sourceName, str(versions.VersionFromString(version).getSourceVersion()))) schema.resetTable(cu, 'tmpNVF') # This is painful, but it converts the source name from a blob to # a string db.bulkload("tmpNVF", [ (x[0],) + x[1] for x in enumerate(addList) ], ["idx", "name", "version"], start_transaction = False) cu.execute(""" SELECT ChangeLogs.name, ChangeLogs.message, tmpNVF.name FROM tmpNVF JOIN Items AS SourceItems ON tmpNVF.name = SourceItems.item LEFT OUTER JOIN Versions AS SourceVersion ON tmpNVF.version = SourceVersion.version LEFT OUTER JOIN Nodes ON SourceItems.itemId = Nodes.itemId AND SourceVersion.versionId = Nodes.versionId LEFT OUTER JOIN ChangeLogs USING (nodeId) ORDER BY tmpNVF.idx """) for ( (name, version, ts, finalTs, sourceName, metadata), (clName, clMessage, troveName) ) in itertools.izip(filteredL, cu): frzVer = versions.strToFrozen(version, [ x for x in ts.split(":") ]) ver = versions.ThawVersion(frzVer) shortdesc = None if metadata: metadata = cu.frombinary(metadata) md = trove.Metadata(metadata) shortdesc = md.get()['shortDesc'] if clName: cl = datamodel.ChangeLog(name = clName, message = clMessage) else: cl = None nodeList.append(name = name, version = ver, mkUrl = mkUrl, changeLog = cl, shortdesc = shortdesc) return nodeList
def expandJobList(db, chgSetList, recurse): """ For each job in the list, find the set of troves which are recursively included in it. The reutnr value is list parallel to chgSetList, each item of which is a sorted list of those troves which are included in the recursive changeset. """ # We mark old groups (ones without weak references) as uncachable # because they're expensive to flatten (and so old that it # hardly matters). if not recurse: return [ [ job ] for job in chgSetList ] cu = db.cursor() schema.resetTable(cu, "tmpNVF") foundGroups = set() foundWeak = set() foundCollections = set() insertList = [] for jobId, job in enumerate(chgSetList): if trove.troveIsGroup(job[0]): foundGroups.add(jobId) insertList.append((jobId, job[0], job[2][0], job[2][1])) db.bulkload("tmpNvf", insertList, [ "idx", "name", "version", "flavor" ], start_transaction = False) db.analyze("tmpNVF") newJobList = [ [ job ] for job in chgSetList ] cu.execute("""SELECT tmpNVF.idx, I_Items.item, I_Versions.version, I_Flavors.flavor, TroveTroves.flags FROM tmpNVF JOIN Items ON tmpNVF.name = Items.item JOIN Versions ON (tmpNVF.version = Versions.version) JOIN Flavors ON (tmpNVF.flavor = Flavors.flavor) JOIN Instances ON Items.itemId = Instances.itemId AND Versions.versionId = Instances.versionId AND Flavors.flavorId = Instances.flavorId JOIN TroveTroves USING (instanceId) JOIN Instances AS I_Instances ON TroveTroves.includedId = I_Instances.instanceId JOIN Items AS I_Items ON I_Instances.itemId = I_Items.itemId JOIN Versions AS I_Versions ON I_Instances.versionId = I_Versions.versionId JOIN Flavors AS I_Flavors ON I_Instances.flavorId = I_Flavors.flavorId WHERE I_Instances.isPresent = 1 ORDER BY I_Items.item, I_Versions.version, I_Flavors.flavor """) for (idx, name, version, flavor, flags) in cu: newJobList[idx].append( (name, (None, None), (version, flavor), True) ) if flags & schema.TROVE_TROVES_WEAKREF > 0: foundWeak.add(idx) if trove.troveIsCollection(name): foundCollections.add(idx) for idx in ((foundGroups & foundCollections) - foundWeak): # groups which contain collections but no weak refs # are uncachable newJobList[idx] = None return newJobList