def _handleContents(self, pathId, fileId, fileStream, configRestoreList, normalRestoreList, oldFileId = None, oldVersion = None, oldfile = None, restoreContents = True): # files with contents need to be tracked so we can stick # their contents in the archive "soon"; config files need # extra magic for tracking since we may have to merge # contents repos = self.repos if not fileStream or not restoreContents: # empty fileStream means there are no contents to restore return hasContents = (files.frozenFileHasContents(fileStream) and not files.frozenFileFlags(fileStream).isEncapsulatedContent()) if not hasContents: return fileFlags = files.frozenFileFlags(fileStream) if self.storeOnlyConfigFiles and not fileFlags.isConfig(): return contentInfo = files.frozenFileContentInfo(fileStream) if fileFlags.isConfig(): tup = (pathId, fileId, contentInfo.sha1(), oldfile, fileId, oldVersion, oldFileId, restoreContents) configRestoreList.append(tup) else: tup = (pathId, fileId, contentInfo.sha1(), restoreContents) normalRestoreList.append(tup)
def install(self, flags, troveCs): if troveCs.getOldVersion(): oldTrv = self.db.getTrove(*troveCs.getOldNameVersionFlavor()) trv = oldTrv.copy() trv.applyChangeSet(troveCs) else: oldTrv = None trv = trove.Trove(troveCs) #if oldTrv and oldTrv.troveInfo.capsule == trv.troveInfo.capsule: # the capsule hasn't changed, so don't reinstall it #return None for pathId, path, fileId, version in trv.iterFileList(capsules=True): # there should only be one... break assert (pathId == trove.CAPSULE_PATHID) if oldTrv: for oldPathId, oldPath, oldFileId, oldVersion in \ oldTrv.iterFileList(capsules = True): # there should only be one... break assert (oldPathId == trove.CAPSULE_PATHID) if (oldFileId == fileId or oldTrv.troveInfo.capsule == trv.troveInfo.capsule): # good enough. this means changing capsule information # in trove info won't fool us into trying to reinstall # capsules which haven't changed. we check the capsule # information as well because derived packages change # the capsule fileIds. ugh. # # we do it in this order to make sure the test suite tests # both sides of the "or" above return self.remove(oldTrv) # is the capsule new or changed? changedFileInfos = [ x for x in troveCs.getChangedFileList() if x[0] == trove.CAPSULE_PATHID ] if changedFileInfos: oldFileId = oldTrv.getFile(pathId)[1] oldFileObjs = self.db.getFileStream(oldFileId) fileObj = files.ThawFile(oldFileObjs, pathId) fileChange = self.changeSet.getFileChange(oldFileId, fileId) fileObj.twm(fileChange, fileObj) sha1 = fileObj.contents.sha1() else: fileStream = self.changeSet.getFileChange(None, fileId) sha1 = files.frozenFileContentInfo(fileStream).sha1() self.installs.append((troveCs, (pathId, path, fileId, sha1))) return (oldTrv, trv)
def install(self, flags, troveCs): if troveCs.getOldVersion(): oldTrv = self.db.getTrove(*troveCs.getOldNameVersionFlavor()) trv = oldTrv.copy() trv.applyChangeSet(troveCs) else: oldTrv = None trv = trove.Trove(troveCs) #if oldTrv and oldTrv.troveInfo.capsule == trv.troveInfo.capsule: # the capsule hasn't changed, so don't reinstall it #return None for pathId, path, fileId, version in trv.iterFileList(capsules = True): # there should only be one... break assert(pathId == trove.CAPSULE_PATHID) if oldTrv: for oldPathId, oldPath, oldFileId, oldVersion in \ oldTrv.iterFileList(capsules = True): # there should only be one... break assert(oldPathId == trove.CAPSULE_PATHID) if (oldFileId == fileId or oldTrv.troveInfo.capsule == trv.troveInfo.capsule): # good enough. this means changing capsule information # in trove info won't fool us into trying to reinstall # capsules which haven't changed. we check the capsule # information as well because derived packages change # the capsule fileIds. ugh. # # we do it in this order to make sure the test suite tests # both sides of the "or" above return self.remove(oldTrv) # is the capsule new or changed? changedFileInfos = [ x for x in troveCs.getChangedFileList() if x[0] == trove.CAPSULE_PATHID ] if changedFileInfos: oldFileId = oldTrv.getFile(pathId)[1] oldFileObjs = self.db.getFileStream(oldFileId) fileObj = files.ThawFile(oldFileObjs, pathId) fileChange = self.changeSet.getFileChange(oldFileId, fileId) fileObj.twm(fileChange, fileObj) sha1 = fileObj.contents.sha1() else: fileStream = self.changeSet.getFileChange(None, fileId) sha1 = files.frozenFileContentInfo(fileStream).sha1() self.installs.append((troveCs, (pathId, path, fileId, sha1))) return (oldTrv, trv)
def removeFile(self, pathId, fileId): stream = self.repos.getFileStream(fileId) sha1 = None if files.frozenFileHasContents(stream): flags = files.frozenFileFlags(stream) if flags.isConfig(): contentInfo = files.frozenFileContentInfo(stream) sha1 = contentInfo.sha1() self.oldFile(pathId, fileId, sha1)
def _processTrove(changeSet, trvCs, mounts, sizes): for pathId, path, fileId, fVer in trvCs.getNewFileList(): fStr = changeSet.getFileChange(None, fileId) fObj = files.frozenFileContentInfo(fStr) if type(fObj) == files.RegularFileStream: for mount in mounts: if path.startswith(mount): blockSize = 4096 realSize = fObj.size() nearestBlock = (realSize / blockSize + 1) * blockSize sizes[mount] += nearestBlock break
def migrate3(self): logMe(2, "fixing missing sha1s") cu = self.db.cursor() cu.execute("""select streamId, stream from filestreams where sha1 is NULL and stream is not NULL""") count = 0 for (streamId, stream) in list(cu): if not files.frozenFileHasContents(stream): continue contentInfo = files.frozenFileContentInfo(stream) cu.execute("update filestreams set sha1=? where streamId=?", cu.binary(contentInfo.sha1()), streamId) count += 1 logMe(2, "fixed up %d rows" % count) cu.execute("""select streamId, stream from filestreams where sha1 is NULL and stream is not NULL""") return True
def _handleContents(self, pathId, fileId, fileStream, configRestoreList, normalRestoreList, oldFileId=None, oldVersion=None, oldfile=None, restoreContents=True): # files with contents need to be tracked so we can stick # their contents in the archive "soon"; config files need # extra magic for tracking since we may have to merge # contents repos = self.repos if not fileStream or not restoreContents: # empty fileStream means there are no contents to restore return hasContents = ( files.frozenFileHasContents(fileStream) and not files.frozenFileFlags(fileStream).isEncapsulatedContent()) if not hasContents: return fileFlags = files.frozenFileFlags(fileStream) if self.storeOnlyConfigFiles and not fileFlags.isConfig(): return contentInfo = files.frozenFileContentInfo(fileStream) if fileFlags.isConfig(): tup = (pathId, fileId, contentInfo.sha1(), oldfile, fileId, oldVersion, oldFileId, restoreContents) configRestoreList.append(tup) else: tup = (pathId, fileId, contentInfo.sha1(), restoreContents) normalRestoreList.append(tup)
def migrate(self): self.message('WARNING: do NOT interrupt this migration, you will leave your DB in a messy state') updateCursor = self.db.cursor() self.cu.execute(""" CREATE TEMPORARY TABLE tmpSha1s( streamId INTEGER, sha1 BINARY(20))""", start_transaction=False) self.cu.execute('CREATE INDEX tmpSha1sIdx ON tmpSha1s(streamId)') total = self.cu.execute('SELECT max(streamId) FROM FileStreams').fetchall()[0][0] gpct = 0 for idx, (streamId, fileId, stream) in \ enumerate(updateCursor.execute("SELECT streamId, fileId, stream FROM " "FileStreams ORDER BY StreamId")): if stream and files.frozenFileHasContents(stream): contents = files.frozenFileContentInfo(stream) sha1 = contents.sha1() self.cu.execute("INSERT INTO tmpSha1s (streamId, sha1) VALUES (?,?)", (streamId, self.cu.binary(sha1))) newPct = (streamId * 100)/total if newPct >= pct: logMe(3, 'Calculating sha1 for fileStream %s/%s (%02d%%)...' % (streamId, total, pct)) pct = newPct + 5 logMe(2, 'Populating FileStream Table with sha1s...') # delay this as long as possible, any CTRL-C after this point # will make future migrations fail. self.cu.execute("ALTER TABLE FileStreams ADD COLUMN " "sha1 %(BINARY20)s" % self.db.keywords) self.cu.execute(""" UPDATE FileStreams SET sha1 = ( SELECT sha1 FROM tmpSha1s WHERE FileStreams.streamid = tmpSha1s.streamid ) """) self.cu.execute('DROP TABLE tmpSha1s') # because of the foreign key referential mess, we need to # destroy the FKs relationships, recreate the Entitlement # tables, and restore the data logMe(2, 'Updating Entitlements...') self.cu.execute("CREATE TABLE Entitlements2 AS SELECT * FROM " "Entitlements") self.cu.execute("CREATE TABLE EntitlementGroups2 AS SELECT * FROM " "EntitlementGroups") self.cu.execute("CREATE TABLE EntitlementOwners2 AS SELECT * FROM " "EntitlementOwners") self.cu.execute("DROP TABLE Entitlements") self.cu.execute("DROP TABLE EntitlementOwners") self.cu.execute("DROP TABLE EntitlementGroups") self.db.loadSchema() schema.createEntitlements(self.db) self.cu.execute("INSERT INTO EntitlementGroups (entGroup, entGroupId) " "SELECT entGroup, entGroupId FROM EntitlementGroups2") self.cu.execute("INSERT INTO EntitlementAccessMap (entGroupId, " "userGroupId) SELECT entGroupId, userGroupId FROM " "EntitlementGroups2") self.cu.execute("INSERT INTO Entitlements SELECT * FROM Entitlements2") self.cu.execute("INSERT INTO EntitlementOwners SELECT * FROM EntitlementOwners2") self.cu.execute("DROP TABLE Entitlements2") self.cu.execute("DROP TABLE EntitlementGroups2") self.cu.execute("DROP TABLE EntitlementOwners2") logMe(2, "Updating the Permissions table...") self.cu.execute("ALTER TABLE Permissions ADD COLUMN " "canRemove INTEGER NOT NULL DEFAULT 0" % self.db.keywords) logMe(2, "Updating Instances table...") self.db.renameColumn("Instances", "isRedirect", "troveType") self.db.loadSchema() return self.Version
def migrate(self): self.message( 'WARNING: do NOT interrupt this migration, you will leave your DB in a messy state' ) updateCursor = self.db.cursor() self.cu.execute(""" CREATE TEMPORARY TABLE tmpSha1s( streamId INTEGER, sha1 BINARY(20))""", start_transaction=False) self.cu.execute('CREATE INDEX tmpSha1sIdx ON tmpSha1s(streamId)') total = self.cu.execute( 'SELECT max(streamId) FROM FileStreams').fetchall()[0][0] gpct = 0 for idx, (streamId, fileId, stream) in \ enumerate(updateCursor.execute("SELECT streamId, fileId, stream FROM " "FileStreams ORDER BY StreamId")): if stream and files.frozenFileHasContents(stream): contents = files.frozenFileContentInfo(stream) sha1 = contents.sha1() self.cu.execute( "INSERT INTO tmpSha1s (streamId, sha1) VALUES (?,?)", (streamId, self.cu.binary(sha1))) newPct = (streamId * 100) / total if newPct >= pct: logMe( 3, 'Calculating sha1 for fileStream %s/%s (%02d%%)...' % (streamId, total, pct)) pct = newPct + 5 logMe(2, 'Populating FileStream Table with sha1s...') # delay this as long as possible, any CTRL-C after this point # will make future migrations fail. self.cu.execute("ALTER TABLE FileStreams ADD COLUMN " "sha1 %(BINARY20)s" % self.db.keywords) self.cu.execute(""" UPDATE FileStreams SET sha1 = ( SELECT sha1 FROM tmpSha1s WHERE FileStreams.streamid = tmpSha1s.streamid ) """) self.cu.execute('DROP TABLE tmpSha1s') # because of the foreign key referential mess, we need to # destroy the FKs relationships, recreate the Entitlement # tables, and restore the data logMe(2, 'Updating Entitlements...') self.cu.execute("CREATE TABLE Entitlements2 AS SELECT * FROM " "Entitlements") self.cu.execute("CREATE TABLE EntitlementGroups2 AS SELECT * FROM " "EntitlementGroups") self.cu.execute("CREATE TABLE EntitlementOwners2 AS SELECT * FROM " "EntitlementOwners") self.cu.execute("DROP TABLE Entitlements") self.cu.execute("DROP TABLE EntitlementOwners") self.cu.execute("DROP TABLE EntitlementGroups") self.db.loadSchema() schema.createEntitlements(self.db) self.cu.execute("INSERT INTO EntitlementGroups (entGroup, entGroupId) " "SELECT entGroup, entGroupId FROM EntitlementGroups2") self.cu.execute("INSERT INTO EntitlementAccessMap (entGroupId, " "userGroupId) SELECT entGroupId, userGroupId FROM " "EntitlementGroups2") self.cu.execute("INSERT INTO Entitlements SELECT * FROM Entitlements2") self.cu.execute( "INSERT INTO EntitlementOwners SELECT * FROM EntitlementOwners2") self.cu.execute("DROP TABLE Entitlements2") self.cu.execute("DROP TABLE EntitlementGroups2") self.cu.execute("DROP TABLE EntitlementOwners2") logMe(2, "Updating the Permissions table...") self.cu.execute("ALTER TABLE Permissions ADD COLUMN " "canRemove INTEGER NOT NULL DEFAULT 0" % self.db.keywords) logMe(2, "Updating Instances table...") self.db.renameColumn("Instances", "isRedirect", "troveType") self.db.loadSchema() return self.Version