def _handleContents(self, pathId, fileId, fileStream,
configRestoreList, normalRestoreList,
oldFileId = None, oldVersion = None, oldfile = None,
restoreContents = True):
# files with contents need to be tracked so we can stick
# their contents in the archive "soon"; config files need
# extra magic for tracking since we may have to merge
# contents
repos = self.repos
if not fileStream or not restoreContents:
# empty fileStream means there are no contents to restore
return
hasContents = (files.frozenFileHasContents(fileStream) and
not files.frozenFileFlags(fileStream).isEncapsulatedContent())
if not hasContents:
return
fileFlags = files.frozenFileFlags(fileStream)
if self.storeOnlyConfigFiles and not fileFlags.isConfig():
return
contentInfo = files.frozenFileContentInfo(fileStream)
if fileFlags.isConfig():
tup = (pathId, fileId, contentInfo.sha1(),
oldfile, fileId, oldVersion, oldFileId,
restoreContents)
configRestoreList.append(tup)
else:
tup = (pathId, fileId, contentInfo.sha1(),
restoreContents)
normalRestoreList.append(tup)
def install(self, flags, troveCs):
if troveCs.getOldVersion():
oldTrv = self.db.getTrove(*troveCs.getOldNameVersionFlavor())
trv = oldTrv.copy()
trv.applyChangeSet(troveCs)
else:
oldTrv = None
trv = trove.Trove(troveCs)
#if oldTrv and oldTrv.troveInfo.capsule == trv.troveInfo.capsule:
# the capsule hasn't changed, so don't reinstall it
#return None
for pathId, path, fileId, version in trv.iterFileList(capsules=True):
# there should only be one...
break
assert (pathId == trove.CAPSULE_PATHID)
if oldTrv:
for oldPathId, oldPath, oldFileId, oldVersion in \
oldTrv.iterFileList(capsules = True):
# there should only be one...
break
assert (oldPathId == trove.CAPSULE_PATHID)
if (oldFileId == fileId
or oldTrv.troveInfo.capsule == trv.troveInfo.capsule):
# good enough. this means changing capsule information
# in trove info won't fool us into trying to reinstall
# capsules which haven't changed. we check the capsule
# information as well because derived packages change
# the capsule fileIds. ugh.
#
# we do it in this order to make sure the test suite tests
# both sides of the "or" above
return
self.remove(oldTrv)
# is the capsule new or changed?
changedFileInfos = [
x for x in troveCs.getChangedFileList()
if x[0] == trove.CAPSULE_PATHID
]
if changedFileInfos:
oldFileId = oldTrv.getFile(pathId)[1]
oldFileObjs = self.db.getFileStream(oldFileId)
fileObj = files.ThawFile(oldFileObjs, pathId)
fileChange = self.changeSet.getFileChange(oldFileId, fileId)
fileObj.twm(fileChange, fileObj)
sha1 = fileObj.contents.sha1()
else:
fileStream = self.changeSet.getFileChange(None, fileId)
sha1 = files.frozenFileContentInfo(fileStream).sha1()
self.installs.append((troveCs, (pathId, path, fileId, sha1)))
return (oldTrv, trv)
def install(self, flags, troveCs):
if troveCs.getOldVersion():
oldTrv = self.db.getTrove(*troveCs.getOldNameVersionFlavor())
trv = oldTrv.copy()
trv.applyChangeSet(troveCs)
else:
oldTrv = None
trv = trove.Trove(troveCs)
#if oldTrv and oldTrv.troveInfo.capsule == trv.troveInfo.capsule:
# the capsule hasn't changed, so don't reinstall it
#return None
for pathId, path, fileId, version in trv.iterFileList(capsules = True):
# there should only be one...
break
assert(pathId == trove.CAPSULE_PATHID)
if oldTrv:
for oldPathId, oldPath, oldFileId, oldVersion in \
oldTrv.iterFileList(capsules = True):
# there should only be one...
break
assert(oldPathId == trove.CAPSULE_PATHID)
if (oldFileId == fileId or
oldTrv.troveInfo.capsule == trv.troveInfo.capsule):
# good enough. this means changing capsule information
# in trove info won't fool us into trying to reinstall
# capsules which haven't changed. we check the capsule
# information as well because derived packages change
# the capsule fileIds. ugh.
#
# we do it in this order to make sure the test suite tests
# both sides of the "or" above
return
self.remove(oldTrv)
# is the capsule new or changed?
changedFileInfos = [ x for x in troveCs.getChangedFileList()
if x[0] == trove.CAPSULE_PATHID ]
if changedFileInfos:
oldFileId = oldTrv.getFile(pathId)[1]
oldFileObjs = self.db.getFileStream(oldFileId)
fileObj = files.ThawFile(oldFileObjs, pathId)
fileChange = self.changeSet.getFileChange(oldFileId, fileId)
fileObj.twm(fileChange, fileObj)
sha1 = fileObj.contents.sha1()
else:
fileStream = self.changeSet.getFileChange(None, fileId)
sha1 = files.frozenFileContentInfo(fileStream).sha1()
self.installs.append((troveCs, (pathId, path, fileId, sha1)))
return (oldTrv, trv)
def removeFile(self, pathId, fileId):
stream = self.repos.getFileStream(fileId)
sha1 = None
if files.frozenFileHasContents(stream):
flags = files.frozenFileFlags(stream)
if flags.isConfig():
contentInfo = files.frozenFileContentInfo(stream)
sha1 = contentInfo.sha1()
self.oldFile(pathId, fileId, sha1)
def removeFile(self, pathId, fileId):
stream = self.repos.getFileStream(fileId)
sha1 = None
if files.frozenFileHasContents(stream):
flags = files.frozenFileFlags(stream)
if flags.isConfig():
contentInfo = files.frozenFileContentInfo(stream)
sha1 = contentInfo.sha1()
self.oldFile(pathId, fileId, sha1)
def _processTrove(changeSet, trvCs, mounts, sizes):
for pathId, path, fileId, fVer in trvCs.getNewFileList():
fStr = changeSet.getFileChange(None, fileId)
fObj = files.frozenFileContentInfo(fStr)
if type(fObj) == files.RegularFileStream:
for mount in mounts:
if path.startswith(mount):
blockSize = 4096
realSize = fObj.size()
nearestBlock = (realSize / blockSize + 1) * blockSize
sizes[mount] += nearestBlock
break
def migrate3(self):
logMe(2, "fixing missing sha1s")
cu = self.db.cursor()
cu.execute("""select streamId, stream from filestreams
where sha1 is NULL and stream is not NULL""")
count = 0
for (streamId, stream) in list(cu):
if not files.frozenFileHasContents(stream): continue
contentInfo = files.frozenFileContentInfo(stream)
cu.execute("update filestreams set sha1=? where streamId=?",
cu.binary(contentInfo.sha1()), streamId)
count += 1
logMe(2, "fixed up %d rows" % count)
cu.execute("""select streamId, stream from filestreams
where sha1 is NULL and stream is not NULL""")
return True
def migrate3(self):
logMe(2, "fixing missing sha1s")
cu = self.db.cursor()
cu.execute("""select streamId, stream from filestreams
where sha1 is NULL and stream is not NULL""")
count = 0
for (streamId, stream) in list(cu):
if not files.frozenFileHasContents(stream): continue
contentInfo = files.frozenFileContentInfo(stream)
cu.execute("update filestreams set sha1=? where streamId=?",
cu.binary(contentInfo.sha1()), streamId)
count += 1
logMe(2, "fixed up %d rows" % count)
cu.execute("""select streamId, stream from filestreams
where sha1 is NULL and stream is not NULL""")
return True
def _handleContents(self,
pathId,
fileId,
fileStream,
configRestoreList,
normalRestoreList,
oldFileId=None,
oldVersion=None,
oldfile=None,
restoreContents=True):
# files with contents need to be tracked so we can stick
# their contents in the archive "soon"; config files need
# extra magic for tracking since we may have to merge
# contents
repos = self.repos
if not fileStream or not restoreContents:
# empty fileStream means there are no contents to restore
return
hasContents = (
files.frozenFileHasContents(fileStream)
and not files.frozenFileFlags(fileStream).isEncapsulatedContent())
if not hasContents:
return
fileFlags = files.frozenFileFlags(fileStream)
if self.storeOnlyConfigFiles and not fileFlags.isConfig():
return
contentInfo = files.frozenFileContentInfo(fileStream)
if fileFlags.isConfig():
tup = (pathId, fileId, contentInfo.sha1(), oldfile, fileId,
oldVersion, oldFileId, restoreContents)
configRestoreList.append(tup)
else:
tup = (pathId, fileId, contentInfo.sha1(), restoreContents)
normalRestoreList.append(tup)
def migrate(self):
self.message('WARNING: do NOT interrupt this migration, you will leave your DB in a messy state')
updateCursor = self.db.cursor()
self.cu.execute("""
CREATE TEMPORARY TABLE tmpSha1s(
streamId INTEGER,
sha1 BINARY(20))""",
start_transaction=False)
self.cu.execute('CREATE INDEX tmpSha1sIdx ON tmpSha1s(streamId)')
total = self.cu.execute('SELECT max(streamId) FROM FileStreams').fetchall()[0][0]
gpct = 0
for idx, (streamId, fileId, stream) in \
enumerate(updateCursor.execute("SELECT streamId, fileId, stream FROM "
"FileStreams ORDER BY StreamId")):
if stream and files.frozenFileHasContents(stream):
contents = files.frozenFileContentInfo(stream)
sha1 = contents.sha1()
self.cu.execute("INSERT INTO tmpSha1s (streamId, sha1) VALUES (?,?)",
(streamId, self.cu.binary(sha1)))
newPct = (streamId * 100)/total
if newPct >= pct:
logMe(3, 'Calculating sha1 for fileStream %s/%s (%02d%%)...' % (streamId, total, pct))
pct = newPct + 5
logMe(2, 'Populating FileStream Table with sha1s...')
# delay this as long as possible, any CTRL-C after this point
# will make future migrations fail.
self.cu.execute("ALTER TABLE FileStreams ADD COLUMN "
"sha1 %(BINARY20)s"
% self.db.keywords)
self.cu.execute("""
UPDATE FileStreams
SET sha1 = (
SELECT sha1 FROM tmpSha1s
WHERE FileStreams.streamid = tmpSha1s.streamid )
""")
self.cu.execute('DROP TABLE tmpSha1s')
# because of the foreign key referential mess, we need to
# destroy the FKs relationships, recreate the Entitlement
# tables, and restore the data
logMe(2, 'Updating Entitlements...')
self.cu.execute("CREATE TABLE Entitlements2 AS SELECT * FROM "
"Entitlements")
self.cu.execute("CREATE TABLE EntitlementGroups2 AS SELECT * FROM "
"EntitlementGroups")
self.cu.execute("CREATE TABLE EntitlementOwners2 AS SELECT * FROM "
"EntitlementOwners")
self.cu.execute("DROP TABLE Entitlements")
self.cu.execute("DROP TABLE EntitlementOwners")
self.cu.execute("DROP TABLE EntitlementGroups")
self.db.loadSchema()
schema.createEntitlements(self.db)
self.cu.execute("INSERT INTO EntitlementGroups (entGroup, entGroupId) "
"SELECT entGroup, entGroupId FROM EntitlementGroups2")
self.cu.execute("INSERT INTO EntitlementAccessMap (entGroupId, "
"userGroupId) SELECT entGroupId, userGroupId FROM "
"EntitlementGroups2")
self.cu.execute("INSERT INTO Entitlements SELECT * FROM Entitlements2")
self.cu.execute("INSERT INTO EntitlementOwners SELECT * FROM EntitlementOwners2")
self.cu.execute("DROP TABLE Entitlements2")
self.cu.execute("DROP TABLE EntitlementGroups2")
self.cu.execute("DROP TABLE EntitlementOwners2")
logMe(2, "Updating the Permissions table...")
self.cu.execute("ALTER TABLE Permissions ADD COLUMN "
"canRemove INTEGER NOT NULL DEFAULT 0"
% self.db.keywords)
logMe(2, "Updating Instances table...")
self.db.renameColumn("Instances", "isRedirect", "troveType")
self.db.loadSchema()
return self.Version
def migrate(self):
self.message(
'WARNING: do NOT interrupt this migration, you will leave your DB in a messy state'
)
updateCursor = self.db.cursor()
self.cu.execute("""
CREATE TEMPORARY TABLE tmpSha1s(
streamId INTEGER,
sha1 BINARY(20))""",
start_transaction=False)
self.cu.execute('CREATE INDEX tmpSha1sIdx ON tmpSha1s(streamId)')
total = self.cu.execute(
'SELECT max(streamId) FROM FileStreams').fetchall()[0][0]
gpct = 0
for idx, (streamId, fileId, stream) in \
enumerate(updateCursor.execute("SELECT streamId, fileId, stream FROM "
"FileStreams ORDER BY StreamId")):
if stream and files.frozenFileHasContents(stream):
contents = files.frozenFileContentInfo(stream)
sha1 = contents.sha1()
self.cu.execute(
"INSERT INTO tmpSha1s (streamId, sha1) VALUES (?,?)",
(streamId, self.cu.binary(sha1)))
newPct = (streamId * 100) / total
if newPct >= pct:
logMe(
3, 'Calculating sha1 for fileStream %s/%s (%02d%%)...' %
(streamId, total, pct))
pct = newPct + 5
logMe(2, 'Populating FileStream Table with sha1s...')
# delay this as long as possible, any CTRL-C after this point
# will make future migrations fail.
self.cu.execute("ALTER TABLE FileStreams ADD COLUMN "
"sha1 %(BINARY20)s" % self.db.keywords)
self.cu.execute("""
UPDATE FileStreams
SET sha1 = (
SELECT sha1 FROM tmpSha1s
WHERE FileStreams.streamid = tmpSha1s.streamid )
""")
self.cu.execute('DROP TABLE tmpSha1s')
# because of the foreign key referential mess, we need to
# destroy the FKs relationships, recreate the Entitlement
# tables, and restore the data
logMe(2, 'Updating Entitlements...')
self.cu.execute("CREATE TABLE Entitlements2 AS SELECT * FROM "
"Entitlements")
self.cu.execute("CREATE TABLE EntitlementGroups2 AS SELECT * FROM "
"EntitlementGroups")
self.cu.execute("CREATE TABLE EntitlementOwners2 AS SELECT * FROM "
"EntitlementOwners")
self.cu.execute("DROP TABLE Entitlements")
self.cu.execute("DROP TABLE EntitlementOwners")
self.cu.execute("DROP TABLE EntitlementGroups")
self.db.loadSchema()
schema.createEntitlements(self.db)
self.cu.execute("INSERT INTO EntitlementGroups (entGroup, entGroupId) "
"SELECT entGroup, entGroupId FROM EntitlementGroups2")
self.cu.execute("INSERT INTO EntitlementAccessMap (entGroupId, "
"userGroupId) SELECT entGroupId, userGroupId FROM "
"EntitlementGroups2")
self.cu.execute("INSERT INTO Entitlements SELECT * FROM Entitlements2")
self.cu.execute(
"INSERT INTO EntitlementOwners SELECT * FROM EntitlementOwners2")
self.cu.execute("DROP TABLE Entitlements2")
self.cu.execute("DROP TABLE EntitlementGroups2")
self.cu.execute("DROP TABLE EntitlementOwners2")
logMe(2, "Updating the Permissions table...")
self.cu.execute("ALTER TABLE Permissions ADD COLUMN "
"canRemove INTEGER NOT NULL DEFAULT 0" %
self.db.keywords)
logMe(2, "Updating Instances table...")
self.db.renameColumn("Instances", "isRedirect", "troveType")
self.db.loadSchema()
return self.Version