def _handleContents(self, pathId, fileId, fileStream,
configRestoreList, normalRestoreList,
oldFileId = None, oldVersion = None, oldfile = None,
restoreContents = True):
# files with contents need to be tracked so we can stick
# their contents in the archive "soon"; config files need
# extra magic for tracking since we may have to merge
# contents
repos = self.repos
if not fileStream or not restoreContents:
# empty fileStream means there are no contents to restore
return
hasContents = (files.frozenFileHasContents(fileStream) and
not files.frozenFileFlags(fileStream).isEncapsulatedContent())
if not hasContents:
return
fileFlags = files.frozenFileFlags(fileStream)
if self.storeOnlyConfigFiles and not fileFlags.isConfig():
return
contentInfo = files.frozenFileContentInfo(fileStream)
if fileFlags.isConfig():
tup = (pathId, fileId, contentInfo.sha1(),
oldfile, fileId, oldVersion, oldFileId,
restoreContents)
configRestoreList.append(tup)
else:
tup = (pathId, fileId, contentInfo.sha1(),
restoreContents)
normalRestoreList.append(tup)
def removeFile(self, pathId, fileId):
stream = self.repos.getFileStream(fileId)
sha1 = None
if files.frozenFileHasContents(stream):
flags = files.frozenFileFlags(stream)
if flags.isConfig():
contentInfo = files.frozenFileContentInfo(stream)
sha1 = contentInfo.sha1()
self.oldFile(pathId, fileId, sha1)
def removeFile(self, pathId, fileId):
stream = self.repos.getFileStream(fileId)
sha1 = None
if files.frozenFileHasContents(stream):
flags = files.frozenFileFlags(stream)
if flags.isConfig():
contentInfo = files.frozenFileContentInfo(stream)
sha1 = contentInfo.sha1()
self.oldFile(pathId, fileId, sha1)
def migrate3(self):
logMe(2, "fixing missing sha1s")
cu = self.db.cursor()
cu.execute("""select streamId, stream from filestreams
where sha1 is NULL and stream is not NULL""")
count = 0
for (streamId, stream) in list(cu):
if not files.frozenFileHasContents(stream): continue
contentInfo = files.frozenFileContentInfo(stream)
cu.execute("update filestreams set sha1=? where streamId=?",
cu.binary(contentInfo.sha1()), streamId)
count += 1
logMe(2, "fixed up %d rows" % count)
cu.execute("""select streamId, stream from filestreams
where sha1 is NULL and stream is not NULL""")
return True
def migrate3(self):
logMe(2, "fixing missing sha1s")
cu = self.db.cursor()
cu.execute("""select streamId, stream from filestreams
where sha1 is NULL and stream is not NULL""")
count = 0
for (streamId, stream) in list(cu):
if not files.frozenFileHasContents(stream): continue
contentInfo = files.frozenFileContentInfo(stream)
cu.execute("update filestreams set sha1=? where streamId=?",
cu.binary(contentInfo.sha1()), streamId)
count += 1
logMe(2, "fixed up %d rows" % count)
cu.execute("""select streamId, stream from filestreams
where sha1 is NULL and stream is not NULL""")
return True
def _handleContents(self,
pathId,
fileId,
fileStream,
configRestoreList,
normalRestoreList,
oldFileId=None,
oldVersion=None,
oldfile=None,
restoreContents=True):
# files with contents need to be tracked so we can stick
# their contents in the archive "soon"; config files need
# extra magic for tracking since we may have to merge
# contents
repos = self.repos
if not fileStream or not restoreContents:
# empty fileStream means there are no contents to restore
return
hasContents = (
files.frozenFileHasContents(fileStream)
and not files.frozenFileFlags(fileStream).isEncapsulatedContent())
if not hasContents:
return
fileFlags = files.frozenFileFlags(fileStream)
if self.storeOnlyConfigFiles and not fileFlags.isConfig():
return
contentInfo = files.frozenFileContentInfo(fileStream)
if fileFlags.isConfig():
tup = (pathId, fileId, contentInfo.sha1(), oldfile, fileId,
oldVersion, oldFileId, restoreContents)
configRestoreList.append(tup)
else:
tup = (pathId, fileId, contentInfo.sha1(), restoreContents)
normalRestoreList.append(tup)
def migrate(self):
self.message('WARNING: do NOT interrupt this migration, you will leave your DB in a messy state')
updateCursor = self.db.cursor()
self.cu.execute("""
CREATE TEMPORARY TABLE tmpSha1s(
streamId INTEGER,
sha1 BINARY(20))""",
start_transaction=False)
self.cu.execute('CREATE INDEX tmpSha1sIdx ON tmpSha1s(streamId)')
total = self.cu.execute('SELECT max(streamId) FROM FileStreams').fetchall()[0][0]
gpct = 0
for idx, (streamId, fileId, stream) in \
enumerate(updateCursor.execute("SELECT streamId, fileId, stream FROM "
"FileStreams ORDER BY StreamId")):
if stream and files.frozenFileHasContents(stream):
contents = files.frozenFileContentInfo(stream)
sha1 = contents.sha1()
self.cu.execute("INSERT INTO tmpSha1s (streamId, sha1) VALUES (?,?)",
(streamId, self.cu.binary(sha1)))
newPct = (streamId * 100)/total
if newPct >= pct:
logMe(3, 'Calculating sha1 for fileStream %s/%s (%02d%%)...' % (streamId, total, pct))
pct = newPct + 5
logMe(2, 'Populating FileStream Table with sha1s...')
# delay this as long as possible, any CTRL-C after this point
# will make future migrations fail.
self.cu.execute("ALTER TABLE FileStreams ADD COLUMN "
"sha1 %(BINARY20)s"
% self.db.keywords)
self.cu.execute("""
UPDATE FileStreams
SET sha1 = (
SELECT sha1 FROM tmpSha1s
WHERE FileStreams.streamid = tmpSha1s.streamid )
""")
self.cu.execute('DROP TABLE tmpSha1s')
# because of the foreign key referential mess, we need to
# destroy the FKs relationships, recreate the Entitlement
# tables, and restore the data
logMe(2, 'Updating Entitlements...')
self.cu.execute("CREATE TABLE Entitlements2 AS SELECT * FROM "
"Entitlements")
self.cu.execute("CREATE TABLE EntitlementGroups2 AS SELECT * FROM "
"EntitlementGroups")
self.cu.execute("CREATE TABLE EntitlementOwners2 AS SELECT * FROM "
"EntitlementOwners")
self.cu.execute("DROP TABLE Entitlements")
self.cu.execute("DROP TABLE EntitlementOwners")
self.cu.execute("DROP TABLE EntitlementGroups")
self.db.loadSchema()
schema.createEntitlements(self.db)
self.cu.execute("INSERT INTO EntitlementGroups (entGroup, entGroupId) "
"SELECT entGroup, entGroupId FROM EntitlementGroups2")
self.cu.execute("INSERT INTO EntitlementAccessMap (entGroupId, "
"userGroupId) SELECT entGroupId, userGroupId FROM "
"EntitlementGroups2")
self.cu.execute("INSERT INTO Entitlements SELECT * FROM Entitlements2")
self.cu.execute("INSERT INTO EntitlementOwners SELECT * FROM EntitlementOwners2")
self.cu.execute("DROP TABLE Entitlements2")
self.cu.execute("DROP TABLE EntitlementGroups2")
self.cu.execute("DROP TABLE EntitlementOwners2")
logMe(2, "Updating the Permissions table...")
self.cu.execute("ALTER TABLE Permissions ADD COLUMN "
"canRemove INTEGER NOT NULL DEFAULT 0"
% self.db.keywords)
logMe(2, "Updating Instances table...")
self.db.renameColumn("Instances", "isRedirect", "troveType")
self.db.loadSchema()
return self.Version
def migrate(self):
self.message(
'WARNING: do NOT interrupt this migration, you will leave your DB in a messy state'
)
updateCursor = self.db.cursor()
self.cu.execute("""
CREATE TEMPORARY TABLE tmpSha1s(
streamId INTEGER,
sha1 BINARY(20))""",
start_transaction=False)
self.cu.execute('CREATE INDEX tmpSha1sIdx ON tmpSha1s(streamId)')
total = self.cu.execute(
'SELECT max(streamId) FROM FileStreams').fetchall()[0][0]
gpct = 0
for idx, (streamId, fileId, stream) in \
enumerate(updateCursor.execute("SELECT streamId, fileId, stream FROM "
"FileStreams ORDER BY StreamId")):
if stream and files.frozenFileHasContents(stream):
contents = files.frozenFileContentInfo(stream)
sha1 = contents.sha1()
self.cu.execute(
"INSERT INTO tmpSha1s (streamId, sha1) VALUES (?,?)",
(streamId, self.cu.binary(sha1)))
newPct = (streamId * 100) / total
if newPct >= pct:
logMe(
3, 'Calculating sha1 for fileStream %s/%s (%02d%%)...' %
(streamId, total, pct))
pct = newPct + 5
logMe(2, 'Populating FileStream Table with sha1s...')
# delay this as long as possible, any CTRL-C after this point
# will make future migrations fail.
self.cu.execute("ALTER TABLE FileStreams ADD COLUMN "
"sha1 %(BINARY20)s" % self.db.keywords)
self.cu.execute("""
UPDATE FileStreams
SET sha1 = (
SELECT sha1 FROM tmpSha1s
WHERE FileStreams.streamid = tmpSha1s.streamid )
""")
self.cu.execute('DROP TABLE tmpSha1s')
# because of the foreign key referential mess, we need to
# destroy the FKs relationships, recreate the Entitlement
# tables, and restore the data
logMe(2, 'Updating Entitlements...')
self.cu.execute("CREATE TABLE Entitlements2 AS SELECT * FROM "
"Entitlements")
self.cu.execute("CREATE TABLE EntitlementGroups2 AS SELECT * FROM "
"EntitlementGroups")
self.cu.execute("CREATE TABLE EntitlementOwners2 AS SELECT * FROM "
"EntitlementOwners")
self.cu.execute("DROP TABLE Entitlements")
self.cu.execute("DROP TABLE EntitlementOwners")
self.cu.execute("DROP TABLE EntitlementGroups")
self.db.loadSchema()
schema.createEntitlements(self.db)
self.cu.execute("INSERT INTO EntitlementGroups (entGroup, entGroupId) "
"SELECT entGroup, entGroupId FROM EntitlementGroups2")
self.cu.execute("INSERT INTO EntitlementAccessMap (entGroupId, "
"userGroupId) SELECT entGroupId, userGroupId FROM "
"EntitlementGroups2")
self.cu.execute("INSERT INTO Entitlements SELECT * FROM Entitlements2")
self.cu.execute(
"INSERT INTO EntitlementOwners SELECT * FROM EntitlementOwners2")
self.cu.execute("DROP TABLE Entitlements2")
self.cu.execute("DROP TABLE EntitlementGroups2")
self.cu.execute("DROP TABLE EntitlementOwners2")
logMe(2, "Updating the Permissions table...")
self.cu.execute("ALTER TABLE Permissions ADD COLUMN "
"canRemove INTEGER NOT NULL DEFAULT 0" %
self.db.keywords)
logMe(2, "Updating Instances table...")
self.db.renameColumn("Instances", "isRedirect", "troveType")
self.db.loadSchema()
return self.Version