def _restoreNormal(self, cs, normalRestoreList, preRestored): ptrRestores = [] ptrRefsAdded = {} lastRestore = None # restore each pathId,fileId combo once while normalRestoreList: (pathId, fileId, sha1, restoreContents) = normalRestoreList.pop(0) if preRestored is not None and sha1 in preRestored: continue if (pathId, fileId) == lastRestore: continue lastRestore = (pathId, fileId) try: (contType, fileContents) = cs.getFileContents(pathId, fileId, compressed=True) except KeyError: raise errors.IntegrityError( "Missing file contents for pathId %s, fileId %s" % (sha1helper.md5ToString(pathId), sha1helper.sha1ToString(fileId))) if contType == changeset.ChangedFileTypes.ptr: ptrRestores.append(sha1) target = util.decompressString(fileContents.get().read()) if util.tupleListBsearchInsert( normalRestoreList, (target[:16], target[16:], sha1, True), self.ptrCmp): # Item was inserted. This creates a reference in the # datastore; keep track of it to prevent a duplicate # reference count. ptrRefsAdded[sha1] = True continue assert (contType == changeset.ChangedFileTypes.file) self.addFileContents(sha1, fileContents, restoreContents, 0, precompressed=True) for sha1 in ptrRestores: # Increment the reference count for items which were ptr's # to a different file. if sha1 in ptrRefsAdded: del ptrRefsAdded[sha1] else: self.addFileContents(sha1, None, False, 0)
def testTupleListBsearchInsert(self): def fn(a, b): if a[1] == b[1]: return 0 elif a[1] < b[1]: return -1 return 1 # this runs all of the inserts twice to make sure duplicates don't # get added l = [] util.tupleListBsearchInsert(l, ('v', 5), fn) assert(l == [('v', 5)]) util.tupleListBsearchInsert(l, ('v', 5), fn) assert(l == [('v', 5)]) util.tupleListBsearchInsert(l, ('e', 22), fn) assert(l == [('v', 5), ('e', 22)]) util.tupleListBsearchInsert(l, ('e', 22), fn) assert(l == [('v', 5), ('e', 22)]) util.tupleListBsearchInsert(l, ('b', 25), fn) assert(l == [('v', 5), ('e', 22), ('b', 25)]) util.tupleListBsearchInsert(l, ('b', 25), fn) assert(l == [('v', 5), ('e', 22), ('b', 25)]) util.tupleListBsearchInsert(l, ('y', 2), fn) assert(l == [('y', 2), ('v', 5), ('e', 22), ('b', 25)]) util.tupleListBsearchInsert(l, ('y', 2), fn) assert(l == [('y', 2), ('v', 5), ('e', 22), ('b', 25)]) util.tupleListBsearchInsert(l, ('g', 20), fn) assert(l == [('y', 2), ('v', 5), ('g', 20), ('e', 22), ('b', 25)]) util.tupleListBsearchInsert(l, ('g', 20), fn) assert(l == [('y', 2), ('v', 5), ('g', 20), ('e', 22), ('b', 25)]) util.tupleListBsearchInsert(l, ('t', 18), fn) assert(l == [('y', 2), ('v', 5), ('t', 18), ('g', 20), ('e', 22), ('b', 25)]) util.tupleListBsearchInsert(l, ('t', 18), fn) assert(l == [('y', 2), ('v', 5), ('t', 18), ('g', 20), ('e', 22), ('b', 25)])
def testTupleListBsearchInsert(self): def fn(a, b): if a[1] == b[1]: return 0 elif a[1] < b[1]: return -1 return 1 # this runs all of the inserts twice to make sure duplicates don't # get added l = [] util.tupleListBsearchInsert(l, ('v', 5), fn) assert(l == [('v', 5)]) util.tupleListBsearchInsert(l, ('v', 5), fn) assert(l == [('v', 5)]) util.tupleListBsearchInsert(l, ('e', 22), fn) assert(l == [('v', 5), ('e', 22)]) util.tupleListBsearchInsert(l, ('e', 22), fn) assert(l == [('v', 5), ('e', 22)]) util.tupleListBsearchInsert(l, ('b', 25), fn) assert(l == [('v', 5), ('e', 22), ('b', 25)]) util.tupleListBsearchInsert(l, ('b', 25), fn) assert(l == [('v', 5), ('e', 22), ('b', 25)]) util.tupleListBsearchInsert(l, ('y', 2), fn) assert(l == [('y', 2), ('v', 5), ('e', 22), ('b', 25)]) util.tupleListBsearchInsert(l, ('y', 2), fn) assert(l == [('y', 2), ('v', 5), ('e', 22), ('b', 25)]) util.tupleListBsearchInsert(l, ('g', 20), fn) assert(l == [('y', 2), ('v', 5), ('g', 20), ('e', 22), ('b', 25)]) util.tupleListBsearchInsert(l, ('g', 20), fn) assert(l == [('y', 2), ('v', 5), ('g', 20), ('e', 22), ('b', 25)]) util.tupleListBsearchInsert(l, ('t', 18), fn) assert(l == [('y', 2), ('v', 5), ('t', 18), ('g', 20), ('e', 22), ('b', 25)]) util.tupleListBsearchInsert(l, ('t', 18), fn) assert(l == [('y', 2), ('v', 5), ('t', 18), ('g', 20), ('e', 22), ('b', 25)])
def __init__(self, repos, cs, fileHostFilter = [], callback = None, resetTimestamps = False, allowIncomplete = False, hidden = False, mirror = False, excludeCapsuleContents = False): self.repos = repos self.cs = cs self.invalidateRollbacksFlag = False newList = [ x for x in cs.iterNewTroveList() ] if resetTimestamps: # This depends intimiately on the versions cache. We don't # change the timestamps on each version, because the cache # ensures they are all a single underlying object. Slick, # but brittle? updated = {} for csTrove in newList: ver = csTrove.getNewVersion() if ver in updated: pass else: oldVer = ver.copy() ver.trailingRevision().resetTimeStamp() updated[oldVer] = ver del updated troveNo, configRestoreList, normalRestoreList = \ self._createInstallTroveObjects(fileHostFilter = fileHostFilter, callback = callback, mirror = mirror, hidden = hidden, allowIncomplete = allowIncomplete, excludeCapsuleContents = excludeCapsuleContents) configRestoreList, normalRestoreList = \ self._filterRestoreList(configRestoreList, normalRestoreList) # use a key to select data up to, but not including, the first # version. We can't sort on version because we don't have timestamps configRestoreList.sort(key=lambda x: x[0:5]) normalRestoreList.sort(key=lambda x: x[0:3]) # config files are cached, so we don't have to worry about not # restoring the same fileId/pathId twice for (pathId, newFileId, sha1, oldfile, newFileId, oldVersion, oldFileId, restoreContents) in configRestoreList: if cs.configFileIsDiff(pathId, newFileId): (contType, fileContents) = cs.getFileContents(pathId, newFileId) # the content for this file is in the form of a # diff, which we need to apply against the file in # the repository assert(oldVersion) try: f = self.repos.getFileContents( [(oldFileId, oldVersion, oldfile)])[0].get() except KeyError: raise errors.IntegrityError( "Missing file contents for pathId %s, fileId %s" % ( sha1helper.md5ToString(pathId), sha1helper.sha1ToString(fileId))) oldLines = f.readlines() f.close() del f diff = fileContents.get().readlines() (newLines, failedHunks) = patch.patch(oldLines, diff) fileContents = filecontents.FromString( "".join(newLines)) assert(not failedHunks) else: # config files are not always available compressed (due # to the config file cache) fileContents = filecontents.FromChangeSet(cs, pathId, newFileId) self.addFileContents(sha1, fileContents, restoreContents, 1) ptrRestores = [] ptrRefsAdded = {} lastRestore = None # restore each pathId,fileId combo once while normalRestoreList: (pathId, fileId, sha1, restoreContents) = normalRestoreList.pop(0) if (pathId, fileId) == lastRestore: continue lastRestore = (pathId, fileId) try: (contType, fileContents) = cs.getFileContents(pathId, fileId, compressed = True) except KeyError: raise errors.IntegrityError( "Missing file contents for pathId %s, fileId %s" % ( sha1helper.md5ToString(pathId), sha1helper.sha1ToString(fileId))) if contType == changeset.ChangedFileTypes.ptr: ptrRestores.append(sha1) target = util.decompressString(fileContents.get().read()) if util.tupleListBsearchInsert(normalRestoreList, (target[:16], target[16:], sha1, True), self.ptrCmp): # Item was inserted. This creates a reference in the # datastore; keep track of it to prevent a duplicate # reference count. ptrRefsAdded[sha1] = True continue assert(contType == changeset.ChangedFileTypes.file) self.addFileContents(sha1, fileContents, restoreContents, 0, precompressed = True) for sha1 in ptrRestores: # Increment the reference count for items which were ptr's # to a different file. if sha1 in ptrRefsAdded: del ptrRefsAdded[sha1] else: self.addFileContents(sha1, None, False, 0) #del configRestoreList #del normalRestoreList for csTrove in newList: if csTrove.troveType() != trove.TROVE_TYPE_REMOVED: continue troveNo += 1 if callback: callback.creatingDatabaseTransaction(troveNo, len(newList)) self.markTroveRemoved(csTrove.getName(), csTrove.getNewVersion(), csTrove.getNewFlavor()) for (troveName, version, flavor) in cs.getOldTroveList(): trv = self.repos.getTrove(troveName, version, flavor) self.oldTrove(trv, None, troveName, version, flavor)
def __init__(self, repos, cs, fileHostFilter=[], callback=None, resetTimestamps=False, allowIncomplete=False, hidden=False, mirror=False, excludeCapsuleContents=False): self.repos = repos self.cs = cs self.invalidateRollbacksFlag = False newList = [x for x in cs.iterNewTroveList()] if resetTimestamps: # This depends intimiately on the versions cache. We don't # change the timestamps on each version, because the cache # ensures they are all a single underlying object. Slick, # but brittle? updated = {} for csTrove in newList: ver = csTrove.getNewVersion() if ver in updated: pass else: oldVer = ver.copy() ver.trailingRevision().resetTimeStamp() updated[oldVer] = ver del updated troveNo, configRestoreList, normalRestoreList = \ self._createInstallTroveObjects(fileHostFilter = fileHostFilter, callback = callback, mirror = mirror, hidden = hidden, allowIncomplete = allowIncomplete, excludeCapsuleContents = excludeCapsuleContents) configRestoreList, normalRestoreList = \ self._filterRestoreList(configRestoreList, normalRestoreList) # use a key to select data up to, but not including, the first # version. We can't sort on version because we don't have timestamps configRestoreList.sort(key=lambda x: x[0:5]) normalRestoreList.sort(key=lambda x: x[0:3]) # config files are cached, so we don't have to worry about not # restoring the same fileId/pathId twice for (pathId, newFileId, sha1, oldfile, newFileId, oldVersion, oldFileId, restoreContents) in configRestoreList: if cs.configFileIsDiff(pathId, newFileId): (contType, fileContents) = cs.getFileContents(pathId, newFileId) # the content for this file is in the form of a # diff, which we need to apply against the file in # the repository assert (oldVersion) try: f = self.repos.getFileContents([(oldFileId, oldVersion, oldfile)])[0].get() except KeyError: raise errors.IntegrityError( "Missing file contents for pathId %s, fileId %s" % (sha1helper.md5ToString(pathId), sha1helper.sha1ToString(fileId))) oldLines = f.readlines() f.close() del f diff = fileContents.get().readlines() (newLines, failedHunks) = patch.patch(oldLines, diff) fileContents = filecontents.FromString("".join(newLines)) assert (not failedHunks) else: # config files are not always available compressed (due # to the config file cache) fileContents = filecontents.FromChangeSet( cs, pathId, newFileId) self.addFileContents(sha1, fileContents, restoreContents, 1) ptrRestores = [] ptrRefsAdded = {} lastRestore = None # restore each pathId,fileId combo once while normalRestoreList: (pathId, fileId, sha1, restoreContents) = normalRestoreList.pop(0) if (pathId, fileId) == lastRestore: continue lastRestore = (pathId, fileId) try: (contType, fileContents) = cs.getFileContents(pathId, fileId, compressed=True) except KeyError: raise errors.IntegrityError( "Missing file contents for pathId %s, fileId %s" % (sha1helper.md5ToString(pathId), sha1helper.sha1ToString(fileId))) if contType == changeset.ChangedFileTypes.ptr: ptrRestores.append(sha1) target = util.decompressString(fileContents.get().read()) if util.tupleListBsearchInsert( normalRestoreList, (target[:16], target[16:], sha1, True), self.ptrCmp): # Item was inserted. This creates a reference in the # datastore; keep track of it to prevent a duplicate # reference count. ptrRefsAdded[sha1] = True continue assert (contType == changeset.ChangedFileTypes.file) self.addFileContents(sha1, fileContents, restoreContents, 0, precompressed=True) for sha1 in ptrRestores: # Increment the reference count for items which were ptr's # to a different file. if sha1 in ptrRefsAdded: del ptrRefsAdded[sha1] else: self.addFileContents(sha1, None, False, 0) #del configRestoreList #del normalRestoreList for csTrove in newList: if csTrove.troveType() != trove.TROVE_TYPE_REMOVED: continue troveNo += 1 if callback: callback.creatingDatabaseTransaction(troveNo, len(newList)) self.markTroveRemoved(csTrove.getName(), csTrove.getNewVersion(), csTrove.getNewFlavor()) for (troveName, version, flavor) in cs.getOldTroveList(): trv = self.repos.getTrove(troveName, version, flavor) self.oldTrove(trv, None, troveName, version, flavor)