def testConfigFilesRaisePathIdsConflict(self): # test to make sure that one changeset's config cache doesn't # override another's cs1 = changeset.ChangeSet() cs2 = changeset.ChangeSet() mergeSet = changeset.ReadOnlyChangeSet() # build two changesets, both with config file diffs that have the same # pathid and fileid cs1.addFileContents('0' * 16, '0' * 20, changeset.ChangedFileTypes.diff, filecontents.FromString('first'), cfgFile = True) cs2.addFileContents('0' * 16, '0' * 20, changeset.ChangedFileTypes.diff, filecontents.FromString('second'), cfgFile = True) mergeSet.merge(cs1) # second merge now handled without ChangeSetKeyConflictError: CNY-3635 mergeSet.merge(cs1) cs1 = changeset.ChangeSet() cs1.addFileContents('0' * 16, '0' * 20, changeset.ChangedFileTypes.diff, filecontents.FromString('first'), cfgFile = True) try: cs1.addFileContents('0' * 16, '0' * 20, changeset.ChangedFileTypes.diff, filecontents.FromString('second'), cfgFile = True) except changeset.ChangeSetKeyConflictError, e: assert str(e) == 'ChangeSetKeyConflictError: 30303030303030303030303030303030,3030303030303030303030303030303030303030'
def testConfigFileMerges(self): # make sure config files are merged properly; at one point we only # merged diffs, now all our merged cs1 = changeset.ReadOnlyChangeSet() cs2 = changeset.ChangeSet() cs2.addFileContents('0' * 16, '0' * 20, changeset.ChangedFileTypes.file, filecontents.FromString('first'), cfgFile = True) cs2.addFileContents('1' * 16, '1' * 20, changeset.ChangedFileTypes.file, filecontents.FromString('first'), cfgFile = True) cs1.merge(cs2) assert(len(cs1.configCache) == 2)
def __init__(self, **kwargs): _File.__init__(self, **kwargs) if isinstance(self.contents, str): self.contents = filecontents.FromString(self.contents) else: self.contents = filecontents.FromFile(self.contents) self._enforceMutuallyExclusiveFlags()
def testOne(s): # test compression of large files for CNY-1896 d = changeset.DictAsCsf({ 'id': (changeset.ChangedFileTypes.file, filecontents.FromString(s), False) }) f = d.getNextFile()[2] gzf = gzip.GzipFile('', 'r', fileobj=f) assert (gzf.read() == s) return f
def testJournal(self): class Journal: def __init__(self): self.perms = [] self.devnodes = [] def lchown(self, root, target, uid, gid): self.perms.append((root, target, uid, gid)) def mknod(self, root, target, devtype, major, minor, mode, uid, gid): self.devnodes.append((root, target, devtype, major, minor, mode, uid, gid)) path = tempfile.mkdtemp() try: journal = Journal() filelist = [] for (name, cls) in (('/dev/block', files.BlockDevice), ('/dev/char', files.CharacterDevice)): d = cls(None) d.inode.perms.set(0604) d.inode.mtime.set(0100) d.inode.owner.set("daemon") d.inode.group.set("uucp") # to make sure that referenced names "exist" files.userCache.nameCache['daemon'] = 2 files.groupCache.nameCache['uucp'] = 14 d.flags.set(0) d.devt.major.set(3) d.devt.minor.set(1) filelist.append((name, d)) for name, d in filelist: p = path + name d.restore(None, path, p, journal=journal) assert(journal.devnodes == [(path, path + '/dev/block', 'b', 3, 1, 0604, 'daemon', 'uucp'), (path, path + '/dev/char', 'c', 3, 1, 0604, 'daemon', 'uucp')]) d = files.RegularFile(None) d.inode.perms.set(1755) d.inode.owner.set('root') d.inode.group.set('root') d.inode.mtime.set(0100) contents = filecontents.FromString('Hello, world') d.restore(contents, path, path + '/sbin/ping', journal=journal) assert(journal.perms == [(path, path + '/sbin/ping', 'root', 'root')]) finally: shutil.rmtree(path)
def save(self, path): # return early if we aren't going to have permission to save try: fd, cacheName = tempfile.mkstemp( prefix=os.path.basename(path) + '.', dir=os.path.dirname(path)) os.close(fd) except (IOError, OSError): # may not have permissions; say, not running as root return cs = changeset.ChangeSet() for withFiles, trv in self.cache.values(): # we just assume everything in the cache is w/o files. it's # fine for system model, safe, and we don't need the cache # anywhere else cs.newTrove(trv.diff(None, absolute = True)[0]) # NB: "fileid" and pathid got reversed here by mistake, try not to # think too hard about it. cs.addFileContents( self._fileId, self._troveCacheVersionPathId, changeset.ChangedFileTypes.file, filecontents.FromString("%d %d" % self.VERSION), False) self._cs = cs self._saveTimestamps() self._saveDeps() self._saveDepSolutions() self._saveFileCache() self._cs = None try: try: cs.writeToFile(cacheName) if util.exists(path): os.chmod(cacheName, os.stat(path).st_mode) else: os.chmod(cacheName, 0644) os.rename(cacheName, path) except (IOError, OSError): # may not have permissions; say, not running as root pass finally: try: if os.path.exists(cacheName): os.remove(cacheName) except OSError: pass
def testFileChanges(self): (built, d) = self.buildRecipe(recipes.testRecipe1, "TestRecipe1") (name, version, flavor) = built[0] version = versions.VersionFromString(version) (fd, path) = tempfile.mkstemp() os.close(fd) repos = self.openRepository() repos.createChangeSetFile( [(name, (None, None), (version, flavor), 1)], path) infc = filecontainer.FileContainer(open(path)) os.unlink(path) (fd, path) = tempfile.mkstemp() os.close(fd) outfc = filecontainer.FileContainer(open(path, "w")) n = 0 while True: next = infc.getNextFile() if not next: break (name, tag, f) = next n += 1 if n == 2 or n == 5: # modify the file a bit contents = gzip.GzipFile(None, "r", fileobj = f).read() contents = chr(ord(contents[0]) ^ 0xff) + contents[1:] outfc.addFile(name, filecontents.FromString(contents), tag) else: contents = filecontents.FromFile(f) outfc.addFile(name, filecontents.FromFile(f), tag, precompressed=True) infc.close() outfc.close() self.resetRepository() repos = self.openRepository() try: self.assertRaises(errors.IntegrityError, self.updatePkg, self.rootDir, path) self.assertRaises(errors.IntegrityError, repos.commitChangeSetFile, path) finally: os.unlink(path)
def testFileId(self): # this test verifies that the value produced as the fileId # of a known stream matches its pre-calculated value. f = files.RegularFile(None) f.inode.perms.set(0604) f.inode.mtime.set(0100) f.inode.owner.set("daemon") f.inode.group.set("uucp") # to make sure that referenced names "exist" files.userCache.nameCache['daemon'] = 2 files.groupCache.nameCache['uucp'] = 14 s = "hello world" contents = filecontents.FromString(s) f.contents = files.RegularFileStream() f.contents.size.set(len(s)) f.contents.sha1.set(sha1helper.sha1String(s)) f.flags.set(0) expectedId = '567355867fbbcb2be55d35c3d229a7df8152fdbc' self.assertEqual(f.freeze(), '-\x01\x00"\x01\x00\x08\x00\x00\x00\x00\x00\x00\x00\x0b\x02\x00\x14*\xael5\xc9O\xcf\xb4\x15\xdb\xe9_@\x8b\x9c\xe9\x1e\xe8F\xed\x03\x00\x04\x00\x00\x00\x00\x05\x00\x1c\x01\x00\x02\x01\x84\x02\x00\x04\x00\x00\x00@\x03\x00\x06daemon\x04\x00\x04uucp') self.assertEqual(sha1helper.sha1ToString(f.fileId()), expectedId)
def testChangeSetMerge(self): os.chdir(self.workDir) cs1 = changeset.ChangeSet() p1 = '0' * 16 f1 = '0' * 20 cs1.addFileContents(p1, f1, changeset.ChangedFileTypes.file, filecontents.FromString('zero'), False) assert (cs1.writeToFile('foo.ccs') == 129) cs2 = changeset.ReadOnlyChangeSet() cs2.merge(cs1) assert (cs2.writeToFile('foo.ccs') == 129) cs2.reset() assert (cs2.writeToFile('foo.ccs') == 129) cs2.reset() cs3 = changeset.ReadOnlyChangeSet() cs3.merge(cs2) assert (cs3.writeToFile('foo.ccs') == 129) cs3.reset() assert (cs3.writeToFile('foo.ccs') == 129)
def _restoreConfig(self, cs, configRestoreList): # config files are cached, so we don't have to worry about not # restoring the same fileId/pathId twice for (pathId, newFileId, sha1, oldfile, newFileId, oldVersion, oldFileId, restoreContents) in configRestoreList: if cs.configFileIsDiff(pathId, newFileId): (contType, fileContents) = cs.getFileContents(pathId, newFileId) # the content for this file is in the form of a # diff, which we need to apply against the file in # the repository assert (oldVersion) try: f = self.repos.getFileContents([(oldFileId, oldVersion, oldfile)])[0].get() except KeyError: raise errors.IntegrityError( "Missing file contents for pathId %s, fileId %s" % (sha1helper.md5ToString(pathId), sha1helper.sha1ToString(oldFileId))) oldLines = f.readlines() f.close() del f diff = fileContents.get().readlines() (newLines, failedHunks) = patch.patch(oldLines, diff) fileContents = filecontents.FromString("".join(newLines)) assert (not failedHunks) else: # config files are not always available compressed (due # to the config file cache) fileContents = filecontents.FromChangeSet( cs, pathId, newFileId) self.addFileContents(sha1, fileContents, restoreContents, 1)
def test07_BadRpmCapsule(self): cmp = self.addRPMComponent("ghost:rpm=1.0", 'epoch-1.0-1.i386.rpm') repos = self.openRepository() orig = self.workDir + '/ghost.ccs' modified = self.workDir + '/ghost-new.ccs' repos.createChangeSetFile([ (cmp.getName(), (None, None), cmp.getNameVersionFlavor()[1:], True) ], orig) fc = filecontainer.FileContainer( util.ExtendedFile(orig, buffering = False)) newFc = filecontainer.FileContainer( util.ExtendedFile(modified, "w", buffering = False)) # CONARYCHANGESET (name, tag, contents) = fc.getNextFile() newFc.addFile(name, filecontents.FromFile(contents), tag, precompressed = True) # the RPM (name, tag, contents) = fc.getNextFile() contents = filecontents.FromString("busted!") newFc.addFile(name, contents, tag) cs = changeset.ChangeSetFromFile(modified) # this throws away the output about the install failing self.assertRaises(files.Sha1Exception, self.captureOutput, self.updatePkg, self.rootDir, cs)
def _addPhantomContents(self, changeSet, trv, header): """Fabricate files for the given RPM header""" for (path, owner, group, mode, size, rdev, flags, vflags, linkto, mtime) in itertools.izip( header[rpmhelper.OLDFILENAMES], header[rpmhelper.FILEUSERNAME], header[rpmhelper.FILEGROUPNAME], header[rpmhelper.FILEMODES], header[rpmhelper.FILESIZES], header[rpmhelper.FILERDEVS], header[rpmhelper.FILEFLAGS], header[rpmhelper.FILEVERIFYFLAGS], header[rpmhelper.FILELINKTOS], header[rpmhelper.FILEMTIMES], ): fullPath = util.joinPaths(self.root, path) fakestat = FakeStat(mode, 0, None, 1, owner, group, size, mtime, mtime, mtime, st_rdev=rdev, linkto=linkto) pathId = os.urandom(16) # Adapted from conary.build.source.addCapsule.doRPM kind = 'regular' if flags & rpmhelper.RPMFILE_GHOST: kind = 'initial' elif flags & (rpmhelper.RPMFILE_CONFIG | rpmhelper.RPMFILE_MISSINGOK | rpmhelper.RPMFILE_NOREPLACE): if size: kind = 'config' else: kind = 'initial' elif vflags: if (stat.S_ISREG(mode) and not (vflags & rpmhelper.RPMVERIFY_FILEDIGEST) or (stat.S_ISLNK(mode) and not (vflags & rpmhelper.RPMVERIFY_LINKTO))): kind = 'initial' # Ignore failures trying to sha1 missing/inaccessible files as long # as those files are flagged initial contents (ghost) fileStream = files.FileFromFilesystem(fullPath, pathId, statBuf=fakestat, sha1FailOk=True) if kind == 'config': fileStream.flags.isConfig(set=True) elif kind == 'initial': fileStream.flags.isInitialContents(set=True) else: assert kind == 'regular' # From conary.build.capsulepolicy.Payload if (isinstance(fileStream, files.RegularFile) and not fileStream.flags.isConfig() and not (fileStream.flags.isInitialContents() and not fileStream.contents.size())): fileStream.flags.isEncapsulatedContent(set=True) fileId = fileStream.fileId() trv.addFile(pathId, path, trv.getVersion(), fileId) changeSet.addFile(None, fileId, fileStream.freeze()) # Config file contents have to go into the database, so snag the # contents from the filesystem and put them in the changeset. if (fileStream.hasContents and not fileStream.flags.isEncapsulatedContent()): if fileStream.contents.sha1() == sha1helper.sha1Empty: # Missing/ghost config file. Hopefully it is supposed to be # empty, but even if not then the fake SHA-1 will be the # SHA-1 of the empty string since there's no hint of what # it was supposed to be. contents = filecontents.FromString('') else: contents = filecontents.FromFilesystem(fullPath) changeSet.addFileContents( pathId, fileId, contType=changeset.ChangedFileTypes.file, contents=contents, cfgFile=fileStream.flags.isConfig(), )
def _savePickle(self, pathId, data): pickled = cPickle.dumps(data, 2) self._cs.addFileContents(pathId, self._fileId, changeset.ChangedFileTypes.file, filecontents.FromString(pickled), False)
cs1.addFileContents('0' * 16, '0' * 20, changeset.ChangedFileTypes.diff, filecontents.FromString('first'), cfgFile = True) try: cs1.addFileContents('0' * 16, '0' * 20, changeset.ChangedFileTypes.diff, filecontents.FromString('second'), cfgFile = True) except changeset.ChangeSetKeyConflictError, e: assert str(e) == 'ChangeSetKeyConflictError: 30303030303030303030303030303030,3030303030303030303030303030303030303030' else: assert(0) cs1 = changeset.ChangeSet() # this is blatantly illegal; diff non-config files! cs1.addFileContents('0' * 16, '0' * 20, changeset.ChangedFileTypes.diff, filecontents.FromString('first'), cfgFile = False) try: cs1.addFileContents('0' * 16, '0' * 20, changeset.ChangedFileTypes.diff, filecontents.FromString('second'), cfgFile = False) except changeset.ChangeSetKeyConflictError, e: assert str(e) == 'ChangeSetKeyConflictError: 30303030303030303030303030303030,3030303030303030303030303030303030303030' else: assert(0) # build a changeset, both with two config files with the same # pathid and fileid. One is a diff and the other is not. This should # be ok the diff will be used. cs1 = changeset.ChangeSet() cs1.addFileContents('0' * 16, '0' * 20,
def testOwnership(self): f = files.RegularFile(None) f.inode.perms.set(0604) f.inode.mtime.set(0100) f.inode.owner.set("daemon") f.inode.group.set("uucp") # to make sure that referenced names "exist" files.userCache.nameCache['daemon'] = 2 files.userCache.nameCache['uucp'] = 10 files.groupCache.nameCache['uucp'] = 14 s = "hello world" contents = filecontents.FromString(s) f.contents = files.RegularFileStream() f.contents.size.set(len(s)) f.contents.sha1.set(sha1helper.sha1String(s)) f.flags.set(0) # and setuid root fr = files.RegularFile(None) fr.inode.perms.set(06755) fr.inode.mtime.set(0100) fr.inode.owner.set("root") fr.inode.group.set("root") fr.contents = files.RegularFileStream() fr.contents.size.set(len(s)) fr.contents.sha1.set(sha1helper.sha1String(s)) fr.flags.set(0) # and unwriteable fo = files.RegularFile(None) fo.inode.perms.set(0444) fo.inode.mtime.set(0100) fo.inode.owner.set("root") fo.inode.group.set("root") fo.contents = files.RegularFileStream() fo.contents.size.set(len(s)) fo.contents.sha1.set(sha1helper.sha1String(s)) fo.flags.set(0) # and secret fs = files.RegularFile(None) fs.inode.perms.set(0400) fs.inode.mtime.set(0100) fs.inode.owner.set("root") fs.inode.group.set("root") fs.contents = files.RegularFileStream() fs.contents.size.set(len(s)) fs.contents.sha1.set(sha1helper.sha1String(s)) fs.flags.set(0) f2 = f.copy() assert(f == f2) d = tempfile.mkdtemp() # before we mimic root, test a non-root of setu/gid file pr = d+"/setuid" fr.restore(contents, d, pr) assert not os.stat(pr).st_mode & 04000 try: self.mimicRoot() p = d + "/file" f.restore(contents, d, p) assert self.compareChownLog([ (p, 2, 14) ]) self.chownLog = [] f.inode.owner.set("rootroot") self.logCheck(f.restore, (contents, d, p), "warning: user rootroot does not exist - using root") assert self.compareChownLog([ (p, 0, 14) ]) self.chownLog = [] f.inode.owner.set("uucp") f.inode.group.set("grpgrp") self.logCheck(f.restore, (contents, d, p), "warning: group grpgrp does not exist - using root") assert self.compareChownLog([ (p, 10, 0) ]) self.chmodLog = [] pr = d+"/setuid" fr.restore(contents, d, pr) assert self.compareChmodLog([ (pr, 06755) ]) assert os.stat(pr).st_mode & 07777 == 06755 self.chmodLog = [] po = d+"/unwriteable" fo.restore(contents, d, po) assert self.compareChmodLog([ (po, 0444) ]) assert os.stat(po).st_mode & 07777 == 0444 self.chmodLog = [] ps = d+"/secret" fs.restore(contents, d, ps) assert self.compareChmodLog([ (ps, 0400) ]) assert os.stat(ps).st_mode & 07777 == 0400 self.chmodLog = [] finally: self.realRoot() shutil.rmtree(d)
def createChangeSet(self, origTroveList, recurse = True, withFiles = True, withFileContents = True, excludeAutoSource = False, mirrorMode = False, roleIds = None): """ @param origTroveList: a list of C{(troveName, flavor, oldVersion, newVersion, absolute)} tuples. If C{oldVersion == None} and C{absolute == 0}, then the trove is assumed to be new for the purposes of the change set. If C{newVersion == None} then the trove is being removed. If recurse is set, this yields one result for the entire troveList. If recurse is not set, it yields one result per troveList entry. """ cs = changeset.ChangeSet() externalTroveList = [] externalFileList = [] removedTroveList = [] dupFilter = set() # make a copy to remove things from troveList = origTroveList[:] # def createChangeSet begins here troveWrapper = _TroveListWrapper(troveList, self.troveStore, withFiles, roleIds = roleIds) for (job, old, new, streams) in troveWrapper: (troveName, (oldVersion, oldFlavor), (newVersion, newFlavor), absolute) = job # make sure we haven't already generated this changeset; since # troves can be included from other troves we could try # to generate quite a few duplicates if job in dupFilter: continue else: dupFilter.add(job) done = False if not newVersion: if oldVersion.getHost() not in self.serverNameList: externalTroveList.append((troveName, (oldVersion, oldFlavor), (None, None), absolute)) else: # remove this trove and any trove contained in it cs.oldTrove(troveName, oldVersion, oldFlavor) for (name, version, flavor) in \ old.iterTroveList(strongRefs=True): troveWrapper.append((name, (version, flavor), (None, None), absolute), False) done = True elif (newVersion.getHost() not in self.serverNameList or (oldVersion and oldVersion.getHost() not in self.serverNameList)): # don't try to make changesets between repositories; the # client can do that itself # we don't generate chagnesets between removed and # present troves; that's up to the client externalTroveList.append((troveName, (oldVersion, oldFlavor), (newVersion, newFlavor), absolute)) done = True elif (oldVersion and old.type() == trove.TROVE_TYPE_REMOVED): removedTroveList.append((troveName, (oldVersion, oldFlavor), (newVersion, newFlavor), absolute)) done = True if done: if not recurse: yield (cs, externalTroveList, externalFileList, removedTroveList) cs = changeset.ChangeSet() externalTroveList = [] externalFileList = [] removedTroveList = [] continue (troveChgSet, filesNeeded, pkgsNeeded) = \ new.diff(old, absolute = absolute) if recurse: for refJob in pkgsNeeded: refOldVersion = refJob[1][0] refNewVersion = refJob[2][0] if (refNewVersion and (refNewVersion.getHost() not in self.serverNameList) or (refOldVersion and refOldVersion.getHost() not in self.serverNameList) ): # don't try to make changesets between repositories; the # client can do that itself externalTroveList.append(refJob) else: troveWrapper.append(refJob, True) cs.newTrove(troveChgSet) if job in origTroveList and job[2][0] is not None: # add the primary w/ timestamps on the version try: primary = troveChgSet.getNewNameVersionFlavor() cs.addPrimaryTrove(*primary) except KeyError: # primary troves could be in the externalTroveList, in # which case they aren't primries pass # sort the set of files we need into bins based on the server # name getList = [] localFilesNeeded = [] for (pathId, oldFileId, oldFileVersion, newFileId, newFileVersion) in filesNeeded: # if either the old or new file version is on a different # repository, creating this diff is someone else's problem if (newFileVersion.getHost() not in self.serverNameList or (oldFileVersion and oldFileVersion.getHost() not in self.serverNameList)): externalFileList.append((pathId, troveName, (oldVersion, oldFlavor, oldFileId, oldFileVersion), (newVersion, newFlavor, newFileId, newFileVersion))) else: localFilesNeeded.append((pathId, oldFileId, oldFileVersion, newFileId, newFileVersion)) if oldFileVersion: getList.append((pathId, oldFileId, oldFileVersion)) getList.append((pathId, newFileId, newFileVersion)) # Walk this in reverse order. This may seem odd, but the # order in the final changeset is set by sorting that happens # in the change set object itself. The only reason we sort # here at all is to make sure PTR file types come before the # file they refer to. Reverse shorting makes this a bit easier. localFilesNeeded.sort() localFilesNeeded.reverse() ptrTable = {} for (pathId, oldFileId, oldFileVersion, newFileId, \ newFileVersion) in localFilesNeeded: oldFile = None if oldFileVersion: oldFile = files.ThawFile(streams[oldFileId], pathId) oldCont = None newCont = None newFile = files.ThawFile(streams[newFileId], pathId) # Skip identical fileids when mirroring, but always use # absolute file changes if there is any difference. See note # below. forceAbsolute = (mirrorMode and oldFileId and oldFileId != newFileId) if forceAbsolute: (filecs, contentsHash) = changeset.fileChangeSet(pathId, None, newFile) else: (filecs, contentsHash) = changeset.fileChangeSet(pathId, oldFile, newFile) cs.addFile(oldFileId, newFileId, filecs) if (not withFileContents or (excludeAutoSource and newFile.flags.isAutoSource()) or (newFile.flags.isEncapsulatedContent() and not newFile.flags.isCapsuleOverride())): continue # this test catches files which have changed from not # config files to config files; these need to be included # unconditionally so we always have the pristine contents # to include in the local database # Also include contents of config files when mirroring if the # fileid changed, even if the SHA-1 did not. # cf CNY-1570, CNY-1699, CNY-2210 if (contentsHash or (oldFile and newFile.flags.isConfig() and not oldFile.flags.isConfig()) or (forceAbsolute and newFile.hasContents) ): if oldFileVersion and oldFile.hasContents: oldCont = self.getFileContents( [ (oldFileId, oldFileVersion, oldFile) ])[0] newCont = self.getFileContents( [ (newFileId, newFileVersion, newFile) ])[0] (contType, cont) = changeset.fileContentsDiff(oldFile, oldCont, newFile, newCont, mirrorMode = mirrorMode) # we don't let config files be ptr types; if they were # they could be ptrs to things which aren't config files, # which would completely hose the sort order we use. this # could be relaxed someday to let them be ptr's to other # config files if not newFile.flags.isConfig() and \ contType == changeset.ChangedFileTypes.file: contentsHash = newFile.contents.sha1() ptr = ptrTable.get(contentsHash, None) if ptr is not None: contType = changeset.ChangedFileTypes.ptr cont = filecontents.FromString(ptr) else: ptrTable[contentsHash] = pathId + newFileId if not newFile.flags.isConfig() and \ contType == changeset.ChangedFileTypes.file: cont = filecontents.CompressedFromDataStore( self.contentsStore, newFile.contents.sha1()) compressed = True else: compressed = False # ptr entries are not compressed, whether or not they # are config files. override the compressed rule from # above if contType == changeset.ChangedFileTypes.ptr: compressed = False cs.addFileContents(pathId, newFileId, contType, cont, newFile.flags.isConfig(), compressed = compressed) if not recurse: yield cs, externalTroveList, externalFileList, removedTroveList cs = changeset.ChangeSet() externalTroveList = [] externalFileList = [] removedTroveList = [] if recurse: yield cs, externalTroveList, externalFileList, removedTroveList
def __init__(self, repos, cs, fileHostFilter=[], callback=None, resetTimestamps=False, allowIncomplete=False, hidden=False, mirror=False, excludeCapsuleContents=False): self.repos = repos self.cs = cs self.invalidateRollbacksFlag = False newList = [x for x in cs.iterNewTroveList()] if resetTimestamps: # This depends intimiately on the versions cache. We don't # change the timestamps on each version, because the cache # ensures they are all a single underlying object. Slick, # but brittle? updated = {} for csTrove in newList: ver = csTrove.getNewVersion() if ver in updated: pass else: oldVer = ver.copy() ver.trailingRevision().resetTimeStamp() updated[oldVer] = ver del updated troveNo, configRestoreList, normalRestoreList = \ self._createInstallTroveObjects(fileHostFilter = fileHostFilter, callback = callback, mirror = mirror, hidden = hidden, allowIncomplete = allowIncomplete, excludeCapsuleContents = excludeCapsuleContents) configRestoreList, normalRestoreList = \ self._filterRestoreList(configRestoreList, normalRestoreList) # use a key to select data up to, but not including, the first # version. We can't sort on version because we don't have timestamps configRestoreList.sort(key=lambda x: x[0:5]) normalRestoreList.sort(key=lambda x: x[0:3]) # config files are cached, so we don't have to worry about not # restoring the same fileId/pathId twice for (pathId, newFileId, sha1, oldfile, newFileId, oldVersion, oldFileId, restoreContents) in configRestoreList: if cs.configFileIsDiff(pathId, newFileId): (contType, fileContents) = cs.getFileContents(pathId, newFileId) # the content for this file is in the form of a # diff, which we need to apply against the file in # the repository assert (oldVersion) try: f = self.repos.getFileContents([(oldFileId, oldVersion, oldfile)])[0].get() except KeyError: raise errors.IntegrityError( "Missing file contents for pathId %s, fileId %s" % (sha1helper.md5ToString(pathId), sha1helper.sha1ToString(fileId))) oldLines = f.readlines() f.close() del f diff = fileContents.get().readlines() (newLines, failedHunks) = patch.patch(oldLines, diff) fileContents = filecontents.FromString("".join(newLines)) assert (not failedHunks) else: # config files are not always available compressed (due # to the config file cache) fileContents = filecontents.FromChangeSet( cs, pathId, newFileId) self.addFileContents(sha1, fileContents, restoreContents, 1) ptrRestores = [] ptrRefsAdded = {} lastRestore = None # restore each pathId,fileId combo once while normalRestoreList: (pathId, fileId, sha1, restoreContents) = normalRestoreList.pop(0) if (pathId, fileId) == lastRestore: continue lastRestore = (pathId, fileId) try: (contType, fileContents) = cs.getFileContents(pathId, fileId, compressed=True) except KeyError: raise errors.IntegrityError( "Missing file contents for pathId %s, fileId %s" % (sha1helper.md5ToString(pathId), sha1helper.sha1ToString(fileId))) if contType == changeset.ChangedFileTypes.ptr: ptrRestores.append(sha1) target = util.decompressString(fileContents.get().read()) if util.tupleListBsearchInsert( normalRestoreList, (target[:16], target[16:], sha1, True), self.ptrCmp): # Item was inserted. This creates a reference in the # datastore; keep track of it to prevent a duplicate # reference count. ptrRefsAdded[sha1] = True continue assert (contType == changeset.ChangedFileTypes.file) self.addFileContents(sha1, fileContents, restoreContents, 0, precompressed=True) for sha1 in ptrRestores: # Increment the reference count for items which were ptr's # to a different file. if sha1 in ptrRefsAdded: del ptrRefsAdded[sha1] else: self.addFileContents(sha1, None, False, 0) #del configRestoreList #del normalRestoreList for csTrove in newList: if csTrove.troveType() != trove.TROVE_TYPE_REMOVED: continue troveNo += 1 if callback: callback.creatingDatabaseTransaction(troveNo, len(newList)) self.markTroveRemoved(csTrove.getName(), csTrove.getNewVersion(), csTrove.getNewFlavor()) for (troveName, version, flavor) in cs.getOldTroveList(): trv = self.repos.getTrove(troveName, version, flavor) self.oldTrove(trv, None, troveName, version, flavor)