def testConfigFilesRaisePathIdsConflict(self): # test to make sure that one changeset's config cache doesn't # override another's cs1 = changeset.ChangeSet() cs2 = changeset.ChangeSet() mergeSet = changeset.ReadOnlyChangeSet() # build two changesets, both with config file diffs that have the same # pathid and fileid cs1.addFileContents('0' * 16, '0' * 20, changeset.ChangedFileTypes.diff, filecontents.FromString('first'), cfgFile = True) cs2.addFileContents('0' * 16, '0' * 20, changeset.ChangedFileTypes.diff, filecontents.FromString('second'), cfgFile = True) mergeSet.merge(cs1) # second merge now handled without ChangeSetKeyConflictError: CNY-3635 mergeSet.merge(cs1) cs1 = changeset.ChangeSet() cs1.addFileContents('0' * 16, '0' * 20, changeset.ChangedFileTypes.diff, filecontents.FromString('first'), cfgFile = True) try: cs1.addFileContents('0' * 16, '0' * 20, changeset.ChangedFileTypes.diff, filecontents.FromString('second'), cfgFile = True) except changeset.ChangeSetKeyConflictError, e: assert str(e) == 'ChangeSetKeyConflictError: 30303030303030303030303030303030,3030303030303030303030303030303030303030'
def testFileIdWrong(self): # create an absolute changeset cs = changeset.ChangeSet() # add a pkg diff flavor = deps.deps.parseFlavor('') v = versions.VersionFromString('/%s/1.0-1-1' %self.cfg.buildLabel.asString()).copy() v.resetTimeStamps() t = trove.Trove('test:test', v, flavor, None) path = self.workDir + '/blah' f = open(path, 'w') f.write('hello, world!\n') f.close() pathId = sha1helper.md5String('/blah') f = files.FileFromFilesystem(path, pathId) # add the file, but munge the fileid brokenFileId = ''.join(reversed(f.fileId())) cs.addFile(None, brokenFileId, f.freeze()) t.addFile(pathId, '/blah', v, brokenFileId) t.computeDigests() diff = t.diff(None, absolute = 1)[0] cs.newTrove(diff) repos = self.openRepository() try: repos.commitChangeSet(cs) assert 0, "Integrity Error not raised" except errors.TroveIntegrityError, e: assert(str(e) == 'fileObj.fileId() != fileId in changeset for ' 'pathId %s' % sha1helper.md5ToString(pathId))
def getChangeSetForCapsuleChanges(self, callback): self.loadPlugins() changeSet = changeset.ChangeSet() for plugin in self._loadedPlugins.itervalues(): callback.capsuleSyncScan(plugin.kind) plugin.addCapsuleChangesToChangeSet(changeSet, callback) return changeSet
def _createCs(version): # create an absolute changeset flavor = deps.parseFlavor('') cs = changeset.ChangeSet() # add a pkg diff v = versions.VersionFromString(version, timeStamps=[1.000]) old = trove.Trove('test', v, flavor, None) old.setIsCollection(True) old.addTrove('test:foo', v, flavor, byDefault=True) old.addTrove('test:bar', v, flavor, byDefault=False) old.computeDigests() # add the 'test' package diff = old.diff(None)[0] cs.newTrove(diff) cs.addPrimaryTrove('test', v, flavor) # add the test:foo component oldfoo = trove.Trove('test:foo', v, flavor, None) oldfoo.computeDigests() diff = oldfoo.diff(None)[0] cs.newTrove(diff) # add the test:bar component oldbar = trove.Trove('test:bar', v, flavor, None) oldbar.computeDigests() diff = oldbar.diff(None)[0] cs.newTrove(diff) return cs
def testFileObjMissing(self): # create an absolute changeset cs = changeset.ChangeSet() # add a pkg diff flavor = deps.deps.parseFlavor('') v = versions.VersionFromString('/%s/1.0-1-1' %self.cfg.buildLabel.asString()).copy() v.resetTimeStamps() t = trove.Trove('test:test', v, flavor, None) path = self.workDir + '/blah' f = open(path, 'w') f.write('hello, world!\n') f.close() pathId = sha1helper.md5String('/blah') f = files.FileFromFilesystem(path, pathId) # add the file, and SKIP including # the filestream by using cs.addFile(). This creates an # incomplete changeset t.addFile(pathId, '/blah', v, f.fileId()) cs.addFileContents(pathId, f.fileId(), changeset.ChangedFileTypes.file, filecontents.FromFilesystem(path), f.flags.isConfig()) t.computeDigests() diff = t.diff(None, absolute = 1)[0] cs.newTrove(diff) repos = self.openRepository() try: repos.commitChangeSet(cs) assert 0, "Did not raise IntegrityError" except errors.IntegrityError, e: assert(str(e).startswith("Incomplete changeset specified: missing pathId e806729b6a2b568fa7e77c3efa3a9684 fileId"))
def testFileContentsMissing(self): # currently causes a 500 error #raise testhelp.SkipTestException # create an absolute changeset cs = changeset.ChangeSet() # add a pkg diff flavor = deps.deps.parseFlavor('') v = versions.VersionFromString('/%s/1.0-1-1' %self.cfg.buildLabel.asString()).copy() v.resetTimeStamps() t = trove.Trove('test:test', v, flavor, None) path = self.workDir + '/blah' f = open(path, 'w') f.write('hello, world!\n') f.close() pathId = sha1helper.md5String('/blah') f = files.FileFromFilesystem(path, pathId) # add the file, but munge the fileid fileId = f.fileId() cs.addFile(None, fileId, f.freeze()) t.addFile(pathId, '/blah', v, fileId) # skip adding the file contents t.computeDigests() diff = t.diff(None, absolute = 1)[0] cs.newTrove(diff) repos = self.openRepository() try: repos.commitChangeSet(cs) assert 0, "Did not raise integrity error" except errors.IntegrityError, e: assert(str(e).startswith("Missing file contents for pathId e806729b6a2b568fa7e77c3efa3a9684, fileId"))
def _createTroves(self, troveAndPathList): cs = changeset.ChangeSet() troveList = [x[0] for x in troveAndPathList] previousVersionMap = self._targetNewTroves(troveList) self._addAllNewFiles(cs, troveAndPathList, previousVersionMap) for trv in troveList: trv.computeDigests() trvCs = trv.diff(None, absolute=True)[0] cs.newTrove(trvCs) return cs
def testConfigFileMerges(self): # make sure config files are merged properly; at one point we only # merged diffs, now all our merged cs1 = changeset.ReadOnlyChangeSet() cs2 = changeset.ChangeSet() cs2.addFileContents('0' * 16, '0' * 20, changeset.ChangedFileTypes.file, filecontents.FromString('first'), cfgFile = True) cs2.addFileContents('1' * 16, '1' * 20, changeset.ChangedFileTypes.file, filecontents.FromString('first'), cfgFile = True) cs1.merge(cs2) assert(len(cs1.configCache) == 2)
def testFileUpdateMissingKey(self): fingerprint = '95B457D16843B21EA3FC73BBC7C32FC1F94E405E' # make a changeset with useful stuff that could be installed cs = changeset.ChangeSet() flavor = deps.parseFlavor('') v = versions.VersionFromString('/%s/1.0-1-1' % self.cfg.buildLabel.asString()).copy() v.resetTimeStamps() t = trove.Trove('test:test', v, flavor, None) path = self.workDir + '/blah' f = open(path, 'w') f.write('hello, world!\n') f.close() pathId = sha1helper.md5String('/blah') f = files.FileFromFilesystem(path, pathId) fileId = f.fileId() t.addFile(pathId, '/blah', v, fileId) cs.addFile(None, fileId, f.freeze()) cs.addFileContents(pathId, fileId, changeset.ChangedFileTypes.file, filecontents.FromFilesystem(path), f.flags.isConfig()) diff = t.diff(None, absolute=1)[0] cs.newTrove(diff) cs.addPrimaryTrove('test:test', v, flavor) # sign the changeset csPath = os.path.join(self.workDir, 'test-1.0-1.ccs') cs = cook.signAbsoluteChangeset(cs, fingerprint) cs.writeToFile(csPath) tmpPath = mkdtemp() keyCache = openpgpkey.getKeyCache() newKeyCache = openpgpkey.OpenPGPKeyFileCache() pubRing = self.cfg.pubRing self.cfg.pubRing = [tmpPath + '/pubring.gpg'] keyCacheCallback = openpgpkey.KeyCacheCallback(None, self.cfg) newKeyCache.setCallback(keyCacheCallback) openpgpkey.setKeyCache(newKeyCache) try: self.updatePkg(self.rootDir, csPath) finally: self.cfg.pubRing = pubRing openpgpkey.setKeyCache(keyCache)
def save(self, path): # return early if we aren't going to have permission to save try: fd, cacheName = tempfile.mkstemp( prefix=os.path.basename(path) + '.', dir=os.path.dirname(path)) os.close(fd) except (IOError, OSError): # may not have permissions; say, not running as root return cs = changeset.ChangeSet() for withFiles, trv in self.cache.values(): # we just assume everything in the cache is w/o files. it's # fine for system model, safe, and we don't need the cache # anywhere else cs.newTrove(trv.diff(None, absolute = True)[0]) # NB: "fileid" and pathid got reversed here by mistake, try not to # think too hard about it. cs.addFileContents( self._fileId, self._troveCacheVersionPathId, changeset.ChangedFileTypes.file, filecontents.FromString("%d %d" % self.VERSION), False) self._cs = cs self._saveTimestamps() self._saveDeps() self._saveDepSolutions() self._saveFileCache() self._cs = None try: try: cs.writeToFile(cacheName) if util.exists(path): os.chmod(cacheName, os.stat(path).st_mode) else: os.chmod(cacheName, 0644) os.rename(cacheName, path) except (IOError, OSError): # may not have permissions; say, not running as root pass finally: try: if os.path.exists(cacheName): os.remove(cacheName) except OSError: pass
def _addUnownedNewFiles(self, newFileList): if not newFileList: return cs = changeset.ChangeSet() ver = versions.VersionFromString( '/localhost@local:LOCAL/1.0-1-1').copy() ver.resetTimeStamps() trv = trove.Trove("@new:files", ver, deps.Flavor()) for path in newFileList: self._addFile(cs, trv, path) trvDiff = trv.diff(None, absolute=False)[0] cs.newTrove(trvDiff) self._handleChangeSet([trv.getNameVersionFlavor()], cs)
def testChangeSetFromFile(self): # ensure that absolute changesets that are read from disk # that contain config files write out changesets to a file # that do not change the file type to a diff. # set up a file with some contents cont = self.workDir + '/contents' f = open(cont, 'w') f.write('hello, world!\n') f.close() pathId = sha1helper.md5FromString('0' * 32) f = files.FileFromFilesystem(cont, pathId) f.flags.isConfig(1) # create an absolute changeset cs = changeset.ChangeSet() # add a pkg diff v = versions.VersionFromString('/localhost@rpl:devel/1.0-1-1', timeStamps=[1.000]) flavor = deps.parseFlavor('') t = trove.Trove('test', v, flavor, None) t.addFile(pathId, '/contents', v, f.fileId()) diff = t.diff(None, absolute=1)[0] cs.newTrove(diff) # add the file and file contents cs.addFile(None, f.fileId(), f.freeze()) cs.addFileContents(pathId, f.fileId(), changeset.ChangedFileTypes.file, filecontents.FromFilesystem(cont), f.flags.isConfig()) # write out the changeset cs.writeToFile(self.workDir + '/foo.ccs') # read it back in cs2 = changeset.ChangeSetFromFile(self.workDir + '/foo.ccs') # write it out again (there was a bug where all config files # became diffs) cs2.writeToFile(self.workDir + '/bar.ccs') # read it again cs3 = changeset.ChangeSetFromFile(self.workDir + '/bar.ccs') # verify that the file is a file, not a diff ctype, contents = cs3.getFileContents(pathId, f.fileId()) assert (ctype == changeset.ChangedFileTypes.file)
def testReset(self): class UncloseableFile(StringIO.StringIO): def close(self): assert(0) # Make sure writing a changeset doesn't close the filecontent objects # inside of it. otherCs = changeset.ChangeSet() otherCs.addFileContents("0pathId", "0fileId", changeset.ChangedFileTypes.file, filecontents.FromFile(UncloseableFile("foo")), 1) otherCs.addFileContents("1pathId", "1fileId", changeset.ChangedFileTypes.file, filecontents.FromFile(UncloseableFile("foo")), 0) otherCs.writeToFile(self.workDir + "/test1.ccs") cs = changeset.ReadOnlyChangeSet() cs.merge(otherCs) cs.writeToFile(self.workDir + "/test2.ccs") assert(open(self.workDir + "/test1.ccs").read() == (open(self.workDir + "/test2.ccs").read())) cs.reset() cs.writeToFile(self.workDir + "/test3.ccs") assert(open(self.workDir + "/test1.ccs").read() == (open(self.workDir + "/test3.ccs").read())) otherCs = changeset.ChangeSetFromFile(self.workDir + "/test1.ccs") cs1 = changeset.ReadOnlyChangeSet() cs1.merge(otherCs) otherCs.reset() cs2 = changeset.ReadOnlyChangeSet() cs2.merge(otherCs) assert([ x[0] for x in cs1.fileQueue ] == [ x[0] for x in cs2.fileQueue ]) assert(cs1.configCache == cs2.configCache)
def testServerErrors(self): # try to get the repository to raise the errors from this class # by doing Bad Things. repos = self.openRepository() # add a pkg diff t = trove.Trove('foo', ThawVersion('/localhost@rpl:1/1.0:1.0-1-1'), deps.parseFlavor('~!foo'), None) # create an absolute changeset cs = changeset.ChangeSet() cs.newTrove(t.diff(None)[0]) try: repos.commitChangeSet(cs) except errors.TroveChecksumMissing, err: assert ( str(err) == 'Checksum Missing Error: Trove foo=/localhost@rpl:1/1.0-1-1[~!foo] has no sha1 checksum calculated, so it was rejected. Please upgrade conary.' )
def testChangeSetMerge(self): os.chdir(self.workDir) cs1 = changeset.ChangeSet() p1 = '0' * 16 f1 = '0' * 20 cs1.addFileContents(p1, f1, changeset.ChangedFileTypes.file, filecontents.FromString('zero'), False) assert (cs1.writeToFile('foo.ccs') == 129) cs2 = changeset.ReadOnlyChangeSet() cs2.merge(cs1) assert (cs2.writeToFile('foo.ccs') == 129) cs2.reset() assert (cs2.writeToFile('foo.ccs') == 129) cs2.reset() cs3 = changeset.ReadOnlyChangeSet() cs3.merge(cs2) assert (cs3.writeToFile('foo.ccs') == 129) cs3.reset() assert (cs3.writeToFile('foo.ccs') == 129)
def testCommitChangesetByDefaultExcluded(self): # this test creates two changesets. The first includes # test=1.0-1-1, test:foo=1.0-1-1 and a reference to # test:bar-1.0-1-1 (but not included). This gets committed # to the database. # The next version of the test package no longer has a test:bar # component. A changeset is generated between test=1.0-1-1 and # test=1.1-1-1. The resulting changeset is committed to the # database cl = conaryclient.ConaryClient(self.cfg) versionPrefix = '/' + self.cfg.buildLabel.asString() + '/' flavor = parseFlavor('') # create an absolute changeset cs = changeset.ChangeSet() # add a pkg diff v1 = VersionFromString(versionPrefix + '1.0-1-1', timeStamps=[1.000]) old = trove.Trove('test', v1, flavor, None) old.addTrove('test:foo', v1, flavor, byDefault=True) old.addTrove('test:bar', v1, flavor, byDefault=False) old.setIsCollection(True) old.computeDigests() # add the 'test' package diff = old.diff(None)[0] cs.newTrove(diff) cs.addPrimaryTrove('test', v1, flavor) # add the test:foo component oldfoo = trove.Trove('test:foo', v1, flavor, None) oldfoo.computeDigests() diff = oldfoo.diff(None)[0] cs.newTrove(diff) # note that we do not add a test:bar diff to the # changeset. this is because it's not included by default # (byDefault=False) # commit the first changeset to the database cs.writeToFile(self.workDir + '/first.ccs') cs = changeset.ChangeSetFromFile(self.workDir + '/first.ccs') cs = cl.updateChangeSet([ ("test", (None, None), (v1, flavor), True) ], fromChangesets = [cs])[0] cl.applyUpdate(cs) # create a relative changeset cs = changeset.ChangeSet() v2 = VersionFromString(versionPrefix + '1.1-1-1', timeStamps=[2.000]) new = trove.Trove('test', v2, flavor, None) new.addTrove('test:foo', v2, flavor, byDefault=True) new.setIsCollection(True) new.computeDigests() # add the new 'test' package diff = new.diff(old)[0] cs.newTrove(diff) cs.addPrimaryTrove('test', v2, flavor) # add new test:foo component newfoo = trove.Trove('test:foo', v2, flavor, None) newfoo.computeDigests() diff = newfoo.diff(oldfoo)[0] cs.newTrove(diff) # mark test:bar as being removed cs.oldTrove('test:bar', v1, flavor) cs.writeToFile(self.workDir + '/second.ccs') cs = changeset.ChangeSetFromFile(self.workDir + '/second.ccs') cs = cl.updateChangeSet([ ('test', (None, None), (v2, flavor), True) ], fromChangesets = [cs])[0] # commit the second changeset to the database cl.applyUpdate(cs) db = cl.db e = [('test', v2, flavor)] assert(db.findTrove(None, ('test', '1.1-1-1', None)) == e) e = [('test:foo', v2, flavor)] assert(db.findTrove(None, ('test:foo', '1.1-1-1', None)) == e)
def testChangeSetDumpOffset(self): """Stress test offset arg to dumpIter""" # Make a changeset with one regular file cs = changeset.ChangeSet() pathId = '0' * 16 fileId = '0' * 20 contents = 'contents' store = datastore.FlatDataStore(self.workDir) sha1 = sha1helper.sha1String(contents) store.addFile(StringIO(contents), sha1) rawFile = store.openRawFile(sha1) rawSize = os.fstat(rawFile.fileno()).st_size contObj = filecontents.CompressedFromDataStore(store, sha1) cs.addFileContents(pathId, fileId, changeset.ChangedFileTypes.file, contObj, cfgFile=False, compressed=True) # Test dumping a fully populated changeset with every possible resume # point path = os.path.join(self.workDir, 'full.ccs') size = cs.writeToFile(path) expected = open(path).read() self.assertEqual(len(expected), size) fc = filecontainer.FileContainer( util.ExtendedFile(path, 'r', buffering=False)) def noop(name, tag, size, subfile): assert tag[2:] != changeset.ChangedFileTypes.refr[4:] return tag, size, subfile for offset in range(size + 1): fc.reset() actual = ''.join(fc.dumpIter(noop, (), offset)) self.assertEqual(actual, expected[offset:]) # Test dumping a changeset with contents stripped out path = os.path.join(self.workDir, 'stubby.ccs') size2 = cs.writeToFile(path, withReferences=True) self.assertEqual(size2, size) fc = filecontainer.FileContainer( util.ExtendedFile(path, 'r', buffering=False)) expect_reference = '%s %d' % (sha1.encode('hex'), rawSize) def addfile(name, tag, size, subfile, dummy): self.assertEqual(dummy, 'dummy') if name == 'CONARYCHANGESET': return tag, size, subfile elif name == pathId + fileId: self.assertEqual(tag[2:], changeset.ChangedFileTypes.refr[4:]) self.assertEqual(subfile.read(), expect_reference) tag = tag[0:2] + changeset.ChangedFileTypes.file[4:] rawFile.seek(0) return tag, rawSize, rawFile else: assert False for offset in range(size + 1): fc.reset() actual = ''.join(fc.dumpIter(addfile, ('dummy', ), offset)) self.assertEqual(actual, expected[offset:])
mergeSet.merge(cs1) cs1 = changeset.ChangeSet() cs1.addFileContents('0' * 16, '0' * 20, changeset.ChangedFileTypes.diff, filecontents.FromString('first'), cfgFile = True) try: cs1.addFileContents('0' * 16, '0' * 20, changeset.ChangedFileTypes.diff, filecontents.FromString('second'), cfgFile = True) except changeset.ChangeSetKeyConflictError, e: assert str(e) == 'ChangeSetKeyConflictError: 30303030303030303030303030303030,3030303030303030303030303030303030303030' else: assert(0) cs1 = changeset.ChangeSet() # this is blatantly illegal; diff non-config files! cs1.addFileContents('0' * 16, '0' * 20, changeset.ChangedFileTypes.diff, filecontents.FromString('first'), cfgFile = False) try: cs1.addFileContents('0' * 16, '0' * 20, changeset.ChangedFileTypes.diff, filecontents.FromString('second'), cfgFile = False) except changeset.ChangeSetKeyConflictError, e: assert str(e) == 'ChangeSetKeyConflictError: 30303030303030303030303030303030,3030303030303030303030303030303030303030' else: assert(0) # build a changeset, both with two config files with the same # pathid and fileid. One is a diff and the other is not. This should
# create an absolute changeset cs = changeset.ChangeSet() cs.newTrove(t.diff(None)[0]) try: repos.commitChangeSet(cs) except errors.TroveChecksumMissing, err: assert ( str(err) == 'Checksum Missing Error: Trove foo=/localhost@rpl:1/1.0-1-1[~!foo] has no sha1 checksum calculated, so it was rejected. Please upgrade conary.' ) else: assert (0) t.computeDigests() # should be renamed computeChecksum t.setSize(1) # now modify the trove after computing the sums cs = changeset.ChangeSet() cs.newTrove(t.diff(None)[0]) try: repos.commitChangeSet(cs) except errors.TroveIntegrityError, err: assert ( str(err) == 'Trove Integrity Error: foo=/localhost@rpl:1/1.0-1-1[~!foo] checksum does not match precalculated value' ) else: assert (0) t.troveInfo.troveVersion.set(100000) t.computeDigests() cs = changeset.ChangeSet() cs.newTrove(t.diff(None)[0])
def _createBranchOrShadow(self, newLabel, troveList, shadow, branchType=BRANCH_ALL, sigKeyId=None, allowEmptyShadow=False): cs = changeset.ChangeSet() seen = set(troveList) sourceTroveList = set() troveList = set(troveList) dupList = [] needsCommit = False newLabel = versions.Label(newLabel) while troveList: troves = self.repos.getTroves(troveList) troveList = set() branchedTroves = {} if sourceTroveList: for st in sourceTroveList: try: sourceTrove = self.repos.getTrove(*st) except repoerrors.TroveMissing: if allowEmptyShadow: st[1].resetTimeStamps() sourceTrove = trove.Trove(*st) else: raise troves.append(sourceTrove) sourceTroveList = set() if shadow: laterShadows = self._checkForLaterShadows(newLabel, troves) # we disallow shadowing an earlier trove if a later # later one has already been shadowed - it makes our # cvc merge algorithm not work well. if laterShadows: msg = [] for n, v, f, shadowedVer in laterShadows: msg.append('''\ Cannot shadow backwards - already shadowed %s=%s[%s] cannot shadow earlier trove %s=%s[%s] ''' % (n, shadowedVer, f, n, v, f)) raise BranchError('\n\n'.join(msg)) for trv in troves: if trv.isRedirect(): raise errors.ShadowRedirect(*trv.getNameVersionFlavor()) # add contained troves to the todo-list newTroves = [ x for x in trv.iterTroveList(strongRefs=True, weakRefs=True) if x not in seen ] troveList.update(newTroves) seen.update(newTroves) troveName = trv.getName() if troveName.endswith(':source'): if not (branchType & self.BRANCH_SOURCE): continue elif branchType & self.BRANCH_SOURCE: # name doesn't end in :source - if we want # to shadow the listed troves' sources, do so now # XXX this can go away once we don't care about # pre-troveInfo troves if not trv.getSourceName(): from conary.lib import log log.warning('%s has no source information' % troveName) sourceName = troveName else: sourceName = trv.getSourceName() key = (sourceName, trv.getVersion().getSourceVersion(False), deps.Flavor()) if key not in seen: seen.add(key) sourceTroveList.add(key) if not (branchType & self.BRANCH_BINARY): continue if shadow: branchedVersion = trv.getVersion().createShadow(newLabel) else: branchedVersion = trv.getVersion().createBranch( newLabel, withVerRel=1) branchedTrove = trv.copy() branchedTrove.changeVersion(branchedVersion) #this clears the digital signatures from the shadow branchedTrove.troveInfo.sigs.reset() # this flattens the old metadata and removes signatures branchedTrove.copyMetadata(trv) # FIXME we should add a new digital signature in cases # where we can (aka user is at kb and can provide secret key for ((name, version, flavor), byDefault, isStrong) \ in trv.iterTroveListInfo(): if shadow: branchedVersion = version.createShadow(newLabel) else: branchedVersion = version.createBranch(newLabel, withVerRel=1) branchedTrove.delTrove(name, version, flavor, missingOkay=False, weakRef=not isStrong) branchedTrove.addTrove(name, branchedVersion, flavor, byDefault=byDefault, weakRef=not isStrong) key = (trv.getName(), branchedTrove.getVersion(), trv.getFlavor()) if sigKeyId is not None: branchedTrove.addDigitalSignature(sigKeyId) else: # if no sigKeyId, just add sha1s branchedTrove.computeDigests() # use a relative changeset if we're staying on the same host if branchedTrove.getVersion().trailingLabel().getHost() == \ trv.getVersion().trailingLabel().getHost(): branchedTroves[key] = branchedTrove.diff(trv, absolute=False)[0] else: branchedTroves[key] = branchedTrove.diff(None, absolute=True)[0] # check for duplicates hasTroves = self.repos.hasTroves(branchedTroves) for (name, version, flavor), troveCs in branchedTroves.iteritems(): if hasTroves[name, version, flavor]: dupList.append((name, version.branch())) else: cs.newTrove(troveCs) cs.addPrimaryTrove(name, version, flavor) needsCommit = True if not needsCommit: cs = None return dupList, cs
def createChangeSet(self, origTroveList, recurse = True, withFiles = True, withFileContents = True, excludeAutoSource = False, mirrorMode = False, roleIds = None): """ @param origTroveList: a list of C{(troveName, flavor, oldVersion, newVersion, absolute)} tuples. If C{oldVersion == None} and C{absolute == 0}, then the trove is assumed to be new for the purposes of the change set. If C{newVersion == None} then the trove is being removed. If recurse is set, this yields one result for the entire troveList. If recurse is not set, it yields one result per troveList entry. """ cs = changeset.ChangeSet() externalTroveList = [] externalFileList = [] removedTroveList = [] dupFilter = set() # make a copy to remove things from troveList = origTroveList[:] # def createChangeSet begins here troveWrapper = _TroveListWrapper(troveList, self.troveStore, withFiles, roleIds = roleIds) for (job, old, new, streams) in troveWrapper: (troveName, (oldVersion, oldFlavor), (newVersion, newFlavor), absolute) = job # make sure we haven't already generated this changeset; since # troves can be included from other troves we could try # to generate quite a few duplicates if job in dupFilter: continue else: dupFilter.add(job) done = False if not newVersion: if oldVersion.getHost() not in self.serverNameList: externalTroveList.append((troveName, (oldVersion, oldFlavor), (None, None), absolute)) else: # remove this trove and any trove contained in it cs.oldTrove(troveName, oldVersion, oldFlavor) for (name, version, flavor) in \ old.iterTroveList(strongRefs=True): troveWrapper.append((name, (version, flavor), (None, None), absolute), False) done = True elif (newVersion.getHost() not in self.serverNameList or (oldVersion and oldVersion.getHost() not in self.serverNameList)): # don't try to make changesets between repositories; the # client can do that itself # we don't generate chagnesets between removed and # present troves; that's up to the client externalTroveList.append((troveName, (oldVersion, oldFlavor), (newVersion, newFlavor), absolute)) done = True elif (oldVersion and old.type() == trove.TROVE_TYPE_REMOVED): removedTroveList.append((troveName, (oldVersion, oldFlavor), (newVersion, newFlavor), absolute)) done = True if done: if not recurse: yield (cs, externalTroveList, externalFileList, removedTroveList) cs = changeset.ChangeSet() externalTroveList = [] externalFileList = [] removedTroveList = [] continue (troveChgSet, filesNeeded, pkgsNeeded) = \ new.diff(old, absolute = absolute) if recurse: for refJob in pkgsNeeded: refOldVersion = refJob[1][0] refNewVersion = refJob[2][0] if (refNewVersion and (refNewVersion.getHost() not in self.serverNameList) or (refOldVersion and refOldVersion.getHost() not in self.serverNameList) ): # don't try to make changesets between repositories; the # client can do that itself externalTroveList.append(refJob) else: troveWrapper.append(refJob, True) cs.newTrove(troveChgSet) if job in origTroveList and job[2][0] is not None: # add the primary w/ timestamps on the version try: primary = troveChgSet.getNewNameVersionFlavor() cs.addPrimaryTrove(*primary) except KeyError: # primary troves could be in the externalTroveList, in # which case they aren't primries pass # sort the set of files we need into bins based on the server # name getList = [] localFilesNeeded = [] for (pathId, oldFileId, oldFileVersion, newFileId, newFileVersion) in filesNeeded: # if either the old or new file version is on a different # repository, creating this diff is someone else's problem if (newFileVersion.getHost() not in self.serverNameList or (oldFileVersion and oldFileVersion.getHost() not in self.serverNameList)): externalFileList.append((pathId, troveName, (oldVersion, oldFlavor, oldFileId, oldFileVersion), (newVersion, newFlavor, newFileId, newFileVersion))) else: localFilesNeeded.append((pathId, oldFileId, oldFileVersion, newFileId, newFileVersion)) if oldFileVersion: getList.append((pathId, oldFileId, oldFileVersion)) getList.append((pathId, newFileId, newFileVersion)) # Walk this in reverse order. This may seem odd, but the # order in the final changeset is set by sorting that happens # in the change set object itself. The only reason we sort # here at all is to make sure PTR file types come before the # file they refer to. Reverse shorting makes this a bit easier. localFilesNeeded.sort() localFilesNeeded.reverse() ptrTable = {} for (pathId, oldFileId, oldFileVersion, newFileId, \ newFileVersion) in localFilesNeeded: oldFile = None if oldFileVersion: oldFile = files.ThawFile(streams[oldFileId], pathId) oldCont = None newCont = None newFile = files.ThawFile(streams[newFileId], pathId) # Skip identical fileids when mirroring, but always use # absolute file changes if there is any difference. See note # below. forceAbsolute = (mirrorMode and oldFileId and oldFileId != newFileId) if forceAbsolute: (filecs, contentsHash) = changeset.fileChangeSet(pathId, None, newFile) else: (filecs, contentsHash) = changeset.fileChangeSet(pathId, oldFile, newFile) cs.addFile(oldFileId, newFileId, filecs) if (not withFileContents or (excludeAutoSource and newFile.flags.isAutoSource()) or (newFile.flags.isEncapsulatedContent() and not newFile.flags.isCapsuleOverride())): continue # this test catches files which have changed from not # config files to config files; these need to be included # unconditionally so we always have the pristine contents # to include in the local database # Also include contents of config files when mirroring if the # fileid changed, even if the SHA-1 did not. # cf CNY-1570, CNY-1699, CNY-2210 if (contentsHash or (oldFile and newFile.flags.isConfig() and not oldFile.flags.isConfig()) or (forceAbsolute and newFile.hasContents) ): if oldFileVersion and oldFile.hasContents: oldCont = self.getFileContents( [ (oldFileId, oldFileVersion, oldFile) ])[0] newCont = self.getFileContents( [ (newFileId, newFileVersion, newFile) ])[0] (contType, cont) = changeset.fileContentsDiff(oldFile, oldCont, newFile, newCont, mirrorMode = mirrorMode) # we don't let config files be ptr types; if they were # they could be ptrs to things which aren't config files, # which would completely hose the sort order we use. this # could be relaxed someday to let them be ptr's to other # config files if not newFile.flags.isConfig() and \ contType == changeset.ChangedFileTypes.file: contentsHash = newFile.contents.sha1() ptr = ptrTable.get(contentsHash, None) if ptr is not None: contType = changeset.ChangedFileTypes.ptr cont = filecontents.FromString(ptr) else: ptrTable[contentsHash] = pathId + newFileId if not newFile.flags.isConfig() and \ contType == changeset.ChangedFileTypes.file: cont = filecontents.CompressedFromDataStore( self.contentsStore, newFile.contents.sha1()) compressed = True else: compressed = False # ptr entries are not compressed, whether or not they # are config files. override the compressed rule from # above if contType == changeset.ChangedFileTypes.ptr: compressed = False cs.addFileContents(pathId, newFileId, contType, cont, newFile.flags.isConfig(), compressed = compressed) if not recurse: yield cs, externalTroveList, externalFileList, removedTroveList cs = changeset.ChangeSet() externalTroveList = [] externalFileList = [] removedTroveList = [] if recurse: yield cs, externalTroveList, externalFileList, removedTroveList