def testConfigFilesRaisePathIdsConflict(self):
# test to make sure that one changeset's config cache doesn't
# override another's
cs1 = changeset.ChangeSet()
cs2 = changeset.ChangeSet()
mergeSet = changeset.ReadOnlyChangeSet()
# build two changesets, both with config file diffs that have the same
# pathid and fileid
cs1.addFileContents('0' * 16, '0' * 20, changeset.ChangedFileTypes.diff,
filecontents.FromString('first'), cfgFile = True)
cs2.addFileContents('0' * 16, '0' * 20, changeset.ChangedFileTypes.diff,
filecontents.FromString('second'), cfgFile = True)
mergeSet.merge(cs1)
# second merge now handled without ChangeSetKeyConflictError: CNY-3635
mergeSet.merge(cs1)
cs1 = changeset.ChangeSet()
cs1.addFileContents('0' * 16, '0' * 20, changeset.ChangedFileTypes.diff,
filecontents.FromString('first'), cfgFile = True)
try:
cs1.addFileContents('0' * 16, '0' * 20,
changeset.ChangedFileTypes.diff,
filecontents.FromString('second'),
cfgFile = True)
except changeset.ChangeSetKeyConflictError, e:
assert str(e) == 'ChangeSetKeyConflictError: 30303030303030303030303030303030,3030303030303030303030303030303030303030'
def testFileIdWrong(self):
# create an absolute changeset
cs = changeset.ChangeSet()
# add a pkg diff
flavor = deps.deps.parseFlavor('')
v = versions.VersionFromString('/%s/1.0-1-1'
%self.cfg.buildLabel.asString()).copy()
v.resetTimeStamps()
t = trove.Trove('test:test', v, flavor, None)
path = self.workDir + '/blah'
f = open(path, 'w')
f.write('hello, world!\n')
f.close()
pathId = sha1helper.md5String('/blah')
f = files.FileFromFilesystem(path, pathId)
# add the file, but munge the fileid
brokenFileId = ''.join(reversed(f.fileId()))
cs.addFile(None, brokenFileId, f.freeze())
t.addFile(pathId, '/blah', v, brokenFileId)
t.computeDigests()
diff = t.diff(None, absolute = 1)[0]
cs.newTrove(diff)
repos = self.openRepository()
try:
repos.commitChangeSet(cs)
assert 0, "Integrity Error not raised"
except errors.TroveIntegrityError, e:
assert(str(e) == 'fileObj.fileId() != fileId in changeset for '
'pathId %s' % sha1helper.md5ToString(pathId))
def getChangeSetForCapsuleChanges(self, callback):
self.loadPlugins()
changeSet = changeset.ChangeSet()
for plugin in self._loadedPlugins.itervalues():
callback.capsuleSyncScan(plugin.kind)
plugin.addCapsuleChangesToChangeSet(changeSet, callback)
return changeSet
def _createCs(version):
# create an absolute changeset
flavor = deps.parseFlavor('')
cs = changeset.ChangeSet()
# add a pkg diff
v = versions.VersionFromString(version, timeStamps=[1.000])
old = trove.Trove('test', v, flavor, None)
old.setIsCollection(True)
old.addTrove('test:foo', v, flavor, byDefault=True)
old.addTrove('test:bar', v, flavor, byDefault=False)
old.computeDigests()
# add the 'test' package
diff = old.diff(None)[0]
cs.newTrove(diff)
cs.addPrimaryTrove('test', v, flavor)
# add the test:foo component
oldfoo = trove.Trove('test:foo', v, flavor, None)
oldfoo.computeDigests()
diff = oldfoo.diff(None)[0]
cs.newTrove(diff)
# add the test:bar component
oldbar = trove.Trove('test:bar', v, flavor, None)
oldbar.computeDigests()
diff = oldbar.diff(None)[0]
cs.newTrove(diff)
return cs
def testFileObjMissing(self):
# create an absolute changeset
cs = changeset.ChangeSet()
# add a pkg diff
flavor = deps.deps.parseFlavor('')
v = versions.VersionFromString('/%s/1.0-1-1'
%self.cfg.buildLabel.asString()).copy()
v.resetTimeStamps()
t = trove.Trove('test:test', v, flavor, None)
path = self.workDir + '/blah'
f = open(path, 'w')
f.write('hello, world!\n')
f.close()
pathId = sha1helper.md5String('/blah')
f = files.FileFromFilesystem(path, pathId)
# add the file, and SKIP including
# the filestream by using cs.addFile(). This creates an
# incomplete changeset
t.addFile(pathId, '/blah', v, f.fileId())
cs.addFileContents(pathId, f.fileId(), changeset.ChangedFileTypes.file,
filecontents.FromFilesystem(path),
f.flags.isConfig())
t.computeDigests()
diff = t.diff(None, absolute = 1)[0]
cs.newTrove(diff)
repos = self.openRepository()
try:
repos.commitChangeSet(cs)
assert 0, "Did not raise IntegrityError"
except errors.IntegrityError, e:
assert(str(e).startswith("Incomplete changeset specified: missing pathId e806729b6a2b568fa7e77c3efa3a9684 fileId"))
def testFileContentsMissing(self):
# currently causes a 500 error
#raise testhelp.SkipTestException
# create an absolute changeset
cs = changeset.ChangeSet()
# add a pkg diff
flavor = deps.deps.parseFlavor('')
v = versions.VersionFromString('/%s/1.0-1-1'
%self.cfg.buildLabel.asString()).copy()
v.resetTimeStamps()
t = trove.Trove('test:test', v, flavor, None)
path = self.workDir + '/blah'
f = open(path, 'w')
f.write('hello, world!\n')
f.close()
pathId = sha1helper.md5String('/blah')
f = files.FileFromFilesystem(path, pathId)
# add the file, but munge the fileid
fileId = f.fileId()
cs.addFile(None, fileId, f.freeze())
t.addFile(pathId, '/blah', v, fileId)
# skip adding the file contents
t.computeDigests()
diff = t.diff(None, absolute = 1)[0]
cs.newTrove(diff)
repos = self.openRepository()
try:
repos.commitChangeSet(cs)
assert 0, "Did not raise integrity error"
except errors.IntegrityError, e:
assert(str(e).startswith("Missing file contents for pathId e806729b6a2b568fa7e77c3efa3a9684, fileId"))
def _createTroves(self, troveAndPathList):
cs = changeset.ChangeSet()
troveList = [x[0] for x in troveAndPathList]
previousVersionMap = self._targetNewTroves(troveList)
self._addAllNewFiles(cs, troveAndPathList, previousVersionMap)
for trv in troveList:
trv.computeDigests()
trvCs = trv.diff(None, absolute=True)[0]
cs.newTrove(trvCs)
return cs
def testConfigFileMerges(self):
# make sure config files are merged properly; at one point we only
# merged diffs, now all our merged
cs1 = changeset.ReadOnlyChangeSet()
cs2 = changeset.ChangeSet()
cs2.addFileContents('0' * 16, '0' * 20, changeset.ChangedFileTypes.file,
filecontents.FromString('first'), cfgFile = True)
cs2.addFileContents('1' * 16, '1' * 20, changeset.ChangedFileTypes.file,
filecontents.FromString('first'), cfgFile = True)
cs1.merge(cs2)
assert(len(cs1.configCache) == 2)
def testFileUpdateMissingKey(self):
fingerprint = '95B457D16843B21EA3FC73BBC7C32FC1F94E405E'
# make a changeset with useful stuff that could be installed
cs = changeset.ChangeSet()
flavor = deps.parseFlavor('')
v = versions.VersionFromString('/%s/1.0-1-1' %
self.cfg.buildLabel.asString()).copy()
v.resetTimeStamps()
t = trove.Trove('test:test', v, flavor, None)
path = self.workDir + '/blah'
f = open(path, 'w')
f.write('hello, world!\n')
f.close()
pathId = sha1helper.md5String('/blah')
f = files.FileFromFilesystem(path, pathId)
fileId = f.fileId()
t.addFile(pathId, '/blah', v, fileId)
cs.addFile(None, fileId, f.freeze())
cs.addFileContents(pathId, fileId, changeset.ChangedFileTypes.file,
filecontents.FromFilesystem(path),
f.flags.isConfig())
diff = t.diff(None, absolute=1)[0]
cs.newTrove(diff)
cs.addPrimaryTrove('test:test', v, flavor)
# sign the changeset
csPath = os.path.join(self.workDir, 'test-1.0-1.ccs')
cs = cook.signAbsoluteChangeset(cs, fingerprint)
cs.writeToFile(csPath)
tmpPath = mkdtemp()
keyCache = openpgpkey.getKeyCache()
newKeyCache = openpgpkey.OpenPGPKeyFileCache()
pubRing = self.cfg.pubRing
self.cfg.pubRing = [tmpPath + '/pubring.gpg']
keyCacheCallback = openpgpkey.KeyCacheCallback(None, self.cfg)
newKeyCache.setCallback(keyCacheCallback)
openpgpkey.setKeyCache(newKeyCache)
try:
self.updatePkg(self.rootDir, csPath)
finally:
self.cfg.pubRing = pubRing
openpgpkey.setKeyCache(keyCache)
def save(self, path):
# return early if we aren't going to have permission to save
try:
fd, cacheName = tempfile.mkstemp(
prefix=os.path.basename(path) + '.',
dir=os.path.dirname(path))
os.close(fd)
except (IOError, OSError):
# may not have permissions; say, not running as root
return
cs = changeset.ChangeSet()
for withFiles, trv in self.cache.values():
# we just assume everything in the cache is w/o files. it's
# fine for system model, safe, and we don't need the cache
# anywhere else
cs.newTrove(trv.diff(None, absolute = True)[0])
# NB: "fileid" and pathid got reversed here by mistake, try not to
# think too hard about it.
cs.addFileContents(
self._fileId,
self._troveCacheVersionPathId,
changeset.ChangedFileTypes.file,
filecontents.FromString("%d %d" % self.VERSION),
False)
self._cs = cs
self._saveTimestamps()
self._saveDeps()
self._saveDepSolutions()
self._saveFileCache()
self._cs = None
try:
try:
cs.writeToFile(cacheName)
if util.exists(path):
os.chmod(cacheName, os.stat(path).st_mode)
else:
os.chmod(cacheName, 0644)
os.rename(cacheName, path)
except (IOError, OSError):
# may not have permissions; say, not running as root
pass
finally:
try:
if os.path.exists(cacheName):
os.remove(cacheName)
except OSError:
pass
def _addUnownedNewFiles(self, newFileList):
if not newFileList: return
cs = changeset.ChangeSet()
ver = versions.VersionFromString(
'/localhost@local:LOCAL/1.0-1-1').copy()
ver.resetTimeStamps()
trv = trove.Trove("@new:files", ver, deps.Flavor())
for path in newFileList:
self._addFile(cs, trv, path)
trvDiff = trv.diff(None, absolute=False)[0]
cs.newTrove(trvDiff)
self._handleChangeSet([trv.getNameVersionFlavor()], cs)
def testChangeSetFromFile(self):
# ensure that absolute changesets that are read from disk
# that contain config files write out changesets to a file
# that do not change the file type to a diff.
# set up a file with some contents
cont = self.workDir + '/contents'
f = open(cont, 'w')
f.write('hello, world!\n')
f.close()
pathId = sha1helper.md5FromString('0' * 32)
f = files.FileFromFilesystem(cont, pathId)
f.flags.isConfig(1)
# create an absolute changeset
cs = changeset.ChangeSet()
# add a pkg diff
v = versions.VersionFromString('/localhost@rpl:devel/1.0-1-1',
timeStamps=[1.000])
flavor = deps.parseFlavor('')
t = trove.Trove('test', v, flavor, None)
t.addFile(pathId, '/contents', v, f.fileId())
diff = t.diff(None, absolute=1)[0]
cs.newTrove(diff)
# add the file and file contents
cs.addFile(None, f.fileId(), f.freeze())
cs.addFileContents(pathId, f.fileId(), changeset.ChangedFileTypes.file,
filecontents.FromFilesystem(cont),
f.flags.isConfig())
# write out the changeset
cs.writeToFile(self.workDir + '/foo.ccs')
# read it back in
cs2 = changeset.ChangeSetFromFile(self.workDir + '/foo.ccs')
# write it out again (there was a bug where all config files
# became diffs)
cs2.writeToFile(self.workDir + '/bar.ccs')
# read it again
cs3 = changeset.ChangeSetFromFile(self.workDir + '/bar.ccs')
# verify that the file is a file, not a diff
ctype, contents = cs3.getFileContents(pathId, f.fileId())
assert (ctype == changeset.ChangedFileTypes.file)
def testReset(self):
class UncloseableFile(StringIO.StringIO):
def close(self):
assert(0)
# Make sure writing a changeset doesn't close the filecontent objects
# inside of it.
otherCs = changeset.ChangeSet()
otherCs.addFileContents("0pathId", "0fileId",
changeset.ChangedFileTypes.file,
filecontents.FromFile(UncloseableFile("foo")),
1)
otherCs.addFileContents("1pathId", "1fileId",
changeset.ChangedFileTypes.file,
filecontents.FromFile(UncloseableFile("foo")),
0)
otherCs.writeToFile(self.workDir + "/test1.ccs")
cs = changeset.ReadOnlyChangeSet()
cs.merge(otherCs)
cs.writeToFile(self.workDir + "/test2.ccs")
assert(open(self.workDir + "/test1.ccs").read() ==
(open(self.workDir + "/test2.ccs").read()))
cs.reset()
cs.writeToFile(self.workDir + "/test3.ccs")
assert(open(self.workDir + "/test1.ccs").read() ==
(open(self.workDir + "/test3.ccs").read()))
otherCs = changeset.ChangeSetFromFile(self.workDir + "/test1.ccs")
cs1 = changeset.ReadOnlyChangeSet()
cs1.merge(otherCs)
otherCs.reset()
cs2 = changeset.ReadOnlyChangeSet()
cs2.merge(otherCs)
assert([ x[0] for x in cs1.fileQueue ] ==
[ x[0] for x in cs2.fileQueue ])
assert(cs1.configCache == cs2.configCache)
def testServerErrors(self):
# try to get the repository to raise the errors from this class
# by doing Bad Things.
repos = self.openRepository()
# add a pkg diff
t = trove.Trove('foo', ThawVersion('/localhost@rpl:1/1.0:1.0-1-1'),
deps.parseFlavor('~!foo'), None)
# create an absolute changeset
cs = changeset.ChangeSet()
cs.newTrove(t.diff(None)[0])
try:
repos.commitChangeSet(cs)
except errors.TroveChecksumMissing, err:
assert (
str(err) ==
'Checksum Missing Error: Trove foo=/localhost@rpl:1/1.0-1-1[~!foo] has no sha1 checksum calculated, so it was rejected. Please upgrade conary.'
)
def testChangeSetMerge(self):
os.chdir(self.workDir)
cs1 = changeset.ChangeSet()
p1 = '0' * 16
f1 = '0' * 20
cs1.addFileContents(p1, f1, changeset.ChangedFileTypes.file,
filecontents.FromString('zero'), False)
assert (cs1.writeToFile('foo.ccs') == 129)
cs2 = changeset.ReadOnlyChangeSet()
cs2.merge(cs1)
assert (cs2.writeToFile('foo.ccs') == 129)
cs2.reset()
assert (cs2.writeToFile('foo.ccs') == 129)
cs2.reset()
cs3 = changeset.ReadOnlyChangeSet()
cs3.merge(cs2)
assert (cs3.writeToFile('foo.ccs') == 129)
cs3.reset()
assert (cs3.writeToFile('foo.ccs') == 129)
def testCommitChangesetByDefaultExcluded(self):
# this test creates two changesets. The first includes
# test=1.0-1-1, test:foo=1.0-1-1 and a reference to
# test:bar-1.0-1-1 (but not included). This gets committed
# to the database.
# The next version of the test package no longer has a test:bar
# component. A changeset is generated between test=1.0-1-1 and
# test=1.1-1-1. The resulting changeset is committed to the
# database
cl = conaryclient.ConaryClient(self.cfg)
versionPrefix = '/' + self.cfg.buildLabel.asString() + '/'
flavor = parseFlavor('')
# create an absolute changeset
cs = changeset.ChangeSet()
# add a pkg diff
v1 = VersionFromString(versionPrefix + '1.0-1-1',
timeStamps=[1.000])
old = trove.Trove('test', v1, flavor, None)
old.addTrove('test:foo', v1, flavor, byDefault=True)
old.addTrove('test:bar', v1, flavor, byDefault=False)
old.setIsCollection(True)
old.computeDigests()
# add the 'test' package
diff = old.diff(None)[0]
cs.newTrove(diff)
cs.addPrimaryTrove('test', v1, flavor)
# add the test:foo component
oldfoo = trove.Trove('test:foo', v1, flavor, None)
oldfoo.computeDigests()
diff = oldfoo.diff(None)[0]
cs.newTrove(diff)
# note that we do not add a test:bar diff to the
# changeset. this is because it's not included by default
# (byDefault=False)
# commit the first changeset to the database
cs.writeToFile(self.workDir + '/first.ccs')
cs = changeset.ChangeSetFromFile(self.workDir + '/first.ccs')
cs = cl.updateChangeSet([ ("test", (None, None), (v1, flavor), True) ],
fromChangesets = [cs])[0]
cl.applyUpdate(cs)
# create a relative changeset
cs = changeset.ChangeSet()
v2 = VersionFromString(versionPrefix + '1.1-1-1',
timeStamps=[2.000])
new = trove.Trove('test', v2, flavor, None)
new.addTrove('test:foo', v2, flavor, byDefault=True)
new.setIsCollection(True)
new.computeDigests()
# add the new 'test' package
diff = new.diff(old)[0]
cs.newTrove(diff)
cs.addPrimaryTrove('test', v2, flavor)
# add new test:foo component
newfoo = trove.Trove('test:foo', v2, flavor, None)
newfoo.computeDigests()
diff = newfoo.diff(oldfoo)[0]
cs.newTrove(diff)
# mark test:bar as being removed
cs.oldTrove('test:bar', v1, flavor)
cs.writeToFile(self.workDir + '/second.ccs')
cs = changeset.ChangeSetFromFile(self.workDir + '/second.ccs')
cs = cl.updateChangeSet([ ('test', (None, None), (v2, flavor), True) ],
fromChangesets = [cs])[0]
# commit the second changeset to the database
cl.applyUpdate(cs)
db = cl.db
e = [('test', v2, flavor)]
assert(db.findTrove(None, ('test', '1.1-1-1', None)) == e)
e = [('test:foo', v2, flavor)]
assert(db.findTrove(None, ('test:foo', '1.1-1-1', None)) == e)
def testChangeSetDumpOffset(self):
"""Stress test offset arg to dumpIter"""
# Make a changeset with one regular file
cs = changeset.ChangeSet()
pathId = '0' * 16
fileId = '0' * 20
contents = 'contents'
store = datastore.FlatDataStore(self.workDir)
sha1 = sha1helper.sha1String(contents)
store.addFile(StringIO(contents), sha1)
rawFile = store.openRawFile(sha1)
rawSize = os.fstat(rawFile.fileno()).st_size
contObj = filecontents.CompressedFromDataStore(store, sha1)
cs.addFileContents(pathId,
fileId,
changeset.ChangedFileTypes.file,
contObj,
cfgFile=False,
compressed=True)
# Test dumping a fully populated changeset with every possible resume
# point
path = os.path.join(self.workDir, 'full.ccs')
size = cs.writeToFile(path)
expected = open(path).read()
self.assertEqual(len(expected), size)
fc = filecontainer.FileContainer(
util.ExtendedFile(path, 'r', buffering=False))
def noop(name, tag, size, subfile):
assert tag[2:] != changeset.ChangedFileTypes.refr[4:]
return tag, size, subfile
for offset in range(size + 1):
fc.reset()
actual = ''.join(fc.dumpIter(noop, (), offset))
self.assertEqual(actual, expected[offset:])
# Test dumping a changeset with contents stripped out
path = os.path.join(self.workDir, 'stubby.ccs')
size2 = cs.writeToFile(path, withReferences=True)
self.assertEqual(size2, size)
fc = filecontainer.FileContainer(
util.ExtendedFile(path, 'r', buffering=False))
expect_reference = '%s %d' % (sha1.encode('hex'), rawSize)
def addfile(name, tag, size, subfile, dummy):
self.assertEqual(dummy, 'dummy')
if name == 'CONARYCHANGESET':
return tag, size, subfile
elif name == pathId + fileId:
self.assertEqual(tag[2:], changeset.ChangedFileTypes.refr[4:])
self.assertEqual(subfile.read(), expect_reference)
tag = tag[0:2] + changeset.ChangedFileTypes.file[4:]
rawFile.seek(0)
return tag, rawSize, rawFile
else:
assert False
for offset in range(size + 1):
fc.reset()
actual = ''.join(fc.dumpIter(addfile, ('dummy', ), offset))
self.assertEqual(actual, expected[offset:])
mergeSet.merge(cs1)
cs1 = changeset.ChangeSet()
cs1.addFileContents('0' * 16, '0' * 20, changeset.ChangedFileTypes.diff,
filecontents.FromString('first'), cfgFile = True)
try:
cs1.addFileContents('0' * 16, '0' * 20,
changeset.ChangedFileTypes.diff,
filecontents.FromString('second'),
cfgFile = True)
except changeset.ChangeSetKeyConflictError, e:
assert str(e) == 'ChangeSetKeyConflictError: 30303030303030303030303030303030,3030303030303030303030303030303030303030'
else:
assert(0)
cs1 = changeset.ChangeSet()
# this is blatantly illegal; diff non-config files!
cs1.addFileContents('0' * 16, '0' * 20, changeset.ChangedFileTypes.diff,
filecontents.FromString('first'), cfgFile = False)
try:
cs1.addFileContents('0' * 16, '0' * 20,
changeset.ChangedFileTypes.diff,
filecontents.FromString('second'),
cfgFile = False)
except changeset.ChangeSetKeyConflictError, e:
assert str(e) == 'ChangeSetKeyConflictError: 30303030303030303030303030303030,3030303030303030303030303030303030303030'
else:
assert(0)
# build a changeset, both with two config files with the same
# pathid and fileid. One is a diff and the other is not. This should
# create an absolute changeset
cs = changeset.ChangeSet()
cs.newTrove(t.diff(None)[0])
try:
repos.commitChangeSet(cs)
except errors.TroveChecksumMissing, err:
assert (
str(err) ==
'Checksum Missing Error: Trove foo=/localhost@rpl:1/1.0-1-1[~!foo] has no sha1 checksum calculated, so it was rejected. Please upgrade conary.'
)
else:
assert (0)
t.computeDigests() # should be renamed computeChecksum
t.setSize(1) # now modify the trove after computing the sums
cs = changeset.ChangeSet()
cs.newTrove(t.diff(None)[0])
try:
repos.commitChangeSet(cs)
except errors.TroveIntegrityError, err:
assert (
str(err) ==
'Trove Integrity Error: foo=/localhost@rpl:1/1.0-1-1[~!foo] checksum does not match precalculated value'
)
else:
assert (0)
t.troveInfo.troveVersion.set(100000)
t.computeDigests()
cs = changeset.ChangeSet()
cs.newTrove(t.diff(None)[0])
def _createBranchOrShadow(self,
newLabel,
troveList,
shadow,
branchType=BRANCH_ALL,
sigKeyId=None,
allowEmptyShadow=False):
cs = changeset.ChangeSet()
seen = set(troveList)
sourceTroveList = set()
troveList = set(troveList)
dupList = []
needsCommit = False
newLabel = versions.Label(newLabel)
while troveList:
troves = self.repos.getTroves(troveList)
troveList = set()
branchedTroves = {}
if sourceTroveList:
for st in sourceTroveList:
try:
sourceTrove = self.repos.getTrove(*st)
except repoerrors.TroveMissing:
if allowEmptyShadow:
st[1].resetTimeStamps()
sourceTrove = trove.Trove(*st)
else:
raise
troves.append(sourceTrove)
sourceTroveList = set()
if shadow:
laterShadows = self._checkForLaterShadows(newLabel, troves)
# we disallow shadowing an earlier trove if a later
# later one has already been shadowed - it makes our
# cvc merge algorithm not work well.
if laterShadows:
msg = []
for n, v, f, shadowedVer in laterShadows:
msg.append('''\
Cannot shadow backwards - already shadowed
%s=%s[%s]
cannot shadow earlier trove
%s=%s[%s]
''' % (n, shadowedVer, f, n, v, f))
raise BranchError('\n\n'.join(msg))
for trv in troves:
if trv.isRedirect():
raise errors.ShadowRedirect(*trv.getNameVersionFlavor())
# add contained troves to the todo-list
newTroves = [
x
for x in trv.iterTroveList(strongRefs=True, weakRefs=True)
if x not in seen
]
troveList.update(newTroves)
seen.update(newTroves)
troveName = trv.getName()
if troveName.endswith(':source'):
if not (branchType & self.BRANCH_SOURCE):
continue
elif branchType & self.BRANCH_SOURCE:
# name doesn't end in :source - if we want
# to shadow the listed troves' sources, do so now
# XXX this can go away once we don't care about
# pre-troveInfo troves
if not trv.getSourceName():
from conary.lib import log
log.warning('%s has no source information' % troveName)
sourceName = troveName
else:
sourceName = trv.getSourceName()
key = (sourceName,
trv.getVersion().getSourceVersion(False),
deps.Flavor())
if key not in seen:
seen.add(key)
sourceTroveList.add(key)
if not (branchType & self.BRANCH_BINARY):
continue
if shadow:
branchedVersion = trv.getVersion().createShadow(newLabel)
else:
branchedVersion = trv.getVersion().createBranch(
newLabel, withVerRel=1)
branchedTrove = trv.copy()
branchedTrove.changeVersion(branchedVersion)
#this clears the digital signatures from the shadow
branchedTrove.troveInfo.sigs.reset()
# this flattens the old metadata and removes signatures
branchedTrove.copyMetadata(trv)
# FIXME we should add a new digital signature in cases
# where we can (aka user is at kb and can provide secret key
for ((name, version, flavor), byDefault, isStrong) \
in trv.iterTroveListInfo():
if shadow:
branchedVersion = version.createShadow(newLabel)
else:
branchedVersion = version.createBranch(newLabel,
withVerRel=1)
branchedTrove.delTrove(name,
version,
flavor,
missingOkay=False,
weakRef=not isStrong)
branchedTrove.addTrove(name,
branchedVersion,
flavor,
byDefault=byDefault,
weakRef=not isStrong)
key = (trv.getName(), branchedTrove.getVersion(),
trv.getFlavor())
if sigKeyId is not None:
branchedTrove.addDigitalSignature(sigKeyId)
else:
# if no sigKeyId, just add sha1s
branchedTrove.computeDigests()
# use a relative changeset if we're staying on the same host
if branchedTrove.getVersion().trailingLabel().getHost() == \
trv.getVersion().trailingLabel().getHost():
branchedTroves[key] = branchedTrove.diff(trv,
absolute=False)[0]
else:
branchedTroves[key] = branchedTrove.diff(None,
absolute=True)[0]
# check for duplicates
hasTroves = self.repos.hasTroves(branchedTroves)
for (name, version, flavor), troveCs in branchedTroves.iteritems():
if hasTroves[name, version, flavor]:
dupList.append((name, version.branch()))
else:
cs.newTrove(troveCs)
cs.addPrimaryTrove(name, version, flavor)
needsCommit = True
if not needsCommit:
cs = None
return dupList, cs
def createChangeSet(self, origTroveList, recurse = True,
withFiles = True, withFileContents = True,
excludeAutoSource = False,
mirrorMode = False, roleIds = None):
"""
@param origTroveList: a list of
C{(troveName, flavor, oldVersion, newVersion, absolute)} tuples.
If C{oldVersion == None} and C{absolute == 0}, then the trove is
assumed to be new for the purposes of the change set.
If C{newVersion == None} then the trove is being removed.
If recurse is set, this yields one result for the entire troveList.
If recurse is not set, it yields one result per troveList entry.
"""
cs = changeset.ChangeSet()
externalTroveList = []
externalFileList = []
removedTroveList = []
dupFilter = set()
# make a copy to remove things from
troveList = origTroveList[:]
# def createChangeSet begins here
troveWrapper = _TroveListWrapper(troveList, self.troveStore, withFiles,
roleIds = roleIds)
for (job, old, new, streams) in troveWrapper:
(troveName, (oldVersion, oldFlavor),
(newVersion, newFlavor), absolute) = job
# make sure we haven't already generated this changeset; since
# troves can be included from other troves we could try
# to generate quite a few duplicates
if job in dupFilter:
continue
else:
dupFilter.add(job)
done = False
if not newVersion:
if oldVersion.getHost() not in self.serverNameList:
externalTroveList.append((troveName,
(oldVersion, oldFlavor),
(None, None), absolute))
else:
# remove this trove and any trove contained in it
cs.oldTrove(troveName, oldVersion, oldFlavor)
for (name, version, flavor) in \
old.iterTroveList(strongRefs=True):
troveWrapper.append((name, (version, flavor),
(None, None), absolute),
False)
done = True
elif (newVersion.getHost() not in self.serverNameList
or (oldVersion and
oldVersion.getHost() not in self.serverNameList)):
# don't try to make changesets between repositories; the
# client can do that itself
# we don't generate chagnesets between removed and
# present troves; that's up to the client
externalTroveList.append((troveName, (oldVersion, oldFlavor),
(newVersion, newFlavor), absolute))
done = True
elif (oldVersion and old.type() == trove.TROVE_TYPE_REMOVED):
removedTroveList.append((troveName, (oldVersion, oldFlavor),
(newVersion, newFlavor), absolute))
done = True
if done:
if not recurse:
yield (cs, externalTroveList, externalFileList,
removedTroveList)
cs = changeset.ChangeSet()
externalTroveList = []
externalFileList = []
removedTroveList = []
continue
(troveChgSet, filesNeeded, pkgsNeeded) = \
new.diff(old, absolute = absolute)
if recurse:
for refJob in pkgsNeeded:
refOldVersion = refJob[1][0]
refNewVersion = refJob[2][0]
if (refNewVersion and
(refNewVersion.getHost() not in self.serverNameList)
or (refOldVersion and
refOldVersion.getHost() not in self.serverNameList)
):
# don't try to make changesets between repositories; the
# client can do that itself
externalTroveList.append(refJob)
else:
troveWrapper.append(refJob, True)
cs.newTrove(troveChgSet)
if job in origTroveList and job[2][0] is not None:
# add the primary w/ timestamps on the version
try:
primary = troveChgSet.getNewNameVersionFlavor()
cs.addPrimaryTrove(*primary)
except KeyError:
# primary troves could be in the externalTroveList, in
# which case they aren't primries
pass
# sort the set of files we need into bins based on the server
# name
getList = []
localFilesNeeded = []
for (pathId, oldFileId, oldFileVersion, newFileId, newFileVersion) in filesNeeded:
# if either the old or new file version is on a different
# repository, creating this diff is someone else's problem
if (newFileVersion.getHost() not in self.serverNameList
or (oldFileVersion and
oldFileVersion.getHost() not in self.serverNameList)):
externalFileList.append((pathId, troveName,
(oldVersion, oldFlavor, oldFileId, oldFileVersion),
(newVersion, newFlavor, newFileId, newFileVersion)))
else:
localFilesNeeded.append((pathId, oldFileId, oldFileVersion,
newFileId, newFileVersion))
if oldFileVersion:
getList.append((pathId, oldFileId, oldFileVersion))
getList.append((pathId, newFileId, newFileVersion))
# Walk this in reverse order. This may seem odd, but the
# order in the final changeset is set by sorting that happens
# in the change set object itself. The only reason we sort
# here at all is to make sure PTR file types come before the
# file they refer to. Reverse shorting makes this a bit easier.
localFilesNeeded.sort()
localFilesNeeded.reverse()
ptrTable = {}
for (pathId, oldFileId, oldFileVersion, newFileId, \
newFileVersion) in localFilesNeeded:
oldFile = None
if oldFileVersion:
oldFile = files.ThawFile(streams[oldFileId], pathId)
oldCont = None
newCont = None
newFile = files.ThawFile(streams[newFileId], pathId)
# Skip identical fileids when mirroring, but always use
# absolute file changes if there is any difference. See note
# below.
forceAbsolute = (mirrorMode and oldFileId
and oldFileId != newFileId)
if forceAbsolute:
(filecs, contentsHash) = changeset.fileChangeSet(pathId,
None,
newFile)
else:
(filecs, contentsHash) = changeset.fileChangeSet(pathId,
oldFile,
newFile)
cs.addFile(oldFileId, newFileId, filecs)
if (not withFileContents
or (excludeAutoSource and newFile.flags.isAutoSource())
or (newFile.flags.isEncapsulatedContent()
and not newFile.flags.isCapsuleOverride())):
continue
# this test catches files which have changed from not
# config files to config files; these need to be included
# unconditionally so we always have the pristine contents
# to include in the local database
# Also include contents of config files when mirroring if the
# fileid changed, even if the SHA-1 did not.
# cf CNY-1570, CNY-1699, CNY-2210
if (contentsHash
or (oldFile and newFile.flags.isConfig()
and not oldFile.flags.isConfig())
or (forceAbsolute and newFile.hasContents)
):
if oldFileVersion and oldFile.hasContents:
oldCont = self.getFileContents(
[ (oldFileId, oldFileVersion, oldFile) ])[0]
newCont = self.getFileContents(
[ (newFileId, newFileVersion, newFile) ])[0]
(contType, cont) = changeset.fileContentsDiff(oldFile,
oldCont, newFile, newCont,
mirrorMode = mirrorMode)
# we don't let config files be ptr types; if they were
# they could be ptrs to things which aren't config files,
# which would completely hose the sort order we use. this
# could be relaxed someday to let them be ptr's to other
# config files
if not newFile.flags.isConfig() and \
contType == changeset.ChangedFileTypes.file:
contentsHash = newFile.contents.sha1()
ptr = ptrTable.get(contentsHash, None)
if ptr is not None:
contType = changeset.ChangedFileTypes.ptr
cont = filecontents.FromString(ptr)
else:
ptrTable[contentsHash] = pathId + newFileId
if not newFile.flags.isConfig() and \
contType == changeset.ChangedFileTypes.file:
cont = filecontents.CompressedFromDataStore(
self.contentsStore,
newFile.contents.sha1())
compressed = True
else:
compressed = False
# ptr entries are not compressed, whether or not they
# are config files. override the compressed rule from
# above
if contType == changeset.ChangedFileTypes.ptr:
compressed = False
cs.addFileContents(pathId, newFileId, contType, cont,
newFile.flags.isConfig(),
compressed = compressed)
if not recurse:
yield cs, externalTroveList, externalFileList, removedTroveList
cs = changeset.ChangeSet()
externalTroveList = []
externalFileList = []
removedTroveList = []
if recurse:
yield cs, externalTroveList, externalFileList, removedTroveList
