def testFileObjMissing(self): # create an absolute changeset cs = changeset.ChangeSet() # add a pkg diff flavor = deps.deps.parseFlavor('') v = versions.VersionFromString('/%s/1.0-1-1' %self.cfg.buildLabel.asString()).copy() v.resetTimeStamps() t = trove.Trove('test:test', v, flavor, None) path = self.workDir + '/blah' f = open(path, 'w') f.write('hello, world!\n') f.close() pathId = sha1helper.md5String('/blah') f = files.FileFromFilesystem(path, pathId) # add the file, and SKIP including # the filestream by using cs.addFile(). This creates an # incomplete changeset t.addFile(pathId, '/blah', v, f.fileId()) cs.addFileContents(pathId, f.fileId(), changeset.ChangedFileTypes.file, filecontents.FromFilesystem(path), f.flags.isConfig()) t.computeDigests() diff = t.diff(None, absolute = 1)[0] cs.newTrove(diff) repos = self.openRepository() try: repos.commitChangeSet(cs) assert 0, "Did not raise IntegrityError" except errors.IntegrityError, e: assert(str(e).startswith("Incomplete changeset specified: missing pathId e806729b6a2b568fa7e77c3efa3a9684 fileId"))
def _addFile(self, cs, trv, path): pathId = sha1helper.md5String(path) absPath = self.cfg.root + path fileObj = files.FileFromFilesystem(absPath, pathId) fileId = fileObj.fileId() trv.addFile(pathId, path, trv.getVersion(), fileId) cs.addFile(None, fileId, fileObj.freeze()) if fileObj.hasContents: cs.addFileContents(pathId, fileId, changeset.ChangedFileTypes.file, filecontents.FromFilesystem(absPath), False)
def testFileUpdateMissingKey(self): fingerprint = '95B457D16843B21EA3FC73BBC7C32FC1F94E405E' # make a changeset with useful stuff that could be installed cs = changeset.ChangeSet() flavor = deps.parseFlavor('') v = versions.VersionFromString('/%s/1.0-1-1' % self.cfg.buildLabel.asString()).copy() v.resetTimeStamps() t = trove.Trove('test:test', v, flavor, None) path = self.workDir + '/blah' f = open(path, 'w') f.write('hello, world!\n') f.close() pathId = sha1helper.md5String('/blah') f = files.FileFromFilesystem(path, pathId) fileId = f.fileId() t.addFile(pathId, '/blah', v, fileId) cs.addFile(None, fileId, f.freeze()) cs.addFileContents(pathId, fileId, changeset.ChangedFileTypes.file, filecontents.FromFilesystem(path), f.flags.isConfig()) diff = t.diff(None, absolute=1)[0] cs.newTrove(diff) cs.addPrimaryTrove('test:test', v, flavor) # sign the changeset csPath = os.path.join(self.workDir, 'test-1.0-1.ccs') cs = cook.signAbsoluteChangeset(cs, fingerprint) cs.writeToFile(csPath) tmpPath = mkdtemp() keyCache = openpgpkey.getKeyCache() newKeyCache = openpgpkey.OpenPGPKeyFileCache() pubRing = self.cfg.pubRing self.cfg.pubRing = [tmpPath + '/pubring.gpg'] keyCacheCallback = openpgpkey.KeyCacheCallback(None, self.cfg) newKeyCache.setCallback(keyCacheCallback) openpgpkey.setKeyCache(newKeyCache) try: self.updatePkg(self.rootDir, csPath) finally: self.cfg.pubRing = pubRing openpgpkey.setKeyCache(keyCache)
def getFileContentsObjects(self, server, fileList, callback, outF, compressed): if not isinstance(self.c[server], ShimServerProxy): return netclient.NetworkRepositoryClient.getFileContentsObjects( self, server, fileList, callback, outF, compressed) filePaths = self.c[server].getFileContents(fileList) fileObjList = [] for path in filePaths: if compressed: fileObjList.append( filecontents.FromFilesystem(path, compressed = True)) else: f = gzip.GzipFile(path, "r") fileObjList.append(filecontents.FromFile(f)) return fileObjList
def testChangeSetFromFile(self): # ensure that absolute changesets that are read from disk # that contain config files write out changesets to a file # that do not change the file type to a diff. # set up a file with some contents cont = self.workDir + '/contents' f = open(cont, 'w') f.write('hello, world!\n') f.close() pathId = sha1helper.md5FromString('0' * 32) f = files.FileFromFilesystem(cont, pathId) f.flags.isConfig(1) # create an absolute changeset cs = changeset.ChangeSet() # add a pkg diff v = versions.VersionFromString('/localhost@rpl:devel/1.0-1-1', timeStamps=[1.000]) flavor = deps.parseFlavor('') t = trove.Trove('test', v, flavor, None) t.addFile(pathId, '/contents', v, f.fileId()) diff = t.diff(None, absolute=1)[0] cs.newTrove(diff) # add the file and file contents cs.addFile(None, f.fileId(), f.freeze()) cs.addFileContents(pathId, f.fileId(), changeset.ChangedFileTypes.file, filecontents.FromFilesystem(cont), f.flags.isConfig()) # write out the changeset cs.writeToFile(self.workDir + '/foo.ccs') # read it back in cs2 = changeset.ChangeSetFromFile(self.workDir + '/foo.ccs') # write it out again (there was a bug where all config files # became diffs) cs2.writeToFile(self.workDir + '/bar.ccs') # read it again cs3 = changeset.ChangeSetFromFile(self.workDir + '/bar.ccs') # verify that the file is a file, not a diff ctype, contents = cs3.getFileContents(pathId, f.fileId()) assert (ctype == changeset.ChangedFileTypes.file)
def getFileContentsFromTrove(self, n, v, f, pathList, callback = None, compressed = False): server = v.trailingLabel().getHost() if not isinstance(self.c[server], ShimServerProxy): return netclient.NetworkRepositoryClient.getFileContentsFromTrove( self, n, v, f, pathList, callback = callback, compressed = compressed) pathList = [self.fromPath(x) for x in pathList] v = self.fromVersion(v) f = self.fromFlavor(f) filePaths = self.c[server].getFileContentsFromTrove(n,v,f, pathList) fileObjList = [] for path in filePaths: if compressed: fileObjList.append( filecontents.FromFilesystem(path, compressed = True)) else: f = gzip.GzipFile(path, "r") fileObjList.append(filecontents.FromFile(f)) return fileObjList
def _merge(self): changeSet = ChangeSet() deleteDirs = set() doCommit = False # If this is not None then all ephemeral sources will still be fetched # but will be placed in this directory instead. if self.helper.plan.ephemeralSourceDir: ephDir = self.helper.makeEphemeralDir() else: ephDir = None def _addFile(path, contents, isText): if path in oldFiles: # Always recycle pathId if available. pathId, _, oldFileId, oldFileVersion = oldFiles[path] else: pathId = hashlib.md5(path).digest() oldFileId = oldFileVersion = None fileHelper = filetypes.RegularFile(contents=contents, config=isText) fileStream = fileHelper.get(pathId) fileStream.flags.isSource(set=True) fileId = fileStream.fileId() # If the fileId matches, recycle the fileVersion too. if fileId == oldFileId: fileVersion = oldFileVersion else: fileVersion = newTrove.getVersion() filesToAdd[fileId] = (fileStream, fileHelper.contents, isText) newTrove.addFile(pathId, path, fileVersion, fileId) for package, (recipeText, recipeObj), oldTrove in zip( self.packages, self.recipes, self.oldTroves): filesToAdd = {} oldFiles = {} if oldTrove is not None: for pathId, path, fileId, fileVer in oldTrove.iterFileList(): oldFiles[path] = (pathId, path, fileId, fileVer) newTrove = Trove(package.name, package.nextVersion, deps.Flavor()) newTrove.setFactory(package.targetConfig.factory) # Add upstream files to new trove. Recycle pathids from the old # version. # LAZY: assume that everything other than the recipe is binary. # Conary has a magic module, but it only accepts filenames! for path, contents in package.recipeFiles.iteritems(): isText = path == package.getRecipeName() _addFile(path, contents, isText) # Collect requested auto sources from recipe. Unknown recipe types # will not be loaded so recipeObj will be the class, so assume # these have no sources. if not inspect.isclass(recipeObj): recipeFiles = dict((os.path.basename(x.getPath()), x) for x in recipeObj.getSourcePathList()) newFiles = set(x[1] for x in newTrove.iterFileList()) needFiles = set(recipeFiles) - newFiles for autoPath in needFiles: source = recipeFiles[autoPath] if (autoPath in oldFiles and not self.helper.plan.refreshSources and not source.ephemeral): # File exists in old version. pathId, path, fileId, fileVer = oldFiles[autoPath] newTrove.addFile(pathId, path, fileVer, fileId) continue if source.ephemeral and not ephDir: continue # File doesn't exist; need to create it. if source.ephemeral: laUrl = lookaside.laUrl(source.getPath()) tempDir = joinPaths(ephDir, os.path.dirname(laUrl.filePath())) mkdirChain(tempDir) else: tempDir = tempfile.mkdtemp() deleteDirs.add(tempDir) snapshot = _getSnapshot(self.helper, package, source, tempDir) if not source.ephemeral and snapshot: autoPathId = hashlib.md5(autoPath).digest() autoObj = FileFromFilesystem(snapshot, autoPathId) autoObj.flags.isAutoSource(set=True) autoObj.flags.isSource(set=True) autoFileId = autoObj.fileId() autoContents = filecontents.FromFilesystem(snapshot) filesToAdd[autoFileId] = (autoObj, autoContents, False) newTrove.addFile(autoPathId, autoPath, newTrove.getVersion(), autoFileId) # If the old and new troves are identical, just use the old one. if oldTrove and _sourcesIdentical( oldTrove, newTrove, [self.oldChangeSet, filesToAdd]): package.setDownstreamVersion(oldTrove.getVersion()) log.debug('Skipped %s=%s', oldTrove.getName(), oldTrove.getVersion()) continue # Add files and contents to changeset. for fileId, (fileObj, fileContents, cfgFile) in filesToAdd.items(): changeSet.addFileContents(fileObj.pathId(), fileObj.fileId(), ChangedFileTypes.file, fileContents, cfgFile) changeSet.addFile(None, fileObj.fileId(), fileObj.freeze()) # Create a changelog entry. changeLog = ChangeLog( name=self.helper.cfg.name, contact=self.helper.cfg.contact, message=self.helper.plan.commitMessage + '\n') newTrove.changeChangeLog(changeLog) # Calculate trove digests and add the trove to the changeset newTrove.invalidateDigests() newTrove.computeDigests() newTroveCs = newTrove.diff(None, absolute=True)[0] changeSet.newTrove(newTroveCs) doCommit = True package.setDownstreamVersion(newTrove.getVersion()) log.debug('Created %s=%s', newTrove.getName(), newTrove.getVersion()) if doCommit: cook.signAbsoluteChangesetByConfig(changeSet, self.helper.cfg) f = tempfile.NamedTemporaryFile(dir=os.getcwd(), suffix='.ccs', delete=False) f.close() changeSet.writeToFile(f.name) try: self.helper.getRepos().commitChangeSet(changeSet) except: log.error("Error committing changeset to repository, " "failed changeset is saved at %s", f.name) raise else: os.unlink(f.name) for path in deleteDirs: shutil.rmtree(path)
def _addPhantomContents(self, changeSet, trv, header): """Fabricate files for the given RPM header""" for (path, owner, group, mode, size, rdev, flags, vflags, linkto, mtime) in itertools.izip( header[rpmhelper.OLDFILENAMES], header[rpmhelper.FILEUSERNAME], header[rpmhelper.FILEGROUPNAME], header[rpmhelper.FILEMODES], header[rpmhelper.FILESIZES], header[rpmhelper.FILERDEVS], header[rpmhelper.FILEFLAGS], header[rpmhelper.FILEVERIFYFLAGS], header[rpmhelper.FILELINKTOS], header[rpmhelper.FILEMTIMES], ): fullPath = util.joinPaths(self.root, path) fakestat = FakeStat(mode, 0, None, 1, owner, group, size, mtime, mtime, mtime, st_rdev=rdev, linkto=linkto) pathId = os.urandom(16) # Adapted from conary.build.source.addCapsule.doRPM kind = 'regular' if flags & rpmhelper.RPMFILE_GHOST: kind = 'initial' elif flags & (rpmhelper.RPMFILE_CONFIG | rpmhelper.RPMFILE_MISSINGOK | rpmhelper.RPMFILE_NOREPLACE): if size: kind = 'config' else: kind = 'initial' elif vflags: if (stat.S_ISREG(mode) and not (vflags & rpmhelper.RPMVERIFY_FILEDIGEST) or (stat.S_ISLNK(mode) and not (vflags & rpmhelper.RPMVERIFY_LINKTO))): kind = 'initial' # Ignore failures trying to sha1 missing/inaccessible files as long # as those files are flagged initial contents (ghost) fileStream = files.FileFromFilesystem(fullPath, pathId, statBuf=fakestat, sha1FailOk=True) if kind == 'config': fileStream.flags.isConfig(set=True) elif kind == 'initial': fileStream.flags.isInitialContents(set=True) else: assert kind == 'regular' # From conary.build.capsulepolicy.Payload if (isinstance(fileStream, files.RegularFile) and not fileStream.flags.isConfig() and not (fileStream.flags.isInitialContents() and not fileStream.contents.size())): fileStream.flags.isEncapsulatedContent(set=True) fileId = fileStream.fileId() trv.addFile(pathId, path, trv.getVersion(), fileId) changeSet.addFile(None, fileId, fileStream.freeze()) # Config file contents have to go into the database, so snag the # contents from the filesystem and put them in the changeset. if (fileStream.hasContents and not fileStream.flags.isEncapsulatedContent()): if fileStream.contents.sha1() == sha1helper.sha1Empty: # Missing/ghost config file. Hopefully it is supposed to be # empty, but even if not then the fake SHA-1 will be the # SHA-1 of the empty string since there's no hint of what # it was supposed to be. contents = filecontents.FromString('') else: contents = filecontents.FromFilesystem(fullPath) changeSet.addFileContents( pathId, fileId, contType=changeset.ChangedFileTypes.file, contents=contents, cfgFile=fileStream.flags.isConfig(), )
def install(self, flags, troveCs): ACTION_RESTORE = 1 ACTION_SKIP = 2 ACTION_CONFLICT = 3 rc = SingleCapsuleOperation.install(self, flags, troveCs) if rc is None: # parent class thinks we should just ignore this troveCs; I'm # not going to argue with it (it's probably because the capsule # hasn't changed return None (oldTrv, trv) = rc trvInfo = troveCs.getNewNameVersionFlavor() oldTrvInfo = troveCs.getOldNameVersionFlavor() hasCapsule = troveCs.hasCapsule() # Updates the fsJob metadata for installing the current trove. # It assumes files are replaced on install, and complains if something # is in the way unless the appropriate flags are set. This is a very # much simplified version of FilesystemJob._singleTrove() which maps # out a complete install strategy for native packages. Note that # we walk all of the files in this trove, not just the new files # or the changed files, because RPM installs all of the files. toRestore = [] changedByPathId = dict((x[0], x) for x in troveCs.getChangedFileList()) # things which aren't change, new, or removed are unchanged unchangedByPathId = (set(x[0] for x in trv.iterFileList()) - set(changedByPathId.iterkeys()) - set(x[0] for x in troveCs.getNewFileList()) - set(troveCs.getOldFileList())) l = [] for oldFileInfo in troveCs.getChangedFileList(): oldFileId, oldVersion = oldTrv.getFile(oldFileInfo[0])[1:3] l.append((oldFileInfo[0], oldFileId, oldVersion)) for unchangedPathId in unchangedByPathId: unchangedFileId, unchangedFileVersion = \ trv.getFile(unchangedPathId)[1:3] l.append((unchangedPathId, unchangedFileId, unchangedFileVersion)) fileObjs = self.db.getFileVersions(l) fileObjsByPathId = dict([(x[0], y) for x, y in itertools.izip(l, fileObjs)]) for fileInfo in trv.iterFileList(): pathId, path, fileId, version = fileInfo if os.path.dirname(path) in self.netSharedPath: # we do nothing. really. nothing. # # we don't back it up. we don't mark it as removed in # our database. we don't look for conflicts. nothing. continue if pathId in changedByPathId: oldFileId = oldTrv.getFile(pathId)[1] fileChange = self.changeSet.getFileChange(oldFileId, fileId) if (oldFileId == fileId): # only the version number changed; we don't need # to merge anything here fileObj = fileObjsByPathId[pathId] elif fileChange[0] == '\x01': fileObj = fileObjsByPathId[pathId] fileObj.twm(fileChange, fileObj) else: fileObj = files.ThawFile(fileChange, pathId) elif pathId in unchangedByPathId: fileObj = fileObjsByPathId[pathId] else: # if it's not changed and it's not unchanged, it must be new fileStream = self.changeSet.getFileChange(None, fileId) fileObj = files.ThawFile(fileStream, pathId) absolutePath = util.joinPaths(self.root, path) if (fileObj.flags.isCapsuleAddition()): # this was added to the package outside of the RPM; we don't # have any responsibility for it continue elif (trove.conaryContents(hasCapsule, pathId, fileObj) and fileObj.lsTag != 'd'): # this content isn't part of the capsule; remember to put # it back when RPM is done self.preservePath(path, unlink=True) continue s = util.lstat(absolutePath) if not s: # there is nothing in the way, so there is nothing which # concerns us here. Track the file for later. toRestore.append((fileInfo, fileObj)) continue action = ACTION_CONFLICT existingOwners = list( self.db.iterFindPathReferences(path, justPresent=True, withStream=True)) if existingOwners: # Don't complain about files owned by the previous version # of this trove. l = [x for x in existingOwners if x[0:3] == oldTrvInfo] if l: existingOwners.remove(l[0]) if not existingOwners: action = ACTION_RESTORE elif stat.S_ISDIR(s.st_mode) and fileObj.lsTag == 'd': # Don't let existing directories stop us from taking over # ownership of the directory action = ACTION_RESTORE elif fileObj.flags.isInitialContents(): # Initial contents files may be restored on top of things # already in the filesystem. They're ghosts or config files # and RPM will get the contents right either way, and we # should remove them either way. action = ACTION_RESTORE if action == ACTION_CONFLICT and not existingOwners: # Check for "conflicts" that might just be a view across a # symlink. if self.fsJob.findAliasedRemovals(absolutePath): action = ACTION_RESTORE if action == ACTION_CONFLICT and existingOwners: if fileId in [x[4] for x in existingOwners]: # The files share metadata same. Whatever it looks like on # disk, RPM is going to blow it away with the new one. for info in existingOwners: self.fsJob.sharedFile(info[0], info[1], info[2], info[3]) action = ACTION_RESTORE elif path.startswith('/usr/share/doc/'): # Mirror badness Red Hat patches into RPM for rhel4 # and rhel5 action = ACTION_RESTORE else: existingFiles = [ files.ThawFile(x[5], pathId) for x in existingOwners ] compatibility = [ 1 for x in existingFiles if fileObj.compatibleWith(x) ] if 1 in compatibility: # files can be shared even though the fileId's # are different for info in existingOwners: self.fsJob.sharedFile(info[0], info[1], info[2], info[3]) action = ACTION_RESTORE elif 1 in [ files.rpmFileColorCmp(x, fileObj) for x in existingFiles ]: # rpm file colors and the default rpm setting for # file color policy make elf64 files silently replace # elf32 files. follow that behavior here. # # no, i'm not making this up # # yes, really action = ACTION_SKIP elif (self._checkReplaceManagedFiles(flags, path) or 1 in [ files.rpmFileColorCmp(fileObj, x) for x in existingFiles ]): # The files are different. Bail unless we're supposed # to replace managed files. existingFile = files.FileFromFilesystem( absolutePath, pathId) for info in existingOwners: self.fsJob.userRemoval( fileObj=existingFile, content=filecontents.FromFilesystem( absolutePath), *info[0:4]) action = ACTION_RESTORE else: # it's not up to us to decide if this is a true # conflict; the database layer will do that for # us (see checkPathConflicts) action = ACTION_RESTORE elif flags.replaceUnmanagedFiles: # we don't own it, but it's on disk. RPM will just write over # it and we have the flag saying we're good with that action = ACTION_RESTORE if action == ACTION_RESTORE: # We may proceed, and RPM will replace this file for us. We # need to track that it's being restored to avoid conflicts # with other restorations though. toRestore.append((fileInfo, fileObj)) elif action == ACTION_CONFLICT: # The file exists already, we can't share it, and we're not # allowed to overwrite it. self._error( errors.FileInWayError(util.normpath(path), troveCs.getName(), troveCs.getNewVersion(), troveCs.getNewFlavor())) else: assert (action == ACTION_SKIP) self.preservePath(path, unlink=False) self.fsJob.userRemoval(trv.getName(), trv.getVersion(), trv.getFlavor(), pathId) # toRestore is the list of what is going to be restored. We need to get # the fileObjects which will be created so we can track them in the # filesystem job. This lets the filesystem job look for resolveable # conflicts within this update. We handle newly created files first # and files which have changed (so we have to look up the diff) # a bit later. for fileInfo, fileObj in toRestore: fullPath = util.joinPaths(self.root, path) self.fsJob._restore(fileObj, fullPath, trvInfo, "restoring %s from RPM", restoreFile=False, fileId=fileId)