Exemple #1
0
 def get(self, request, hostname, troveString, pathHash):
     repos, path, ver, fileObj = self._getFileInfo(hostname, troveString, 
                                                   pathHash)
     flags = fileObj.flags
     return models.TroveFile(hostname=hostname,
                     trove=troveString,
                     pathId=sha1helper.md5ToString(fileObj.pathId()), 
                     pathHash=pathHash, 
                     path=path, 
                     fileVersion=ver, 
                     fileId=sha1helper.sha1ToString(fileObj.fileId()), 
                     tags=','.join(fileObj.tags()),
                     isConfig=flags.isConfig(),
                     isInitialContents=flags.isInitialContents(),
                     isSource=flags.isSource(),
                     isAutoSource=flags.isAutoSource(),
                     isTransient=flags.isTransient(),
                     size=fileObj.contents.size(),
                     sha1=sha1helper.sha1ToString(fileObj.contents.sha1()),
                     permissions=fileObj.inode.permsString(),
                     mtime=fileObj.inode.mtime(),
                     owner=fileObj.inode.owner(),
                     group=fileObj.inode.group(),
                     provides=fileObj.provides(),
                     requires=fileObj.requires())
Exemple #2
0
    def testFileIdWrong(self):
        # create an absolute changeset
        cs = changeset.ChangeSet()

        # add a pkg diff
        flavor = deps.deps.parseFlavor('')
        v = versions.VersionFromString('/%s/1.0-1-1'
                                       %self.cfg.buildLabel.asString()).copy()
        v.resetTimeStamps()
        t = trove.Trove('test:test', v, flavor, None)
        path = self.workDir + '/blah'
        f = open(path, 'w')
        f.write('hello, world!\n')
        f.close()
        pathId = sha1helper.md5String('/blah')
        f = files.FileFromFilesystem(path, pathId)
        # add the file, but munge the fileid
        brokenFileId = ''.join(reversed(f.fileId()))
        cs.addFile(None, brokenFileId, f.freeze())
        t.addFile(pathId, '/blah', v, brokenFileId)
        t.computeDigests()

        diff = t.diff(None, absolute = 1)[0]
        cs.newTrove(diff)

        repos = self.openRepository()
        try:
            repos.commitChangeSet(cs)
            assert 0, "Integrity Error not raised"
        except errors.TroveIntegrityError, e:
            assert(str(e) == 'fileObj.fileId() != fileId in changeset for '
                             'pathId %s' % sha1helper.md5ToString(pathId))
Exemple #3
0
 def _getCacheFilename(self, name, version, flavor, compNames):
     # hash the version and flavor to give a unique filename
     versionFlavor = '%s %s' % (version.asString(), flavor.freeze())
     if compNames:
         # we could be generating a different set of troves,
         # say, :runtime only instead of :runtime + :devel, so add
         # in the components we're creating
         versionFlavor += ' %s' % ' '.join(compNames)
     h = sha1helper.md5ToString(sha1helper.md5String(versionFlavor))
     return '%s-%s.ccs' %(name, h)
Exemple #4
0
 def get(self, request, hostname, troveString):
     repos = self.getRepos()
     name, version, flavor = self._getTuple(troveString)
     trv = repos.getTrove(name, version, flavor, withFiles=True)
     fileList = []
     for pathId, path, fileId, fileVersion in trv.iterFileList():
         pathHash = sha1helper.md5String(path)
         fileList.append(models.TroveFile(
                     hostname=hostname,
                     pathId=sha1helper.md5ToString(pathId), 
                     pathHash=sha1helper.md5ToString(pathHash), 
                     path=path, 
                     fileId=sha1helper.sha1ToString(fileId), 
                     trove=troveString,
                     fileVersion=fileVersion))
     troveModel = models.Trove(hostname=hostname, 
                               name=name, version=version, flavor=flavor,
                               files=fileList)
     return troveModel
Exemple #5
0
    def _getCheckFilesList(self,
                           csTrove,
                           troveInfo,
                           fileHostFilter,
                           configRestoreList,
                           normalRestoreList,
                           restoreContents=True):
        checkFilesList = []

        for (pathId, path, fileId, newVersion) in csTrove.getNewFileList():
            if (fileHostFilter and newVersion.getHost() not in fileHostFilter):
                fileObj = None
                fileStream = None
            else:
                # New files don't always have streams in the changeset.
                # Filesets and clones don't include them for files which
                # are already known to be in the repository.
                fileStream = self.cs.getFileChange(None, fileId)

                if fileStream is None:
                    if not fileHostFilter:
                        # We are trying to commit to a database, but the
                        # diff returned nothing
                        raise KeyError

                    checkFilesList.append((pathId, fileId, newVersion))
                    fileObj = None
                else:
                    fileObj = files.ThawFile(fileStream, pathId)
                    if fileObj and fileObj.fileId() != fileId:
                        raise trove.TroveIntegrityError(
                            csTrove.getName(), csTrove.getNewVersion(),
                            csTrove.getNewFlavor(),
                            "fileObj.fileId() != fileId in changeset "
                            "for pathId %s" % sha1helper.md5ToString(pathId))

            self.addFileVersion(troveInfo,
                                pathId,
                                path,
                                fileId,
                                newVersion,
                                fileStream=fileStream,
                                withContents=restoreContents)

            self._handleContents(pathId,
                                 fileId,
                                 fileStream,
                                 configRestoreList,
                                 normalRestoreList,
                                 restoreContents=restoreContents)

        return checkFilesList
Exemple #6
0
    def _restoreNormal(self, cs, normalRestoreList, preRestored):
        ptrRestores = []
        ptrRefsAdded = {}
        lastRestore = None  # restore each pathId,fileId combo once
        while normalRestoreList:
            (pathId, fileId, sha1, restoreContents) = normalRestoreList.pop(0)
            if preRestored is not None and sha1 in preRestored:
                continue
            if (pathId, fileId) == lastRestore:
                continue

            lastRestore = (pathId, fileId)

            try:
                (contType, fileContents) = cs.getFileContents(pathId,
                                                              fileId,
                                                              compressed=True)
            except KeyError:
                raise errors.IntegrityError(
                    "Missing file contents for pathId %s, fileId %s" %
                    (sha1helper.md5ToString(pathId),
                     sha1helper.sha1ToString(fileId)))
            if contType == changeset.ChangedFileTypes.ptr:
                ptrRestores.append(sha1)
                target = util.decompressString(fileContents.get().read())

                if util.tupleListBsearchInsert(
                        normalRestoreList,
                    (target[:16], target[16:], sha1, True), self.ptrCmp):
                    # Item was inserted. This creates a reference in the
                    # datastore; keep track of it to prevent a duplicate
                    # reference count.
                    ptrRefsAdded[sha1] = True

                continue

            assert (contType == changeset.ChangedFileTypes.file)
            self.addFileContents(sha1,
                                 fileContents,
                                 restoreContents,
                                 0,
                                 precompressed=True)

        for sha1 in ptrRestores:
            # Increment the reference count for items which were ptr's
            # to a different file.
            if sha1 in ptrRefsAdded:
                del ptrRefsAdded[sha1]
            else:
                self.addFileContents(sha1, None, False, 0)
Exemple #7
0
    def _write(self, f):
        """
        Returns a string representing file information for this trove
        trove, which can later be read by the read() method. This is
        only used to create the Conary control file when dealing with
        :source component checkins, so things like trove dependency
        information is not needed.  The format of the string is:

        name <name>
        version <version>
        branch <branch>
        (lastmerged <version>)?
        (factory <name>)?
        <file count>
        PATHID1 PATH1 FILEID1 ISCONFIG1 REFRESH1 VERSION1
        PATHID2 PATH2 FILEID2 ISCONFIG2 REFRESH2 VERSION2
        .
        .
        .
        PATHIDn PATHn FILEIDn ISCONFIGn REFRESHn VERSIONn
        """
        assert(len(self.strongTroves) == 0)
        assert(len(self.weakTroves) == 0)

        f.write("name %s\n" % self.getName())
        f.write("version %s\n" % self.getVersion().freeze())
        f.write("branch %s\n" % self.getBranch().freeze())
        if self.getLastMerged() is not None:
            f.write("lastmerged %s\n" % self.getLastMerged().freeze())
        if self.getFactory():
            f.write("factory %s\n" % self.getFactory())

        rc = []
        rc.append("%d\n" % (len(list(self.iterFileList()))))

        rc += [ "%s %s %s %s %s\n" % (sha1helper.md5ToString(x[0]),
                                x[1],
                                sha1helper.sha1ToString(x[2]),
                                self.fileInfo[x[0]],
                                x[3].asString())
                for x in sorted(self.iterFileList()) ]

        f.write("".join(rc))
Exemple #8
0
    def _write(self, f):
        """
        Returns a string representing file information for this trove
        trove, which can later be read by the read() method. This is
        only used to create the Conary control file when dealing with
        :source component checkins, so things like trove dependency
        information is not needed.  The format of the string is:

        name <name>
        version <version>
        branch <branch>
        (lastmerged <version>)?
        (factory <name>)?
        <file count>
        PATHID1 PATH1 FILEID1 ISCONFIG1 REFRESH1 VERSION1
        PATHID2 PATH2 FILEID2 ISCONFIG2 REFRESH2 VERSION2
        .
        .
        .
        PATHIDn PATHn FILEIDn ISCONFIGn REFRESHn VERSIONn
        """
        assert (len(self.strongTroves) == 0)
        assert (len(self.weakTroves) == 0)

        f.write("name %s\n" % self.getName())
        f.write("version %s\n" % self.getVersion().freeze())
        f.write("branch %s\n" % self.getBranch().freeze())
        if self.getLastMerged() is not None:
            f.write("lastmerged %s\n" % self.getLastMerged().freeze())
        if self.getFactory():
            f.write("factory %s\n" % self.getFactory())

        rc = []
        rc.append("%d\n" % (len(list(self.iterFileList()))))

        rc += [
            "%s %s %s %s %s\n" %
            (sha1helper.md5ToString(x[0]), x[1], sha1helper.sha1ToString(
                x[2]), self.fileInfo[x[0]], x[3].asString())
            for x in sorted(self.iterFileList())
        ]

        f.write("".join(rc))
Exemple #9
0
    def _getCheckFilesList(self, csTrove, troveInfo, fileHostFilter,
            configRestoreList, normalRestoreList, restoreContents = True):
        checkFilesList = []

        for (pathId, path, fileId, newVersion) in csTrove.getNewFileList():
            if (fileHostFilter
                and newVersion.getHost() not in fileHostFilter):
                fileObj = None
                fileStream = None
            else:
                # New files don't always have streams in the changeset.
                # Filesets and clones don't include them for files which
                # are already known to be in the repository.
                fileStream = self.cs.getFileChange(None, fileId)

                if fileStream is None:
                    if not fileHostFilter:
                        # We are trying to commit to a database, but the
                        # diff returned nothing
                        raise KeyError

                    checkFilesList.append((pathId, fileId, newVersion))
                    fileObj = None
                else:
                    fileObj = files.ThawFile(fileStream, pathId)
                    if fileObj and fileObj.fileId() != fileId:
                        raise trove.TroveIntegrityError(csTrove.getName(),
                              csTrove.getNewVersion(), csTrove.getNewFlavor(),
                              "fileObj.fileId() != fileId in changeset "
                              "for pathId %s" %
                                    sha1helper.md5ToString(pathId))

            self.addFileVersion(troveInfo, pathId, path, fileId,
                                newVersion, fileStream = fileStream,
                                withContents = restoreContents)

            self._handleContents(pathId, fileId, fileStream, configRestoreList,
                                 normalRestoreList,
                                 restoreContents = restoreContents)

        return checkFilesList
Exemple #10
0
    def _restoreConfig(self, cs, configRestoreList):
        # config files are cached, so we don't have to worry about not
        # restoring the same fileId/pathId twice
        for (pathId, newFileId, sha1, oldfile, newFileId, oldVersion,
             oldFileId, restoreContents) in configRestoreList:
            if cs.configFileIsDiff(pathId, newFileId):
                (contType,
                 fileContents) = cs.getFileContents(pathId, newFileId)

                # the content for this file is in the form of a
                # diff, which we need to apply against the file in
                # the repository
                assert (oldVersion)

                try:
                    f = self.repos.getFileContents([(oldFileId, oldVersion,
                                                     oldfile)])[0].get()
                except KeyError:
                    raise errors.IntegrityError(
                        "Missing file contents for pathId %s, fileId %s" %
                        (sha1helper.md5ToString(pathId),
                         sha1helper.sha1ToString(oldFileId)))

                oldLines = f.readlines()
                f.close()
                del f
                diff = fileContents.get().readlines()
                (newLines, failedHunks) = patch.patch(oldLines, diff)
                fileContents = filecontents.FromString("".join(newLines))

                assert (not failedHunks)
            else:
                # config files are not always available compressed (due
                # to the config file cache)
                fileContents = filecontents.FromChangeSet(
                    cs, pathId, newFileId)

            self.addFileContents(sha1, fileContents, restoreContents, 1)
Exemple #11
0
    def formatFile(self, pathId, path, fileId, version, fileObj=None,
                   prefix='', indent=0):
        taglist = ''
        sha1 = ''
        id = ''
        flavor = ''

        dcfg = self.dcfg
        verbose = dcfg.isVerbose()

        if verbose and isinstance(fileObj, files.SymbolicLink):
            name = "%s -> %s" % (path, fileObj.target())
        else:
            name = path
        if dcfg.fileFlavors:
            if not fileObj.flavor().isEmpty():
                flavor = '[%s]' % fileObj.flavor()

        if dcfg.tags:
            tags = []
            if fileObj.tags:
                tags.extend(fileObj.tags)
            if fileObj.flags.isInitialContents():
                tags.append('initialContents')
            if fileObj.flags.isAutoSource():
                tags.append('autosource')
            if fileObj.flags.isConfig():
                tags.append('config')
            if fileObj.flags.isTransient():
                tags.append('transient')
            if tags:
                taglist = ' {' + ' '.join(tags) + '}'
        if dcfg.sha1s:
            if hasattr(fileObj, 'contents') and fileObj.contents:
                sha1 = sha1ToString(fileObj.contents.sha1()) + ' '
            else:
                sha1 = ' '*41

        if dcfg.ids and pathId:
            id = md5ToString(pathId) + ' ' + sha1ToString(fileId) + ', '
        if dcfg.fileVersions:
            if dcfg.useFullVersions():
                verStr = '    %s' % version
            elif dcfg.showLabels:
                verStr = '    %s/%s' % (version.branch().label(), version.trailingRevision())
            else:
                verStr = '    %s' % version.trailingRevision()
        else:
            verStr = ''

        spacer = '  ' * indent

        if fileObj:
            owner = fileObj.inode.owner()
            if owner[0] == '+':
                owner = owner[1:]
            group = fileObj.inode.group()
            if group[0] == '+':
                group = group[1:]

        if verbose:
            ln = "%s%s%s%s%s    1 %-8s %-8s %s %s %s%s%s%s" % \
              (spacer,
               prefix, id, sha1, fileObj.modeString(), owner,
               group, fileObj.sizeString(),
               fileObj.timeString(), name, flavor, taglist, verStr)
        else:
            ln = "%s%s%s%s%s%s%s" % (spacer, id, sha1, path, flavor,
                                     taglist, verStr)

        yield ln

        if dcfg.fileDeps:
            for ln in self.formatDeps(fileObj.provides(), fileObj.requires(),
                                      indent + 1, showEmpty = False):
                yield ln
Exemple #12
0
    def _createInstallTroveObjects(self, fileHostFilter = [],
                                   callback = None, hidden = False,
                                   mirror = False, allowIncomplete = False,
                                   excludeCapsuleContents = False):
        # create the trove objects which need to be installed; the
        # file objects which map up with them are created later, but
        # we do need a map from pathId to the path and version of the
        # file we need, so build up a dictionary with that information

        configRestoreList = []
        normalRestoreList = []
        checkFilesList = []

        newList = [ x for x in self.cs.iterNewTroveList() ]
        repos = self.repos
        cs = self.cs

        oldTrovesNeeded = [ x.getOldNameVersionFlavor() for x in
                                newList if x.getOldVersion() ]
        oldTroveIter = repos.iterTroves(oldTrovesNeeded, hidden = True)

        troveNo = 0
        for csTrove in newList:
            if csTrove.troveType() == trove.TROVE_TYPE_REMOVED:
                # deal with these later on to ensure any changesets which
                # are relative to removed troves can be processed
                continue

            troveNo += 1

            if callback:
                callback.creatingDatabaseTransaction(troveNo, len(newList))

            newVersion = csTrove.getNewVersion()
            oldTroveVersion = csTrove.getOldVersion()
            oldTroveFlavor = csTrove.getOldFlavor()
            troveName = csTrove.getName()
            troveFlavor = csTrove.getNewFlavor()

            if repos.hasTrove(troveName, newVersion, troveFlavor):
                raise errors.CommitError, \
                       "version %s of %s already exists" % \
                        (newVersion.asString(), csTrove.getName())

            if oldTroveVersion:
                newTrove = oldTroveIter.next()
                assert(newTrove.getNameVersionFlavor() ==
                        csTrove.getOldNameVersionFlavor())
                self.oldTrove(newTrove, csTrove, troveName, oldTroveVersion,
                              oldTroveFlavor)

                oldCompatClass = newTrove.getCompatibilityClass()

                if csTrove.isRollbackFence(
                                   oldCompatibilityClass = oldCompatClass,
                                   update = True):
                    self.invalidateRollbacks(set = True)
            else:
                newTrove = trove.Trove(csTrove.getName(), newVersion,
                                       troveFlavor, csTrove.getChangeLog(),
                                       setVersion = False)
                # FIXME: we reset the trove version
                # since in this case we need to use the fileMap returned
                # from applyChangeSet
                allowIncomplete = True

            newFileMap = newTrove.applyChangeSet(csTrove,
                                     needNewFileMap=True,
                                     allowIncomplete=allowIncomplete)
            skipContents = (excludeCapsuleContents and
                              newTrove.troveInfo.capsule.type and
                              newTrove.troveInfo.capsule.type())

            if newTrove.troveInfo.incomplete():
                log.warning('trove %s has schema version %s, which contains'
                        ' information not handled by this client.  This'
                        ' version of Conary understands schema version %s.'
                        ' Dropping extra information.  Please upgrade conary.',
                        newTrove.getName(), newTrove.troveInfo.troveVersion(),
                        trove.TROVE_VERSION)

            self.checkTroveCompleteness(newTrove)

            self.checkTroveSignatures(newTrove, callback=callback)

            if oldTroveVersion is not None:
                troveInfo = self.addTrove(
                        (troveName, oldTroveVersion, oldTroveFlavor), newTrove,
                        csTrove, hidden = hidden)
            else:
                troveInfo = self.addTrove(None, newTrove, csTrove,
                                          hidden = hidden)

            checkFilesList += self._getCheckFilesList(csTrove, troveInfo,
                fileHostFilter, configRestoreList, normalRestoreList,
                restoreContents = not skipContents)

            for (pathId, path, fileId, newVersion) in \
                            newTrove.iterFileList(members = True,
                                                  capsules = True):
                # handle files which haven't changed; we know which those
                # are because they're in the merged trove but they aren't
                # in the newFileMap
                if pathId in newFileMap:
                    continue

                self.addFileVersion(troveInfo, pathId, path, fileId,
                                    newVersion, withContents = not skipContents)

            filesNeeded = []
            for i, (pathId, path, fileId, newVersion) in enumerate(csTrove.getChangedFileList()):
                tup = newFileMap[pathId]
                (oldPath, oldFileId, oldVersion) = tup[-3:]
                if path is None:
                    path = oldPath
                if fileId is None:
                    oldFileId = fileId
                if newVersion is None:
                    newVersion = oldVersion

                if (fileHostFilter
                    and newVersion.getHost() not in fileHostFilter):
                    fileStream = None
                elif (oldVersion == newVersion and oldFileId == fileId):
                    # the file didn't change between versions; we can just
                    # ignore it
                    fileStream = None
                else:
                    fileStream = cs.getFileChange(oldFileId, fileId)

                    if fileStream and fileStream[0] == "\x01":
                        if len(fileStream) != 2:
                            # This is awful, but this is how we say a file
                            # stream didn't change. Omitting it or at least ''
                            # would be nicer, but would break clients.
                            filesNeeded.append((i, (pathId, oldFileId,
                                                    oldVersion)))
                            continue

                        fileStream = None

                # None is the file object
                self.addFileVersion(troveInfo, pathId, path, fileId,
                                    newVersion, fileStream = fileStream,
                                    withContents = not skipContents)

                if fileStream is not None:
                    self._handleContents(pathId, fileId, fileStream,
                                    configRestoreList, normalRestoreList,
                                    oldFileId = oldFileId,
                                    oldVersion = oldVersion,
                                    oldfile = None,
                                    restoreContents = not skipContents)

            oldFileObjects = list(repos.getFileVersions(
                                        [ x[1] for x in filesNeeded ]))

            for i, (pathId, path, fileId, newVersion) in enumerate(csTrove.getChangedFileList()):
                if not filesNeeded or filesNeeded[0][0] != i:
                    continue
                filesNeeded.pop(0)

                tup = newFileMap[pathId]
                (oldPath, oldFileId, oldVersion) = tup[-3:]
                if path is None:
                    path = oldPath
                if fileId is None:
                    oldFileId = fileId
                if newVersion is None:
                    newVersion = oldVersion

                restoreContents = not skipContents

                diff = cs.getFileChange(oldFileId, fileId)

                # stored as a diff (the file type is the same
                # and (for *repository* commits) the file
                # is in the same repository between versions
                oldfile = oldFileObjects.pop(0)
                fileObj = oldfile.copy()
                fileObj.twm(diff, oldfile)
                assert(fileObj.pathId() == pathId)
                fileStream = fileObj.freeze()

                if (not mirror) and (
                    fileObj.hasContents and fileObj.contents.sha1() == oldfile.contents.sha1()
                    and not (fileObj.flags.isConfig() and not oldfile.flags.isConfig())):
                    # don't restore the contents here. we don't
                    # need them, and they may be relative to
                    # something from a different repository
                    restoreContents = False

                if fileObj and fileObj.fileId() != fileId:
                    raise trove.TroveIntegrityError(csTrove.getName(),
                          csTrove.getNewVersion(), csTrove.getNewFlavor(),
                          "fileObj.fileId() != fileId in changeset")

                self.addFileVersion(troveInfo, pathId, path, fileId,
                                    newVersion, fileStream = fileStream,
                                    withContents = restoreContents)

                self._handleContents(pathId, fileId, fileStream,
                                configRestoreList, normalRestoreList,
                                oldFileId = oldFileId,
                                oldVersion = oldVersion,
                                oldfile = oldfile,
                                restoreContents = restoreContents)

            del newFileMap
            self.addTroveDone(troveInfo, mirror=mirror)

        try:
            # we need to actualize this, not just get a generator
            list(repos.getFileVersions(checkFilesList))
        except errors.FileStreamMissing, e:
            info = [ x for x in checkFilesList if x[1] == e.fileId ]
            (pathId, fileId) = info[0][0:2]
            # Missing from the repo; raise exception
            raise errors.IntegrityError(
                "Incomplete changeset specified: missing pathId %s "
                "fileId %s" % (sha1helper.md5ToString(pathId),
                               sha1helper.sha1ToString(fileId)))
Exemple #13
0
def verifySignatures(f, validKeys=None):
    """
    Given an extended file, compute signatures
    """
    f.seek(0)
    h = readHeader(f)

    # Cheap test first: verify MD5 sig
    sigmd5 = h.get(SIG_MD5, None)
    if sigmd5 is not None:
        f.seek(0)
        readSignatureHeader(f)

        # verify md5 digest
        md5 = digestlib.md5()
        util.copyfileobj(f, NullWriter(), digest=md5)
        if md5.digest() != sigmd5:
            raise MD5SignatureError(
                "The MD5 digest fails to verify: expected %s, got %s" %
                (sha1helper.md5ToString(sigmd5), md5.hexdigest()))

    # Don't bother if no gpg signature was present, or no valid keys were
    # presented
    if validKeys is None:
        return
    sigString = h.get(SIG_GPG, None)
    if sigString is None:
        return
    # Skip to immutable header region
    f.seek(0)
    readSignatureHeader(f)
    sig = openpgpfile.readSignature(sigString)

    keyId = sig.getSignerKeyId()
    matchingKeys = [x for x in validKeys if x.hasKeyId(keyId)]
    if not matchingKeys:
        raise PGPSignatureError("Signature generated with key %s does "
                                "not match valid keys %s" %
                                (keyId, ', '.join(x.getKeyId()
                                                  for x in validKeys)))

    key = matchingKeys[0]

    # signature verification assumes a seekable stream and will seek to the
    # beginning; use a SeekableNestedFile
    size = h.getHeaderPlusPayloadSize()
    if size is None:
        pos = f.tell()
        f.seek(0, 2)
        size = f.tell()
        f.seek(pos, 0)
    snf = None
    if hasattr(f, 'pread'):
        extFile = f
    elif hasattr(f, 'name'):
        extFile = util.ExtendedFile(f.name, buffering=False)
    else:
        # worst case scenario, we slurp everything in memory
        extFile = util.ExtendedStringIO(f.read())
        snf = extFile
    if snf is None:
        snf = util.SeekableNestedFile(extFile, start=f.tell(), size=size)
    try:
        sig.verifyDocument(key.getCryptoKey(), snf)
    except openpgpfile.SignatureError:
        raise PGPSignatureError
Exemple #14
0
def getTrove(cu, roleIds, name, version, flavor, mkUrl = None,
             thisHost = None, displayFlavor = None, excludeCapsules = False):

    def buildTupleList(tuples, name, mkUrl = mkUrl):
        l = getattr(datamodel.SingleTrove, name)()
        for troveInfo in sorted(tuples.iter()):
            l.append(name = troveInfo.name(), version = troveInfo.version(),
                     flavor = troveInfo.flavor(), mkUrl = mkUrl)

        return l

    def fileQuery(gfcu, filesInstanceId, dirName = None):
        # XXX restricing by dirName seems and obvious thing to do here,
        # but it actually slows things down??
        #
        # the distinct here is unfortunate, but conary repositories had
        # a bug for about a year which caused it to store duplicate paths
        # if a path was committed for the first time duplicate times in
        # a single commit job
        gfcu.execute("""
            SELECT DISTINCT dirName, basename, version, pathId, fileId
                FROM TroveFiles
                JOIN Versions USING (versionId)
                JOIN FileStreams ON (TroveFiles.streamId = FileStreams.streamId)
                JOIN FilePaths ON (TroveFiles.filePathId = FilePaths.filePathId)
                JOIN DirNames ON
                    FilePaths.dirNameId = DirNames.dirNameId
                JOIN Basenames ON (FilePaths.baseNameId = Basenames.baseNameId)
                WHERE TroveFiles.instanceId = ? ORDER BY dirName, basename
        """, filesInstanceId)

    cu.execute("""
        SELECT Instances.instanceId, Nodes.timeStamps FROM Instances
            JOIN Nodes USING (itemId, versionId)
            JOIN Items USING (itemId)
            JOIN Versions ON (Instances.versionId = Versions.versionId)
            JOIN Flavors ON (Instances.flavorId = Flavors.flavorId)
            JOIN UserGroupInstancesCache AS ugi
                ON (instances.instanceId = ugi.instanceId AND
                    ugi.userGroupId in (%s))
        WHERE
            item = ? AND version = ? AND flavor = ?
    """ % ",".join( str(x) for x in roleIds), name, version,
        deps.parseFlavor(flavor).freeze())

    l = [ (x[0], x[1]) for x in cu ]
    if not l:
        return None

    instanceId, timeStamps = l[0]
    frzVer = versions.strToFrozen(version, timeStamps.split(":"))
    verobj = versions.ThawVersion(frzVer)

    tupleLists = [ ( trove._TROVEINFO_TAG_BUILDDEPS, 'builddeps' ),
                   ( trove._TROVEINFO_TAG_POLICY_PROV, 'policyprovider' ),
                   ( trove._TROVEINFO_TAG_LOADEDTROVES, 'loadedtroves' ),
                   ( trove._TROVEINFO_TAG_COPIED_FROM, 'copiedfrom' ),
                   ( trove._TROVEINFO_TAG_DERIVEDFROM, 'derivedfrom' ) ]

    cu.execute("""
    SELECT infoType, data FROM TroveInfo WHERE instanceId = ? AND
        infoType IN (%s)
                """ % ",".join(str(x) for x in
                        [ trove._TROVEINFO_TAG_SOURCENAME,
                          trove._TROVEINFO_TAG_CLONEDFROM,
                          trove._TROVEINFO_TAG_CLONEDFROMLIST,
                          trove._TROVEINFO_TAG_BUILDTIME,
                          trove._TROVEINFO_TAG_SIZE,
                          trove._TROVEINFO_TAG_METADATA,
                          trove._TROVEINFO_TAG_CAPSULE,
                        ] + [ x[0] for x in tupleLists ]
                ), instanceId)

    troveInfo = {}
    for infoType, data in cu:
        data = cu.frombinary(data)
        infoClass = trove.TroveInfo.streamDict[infoType][1]
        troveInfo[infoType] = infoClass(data)

    kwargs = { 'name' : name,
               'version' : verobj,
               'flavor' : flavor }

    if displayFlavor is not None:
        kwargs['displayflavor'] = displayFlavor

    if trove._TROVEINFO_TAG_BUILDTIME in troveInfo:
        kwargs['buildtime'] = int(troveInfo[trove._TROVEINFO_TAG_BUILDTIME]())

    if trove._TROVEINFO_TAG_SOURCENAME in troveInfo:
        kwargs['source'] = (troveInfo[trove._TROVEINFO_TAG_SOURCENAME](),
            verobj.getSourceVersion(), '')

    if trove._TROVEINFO_TAG_SIZE in troveInfo:
        kwargs['size'] = troveInfo[trove._TROVEINFO_TAG_SIZE]()

    if trove._TROVEINFO_TAG_METADATA in troveInfo:
        md = troveInfo[trove._TROVEINFO_TAG_METADATA].get()
        kwargs['shortdesc'] = md['shortDesc']
        kwargs['longdesc'] = md['longDesc']

        if md['licenses']:
            kwargs['license'] = [ x for x in md['licenses' ]]
        if md['crypto']:
            kwargs['crypto'] = [ x for x in md['crypto'] ]

    for (tag, tagName) in tupleLists:
        if tag in troveInfo:
            kwargs[tagName] = buildTupleList(troveInfo[tag], tagName,
                                             mkUrl = mkUrl)

    t = datamodel.SingleTrove(mkUrl = mkUrl, thisHost = thisHost, **kwargs)

    if trove._TROVEINFO_TAG_CLONEDFROMLIST in troveInfo:
        clonedFromList = troveInfo[trove._TROVEINFO_TAG_CLONEDFROMLIST]
    elif (trove._TROVEINFO_TAG_CLONEDFROM in troveInfo):
        clonedFromList = [ troveInfo[trove._TROVEINFO_TAG_CLONEDFROM]() ]
    else:
        clonedFromList = []

    for ver in clonedFromList:
        t.addClonedFrom(name, ver, flavor, mkUrl = mkUrl)

    hasCapsule = False
    if trove._TROVEINFO_TAG_CAPSULE in troveInfo:
        if troveInfo[trove._TROVEINFO_TAG_CAPSULE].type():
            hasCapsule = True

    fileQuery(cu, instanceId)

    for (dirName, baseName, fileVersion, pathId, fileId) in cu:
        dirName = cu.frombinary(dirName)
        baseName = cu.frombinary(baseName)
        if pathId == trove.CAPSULE_PATHID:
            isCapsule = 1
            contentAvailable = not excludeCapsules
        else:
            isCapsule = None
            contentAvailable = not hasCapsule

        fileObj = datamodel.FileReference(
                        path = os.path.join(dirName, baseName),
                        version = fileVersion,
                        pathId = md5ToString(cu.frombinary(pathId)),
                        fileId = sha1ToString(cu.frombinary(fileId)),
                        isCapsule = isCapsule,
                        contentAvailable = contentAvailable,
                        mkUrl = mkUrl, thisHost = thisHost)
        t.addFile(fileObj)

    cu.execute("""
        SELECT item, version, flavor, TroveTroves.includedId, Nodes.timeStamps
          FROM TroveTroves
            JOIN Instances ON (Instances.instanceId = TroveTroves.includedId)
            JOIN Nodes USING (itemId, versionId)
            JOIN Items USING (itemId)
            JOIN Versions ON (Versions.versionId = Instances.versionId)
            JOIN Flavors ON (Flavors.flavorId = Instances.flavorId)
            WHERE
                TroveTroves.instanceId = ? AND
                (TroveTroves.flags & %d) = 0
            ORDER BY item, version, flavor
    """ % schema.TROVE_TROVES_WEAKREF, instanceId)

    for (subName, subVersion, subFlavor, refInstanceId, subTS) in cu:
        subFlavor = str(deps.ThawFlavor(subFlavor))
        frzVer = versions.strToFrozen(subVersion,
                [ x for x in subTS.split(":") ])
        subV = versions.ThawVersion(frzVer)
        t.addReferencedTrove(subName, subV, subFlavor, mkUrl = mkUrl)

        # It would be far better to use file tags to identify these build
        # logs, but it's significantly slower as well because they're in
        # the file objects rather than the trove (and those file objects
        # could be stored on a different repository)
        if not subName.endswith(':debuginfo'):
            continue

        fileQuery(cu, refInstanceId, dirName = '/usr/src/debug/buildlogs')
        logHost = subV.getHost()
        for (dirName, baseName, fileVersion, pathId, fileId) in cu:
            if (dirName) != '/usr/src/debug/buildlogs':
                continue

            if baseName.endswith('-log.bz2'):
                t.setBuildLog(logHost, sha1ToString(fileId))
            elif baseName.endswith('-xml.bz2'):
                t.setXMLBuildLog(logHost, sha1ToString(fileId))

    return t
Exemple #15
0
    def formatFile(self,
                   pathId,
                   path,
                   fileId,
                   version,
                   fileObj=None,
                   prefix='',
                   indent=0):
        taglist = ''
        sha1 = ''
        id = ''
        flavor = ''

        dcfg = self.dcfg
        verbose = dcfg.isVerbose()

        if verbose and isinstance(fileObj, files.SymbolicLink):
            name = "%s -> %s" % (path, fileObj.target())
        else:
            name = path
        if dcfg.fileFlavors:
            if not fileObj.flavor().isEmpty():
                flavor = '[%s]' % fileObj.flavor()

        if dcfg.tags:
            tags = []
            if fileObj.tags:
                tags.extend(fileObj.tags)
            if fileObj.flags.isInitialContents():
                tags.append('initialContents')
            if fileObj.flags.isAutoSource():
                tags.append('autosource')
            if fileObj.flags.isConfig():
                tags.append('config')
            if fileObj.flags.isTransient():
                tags.append('transient')
            if tags:
                taglist = ' {' + ' '.join(tags) + '}'
        if dcfg.sha1s:
            if hasattr(fileObj, 'contents') and fileObj.contents:
                sha1 = sha1ToString(fileObj.contents.sha1()) + ' '
            else:
                sha1 = ' ' * 41

        if dcfg.ids and pathId:
            id = md5ToString(pathId) + ' ' + sha1ToString(fileId) + ', '
        if dcfg.fileVersions:
            if dcfg.useFullVersions():
                verStr = '    %s' % version
            elif dcfg.showLabels:
                verStr = '    %s/%s' % (version.branch().label(),
                                        version.trailingRevision())
            else:
                verStr = '    %s' % version.trailingRevision()
        else:
            verStr = ''

        spacer = '  ' * indent

        if fileObj:
            owner = fileObj.inode.owner()
            if owner[0] == '+':
                owner = owner[1:]
            group = fileObj.inode.group()
            if group[0] == '+':
                group = group[1:]

        if verbose:
            ln = "%s%s%s%s%s    1 %-8s %-8s %s %s %s%s%s%s" % \
              (spacer,
               prefix, id, sha1, fileObj.modeString(), owner,
               group, fileObj.sizeString(),
               fileObj.timeString(), name, flavor, taglist, verStr)
        else:
            ln = "%s%s%s%s%s%s%s" % (spacer, id, sha1, path, flavor, taglist,
                                     verStr)

        yield ln

        if dcfg.fileDeps:
            for ln in self.formatDeps(fileObj.provides(),
                                      fileObj.requires(),
                                      indent + 1,
                                      showEmpty=False):
                yield ln
Exemple #16
0
    def __init__(self,
                 repos,
                 cs,
                 fileHostFilter=[],
                 callback=None,
                 resetTimestamps=False,
                 allowIncomplete=False,
                 hidden=False,
                 mirror=False,
                 excludeCapsuleContents=False):

        self.repos = repos
        self.cs = cs
        self.invalidateRollbacksFlag = False

        newList = [x for x in cs.iterNewTroveList()]

        if resetTimestamps:
            # This depends intimiately on the versions cache. We don't
            # change the timestamps on each version, because the cache
            # ensures they are all a single underlying object. Slick,
            # but brittle?
            updated = {}

            for csTrove in newList:
                ver = csTrove.getNewVersion()
                if ver in updated:
                    pass
                else:
                    oldVer = ver.copy()
                    ver.trailingRevision().resetTimeStamp()
                    updated[oldVer] = ver

            del updated

        troveNo, configRestoreList, normalRestoreList = \
            self._createInstallTroveObjects(fileHostFilter = fileHostFilter,
                                            callback = callback,
                                            mirror = mirror, hidden = hidden,
                                            allowIncomplete = allowIncomplete,
                                            excludeCapsuleContents =
                                                excludeCapsuleContents)
        configRestoreList, normalRestoreList = \
            self._filterRestoreList(configRestoreList, normalRestoreList)

        # use a key to select data up to, but not including, the first
        # version.  We can't sort on version because we don't have timestamps
        configRestoreList.sort(key=lambda x: x[0:5])
        normalRestoreList.sort(key=lambda x: x[0:3])

        # config files are cached, so we don't have to worry about not
        # restoring the same fileId/pathId twice
        for (pathId, newFileId, sha1, oldfile, newFileId, oldVersion,
             oldFileId, restoreContents) in configRestoreList:
            if cs.configFileIsDiff(pathId, newFileId):
                (contType,
                 fileContents) = cs.getFileContents(pathId, newFileId)

                # the content for this file is in the form of a
                # diff, which we need to apply against the file in
                # the repository
                assert (oldVersion)

                try:
                    f = self.repos.getFileContents([(oldFileId, oldVersion,
                                                     oldfile)])[0].get()
                except KeyError:
                    raise errors.IntegrityError(
                        "Missing file contents for pathId %s, fileId %s" %
                        (sha1helper.md5ToString(pathId),
                         sha1helper.sha1ToString(fileId)))

                oldLines = f.readlines()
                f.close()
                del f
                diff = fileContents.get().readlines()
                (newLines, failedHunks) = patch.patch(oldLines, diff)
                fileContents = filecontents.FromString("".join(newLines))

                assert (not failedHunks)
            else:
                # config files are not always available compressed (due
                # to the config file cache)
                fileContents = filecontents.FromChangeSet(
                    cs, pathId, newFileId)

            self.addFileContents(sha1, fileContents, restoreContents, 1)

        ptrRestores = []
        ptrRefsAdded = {}
        lastRestore = None  # restore each pathId,fileId combo once
        while normalRestoreList:
            (pathId, fileId, sha1, restoreContents) = normalRestoreList.pop(0)
            if (pathId, fileId) == lastRestore:
                continue

            lastRestore = (pathId, fileId)

            try:
                (contType, fileContents) = cs.getFileContents(pathId,
                                                              fileId,
                                                              compressed=True)
            except KeyError:
                raise errors.IntegrityError(
                    "Missing file contents for pathId %s, fileId %s" %
                    (sha1helper.md5ToString(pathId),
                     sha1helper.sha1ToString(fileId)))
            if contType == changeset.ChangedFileTypes.ptr:
                ptrRestores.append(sha1)
                target = util.decompressString(fileContents.get().read())

                if util.tupleListBsearchInsert(
                        normalRestoreList,
                    (target[:16], target[16:], sha1, True), self.ptrCmp):
                    # Item was inserted. This creates a reference in the
                    # datastore; keep track of it to prevent a duplicate
                    # reference count.
                    ptrRefsAdded[sha1] = True

                continue

            assert (contType == changeset.ChangedFileTypes.file)
            self.addFileContents(sha1,
                                 fileContents,
                                 restoreContents,
                                 0,
                                 precompressed=True)

        for sha1 in ptrRestores:
            # Increment the reference count for items which were ptr's
            # to a different file.
            if sha1 in ptrRefsAdded:
                del ptrRefsAdded[sha1]
            else:
                self.addFileContents(sha1, None, False, 0)

        #del configRestoreList
        #del normalRestoreList

        for csTrove in newList:
            if csTrove.troveType() != trove.TROVE_TYPE_REMOVED:
                continue

            troveNo += 1

            if callback:
                callback.creatingDatabaseTransaction(troveNo, len(newList))

            self.markTroveRemoved(csTrove.getName(), csTrove.getNewVersion(),
                                  csTrove.getNewFlavor())

        for (troveName, version, flavor) in cs.getOldTroveList():
            trv = self.repos.getTrove(troveName, version, flavor)
            self.oldTrove(trv, None, troveName, version, flavor)
Exemple #17
0
    def __init__(self, repos, cs, fileHostFilter = [], callback = None,
                 resetTimestamps = False, allowIncomplete = False,
                 hidden = False, mirror = False,
                 excludeCapsuleContents = False):

        self.repos = repos
        self.cs = cs
        self.invalidateRollbacksFlag = False

        newList = [ x for x in cs.iterNewTroveList() ]

        if resetTimestamps:
            # This depends intimiately on the versions cache. We don't
            # change the timestamps on each version, because the cache
            # ensures they are all a single underlying object. Slick,
            # but brittle?
            updated = {}

            for csTrove in newList:
                ver = csTrove.getNewVersion()
                if ver in updated:
                    pass
                else:
                    oldVer = ver.copy()
                    ver.trailingRevision().resetTimeStamp()
                    updated[oldVer] = ver

            del updated

        troveNo, configRestoreList, normalRestoreList = \
            self._createInstallTroveObjects(fileHostFilter = fileHostFilter,
                                            callback = callback,
                                            mirror = mirror, hidden = hidden,
                                            allowIncomplete = allowIncomplete,
                                            excludeCapsuleContents =
                                                excludeCapsuleContents)
        configRestoreList, normalRestoreList = \
            self._filterRestoreList(configRestoreList, normalRestoreList)

        # use a key to select data up to, but not including, the first
        # version.  We can't sort on version because we don't have timestamps
        configRestoreList.sort(key=lambda x: x[0:5])
        normalRestoreList.sort(key=lambda x: x[0:3])

        # config files are cached, so we don't have to worry about not
        # restoring the same fileId/pathId twice
        for (pathId, newFileId, sha1, oldfile, newFileId,
             oldVersion, oldFileId, restoreContents) in configRestoreList:
            if cs.configFileIsDiff(pathId, newFileId):
                (contType, fileContents) = cs.getFileContents(pathId, newFileId)

                # the content for this file is in the form of a
                # diff, which we need to apply against the file in
                # the repository
                assert(oldVersion)

                try:
                    f = self.repos.getFileContents(
                                    [(oldFileId, oldVersion, oldfile)])[0].get()
                except KeyError:
                    raise errors.IntegrityError(
                        "Missing file contents for pathId %s, fileId %s" % (
                                        sha1helper.md5ToString(pathId),
                                        sha1helper.sha1ToString(fileId)))

                oldLines = f.readlines()
                f.close()
                del f
                diff = fileContents.get().readlines()
                (newLines, failedHunks) = patch.patch(oldLines,
                                                      diff)
                fileContents = filecontents.FromString(
                                                "".join(newLines))

                assert(not failedHunks)
            else:
                # config files are not always available compressed (due
                # to the config file cache)
                fileContents = filecontents.FromChangeSet(cs, pathId, newFileId)

            self.addFileContents(sha1, fileContents, restoreContents, 1)

        ptrRestores = []
        ptrRefsAdded = {}
        lastRestore = None         # restore each pathId,fileId combo once
        while normalRestoreList:
            (pathId, fileId, sha1, restoreContents) = normalRestoreList.pop(0)
            if (pathId, fileId) == lastRestore:
                continue

            lastRestore = (pathId, fileId)

            try:
                (contType, fileContents) = cs.getFileContents(pathId, fileId,
                                                              compressed = True)
            except KeyError:
                raise errors.IntegrityError(
                        "Missing file contents for pathId %s, fileId %s" % (
                                        sha1helper.md5ToString(pathId),
                                        sha1helper.sha1ToString(fileId)))
            if contType == changeset.ChangedFileTypes.ptr:
                ptrRestores.append(sha1)
                target = util.decompressString(fileContents.get().read())

                if util.tupleListBsearchInsert(normalRestoreList,
                                (target[:16], target[16:], sha1, True),
                                self.ptrCmp):
                    # Item was inserted. This creates a reference in the
                    # datastore; keep track of it to prevent a duplicate
                    # reference count.
                    ptrRefsAdded[sha1] = True

                continue

            assert(contType == changeset.ChangedFileTypes.file)
            self.addFileContents(sha1, fileContents, restoreContents, 0,
                                 precompressed = True)

        for sha1 in ptrRestores:
            # Increment the reference count for items which were ptr's
            # to a different file.
            if sha1 in ptrRefsAdded:
                del ptrRefsAdded[sha1]
            else:
                self.addFileContents(sha1, None, False, 0)

        #del configRestoreList
        #del normalRestoreList

        for csTrove in newList:
            if csTrove.troveType() != trove.TROVE_TYPE_REMOVED:
                continue

            troveNo += 1

            if callback:
                callback.creatingDatabaseTransaction(troveNo, len(newList))

            self.markTroveRemoved(csTrove.getName(), csTrove.getNewVersion(),
                                  csTrove.getNewFlavor())

        for (troveName, version, flavor) in cs.getOldTroveList():
            trv = self.repos.getTrove(troveName, version, flavor)
            self.oldTrove(trv, None, troveName, version, flavor)
Exemple #18
0
    def _createInstallTroveObjects(
        self,
        fileHostFilter=[],
        callback=None,
        hidden=False,
        mirror=False,
        allowIncomplete=False,
    ):
        # create the trove objects which need to be installed; the
        # file objects which map up with them are created later, but
        # we do need a map from pathId to the path and version of the
        # file we need, so build up a dictionary with that information

        configRestoreList = []
        normalRestoreList = []
        checkFilesList = []

        newList = [x for x in self.cs.iterNewTroveList()]
        repos = self.repos
        cs = self.cs

        oldTrovesNeeded = [
            x.getOldNameVersionFlavor() for x in newList if x.getOldVersion()
        ]
        oldTroveIter = repos.iterTroves(oldTrovesNeeded, hidden=True)

        troveNo = 0
        for csTrove in newList:
            if csTrove.troveType() == trove.TROVE_TYPE_REMOVED:
                # deal with these later on to ensure any changesets which
                # are relative to removed troves can be processed
                continue

            troveNo += 1

            if callback:
                callback.creatingDatabaseTransaction(troveNo, len(newList))

            newVersion = csTrove.getNewVersion()
            oldTroveVersion = csTrove.getOldVersion()
            oldTroveFlavor = csTrove.getOldFlavor()
            troveName = csTrove.getName()
            troveFlavor = csTrove.getNewFlavor()

            if repos.hasTrove(troveName, newVersion, troveFlavor):
                raise errors.CommitError, \
                       "version %s of %s already exists" % \
                        (newVersion.asString(), csTrove.getName())

            if oldTroveVersion:
                newTrove = oldTroveIter.next()
                assert (newTrove.getNameVersionFlavor() ==
                        csTrove.getOldNameVersionFlavor())
                self.oldTrove(newTrove, csTrove, troveName, oldTroveVersion,
                              oldTroveFlavor)

                oldCompatClass = newTrove.getCompatibilityClass()

                if csTrove.isRollbackFence(
                        oldCompatibilityClass=oldCompatClass, update=True):
                    self.invalidateRollbacks(set=True)
            else:
                newTrove = trove.Trove(csTrove.getName(),
                                       newVersion,
                                       troveFlavor,
                                       csTrove.getChangeLog(),
                                       setVersion=False)
                # FIXME: we reset the trove version
                # since in this case we need to use the fileMap returned
                # from applyChangeSet
                allowIncomplete = True

            newFileMap = newTrove.applyChangeSet(
                csTrove, needNewFileMap=True, allowIncomplete=allowIncomplete)
            if newTrove.troveInfo.incomplete():
                log.warning(
                    'trove %s has schema version %s, which contains'
                    ' information not handled by this client.  This'
                    ' version of Conary understands schema version %s.'
                    ' Dropping extra information.  Please upgrade conary.',
                    newTrove.getName(), newTrove.troveInfo.troveVersion(),
                    trove.TROVE_VERSION)

            self.checkTroveCompleteness(newTrove)

            self.checkTroveSignatures(newTrove, callback=callback)

            if oldTroveVersion is not None:
                troveInfo = self.addTrove(
                    (troveName, oldTroveVersion, oldTroveFlavor),
                    newTrove,
                    csTrove,
                    hidden=hidden)
            else:
                troveInfo = self.addTrove(None,
                                          newTrove,
                                          csTrove,
                                          hidden=hidden)

            checkFilesList += self._getCheckFilesList(csTrove,
                                                      troveInfo,
                                                      fileHostFilter,
                                                      configRestoreList,
                                                      normalRestoreList,
                                                      restoreContents=True)

            for (pathId, path, fileId, newVersion) in \
                            newTrove.iterFileList(members = True,
                                                  capsules = True):
                # handle files which haven't changed; we know which those
                # are because they're in the merged trove but they aren't
                # in the newFileMap
                if pathId in newFileMap:
                    continue

                self.addFileVersion(troveInfo,
                                    pathId,
                                    path,
                                    fileId,
                                    newVersion,
                                    withContents=True)

            filesNeeded = []
            for i, (pathId, path, fileId,
                    newVersion) in enumerate(csTrove.getChangedFileList()):
                tup = newFileMap[pathId]
                (oldPath, oldFileId, oldVersion) = tup[-3:]
                if path is None:
                    path = oldPath
                if fileId is None:
                    oldFileId = fileId
                if newVersion is None:
                    newVersion = oldVersion

                if (fileHostFilter
                        and newVersion.getHost() not in fileHostFilter):
                    fileStream = None
                elif (oldVersion == newVersion and oldFileId == fileId):
                    # the file didn't change between versions; we can just
                    # ignore it
                    fileStream = None
                else:
                    fileStream = cs.getFileChange(oldFileId, fileId)

                    if fileStream and fileStream[0] == "\x01":
                        if len(fileStream) != 2:
                            # This is awful, but this is how we say a file
                            # stream didn't change. Omitting it or at least ''
                            # would be nicer, but would break clients.
                            filesNeeded.append(
                                (i, (pathId, oldFileId, oldVersion)))
                            continue

                        fileStream = None

                # None is the file object
                self.addFileVersion(troveInfo,
                                    pathId,
                                    path,
                                    fileId,
                                    newVersion,
                                    fileStream=fileStream,
                                    withContents=True)

                if fileStream is not None:
                    self._handleContents(pathId,
                                         fileId,
                                         fileStream,
                                         configRestoreList,
                                         normalRestoreList,
                                         oldFileId=oldFileId,
                                         oldVersion=oldVersion,
                                         oldfile=None,
                                         restoreContents=True)

            oldFileObjects = list(
                repos.getFileVersions([x[1] for x in filesNeeded]))

            for i, (pathId, path, fileId,
                    newVersion) in enumerate(csTrove.getChangedFileList()):
                if not filesNeeded or filesNeeded[0][0] != i:
                    continue
                filesNeeded.pop(0)

                tup = newFileMap[pathId]
                (oldPath, oldFileId, oldVersion) = tup[-3:]
                if path is None:
                    path = oldPath
                if fileId is None:
                    oldFileId = fileId
                if newVersion is None:
                    newVersion = oldVersion

                restoreContents = True

                diff = cs.getFileChange(oldFileId, fileId)

                # stored as a diff (the file type is the same
                # and (for *repository* commits) the file
                # is in the same repository between versions
                oldfile = oldFileObjects.pop(0)
                fileObj = oldfile.copy()
                fileObj.twm(diff, oldfile)
                assert (fileObj.pathId() == pathId)
                fileStream = fileObj.freeze()

                if (not mirror) and (
                        fileObj.hasContents
                        and fileObj.contents.sha1() == oldfile.contents.sha1()
                        and not (fileObj.flags.isConfig()
                                 and not oldfile.flags.isConfig())):
                    # don't restore the contents here. we don't
                    # need them, and they may be relative to
                    # something from a different repository
                    restoreContents = False

                if fileObj and fileObj.fileId() != fileId:
                    raise trove.TroveIntegrityError(
                        csTrove.getName(), csTrove.getNewVersion(),
                        csTrove.getNewFlavor(),
                        "fileObj.fileId() != fileId in changeset")

                self.addFileVersion(troveInfo,
                                    pathId,
                                    path,
                                    fileId,
                                    newVersion,
                                    fileStream=fileStream,
                                    withContents=restoreContents)

                self._handleContents(pathId,
                                     fileId,
                                     fileStream,
                                     configRestoreList,
                                     normalRestoreList,
                                     oldFileId=oldFileId,
                                     oldVersion=oldVersion,
                                     oldfile=oldfile,
                                     restoreContents=restoreContents)

            del newFileMap
            self.addTroveDone(troveInfo, mirror=mirror)

        try:
            # we need to actualize this, not just get a generator
            list(repos.getFileVersions(checkFilesList))
        except errors.FileStreamMissing, e:
            info = [x for x in checkFilesList if x[1] == e.fileId]
            (pathId, fileId) = info[0][0:2]
            # Missing from the repo; raise exception
            raise errors.IntegrityError(
                "Incomplete changeset specified: missing pathId %s "
                "fileId %s" % (sha1helper.md5ToString(pathId),
                               sha1helper.sha1ToString(fileId)))
Exemple #19
0
def verifySignatures(f, validKeys = None):
    """
    Given an extended file, compute signatures
    """
    f.seek(0)
    h = readHeader(f)

    # Cheap test first: verify MD5 sig
    sigmd5 = h.get(SIG_MD5, None)
    if sigmd5 is not None:
        f.seek(0)
        readSignatureHeader(f)

        # verify md5 digest
        md5 = digestlib.md5()
        util.copyfileobj(f, NullWriter(), digest = md5)
        if md5.digest() != sigmd5:
            raise MD5SignatureError(
                "The MD5 digest fails to verify: expected %s, got %s" %
                    (sha1helper.md5ToString(sigmd5), md5.hexdigest()))

    # Don't bother if no gpg signature was present, or no valid keys were
    # presented
    if validKeys is None:
        return
    sigString = h.get(SIG_GPG, None)
    if sigString is None:
        return
    # Skip to immutable header region
    f.seek(0)
    readSignatureHeader(f)
    sig = openpgpfile.readSignature(sigString)

    keyId = sig.getSignerKeyId()
    matchingKeys = [ x for x in validKeys if x.hasKeyId(keyId) ]
    if not matchingKeys:
        raise PGPSignatureError("Signature generated with key %s does "
              "not match valid keys %s" %
              (keyId, ', '.join(x.getKeyId() for x in validKeys)))

    key = matchingKeys[0]

    # signature verification assumes a seekable stream and will seek to the
    # beginning; use a SeekableNestedFile
    size = h.getHeaderPlusPayloadSize()
    if size is None:
        pos = f.tell()
        f.seek(0, 2)
        size = f.tell()
        f.seek(pos, 0)
    snf = None
    if hasattr(f, 'pread'):
        extFile = f
    elif hasattr(f, 'name'):
        extFile = util.ExtendedFile(f.name, buffering = False)
    else:
        # worst case scenario, we slurp everything in memory
        extFile = util.ExtendedStringIO(f.read())
        snf = extFile
    if snf is None:
        snf = util.SeekableNestedFile(extFile, start = f.tell(), size = size)
    try:
        sig.verifyDocument(key.getCryptoKey(), snf)
    except openpgpfile.SignatureError:
        raise PGPSignatureError
Exemple #20
0
 def checkMd5(self, path, md5):
     d = md5String(open(self.rootDir + path).read())
     self.assertEquals(md5ToString(d), md5)
Exemple #21
0
def getTrove(cu, roleIds, name, version, flavor, mkUrl = None,
        thisHost = None, displayFlavor = None):

    def buildTupleList(tuples, name, mkUrl = mkUrl):
        l = getattr(datamodel.SingleTrove, name)()
        for troveInfo in sorted(tuples.iter()):
            l.append(name = troveInfo.name(), version = troveInfo.version(),
                     flavor = troveInfo.flavor(), mkUrl = mkUrl)

        return l

    def fileQuery(gfcu, filesInstanceId, dirName = None):
        # XXX restricing by dirName seems and obvious thing to do here,
        # but it actually slows things down??
        #
        # the distinct here is unfortunate, but conary repositories had
        # a bug for about a year which caused it to store duplicate paths
        # if a path was committed for the first time duplicate times in
        # a single commit job
        gfcu.execute("""
            SELECT DISTINCT dirName, basename, version, pathId, fileId
                FROM TroveFiles
                JOIN Versions USING (versionId)
                JOIN FileStreams ON (TroveFiles.streamId = FileStreams.streamId)
                JOIN FilePaths ON (TroveFiles.filePathId = FilePaths.filePathId)
                JOIN DirNames ON
                    FilePaths.dirNameId = DirNames.dirNameId
                JOIN Basenames ON (FilePaths.baseNameId = Basenames.baseNameId)
                WHERE TroveFiles.instanceId = ? ORDER BY dirName, basename
        """, filesInstanceId)

    cu.execute("""
        SELECT Instances.instanceId, Nodes.timeStamps FROM Instances
            JOIN Nodes USING (itemId, versionId)
            JOIN Items USING (itemId)
            JOIN Versions ON (Instances.versionId = Versions.versionId)
            JOIN Flavors ON (Instances.flavorId = Flavors.flavorId)
            JOIN UserGroupInstancesCache AS ugi
                ON (instances.instanceId = ugi.instanceId AND
                    ugi.userGroupId in (%s))
        WHERE
            item = ? AND version = ? AND flavor = ?
    """ % ",".join( str(x) for x in roleIds), name, version,
        deps.parseFlavor(flavor).freeze())

    l = [ (x[0], x[1]) for x in cu ]
    if not l:
        return None

    instanceId, timeStamps = l[0]
    frzVer = versions.strToFrozen(version, timeStamps.split(":"))
    verobj = versions.ThawVersion(frzVer)

    tupleLists = [ ( trove._TROVEINFO_TAG_BUILDDEPS, 'builddeps' ),
                   ( trove._TROVEINFO_TAG_POLICY_PROV, 'policyprovider' ),
                   ( trove._TROVEINFO_TAG_LOADEDTROVES, 'loadedtroves' ),
                   ( trove._TROVEINFO_TAG_COPIED_FROM, 'copiedfrom' ),
                   ( trove._TROVEINFO_TAG_DERIVEDFROM, 'derivedfrom' ) ]

    cu.execute("""
    SELECT infoType, data FROM TroveInfo WHERE instanceId = ? AND
        infoType IN (%s)
                """ % ",".join(str(x) for x in
                        [ trove._TROVEINFO_TAG_SOURCENAME,
                          trove._TROVEINFO_TAG_CLONEDFROM,
                          trove._TROVEINFO_TAG_CLONEDFROMLIST,
                          trove._TROVEINFO_TAG_BUILDTIME,
                          trove._TROVEINFO_TAG_SIZE,
                          trove._TROVEINFO_TAG_METADATA,
                          trove._TROVEINFO_TAG_CAPSULE,
                        ] + [ x[0] for x in tupleLists ]
                ), instanceId)

    troveInfo = {}
    for infoType, data in cu:
        data = cu.frombinary(data)
        infoClass = trove.TroveInfo.streamDict[infoType][1]
        troveInfo[infoType] = infoClass(data)

    kwargs = { 'name' : name,
               'version' : verobj,
               'flavor' : flavor }

    if displayFlavor is not None:
        kwargs['displayflavor'] = displayFlavor

    if trove._TROVEINFO_TAG_BUILDTIME in troveInfo:
        kwargs['buildtime'] = int(troveInfo[trove._TROVEINFO_TAG_BUILDTIME]())

    if trove._TROVEINFO_TAG_SOURCENAME in troveInfo:
        kwargs['source'] = (troveInfo[trove._TROVEINFO_TAG_SOURCENAME](),
            verobj.getSourceVersion(), '')

    if trove._TROVEINFO_TAG_SIZE in troveInfo:
        kwargs['size'] = troveInfo[trove._TROVEINFO_TAG_SIZE]()

    if trove._TROVEINFO_TAG_METADATA in troveInfo:
        md = troveInfo[trove._TROVEINFO_TAG_METADATA].get()
        kwargs['shortdesc'] = md['shortDesc']
        kwargs['longdesc'] = md['longDesc']

        if md['licenses']:
            kwargs['license'] = [ x for x in md['licenses' ]]
        if md['crypto']:
            kwargs['crypto'] = [ x for x in md['crypto'] ]

    for (tag, tagName) in tupleLists:
        if tag in troveInfo:
            kwargs[tagName] = buildTupleList(troveInfo[tag], tagName,
                                             mkUrl = mkUrl)

    t = datamodel.SingleTrove(mkUrl = mkUrl, thisHost = thisHost, **kwargs)

    if trove._TROVEINFO_TAG_CLONEDFROMLIST in troveInfo:
        clonedFromList = troveInfo[trove._TROVEINFO_TAG_CLONEDFROMLIST]
    elif (trove._TROVEINFO_TAG_CLONEDFROM in troveInfo):
        clonedFromList = [ troveInfo[trove._TROVEINFO_TAG_CLONEDFROM]() ]
    else:
        clonedFromList = []

    for ver in clonedFromList:
        t.addClonedFrom(name, ver, flavor, mkUrl = mkUrl)

    hasCapsule = False
    if trove._TROVEINFO_TAG_CAPSULE in troveInfo:
        if troveInfo[trove._TROVEINFO_TAG_CAPSULE].type():
            hasCapsule = True

    fileQuery(cu, instanceId)

    for (dirName, baseName, fileVersion, pathId, fileId) in cu:
        dirName = cu.frombinary(dirName)
        baseName = cu.frombinary(baseName)
        if pathId == trove.CAPSULE_PATHID:
            isCapsule = 1
            contentAvailable = True
        else:
            isCapsule = None
            contentAvailable = not hasCapsule

        fileObj = datamodel.FileReference(
                        path = os.path.join(dirName, baseName),
                        version = fileVersion,
                        pathId = md5ToString(cu.frombinary(pathId)),
                        fileId = sha1ToString(cu.frombinary(fileId)),
                        isCapsule = isCapsule,
                        contentAvailable = contentAvailable,
                        mkUrl = mkUrl, thisHost = thisHost)
        t.addFile(fileObj)

    cu.execute("""
        SELECT item, version, flavor, TroveTroves.includedId, Nodes.timeStamps
          FROM TroveTroves
            JOIN Instances ON (Instances.instanceId = TroveTroves.includedId)
            JOIN Nodes USING (itemId, versionId)
            JOIN Items USING (itemId)
            JOIN Versions ON (Versions.versionId = Instances.versionId)
            JOIN Flavors ON (Flavors.flavorId = Instances.flavorId)
            WHERE
                TroveTroves.instanceId = ? AND
                (TroveTroves.flags & %d) = 0
            ORDER BY item, version, flavor
    """ % schema.TROVE_TROVES_WEAKREF, instanceId)

    for (subName, subVersion, subFlavor, refInstanceId, subTS) in cu:
        subFlavor = str(deps.ThawFlavor(subFlavor))
        frzVer = versions.strToFrozen(subVersion,
                [ x for x in subTS.split(":") ])
        subV = versions.ThawVersion(frzVer)
        t.addReferencedTrove(subName, subV, subFlavor, mkUrl = mkUrl)

        # It would be far better to use file tags to identify these build
        # logs, but it's significantly slower as well because they're in
        # the file objects rather than the trove (and those file objects
        # could be stored on a different repository)
        if not subName.endswith(':debuginfo'):
            continue

        fileQuery(cu, refInstanceId, dirName = '/usr/src/debug/buildlogs')
        logHost = subV.getHost()
        for (dirName, baseName, fileVersion, pathId, fileId) in cu:
            if (dirName) != '/usr/src/debug/buildlogs':
                continue

            if baseName.endswith('-log.bz2'):
                t.setBuildLog(logHost, sha1ToString(fileId))
            elif baseName.endswith('-xml.bz2'):
                t.setXMLBuildLog(logHost, sha1ToString(fileId))

    return t