예제 #1
0
    def _read_info(cls, stream, gen, schema, version):
        # Read the schema and segments from the TOC file

        # Read the pickled schema bytes
        pick = stream.read_string()
        # If the user passed a schema, use it, otherwise unpickle the schema
        # we just read
        if not schema:
            schema = pickle.loads(pick)

        # Read the list of segments
        numsegments = stream.read_varint()
        segments = []
        for _ in xrange(numsegments):
            segtype = stream.read_string()  # @UnusedVariable
            segment = pickle.loads(stream.read_string())
            segments.append(segment)

        return schema, segments
예제 #2
0
    def read(cls, storage, indexname, gen=None, schema=None):
        if gen is None:
            gen = cls._latest_generation(storage, indexname)
            if gen < 0:
                raise EmptyIndexError("Index %r does not exist in %r" %
                                      (indexname, storage))

        # Read the content of this index from the .toc file.
        tocfilename = cls._filename(indexname, gen)
        stream = storage.open_file(tocfilename)

        def check_size(name, target):
            sz = stream.read_varint()
            if sz != target:
                raise IndexError("Index was created on different architecture:"
                                 " saved %s = %s, this computer = %s" %
                                 (name, sz, target))

        check_size("int", _INT_SIZE)
        check_size("long", _LONG_SIZE)
        check_size("float", _FLOAT_SIZE)

        if not stream.read_int() == -12345:
            raise IndexError("Number misread: byte order problem")

        version = stream.read_int()
        release = (stream.read_varint(), stream.read_varint(),
                   stream.read_varint())

        if version != _CURRENT_TOC_VERSION:
            if version in toc_loaders:
                loader = toc_loaders[version]
                schema, segments = loader(stream, gen, schema, version)
            else:
                raise IndexVersionError("Can't read format %s" % version,
                                        version)
        else:
            # If the user supplied a schema object with the constructor, don't
            # load the pickled schema from the saved index.
            if schema:
                stream.skip_string()
            else:
                schema = pickle.loads(stream.read_string())
            schema = ensure_schema(schema)

            # Generation
            index_gen = stream.read_int()
            assert gen == index_gen

            _ = stream.read_int()  # Unused
            segments = stream.read_pickle()

        stream.close()
        return cls(schema, segments, gen, version=version, release=release)
    def read(cls, storage, indexname, gen=None, schema=None):
        if gen is None:
            gen = cls._latest_generation(storage, indexname)
            if gen < 0:
                raise EmptyIndexError("Index %r does not exist in %r"
                                      % (indexname, storage))

        # Read the content of this index from the .toc file.
        tocfilename = cls._filename(indexname, gen)
        stream = storage.open_file(tocfilename)

        def check_size(name, target):
            sz = stream.read_varint()
            if sz != target:
                raise IndexError("Index was created on different architecture:"
                                 " saved %s = %s, this computer = %s"
                                 % (name, sz, target))

        check_size("int", _INT_SIZE)
        check_size("long", _LONG_SIZE)
        check_size("float", _FLOAT_SIZE)

        if not stream.read_int() == -12345:
            raise IndexError("Number misread: byte order problem")

        version = stream.read_int()
        release = (stream.read_varint(), stream.read_varint(),
                   stream.read_varint())

        if version != _CURRENT_TOC_VERSION:
            if version in toc_loaders:
                loader = toc_loaders[version]
                schema, segments = loader(stream, gen, schema, version)
            else:
                raise IndexVersionError("Can't read format %s" % version,
                                        version)
        else:
            # If the user supplied a schema object with the constructor, don't
            # load the pickled schema from the saved index.
            if schema:
                stream.skip_string()
            else:
                schema = pickle.loads(stream.read_string())
            schema = ensure_schema(schema)

            # Generation
            index_gen = stream.read_int()
            assert gen == index_gen

            _ = stream.read_int()  # Unused
            segments = stream.read_pickle()

        stream.close()
        return cls(schema, segments, gen, version=version, release=release)