def _read_info(cls, stream, gen, schema, version):
# Read the schema and segments from the TOC file
# Read the pickled schema bytes
pick = stream.read_string()
# If the user passed a schema, use it, otherwise unpickle the schema
# we just read
if not schema:
schema = pickle.loads(pick)
# Read the list of segments
numsegments = stream.read_varint()
segments = []
for _ in xrange(numsegments):
segtype = stream.read_string() # @UnusedVariable
segment = pickle.loads(stream.read_string())
segments.append(segment)
return schema, segments
def read(cls, storage, indexname, gen=None, schema=None):
if gen is None:
gen = cls._latest_generation(storage, indexname)
if gen < 0:
raise EmptyIndexError("Index %r does not exist in %r" %
(indexname, storage))
# Read the content of this index from the .toc file.
tocfilename = cls._filename(indexname, gen)
stream = storage.open_file(tocfilename)
def check_size(name, target):
sz = stream.read_varint()
if sz != target:
raise IndexError("Index was created on different architecture:"
" saved %s = %s, this computer = %s" %
(name, sz, target))
check_size("int", _INT_SIZE)
check_size("long", _LONG_SIZE)
check_size("float", _FLOAT_SIZE)
if not stream.read_int() == -12345:
raise IndexError("Number misread: byte order problem")
version = stream.read_int()
release = (stream.read_varint(), stream.read_varint(),
stream.read_varint())
if version != _CURRENT_TOC_VERSION:
if version in toc_loaders:
loader = toc_loaders[version]
schema, segments = loader(stream, gen, schema, version)
else:
raise IndexVersionError("Can't read format %s" % version,
version)
else:
# If the user supplied a schema object with the constructor, don't
# load the pickled schema from the saved index.
if schema:
stream.skip_string()
else:
schema = pickle.loads(stream.read_string())
schema = ensure_schema(schema)
# Generation
index_gen = stream.read_int()
assert gen == index_gen
_ = stream.read_int() # Unused
segments = stream.read_pickle()
stream.close()
return cls(schema, segments, gen, version=version, release=release)
def read(cls, storage, indexname, gen=None, schema=None):
if gen is None:
gen = cls._latest_generation(storage, indexname)
if gen < 0:
raise EmptyIndexError("Index %r does not exist in %r"
% (indexname, storage))
# Read the content of this index from the .toc file.
tocfilename = cls._filename(indexname, gen)
stream = storage.open_file(tocfilename)
def check_size(name, target):
sz = stream.read_varint()
if sz != target:
raise IndexError("Index was created on different architecture:"
" saved %s = %s, this computer = %s"
% (name, sz, target))
check_size("int", _INT_SIZE)
check_size("long", _LONG_SIZE)
check_size("float", _FLOAT_SIZE)
if not stream.read_int() == -12345:
raise IndexError("Number misread: byte order problem")
version = stream.read_int()
release = (stream.read_varint(), stream.read_varint(),
stream.read_varint())
if version != _CURRENT_TOC_VERSION:
if version in toc_loaders:
loader = toc_loaders[version]
schema, segments = loader(stream, gen, schema, version)
else:
raise IndexVersionError("Can't read format %s" % version,
version)
else:
# If the user supplied a schema object with the constructor, don't
# load the pickled schema from the saved index.
if schema:
stream.skip_string()
else:
schema = pickle.loads(stream.read_string())
schema = ensure_schema(schema)
# Generation
index_gen = stream.read_int()
assert gen == index_gen
_ = stream.read_int() # Unused
segments = stream.read_pickle()
stream.close()
return cls(schema, segments, gen, version=version, release=release)
