def _write_index(self, fp, index):
index_offset = fp.tell()
compressed = compress(dumps(index))
write_int8_str(fp, compressed)
fp.seek(len(self.MAGIC))
write_int8(fp, index_offset)
assert fp.tell() == len(self.MAGIC) + 8
def read_write_int8(self):
s = BytesIO()
for x in (0, 1, 2**60):
s.seek(0)
write_int8(s, x)
s.seek(0)
assert x == read_int8(s)
def generate_shelf(klass, file, items):
"""(File, [(str, str)])
This returns a generator that writes a new Shelf into file,
iterating once through the given items.
The use of an iterator makes it possible to build a new Shelf
incrementally.
"""
file.seek_end()
if not file.tell() == 0:
raise ValueError("Expected %s to be empty." % file)
write(file, klass.prefix)
if not items:
# Just write an empty transaction.
write_int8(file, 0)
# Write an empty index array.
offset_map = OffsetMap(file)
else:
# Write a transaction here with the given items.
transaction_start = file.tell()
# Write a placeholder for the length.
write_int8(file, 0)
# Loop over the items, writing their records.
# Keep track of max_key and max_offset.
max_key = 0
max_offset = 0
n = 0
for name, value in items:
max_key = max(max_key, str_to_int8(name))
max_offset = max(max_offset, file.tell())
write_int8(file, len(name) + len(value))
write(file, name)
write(file, value)
n += 1
yield n
transaction_end = file.tell()
# Write the correct transaction length.
file.seek(transaction_start)
write_int8(file, transaction_end - transaction_start - 8)
# Write the empty array with the calculated dimensions.
file.seek(transaction_end)
for step in OffsetMap.generate(file, max_key, max_offset):
yield step
offset_map = OffsetMap(file)
# Now read through the records and record the offsets in the array.
file.seek(transaction_start + 8)
while file.tell() < transaction_end:
position = file.tell()
record_length = read_int8(file)
name = read(file, 8)
k = str_to_int8(name)
offset_map[k] = position
file.seek(position + 8 + record_length)
n -= 1
yield n
for index in offset_map.gen_stitch():
yield index
def generate_shelf(klass, file, items):
"""(File, [(str, str)])
This returns a generator that writes a new Shelf into file,
iterating once through the given items.
The use of an iterator makes it possible to build a new Shelf
incrementally.
"""
file.seek_end()
if not file.tell() == 0:
raise ValueError("Expected %s to be empty." % file)
write(file, klass.prefix)
if not items:
# Just write an empty transaction.
write_int8(file, 0)
# Write an empty index array.
offset_map = OffsetMap(file)
else:
# Write a transaction here with the given items.
transaction_start = file.tell()
# Write a placeholder for the length.
write_int8(file, 0)
# Loop over the items, writing their records.
# Keep track of max_key and max_offset.
max_key = 0
max_offset = 0
n = 0
for name, value in items:
max_key = max(max_key, str_to_int8(name))
max_offset = max(max_offset, file.tell())
write_int8(file, len(name) + len(value))
write(file, name)
write(file, value)
n += 1
yield n
transaction_end = file.tell()
# Write the correct transaction length.
file.seek(transaction_start)
write_int8(file, transaction_end - transaction_start - 8)
# Write the empty array with the calculated dimensions.
file.seek(transaction_end)
for step in OffsetMap.generate(file, max_key, max_offset):
yield step
offset_map = OffsetMap(file)
# Now read through the records and record the offsets in the array.
file.seek(transaction_start + 8)
while file.tell() < transaction_end:
position = file.tell()
record_length = read_int8(file)
name = read(file, 8)
k = str_to_int8(name)
offset_map[k] = position
file.seek(position + 8 + record_length)
n -= 1
yield n
for index in offset_map.gen_stitch():
yield index
def store(self, name_value_sequence):
"""([(str, str)]) -> [(str, int|None, int)]
Record all of the items in the sequence.
Return a list of triples, each giving a name, an old position (or None
if this is a new name), and a new position.
"""
self.file.seek_end()
start = self.file.tell()
write_int8(self.file, 0)
result = []
index = {}
try:
for name, value in name_value_sequence:
new_position = self.file.tell()
old_position = self.get_position(name)
index[name] = new_position
result.append((name, old_position, new_position))
write_int8(self.file, len(name) + len(value))
write(self.file, name)
write(self.file, value)
except:
# Revert before raising.
self.file.seek(start)
self.file.truncate()
raise
end = self.file.tell()
self.file.seek(start)
write_int8(self.file, end - start - 8)
self.file.seek(end)
self.memory_index.update(index)
return result
def store(self, name_value_sequence):
"""([(str, str)]) -> [(str, int|None, int)]
Record all of the items in the sequence.
Return a list of triples, each giving a name, an old position (or None
if this is a new name), and a new position.
"""
self.file.seek_end()
start = self.file.tell()
write_int8(self.file, 0)
result = []
index = {}
try:
for name, value in name_value_sequence:
new_position = self.file.tell()
old_position = self.get_position(name)
index[name] = new_position
result.append((name, old_position, new_position))
write_int8(self.file, len(name) + len(value))
write(self.file, name)
write(self.file, value)
except:
# Revert before raising.
self.file.seek(start)
self.file.truncate()
raise
end = self.file.tell()
self.file.seek(start)
write_int8(self.file, end - start - 8)
self.file.seek(end)
self.memory_index.update(index)
return result
def _write_header(self, fp):
fp.seek(0, 2)
assert fp.tell() == 0
write(fp, self.MAGIC)
write_int8(fp, 0) # index offset
