def read_write_int8(self):
s = BytesIO()
for x in (0, 1, 2**60):
s.seek(0)
write_int8(s, x)
s.seek(0)
assert x == read_int8(s)
def __init__(self, file=None, items=None, repair=False, readonly=False):
"""(File:str:None, [(str:str)], boolean)
"""
if file is None:
file = File()
assert not readonly
assert not repair
elif not hasattr(file, 'seek'):
file = File(file, readonly=readonly)
if not readonly:
file.obtain_lock()
file.seek(0, 2) # seek end
if file.tell() == 0:
# The file is empty.
for result in self.generate_shelf(file=file, items=items or []):
pass
else:
assert items is None
# The file is not empty.
assert self.has_format(file)
self.file = file
self.file.seek(len(self.prefix))
n = read_int8(self.file) # bytes in first transaction
self.file.seek(self.file.tell() + n)
self.offset_map = OffsetMap(self.file)
# Initialize the memory index.
self.memory_index = {}
while True:
transaction_offsets = read_transaction_offsets(
self.file, repair=repair)
if transaction_offsets is None:
break
self.memory_index.update(transaction_offsets)
self.file.seek_end()
self.unused_name_generator = None
def __init__(self, file=None, items=None, repair=False, readonly=False):
"""(File:str:None, [(str:str)], boolean)
"""
if file is None:
file = File()
assert not readonly
assert not repair
elif not hasattr(file, 'seek'):
file = File(file, readonly=readonly)
if not readonly:
file.obtain_lock()
file.seek(0, 2) # seek end
if file.tell() == 0:
# The file is empty.
for result in self.generate_shelf(file=file, items=items or []):
pass
else:
assert items is None
# The file is not empty.
assert self.has_format(file)
self.file = file
self.file.seek(len(self.prefix))
n = read_int8(self.file) # bytes in first transaction
self.file.seek(self.file.tell() + n)
self.offset_map = OffsetMap(self.file)
# Initialize the memory index.
self.memory_index = {}
while True:
transaction_offsets = read_transaction_offsets(self.file,
repair=repair)
if transaction_offsets is None:
break
self.memory_index.update(transaction_offsets)
self.file.seek_end()
self.unused_name_generator = None
def generate_shelf(klass, file, items):
"""(File, [(str, str)])
This returns a generator that writes a new Shelf into file,
iterating once through the given items.
The use of an iterator makes it possible to build a new Shelf
incrementally.
"""
file.seek_end()
if not file.tell() == 0:
raise ValueError("Expected %s to be empty." % file)
write(file, klass.prefix)
if not items:
# Just write an empty transaction.
write_int8(file, 0)
# Write an empty index array.
offset_map = OffsetMap(file)
else:
# Write a transaction here with the given items.
transaction_start = file.tell()
# Write a placeholder for the length.
write_int8(file, 0)
# Loop over the items, writing their records.
# Keep track of max_key and max_offset.
max_key = 0
max_offset = 0
n = 0
for name, value in items:
max_key = max(max_key, str_to_int8(name))
max_offset = max(max_offset, file.tell())
write_int8(file, len(name) + len(value))
write(file, name)
write(file, value)
n += 1
yield n
transaction_end = file.tell()
# Write the correct transaction length.
file.seek(transaction_start)
write_int8(file, transaction_end - transaction_start - 8)
# Write the empty array with the calculated dimensions.
file.seek(transaction_end)
for step in OffsetMap.generate(file, max_key, max_offset):
yield step
offset_map = OffsetMap(file)
# Now read through the records and record the offsets in the array.
file.seek(transaction_start + 8)
while file.tell() < transaction_end:
position = file.tell()
record_length = read_int8(file)
name = read(file, 8)
k = str_to_int8(name)
offset_map[k] = position
file.seek(position + 8 + record_length)
n -= 1
yield n
for index in offset_map.gen_stitch():
yield index
def generate_shelf(klass, file, items):
"""(File, [(str, str)])
This returns a generator that writes a new Shelf into file,
iterating once through the given items.
The use of an iterator makes it possible to build a new Shelf
incrementally.
"""
file.seek_end()
if not file.tell() == 0:
raise ValueError("Expected %s to be empty." % file)
write(file, klass.prefix)
if not items:
# Just write an empty transaction.
write_int8(file, 0)
# Write an empty index array.
offset_map = OffsetMap(file)
else:
# Write a transaction here with the given items.
transaction_start = file.tell()
# Write a placeholder for the length.
write_int8(file, 0)
# Loop over the items, writing their records.
# Keep track of max_key and max_offset.
max_key = 0
max_offset = 0
n = 0
for name, value in items:
max_key = max(max_key, str_to_int8(name))
max_offset = max(max_offset, file.tell())
write_int8(file, len(name) + len(value))
write(file, name)
write(file, value)
n += 1
yield n
transaction_end = file.tell()
# Write the correct transaction length.
file.seek(transaction_start)
write_int8(file, transaction_end - transaction_start - 8)
# Write the empty array with the calculated dimensions.
file.seek(transaction_end)
for step in OffsetMap.generate(file, max_key, max_offset):
yield step
offset_map = OffsetMap(file)
# Now read through the records and record the offsets in the array.
file.seek(transaction_start + 8)
while file.tell() < transaction_end:
position = file.tell()
record_length = read_int8(file)
name = read(file, 8)
k = str_to_int8(name)
offset_map[k] = position
file.seek(position + 8 + record_length)
n -= 1
yield n
for index in offset_map.gen_stitch():
yield index
def read_transaction_offsets(file, repair=False):
"""
Read the offsets of one (Shelf-format) transaction in the file.
If repair is True and the file ends in something other than a well
formed transaction, the file is truncated to remove the ugly
ending. This ugliness might occur if the power fails in the middle
of writing a transaction.
"""
transaction_start = transaction_end = position = file.tell()
try:
transaction_length = read_int8(file)
transaction_end = transaction_start + 8 + transaction_length
transaction_offsets = {}
while file.tell() < transaction_end:
position = file.tell()
record_length = read_int8(file)
identifier = read(file, 8)
transaction_offsets[identifier] = position
file.seek(position + 8 + record_length)
if file.tell() != transaction_end:
raise ShortRead
return transaction_offsets
except ShortRead:
position = file.tell()
if position > transaction_start:
if repair:
file.seek(transaction_start)
file.truncate()
else:
e = sys.exc_info()[1]
e.args = repr(
dict(transaction_start=transaction_start,
transaction_end=transaction_end,
position=position))
raise
return None
def read_transaction_offsets(file, repair=False):
"""
Read the offsets of one (Shelf-format) transaction in the file.
If repair is True and the file ends in something other than a well
formed transaction, the file is truncated to remove the ugly
ending. This ugliness might occur if the power fails in the middle
of writing a transaction.
"""
transaction_start = transaction_end = position = file.tell()
try:
transaction_length = read_int8(file)
transaction_end = transaction_start + 8 + transaction_length
transaction_offsets = {}
while file.tell() < transaction_end:
position = file.tell()
record_length = read_int8(file)
identifier = read(file, 8)
transaction_offsets[identifier] = position
file.seek(position + 8 + record_length)
if file.tell() != transaction_end:
raise ShortRead
return transaction_offsets
except ShortRead:
position = file.tell()
if position > transaction_start:
if repair:
file.seek(transaction_start)
file.truncate()
else:
e = sys.exc_info()[1]
e.args = repr(dict(
transaction_start=transaction_start,
transaction_end = transaction_end,
position=position))
raise
return None
def _build_index(self, repair):
self.fp.seek(0)
if read(self.fp, len(self.MAGIC)) != self.MAGIC:
raise IOError("invalid storage (missing magic in %r)" % self.fp)
index_offset = read_int8(self.fp)
assert index_offset > 0
self.fp.seek(index_offset)
tmp_index = loads(decompress(read_int8_str(self.fp)))
self.index = {}
def oid_as_bytes(oid):
if isinstance(oid, byte_string):
return oid
else:
return oid.encode('latin1')
for tmp_oid in tmp_index:
self.index[oid_as_bytes(tmp_oid)] = tmp_index[tmp_oid]
del tmp_index
while 1:
# Read one transaction each time here.
oids = {}
transaction_offset = self.fp.tell()
try:
while 1:
object_record_offset = self.fp.tell()
record = self._read_block()
if len(record) == 0:
break # normal termination
if len(record) < 12:
raise ShortRead("Bad record size")
oid = record[0:8]
oids[oid] = object_record_offset
# We've reached the normal end of a transaction.
self.index.update(oids)
oids.clear()
except (ValueError, IOError):
if self.fp.tell() > transaction_offset:
if not repair:
raise
# The transaction was malformed. Attempt repair.
self.fp.seek(transaction_offset)
self.fp.truncate()
break
