def read_write_int8(self): s = BytesIO() for x in (0, 1, 2**60): s.seek(0) write_int8(s, x) s.seek(0) assert x == read_int8(s)
def __init__(self, file=None, items=None, repair=False, readonly=False): """(File:str:None, [(str:str)], boolean) """ if file is None: file = File() assert not readonly assert not repair elif not hasattr(file, 'seek'): file = File(file, readonly=readonly) if not readonly: file.obtain_lock() file.seek(0, 2) # seek end if file.tell() == 0: # The file is empty. for result in self.generate_shelf(file=file, items=items or []): pass else: assert items is None # The file is not empty. assert self.has_format(file) self.file = file self.file.seek(len(self.prefix)) n = read_int8(self.file) # bytes in first transaction self.file.seek(self.file.tell() + n) self.offset_map = OffsetMap(self.file) # Initialize the memory index. self.memory_index = {} while True: transaction_offsets = read_transaction_offsets( self.file, repair=repair) if transaction_offsets is None: break self.memory_index.update(transaction_offsets) self.file.seek_end() self.unused_name_generator = None
def __init__(self, file=None, items=None, repair=False, readonly=False): """(File:str:None, [(str:str)], boolean) """ if file is None: file = File() assert not readonly assert not repair elif not hasattr(file, 'seek'): file = File(file, readonly=readonly) if not readonly: file.obtain_lock() file.seek(0, 2) # seek end if file.tell() == 0: # The file is empty. for result in self.generate_shelf(file=file, items=items or []): pass else: assert items is None # The file is not empty. assert self.has_format(file) self.file = file self.file.seek(len(self.prefix)) n = read_int8(self.file) # bytes in first transaction self.file.seek(self.file.tell() + n) self.offset_map = OffsetMap(self.file) # Initialize the memory index. self.memory_index = {} while True: transaction_offsets = read_transaction_offsets(self.file, repair=repair) if transaction_offsets is None: break self.memory_index.update(transaction_offsets) self.file.seek_end() self.unused_name_generator = None
def generate_shelf(klass, file, items): """(File, [(str, str)]) This returns a generator that writes a new Shelf into file, iterating once through the given items. The use of an iterator makes it possible to build a new Shelf incrementally. """ file.seek_end() if not file.tell() == 0: raise ValueError("Expected %s to be empty." % file) write(file, klass.prefix) if not items: # Just write an empty transaction. write_int8(file, 0) # Write an empty index array. offset_map = OffsetMap(file) else: # Write a transaction here with the given items. transaction_start = file.tell() # Write a placeholder for the length. write_int8(file, 0) # Loop over the items, writing their records. # Keep track of max_key and max_offset. max_key = 0 max_offset = 0 n = 0 for name, value in items: max_key = max(max_key, str_to_int8(name)) max_offset = max(max_offset, file.tell()) write_int8(file, len(name) + len(value)) write(file, name) write(file, value) n += 1 yield n transaction_end = file.tell() # Write the correct transaction length. file.seek(transaction_start) write_int8(file, transaction_end - transaction_start - 8) # Write the empty array with the calculated dimensions. file.seek(transaction_end) for step in OffsetMap.generate(file, max_key, max_offset): yield step offset_map = OffsetMap(file) # Now read through the records and record the offsets in the array. file.seek(transaction_start + 8) while file.tell() < transaction_end: position = file.tell() record_length = read_int8(file) name = read(file, 8) k = str_to_int8(name) offset_map[k] = position file.seek(position + 8 + record_length) n -= 1 yield n for index in offset_map.gen_stitch(): yield index
def read_transaction_offsets(file, repair=False): """ Read the offsets of one (Shelf-format) transaction in the file. If repair is True and the file ends in something other than a well formed transaction, the file is truncated to remove the ugly ending. This ugliness might occur if the power fails in the middle of writing a transaction. """ transaction_start = transaction_end = position = file.tell() try: transaction_length = read_int8(file) transaction_end = transaction_start + 8 + transaction_length transaction_offsets = {} while file.tell() < transaction_end: position = file.tell() record_length = read_int8(file) identifier = read(file, 8) transaction_offsets[identifier] = position file.seek(position + 8 + record_length) if file.tell() != transaction_end: raise ShortRead return transaction_offsets except ShortRead: position = file.tell() if position > transaction_start: if repair: file.seek(transaction_start) file.truncate() else: e = sys.exc_info()[1] e.args = repr( dict(transaction_start=transaction_start, transaction_end=transaction_end, position=position)) raise return None
def read_transaction_offsets(file, repair=False): """ Read the offsets of one (Shelf-format) transaction in the file. If repair is True and the file ends in something other than a well formed transaction, the file is truncated to remove the ugly ending. This ugliness might occur if the power fails in the middle of writing a transaction. """ transaction_start = transaction_end = position = file.tell() try: transaction_length = read_int8(file) transaction_end = transaction_start + 8 + transaction_length transaction_offsets = {} while file.tell() < transaction_end: position = file.tell() record_length = read_int8(file) identifier = read(file, 8) transaction_offsets[identifier] = position file.seek(position + 8 + record_length) if file.tell() != transaction_end: raise ShortRead return transaction_offsets except ShortRead: position = file.tell() if position > transaction_start: if repair: file.seek(transaction_start) file.truncate() else: e = sys.exc_info()[1] e.args = repr(dict( transaction_start=transaction_start, transaction_end = transaction_end, position=position)) raise return None
def _build_index(self, repair): self.fp.seek(0) if read(self.fp, len(self.MAGIC)) != self.MAGIC: raise IOError("invalid storage (missing magic in %r)" % self.fp) index_offset = read_int8(self.fp) assert index_offset > 0 self.fp.seek(index_offset) tmp_index = loads(decompress(read_int8_str(self.fp))) self.index = {} def oid_as_bytes(oid): if isinstance(oid, byte_string): return oid else: return oid.encode('latin1') for tmp_oid in tmp_index: self.index[oid_as_bytes(tmp_oid)] = tmp_index[tmp_oid] del tmp_index while 1: # Read one transaction each time here. oids = {} transaction_offset = self.fp.tell() try: while 1: object_record_offset = self.fp.tell() record = self._read_block() if len(record) == 0: break # normal termination if len(record) < 12: raise ShortRead("Bad record size") oid = record[0:8] oids[oid] = object_record_offset # We've reached the normal end of a transaction. self.index.update(oids) oids.clear() except (ValueError, IOError): if self.fp.tell() > transaction_offset: if not repair: raise # The transaction was malformed. Attempt repair. self.fp.seek(transaction_offset) self.fp.truncate() break