def _write_index(self, fp, index): index_offset = fp.tell() compressed = compress(dumps(index)) write_int8_str(fp, compressed) fp.seek(len(self.MAGIC)) write_int8(fp, index_offset) assert fp.tell() == len(self.MAGIC) + 8
def read_write_int8(self): s = BytesIO() for x in (0, 1, 2**60): s.seek(0) write_int8(s, x) s.seek(0) assert x == read_int8(s)
def generate_shelf(klass, file, items): """(File, [(str, str)]) This returns a generator that writes a new Shelf into file, iterating once through the given items. The use of an iterator makes it possible to build a new Shelf incrementally. """ file.seek_end() if not file.tell() == 0: raise ValueError("Expected %s to be empty." % file) write(file, klass.prefix) if not items: # Just write an empty transaction. write_int8(file, 0) # Write an empty index array. offset_map = OffsetMap(file) else: # Write a transaction here with the given items. transaction_start = file.tell() # Write a placeholder for the length. write_int8(file, 0) # Loop over the items, writing their records. # Keep track of max_key and max_offset. max_key = 0 max_offset = 0 n = 0 for name, value in items: max_key = max(max_key, str_to_int8(name)) max_offset = max(max_offset, file.tell()) write_int8(file, len(name) + len(value)) write(file, name) write(file, value) n += 1 yield n transaction_end = file.tell() # Write the correct transaction length. file.seek(transaction_start) write_int8(file, transaction_end - transaction_start - 8) # Write the empty array with the calculated dimensions. file.seek(transaction_end) for step in OffsetMap.generate(file, max_key, max_offset): yield step offset_map = OffsetMap(file) # Now read through the records and record the offsets in the array. file.seek(transaction_start + 8) while file.tell() < transaction_end: position = file.tell() record_length = read_int8(file) name = read(file, 8) k = str_to_int8(name) offset_map[k] = position file.seek(position + 8 + record_length) n -= 1 yield n for index in offset_map.gen_stitch(): yield index
def store(self, name_value_sequence): """([(str, str)]) -> [(str, int|None, int)] Record all of the items in the sequence. Return a list of triples, each giving a name, an old position (or None if this is a new name), and a new position. """ self.file.seek_end() start = self.file.tell() write_int8(self.file, 0) result = [] index = {} try: for name, value in name_value_sequence: new_position = self.file.tell() old_position = self.get_position(name) index[name] = new_position result.append((name, old_position, new_position)) write_int8(self.file, len(name) + len(value)) write(self.file, name) write(self.file, value) except: # Revert before raising. self.file.seek(start) self.file.truncate() raise end = self.file.tell() self.file.seek(start) write_int8(self.file, end - start - 8) self.file.seek(end) self.memory_index.update(index) return result
def _write_header(self, fp): fp.seek(0, 2) assert fp.tell() == 0 write(fp, self.MAGIC) write_int8(fp, 0) # index offset