def write_header(self, fh): fh.seek(self.start) fh.write(self.MAGIC_BYTES) fh.write(int_to_bytes(self.size, self.SIZE_SIZE)) fh.write(int_to_bytes(self.next, self.SIZE_NEXT)) fh.write(int_to_bytes(self.prev, self.SIZE_PREV)) fh.write(int_to_bytes(self.next_empty, self.SIZE_NEXT_EMPTY))
def __init__(self, fh, block_size=1024, initialize=False): self.header_structure = BlockStructure(fh, block_size=block_size, initialize=initialize, fill=int_to_bytes(-1, 4)) self.header = BlockStructureOrderedDataIO(fh, self.header_structure) self.super_blocks = self.read_structure(fh, self.header_structure)
def add_block(self, fh, block_size, after=None, fill=None): if fill is None: fill = int_to_bytes(-1, 4) if after is None: after = self.blocks[-1] prior_block = after next_block = Block.read_block(fh, after.next) if after.next != -1 else None prior_block_pos = after.start next_block_pos = after.next pos = fh.seek(0, os.SEEK_END) block = Block(pos, block_size, next_block_pos, prior_block_pos) pos = self.write_new_block(fh, block, fill=fill) prior_block.next = pos prior_block.write_header(fh) block.write_header(fh) if next_block is not None: next_block.prev = block.start next_block.write_header(fh) self.blocks.append(block) fh.flush() return block
def init_structure(self, fh, position, block_size, fill=None): if fill is None: fill = int_to_bytes(-1, 4) block = Block(position, block_size, -1, -1) self.write_new_block(fh, block, fill) return [block]
def add_structure(self, fh, block_size, fill=None): pos = fh.seek(0, os.SEEK_END) block_structure = BlockStructure(fh, position=pos, initialize=True, block_size=block_size, fill=fill) self.header.write(int_to_bytes(pos)) fh.flush() return block_structure
def test_bad_magic(self): bs = BlockStructure(self.f, block_size=16, initialize=True) orig = Block.MAGIC_BYTES Block.MAGIC_BYTES = int_to_bytes(123424736, 4) bs.add_block(self.f, 16, fill=b'\x00\x00\x00\x10') Block.MAGIC_BYTES = orig self.reopen_file() with pytest.raises(PyDBInternalError) as ex: BlockStructure(self.f) expected = "Internal Error. Possibly corrupt database. Not a block at start position: 0." assert ex.value.message == expected
def encode_metadata(self, io, pos=0): io.seek(pos) io.write(self.MAGIC_BYTES) io.write(int_to_bytes(len(self.class_name))) io.write(string_to_bytes(self.class_name)) io.write(int_to_bytes(len(self.column_names))) for x in self.column_names: io.write(int_to_bytes(len(x))) io.write(string_to_bytes(x)) io.write(int_to_bytes(len(self.primary_key))) io.write(string_to_bytes(self.primary_key)) io.write(int_to_bytes(len(self.unique_keys))) for x in self.unique_keys: io.write(int_to_bytes(len(x))) io.write(string_to_bytes(x)) io.write(int_to_bytes(self.row_count))
def test_int_to_bytes(): assert int_to_bytes(2989135076627009422, size=8) == b'){\x8a\x83\xdc\x03O\x8e'
def write_header(self): self.io.seek(0) self.io.write(int_to_bytes(self.count, self.SIZE_HEADER))
def encode_header(self): res = int_to_bytes(self.null, self.SIZE_NULL) res += int_to_bytes(self.size, self.SIZE_SIZE) return res
def encode_value(self, val): return int_to_bytes(val, self.size)
class TableMetadata(object): MAGIC_VALUE = 2123427274 MAGIC_BYTES = int_to_bytes(MAGIC_VALUE, 4) INT_BYTE_LEN = 4 COLUMN_BYTE_LEN = 128 CLASS_BYTE_LEN = 512 def __init__(self, cls): self.class_name = get_qualified_name(cls) self.columns = cls._get_columns() self.column_names = [x[0] for x in self.columns] self.primary_key = ([x for x, y in self.columns if y.primary_key] + [None])[0] self.unique_keys = [x for x, y in self.columns if y.unique] self.row_count = 0 self.check_valid() def check_valid(self): pass def decode_metadata(self, io): reader = SafeReader(io) magic = reader.next_int(self.INT_BYTE_LEN) if magic != self.MAGIC_VALUE: raise PyDBInternalError("Invalid metatadata.") class_name = reader.next_string(self.CLASS_BYTE_LEN) size_cols = reader.next_int(self.INT_BYTE_LEN) col_names = [] for _ in range(size_cols): col_name = reader.next_string(self.COLUMN_BYTE_LEN) col_names.append(col_name) primary_key = reader.next_string(self.COLUMN_BYTE_LEN) or None unique_keys_count = reader.next_int(self.INT_BYTE_LEN) unique_keys = [] for _ in range(unique_keys_count): unique_key = reader.next_string(self.COLUMN_BYTE_LEN) unique_keys.append(unique_key) row_count = reader.next_int(self.INT_BYTE_LEN) self.check_compatibility(class_name, col_names, row_count, primary_key, unique_keys) #Updating attributes that might have changed. self.row_count = row_count def encode_metadata(self, io, pos=0): io.seek(pos) io.write(self.MAGIC_BYTES) io.write(int_to_bytes(len(self.class_name))) io.write(string_to_bytes(self.class_name)) io.write(int_to_bytes(len(self.column_names))) for x in self.column_names: io.write(int_to_bytes(len(x))) io.write(string_to_bytes(x)) io.write(int_to_bytes(len(self.primary_key))) io.write(string_to_bytes(self.primary_key)) io.write(int_to_bytes(len(self.unique_keys))) for x in self.unique_keys: io.write(int_to_bytes(len(x))) io.write(string_to_bytes(x)) io.write(int_to_bytes(self.row_count)) def check_compatibility(self, class_name, col_names, row_count, primary_key, unique_keys): if class_name != self.class_name: raise PyDBMetadataError("Class name differs.") if col_names != self.column_names: raise PyDBMetadataError("Column names differ.") if primary_key != self.primary_key: raise PyDBMetadataError("Primary key differs.") if unique_keys != self.unique_keys: raise PyDBMetadataError("Unique keys differ.")
class Block(object): """ | MAGIC | SIZE | NEXT | PREV | NEXT_EMPTY | DATA... | """ MAGIC_VALUE = -1208913507 MAGIC_BYTES = int_to_bytes(MAGIC_VALUE, 4) SIZE_MAGIC = 4 SIZE_SIZE = 4 SIZE_NEXT = 4 SIZE_PREV = 4 SIZE_NEXT_EMPTY = 4 SIZE_HEADER = SIZE_MAGIC + SIZE_SIZE + SIZE_NEXT + SIZE_PREV + SIZE_NEXT_EMPTY def __init__(self, start, size, nxt, prev, next_empty=0): self.start = start self.size = size self.next = nxt self.prev = prev self.next_empty = next_empty def get_header_size(self): return self.SIZE_HEADER def write_header(self, fh): fh.seek(self.start) fh.write(self.MAGIC_BYTES) fh.write(int_to_bytes(self.size, self.SIZE_SIZE)) fh.write(int_to_bytes(self.next, self.SIZE_NEXT)) fh.write(int_to_bytes(self.prev, self.SIZE_PREV)) fh.write(int_to_bytes(self.next_empty, self.SIZE_NEXT_EMPTY)) def fill_data(self, fh, data): if self.size % len(data) != 0: raise PyDBInternalError("Can't fill data. Not aligned.") fh.seek(self.start + self.get_header_size()) for _ in range(self.size // len(data)): fh.write(data) def write_data(self, fh, position, data): """ Writes data at the `position`, but doesn't update self.next_empty. """ if position < 0 or position + len(data) > self.size: raise PyDBInternalError("Invalid position to write in.") fh.seek(self.start + self.get_header_size() + position) fh.write(data) def __repr__(self): return ("Block(start={s.start}, size={s.size}, nxt={s.next}, " "prev={s.prev}, next_empty={s.next_empty})").format(s=self) @classmethod def read_block(cls, fh, start): fh.seek(start) magic = fh.read(cls.SIZE_MAGIC) if magic != cls.MAGIC_BYTES: raise PyDBInternalError( "Not a block at start position: {}.".format(start)) size = bytes_to_int(fh.read(cls.SIZE_SIZE)) nxt = bytes_to_int(fh.read(cls.SIZE_NEXT)) prev = bytes_to_int(fh.read(cls.SIZE_PREV)) next_empty = bytes_to_int(fh.read(cls.SIZE_NEXT_EMPTY)) return cls(start, size, nxt, prev, next_empty=next_empty)