def write_header(self, fh):
fh.seek(self.start)
fh.write(self.MAGIC_BYTES)
fh.write(int_to_bytes(self.size, self.SIZE_SIZE))
fh.write(int_to_bytes(self.next, self.SIZE_NEXT))
fh.write(int_to_bytes(self.prev, self.SIZE_PREV))
fh.write(int_to_bytes(self.next_empty, self.SIZE_NEXT_EMPTY))
def __init__(self, fh, block_size=1024, initialize=False):
self.header_structure = BlockStructure(fh,
block_size=block_size,
initialize=initialize,
fill=int_to_bytes(-1, 4))
self.header = BlockStructureOrderedDataIO(fh, self.header_structure)
self.super_blocks = self.read_structure(fh, self.header_structure)
def add_block(self, fh, block_size, after=None, fill=None):
if fill is None:
fill = int_to_bytes(-1, 4)
if after is None:
after = self.blocks[-1]
prior_block = after
next_block = Block.read_block(fh,
after.next) if after.next != -1 else None
prior_block_pos = after.start
next_block_pos = after.next
pos = fh.seek(0, os.SEEK_END)
block = Block(pos, block_size, next_block_pos, prior_block_pos)
pos = self.write_new_block(fh, block, fill=fill)
prior_block.next = pos
prior_block.write_header(fh)
block.write_header(fh)
if next_block is not None:
next_block.prev = block.start
next_block.write_header(fh)
self.blocks.append(block)
fh.flush()
return block
def init_structure(self, fh, position, block_size, fill=None):
if fill is None:
fill = int_to_bytes(-1, 4)
block = Block(position, block_size, -1, -1)
self.write_new_block(fh, block, fill)
return [block]
def add_structure(self, fh, block_size, fill=None):
pos = fh.seek(0, os.SEEK_END)
block_structure = BlockStructure(fh,
position=pos,
initialize=True,
block_size=block_size,
fill=fill)
self.header.write(int_to_bytes(pos))
fh.flush()
return block_structure
def test_bad_magic(self):
bs = BlockStructure(self.f, block_size=16, initialize=True)
orig = Block.MAGIC_BYTES
Block.MAGIC_BYTES = int_to_bytes(123424736, 4)
bs.add_block(self.f, 16, fill=b'\x00\x00\x00\x10')
Block.MAGIC_BYTES = orig
self.reopen_file()
with pytest.raises(PyDBInternalError) as ex:
BlockStructure(self.f)
expected = "Internal Error. Possibly corrupt database. Not a block at start position: 0."
assert ex.value.message == expected
def encode_metadata(self, io, pos=0):
io.seek(pos)
io.write(self.MAGIC_BYTES)
io.write(int_to_bytes(len(self.class_name)))
io.write(string_to_bytes(self.class_name))
io.write(int_to_bytes(len(self.column_names)))
for x in self.column_names:
io.write(int_to_bytes(len(x)))
io.write(string_to_bytes(x))
io.write(int_to_bytes(len(self.primary_key)))
io.write(string_to_bytes(self.primary_key))
io.write(int_to_bytes(len(self.unique_keys)))
for x in self.unique_keys:
io.write(int_to_bytes(len(x)))
io.write(string_to_bytes(x))
io.write(int_to_bytes(self.row_count))
def test_int_to_bytes():
assert int_to_bytes(2989135076627009422,
size=8) == b'){\x8a\x83\xdc\x03O\x8e'
def write_header(self):
self.io.seek(0)
self.io.write(int_to_bytes(self.count, self.SIZE_HEADER))
def encode_header(self):
res = int_to_bytes(self.null, self.SIZE_NULL)
res += int_to_bytes(self.size, self.SIZE_SIZE)
return res
def encode_value(self, val):
return int_to_bytes(val, self.size)
class TableMetadata(object):
MAGIC_VALUE = 2123427274
MAGIC_BYTES = int_to_bytes(MAGIC_VALUE, 4)
INT_BYTE_LEN = 4
COLUMN_BYTE_LEN = 128
CLASS_BYTE_LEN = 512
def __init__(self, cls):
self.class_name = get_qualified_name(cls)
self.columns = cls._get_columns()
self.column_names = [x[0] for x in self.columns]
self.primary_key = ([x for x, y in self.columns if y.primary_key] +
[None])[0]
self.unique_keys = [x for x, y in self.columns if y.unique]
self.row_count = 0
self.check_valid()
def check_valid(self):
pass
def decode_metadata(self, io):
reader = SafeReader(io)
magic = reader.next_int(self.INT_BYTE_LEN)
if magic != self.MAGIC_VALUE:
raise PyDBInternalError("Invalid metatadata.")
class_name = reader.next_string(self.CLASS_BYTE_LEN)
size_cols = reader.next_int(self.INT_BYTE_LEN)
col_names = []
for _ in range(size_cols):
col_name = reader.next_string(self.COLUMN_BYTE_LEN)
col_names.append(col_name)
primary_key = reader.next_string(self.COLUMN_BYTE_LEN) or None
unique_keys_count = reader.next_int(self.INT_BYTE_LEN)
unique_keys = []
for _ in range(unique_keys_count):
unique_key = reader.next_string(self.COLUMN_BYTE_LEN)
unique_keys.append(unique_key)
row_count = reader.next_int(self.INT_BYTE_LEN)
self.check_compatibility(class_name, col_names, row_count, primary_key,
unique_keys)
#Updating attributes that might have changed.
self.row_count = row_count
def encode_metadata(self, io, pos=0):
io.seek(pos)
io.write(self.MAGIC_BYTES)
io.write(int_to_bytes(len(self.class_name)))
io.write(string_to_bytes(self.class_name))
io.write(int_to_bytes(len(self.column_names)))
for x in self.column_names:
io.write(int_to_bytes(len(x)))
io.write(string_to_bytes(x))
io.write(int_to_bytes(len(self.primary_key)))
io.write(string_to_bytes(self.primary_key))
io.write(int_to_bytes(len(self.unique_keys)))
for x in self.unique_keys:
io.write(int_to_bytes(len(x)))
io.write(string_to_bytes(x))
io.write(int_to_bytes(self.row_count))
def check_compatibility(self, class_name, col_names, row_count,
primary_key, unique_keys):
if class_name != self.class_name:
raise PyDBMetadataError("Class name differs.")
if col_names != self.column_names:
raise PyDBMetadataError("Column names differ.")
if primary_key != self.primary_key:
raise PyDBMetadataError("Primary key differs.")
if unique_keys != self.unique_keys:
raise PyDBMetadataError("Unique keys differ.")
class Block(object):
"""
| MAGIC | SIZE | NEXT | PREV | NEXT_EMPTY | DATA... |
"""
MAGIC_VALUE = -1208913507
MAGIC_BYTES = int_to_bytes(MAGIC_VALUE, 4)
SIZE_MAGIC = 4
SIZE_SIZE = 4
SIZE_NEXT = 4
SIZE_PREV = 4
SIZE_NEXT_EMPTY = 4
SIZE_HEADER = SIZE_MAGIC + SIZE_SIZE + SIZE_NEXT + SIZE_PREV + SIZE_NEXT_EMPTY
def __init__(self, start, size, nxt, prev, next_empty=0):
self.start = start
self.size = size
self.next = nxt
self.prev = prev
self.next_empty = next_empty
def get_header_size(self):
return self.SIZE_HEADER
def write_header(self, fh):
fh.seek(self.start)
fh.write(self.MAGIC_BYTES)
fh.write(int_to_bytes(self.size, self.SIZE_SIZE))
fh.write(int_to_bytes(self.next, self.SIZE_NEXT))
fh.write(int_to_bytes(self.prev, self.SIZE_PREV))
fh.write(int_to_bytes(self.next_empty, self.SIZE_NEXT_EMPTY))
def fill_data(self, fh, data):
if self.size % len(data) != 0:
raise PyDBInternalError("Can't fill data. Not aligned.")
fh.seek(self.start + self.get_header_size())
for _ in range(self.size // len(data)):
fh.write(data)
def write_data(self, fh, position, data):
"""
Writes data at the `position`, but doesn't update self.next_empty.
"""
if position < 0 or position + len(data) > self.size:
raise PyDBInternalError("Invalid position to write in.")
fh.seek(self.start + self.get_header_size() + position)
fh.write(data)
def __repr__(self):
return ("Block(start={s.start}, size={s.size}, nxt={s.next}, "
"prev={s.prev}, next_empty={s.next_empty})").format(s=self)
@classmethod
def read_block(cls, fh, start):
fh.seek(start)
magic = fh.read(cls.SIZE_MAGIC)
if magic != cls.MAGIC_BYTES:
raise PyDBInternalError(
"Not a block at start position: {}.".format(start))
size = bytes_to_int(fh.read(cls.SIZE_SIZE))
nxt = bytes_to_int(fh.read(cls.SIZE_NEXT))
prev = bytes_to_int(fh.read(cls.SIZE_PREV))
next_empty = bytes_to_int(fh.read(cls.SIZE_NEXT_EMPTY))
return cls(start, size, nxt, prev, next_empty=next_empty)