class Storage():
def __init__(self, db_path):
self._current_memtable = Memtable()
self._sstable_group = SSTableGroup.from_directory(db_path)
pass
def put(self, key, value):
self._current_memtable.put(key, value)
# TODO: Check memtable size
# If it's smaller than THRESHOLD:
# Add key to memtable
# Otherwise:
# Write memtable to the new SSTable on disk(table + table index)
# Load that SSTable's index into memory
# Create new memtable
# Add key to the new memtable
def delete(self, key):
# TODO: Same as put
self._current_memtable.delete(key)
def get(self, key):
result = self._current_memtable.get(key)
if result is not None:
return result.get_value()
result = self._sstable_group.get(key)
if result is not None:
return result.get_value()
return None
def test_deletion():
memtable = Memtable()
memtable.put("key1", "value1")
memtable.put("key1", "value1-changed")
memtable.delete("key1")
assert memtable.get('key1').is_tombstone() == True
def test_get_value_memtable_created(test_folder):
memtable = Memtable()
memtable.put('1', '1')
memtable.put('2', '2')
memtable.put('3', '3')
memtable.put('2', 'two')
memtable.delete('3')
sstable = SSTable.from_memtable(memtable, test_folder, 'test')
assert Item('1', '1', False) == sstable.get('1')
assert Item('2', 'two', False) == sstable.get('2')
assert Item('3', None, True) == sstable.get('3')
assert None == sstable.get('4')
def test_binary_output():
memtable = Memtable()
memtable.put(1, 1)
memtable.put(2, 2)
memtable.put(3, 3)
memtable.delete(3)
binary_sstable, binary_index, index = memtable.convert_to_binary_and_index()
assert binary_sstable == b'\x01\x00\x00\x001\x01\x00\x00\x00\x001\x01\x00\x00\x002\x01\x00\x00\x00\x002\x01\x00\x00\x003\x00\x00\x00\x00\x80'
assert binary_index == b'\x93\xac\x16\x94\x00\x00\x00\x00\x17\xe2)\x01\x0b\x00\x00\x00\xb4\xa1\xc7\x0f\x16\x00\x00\x00'
assert index.get_item_offset('1') == 0
assert index.get_item_offset('2') == 11
assert index.get_item_offset('3') == 22
def test_retrieval():
memtable = Memtable()
memtable.put("key1", "value1")
memtable.put("key2", "value2")
memtable.put("key2", "value2-changed")
assert memtable.get('key1') == Item('key1', "value1", False)
assert memtable.get('key2') == Item('key2', "value2-changed", False)
def test_creation_from_memtable(test_folder):
memtable = Memtable()
memtable.put(1, 1)
SSTable.from_memtable(memtable, test_folder, 'test')
expected_table_path = os.path.join(test_folder, "test.table")
expected_index_path = os.path.join(test_folder, "test.index")
assert os.path.isfile(expected_table_path)
assert os.path.isfile(expected_index_path)
with open(expected_table_path, 'rb') as f:
table = f.read()
assert table == b'\x01\x00\x00\x001\x01\x00\x00\x00\x001'
with open(expected_index_path, 'rb') as f:
index = f.read()
assert index == b'\x93\xac\x16\x94\x00\x00\x00\x00'
def test_loading_from_folder(test_folder):
now = datetime.now()
hour_after = datetime.now() + timedelta(hours=1)
hour_before = datetime.now() - timedelta(hours=1)
day_before = datetime.now() - timedelta(days=1)
mm1 = Memtable()
mm1.put(1, 1)
mm1.delete(3)
mm2 = Memtable()
mm2.put(2, 2)
mm3 = Memtable()
mm3.put(1, "old value")
mm3.put(3, "should be deleted")
mm3.put(4, 4)
SSTable.from_memtable(mm1, test_folder,
now.strftime(FILE_NAME_DATE_FORMAT))
SSTable.from_memtable(mm2, test_folder,
hour_before.strftime(FILE_NAME_DATE_FORMAT))
SSTable.from_memtable(mm3, test_folder,
day_before.strftime(FILE_NAME_DATE_FORMAT))
storage = Storage(test_folder)
storage.put(4, "in memtable")
assert "1" == storage.get(1)
assert "2" == storage.get(2)
assert None == storage.get(3)
assert "in memtable" == storage.get(4)
def test_loading_from_folder(test_folder):
now = datetime.now()
hour_after = datetime.now() + timedelta(hours=1)
hour_before = datetime.now() - timedelta(hours=1)
day_before = datetime.now() - timedelta(days=1)
mm1 = Memtable()
mm1.put(1, 1)
mm1.delete(3)
mm2 = Memtable()
mm2.put(2, 2)
mm3 = Memtable()
mm3.put(1, "old value")
mm3.put(3, "should be deleted")
mm3.put(4, 4)
SSTable.from_memtable(mm1, test_folder,
now.strftime(FILE_NAME_DATE_FORMAT))
SSTable.from_memtable(mm2, test_folder,
hour_before.strftime(FILE_NAME_DATE_FORMAT))
SSTable.from_memtable(mm3, test_folder,
day_before.strftime(FILE_NAME_DATE_FORMAT))
sstable_group = SSTableGroup.from_directory(test_folder)
assert Item(1, 1, False) == sstable_group.get(1)
assert Item(2, 2, False) == sstable_group.get(2)
assert Item(3, None, True) == sstable_group.get(3)
assert Item(4, 4, False) == sstable_group.get(4)
mm4 = Memtable()
mm4.put(5, 5)
mm4.delete(1)
new_sstable = SSTable.from_memtable(
mm4, test_folder, hour_after.strftime(FILE_NAME_DATE_FORMAT))
sstable_group.add_table(new_sstable)
assert Item(1, None, True) == sstable_group.get(1)
assert Item(5, 5, False) == sstable_group.get(5)
def __init__(self, db_path):
self._current_memtable = Memtable()
self._sstable_group = SSTableGroup.from_directory(db_path)
pass
def test_sorted_items_iterator():
memtable = Memtable()
memtable.put("key2", "value2")
memtable.put("key1", "value1")
memtable.put("4", "4")
memtable.put("key3", "value3")
memtable.put("33", "33")
memtable.put("5", "5")
output = ''
for key, _ in memtable.get_sorted_items():
output += key + ":"
assert output == "33:4:5:key1:key2:key3:"
def test_size():
key1_size = sys.getsizeof("key1") + sys.getsizeof("value1")
key1_changed_size = sys.getsizeof("key1") + sys.getsizeof("value1-changed")
key2_size = sys.getsizeof("key2") + sys.getsizeof("value2")
key3_size = sys.getsizeof("key3") + sys.getsizeof("value3")
key3_deleted_size = sys.getsizeof("key3") + sys.getsizeof(None)
memtable = Memtable()
memtable.put("key1", "value1")
memtable.put("key2", "value2")
assert memtable.get_size() == key1_size + key2_size
memtable.put("key1", "value1-changed")
memtable.put("key3", "value3")
assert memtable.get_size() == key1_changed_size + key2_size + key3_size
memtable.delete("key3")
assert memtable.get_size() == key1_changed_size + key2_size + key3_deleted_size