class Storage(): def __init__(self, db_path): self._current_memtable = Memtable() self._sstable_group = SSTableGroup.from_directory(db_path) pass def put(self, key, value): self._current_memtable.put(key, value) # TODO: Check memtable size # If it's smaller than THRESHOLD: # Add key to memtable # Otherwise: # Write memtable to the new SSTable on disk(table + table index) # Load that SSTable's index into memory # Create new memtable # Add key to the new memtable def delete(self, key): # TODO: Same as put self._current_memtable.delete(key) def get(self, key): result = self._current_memtable.get(key) if result is not None: return result.get_value() result = self._sstable_group.get(key) if result is not None: return result.get_value() return None
def test_deletion(): memtable = Memtable() memtable.put("key1", "value1") memtable.put("key1", "value1-changed") memtable.delete("key1") assert memtable.get('key1').is_tombstone() == True
def test_get_value_memtable_created(test_folder): memtable = Memtable() memtable.put('1', '1') memtable.put('2', '2') memtable.put('3', '3') memtable.put('2', 'two') memtable.delete('3') sstable = SSTable.from_memtable(memtable, test_folder, 'test') assert Item('1', '1', False) == sstable.get('1') assert Item('2', 'two', False) == sstable.get('2') assert Item('3', None, True) == sstable.get('3') assert None == sstable.get('4')
def test_binary_output(): memtable = Memtable() memtable.put(1, 1) memtable.put(2, 2) memtable.put(3, 3) memtable.delete(3) binary_sstable, binary_index, index = memtable.convert_to_binary_and_index() assert binary_sstable == b'\x01\x00\x00\x001\x01\x00\x00\x00\x001\x01\x00\x00\x002\x01\x00\x00\x00\x002\x01\x00\x00\x003\x00\x00\x00\x00\x80' assert binary_index == b'\x93\xac\x16\x94\x00\x00\x00\x00\x17\xe2)\x01\x0b\x00\x00\x00\xb4\xa1\xc7\x0f\x16\x00\x00\x00' assert index.get_item_offset('1') == 0 assert index.get_item_offset('2') == 11 assert index.get_item_offset('3') == 22
def test_retrieval(): memtable = Memtable() memtable.put("key1", "value1") memtable.put("key2", "value2") memtable.put("key2", "value2-changed") assert memtable.get('key1') == Item('key1', "value1", False) assert memtable.get('key2') == Item('key2', "value2-changed", False)
def test_creation_from_memtable(test_folder): memtable = Memtable() memtable.put(1, 1) SSTable.from_memtable(memtable, test_folder, 'test') expected_table_path = os.path.join(test_folder, "test.table") expected_index_path = os.path.join(test_folder, "test.index") assert os.path.isfile(expected_table_path) assert os.path.isfile(expected_index_path) with open(expected_table_path, 'rb') as f: table = f.read() assert table == b'\x01\x00\x00\x001\x01\x00\x00\x00\x001' with open(expected_index_path, 'rb') as f: index = f.read() assert index == b'\x93\xac\x16\x94\x00\x00\x00\x00'
def test_loading_from_folder(test_folder): now = datetime.now() hour_after = datetime.now() + timedelta(hours=1) hour_before = datetime.now() - timedelta(hours=1) day_before = datetime.now() - timedelta(days=1) mm1 = Memtable() mm1.put(1, 1) mm1.delete(3) mm2 = Memtable() mm2.put(2, 2) mm3 = Memtable() mm3.put(1, "old value") mm3.put(3, "should be deleted") mm3.put(4, 4) SSTable.from_memtable(mm1, test_folder, now.strftime(FILE_NAME_DATE_FORMAT)) SSTable.from_memtable(mm2, test_folder, hour_before.strftime(FILE_NAME_DATE_FORMAT)) SSTable.from_memtable(mm3, test_folder, day_before.strftime(FILE_NAME_DATE_FORMAT)) storage = Storage(test_folder) storage.put(4, "in memtable") assert "1" == storage.get(1) assert "2" == storage.get(2) assert None == storage.get(3) assert "in memtable" == storage.get(4)
def test_loading_from_folder(test_folder): now = datetime.now() hour_after = datetime.now() + timedelta(hours=1) hour_before = datetime.now() - timedelta(hours=1) day_before = datetime.now() - timedelta(days=1) mm1 = Memtable() mm1.put(1, 1) mm1.delete(3) mm2 = Memtable() mm2.put(2, 2) mm3 = Memtable() mm3.put(1, "old value") mm3.put(3, "should be deleted") mm3.put(4, 4) SSTable.from_memtable(mm1, test_folder, now.strftime(FILE_NAME_DATE_FORMAT)) SSTable.from_memtable(mm2, test_folder, hour_before.strftime(FILE_NAME_DATE_FORMAT)) SSTable.from_memtable(mm3, test_folder, day_before.strftime(FILE_NAME_DATE_FORMAT)) sstable_group = SSTableGroup.from_directory(test_folder) assert Item(1, 1, False) == sstable_group.get(1) assert Item(2, 2, False) == sstable_group.get(2) assert Item(3, None, True) == sstable_group.get(3) assert Item(4, 4, False) == sstable_group.get(4) mm4 = Memtable() mm4.put(5, 5) mm4.delete(1) new_sstable = SSTable.from_memtable( mm4, test_folder, hour_after.strftime(FILE_NAME_DATE_FORMAT)) sstable_group.add_table(new_sstable) assert Item(1, None, True) == sstable_group.get(1) assert Item(5, 5, False) == sstable_group.get(5)
def __init__(self, db_path): self._current_memtable = Memtable() self._sstable_group = SSTableGroup.from_directory(db_path) pass
def test_sorted_items_iterator(): memtable = Memtable() memtable.put("key2", "value2") memtable.put("key1", "value1") memtable.put("4", "4") memtable.put("key3", "value3") memtable.put("33", "33") memtable.put("5", "5") output = '' for key, _ in memtable.get_sorted_items(): output += key + ":" assert output == "33:4:5:key1:key2:key3:"
def test_size(): key1_size = sys.getsizeof("key1") + sys.getsizeof("value1") key1_changed_size = sys.getsizeof("key1") + sys.getsizeof("value1-changed") key2_size = sys.getsizeof("key2") + sys.getsizeof("value2") key3_size = sys.getsizeof("key3") + sys.getsizeof("value3") key3_deleted_size = sys.getsizeof("key3") + sys.getsizeof(None) memtable = Memtable() memtable.put("key1", "value1") memtable.put("key2", "value2") assert memtable.get_size() == key1_size + key2_size memtable.put("key1", "value1-changed") memtable.put("key3", "value3") assert memtable.get_size() == key1_changed_size + key2_size + key3_size memtable.delete("key3") assert memtable.get_size() == key1_changed_size + key2_size + key3_deleted_size