def test_read_write_multi_blocks():
pool_folder, conf_path, block_file = _get_common_file_paths()
_clean_up()
pool_file_1 = os.path.join(pool_folder, "pool_0")
pool_file_2 = os.path.join(pool_folder, "pool_1")
def check_content(filepath, target_string):
with open(filepath, "rb") as f:
code = f.read()
assert code == target_string.encode("utf-8")
manager = MemoryManager(pool_folder=pool_folder,
conf_path=conf_path,
block_file=block_file)
block1 = manager.allocate_block(6)
block2 = manager.allocate_block(4)
block1.write("hello")
block2.write("hey")
check_content(pool_file_1, "00010hello")
check_content(pool_file_2, "000100hey0")
assert block1.read(1, 4) == "ello".encode("utf-8")
assert block2.read(2, 2) == "y0".encode("utf-8")
assert block1.read(3, 2) == "lo".encode("utf-8")
assert block2.read(0, 2) == "he".encode("utf-8")
_clean_up()
def test_write_data_to_block():
pool_folder, conf_path, block_file = _get_common_file_paths()
_clean_up()
def check_content(filepath, target_content_string):
with open(filepath, "rb") as f:
bytes = f.read()
assert bytes == target_content_string.encode("utf-8")
manager = MemoryManager(pool_folder=pool_folder,
conf_path=conf_path,
block_file=block_file)
block = manager.allocate_block(9)
block.write("something")
assert len(manager.pools) == 2
assert len(manager.blocks) == 1
filepath1 = os.path.join(pool_folder, "pool_0")
filepath2 = os.path.join(pool_folder, "pool_1")
check_content(filepath1, "00010somet")
check_content(filepath2, "00009hing0")
assert block.used_memory == 9 and block.free_memory == 0 and block.block_size == 9
_clean_up()
def test_block_rewind_and_write():
pool_folder, conf_path, block_file = _get_common_file_paths()
_clean_up()
def check_content(filepath, target_content_string):
with open(filepath, "rb") as f:
bytes = f.read()
assert bytes == target_content_string.encode("utf-8")
manager = MemoryManager(pool_folder=pool_folder,
conf_path=conf_path,
block_file=block_file)
pool_path = os.path.join(pool_folder, "pool_0")
block = manager.allocate_block(5)
block.write("hello")
check_content(pool_path, "00010hello")
try:
block.write("hey")
assert False
except:
pass
block.rewind(2)
block.write("hey")
check_content(pool_path, "00010hehey")
_clean_up()
def test_memory_manager_allocate_multi_blocks():
pool_folder, conf_path, block_file = _get_common_file_paths()
_clean_up()
manager = MemoryManager(conf_path=conf_path,
pool_folder=pool_folder,
block_file=block_file)
block1 = manager.allocate_block(3)
block2 = manager.allocate_block(9)
assert (block1.used_memory == 0 and block1.free_memory == 3
and block1.block_size == 3)
assert (block2.used_memory == 0 and block2.free_memory == 9
and block2.block_size == 9)
def check_content(filepath, target_content_string):
with open(filepath, "rb") as f:
bytes = f.read()
assert bytes == target_content_string.encode("utf-8")
filepath1 = os.path.join(pool_folder, "pool_0")
filepath2 = os.path.join(pool_folder, "pool_1")
filepath3 = os.path.join(pool_folder, "pool_2")
check_content(filepath1, "0001000000")
check_content(filepath2, "0001000000")
check_content(filepath3, "0000700000")
# check second block which span three pools
assert len(block2.memory_segments) == 3
assert (block2.memory_segments[0].pool.pool_id == 0
and block2.memory_segments[0].start_offset == 8
and block2.memory_segments[0].end_offset == 10
and block2.memory_segments[0].length == 2)
assert (block2.memory_segments[1].pool.pool_id == 1
and block2.memory_segments[1].start_offset == 5
and block2.memory_segments[1].end_offset == 10
and block2.memory_segments[1].length == 5)
assert (block2.memory_segments[2].pool.pool_id == 2
and block2.memory_segments[2].start_offset == 5
and block2.memory_segments[2].end_offset == 7
and block2.memory_segments[2].length == 2)
# check first block which only occupy one block
assert len(block1.memory_segments) == 1
assert (block1.memory_segments[0].pool.pool_id == 0
and block1.memory_segments[0].start_offset == 5
and block1.memory_segments[0].end_offset == 8)
_clean_up()
def test_pool_dict():
pool_folder, conf_path, block_file = _get_common_file_paths()
_clean_up()
manager = MemoryManager(pool_folder=pool_folder,
conf_path=conf_path,
block_file=block_file)
manager.allocate_block(100)
assert len(manager.pool_dict) == 20
for i in range(20):
assert i in manager.pool_dict
_clean_up()
def test_memory_manager_allocate_block():
pool_folder, conf_path, block_file = _get_common_file_paths()
_clean_up()
if os.path.exists(pool_folder):
shutil.rmtree(pool_folder)
if os.path.exists(block_file):
os.remove(block_file)
manager = MemoryManager(conf_path=conf_path,
pool_folder=pool_folder,
block_file=block_file)
block = manager.allocate_block(8)
assert block.used_memory == 0 and block.free_memory == 8 and block.block_size == 8
# check files
file_paths = sorted(list(os.listdir(pool_folder)))
assert file_paths == ["pool_0", "pool_1"]
def check_content(file_path, target_content_string):
with open(file_path, "rb") as f:
bytes = f.read()
assert bytes == target_content_string.encode("utf-8")
# check content
check_content(os.path.join(pool_folder, "pool_0"), "0001000000")
check_content(os.path.join(pool_folder, "pool_1"), "0000800000")
manager = MemoryManager(conf_path=conf_path,
pool_folder=pool_folder,
block_file=block_file)
assert len(manager.pools) == 2
assert len(manager.blocks) == 1
block = manager.blocks[0]
assert block.used_memory == 0 and block.free_memory == 8 and block.block_size == 8
pools = manager.pools
pools.sort(key=lambda pool: pool.pool_id)
assert pools[0].filepath == os.path.join(pool_folder, "pool_0")
assert pools[1].filepath == os.path.join(pool_folder, "pool_1")
assert pools[0].pool_allocate_limit == 0
assert pools[0].pool_allocate_offset == 10
assert pools[1].pool_allocate_limit == 2
assert pools[1].pool_allocate_offset == 8
_clean_up()
def test_skiplist_key_value_pairs():
pool_folder, conf_path, block_file = _get_common_file_paths()
_clean_up()
index = SkipListIndex(
MemoryManager(pool_folder=pool_folder,
conf_path=conf_path,
block_file=block_file))
key_value_pairs = []
vis = set()
for _ in range(100):
key = random.randint(1, 10000)
value = random.randint(1, 10000)
while key in vis:
key = random.randint(1, 10000)
vis.add(key)
key_value_pairs.append((key, value))
key_value_pairs.sort()
assert len(key_value_pairs) == 100 and len(set(key_value_pairs)) == 100
for key, value in key_value_pairs:
index.set(key, value)
assert index.key_value_pairs() == key_value_pairs
_clean_up()
def __init__(self, file_name):
self.file_name = file_name
self.stack = Stack()
self.stack.push("EOF")
self.stack.push("Source")
self.parser_table = PARSER_TABLE
self.symbol_table = OOPSymbolTable()
self.semantic_stack = Stack()
self.memory_manager = MemoryManager(1000, 2000)
self.scanner = Scanner(file_name, self.symbol_table)
self.next_token = self.scanner.get_next_token()
self.top_stack = self.stack.top()
self.rule_number = None
self.rule = ""
self.grammar = GRAMMAR
self.error_handler = ErrorHandler(self.scanner)
self.symbol_table.set_error_handler(self.error_handler)
self.semantic_analyzer = SemanticAnalyzer(self.symbol_table,
self.memory_manager,
self.semantic_stack,
self.error_handler)
self.code_generator = CodeGenerator(self.symbol_table,
self.semantic_stack,
self.memory_manager)
self.current_identifier = None
self.follow = FOLLOW
self.non_terminal = 0
self.must_get = False
def test_set_and_get_with_persist():
pool_folder, conf_path, block_file = _get_common_file_paths()
_clean_up()
index = TreeIndex(
MemoryManager(
pool_folder=pool_folder, conf_path=conf_path, block_file=block_file
)
)
index.set(1, 10)
index.set(3, 20)
index.set(3, 100)
assert index.get(1) == 10
assert index.get(3) == 100
assert index.persist() == 2
assert index.persist() == 0
index.set(1, 7)
assert index.get(1) == 7
assert index.get(3) == 100
assert index.persist() == 1
_clean_up()
def test_keys():
pool_folder, conf_path, block_file = _get_common_file_paths()
_clean_up()
index = TreeIndex(
MemoryManager(
pool_folder=pool_folder, conf_path=conf_path, block_file=block_file
)
)
index.set(1, 10)
index.set(4, 4)
index.set(9, 100)
assert list(index.keys()) == [1, 4, 9]
index.set(4, 7)
assert list(index.keys()) == [1, 4, 9]
index.set(20, 1)
assert list(index.keys()) == [1, 4, 9, 20]
index.clear()
assert list(index.keys()) == []
values = set()
for i in range(10000):
value = random.randint(1, 200)
index.set(value, value)
values.add(value)
assert sorted(list(values)) == list(index.keys())
_clean_up()
def test_btree_clear():
pool_folder, conf_path, block_file = _get_common_file_paths()
_clean_up()
index = BTreeIndex(
MemoryManager(pool_folder=pool_folder,
conf_path=conf_path,
block_file=block_file))
values = [value for value in range(1, 11, 1)]
for value in values:
index.set(value, value)
assert len(list(index.keys())) == 10
index.clear()
assert list(index.keys()) == []
index.set(1, 10)
index.set(2, 100)
assert len(list(index.keys())) == 2
index.clear()
assert list(index.keys()) == []
_clean_up()
def test_remove():
pool_folder, conf_path, block_file = _get_common_file_paths()
_clean_up()
index = TreeIndex(
MemoryManager(
pool_folder=pool_folder, conf_path=conf_path, block_file=block_file
)
)
values = set()
for _ in range(1000):
value = random.randint(1, 1200)
while value in values:
value = random.randint(1, 1200)
values.add(value)
index.set(value, value)
for _ in range(2000):
value = random.randint(1, 1200)
if value in values:
values.remove(value)
assert index.remove(value) == True
else:
assert index.remove(value) == False
_clean_up()
def test_basic_btree_set_and_get():
pool_folder, conf_path, block_file = _get_common_file_paths()
_clean_up()
index = BTreeIndex(
MemoryManager(pool_folder=pool_folder,
conf_path=conf_path,
block_file=block_file))
index.set(1, 10)
index.set(3, 100)
index.set(6, 8)
assert index.get(3) == 100
assert index.get(6) == 8
assert index.get(8) == None
index.set(3, 8)
assert index.get(3) == 8
assert list(index.key_value_pairs()) == [(1, 10), (3, 8), (6, 8)]
index.set(10, 100)
assert index.get(1) == 10
assert index.get(3) == 8
assert index.get(6) == 8
assert index.get(10) == 100
assert index.get(15) == None
assert index.get(14) == None
assert list(index.keys()) == [1, 3, 6, 10]
_clean_up()
def test_skiplist_real_scenario_with_compact():
"""almost all things are same as `test_skiplist_real_scenario` except randomly compaction"""
pool_folder, conf_path, block_file = _get_common_file_paths()
_clean_up()
comparision_dict = {}
index = SkipListIndex(
MemoryManager(pool_folder=pool_folder,
conf_path=conf_path,
block_file=block_file))
for _ in range(20000):
ind = random.randint(1, 10)
# set: 3, get: 3, remove: 3, clear: 1
if ind <= 3:
op = "set"
elif ind <= 6:
op = "get"
elif ind <= 9:
op = "remove"
else:
op = "clear"
# dispatch operations
if op == "set":
key, value = random.randint(1, 50), random.randint(1, 1000)
comparision_dict[key] = value
index.set(key, value)
elif op == "get":
key = random.randint(1, 50)
assert comparision_dict.get(key, -1) == index.get(key, -1)
elif op == "remove":
key = random.randint(1, 50)
if key in comparision_dict:
value1 = True
comparision_dict.pop(key)
else:
value1 = False
value2 = index.remove(key)
assert value1 == value2
elif op == "clear":
comparision_dict.clear()
index.clear()
# do compaction randomly
if random.random() < 0.2:
index.compact()
# compare key-value pairs
assert sorted(list(index.key_value_pairs())) == sorted([
(key, value) for key, value in comparision_dict.items()
])
# check skiplist lists
cnt = sum(map(lambda head: not head.right, index._heads))
# it could have at most one empty list, and the only scenario is empty SkipList
assert cnt <= 1
if cnt == 1:
assert len(index._heads) == 1
index._memory_manager.close()
_clean_up()
def test_write_too_much_data_to_block():
pool_folder, conf_path, block_file = _get_common_file_paths()
_clean_up()
manager = MemoryManager(pool_folder=pool_folder,
conf_path=conf_path,
block_file=block_file)
block = manager.allocate_block(5)
assert len(manager.blocks) == 1
assert len(manager.pools) == 1
assert block.used_memory == 0 and block.free_memory == 5 and block.block_size == 5
try:
block.write("hahehe")
assert False
except:
pass
_clean_up()
def test_read_data_from_block():
pool_folder, conf_path, block_file = _get_common_file_paths()
_clean_up()
manager = MemoryManager(pool_folder=pool_folder,
conf_path=conf_path,
block_file=block_file)
block = manager.allocate_block(10)
block.write("helloworld")
manager = MemoryManager(pool_folder=pool_folder,
conf_path=conf_path,
block_file=block_file)
block = manager.blocks[0]
byte_data = block.read(3, 4)
assert byte_data == "lowo".encode("utf-8")
byte_data = block.read(1, 7)
assert byte_data == "ellowor".encode("utf-8")
_clean_up()
def __init__(self):
self.my_shell = Shell()
self.my_file_manager = FileManager(Config.storage_block_size,
Config.storage_track_num,
Config.storage_sec_num)
self.my_memory_manager = MemoryManager(
mode=Config.memory_management_mode,
page_size=Config.memory_page_size,
page_number=Config.memory_page_number,
physical_page=Config.memory_physical_page_number)
self.my_process_manager = ProcessManager(self.my_memory_manager)
self.is_monitoring = False
# start process manager
self.my_process_manager_run_thread = threading.Thread(
target=self.my_process_manager.start_manager)
self.my_process_manager_run_thread.start()
# printer Threading
self.IOdevice_thread = threading.Thread(
target=self.my_process_manager.io_device_handler)
self.IOdevice_thread.start()
def test_block_current_offset():
pool_folder, conf_path, block_file = _get_common_file_paths()
_clean_up()
manager = MemoryManager(pool_folder=pool_folder,
conf_path=conf_path,
block_file=block_file)
block = manager.allocate_block(10)
block.write("hello")
assert block.current_offset == 5
block.write("hey")
assert block.current_offset == 8
block.write("a")
assert block.current_offset == 9
block.rewind(7)
assert block.current_offset == 7
_clean_up()
def test_clear():
pool_folder, conf_path, block_file = _get_common_file_paths()
_clean_up()
index = TreeIndex(
MemoryManager(
pool_folder=pool_folder, conf_path=conf_path, block_file=block_file
)
)
index.set(1, 10)
index.set(100, 8)
index.clear()
assert list(index.keys()) == []
_clean_up()
def test_btree_real_scenario():
pool_folder, conf_path, block_file = _get_common_file_paths()
_clean_up()
comp_dict = {}
index = BTreeIndex(
MemoryManager(pool_folder=pool_folder,
conf_path=conf_path,
block_file=block_file))
ops = ["set", "get", "remove", "clear"]
for _ in range(20000):
op_index = random.randint(1, 10)
# weight: [4, 3, 2, 1]
if op_index <= 4:
op_index = 0
elif op_index <= 7:
op_index = 1
elif op_index <= 9:
op_index = 2
else:
op_index = 3
op = ops[op_index]
if op == "set":
key, value = random.randint(1, 100), random.randint(1, 100)
index.set(key, value)
comp_dict[key] = value
elif op == "get":
key = random.randint(1, 100)
if key not in comp_dict:
assert index.get(key) == None
else:
assert comp_dict[key] == index.get(key)
elif op == "remove":
key = random.randint(1, 100)
if key in comp_dict:
comp_dict.pop(key)
assert index.remove(key) == True
else:
assert index.remove(key) == False
elif op == "clear":
comp_dict.clear()
index.clear()
key_value_pairs = sorted([(key, value)
for key, value in comp_dict.items()])
assert key_value_pairs == list(index.key_value_pairs())
_clean_up()
def __init__(
self,
index_file=None,
index_type=TreeIndex,
conf_path=None,
pool_folder=None,
block_file=None,
):
# If index_file is given
if index_file:
self._index = pickle.loads(open(index_file, "rb").read())
else:
# Create a new index
self._index = index_type(
MemoryManager(conf_path, pool_folder, block_file))
assert isinstance(self._index, KVIndex)
def test_repeated_set_and_get_same_key():
pool_folder, conf_path, block_file = _get_common_file_paths()
_clean_up()
index = SkipListIndex(
MemoryManager(pool_folder=pool_folder,
conf_path=conf_path,
block_file=block_file))
comparison_dict = {}
for _ in range(100):
key, value = random.randint(1, 5), random.randint(1, 100)
comparison_dict[key] = value
index.set(key, value)
assert (sorted([(key, value) for key, value in comparison_dict.items()
]) == index.key_value_pairs())
_clean_up()
def test_skiplist_clear():
pool_folder, conf_path, block_file = _get_common_file_paths()
_clean_up()
index = SkipListIndex(
MemoryManager(pool_folder=pool_folder,
conf_path=conf_path,
block_file=block_file))
for i in range(100):
index.set(random.randint(1, 100), random.randint(1, 1000))
index.clear()
assert index.keys() == []
assert (len(index._heads) == 1 and not index._heads[0].right
and not index._heads[0].down)
_clean_up()
def test_memory_manager_bootstrap():
pool_folder, conf_path, block_file = _get_common_file_paths()
pool_file_path = os.path.join(pool_folder, "pool_0")
manager = MemoryManager(pool_folder=pool_folder,
conf_path=conf_path,
block_file=block_file)
assert len(manager.blocks) == 1
assert len(manager.pools) == 1
pool = manager.pools[0]
block = manager.blocks[0]
assert (pool.pool_id == 0 and pool.filepath == pool_file_path
and pool.pool_allocate_offset == 10
and pool.pool_allocate_limit == 0)
assert block.block_size == 5 and block.used_memory == 0 and block.free_memory == 5
def test_basic_btree_remove():
pool_folder, conf_path, block_file = _get_common_file_paths()
_clean_up()
index = BTreeIndex(
MemoryManager(pool_folder=pool_folder,
conf_path=conf_path,
block_file=block_file))
values = [value for value in range(1, 11, 1)]
for value in values:
index.set(value, value)
random.shuffle(values)
for value in values:
index.remove(value)
values.remove(value)
assert list(index.keys()) == sorted(list(values))
_clean_up()
def test_checkout():
pool_folder, conf_path, block_file = _get_common_file_paths()
_clean_up()
index = TreeIndex(
MemoryManager(
pool_folder=pool_folder, conf_path=conf_path, block_file=block_file
)
)
index.set(1, 10)
index.set(2, 8)
index.set(8, 100)
index.set(2, 4)
# Interesting time leap :-)
retrospect_index_1 = index.checkout(backoff=0)
assert retrospect_index_1.keys() == [1, 2, 8]
retrospect_index_2 = index.checkout(backoff=1)
assert retrospect_index_2.keys() == [1, 2, 8]
retrospect_index_3 = index.checkout(backoff=2)
assert retrospect_index_3.keys() == [1, 2]
retrospect_index_4 = index.checkout(backoff=3)
assert retrospect_index_4.keys() == [1]
# Validate if all indexes are really isolated
s = {
index,
retrospect_index_1,
retrospect_index_2,
retrospect_index_3,
retrospect_index_4,
}
assert len(s) == 5
# If I update index_4, it won't affect current index's result
retrospect_index_4.set(2, 7)
assert retrospect_index_4.key_value_pairs() == [(1, 10), (2, 7)]
assert index.key_value_pairs() == [(1, 10), (2, 4), (8, 100)]
_clean_up()
def test_skiplist_remove():
pool_folder, conf_path, block_file = _get_common_file_paths()
_clean_up()
index = SkipListIndex(
MemoryManager(pool_folder=pool_folder,
conf_path=conf_path,
block_file=block_file))
index.set(1, 10)
index.set(10, 20)
assert index.remove(1) == True
assert index.remove(10) == True
assert index.remove(1) == False
assert index.remove(10) == False
assert index.remove(100) == False
assert index.keys() == []
_clean_up()
def test_btree_keys():
pool_folder, conf_path, block_file = _get_common_file_paths()
_clean_up()
index = BTreeIndex(
MemoryManager(pool_folder=pool_folder,
conf_path=conf_path,
block_file=block_file))
values, vis = [], set()
for _ in range(100):
value = random.randint(1, 10000)
while value in vis:
value = random.randint(1, 10000)
vis.add(value)
values.append(value)
for ind, value in enumerate(values):
index.set(value, value)
assert list(index.keys()) == sorted(list(values[:ind + 1]))
_clean_up()
def test_basic_set_and_get_without_persist():
pool_folder, conf_path, block_file = _get_common_file_paths()
_clean_up()
index = TreeIndex(
MemoryManager(
pool_folder=pool_folder, conf_path=conf_path, block_file=block_file
)
)
index.set(1, 10)
index.set(3, 5)
assert index.get(1) == 10
assert index.get(3) == 5
assert index.get(4, 100) == 100
index.set(3, 20)
assert index.get(1) == 10
assert index.get(3) == 20
_clean_up()
def test_key_value_pairs():
pool_folder, conf_path, block_file = _get_common_file_paths()
_clean_up()
index = TreeIndex(
MemoryManager(
pool_folder=pool_folder, conf_path=conf_path, block_file=block_file
)
)
pair_dict = {}
for _ in range(10000):
key = random.randint(1, 200)
value = random.randint(1, 200)
pair_dict[key] = value
index.set(key, value)
pairs = [(key, value) for key, value in pair_dict.items()]
pairs.sort()
assert list(index.key_value_pairs()) == pairs
_clean_up()
