def test_delete_transfer(self):
self.manager.Node.n = 4
self.manager.insert(2, 32)
self.manager.insert(24, 67)
self.manager.insert(7, 87)
self.manager.insert(15, 45)
self.manager.insert(11, 43)
self.manager.delete(24)
self.assertEqual(self.manager.root, 3)
Node = self.manager.Node
root_block = BufferManager().get_file_block(self.manager.index_file_path, self.manager.root)
root_node = Node.frombytes(root_block.read())
self.assertEqual(root_node.keys, [(11,)])
left_block = BufferManager().get_file_block(self.manager.index_file_path, 1)
left_node = Node.frombytes(left_block.read())
self.assertEqual(left_node.keys, _convert_to_tuple_list([2, 7]))
self.assertEqual(left_node.children, [32, 87, 2])
right_block = BufferManager().get_file_block(self.manager.index_file_path, 2)
right_node = Node.frombytes(right_block.read())
self.assertEqual(right_node.keys, _convert_to_tuple_list([11, 15]))
self.assertEqual(right_node.children, [43, 45, 0])
def setUp(self):
Node = node_factory('<i')
left = Node(True,
[2, 7, 11],
[32, 87, 43, 2])
right = Node(True,
[15, 24, 31],
[45, 67, 89, 0])
with open('foobar', 'wb') as file:
pass
manager = BufferManager()
block = manager.get_file_block('foobar', 0)
with pin(block):
block.write(b'\0' * BufferManager.block_size)
block = manager.get_file_block('foobar', 1)
with pin(block):
block.write(bytes(left))
block = manager.get_file_block('foobar', 2)
with pin(block):
block.write(bytes(right))
self.Node = Node
self.left = left
def test_buffer_manager(self):
manager = BufferManager()
a = manager.get_file_block('foo', 0)
a.pin()
self.assertEqual(a.read(), b'Hello')
b = manager.get_file_block('./foo', 0)
self.assertTrue(a is b) # test cache hit
a.write(b'hello')
# a is not flushed
b = manager.get_file_block('foo', 1)
b.pin()
time.sleep(0.5)
self.assertEqual(b.read(), b' Worl')
with self.assertRaises(RuntimeError):
c = manager.get_file_block('foo',
2) # test buffer run out of space
a.unpin()
b.unpin()
c = manager.get_file_block('foo', 2) # test lru swap
self.assertFalse((os.path.abspath('foo'), 0)
in manager._blocks.keys()) # a should be swapped out
self.assertTrue(
(os.path.abspath('foo'), 1)
in manager._blocks.keys()) # b should remain in the buffer
with open('foo', 'rb') as file:
self.assertEqual(
file.read(),
b'hello World') # test the swapped out block is flushed
def init_my_bm(self):
self.conn, bm_conn = multiprocessing.Pipe()
self.bm = BufferManager(name="bm", conn=bm_conn)
# self.bm = BufferManager(name="bm")
# self.thread = threading.Thread(target=self.listen_to_my_conn)
# self.thread.start()
hub.spawn(self.listen_to_my_conn)
self.bm.start()
def create_param_manager(layers):
"""
@type layers: dict[unicode, pylstm.wrapper.py_layers.BaseLayer]
@rtype: buffer_manager.BufferManager
"""
param_manager = BufferManager()
for name, l in layers.items()[1:]:
sources = {name: (l.get_parameter_size, l.create_param_view)}
param_manager.add(sources, {})
return param_manager
def create_bwd_state_manager(layers):
"""
@type layers: dict[unicode, pylstm.wrapper.py_layers.BaseLayer]
@rtype: buffer_manager.BufferManager
"""
bwd_state_manager = BufferManager()
for name, l in layers.items()[1:]:
sources = {name: (l.get_bwd_state_size, l.create_bwd_state)}
bwd_state_manager.add(sources, {})
return bwd_state_manager
def drop_table(table_name):
metadata = load_metadata()
buffer_manager = BufferManager()
MinisqlFacade.delete_record_all(table_name)
shutil.rmtree('schema/tables/' + table_name + '/', True)
buffer_manager.detach_from_file('schema/tables/' + table_name + '/' +
table_name + '.table')
for index_name in metadata.tables[table_name].indexes:
buffer_manager.detach_from_file('schema/tables/' + table_name +
'/' + index_name + '.index')
metadata.drop_table(table_name)
metadata.dump()
def test_initial_insert(self):
manager = IndexManager('spam', '<id')
manager.insert([42, 7.6], 518)
self.assertEqual(manager.root, 1)
self.assertEqual(manager.first_deleted_block, 0)
self.assertEqual(manager.total_blocks, 2)
block = BufferManager().get_file_block('spam', 1)
Node = manager.Node
node = Node.frombytes(block.read())
self.assertEqual(node.is_leaf, True)
self.assertEqual(node.keys, [(42, 7.6)])
self.assertEqual(node.children, [518, 0])
def __init__(self):
self.console = None
self.__objTemplatesWatingVertices_l = []
self.__objTemplatesWatingTextrue_l = []
self.__objectTemplates_d = {}
self.bufferManager = BufferManager()
self.texMan = TextureManager()
self.__toHereQueue = Queue()
self.__toProcessQueue = Queue()
self.objectLoader = ObjectLoader(self.__toHereQueue, self.__toProcessQueue)
self.__haveThingsToGetFromProcess_i = 0
def test_multiple_delete(self):
manager = IndexManager('spam', '<id')
manager.insert([42, 7.6], 518)
manager.insert([42, 7.6], 212)
manager.insert([233, 66.6], 7)
deleted_num = manager.delete([42, 7.6])
self.assertEqual(deleted_num, 2)
self.assertEqual(manager.root, 1)
self.assertEqual(manager.first_deleted_block, 0)
self.assertEqual(manager.total_blocks, 2)
block = BufferManager().get_file_block('spam', 1)
Node = manager.Node
node = Node.frombytes(block.read())
self.assertEqual(node.is_leaf, True)
self.assertEqual(node.keys, [(233, 66.6)])
self.assertEqual(node.children, [7, 0])
def test_delete_fuse(self):
self.manager.Node.n = 4
self.manager.insert(2, 32)
self.manager.insert(24, 67)
self.manager.insert(7, 87)
self.manager.insert(15, 45)
self.manager.insert(11, 43)
self.manager.delete(24)
self.manager.delete(11)
self.assertEqual(self.manager.root, 1)
Node = self.manager.Node
root_block = BufferManager().get_file_block(self.manager.index_file_path, self.manager.root)
root_node = Node.frombytes(root_block.read())
self.assertEqual(root_node.is_leaf, True)
self.assertEqual(root_node.keys, _convert_to_tuple_list([2, 7, 15]))
self.assertEqual(root_node.children, [32, 87, 45, 0])
def test_persistence(self):
manager = IndexManager('spam', '<id')
manager.insert([42, 7.6], 518)
manager.insert([233, 66.6], 7)
manager.delete([42, 7.6])
manager.dump_header()
manager._manager.flush_all()
del manager
manager = IndexManager('spam', '<id')
self.assertEqual(manager.root, 1)
self.assertEqual(manager.first_deleted_block, 0)
self.assertEqual(manager.total_blocks, 2)
block = BufferManager().get_file_block('spam', 1)
Node = manager.Node
node = Node.frombytes(block.read())
self.assertEqual(node.is_leaf, True)
self.assertEqual(node.keys, [(233, 66.6)])
self.assertEqual(node.children, [7, 0])
def __init__(self, index_file_path, fmt):
"""specify the path of the index file and the format of the keys, return a index manager
if the index file exists, read data from the file
otherwise create it and initialize its header info
multiple index manager on the same file MUSTN'T simultaneously exist"""
self.Node = node_factory(fmt)
self.index_file_path = index_file_path
self._manager = BufferManager()
self.meta_struct = Struct(
'<4i'
) # total blocks, offset of the first deleted block, offset of the root node
try:
meta_block = self._manager.get_file_block(self.index_file_path, 0)
with pin(meta_block):
self.total_blocks, self.first_deleted_block, self.root, self.first_leaf = self.meta_struct.unpack(
meta_block.read()[:self.meta_struct.size])
except FileNotFoundError: # create and initialize an index file if not exits
self.total_blocks, self.first_deleted_block, self.root, self.first_leaf = 1, 0, 0, 0
with open(index_file_path, 'wb') as f:
f.write(
self.meta_struct.pack(self.total_blocks,
self.first_deleted_block, self.root,
self.first_leaf).ljust(
BufferManager.block_size, b'\0'))
def create_in_out_manager(extended_architecture, layers):
"""
@type extended_architecture: dict
@type layers: dict[unicode, pylstm.wrapper.py_layers.BaseLayer]
@rtype: buffer_manager.BufferManager
"""
in_out_manager = BufferManager()
possible_sources = list(extended_architecture.keys())
while possible_sources:
layer = possible_sources[0]
source_set, sink_set = get_forward_closure(layer,
extended_architecture)
for s in source_set:
possible_sources.remove(s)
source_list, sink_list, connection_table = set_up_connection_table(
source_set, sink_set, extended_architecture)
perm = permute_rows(connection_table)
source_list = [source_list[i] for i in perm]
connection_table = np.atleast_2d(connection_table[perm])
source_getters = OrderedDict()
for n in source_list:
source_getters[n] = (layers[n].get_output_buffer_size,
layers[n].create_output_view)
sink_getters = OrderedDict()
for n in sink_list:
sink_getters[n] = (layers[n].get_input_buffer_size,
layers[n].create_input_view)
in_out_manager.add(source_getters, sink_getters, connection_table)
return in_out_manager
from buffer_manager import BufferManager manager = BufferManager()
def test_singleton(self):
manager_a = BufferManager()
manager_b = BufferManager()
self.assertTrue(manager_a is manager_b)
def quit():
buffer_manager = BufferManager()
buffer_manager.flush_all()
def __init__(self, Node, index_file_path, node, key_position):
self.Node = Node
self.index_file_path = index_file_path
self.node = node
self.key_position = key_position
self.manager = BufferManager()
def test_detach(self):
manager = BufferManager()
manager.get_file_block('foo', 0)
manager.detach_from_file('foo')
self.assertFalse(manager._blocks)
def __init__(self, file_path, fmt):
self.buffer_manager = BufferManager()
self.filename = file_path
# Each record in file has 2 extra info: next's record_off and valid bit
self.record_struct = Struct(fmt + 'ci')
self.first_free_rec, self.rec_tail = self._parse_header()
