def test_delete_transfer(self): self.manager.Node.n = 4 self.manager.insert(2, 32) self.manager.insert(24, 67) self.manager.insert(7, 87) self.manager.insert(15, 45) self.manager.insert(11, 43) self.manager.delete(24) self.assertEqual(self.manager.root, 3) Node = self.manager.Node root_block = BufferManager().get_file_block(self.manager.index_file_path, self.manager.root) root_node = Node.frombytes(root_block.read()) self.assertEqual(root_node.keys, [(11,)]) left_block = BufferManager().get_file_block(self.manager.index_file_path, 1) left_node = Node.frombytes(left_block.read()) self.assertEqual(left_node.keys, _convert_to_tuple_list([2, 7])) self.assertEqual(left_node.children, [32, 87, 2]) right_block = BufferManager().get_file_block(self.manager.index_file_path, 2) right_node = Node.frombytes(right_block.read()) self.assertEqual(right_node.keys, _convert_to_tuple_list([11, 15])) self.assertEqual(right_node.children, [43, 45, 0])
def setUp(self): Node = node_factory('<i') left = Node(True, [2, 7, 11], [32, 87, 43, 2]) right = Node(True, [15, 24, 31], [45, 67, 89, 0]) with open('foobar', 'wb') as file: pass manager = BufferManager() block = manager.get_file_block('foobar', 0) with pin(block): block.write(b'\0' * BufferManager.block_size) block = manager.get_file_block('foobar', 1) with pin(block): block.write(bytes(left)) block = manager.get_file_block('foobar', 2) with pin(block): block.write(bytes(right)) self.Node = Node self.left = left
def test_buffer_manager(self): manager = BufferManager() a = manager.get_file_block('foo', 0) a.pin() self.assertEqual(a.read(), b'Hello') b = manager.get_file_block('./foo', 0) self.assertTrue(a is b) # test cache hit a.write(b'hello') # a is not flushed b = manager.get_file_block('foo', 1) b.pin() time.sleep(0.5) self.assertEqual(b.read(), b' Worl') with self.assertRaises(RuntimeError): c = manager.get_file_block('foo', 2) # test buffer run out of space a.unpin() b.unpin() c = manager.get_file_block('foo', 2) # test lru swap self.assertFalse((os.path.abspath('foo'), 0) in manager._blocks.keys()) # a should be swapped out self.assertTrue( (os.path.abspath('foo'), 1) in manager._blocks.keys()) # b should remain in the buffer with open('foo', 'rb') as file: self.assertEqual( file.read(), b'hello World') # test the swapped out block is flushed
def init_my_bm(self): self.conn, bm_conn = multiprocessing.Pipe() self.bm = BufferManager(name="bm", conn=bm_conn) # self.bm = BufferManager(name="bm") # self.thread = threading.Thread(target=self.listen_to_my_conn) # self.thread.start() hub.spawn(self.listen_to_my_conn) self.bm.start()
def create_param_manager(layers): """ @type layers: dict[unicode, pylstm.wrapper.py_layers.BaseLayer] @rtype: buffer_manager.BufferManager """ param_manager = BufferManager() for name, l in layers.items()[1:]: sources = {name: (l.get_parameter_size, l.create_param_view)} param_manager.add(sources, {}) return param_manager
def create_bwd_state_manager(layers): """ @type layers: dict[unicode, pylstm.wrapper.py_layers.BaseLayer] @rtype: buffer_manager.BufferManager """ bwd_state_manager = BufferManager() for name, l in layers.items()[1:]: sources = {name: (l.get_bwd_state_size, l.create_bwd_state)} bwd_state_manager.add(sources, {}) return bwd_state_manager
def drop_table(table_name): metadata = load_metadata() buffer_manager = BufferManager() MinisqlFacade.delete_record_all(table_name) shutil.rmtree('schema/tables/' + table_name + '/', True) buffer_manager.detach_from_file('schema/tables/' + table_name + '/' + table_name + '.table') for index_name in metadata.tables[table_name].indexes: buffer_manager.detach_from_file('schema/tables/' + table_name + '/' + index_name + '.index') metadata.drop_table(table_name) metadata.dump()
def test_initial_insert(self): manager = IndexManager('spam', '<id') manager.insert([42, 7.6], 518) self.assertEqual(manager.root, 1) self.assertEqual(manager.first_deleted_block, 0) self.assertEqual(manager.total_blocks, 2) block = BufferManager().get_file_block('spam', 1) Node = manager.Node node = Node.frombytes(block.read()) self.assertEqual(node.is_leaf, True) self.assertEqual(node.keys, [(42, 7.6)]) self.assertEqual(node.children, [518, 0])
def __init__(self): self.console = None self.__objTemplatesWatingVertices_l = [] self.__objTemplatesWatingTextrue_l = [] self.__objectTemplates_d = {} self.bufferManager = BufferManager() self.texMan = TextureManager() self.__toHereQueue = Queue() self.__toProcessQueue = Queue() self.objectLoader = ObjectLoader(self.__toHereQueue, self.__toProcessQueue) self.__haveThingsToGetFromProcess_i = 0
def test_multiple_delete(self): manager = IndexManager('spam', '<id') manager.insert([42, 7.6], 518) manager.insert([42, 7.6], 212) manager.insert([233, 66.6], 7) deleted_num = manager.delete([42, 7.6]) self.assertEqual(deleted_num, 2) self.assertEqual(manager.root, 1) self.assertEqual(manager.first_deleted_block, 0) self.assertEqual(manager.total_blocks, 2) block = BufferManager().get_file_block('spam', 1) Node = manager.Node node = Node.frombytes(block.read()) self.assertEqual(node.is_leaf, True) self.assertEqual(node.keys, [(233, 66.6)]) self.assertEqual(node.children, [7, 0])
def test_delete_fuse(self): self.manager.Node.n = 4 self.manager.insert(2, 32) self.manager.insert(24, 67) self.manager.insert(7, 87) self.manager.insert(15, 45) self.manager.insert(11, 43) self.manager.delete(24) self.manager.delete(11) self.assertEqual(self.manager.root, 1) Node = self.manager.Node root_block = BufferManager().get_file_block(self.manager.index_file_path, self.manager.root) root_node = Node.frombytes(root_block.read()) self.assertEqual(root_node.is_leaf, True) self.assertEqual(root_node.keys, _convert_to_tuple_list([2, 7, 15])) self.assertEqual(root_node.children, [32, 87, 45, 0])
def test_persistence(self): manager = IndexManager('spam', '<id') manager.insert([42, 7.6], 518) manager.insert([233, 66.6], 7) manager.delete([42, 7.6]) manager.dump_header() manager._manager.flush_all() del manager manager = IndexManager('spam', '<id') self.assertEqual(manager.root, 1) self.assertEqual(manager.first_deleted_block, 0) self.assertEqual(manager.total_blocks, 2) block = BufferManager().get_file_block('spam', 1) Node = manager.Node node = Node.frombytes(block.read()) self.assertEqual(node.is_leaf, True) self.assertEqual(node.keys, [(233, 66.6)]) self.assertEqual(node.children, [7, 0])
def __init__(self, index_file_path, fmt): """specify the path of the index file and the format of the keys, return a index manager if the index file exists, read data from the file otherwise create it and initialize its header info multiple index manager on the same file MUSTN'T simultaneously exist""" self.Node = node_factory(fmt) self.index_file_path = index_file_path self._manager = BufferManager() self.meta_struct = Struct( '<4i' ) # total blocks, offset of the first deleted block, offset of the root node try: meta_block = self._manager.get_file_block(self.index_file_path, 0) with pin(meta_block): self.total_blocks, self.first_deleted_block, self.root, self.first_leaf = self.meta_struct.unpack( meta_block.read()[:self.meta_struct.size]) except FileNotFoundError: # create and initialize an index file if not exits self.total_blocks, self.first_deleted_block, self.root, self.first_leaf = 1, 0, 0, 0 with open(index_file_path, 'wb') as f: f.write( self.meta_struct.pack(self.total_blocks, self.first_deleted_block, self.root, self.first_leaf).ljust( BufferManager.block_size, b'\0'))
def create_in_out_manager(extended_architecture, layers): """ @type extended_architecture: dict @type layers: dict[unicode, pylstm.wrapper.py_layers.BaseLayer] @rtype: buffer_manager.BufferManager """ in_out_manager = BufferManager() possible_sources = list(extended_architecture.keys()) while possible_sources: layer = possible_sources[0] source_set, sink_set = get_forward_closure(layer, extended_architecture) for s in source_set: possible_sources.remove(s) source_list, sink_list, connection_table = set_up_connection_table( source_set, sink_set, extended_architecture) perm = permute_rows(connection_table) source_list = [source_list[i] for i in perm] connection_table = np.atleast_2d(connection_table[perm]) source_getters = OrderedDict() for n in source_list: source_getters[n] = (layers[n].get_output_buffer_size, layers[n].create_output_view) sink_getters = OrderedDict() for n in sink_list: sink_getters[n] = (layers[n].get_input_buffer_size, layers[n].create_input_view) in_out_manager.add(source_getters, sink_getters, connection_table) return in_out_manager
from buffer_manager import BufferManager manager = BufferManager()
def test_singleton(self): manager_a = BufferManager() manager_b = BufferManager() self.assertTrue(manager_a is manager_b)
def quit(): buffer_manager = BufferManager() buffer_manager.flush_all()
def __init__(self, Node, index_file_path, node, key_position): self.Node = Node self.index_file_path = index_file_path self.node = node self.key_position = key_position self.manager = BufferManager()
def test_detach(self): manager = BufferManager() manager.get_file_block('foo', 0) manager.detach_from_file('foo') self.assertFalse(manager._blocks)
def __init__(self, file_path, fmt): self.buffer_manager = BufferManager() self.filename = file_path # Each record in file has 2 extra info: next's record_off and valid bit self.record_struct = Struct(fmt + 'ci') self.first_free_rec, self.rec_tail = self._parse_header()