def create_in_out_manager(extended_architecture, layers): """ @type extended_architecture: dict @type layers: dict[unicode, pylstm.wrapper.py_layers.BaseLayer] @rtype: buffer_manager.BufferManager """ in_out_manager = BufferManager() possible_sources = list(extended_architecture.keys()) while possible_sources: layer = possible_sources[0] source_set, sink_set = get_forward_closure(layer, extended_architecture) for s in source_set: possible_sources.remove(s) source_list, sink_list, connection_table = set_up_connection_table( source_set, sink_set, extended_architecture) perm = permute_rows(connection_table) source_list = [source_list[i] for i in perm] connection_table = np.atleast_2d(connection_table[perm]) source_getters = OrderedDict() for n in source_list: source_getters[n] = (layers[n].get_output_buffer_size, layers[n].create_output_view) sink_getters = OrderedDict() for n in sink_list: sink_getters[n] = (layers[n].get_input_buffer_size, layers[n].create_input_view) in_out_manager.add(source_getters, sink_getters, connection_table) return in_out_manager
def setUp(self): Node = node_factory('<i') left = Node(True, [2, 7, 11], [32, 87, 43, 2]) right = Node(True, [15, 24, 31], [45, 67, 89, 0]) with open('foobar', 'wb') as file: pass manager = BufferManager() block = manager.get_file_block('foobar', 0) with pin(block): block.write(b'\0' * BufferManager.block_size) block = manager.get_file_block('foobar', 1) with pin(block): block.write(bytes(left)) block = manager.get_file_block('foobar', 2) with pin(block): block.write(bytes(right)) self.Node = Node self.left = left
def create_param_manager(layers): """ @type layers: dict[unicode, pylstm.wrapper.py_layers.BaseLayer] @rtype: buffer_manager.BufferManager """ param_manager = BufferManager() for name, l in layers.items()[1:]: sources = {name: (l.get_parameter_size, l.create_param_view)} param_manager.add(sources, {}) return param_manager
def init_my_bm(self): self.conn, bm_conn = multiprocessing.Pipe() self.bm = BufferManager(name="bm", conn=bm_conn) # self.bm = BufferManager(name="bm") # self.thread = threading.Thread(target=self.listen_to_my_conn) # self.thread.start() hub.spawn(self.listen_to_my_conn) self.bm.start()
def create_bwd_state_manager(layers): """ @type layers: dict[unicode, pylstm.wrapper.py_layers.BaseLayer] @rtype: buffer_manager.BufferManager """ bwd_state_manager = BufferManager() for name, l in layers.items()[1:]: sources = {name: (l.get_bwd_state_size, l.create_bwd_state)} bwd_state_manager.add(sources, {}) return bwd_state_manager
def test_delete_transfer(self): self.manager.Node.n = 4 self.manager.insert(2, 32) self.manager.insert(24, 67) self.manager.insert(7, 87) self.manager.insert(15, 45) self.manager.insert(11, 43) self.manager.delete(24) self.assertEqual(self.manager.root, 3) Node = self.manager.Node root_block = BufferManager().get_file_block(self.manager.index_file_path, self.manager.root) root_node = Node.frombytes(root_block.read()) self.assertEqual(root_node.keys, [(11,)]) left_block = BufferManager().get_file_block(self.manager.index_file_path, 1) left_node = Node.frombytes(left_block.read()) self.assertEqual(left_node.keys, _convert_to_tuple_list([2, 7])) self.assertEqual(left_node.children, [32, 87, 2]) right_block = BufferManager().get_file_block(self.manager.index_file_path, 2) right_node = Node.frombytes(right_block.read()) self.assertEqual(right_node.keys, _convert_to_tuple_list([11, 15])) self.assertEqual(right_node.children, [43, 45, 0])
def test_initial_insert(self): manager = IndexManager('spam', '<id') manager.insert([42, 7.6], 518) self.assertEqual(manager.root, 1) self.assertEqual(manager.first_deleted_block, 0) self.assertEqual(manager.total_blocks, 2) block = BufferManager().get_file_block('spam', 1) Node = manager.Node node = Node.frombytes(block.read()) self.assertEqual(node.is_leaf, True) self.assertEqual(node.keys, [(42, 7.6)]) self.assertEqual(node.children, [518, 0])
def __init__(self): self.console = None self.__objTemplatesWatingVertices_l = [] self.__objTemplatesWatingTextrue_l = [] self.__objectTemplates_d = {} self.bufferManager = BufferManager() self.texMan = TextureManager() self.__toHereQueue = Queue() self.__toProcessQueue = Queue() self.objectLoader = ObjectLoader(self.__toHereQueue, self.__toProcessQueue) self.__haveThingsToGetFromProcess_i = 0
class LeafIterator: def __init__(self, Node, index_file_path, node, key_position): self.Node = Node self.index_file_path = index_file_path self.node = node self.key_position = key_position self.manager = BufferManager() def __iter__(self): return self def __next__(self): value = self.node.children[self.key_position] if self.key_position < len(self.node.keys): key = self.node.keys[self.key_position] self.key_position += 1 return key, value else: # jump if value == 0: raise StopIteration else: node_block = self.manager.get_file_block( self.index_file_path, value) with pin(node_block): self.node = self.Node.frombytes(node_block.read()) self.key_position = 1 return self.node.keys[0], self.node.children[0]
def test_multiple_delete(self): manager = IndexManager('spam', '<id') manager.insert([42, 7.6], 518) manager.insert([42, 7.6], 212) manager.insert([233, 66.6], 7) deleted_num = manager.delete([42, 7.6]) self.assertEqual(deleted_num, 2) self.assertEqual(manager.root, 1) self.assertEqual(manager.first_deleted_block, 0) self.assertEqual(manager.total_blocks, 2) block = BufferManager().get_file_block('spam', 1) Node = manager.Node node = Node.frombytes(block.read()) self.assertEqual(node.is_leaf, True) self.assertEqual(node.keys, [(233, 66.6)]) self.assertEqual(node.children, [7, 0])
def test_buffer_manager(self): manager = BufferManager() a = manager.get_file_block('foo', 0) a.pin() self.assertEqual(a.read(), b'Hello') b = manager.get_file_block('./foo', 0) self.assertTrue(a is b) # test cache hit a.write(b'hello') # a is not flushed b = manager.get_file_block('foo', 1) b.pin() time.sleep(0.5) self.assertEqual(b.read(), b' Worl') with self.assertRaises(RuntimeError): c = manager.get_file_block('foo', 2) # test buffer run out of space a.unpin() b.unpin() c = manager.get_file_block('foo', 2) # test lru swap self.assertFalse((os.path.abspath('foo'), 0) in manager._blocks.keys()) # a should be swapped out self.assertTrue( (os.path.abspath('foo'), 1) in manager._blocks.keys()) # b should remain in the buffer with open('foo', 'rb') as file: self.assertEqual( file.read(), b'hello World') # test the swapped out block is flushed
def test_delete_fuse(self): self.manager.Node.n = 4 self.manager.insert(2, 32) self.manager.insert(24, 67) self.manager.insert(7, 87) self.manager.insert(15, 45) self.manager.insert(11, 43) self.manager.delete(24) self.manager.delete(11) self.assertEqual(self.manager.root, 1) Node = self.manager.Node root_block = BufferManager().get_file_block(self.manager.index_file_path, self.manager.root) root_node = Node.frombytes(root_block.read()) self.assertEqual(root_node.is_leaf, True) self.assertEqual(root_node.keys, _convert_to_tuple_list([2, 7, 15])) self.assertEqual(root_node.children, [32, 87, 45, 0])
def test_persistence(self): manager = IndexManager('spam', '<id') manager.insert([42, 7.6], 518) manager.insert([233, 66.6], 7) manager.delete([42, 7.6]) manager.dump_header() manager._manager.flush_all() del manager manager = IndexManager('spam', '<id') self.assertEqual(manager.root, 1) self.assertEqual(manager.first_deleted_block, 0) self.assertEqual(manager.total_blocks, 2) block = BufferManager().get_file_block('spam', 1) Node = manager.Node node = Node.frombytes(block.read()) self.assertEqual(node.is_leaf, True) self.assertEqual(node.keys, [(233, 66.6)]) self.assertEqual(node.children, [7, 0])
def __init__(self, index_file_path, fmt): """specify the path of the index file and the format of the keys, return a index manager if the index file exists, read data from the file otherwise create it and initialize its header info multiple index manager on the same file MUSTN'T simultaneously exist""" self.Node = node_factory(fmt) self.index_file_path = index_file_path self._manager = BufferManager() self.meta_struct = Struct( '<4i' ) # total blocks, offset of the first deleted block, offset of the root node try: meta_block = self._manager.get_file_block(self.index_file_path, 0) with pin(meta_block): self.total_blocks, self.first_deleted_block, self.root, self.first_leaf = self.meta_struct.unpack( meta_block.read()[:self.meta_struct.size]) except FileNotFoundError: # create and initialize an index file if not exits self.total_blocks, self.first_deleted_block, self.root, self.first_leaf = 1, 0, 0, 0 with open(index_file_path, 'wb') as f: f.write( self.meta_struct.pack(self.total_blocks, self.first_deleted_block, self.root, self.first_leaf).ljust( BufferManager.block_size, b'\0'))
def drop_table(table_name): metadata = load_metadata() buffer_manager = BufferManager() MinisqlFacade.delete_record_all(table_name) shutil.rmtree('schema/tables/' + table_name + '/', True) buffer_manager.detach_from_file('schema/tables/' + table_name + '/' + table_name + '.table') for index_name in metadata.tables[table_name].indexes: buffer_manager.detach_from_file('schema/tables/' + table_name + '/' + index_name + '.index') metadata.drop_table(table_name) metadata.dump()
class IndexManager: def __init__(self, index_file_path, fmt): """specify the path of the index file and the format of the keys, return a index manager if the index file exists, read data from the file otherwise create it and initialize its header info multiple index manager on the same file MUSTN'T simultaneously exist""" self.Node = node_factory(fmt) self.index_file_path = index_file_path self._manager = BufferManager() self.meta_struct = Struct( '<4i' ) # total blocks, offset of the first deleted block, offset of the root node try: meta_block = self._manager.get_file_block(self.index_file_path, 0) with pin(meta_block): self.total_blocks, self.first_deleted_block, self.root, self.first_leaf = self.meta_struct.unpack( meta_block.read()[:self.meta_struct.size]) except FileNotFoundError: # create and initialize an index file if not exits self.total_blocks, self.first_deleted_block, self.root, self.first_leaf = 1, 0, 0, 0 with open(index_file_path, 'wb') as f: f.write( self.meta_struct.pack(self.total_blocks, self.first_deleted_block, self.root, self.first_leaf).ljust( BufferManager.block_size, b'\0')) def dump_header(self): """write the header info to the index file MUST be called before the program exits, otherwise the header info in the file won't be updated""" meta_block = self._manager.get_file_block(self.index_file_path, 0) with pin(meta_block): meta_block.write( self.meta_struct.pack(self.total_blocks, self.first_deleted_block, self.root, self.first_leaf).ljust( BufferManager.block_size, b'\0')) def _get_free_block(self): """return a free block and update header info, assuming this block will be used""" if self.first_deleted_block > 0: block_offset = self.first_deleted_block block = self._manager.get_file_block(self.index_file_path, block_offset) s = Struct('<i') next_deleted = s.unpack(block.read()[:s.size])[0] self.first_deleted_block = next_deleted return block else: block_offset = self.total_blocks block = self._manager.get_file_block(self.index_file_path, block_offset) self.total_blocks += 1 return block def _delete_node(self, node, block): """delete node and writes it to block just a shortcut to mark a block as deleted""" with pin(block): node.next_deleted = self.first_deleted_block block.write(bytes(node)) self.first_deleted_block = block.block_offset def _find_leaf(self, key): """find the first leaf node where key may reside key may not really reside in this node, in this case, the index file has no such key""" key = _convert_to_tuple(key) node_block_offset = self.root path_to_parents = [] while True: # find the insert position node_block = self._manager.get_file_block(self.index_file_path, node_block_offset) with pin(node_block): node = self.Node.frombytes(node_block.read()) if node.is_leaf: return node, node_block, path_to_parents else: # continue searching path_to_parents.append(node_block_offset) child_index = bisect.bisect_right(node.keys, key) node_block_offset = node.children[child_index] def _handle_overflow(self, node, block, path_to_parents): if not path_to_parents: # the root overflowed new_block = self._get_free_block() new_node, key, value = node.split(new_block.block_offset) with pin(block), pin(new_block): block.write(bytes(node)) new_block.write(bytes(new_node)) new_root_block = self._get_free_block() with pin(new_root_block): new_root_node = self.Node( False, [key], [block.block_offset, new_block.block_offset]) new_root_block.write(bytes(new_root_node)) self.root = new_root_block.block_offset return else: parent_offset = path_to_parents.pop() new_block = self._get_free_block() new_node, key, value = node.split(new_block.block_offset) with pin(block), pin(new_block): block.write(bytes(node)) new_block.write(bytes(new_node)) parent_block = self._manager.get_file_block( self.index_file_path, parent_offset) parent_node = self.Node.frombytes(parent_block.read()) parent_node.insert(key, value) if len(parent_node.keys) <= self.Node.n: with pin(parent_block): parent_block.write(bytes(parent_node)) else: self._handle_overflow(parent_node, parent_block, path_to_parents) def _handle_underflow(self, node, block, path_to_parents): """handle underflow after deletion will try to transfer from the left sibling first then try to transfer from the right sibling then try to fuse with the left sibling then try to fuse with the right sibling""" if block.block_offset == self.root: if not node.keys: # root has no key at all; this node is no longer needed if node.is_leaf: self.root = 0 self.first_leaf = 0 else: self.root = node.children[0] self._delete_node(node, block) else: block.write(bytes(node)) return # root underflow is not a problem parent_offset = path_to_parents.pop() parent_block = self._manager.get_file_block(self.index_file_path, parent_offset) with pin(parent_block): parent = self.Node.frombytes(parent_block.read()) my_position = bisect.bisect_right(parent.keys, node.keys[0]) if my_position > 0: # try find the left sibling left_sibling_offset = parent.children[my_position - 1] left_sibling_block = self._manager.get_file_block( self.index_file_path, left_sibling_offset) with pin(left_sibling_block): left_sibling = self.Node.frombytes(left_sibling_block.read()) if len(left_sibling.keys) > ceil( node.n / 2): # a transfer is possible node.transfer_from_left(left_sibling, parent, my_position - 1) with pin(block), pin(left_sibling_block), pin(parent_block): block.write(bytes(node)) left_sibling_block.write(bytes(left_sibling)) parent_block.write(bytes(parent)) return else: left_sibling = None # no left sibling if my_position < len(parent.keys) - 1: # try find the right sibling right_sibling_offset = parent.children[my_position + 1] right_sibling_block = self._manager.get_file_block( self.index_file_path, right_sibling_offset) with pin(right_sibling_block): right_sibling = self.Node.frombytes(right_sibling_block.read()) if len(right_sibling.keys) > ceil( node.n / 2): # a transfer is possible node.transfer_from_right(right_sibling, parent, my_position) with pin(block), pin(right_sibling_block), pin(parent_block): block.write(bytes(node)) right_sibling_block.write(bytes(right_sibling)) parent_block.write(bytes(parent)) return else: right_sibling = None # no right sibling if left_sibling is not None: # fuse with left sibling left_sibling.fuse_with(node, parent, my_position - 1) with pin(left_sibling_block): left_sibling_block.write(bytes(left_sibling)) self._delete_node(node, block) if len(parent.keys) >= ceil(node.n / 2): return else: self._handle_underflow(parent, parent_block, path_to_parents) else: # fuse with right sibling node.fuse_with(right_sibling, parent, my_position) with pin(block): block.write(bytes(node)) self._delete_node(right_sibling, right_sibling_block) if len(parent.keys) >= ceil(node.n / 2): return else: self._handle_underflow(parent, parent_block, path_to_parents) def find(self, key): """find the smallest key >= (parameter) key return an iterator from this position raise RuntimeError if the index is empty""" key = _convert_to_tuple(key) if self.root == 0: raise RuntimeError('cannot find from empty index') else: node, node_block, path_to_parents = self._find_leaf(key) key_position = bisect.bisect_left(node.keys, key) return LeafIterator(self.Node, self.index_file_path, node, key_position) def insert(self, key, value): """insert a key-value pair into the index file if key already in this index, raise ValueError""" key = _convert_to_tuple(key) if self.root == 0: block = self._get_free_block() with pin(block): self.root = block.block_offset self.first_leaf = self.root node = self.Node(is_leaf=True, keys=[key], children=[value, 0]) block.write(bytes(node)) else: node, node_block, path_to_parents = self._find_leaf(key) key_position = bisect.bisect_left(node.keys, key) if key_position < len( node.keys) and node.keys[key_position] == key: raise ValueError('duplicate key {}'.format(key)) node.insert(key, value) if len(node.keys) <= node.n: node_block.write(bytes(node)) return else: # split self._handle_overflow(node, node_block, path_to_parents) def delete(self, key): """delete the key-value pair with key equal the parameter if the index file has no such key, raise ValueError""" key = _convert_to_tuple(key) if self.root == 0: raise ValueError('can\'t delete from empty index') else: node, node_block, path_to_parents = self._find_leaf(key) key_position = bisect.bisect_left(node.keys, key) if key_position < len( node.keys) and node.keys[key_position] == key: # key match del node.keys[key_position] del node.children[key_position] if len(node.keys) >= ceil(node.n / 2): node_block.write(bytes(node)) return else: # underflow self._handle_underflow(node, node_block, path_to_parents) else: # key doesn't match raise ValueError('index has no such key {}'.format(key)) def iter_leaves(self): """return an iterator at the beginning of the leaf node chain""" if self.first_leaf == 0: raise RuntimeError('can\'t iter from empty index') first_leaf_block = self._manager.get_file_block( self.index_file_path, self.first_leaf) first_leaf = self.Node.frombytes(first_leaf_block.read()) return LeafIterator(self.Node, self.index_file_path, first_leaf, 0)
class LocalController(app_manager.RyuApp): OFP_VERSIONS = [ofproto_v1_3.OFP_VERSION] def __init__(self, *args, **kwargs): super(LocalController, self).__init__(*args, **kwargs) # self.topo = extract_topo(fourswitch()) self.topo = {} self.time = 0 self.datapaths = {} self.eth_to_port = {} self.flows_final = False self.buffer = dict() self.bm = None self.conn = None self.thread = None # self.bm.setDaemon(True) self.init_my_bm() self.packts_buffed = 0 # self.queue = Queue() # logger.init('buftest' ,logging.INFO) def init_my_bm(self): self.conn, bm_conn = multiprocessing.Pipe() self.bm = BufferManager(name="bm", conn=bm_conn) # self.bm = BufferManager(name="bm") # self.thread = threading.Thread(target=self.listen_to_my_conn) # self.thread.start() hub.spawn(self.listen_to_my_conn) self.bm.start() def listen_to_my_conn(self): while (True): # print("but in the list" + str(len(self.pkg_to_save))) packet_to_me = None try: packet_to_me = self.conn.recv() except Exception as err: # print("I'm waiting for your bilibili") time.sleep(0.01) pass if (packet_to_me): packet = pickle.loads(packet_to_me) # print(packet) dpid = packet["dpid"] pkg = packet["pkg"] in_port = packet["in_port"] self.send_back(pkg, dpid, in_port) @set_ev_cls(ofp_event.EventOFPSwitchFeatures, CONFIG_DISPATCHER) def _switch_features_handler(self, ev): msg = ev.msg datapath = msg.datapath ofproto = datapath.ofproto parser = datapath.ofproto_parser dpid = datapath.id self.datapaths[dpid] = datapath print(self.datapaths) print(ofproto.OFPP_CONTROLLER) actions = [ parser.OFPActionOutput(port=ofproto.OFPP_CONTROLLER, max_len=ofproto.OFPCML_NO_BUFFER) ] # actions = [parser.OFPActionOutput(port=0 ,max_len=ofproto.OFPCML_NO_BUFFER)] inst = [ parser.OFPInstructionActions(type_=ofproto.OFPIT_APPLY_ACTIONS, actions=actions) ] mod = parser.OFPFlowMod(datapath=datapath, priority=0, match=parser.OFPMatch(), instructions=inst) datapath.send_msg(mod) # buf_match = parser.OFPMatch(eth_type=ether_types.ETH_TYPE_IP,ipv4_src="192.168.1.1",ipv4_dst="192.168.1.2") # buf_match = parser.OFPMatch(in_port=1) # self.send_buf_cmd(self.datapaths[1],buf_match) # buf_match = parser.OFPMatch(in_port=2) # self.send_buf_cmd(self.datapaths[1],buf_match) self.disable_dhcp(datapath) # self.install(datapath) @set_ev_cls(ofp_event.EventOFPBarrierReply, MAIN_DISPATCHER) def _barrier_reply_handler(self, ev): #nononononono don't print(ev.msg) print("---------------------------------------------------------") datapath = ev.msg.datapath cmd_pop = self.make_buf_message(consts.BUF_POP, src="10.0.0.1", dst="10.0.0.2", dst_port=None, dpid=None, pkg=None, in_port=None) try: self.conn.send(cmd_pop) except Exception as e: print("Error in barrier!!!!!!!!!!!") print(e) # if(len(self.buffer)>0): # self.send_back(1,datapath) # self.install34(self.datapaths[1]) def install(self, datapath): # time.sleep(2) ofproto = datapath.ofproto parser = datapath.ofproto_parser priority = 2 buffer_id = ofproto.OFP_NO_BUFFER match = parser.OFPMatch(in_port=1) actions = [parser.OFPActionOutput(2)] inst = [ parser.OFPInstructionActions(ofproto.OFPIT_APPLY_ACTIONS, actions) ] mod = parser.OFPFlowMod(datapath=datapath, priority=priority, match=match, instructions=inst) datapath.send_msg(mod) # time.sleep(10) # datapath.send_barrier() match = parser.OFPMatch(in_port=2) actions = [parser.OFPActionOutput(1)] inst = [ parser.OFPInstructionActions(ofproto.OFPIT_APPLY_ACTIONS, actions) ] mod = parser.OFPFlowMod(datapath=datapath, priority=priority, match=match, instructions=inst) datapath.send_msg(mod) # datapath.send_barrier() req = parser.OFPBarrierRequest(datapath, xid=1) datapath.send_msg(req) @set_ev_cls(ofp_event.EventOFPPacketIn, MAIN_DISPATCHER) def _packet_in_handler(self, ev): pkt_in = ev.msg datapath = pkt_in.datapath ofproto = datapath.ofproto in_port = pkt_in.match["in_port"] print(in_port) src, dst, dst_port, dpid, pkg = self.identify_pkg(pkt_in) self.packts_buffed += 1 if (self.packts_buffed == 100): self.install(self.datapaths[1]) # print(pkg) # self.bm.pkg_to_save.append(pkt_in) cmd_to_bm = self.make_buf_message(consts.BUF_PUSH, src, dst, dst_port=dst_port, dpid=dpid, pkg=pkg, in_port=in_port) try: self.conn.send(cmd_to_bm) #for process except Exception as e: print("Error!!!!!!!!!!!!!!!!!!!!!!!!!!") print(e) # ("go and save") def make_buf_message(self, msg_type, src, dst, dst_port, dpid, pkg, in_port): return pickle.dumps({ "msg_type": msg_type, "src": src, "dst": dst, "dst_port": dst_port, "dpid": dpid, "pkg": pkg, "in_port": in_port }) def cal_update(self, src, dst, old, new): n_buf = new[0] match = parser.OFPMatch(eth_type=ether_types.ETH_TYPE_IP, ipv4_dst=ipAdd(dst)) actions_buf = [ parser.OFPActionOutput(port=ofproto.OFPP_CONTROLLER, max_len=ofproto.OFPCML_NO_BUFFER) ] dp_buf = self.datapaths[n_buf] self.add_flow(datapath=dp_buf, priority=233, match=match, actions=actions) match_remove = parser.OFPMatch(eth_type=ether_types.ETH_TYPEk_IP, ipv4_dst=ipAdd(dst)) for node in old: dp = self.datapaths[node] self.remove_flow(datapath=dp, priority=1, match=match) new_reverse = new[::-1] for node in new: dp = self.datapaths[node] actions = [ parser.OFPActionOutput(port=ofproto.OFPP_CONTROLLER, max_len=ofproto.OFPCML_NO_BUFFER) ] self.add_flow(datapath=dp, priority=1, match=match, actions=actions) #drop packet for udp 68 def disable_dhcp(self, datapath): ofproto = datapath.ofproto parser = datapath.ofproto_parser match_dhcp = parser.OFPMatch(eth_type=ether_types.ETH_TYPE_IP, ip_proto=in_proto.IPPROTO_UDP, udp_src=68, udp_dst=67) instruction = [ parser.OFPInstructionActions(ofproto.OFPIT_CLEAR_ACTIONS, []) ] mod = parser.OFPFlowMod(datapath, priority=1, match=match_dhcp, instructions=instruction) datapath.send_msg(mod) def remove_flow(self, datapath, priority, match): ofproto = datapath.ofproto parser = datapath.ofproto_parser mod = parser.OFPFlowMod(datapath=datapath, command=ofproto.OFPFC_DELETE, out_port=ofproto.OFPP_ANY, out_group=ofproto.OFPG_ANY, match=match, priority=priority) datapath.send_msg(mod) def send_back(self, pkg, dpid, in_port): datapath = self.datapaths[dpid] parser = datapath.ofproto_parser ofproto = datapath.ofproto actions = [parser.OFPActionOutput(port=ofproto.OFPP_TABLE)] req = parser.OFPPacketOut(datapath, in_port=in_port, buffer_id=ofproto.OFP_NO_BUFFER, actions=actions, data=pkg) datapath.send_msg(req) # print("sent back") @set_ev_cls(ofp_event.EventOFPSwitchFeatures, CONFIG_DISPATCHER) def _features_handler(self, ev): msg = ev.msg datapath = msg.datapath ofproto = datapath.ofproto parser = datapath.ofproto_parser dpid = datapath.id self.datapaths[dpid] = datapath print("haha" + str(self.datapaths)) actions = [ parser.OFPActionOutput(port=ofproto.OFPP_CONTROLLER, max_len=ofproto.OFPCML_NO_BUFFER) ] inst = [ parser.OFPInstructionActions(type_=ofproto.OFPIT_APPLY_ACTIONS, actions=actions) ] mod = parser.OFPFlowMod(datapath=datapath, priority=0, match=parser.OFPMatch(), instructions=inst) datapath.send_msg(mod) def save_to_buffer(self, flow_id, pkg): if (pkg.datapath.id): flow_id = pkg.datapath.id if (not self.buffer.has_key(flow_id)): self.buffer[flow_id] = [] print(flow_id) self.buffer[flow_id].append(pkg) print(len(self.buffer[flow_id])) def identify_pkg(self, pkt_in): (src, dst, dst_port, dpid, pkg) = ("", "", None, None, None) dpid = pkt_in.datapath.id pkg = packet.Packet(pkt_in.data) # pkg = pkt_in.data ipkg = pkg.get_protocol(ipv4.ipv4) if (ipkg): src = ipkg.src dst = ipkg.dst upkg = pkg.get_protocol(udp.udp) or pkg.get_protocol( tcp.tcp) #udp and tcp are not seperated if (upkg): dst_port = upkg.dst_port # print(ipkg) return src, dst, dst_port, dpid, pkg def add_flow(self, datapath, priority, match): pass def start_update(self): old = [0, 1, 2] new = [0, 3, 2] src = 0 dst = 2 self.cal_update(src, dst, old, new)
def test_singleton(self): manager_a = BufferManager() manager_b = BufferManager() self.assertTrue(manager_a is manager_b)
def test_detach(self): manager = BufferManager() manager.get_file_block('foo', 0) manager.detach_from_file('foo') self.assertFalse(manager._blocks)
def __init__(self, Node, index_file_path, node, key_position): self.Node = Node self.index_file_path = index_file_path self.node = node self.key_position = key_position self.manager = BufferManager()
class ObjectManager: def __init__(self): self.console = None self.__objTemplatesWatingVertices_l = [] self.__objTemplatesWatingTextrue_l = [] self.__objectTemplates_d = {} self.bufferManager = BufferManager() self.texMan = TextureManager() self.__toHereQueue = Queue() self.__toProcessQueue = Queue() self.objectLoader = ObjectLoader(self.__toHereQueue, self.__toProcessQueue) self.__haveThingsToGetFromProcess_i = 0 def update(self) -> None: self.__fillObjectTemplateWithTexture() if self.__haveThingsToGetFromProcess_i: self.__popObjectBlueprintFromProcess() def terminate(self): try: self.objectLoader.terminate() except AttributeError: pass try: self.texMan.terminate() except AssertionError: pass def runProcesses(self): self.objectLoader.start() def requestObject(self, objInitInfo:"ObjectInitInfo") -> Optional[ds.Object]: self.update() if objInitInfo.objTemplateName_s in self.__objectTemplates_d.keys(): if self.__objectTemplates_d[objInitInfo.objTemplateName_s] is None: return None else: objTemplate = self.__objectTemplates_d[objInitInfo.objTemplateName_s] obj = objTemplate.makeObject( objInitInfo.name_s, objInitInfo.level, objInitInfo.initPos_t, objInitInfo.static_b, objInitInfo.colliders_l, objInitInfo.colGroupTargets_l ) self.console.appendLogs("Instancing object: {} ({})".format(objTemplate.templateName_s, objTemplate.refCount_i)) return obj else: print("Object never created:", objInitInfo.objTemplateName_s) raise FileNotFoundError def dumpObjects(self, objTempNames_l:list) -> None: for objTempName_s in objTempNames_l: objTemplate = self.__objectTemplates_d[objTempName_s] objTemplate.refCount_i -= 1 tempName_s = objTemplate.templateName_s self.console.appendLogs("Deleted an object instance: '{}' ({})".format(objTemplate.templateName_s, objTemplate.refCount_i)) if objTemplate.refCount_i <= 0: trashes_l = objTemplate.terminate() for trash_t in trashes_l: self.bufferManager.dumpVertexArray(trash_t[0]) self.bufferManager.dumpBuffer(trash_t[1]) self.bufferManager.dumpBuffer(trash_t[2]) self.bufferManager.dumpBuffer(trash_t[3]) self.texMan.dump(trash_t[4]) del self.__objectTemplates_d[objTempName_s], objTemplate self.console.appendLogs( "Deleted ObjectTemplate: '{}'".format(tempName_s) ) def giveObjectDefineBlueprint(self, objBprint:ObjectDefineBlueprint) -> None: if objBprint.name_s in self.__objectTemplates_d.keys(): if self.__objectTemplates_d[objBprint.name_s] is not None: raise FileExistsError(objBprint.name_s) else: self.__objectTemplates_d[objBprint.name_s] = None objTemp = ObjectTemplate( objBprint.name_s, [self.__makeRendererFromBprint(xx) for xx in objBprint.rendererBlueprints_l], objBprint.colliders_l, objBprint.boundingBox ) self.console.appendLogs("Created an object template: '{}'".format(objTemp.templateName_s)) self.__objTemplatesWatingTextrue_l.append(objTemp) def giveObjectObjStaticBlueprint(self, objBprint:ObjectObjStaticBlueprint) -> None: if objBprint.objFileName_s in self.__objectTemplates_d.keys(): return self.__objectTemplates_d[objBprint.objFileName_s] = None a = self.__findObjMtlDir(objBprint.objFileName_s) if a is None: self.console.appendLogs( "Failed to obj file: '{}'".format(objBprint.objFileName_s) ) else: self.__toProcessQueue.put( (objBprint.objFileName_s, *a) ) self.__haveThingsToGetFromProcess_i += 1 def __fillObjectTemplateWithTexture(self): for x in range(len(self.__objTemplatesWatingTextrue_l) - 1, -1, -1): failedCount_i = 0 objTemplate = self.__objTemplatesWatingTextrue_l[x] # ObjectTemplate for y in range(len(objTemplate.renderers_l) - 1, -1, -1): aRenderer = objTemplate.renderers_l[y] if aRenderer.diffuseMap_i is None: result = self.texMan.request(aRenderer.diffuseMapName_s) if result is None: failedCount_i += 1 continue else: aRenderer.diffuseMap_i = result continue else: pass if failedCount_i <= 0: self.__objectTemplates_d[objTemplate.templateName_s] = objTemplate del self.__objTemplatesWatingTextrue_l[x] def __popObjectBlueprintFromProcess(self): try: objBprint = self.__toHereQueue.get_nowait() except Empty: return None else: self.giveObjectDefineBlueprint(objBprint) def __makeRendererFromBprint(self, renBprint:RendererBlueprint) -> ds.Renderer: newRenderer = ds.Renderer(renBprint.name_s, renBprint.initPos_t, renBprint.static_b); vramUsage_i = 0; ######## VAO ######## newRenderer.vao_i = self.bufferManager.requestVertexArray(); gl.glBindVertexArray(newRenderer.vao_i); ######## Vertex buffer ######## size = renBprint.vertexNdarray.size * renBprint.vertexNdarray.itemsize; newRenderer.vertexSize_i = renBprint.vertexNdarray.size // 3; vramUsage_i += size; newRenderer.vertexArrayBuffer_i = self.bufferManager.requestBuffer(); gl.glBindBuffer(gl.GL_ARRAY_BUFFER, newRenderer.vertexArrayBuffer_i); gl.glBufferData(gl.GL_ARRAY_BUFFER, size, renBprint.vertexNdarray, gl.GL_STATIC_DRAW); gl.glVertexAttribPointer(0, 3, gl.GL_FLOAT, gl.GL_FALSE, 0, None) gl.glEnableVertexAttribArray(0); ######## Texture coord buffer ######## size = renBprint.texCoordNdarray.size * renBprint.texCoordNdarray.itemsize; vramUsage_i += size; newRenderer.textureArrayBuffer_i = self.bufferManager.requestBuffer(); gl.glBindBuffer(gl.GL_ARRAY_BUFFER, newRenderer.textureArrayBuffer_i); # Bind the buffer. gl.glBufferData(gl.GL_ARRAY_BUFFER, size, renBprint.texCoordNdarray, gl.GL_STATIC_DRAW); # Allocate memory. gl.glVertexAttribPointer(1, 2, gl.GL_FLOAT, gl.GL_FALSE, 0, None); # Defines vertex attributes. What are those? gl.glEnableVertexAttribArray(1); ######## ######## size = renBprint.normalNdarray.size * renBprint.normalNdarray.itemsize; vramUsage_i += size; newRenderer.normalArrayBuffe_i = self.bufferManager.requestBuffer(); gl.glBindBuffer(gl.GL_ARRAY_BUFFER, newRenderer.normalArrayBuffe_i); # Bind the buffer. gl.glBufferData(gl.GL_ARRAY_BUFFER, size, renBprint.normalNdarray, gl.GL_STATIC_DRAW); # Allocate memory. gl.glVertexAttribPointer(2, 3, gl.GL_FLOAT, gl.GL_FALSE, 0, None); # Defines vertex attributes. What are those? gl.glEnableVertexAttribArray(2); ######## ######## newRenderer.diffuseMapName_s = renBprint.textureDir_s; newRenderer.textureVerNum_f = renBprint.textureVerNum_f; newRenderer.textureHorNum_f = renBprint.textureHorNum_f; newRenderer.shininess_f = renBprint.shininess_f; newRenderer.specularStrength_f = renBprint.specularStrength_f; newRenderer.vramUsage_i = vramUsage_i; return newRenderer; @staticmethod def __findObjMtlDir(objFileName_s:str) -> Optional[ Tuple[str, str] ]: objFileDir_s = "" mtlFileDir_s = "" for folderDir_s, _, files_l in os.walk(const.MODEL_DIR_s): for file_s in files_l: if objFileDir_s and mtlFileDir_s: break if file_s == objFileName_s + ".obj": if objFileDir_s: raise FileExistsError("There are multiple '{}' files.".format(objFileName_s)) else: objFileDir_s = "{}\\{}".format(folderDir_s, file_s) continue elif file_s == objFileName_s + ".mtl": if mtlFileDir_s: raise FileExistsError("There are multiple '{}' files.".format(mtlFileDir_s)) else: mtlFileDir_s = "{}\\{}".format(folderDir_s, file_s) continue if objFileDir_s and mtlFileDir_s: return objFileDir_s, mtlFileDir_s else: return None
def quit(): buffer_manager = BufferManager() buffer_manager.flush_all()
class Record: # The format of header should be the same for all records files. header_format = '<ii' # will be confirmed by RecordManager header_struct = Struct(header_format) def __init__(self, file_path, fmt): self.buffer_manager = BufferManager() self.filename = file_path # Each record in file has 2 extra info: next's record_off and valid bit self.record_struct = Struct(fmt + 'ci') self.first_free_rec, self.rec_tail = self._parse_header() def insert(self, attributes): """Insert the given record""" record_info = convert_str_to_bytes(attributes) + ( b'1', -1) # valid bit, next free space self.first_free_rec, self.rec_tail = self._parse_header() if self.first_free_rec >= 0: # There are space in free list first_free_blk, local_offset = self._calc(self.first_free_rec) block = self.buffer_manager.get_file_block(self.filename, first_free_blk) with pin(block): data = block.read() records = self._parse_block_data(data, first_free_blk) next_free_rec = records[self.first_free_rec][-1] records[local_offset] = record_info new_data = self._generate_new_data(records, first_free_blk) block.write(new_data) position = self.first_free_rec self.first_free_rec = next_free_rec else: # No space in free list, append the new record to the end of file self.rec_tail += 1 block_offset, local_offset = self._calc(self.rec_tail) block = self.buffer_manager.get_file_block(self.filename, block_offset) with pin(block): data = block.read() records = self._parse_block_data(data, block_offset) records.append(record_info) new_data = self._generate_new_data(records, block_offset) block.write(new_data) position = self.rec_tail self._update_header() return position def remove(self, record_offset): """Remove the record at specified position and update the free list""" self.first_free_rec, self.rec_tail = self._parse_header() block_offset, local_offset = self._calc(record_offset) block = self.buffer_manager.get_file_block(self.filename, block_offset) with pin(block): data = block.read() records = self._parse_block_data(data, block_offset) try: records[local_offset][-1] except IndexError: raise IndexError('The offset points to an empty space') if records[local_offset][-2] == b'0': raise RuntimeError('Cannot remove an empty record') records[local_offset][ -1] = self.first_free_rec # A positive number, putting this position into free list records[local_offset][-2] = b'0' self.first_free_rec = record_offset # update the head of free list new_data = self._generate_new_data(records, block_offset) block.write(new_data) self._update_header() def modify(self, attributes, record_offset): """Modify the record at specified offset""" block_offset, local_offset = self._calc(record_offset) block = self.buffer_manager.get_file_block(self.filename, block_offset) record_info = convert_str_to_bytes(attributes) + ( b'1', -1) # Updated record must be real with pin(block): data = block.read() records = self._parse_block_data(data, block_offset) if records[local_offset][-2] == b'0': raise RuntimeError('Cannot update an empty record') records[local_offset] = record_info new_data = self._generate_new_data(records, block_offset) block.write(new_data) def read(self, record_offset): """ Return the record at the corresponding position """ block_offset, local_offset = self._calc(record_offset) block = self.buffer_manager.get_file_block(self.filename, block_offset) with pin(block): data = block.read() records = self._parse_block_data(data, block_offset) if records[local_offset][-2] == b'0': raise RuntimeError('Cannot read an empty record') return convert_bytes_to_str(tuple(records[local_offset][:-2])) def scanning_select(self, conditions): # condition should be a dict: { attribute offset : {operator : value } } total_blk = self._calc(self.rec_tail)[0] + 1 result_set = [] for block_offset in range(total_blk): block = self.buffer_manager.get_file_block(self.filename, block_offset) records = self._parse_block_data(block.read(), block_offset) result_set += tuple([ convert_bytes_to_str(record[:-2]) for record in records if self._check_condition(record, conditions) is True ]) return result_set def scanning_delete(self, conditions): total_blk = self._calc(self.rec_tail)[0] + 1 record_offset = 0 for block_offset in range(total_blk): block = self.buffer_manager.get_file_block(self.filename, block_offset) records = self._parse_block_data(block.read(), block_offset) for i, record in enumerate(records): if self._check_condition(convert_bytes_to_str(record), conditions): records[i][-2] = b'0' records[i][-1] = self.first_free_rec self.first_free_rec = record_offset record_offset += 1 block.write(self._generate_new_data(records, block_offset)) self._update_header() def scanning_update(self, conditions, attributes): # The file header won't change when updating total_blk = self._calc(self.rec_tail)[0] + 1 new_record = convert_str_to_bytes(attributes) + (b'1', -1) for block_offset in range(total_blk): block = self.buffer_manager.get_file_block(self.filename, block_offset) records = self._parse_block_data(block.read(), block_offset) for i, record in enumerate(records): if self._check_condition(convert_bytes_to_str(record), conditions): records[i] = new_record block.write(self._generate_new_data(records, block_offset)) def _calc(self, record_offset): rec_per_blk = BufferManager.block_size // self.record_struct.size rec_first_blk = (BufferManager.block_size - self.header_struct.size) // self.record_struct.size if record_offset < rec_first_blk: # in 1st block return 0, record_offset else: # not in 1st block block_offset = (record_offset - rec_first_blk) // rec_per_blk + 1 local_offset = record_offset - rec_first_blk - (block_offset - 1) * rec_per_blk return block_offset, local_offset @staticmethod def _check_condition(record, conditions): if record[ -2] == b'0': # check the valid bit, return false when meet empty record return False str_record = convert_bytes_to_str(record[:-2]) for position, condition in conditions.items(): value = str_record[position] for operator_type, value_restriction in condition.items(): if operator_type == '=': if value != value_restriction: return False elif operator_type == '>': if value <= value_restriction: return False elif operator_type == '<': if value >= value_restriction: return False return True def _generate_new_data(self, records, blk_offset): if blk_offset == 0: data = bytearray(self.header_struct.size) else: data = bytearray() for r in records: data += self.record_struct.pack(*r) return data def _parse_block_data(self, data, blk_offset): upper_bound = len(data) if (upper_bound - self.header_struct.size) % self.record_struct.size != 0: upper_bound -= self.record_struct.size if blk_offset == 0: # is the first block, need to consider the header lower_bound = self.header_struct.size else: # not the first block, all data are records lower_bound = 0 records = [ list(self.record_struct.unpack_from(data, offset)) for offset in range(lower_bound, upper_bound, self.record_struct.size) ] return records def _parse_header(self): # Parse the file header, refresh corresponding info # and return the info with a tuple block = self.buffer_manager.get_file_block(self.filename, 0) # Get the first block with pin(block): data = block.read() header_info = self.header_struct.unpack_from(data, 0) return header_info def _update_header(self): # Update the file header after modifying the records block = self.buffer_manager.get_file_block(self.filename, 0) with pin(block): data = block.read() header_info = (self.first_free_rec, self.rec_tail) data[:self.header_struct.size] = self.header_struct.pack( *header_info) block.write(data)
from buffer_manager import BufferManager manager = BufferManager()
def __init__(self, file_path, fmt): self.buffer_manager = BufferManager() self.filename = file_path # Each record in file has 2 extra info: next's record_off and valid bit self.record_struct = Struct(fmt + 'ci') self.first_free_rec, self.rec_tail = self._parse_header()