def test_0_normal_ops(self): shm_mgr = SharedMemoryManager(config, sensitive=False) shm_mgr.connect('test') print('conn_id: ', shm_mgr.current_connection.conn_id) self.assertIsInstance(shm_mgr.current_connection.conn_id, str) for td in DATA: print(f'\n==================={td["name"]}===================\n') # try to remove this KEY first resp = shm_mgr.clean_key(KEY) if not resp.get('succeeded'): self.assertEqual(resp.get('rcode'), ReturnCodes.KEY_ERROR) resp = shm_mgr.create_key(KEY, td['type'], td['2_create']) print('\n-----------Create-----------') print(resp) self.assertTrue(resp.get('succeeded')) resp = shm_mgr.read_key(KEY) print('\n-----------Read-----------') print(resp) self.assertTrue(resp.get('succeeded')) resp = shm_mgr.add_value(KEY, td['2_add']) print('\n-----------Add-----------') print(resp) self.assertTrue(resp.get('succeeded')) resp = shm_mgr.read_key(KEY) print('\n-----------Read-----------') print(resp) self.assertTrue(resp.get('succeeded')) resp = shm_mgr.remove_value(KEY, td['2_remove']) print('\n-----------Remove-----------') print(resp) self.assertTrue(resp.get('succeeded')) resp = shm_mgr.read_key(KEY) print('\n-----------Read-----------') print(resp) self.assertTrue(resp.get('succeeded')) self.assertEqual( resp.get('value')[td['remaining_key']], td['remaining'], ) shm_mgr.disconnect() self.assertEqual(shm_mgr.current_connection, None)
def test_999_backlog(self): global do_not_kill_shm_worker key = 'bl0' shm_mgr = SharedMemoryManager(config, sensitive=False) shm_mgr.connect('test_bl') shm_mgr.create_key(key, SHMContainerTypes.LIST) shm_mgr.lock_key(key) pid = os.fork() if pid == -1: raise OSError('fork failed') if pid == 0: # This socket will cause a warning, so we close it here shm_mgr.current_connection.socket.close() do_not_kill_shm_worker = True shm_mgr1 = SharedMemoryManager(config, sensitive=False) shm_mgr1.connect('test_bl1') print('\n\n==============access-locked-container===============') t0 = time.time() resp = shm_mgr1.read_key(key) t1 = time.time() print(resp) delay = t1 - t0 print(f'delay: {delay}') self.assertEqual(resp.get('succeeded'), True) self.assertTrue(delay >= 1) shm_mgr1.disconnect() else: time.sleep(1) print('------------release-lock-------------') shm_mgr.unlock_key(key) shm_mgr.disconnect() os.waitpid(-1, 0)
def test_1_set(self): KEY_FOR_SET = 'set_test' VALUE_FOR_SET = 'aaaaaaaaaaaaa' print('\n\n=====================set-value====================') shm_mgr = SharedMemoryManager(config, sensitive=False) shm_mgr.connect('test_set') print('conn_id: ', shm_mgr.current_connection.conn_id) self.assertIsInstance(shm_mgr.current_connection.conn_id, str) resp = shm_mgr.create_key(KEY_FOR_SET, SHMContainerTypes.STR) self.assertTrue(resp.get('succeeded')) resp = shm_mgr.set_value(KEY_FOR_SET, VALUE_FOR_SET) self.assertTrue(resp.get('succeeded')) resp = shm_mgr.read_key(KEY_FOR_SET) self.assertTrue(resp.get('succeeded')) self.assertEqual(resp.get('value'), VALUE_FOR_SET) print(resp) shm_mgr.disconnect() self.assertEqual(shm_mgr.current_connection, None)
class BaseCore(): ''' The base model of cores Literally, the core is supposed to be a kernel-like component. It organizes other components to work together and administrate them. Some components are plugable, and some others are necessary. Here is the list of all components: afferents in neverland.afferents, plugable efferents in neverland.efferents, necessary logic handlers in neverland.logic, necessary protocol wrappers in neverland.protocol, necessary In the initial version, all these components are necessary, and afferents could be multiple. ''' EV_MASK = select.EPOLLIN SHM_SOCKET_NAME_TEMPLATE = 'SHM-Core-%d.socket' # SHM container for containing allocated core id # data structure: # [1, 2, 3, 4] SHM_KEY_CORE_ID = 'Core_id' # The shared status of cluster controlling, # enumerated in neverland.core.state.ClusterControllingStates SHM_KEY_CC_STATE = 'Core_CCState' def __init__( self, config, efferent, logic_handler, protocol_wrapper, main_afferent, minor_afferents=tuple(), ): ''' constructor :param config: the config :param efferent: an efferent instance :param logic_handler: a logic handler instance :param protocol_wrapper: a protocol wrapper instance :param main_afferent: the main afferent :param minor_afferents: a group of minor afferents, any iterable type contains afferent instances ''' self.__running = False self.core_id = None self._epoll = select.epoll() self.afferent_mapping = {} self.config = config self.main_afferent = main_afferent self.efferent = efferent self.logic_handler = logic_handler self.protocol_wrapper = protocol_wrapper self.shm_mgr = SharedMemoryManager(self.config) self.plug_afferent(self.main_afferent) for afferent in minor_afferents: self.plug_afferent(afferent) def set_cc_state(self, status): self.shm_mgr.set_value(self.SHM_KEY_CC_STATE, status) def get_cc_state(self): resp = self.shm_mgr.read_key(self.SHM_KEY_CC_STATE) return resp.get('value') @property def cc_state(self): return self.get_cc_state() def init_shm(self): self.shm_mgr.connect(self.SHM_SOCKET_NAME_TEMPLATE % NodeContext.pid) self.shm_mgr.create_key_and_ignore_conflict( self.SHM_KEY_CORE_ID, SHMContainerTypes.LIST, ) self.shm_mgr.create_key_and_ignore_conflict( self.SHM_KEY_CC_STATE, SHMContainerTypes.INT, CCStates.INIT, ) logger.debug(f'init_shm for core of worker {NodeContext.pid} has done') def close_shm(self): self.shm_mgr.disconnect() def self_allocate_core_id(self): ''' Let the core pick up an id for itself ''' try: resp = self.shm_mgr.lock_key(self.SHM_KEY_CORE_ID) except SHMResponseTimeout: # Currently, SHM_MAX_BLOCKING_TIME is 4 seconds and # these works can be definitely done in 4 seconds. # If a SHMResponseTimeout occurred, then there must # be a deadlock raise SHMResponseTimeout( f'deadlock of key: {self.SHM_KEY_CORE_ID}') resp = self.shm_mgr.read_key(self.SHM_KEY_CORE_ID) allocated_id = resp.get('value') if len(allocated_id) == 0: id_ = 0 else: last_id = allocated_id[-1] id_ = last_id + 1 self.shm_mgr.add_value( self.SHM_KEY_CORE_ID, [id_], ) self.core_id = id_ self.shm_mgr.unlock_key(self.SHM_KEY_CORE_ID) logger.debug( f'core of worker {NodeContext.pid} has self-allocated id: {id_}') def plug_afferent(self, afferent): self._epoll.register(afferent.fd, self.EV_MASK) self.afferent_mapping.update({afferent.fd: afferent}) def unplug_afferent(self, fd): ''' remove an afferent from the core :param fd: the file discriptor of the afferent, int ''' if fd not in self.afferent_mapping: return self._epoll.unregister(fd) self.afferent_mapping.pop(fd) def request_to_join_cluster(self): ''' send a request of the node is going to join the cluster ''' entrance = self.config.cluster_entrance identification = self.config.net.identification if entrance is None: raise ConfigError("cluster_entrance is not defined") if identification is None: raise ConfigError("identification is not defined") logger.info('Trying to join cluster...') content = { 'identification': identification, 'ip': get_localhost_ip(), 'listen_port': self.config.net.aff_listen_port, } subject = CCSubjects.JOIN_CLUSTER dest = (entrance.ip, entrance.port) pkt = UDPPacket() pkt.fields = ObjectifiedDict( type=PktTypes.CTRL, dest=dest, subject=subject, content=content, ) pkt.next_hop = dest pkt = self.protocol_wrapper.wrap(pkt) NodeContext.pkt_mgr.repeat_pkt(pkt) logger.info( f'Sending request to cluster entrance {entrance.ip}:{entrance.port}' ) logger.info('[Node Status] WAITING_FOR_JOIN') self.set_cc_state(CCStates.WAITING_FOR_JOIN) def request_to_leave_cluster(self): ''' send a request of the node is going to detach from the cluster ''' entrance = self.config.cluster_entrance identification = self.config.net.identification if entrance is None: raise ConfigError("cluster_entrance is not defined") if identification is None: raise ConfigError("identification is not defined") logger.info('Trying to leave cluster...') content = {"identification": identification} subject = CCSubjects.LEAVE_CLUSTER dest = (entrance.ip, entrance.port) pkt = UDPPacket() pkt.fields = ObjectifiedDict( type=PktTypes.CTRL, dest=dest, subject=subject, content=content, ) pkt.next_hop = dest pkt = self.protocol_wrapper.wrap(pkt) NodeContext.pkt_mgr.repeat_pkt(pkt) logger.info( f'Sent request to cluster entrance {entrance.ip}:{entrance.port}') logger.info('[Node Status] WAITING_FOR_LEAVE') self.set_cc_state(CCStates.WAITING_FOR_LEAVE) def handle_pkt(self, pkt): pkt = self.protocol_wrapper.unwrap(pkt) if not pkt.valid: return try: pkt = self.logic_handler.handle_logic(pkt) except DropPacket: return pkt = self.protocol_wrapper.wrap(pkt) self.efferent.transmit(pkt) def _poll(self): events = self._epoll.poll(POLL_TIMEOUT) for fd, evt in events: afferent = self.afferent_mapping[fd] if evt & select.EPOLLERR: self.unplug_afferent(fd) afferent.destroy() elif evt & select.EPOLLIN: pkt = afferent.recv() self.handle_pkt(pkt) def run(self): self.set_cc_state(CCStates.WORKING) self.__running = True self.main_afferent.listen() addr = self.main_afferent.listen_addr port = self.main_afferent.listen_port logger.info(f'Main afferent is listening on {addr}:{port}') while self.__running: self._poll() def run_for_a_while(self, duration=None, polling_times=None): ''' run the core within the specified duration time or poll times :param duration: the duration time, seconds in int :param polling_times: times the _poll method shall be invoked, int These 2 arguments will not work together, if duration specified, the poll_times will be ignored ''' if duration is None and polling_times is None: raise ArgumentError('no argument passed') self.main_afferent.listen() addr = self.main_afferent.listen_addr port = self.main_afferent.listen_port logger.info(f'Main afferent is listening on {addr}:{port}') if duration is not None: starting_time = time.time() while time.time() - starting_time <= duration: self._poll() elif polling_times is not None: for _ in polling_times: self._poll() def shutdown(self): self.__running = False
def test_2_rcode(self): shm_mgr = SharedMemoryManager(config, sensitive=False) shm_mgr.connect('test_err') print('\n\n=====================key-error-test====================') resp = shm_mgr.add_value( key='not_exists', value='a', backlogging=False, ) print(resp) self.assertEqual(resp.get('succeeded'), False) self.assertEqual(resp.get('rcode'), ReturnCodes.KEY_ERROR) print('\n\n=====================type-error-test====================') resp = shm_mgr.create_key( 'testing', SHMContainerTypes.DICT, value='a', backlogging=False, ) print(resp) self.assertEqual(resp.get('succeeded'), False) self.assertEqual(resp.get('rcode'), ReturnCodes.TYPE_ERROR) print( '\n\n===================unlock-not-locked-test==================') resp = shm_mgr.unlock_key('testing') print(resp) self.assertEqual(resp.get('succeeded'), True) self.assertEqual(resp.get('rcode'), ReturnCodes.NOT_LOCKED) print('\n\n=====================lock-test====================') print('------create-container------') resp = shm_mgr.create_key( 'testing', SHMContainerTypes.LIST, backlogging=False, ) self.assertEqual(resp.get('succeeded'), True) self.assertEqual(resp.get('rcode'), ReturnCodes.OK) print('------lock-container------') resp = shm_mgr.lock_key('testing') print(resp) self.assertEqual(resp.get('succeeded'), True) self.assertEqual(resp.get('rcode'), ReturnCodes.OK) print('------access-locked-container------') shm_mgr1 = SharedMemoryManager(config, sensitive=False) shm_mgr1.connect('test_err_1') resp = shm_mgr1.add_value( 'testing', [1, 2, 3], backlogging=False, ) print(resp) self.assertEqual(resp.get('succeeded'), False) self.assertEqual(resp.get('rcode'), ReturnCodes.LOCKED) print('------unlock-container----------') resp = shm_mgr.unlock_key('testing') print(resp) self.assertEqual(resp.get('succeeded'), True) self.assertEqual(resp.get('rcode'), ReturnCodes.OK) print('------access-locked-container-again------') resp = shm_mgr1.add_value( 'testing', [1, 2, 3], backlogging=False, ) print(resp) self.assertEqual(resp.get('succeeded'), True) self.assertEqual(resp.get('rcode'), ReturnCodes.OK) shm_mgr.disconnect() shm_mgr1.disconnect()
class SpecialPacketManager(): SHM_SOCKET_NAME_TEMPLATE = 'SHM-SpecialPacketManager-%d.socket' def __init__(self, config): self.config = config self.pid = NodeContext.pid self.shm_key_pkts = SHM_KEY_PKTS # These containers are for the SpecialPacketRepeater, the repeater # will also access special packets by the manager. self.shm_key_pkts_to_repeat = SHM_KEY_TMP_PKTS_TO_REPEAT % self.pid self.shm_key_last_repeat_time = SHM_KEY_TMP_LAST_REPEAT_TIME % self.pid self.shm_key_next_repeat_time = SHM_KEY_TMP_NEXT_REPEAT_TIME % self.pid self.shm_key_max_repeat_times = SHM_KEY_TMP_MAX_REPEAT_TIMES % self.pid self.shm_key_repeated_times = SHM_KEY_TMP_REPEATED_TIMES % self.pid def init_shm(self): ''' initialize the shared memory manager ''' self.shm_mgr = SharedMemoryManager(self.config) self.shm_mgr.connect(self.SHM_SOCKET_NAME_TEMPLATE % self.pid) self.shm_mgr.create_key_and_ignore_conflict( self.shm_key_pkts, SHMContainerTypes.DICT, ) self.shm_mgr.create_key_and_ignore_conflict( self.shm_key_pkts_to_repeat, SHMContainerTypes.LIST, ) self.shm_mgr.create_key_and_ignore_conflict( self.shm_key_last_repeat_time, SHMContainerTypes.DICT, ) self.shm_mgr.create_key_and_ignore_conflict( self.shm_key_next_repeat_time, SHMContainerTypes.DICT, ) self.shm_mgr.create_key_and_ignore_conflict( self.shm_key_max_repeat_times, SHMContainerTypes.DICT, ) self.shm_mgr.create_key_and_ignore_conflict( self.shm_key_repeated_times, SHMContainerTypes.DICT, ) def close_shm(self): self.shm_mgr.disconnect() def store_pkt(self, pkt, need_repeat=False, max_rpt_times=5): sn = pkt.fields.sn type_ = pkt.fields.type # The salt field is bytes, so we cannot serialize it in a JSON. # So, we shall encode it into a base64 string before store it. fields = pkt.fields.__to_dict__() salt = fields.get('salt') if salt is not None: salt_b64 = base64.b64encode(salt).decode() fields.update(salt=salt_b64) previous_hop = list(pkt.previous_hop) next_hop = list(pkt.next_hop) if sn is None: raise InvalidPkt( 'Packets to be stored must contain a serial number') value = { sn: { 'type': type_, 'fields': fields, 'previous_hop': previous_hop, 'next_hop': next_hop, } } self.shm_mgr.lock_key(self.shm_key_pkts) self.shm_mgr.add_value(self.shm_key_pkts, value) self.shm_mgr.unlock_key(self.shm_key_pkts) if need_repeat: self.shm_mgr.add_value(self.shm_key_pkts_to_repeat, [sn]) self.set_pkt_max_repeat_times(sn, max_rpt_times) self.set_pkt_repeated_times(sn, 0) hex_type = Converter.int_2_hex(type_) logger.debug(f'Stored a special packet, need_repeat: {need_repeat}, ' f'sn: {sn}, type: {hex_type}, dest: {pkt.fields.dest}') def get_pkt(self, sn): shm_data = self.shm_mgr.get_value(self.shm_key_pkts, sn) shm_value = shm_data.get('value') if shm_value is None: return None # and here, we restore the base64 encoded salt into bytes fields = shm_value.get('fields') salt_b64 = fields.get('salt') salt = base64.b64decode(salt_b64) fields.update(salt=salt) return UDPPacket( fields=fields, type=shm_value.get('type'), previous_hop=shm_value.get('previous_hop'), next_hop=shm_value.get('next_hop'), ) def remove_pkt(self, sn): self.cancel_repeat(sn) self.shm_mgr.lock_key(self.shm_key_pkts) self.shm_mgr.remove_value(self.shm_key_pkts, sn) self.shm_mgr.unlock_key(self.shm_key_pkts) logger.debug(f'Removed a special packet, sn: {sn}') def cancel_repeat(self, sn): self.shm_mgr.remove_value(self.shm_key_pkts_to_repeat, sn) self.shm_mgr.remove_value(self.shm_key_last_repeat_time, sn) self.shm_mgr.remove_value(self.shm_key_next_repeat_time, sn) self.shm_mgr.remove_value(self.shm_key_max_repeat_times, sn) self.shm_mgr.remove_value(self.shm_key_repeated_times, sn) logger.debug(f'Cancelled repeat for a packet, sn: {sn}') def repeat_pkt(self, pkt, max_rpt_times=5): self.store_pkt(pkt, need_repeat=True, max_rpt_times=max_rpt_times) def get_repeating_sn_list(self): shm_data = self.shm_mgr.read_key(self.shm_key_pkts_to_repeat) return shm_data.get('value') def set_pkt_last_repeat_time(self, sn, timestamp): self.shm_mgr.add_value(self.shm_key_last_repeat_time, {sn: timestamp}) logger.debug(f'set_pkt_last_repeat_time, sn: {sn}, ts: {timestamp}') def get_pkt_last_repeat_time(self, sn): shm_data = self.shm_mgr.get_value(self.shm_key_last_repeat_time, sn) return shm_data.get('value') def set_pkt_next_repeat_time(self, sn, timestamp): self.shm_mgr.add_value(self.shm_key_next_repeat_time, {sn: timestamp}) logger.debug(f'set_pkt_next_repeat_time, sn: {sn}, ts: {timestamp}') def get_pkt_next_repeat_time(self, sn): shm_data = self.shm_mgr.get_value(self.shm_key_next_repeat_time, sn) return shm_data.get('value') def set_pkt_max_repeat_times(self, sn, times): self.shm_mgr.add_value(self.shm_key_max_repeat_times, {sn: times}) logger.debug(f'set_pkt_max_repeat_times, sn: {sn}, times: {times}') def get_pkt_max_repeat_times(self, sn): shm_data = self.shm_mgr.get_value(self.shm_key_max_repeat_times, sn) return shm_data.get('value') def set_pkt_repeated_times(self, sn, times): self.shm_mgr.add_value(self.shm_key_repeated_times, {sn: times}) logger.debug(f'set_pkt_repeated_times, sn: {sn}, times: {times}') def get_pkt_repeated_times(self, sn): shm_data = self.shm_mgr.get_value(self.shm_key_repeated_times, sn) return shm_data.get('value') def increase_pkt_repeated_times(self, sn): repeated_times = self.get_pkt_repeated_times(sn) if repeated_times is None: repeated_times = 1 else: repeated_times += 1 self.set_pkt_repeated_times(sn, repeated_times)