def __init__(self, **kwargs): for kw in ['previous_hop', 'next_hop']: if kw not in kwargs: kwargs.update({kw: (None, None)}) for kw in ['fields', 'byte_fields']: if kw not in kwargs: kwargs.update({kw: ObjectifiedDict()}) if 'valid' not in kwargs: kwargs.update(valid=None) ObjectifiedDict.__init__(self, **kwargs)
def request_to_leave_cluster(self): ''' send a request of the node is going to detach from the cluster ''' entrance = self.config.cluster_entrance identification = self.config.net.identification if entrance is None: raise ConfigError("cluster_entrance is not defined") if identification is None: raise ConfigError("identification is not defined") logger.info('Trying to leave cluster...') content = {"identification": identification} subject = CCSubjects.LEAVE_CLUSTER dest = (entrance.ip, entrance.port) pkt = UDPPacket() pkt.fields = ObjectifiedDict( type=PktTypes.CTRL, dest=dest, subject=subject, content=content, ) pkt.next_hop = dest pkt = self.protocol_wrapper.wrap(pkt) NodeContext.pkt_mgr.repeat_pkt(pkt) logger.info( f'Sent request to cluster entrance {entrance.ip}:{entrance.port}') logger.info('[Node Status] WAITING_FOR_LEAVE') self.set_cc_state(CCStates.WAITING_FOR_LEAVE)
def _gen_resp_pkt(self, content, dest): src = (NodeContext.local_ip, NodeContext.core.main_afferent.listen_port) resp_pkt = UDPPacket() resp_pkt.fields = ObjectifiedDict( type=PktTypes.CTRL, src=src, dest=dest, subject=CCSubjects.RESPONSE, content=content, ) relay_nodes = self.get_relay_list() # If we have relay nodes in our cluster, then just give the packet # to a relay node. If we have no relay node, then send the packet to # the requester directly. if len(relay_nodes) == 0: resp_pkt.next_hop = dest else: relay = relay_nodes[0] resp_pkt.next_hop = (relay.get('ip'), relay.get('port')) return resp_pkt
def test_1_wrap_unwrap(self): pkt = UDPPacket() pkt.fields = ObjectifiedDict( serial=1, type=PktTypes.CONN_CTRL, diverged=0x01, src=('127.0.0.1', 65535), dest=('127.0.0.1', 65535), iv_changed=0x01, iv_duration=10000, iv=b'iviviviviv' ) pkt.type = pkt.fields.type pkt = base_wrapper.wrap(pkt) print(pkt.data) print('==================\n') pkt1 = UDPPacket() # pkt1.type = 0x01 pkt1.data = pkt.data pkt1 = base_wrapper.unwrap(pkt1) self.assertEqual(pkt1.valid, True) self.assertEqual(pkt1.type, PktTypes.CONN_CTRL) self.assertEqual(pkt1.fields.src, pkt.fields.src) self.assertEqual(pkt1.fields.dest, pkt.fields.dest) print( str(pkt1.fields) )
def request_to_join_cluster(self): ''' send a request of the node is going to join the cluster ''' entrance = self.config.cluster_entrance identification = self.config.net.identification if entrance is None: raise ConfigError("cluster_entrance is not defined") if identification is None: raise ConfigError("identification is not defined") logger.info('Trying to join cluster...') content = { 'identification': identification, 'ip': get_localhost_ip(), 'listen_port': self.config.net.aff_listen_port, } subject = CCSubjects.JOIN_CLUSTER dest = (entrance.ip, entrance.port) pkt = UDPPacket() pkt.fields = ObjectifiedDict( type=PktTypes.CTRL, dest=dest, subject=subject, content=content, ) pkt.next_hop = dest pkt = self.protocol_wrapper.wrap(pkt) NodeContext.pkt_mgr.repeat_pkt(pkt) logger.info( f'Sending request to cluster entrance {entrance.ip}:{entrance.port}' ) logger.info('[Node Status] WAITING_FOR_JOIN') self.set_cc_state(CCStates.WAITING_FOR_JOIN)
import __code_path__ from neverland.utils import ObjectifiedDict from neverland.components.shm import ( SharedMemoryManager, SHMContainerTypes, ReturnCodes, ) json_config = { 'shm': { 'socket_dir': '/tmp/nl_shm_sock/', 'manager_socket_name': 'manager', } } config = ObjectifiedDict(**json_config) # clean the socket directory if os.path.isdir(config.shm.socket_dir): shutil.rmtree(config.shm.socket_dir) os.mkdir(config.shm.socket_dir) KEY = 'k0' DATA = [ { 'name': 'testing-list', 'type': SHMContainerTypes.LIST, '2_create': [1, 2, 3], '2_add': [4], '2_remove': [1, 2, 4], 'remaining': 3,
def new_conn(self, remote, synchronous=False, timeout=2, interval=0.1): ''' establish a new connection The establishing connection will be placed in slot-2. After the new connection is established, if we have established 2 connections with the specified node already then the connection in slot-0 will be removed and the connection in slot-1 will be moved to slot-0. The new connection will be placed in slot-1. :param remote: remote socket address, (ip, port) :param synchronous: If the sync argument is True, then the new_conn method will try to wait the connection complete and return a connection object. This operation will be blocking until it reaches the timeout or the connection completes. If the sync argument is False, then the new_conn method will return None immediately without waiting. :param timeout: seconds to timeout :param interval: the interval time of connection checking ''' usable_slots = self.get_usable_slots(remote) remote_name = self._remote_sa_2_key(remote) if SLOT_2 not in usable_slots: raise ConnSlotNotAvailable(f'slot-2 to {remote_name} is in using, ' f'cannot establish connection now') iv = os.urandom(self.iv_len) iv_duration = random.randint(*self.iv_duration_range) pkt = UDPPacket() pkt.fields = ObjectifiedDict( type=PktTypes.CONN_CTRL, dest=remote, communicating=1, iv_changed=1, iv_duration=iv_duration, iv=iv, ) pkt.next_hop = remote pkt = NodeContext.protocol_wrapper.wrap(pkt) NodeContext.pkt_mgr.repeat_pkt(pkt) conn_sn = NodeContext.id_generator.gen() conn = { "remote": { "ip": remote[0], "port": remote[1], }, "sn": conn_sn, "state": ConnStates.ESTABLISHING, "slot": SLOT_2, "iv": iv, "iv_duration": iv_duration, } conn = Connection(**conn) # though we have checked the SLOT_2 already, # but it still has the possibility... try: self.store_conn(conn, SLOT_2) except ConnSlotNotAvailable: logger.warn( f'slot-2 to {remote_name} seized, abort the establishment') # The request is sending, now we wait for the response if not synchronous: return None
def handle_request(self, data, data_parsed=False, backlogging=None): ''' handle the request :param data: the data of request :param data_parsed: Tell the method if the data has been parsed. If it has not been parsed then the data shall be bytes. If it has been parsed, then the data shall be an ObjectifiedDict :param backlogging: To override the backlogging option in the data. ''' if not data_parsed: try: data = json.loads(data.decode('utf-8')) if not isinstance(data, dict): raise ValueError except (UnicodeDecodeError, ValueError): # If this was the packet which we sent, then it could not # cause any of these errors, so we can simply ignore it. raise DropPacket data = ObjectifiedDict(**data) if not data.action in Actions: # same as above raise DropPacket if backlogging is None: backlogging = data.backlogging backlogging = True if backlogging is None else backlogging try: self.prehandle_lock(data) except SHMContainerLocked: if backlogging: sn = self.gen_backlog_sn() self.backlogged_requests.update({sn: data}) backlog_count = len(self.backlogged_requests) logger.debug(f'request backlogged, current: {backlog_count}') raise SHMRequestBacklogged else: return self._gen_response_json( conn_id=data.conn_id, succeeded=False, rcode=ReturnCodes.LOCKED, ) if data.action == Actions.CREATE: return self.handle_create(data) if data.action == Actions.READ: return self.handle_read(data) if data.action == Actions.SET: return self.handle_set(data) if data.action == Actions.ADD: return self.handle_add(data) if data.action == Actions.DICT_GET: return self.handle_dict_get(data) if data.action == Actions.DICT_UPDATE: return self.handle_dict_update(data) if data.action == Actions.CLEAN: return self.handle_clean(data) if data.action == Actions.REMOVE: return self.handle_remove(data) if data.action == Actions.LOCK: return self.handle_lock(data) if data.action == Actions.UNLOCK: return self.handle_unlock(data) if data.action == Actions.CONNECT: return self.handle_connect(data) if data.action == Actions.DISCONNECT: return self.handle_disconnect(data)
def parse_udp_pkt(self, pkt): ''' parse a raw UDP packet :param value: value of the field :return: (fields, byte_fields) ''' cur = 0 # cursor fields = ObjectifiedDict() byte_fields = ObjectifiedDict() # parse the header first for field_name, definition in self.header_fmt.__fmt__.items(): field_data = pkt.data[cur:cur + definition.length] # Packet too short, it must be invalid if len(field_data) == 0: raise InvalidPkt('packet too short') try: value = self._unpack_field(field_data, definition.type) except struct.error: raise InvalidPkt('unpack failed') fields.__update__(**{field_name: value}) byte_fields.__update__(**{field_name: field_data}) cur += definition.length body_type = fields.type body_fmt = self._body_fmt_mapping.get(body_type) if body_fmt is None: raise InvalidPkt('invalid type') # parse the body for field_name, definition in body_fmt.__fmt__.items(): field_data = pkt.data[cur:cur + definition.length] # Packet too short, it must be invalid if len(field_data) == 0: raise InvalidPkt('packet too short') try: value = self._unpack_field(field_data, definition.type) except struct.error: raise InvalidPkt('unpack failed') fields.__update__(**{field_name: value}) byte_fields.__update__(**{field_name: field_data}) cur += definition.length return fields, byte_fields
def test_1_to_dict(self): data = {'a': 1, 'b': {'c': 3, 'd': False}} od = ObjectifiedDict(**data) print(od) print(od.__to_dict__())
def test_0_set_get(self): od = ObjectifiedDict() od.a = 1 self.assertEqual(od.a, 1) self.assertEqual(getattr(od, 'a'), 1) print(f'od.a = {od.a}')