def __init__(self, **kwargs):
for kw in ['previous_hop', 'next_hop']:
if kw not in kwargs:
kwargs.update({kw: (None, None)})
for kw in ['fields', 'byte_fields']:
if kw not in kwargs:
kwargs.update({kw: ObjectifiedDict()})
if 'valid' not in kwargs:
kwargs.update(valid=None)
ObjectifiedDict.__init__(self, **kwargs)
def request_to_leave_cluster(self):
''' send a request of the node is going to detach from the cluster
'''
entrance = self.config.cluster_entrance
identification = self.config.net.identification
if entrance is None:
raise ConfigError("cluster_entrance is not defined")
if identification is None:
raise ConfigError("identification is not defined")
logger.info('Trying to leave cluster...')
content = {"identification": identification}
subject = CCSubjects.LEAVE_CLUSTER
dest = (entrance.ip, entrance.port)
pkt = UDPPacket()
pkt.fields = ObjectifiedDict(
type=PktTypes.CTRL,
dest=dest,
subject=subject,
content=content,
)
pkt.next_hop = dest
pkt = self.protocol_wrapper.wrap(pkt)
NodeContext.pkt_mgr.repeat_pkt(pkt)
logger.info(
f'Sent request to cluster entrance {entrance.ip}:{entrance.port}')
logger.info('[Node Status] WAITING_FOR_LEAVE')
self.set_cc_state(CCStates.WAITING_FOR_LEAVE)
def _gen_resp_pkt(self, content, dest):
src = (NodeContext.local_ip,
NodeContext.core.main_afferent.listen_port)
resp_pkt = UDPPacket()
resp_pkt.fields = ObjectifiedDict(
type=PktTypes.CTRL,
src=src,
dest=dest,
subject=CCSubjects.RESPONSE,
content=content,
)
relay_nodes = self.get_relay_list()
# If we have relay nodes in our cluster, then just give the packet
# to a relay node. If we have no relay node, then send the packet to
# the requester directly.
if len(relay_nodes) == 0:
resp_pkt.next_hop = dest
else:
relay = relay_nodes[0]
resp_pkt.next_hop = (relay.get('ip'), relay.get('port'))
return resp_pkt
def test_1_wrap_unwrap(self):
pkt = UDPPacket()
pkt.fields = ObjectifiedDict(
serial=1,
type=PktTypes.CONN_CTRL,
diverged=0x01,
src=('127.0.0.1', 65535),
dest=('127.0.0.1', 65535),
iv_changed=0x01,
iv_duration=10000,
iv=b'iviviviviv'
)
pkt.type = pkt.fields.type
pkt = base_wrapper.wrap(pkt)
print(pkt.data)
print('==================\n')
pkt1 = UDPPacket()
# pkt1.type = 0x01
pkt1.data = pkt.data
pkt1 = base_wrapper.unwrap(pkt1)
self.assertEqual(pkt1.valid, True)
self.assertEqual(pkt1.type, PktTypes.CONN_CTRL)
self.assertEqual(pkt1.fields.src, pkt.fields.src)
self.assertEqual(pkt1.fields.dest, pkt.fields.dest)
print(
str(pkt1.fields)
)
def request_to_join_cluster(self):
''' send a request of the node is going to join the cluster
'''
entrance = self.config.cluster_entrance
identification = self.config.net.identification
if entrance is None:
raise ConfigError("cluster_entrance is not defined")
if identification is None:
raise ConfigError("identification is not defined")
logger.info('Trying to join cluster...')
content = {
'identification': identification,
'ip': get_localhost_ip(),
'listen_port': self.config.net.aff_listen_port,
}
subject = CCSubjects.JOIN_CLUSTER
dest = (entrance.ip, entrance.port)
pkt = UDPPacket()
pkt.fields = ObjectifiedDict(
type=PktTypes.CTRL,
dest=dest,
subject=subject,
content=content,
)
pkt.next_hop = dest
pkt = self.protocol_wrapper.wrap(pkt)
NodeContext.pkt_mgr.repeat_pkt(pkt)
logger.info(
f'Sending request to cluster entrance {entrance.ip}:{entrance.port}'
)
logger.info('[Node Status] WAITING_FOR_JOIN')
self.set_cc_state(CCStates.WAITING_FOR_JOIN)
import __code_path__
from neverland.utils import ObjectifiedDict
from neverland.components.shm import (
SharedMemoryManager,
SHMContainerTypes,
ReturnCodes,
)
json_config = {
'shm': {
'socket_dir': '/tmp/nl_shm_sock/',
'manager_socket_name': 'manager',
}
}
config = ObjectifiedDict(**json_config)
# clean the socket directory
if os.path.isdir(config.shm.socket_dir):
shutil.rmtree(config.shm.socket_dir)
os.mkdir(config.shm.socket_dir)
KEY = 'k0'
DATA = [
{
'name': 'testing-list',
'type': SHMContainerTypes.LIST,
'2_create': [1, 2, 3],
'2_add': [4],
'2_remove': [1, 2, 4],
'remaining': 3,
def new_conn(self, remote, synchronous=False, timeout=2, interval=0.1):
''' establish a new connection
The establishing connection will be placed in slot-2.
After the new connection is established, if we have established 2
connections with the specified node already then the connection in
slot-0 will be removed and the connection in slot-1 will be moved
to slot-0. The new connection will be placed in slot-1.
:param remote: remote socket address, (ip, port)
:param synchronous:
If the sync argument is True, then the new_conn method
will try to wait the connection complete and return a
connection object. This operation will be blocking until
it reaches the timeout or the connection completes.
If the sync argument is False, then the new_conn method
will return None immediately without waiting.
:param timeout: seconds to timeout
:param interval: the interval time of connection checking
'''
usable_slots = self.get_usable_slots(remote)
remote_name = self._remote_sa_2_key(remote)
if SLOT_2 not in usable_slots:
raise ConnSlotNotAvailable(f'slot-2 to {remote_name} is in using, '
f'cannot establish connection now')
iv = os.urandom(self.iv_len)
iv_duration = random.randint(*self.iv_duration_range)
pkt = UDPPacket()
pkt.fields = ObjectifiedDict(
type=PktTypes.CONN_CTRL,
dest=remote,
communicating=1,
iv_changed=1,
iv_duration=iv_duration,
iv=iv,
)
pkt.next_hop = remote
pkt = NodeContext.protocol_wrapper.wrap(pkt)
NodeContext.pkt_mgr.repeat_pkt(pkt)
conn_sn = NodeContext.id_generator.gen()
conn = {
"remote": {
"ip": remote[0],
"port": remote[1],
},
"sn": conn_sn,
"state": ConnStates.ESTABLISHING,
"slot": SLOT_2,
"iv": iv,
"iv_duration": iv_duration,
}
conn = Connection(**conn)
# though we have checked the SLOT_2 already,
# but it still has the possibility...
try:
self.store_conn(conn, SLOT_2)
except ConnSlotNotAvailable:
logger.warn(
f'slot-2 to {remote_name} seized, abort the establishment')
# The request is sending, now we wait for the response
if not synchronous:
return None
def handle_request(self, data, data_parsed=False, backlogging=None):
''' handle the request
:param data: the data of request
:param data_parsed: Tell the method if the data has been parsed.
If it has not been parsed then the data shall
be bytes. If it has been parsed, then the data
shall be an ObjectifiedDict
:param backlogging: To override the backlogging option in the data.
'''
if not data_parsed:
try:
data = json.loads(data.decode('utf-8'))
if not isinstance(data, dict):
raise ValueError
except (UnicodeDecodeError, ValueError):
# If this was the packet which we sent, then it could not
# cause any of these errors, so we can simply ignore it.
raise DropPacket
data = ObjectifiedDict(**data)
if not data.action in Actions:
# same as above
raise DropPacket
if backlogging is None:
backlogging = data.backlogging
backlogging = True if backlogging is None else backlogging
try:
self.prehandle_lock(data)
except SHMContainerLocked:
if backlogging:
sn = self.gen_backlog_sn()
self.backlogged_requests.update({sn: data})
backlog_count = len(self.backlogged_requests)
logger.debug(f'request backlogged, current: {backlog_count}')
raise SHMRequestBacklogged
else:
return self._gen_response_json(
conn_id=data.conn_id,
succeeded=False,
rcode=ReturnCodes.LOCKED,
)
if data.action == Actions.CREATE:
return self.handle_create(data)
if data.action == Actions.READ:
return self.handle_read(data)
if data.action == Actions.SET:
return self.handle_set(data)
if data.action == Actions.ADD:
return self.handle_add(data)
if data.action == Actions.DICT_GET:
return self.handle_dict_get(data)
if data.action == Actions.DICT_UPDATE:
return self.handle_dict_update(data)
if data.action == Actions.CLEAN:
return self.handle_clean(data)
if data.action == Actions.REMOVE:
return self.handle_remove(data)
if data.action == Actions.LOCK:
return self.handle_lock(data)
if data.action == Actions.UNLOCK:
return self.handle_unlock(data)
if data.action == Actions.CONNECT:
return self.handle_connect(data)
if data.action == Actions.DISCONNECT:
return self.handle_disconnect(data)
def parse_udp_pkt(self, pkt):
''' parse a raw UDP packet
:param value: value of the field
:return: (fields, byte_fields)
'''
cur = 0 # cursor
fields = ObjectifiedDict()
byte_fields = ObjectifiedDict()
# parse the header first
for field_name, definition in self.header_fmt.__fmt__.items():
field_data = pkt.data[cur:cur + definition.length]
# Packet too short, it must be invalid
if len(field_data) == 0:
raise InvalidPkt('packet too short')
try:
value = self._unpack_field(field_data, definition.type)
except struct.error:
raise InvalidPkt('unpack failed')
fields.__update__(**{field_name: value})
byte_fields.__update__(**{field_name: field_data})
cur += definition.length
body_type = fields.type
body_fmt = self._body_fmt_mapping.get(body_type)
if body_fmt is None:
raise InvalidPkt('invalid type')
# parse the body
for field_name, definition in body_fmt.__fmt__.items():
field_data = pkt.data[cur:cur + definition.length]
# Packet too short, it must be invalid
if len(field_data) == 0:
raise InvalidPkt('packet too short')
try:
value = self._unpack_field(field_data, definition.type)
except struct.error:
raise InvalidPkt('unpack failed')
fields.__update__(**{field_name: value})
byte_fields.__update__(**{field_name: field_data})
cur += definition.length
return fields, byte_fields
def test_1_to_dict(self):
data = {'a': 1, 'b': {'c': 3, 'd': False}}
od = ObjectifiedDict(**data)
print(od)
print(od.__to_dict__())
def test_0_set_get(self):
od = ObjectifiedDict()
od.a = 1
self.assertEqual(od.a, 1)
self.assertEqual(getattr(od, 'a'), 1)
print(f'od.a = {od.a}')