def on_packet_recvchallenge(self, data):
if chr(data[0]) != 'n':
return self.protocol_error("Handshake 'n' packet expected")
self.peer_distr_version_ = (data[1], data[2])
self.peer_flags_ = util.u32(data, 3)
challenge = util.u32(data, 7)
self.peer_name_ = data[11:].decode("latin1")
self._send_challenge_reply(challenge)
self.state_ = self.RECV_CHALLENGE_ACK
return True
def on_packet_challengereply(self, data):
if data[0] != ord('r'):
return self.protocol_error(
"Unexpected packet (expecting CHALLENGE_REPLY) %s" % data)
peers_challenge = util.u32(data, 1)
peer_digest = data[5:]
LOG.info("challengereply: peer's challenge %s", peers_challenge)
my_cookie = self.get_node().node_opts_.cookie_
if not self.check_digest(digest=peer_digest,
challenge=self.my_challenge_,
cookie=my_cookie):
return self.protocol_error(
"Disallowed node connection (check the cookie)")
self._send_challenge_ack(peers_challenge, my_cookie)
self.packet_len_size_ = 4
self.state_ = self.CONNECTED
self.report_dist_connected()
# TODO: start timer with node_opts_.network_tick_time_
LOG.info("Incoming established with %s", self.peer_name_)
return True
def binary_to_term(data: bytes, options: dict = None) -> (any, bytes):
""" Strip 131 header and unpack if the data was compressed.
:param data: The incoming encoded data with the 131 byte
:param options:
* "atom": "str" | "bytes" | "Atom" (default "Atom").
Returns atoms as strings, as bytes or as atom.Atom objects.
* "byte_string": "str" | "bytes" (default "str").
Returns 8-bit strings as Python str or bytes.
:raises PyCodecError: when the tag is not 131, when compressed
data is incomplete or corrupted
:returns: Remaining unconsumed bytes
"""
if options is None:
options = {}
if data[0] != ETF_VERSION_TAG:
raise PyCodecError("Unsupported external term version")
if data[1] == TAG_COMPRESSED:
do = decompressobj()
decomp_size = util.u32(data, 2)
decomp = do.decompress(data[6:]) + do.flush()
if len(decomp) != decomp_size:
# Data corruption?
raise PyCodecError("Compressed size mismatch with actual")
return binary_to_term_2(decomp, options)
return binary_to_term_2(data[1:], options)
def on_packet_recvname(self, data) -> bool:
""" Handle RECV_NAME command, the first packet in a new connection. """
if data[0] != ord('n'):
return self.protocol_error(
"Unexpected packet (expecting RECV_NAME)")
# Read peer distribution version and compare to ours
peer_max_min = (data[1], data[2])
if dist_protocol.dist_version_check(peer_max_min):
return self.protocol_error(
"Dist protocol version have: %s got: %s" %
(str(dist_protocol.DIST_VSN_PAIR), str(peer_max_min)))
self.peer_distr_version_ = peer_max_min
self.peer_flags_ = util.u32(data[3:7])
self.peer_name_ = data[7:].decode("latin1")
LOG.info("RECV_NAME: %s %s", self.peer_distr_version_, self.peer_name_)
# Report
self._send_packet2(b"sok")
self.my_challenge_ = int(random.random() * 0x7fffffff)
self._send_challenge(self.my_challenge_)
self.state_ = self.WAIT_CHALLENGE_REPLY
return True
def on_packet_recvname(self, data: bytes) -> bytes:
""" Handle RECV_NAME command, the first packet in a new connection. """
if not data.startswith(b'n'):
self.protocol_error("Unexpected packet (expecting RECV_NAME)")
# raise
# Read peer dist_proto version and compare to ours
peer_max_min = (data[1], data[2])
#if version.dist_version_check(peer_max_min):
if not version.check_valid_dist_version(peer_max_min):
self.protocol_error(
"Dist protocol version have: %s got: %s" %
(str(version.DIST_VSN_PAIR), str(peer_max_min)))
# raise
self.peer_distr_version_ = peer_max_min
self.peer_flags_ = util.u32(data[3:7])
self.peer_name_ = data[7:].decode("latin1")
LOG.info("RECV_NAME: %s %s", self.peer_distr_version_, self.peer_name_)
# Report
self._send_packet2(b'sok')
self.my_challenge_ = int(random.random() * 0x7fffffff)
self._send_challenge(self.my_challenge_)
self.state_ = self.WAIT_CHALLENGE_REPLY
return b'' # assume everything is consumed
def _data_received_inner(self) -> bool:
if len(self.unconsumed_data_) < self.packet_len_size_:
# Not ready yet, keep reading
return False
# Dist protocol switches from 2 byte packet length to 4 at some point
if self.packet_len_size_ == 2:
pkt_size = util.u16(self.unconsumed_data_, 0)
offset = 2
else:
pkt_size = util.u32(self.unconsumed_data_, 0)
offset = 4
if len(self.unconsumed_data_) < self.packet_len_size_ + pkt_size:
# Length is already visible but the data is not here yet
return False
#packet = self.unconsumed_data_[offset:]
packet = self.unconsumed_data_[offset:(offset + pkt_size)]
self.unconsumed_data_ = self.unconsumed_data_[(offset + pkt_size):]
# Try to consume some data, remember the unconsumed tail
# Loop while the data is consumed, stop when not consumed anymore
self.on_packet(packet)
LOG.debug("unconsumed: %s, state=%s", self.unconsumed_data_,
self.state_)
return True
def on_incoming_data(self, data: bytes) -> Union[bytes, None]:
if len(data) < self.packet_len_size_:
# Not ready yet, keep reading
return data
# Dist protocol switches from 2 byte packet length to 4 at some point
if self.packet_len_size_ == 2:
pkt_size = util.u16(data, 0)
offset = 2
else:
pkt_size = util.u32(data, 0)
offset = 4
if len(data) < self.packet_len_size_ + pkt_size:
# Length is already visible but the data is not here yet
return data
packet = data[offset:(offset + pkt_size)]
if self.on_packet(packet):
return data[(offset + pkt_size):]
# Protocol error has occured and instead we return None to request
# connection close
return None
def binary_to_term(data: bytes, options: dict = None) -> (any, bytes):
""" Strip 131 header and unpack if the data was compressed.
:param data: The incoming encoded data with the 131 byte
:param options:
* "atom": "str" | "bytes" | "Atom" | "StrictAtom" (default
"Atom"). Returns atoms as strings, as bytes or as atom.Atom
objects.
* "byte_string": "str" | "bytes" | "int_list" (default "str").
Returns 8-bit strings as Python str or bytes or list of integers.
* "decode_hook" : callable. Python function called for each
decoded term.
:raises PyCodecError: when the tag is not 131, when compressed
data is incomplete or corrupted
:returns: Remaining unconsumed bytes
"""
if options is None:
options = {}
if data[0] != ETF_VERSION_TAG:
raise PyCodecError("Unsupported external term version")
if data[1] == TAG_COMPRESSED:
do = decompressobj()
decomp_size = util.u32(data, 2)
decode_data = do.decompress(data[6:]) + do.flush()
if len(decode_data) != decomp_size:
# Data corruption?
raise PyCodecError("Compressed size mismatch with actual")
else:
decode_data = data[1:]
decode_hook = options.get('decode_hook', {})
val, rem = binary_to_term_2(decode_data, options)
type_name_ref = type(val).__name__
if type_name_ref in decode_hook:
return decode_hook.get(type_name_ref)(val), rem
else:
return val, rem
def binary_to_term_2(data: bytes, options: dict = None) -> (any, bytes):
""" Proceed decoding after leading tag has been checked and removed.
Erlang lists are decoded into ``term.List`` object, whose ``elements_``
field contains the data, ``tail_`` field has the optional tail and a
helper function exists to assist with extracting an unicode string.
Atoms are decoded to :py:class:`~Term.atom.Atom` or optionally to bytes
or to strings.
Pids are decoded into :py:class:`~Term.pid.Pid`.
Refs decoded to and :py:class:`~Term.reference.Reference`.
Maps are decoded into Python ``dict``.
Binaries are decoded into ``bytes``
object and bitstrings into a pair of ``(bytes, last_byte_bits:int)``.
:param options: dict(str, _);
* "atom": "str" | "bytes" | "Atom"; default "Atom".
Returns atoms as strings, as bytes or as atom.Atom class objects
* "byte_string": "str" | "bytes" (default "str").
Returns 8-bit strings as Python str or bytes.
:param data: Bytes containing encoded term without 131 header
:rtype: Tuple[Value, Tail: bytes]
:return: The function consumes as much data as
possible and returns the tail. Tail can be used again to parse
another term if there was any.
:raises PyCodecError(str): on various errors
"""
if options is None:
options = {}
create_atom_fn = _get_create_atom_fn(options.get("atom", "Atom"))
create_str_fn = _get_create_str_fn(options.get("byte_string", "str"))
tag = data[0]
if tag in [TAG_ATOM_EXT, TAG_ATOM_UTF8_EXT]:
len_data = len(data)
if len_data < 3:
return incomplete_data("decoding length for an atom name")
len_expected = util.u16(data, 1) + 3
if len_expected > len_data:
return incomplete_data("decoding text for an atom")
name = data[3:len_expected]
enc = 'latin-1' if tag == TAG_ATOM_EXT else 'utf8'
return _bytes_to_atom(name, enc, create_atom_fn), data[len_expected:]
if tag in [TAG_SMALL_ATOM_EXT, TAG_SMALL_ATOM_UTF8_EXT]:
len_data = len(data)
if len_data < 2:
return incomplete_data("decoding length for a small-atom name")
len_expected = data[1] + 2
name = data[2:len_expected]
enc = 'latin-1' if tag == TAG_SMALL_ATOM_EXT else 'utf8'
return _bytes_to_atom(name, enc, create_atom_fn), data[len_expected:]
if tag == TAG_NIL_EXT:
return [], data[1:]
if tag == TAG_STRING_EXT:
len_data = len(data)
if len_data < 3:
return incomplete_data("decoding length for a string")
len_expected = util.u16(data, 1) + 3
if len_expected > len_data:
return incomplete_data()
return create_str_fn(data[3:len_expected]), data[len_expected:]
if tag == TAG_LIST_EXT:
if len(data) < 5:
return incomplete_data("decoding length for a list")
len_expected = util.u32(data, 1)
result_l = []
tail = data[5:]
while len_expected > 0:
term1, tail = binary_to_term_2(tail)
result_l.append(term1)
len_expected -= 1
# Read list tail and set it
list_tail, tail = binary_to_term_2(tail)
if list_tail == NIL:
return result_l, tail
return (result_l, list_tail), tail
if tag == TAG_SMALL_TUPLE_EXT:
if len(data) < 2:
return incomplete_data("decoding length for a small tuple")
len_expected = data[1]
result_t = []
tail = data[2:]
while len_expected > 0:
term1, tail = binary_to_term_2(tail)
result_t.append(term1)
len_expected -= 1
return tuple(result_t), tail
if tag == TAG_LARGE_TUPLE_EXT:
if len(data) < 5:
return incomplete_data("decoding length for a large tuple")
len_expected = util.u32(data, 1)
result_lt = []
tail = data[5:]
while len_expected > 0:
term1, tail = binary_to_term_2(tail)
result_lt.append(term1)
len_expected -= 1
return tuple(result_lt), tail
if tag == TAG_SMALL_INT:
if len(data) < 2:
return incomplete_data("decoding a 8-bit small uint")
return data[1], data[2:]
if tag == TAG_INT:
if len(data) < 5:
return incomplete_data("decoding a 32-bit int")
return util.i32(data, 1), data[5:]
if tag == TAG_PID_EXT:
node, tail = binary_to_term_2(data[1:])
id1 = util.u32(tail, 0)
serial = util.u32(tail, 4)
creation = tail[8]
assert isinstance(node, Atom)
pid = Pid(node_name=node.text_,
id=id1,
serial=serial,
creation=creation)
return pid, tail[9:]
if tag == TAG_NEW_REF_EXT:
if len(data) < 2:
return incomplete_data("decoding length for a new-ref")
term_len = util.u16(data, 1)
node, tail = binary_to_term_2(data[3:])
creation = tail[0]
id_len = 4 * term_len
id1 = tail[1:id_len + 1]
ref = Reference(node_name=node.text_, creation=creation, refid=id1)
return ref, tail[id_len + 1:]
if tag == TAG_MAP_EXT:
if len(data) < 5:
return incomplete_data("decoding length for a map")
len_expected = util.u32(data, 1)
result_m = {}
tail = data[5:]
while len_expected > 0:
term1, tail = binary_to_term_2(tail)
term2, tail = binary_to_term_2(tail)
result_m[term1] = term2
len_expected -= 1
return result_m, tail
if tag == TAG_BINARY_EXT:
len_data = len(data)
if len_data < 5:
return incomplete_data("decoding length for a binary")
len_expected = util.u32(data, 1) + 5
if len_expected > len_data:
return incomplete_data("decoding data for a binary")
# Returned as Python `bytes`
return data[5:len_expected], data[len_expected:]
if tag == TAG_BIT_BINARY_EXT:
len_data = len(data)
if len_data < 6:
return incomplete_data("decoding length for a bit-binary")
len_expected = util.u32(data, 1) + 6
lbb = data[5]
if len_expected > len_data:
return incomplete_data("decoding data for a bit-binary")
# Returned as tuple `(bytes, last_byte_bits:int)`
return (data[6:len_expected], lbb), data[len_expected:]
if tag == TAG_NEW_FLOAT_EXT:
(result_f, ) = struct.unpack(">d", data[1:9])
return result_f, data[9:]
if tag == TAG_SMALL_BIG_EXT:
nbytes = data[1]
# Data is encoded little-endian as bytes (least significant first)
in_bytes = data[3:(3 + nbytes)]
# NOTE: int.from_bytes is Python 3.2+
result_bi = int.from_bytes(in_bytes, byteorder='little')
if data[2] != 0:
result_bi = -result_bi
return result_bi, data[3 + nbytes:]
if tag == TAG_LARGE_BIG_EXT:
nbytes = util.u32(data, 1)
# Data is encoded little-endian as bytes (least significant first)
in_bytes = data[6:(6 + nbytes)]
# NOTE: int.from_bytes is Python 3.2+
result_lbi = int.from_bytes(in_bytes, byteorder='little')
if data[5] != 0:
result_lbi = -result_lbi
return result_lbi, data[6 + nbytes:]
if tag == TAG_NEW_FUN_EXT:
# size = util.u32(data, 1)
arity = data[5]
uniq = data[6:22]
index = util.u32(data, 22)
num_free = util.u32(data, 26)
(mod, tail) = binary_to_term_2(data[30:])
(old_index, tail) = binary_to_term_2(tail)
(old_uniq, tail) = binary_to_term_2(tail)
(pid, tail) = binary_to_term_2(tail)
free_vars = []
while num_free > 0:
(v, tail) = binary_to_term_2(tail)
free_vars.append(v)
num_free -= 1
return Fun(mod=mod,
arity=arity,
pid=pid,
index=index,
uniq=uniq,
old_index=old_index,
old_uniq=old_uniq,
free=free_vars), tail
raise PyCodecError("Unknown tag %d" % data[0])
