def parse_data_var(pos, msg):
size = varint.decode_bytes(msg[pos:])
size_bytes_len = len(varint.encode(size))
pos = pos + size_bytes_len
data = msg[pos:pos + size]
pos = pos + size
return pos, data
def parse_data(buff):
if len(buff) < MAGIC_PACKET_LENGTH + HEADER_ELEMENTS_NUM:
return False
if buf[:MAGIC_PACKET_LENGTH] != MAGIC_PACKET:
return False
decoded_data_list = []
s = MAGIC_PACKET_LENGTH
e = MAGIC_PACKET_LENGTH
element_count = 0
while True:
if serial_data[e] <= 127: # int
b = serial_data[s:e + 1] # slicing to parse byte data
if isinstance(b, bytes):
decoded_data = varint.decode_bytes(b)
decoded_data_list.append(decoded_data)
s = e + 1
element_count += 1
else:
# something to do like handling error
return False
if element_count == HEADER_ELEMENTS_NUM:
break
e += 1
message_header = MessageHeader(*decoded_data_list)
message_body = serial_data[s:]
return message_header, message_body
def decode(b):
# Decode the KeyLength (signed Varint, ZigZag encoded)
klen = varint.decode_bytes(b)
klen = (klen >> 1) ^ -(klen & 1)
# Decode the Key
print klen
print val2fmt(klen)
ksize = struct.calcsize("> {} {}s".format(val2fmt(klen), klen))
(_, k) = struct.unpack("> {} {}s".format(val2fmt(klen), klen), b[:ksize])
# Decode the ValueLength (signed Varint, ZigZag encoded)
vlen = varint.decode_bytes(b[ksize:])
vlen = (vlen >> 1) ^ -(vlen & 1)
# Decode the Value
vsize = struct.calcsize("> {} {}s".format(val2fmt(vlen), vlen))
(_, v) = struct.unpack("> {} {}s".format(val2fmt(vlen), vlen),
b[ksize:ksize + vsize])
# Return the decoded RecordHeader
return (RecordHeader(k, v), ksize + vsize)
def extract_prefix(bytes_):
"""
Extracts the prefix from multicodec prefixed data
:param bytes bytes_: multicodec prefixed data
:return: prefix for the prefixed data
:rtype: bytes
:raises ValueError: when incorrect varint is provided
"""
try:
return varint.decode_bytes(bytes_)
except TypeError:
raise ValueError('incorrect varint provided')
def receive(socket):
bytez = socket.recv(2)
if bytez == '':
return None
length = varint.decode_bytes(bytez)
if length <= 127:
bytez = bytez[1:]
else:
bytez = bytes()
bytez = bytez + socket.recv(length - len(bytez))
msg = ProtocolMessage_pb2.ProtocolMessage()
msg.ParseFromString(bytez)
return msg
def decompress(inbar):
i = 0
outbar = bytearray()
while i < len(inbar):
repcount = varint.decode_bytes(inbar[i:])
while inbar[i] >= 128:
i += 1
i += 1
if repcount:
while repcount:
outbar.append(inbar[i])
repcount -= 1
i += 1
else:
repcount = varint.decode_bytes(inbar[i:])
while inbar[i] >= 128:
i += 1
i += 1
while repcount:
outbar.append(inbar[i])
i += 1
repcount -= 1
return (outbar)
def decode(b):
# Decode the Length
length = varint.decode_bytes(b)
size = varint_size_map[31 - clz(length)]
length = (length >> 1) ^ -(length & 1)
# Verfiy the Length of the Record
if length + size > len(b):
raise RecordInvalidLengthException(\
"BatchLength {} exceeds record length {}".format( \
length, len(b)))
# Encode the Attributes (unused)
nsize = struct.calcsize("> b")
attr = struct.unpack("> b", b[size:size+nsize])
size += nsize
# Decode the TimestampDelta
ts_delta = varint.decode_bytes(b[size:])
size += varint_size_map[31 - clz(ts_delta)]
ts_delta = (ts_delta >> 1) ^ -(ts_delta & 1)
# Decode the OffsetDelta
offs_delta = varint.decode_bytes(b[size:])
size += varint_size_map[31 - clz(offs_delta)]
offs_delta = (offs_delta >> 1) ^ -(offs_delta & 1)
# Decode the KeyLength and Key
klen = varint.decode_bytes(b[size:])
nsize = varint_size_map[31 - clz(klen)]
klen = (klen >> 1) ^ -(klen & 1)
size += nsize
(k, ) = struct.unpack("> {}s".format(klen), b[size:size+klen])
size += klen
# Decode the ValueLength and Value
vlen = varint.decode_bytes(b[size:])
nsize = varint_size_map[31 - clz(vlen)]
vlen = (vlen >> 1) ^ -(vlen & 1)
size += nsize
(v, ) = struct.unpack("> {}s".format(vlen), b[size:size+vlen])
size += vlen
# Decode the number of Headers
nhdrs = varint.decode_bytes(b[size:])
nsize = varint_size_map[31 - clz(nhdrs)]
nhdrs = (nhdrs >> 1) ^ -(nhdrs & 1)
size += nsize
headers = []
for i in range(0, nhdrs):
(hdr, hdr_size) = record_header.decode(b[size:])
headers.append(hdr)
size += hdr_size
return (Record(ts_delta, offs_delta, k, v, headers) ,size)
def _extract_pubkey(self, extra):
if extra:
if extra[0] == self.TX_EXTRA_TAG_PUBKEY:
extra = extra[1:]
self.offset += 1
extra = self._pop_pubkey(extra)
elif extra[0] == self.TX_EXTRA_TAG_ADDITIONAL_PUBKEYS:
extra = extra[1:]
self.offset += 1
keycount = varint.decode_bytes(bytearray(extra))
valen = len(varint.encode(keycount))
extra = extra[valen:]
self.offset += valen
for i in range(keycount):
extra = self._pop_pubkey(extra)
return extra
def data_received(self, data):
self.pending_data += data
while len(self.pending_data) > 0:
length = varint.decode_bytes(self.pending_data)
length_bytes = len(varint.encode(length))
if len(self.pending_data) < length + length_bytes:
break
data = self.pending_data[length_bytes:length_bytes + length]
self.pending_data = self.pending_data[length_bytes + length:]
msg = ProtocolMessage_pb2.ProtocolMessage()
decrypted = self.decrypt(data)
msg.ParseFromString(decrypted)
self.message_received(msg)
def test_varint_to_code():
cc = varint.decode_bytes(b'\x05')
assert cc == 5
cc = varint.decode_bytes(b'\x96\x01')
assert cc == 150
def varint_to_code(buf):
return varint.decode_bytes(binascii.unhexlify(buf))
def parse_number(pos, msg):
n = varint.decode_bytes(msg[pos:])
n_len = len(varint.encode(n))
pos = pos + n_len
return pos, n
def varint_to_code(buf):
return varint.decode_bytes(binascii.unhexlify(buf))