def UnPack(src,key = b''):
if len(src) < 0x20:
raise RuntimeError('Unpack Error!Please check mm protocol!') #协议需要更新
return b''
if not key:
key = Util.sessionKey
#解析包头
nCur= 0
if src[nCur] == struct.unpack('>B',b'\xbf')[0]:
nCur += 1 #跳过协议标志位
nLenHeader = src[nCur] >> 2 #包头长度
bUseCompressed = (src[nCur] & 0x3 == 1) #包体是否使用压缩算法:01使用,02不使用
nCur += 1
nDecryptType = src[nCur] >> 4 #解密算法(固定为AES解密): 05 aes解密 / 07 rsa解密
nLenCookie = src[nCur] & 0xf #cookie长度
nCur += 1
nCur += 4 #服务器版本(当前固定返回4字节0)
uin= struct.unpack('>i',src[nCur:nCur+4])[0] #uin
nCur += 4
cookie_temp = src[nCur:nCur+nLenCookie] #cookie
if cookie_temp and not(cookie_temp == Util.cookie):
Util.cookie = cookie_temp #刷新cookie
nCur += nLenCookie
(nCgi,nCur) = decoder._DecodeVarint(src,nCur) #cgi type
(nLenProtobuf,nCur) = decoder._DecodeVarint(src,nCur) #压缩前protobuf长度
(nLenCompressed,nCur) = decoder._DecodeVarint(src,nCur) #压缩后protobuf长度
logger.debug('包头长度:{}\n是否使用压缩算法:{}\n解密算法:{}\ncookie长度:{}\nuin:{}\ncookie:{}\ncgi type:{}\nprotobuf长度:{}\n压缩后protobuf长度:{}'.format(nLenHeader, bUseCompressed, nDecryptType, nLenCookie, uin, str(Util.cookie), nCgi, nLenProtobuf, nLenCompressed))
#对包体aes解密解压缩
body = src[nLenHeader:] #取包体数据
if bUseCompressed:
protobufData = Util.decompress_and_aesDecrypt(body,key)
else:
protobufData = Util.aesDecrypt(body,key)
logger.debug('解密后数据:%s' % str(protobufData))
return protobufData
def UnPack(src,key):
if len(src) < 20:
return b''
#解析包头
nCur= 0
if src[nCur] == '\xbf':
nCur += 1 #跳过协议标志位
nLenHeader = src[nCur] >> 2 #包头长度
bUseCompressed = (src[nCur] & 0x3 == 1) #包体是否使用压缩算法:01使用,02不使用
nCur += 1
nDecryptType = src[nCur] >> 4 #解密算法(固定为AES解密): 05 aes解密 / 07 rsa解密
nLenCookie = src[nCur] & 0xf #cookie长度
nCur += 1
nCur += 4 #服务器版本(当前固定返回4字节0)
uin= struct.unpack('>I',src[nCur:nCur+4])[0] #uin
nCur += 4
cookie = src[nCur:nCur+nLenCookie] #cookie
nCur += nLenCookie
(nCgi,nCur) = decoder._DecodeVarint(src,nCur) #cgi type
(nLenProtobuf,nCur) = decoder._DecodeVarint(src,nCur) #压缩前protobuf长度
(nLenCompressed,nCur) = decoder._DecodeVarint(src,nCur) #压缩后protobuf长度
logger.debug('包头长度:{}\n是否使用压缩算法:{}\n解密算法:{}\ncookie长度:{}\nuin:{}\ncookie:{}\ncgi type:{}\nprotobuf长度:{}\n压缩后protobuf长度:{}'.format(nLenHeader, bUseCompressed, nDecryptType, nLenCookie, uin, str(cookie), nCgi, nLenProtobuf, nLenCompressed))
#对包体aes解密解压缩
body = src[nLenHeader:] #取包体数据
if bUseCompressed:
protobufData = decompress_and_aesDecrypt(body,key)
else:
protobufData = aesDecrypt(body,key)
logger.debug('解密后数据:%s' % str(protobufData))
return protobufData
def process_incoming_data(self):
message = ""
# First, get some data, and kick it back to the outer routine until we get something
message = message + (yield)
# The first part of the connection will be a 7 byte header identifying this as a Hadoop
# data stream. Verify that. (well, pretend to verify that)
while len(message) < 7:
message = message + (yield)
(header, v, a, i) = unpack_from("4sbbb", message[0:7])
logger.debug("Header decoded: %s %d %d %d" % (header, v, a, i))
message = message[7:]
# Next, we need at least 4 bytes to give us a notion of how much more to read
while len(message) < 4:
message = message + (yield)
size = unpack(">I", message[0:4])
message = message[4:]
# Now we know how much we need to read for the header
while len(message) < size[0]:
message = message + (yield)
ipcconn = IpcConnectionContext_pb2.IpcConnectionContextProto()
ipcconn.ParseFromString(message[0 : size[0]])
logger.info("Logging a connection:\n%s" % (str(ipcconn)))
message = message[size[0] :]
# From here on out, we'll be in an infinite loop, reading RPC Requests
while True:
while len(message) < 4:
message = message + (yield)
size = unpack(">I", message[0:4])
logger.debug("Waiting for at least %d for next RPC" % (size[0]))
message = message[4:]
while len(message) < size[0]:
message = message + (yield)
# OK, we've got enough for this message, let's decode it
rpchead = RpcPayloadHeader_pb2.RpcPayloadHeaderProto()
(headersize, headerposition) = decoder._DecodeVarint(message, 0)
rpchead.ParseFromString(message[headerposition : headerposition + headersize])
message = message[headerposition + headersize :]
(bodysize, bodyposition) = decoder._DecodeVarint(message, 0)
rpcreq = hadoop_rpc_pb2.HadoopRpcRequestProto()
rpcreq.ParseFromString(message[bodyposition : bodyposition + bodysize])
# Stash some information about this reqest so we can look it up on the flipside
logger.info("The Request is for callid %s, %s" % (str(rpchead.callId), rpcreq.methodName))
self.messages[rpchead.callId] = {"time": time.time(), "method": rpcreq.methodName}
message = message[size[0] :]
def _parse_protobuf(self, data):
hdHeader = HighDimHeader()
(length, start) = decoder._DecodeVarint(data, 0)
hdHeader.ParseFromString(data[start:start+length])
data = data[start+length:]
hdRows = []
n = len(data)
start = 0
while start < n:
(length, start) = decoder._DecodeVarint(data, start)
hdRow = Row()
hdRow.ParseFromString(data[start:start+length])
hdRows.append(hdRow)
start += length
return (hdHeader, hdRows)
def _parse_protobuf(self, data):
hdHeader = HighDimHeader()
(length, start) = decoder._DecodeVarint(data, 0)
hdHeader.ParseFromString(data[start:start + length])
data = data[start + length:]
hdRows = []
n = len(data)
start = 0
while start < n:
(length, start) = decoder._DecodeVarint(data, start)
hdRow = Row()
hdRow.ParseFromString(data[start:start + length])
hdRows.append(hdRow)
start += length
return (hdHeader, hdRows)
def _read_next_obj(self):
"""
Each object is encoded as:
- vtype (uint8_t): type of the object
- size (varint): size of the object as varint
- object itself
"""
bufsz = self._prevbuf + self._fobj.read(9 - len(self._prevbuf))
if len(bufsz) == 0:
return None, None
vtype = bufsz[0]
size, pos = _DecodeVarint(bufsz[1:], 0)
if pos + size < 8:
data = bufsz[1 + pos:1 + pos + size]
self._prevbuf = bufsz[1 + pos + size:]
elif pos + size < 8:
data = bufsz[1:]
self._prevbuf = b""
else:
data = bufsz[1 + pos:] + self._fobj.read(size - 8 + pos)
self._prevbuf = b""
return vtype, data
def __init__(self, msg):
def InternalAdd(field_number, wire_type, data):
unknown_field = UnknownField(field_number, wire_type, data)
self._values.append(unknown_field)
self._values = []
msg_des = msg.DESCRIPTOR
# pylint: disable=protected-access
unknown_fields = msg._unknown_fields
if (msg_des.has_options
and msg_des.GetOptions().message_set_wire_format):
local_decoder = decoder.UnknownMessageSetItemDecoder()
for _, buffer in unknown_fields:
(field_number, data) = local_decoder(memoryview(buffer))
InternalAdd(field_number,
wire_format.WIRETYPE_LENGTH_DELIMITED, data)
else:
for tag_bytes, buffer in unknown_fields:
# pylint: disable=protected-access
(tag, _) = decoder._DecodeVarint(tag_bytes, 0)
field_number, wire_type = wire_format.UnpackTag(tag)
if field_number == 0:
raise RuntimeError('Field number 0 is illegal.')
(data,
_) = decoder._DecodeUnknownField(memoryview(buffer), 0,
wire_type)
InternalAdd(field_number, wire_type, data)
def parse_next_message(self):
message = None
remainder = self.message_buffer
message_data = ""
# 1. decode a varint from the top of the stream
# 2. using that as the length, if there's enough in the buffer, try and
# decode try and decode a VehicleMessage after the varint
# 3. if it worked, great, we're oriented in the stream - continue
# 4. if either couldn't be parsed, skip to the next byte and repeat
while message is None and len(self.message_buffer) > 1:
message_length, message_start = _DecodeVarint(self.message_buffer, 0)
# sanity check to make sure we didn't parse some huge number that's
# clearly not the length prefix
if message_length > self.MAX_PROTOBUF_MESSAGE_LENGTH:
self.message_buffer = self.message_buffer[1:]
continue
if message_start + message_length > len(self.message_buffer):
break
message_data = self.message_buffer[message_start:message_start +
message_length]
remainder = self.message_buffer[message_start + message_length:]
message = ProtobufFormatter.deserialize(message_data)
if message is None:
self.message_buffer = self.message_buffer[1:]
self.message_buffer = remainder
return message
def decode_varint(data: bytes) -> int:
"""
Decode a protobuf varint to an int
"""
decoded: Tuple[int, int] = _DecodeVarint(data, 0)
assert isinstance(decoded[0], int)
return decoded[0]
def _to_sentences(self, blob):
from data.CoreNLP_pb2 import Document
d = Document()
from google.protobuf.internal.decoder import _DecodeVarint
s, p = _DecodeVarint(blob, 0)
d.ParseFromString(blob[p:p + s])
return d.sentence
def parse_next_message(self):
message = None
remainder = self.message_buffer
message_data = ""
# 1. decode a varint from the top of the stream
# 2. using that as the length, if there's enough in the buffer, try and
# decode try and decode a VehicleMessage after the varint
# 3. if it worked, great, we're oriented in the stream - continue
# 4. if either couldn't be parsed, skip to the next byte and repeat
while message is None and len(self.message_buffer) > 1:
message_length, message_start = _DecodeVarint(
self.message_buffer, 0)
# sanity check to make sure we didn't parse some huge number that's
# clearly not the length prefix
if message_length > self.MAX_PROTOBUF_MESSAGE_LENGTH:
self.message_buffer = self.message_buffer[1:]
continue
if message_start + message_length > len(self.message_buffer):
break
message_data = self.message_buffer[message_start:message_start +
message_length]
remainder = self.message_buffer[message_start + message_length:]
message = ProtobufFormatter.deserialize(message_data)
if message is None:
self.message_buffer = self.message_buffer[1:]
self.message_buffer = remainder
return message
def _extract_encoded_msg(encoded_tx):
decoded = base64.b64decode(encoded_tx)
msg_len, new_pos = _DecodeVarint(decoded, 0)
tx_type = decoded[new_pos:new_pos + 4]
stdtx_pb2 = dex_pb2.StdTx().FromString(decoded[new_pos + 4:])
return stdtx_pb2.msgs[0]
def Read(f):
"""Reads an event trace file.
Args:
f: The file to read.
Yields:
The events contained in the file.
"""
close = not hasattr(f, 'read')
if close:
f = open(f, 'r')
try:
while True:
pos = f.tell()
l_data = f.read(10) # 10 is always enough
l, l_sz = gpbd._DecodeVarint(l_data, 0)
f.seek(pos + l_sz)
data = f.read(l)
rec = event_pb2.FileRecord()
rec.ParseFromString(data)
for evt in rec.event:
yield evt
finally:
if close:
f.close()
def inext(self):
""" Read a single trace msg container object from the input
file.
The input file is a fixed-size (?!) file, where each
fixed-sized message
"""
if (self.position >= self.size):
raise StopIteration()
(size, self.position) = decoder._DecodeVarint(self.data, self.position)
raw_data = self.data[self.position:self.position + size]
self.position = self.position + size
trace_msg_container = tracemsg_pb2.TraceMsgContainer()
try:
trace_msg_container.ParseFromString(raw_data)
pass
except message.DecodeError as e:
trace_msg_container == None
raise
if trace_msg_container == None:
raise StopIteration()
else:
return trace_msg_container
def decode_uvarint(buf, pos):
"""Decode bytearray into a long."""
# Convert buffer to string
if six.PY2:
buf = str(buf)
value, pos = decoder._DecodeVarint(buf, pos)
return (value, pos)
def sentences_from(self, buf, offset=0):
self.protoc()
from .CoreNLP_pb2 import Document
d = Document()
from google.protobuf.internal.decoder import _DecodeVarint
size, pos = _DecodeVarint(buf, offset)
d.ParseFromString(buf[offset + pos:offset + pos + size])
return d.sentence
def walk_binary(binr):
if type(binr) == str:
with open(binr, 'rb') as fd:
binr = fd.read()
# Search for:
# ".proto" or ".protodevel", as part of the "name" (1) field
cursor = 0
while cursor < len(binr):
cursor = binr.find(b'.proto', cursor)
if cursor == -1:
break
cursor += len('.proto')
cursor += (binr[cursor:cursor + 5] == b'devel') * 5
# Search back for the (1, length-delimited) marker
start = binr.rfind(b'\x0a', max(cursor - 1024, 0), cursor)
if start > 0 and binr[start - 1] == 0x0a == (cursor - start - 1):
start -= 1
# Check whether length byte is coherent
if start == -1:
continue
varint, end = _DecodeVarint(binr, start + 1)
if cursor - end != varint:
continue
# Look just after for subsequent markers
tags = b'\x12\x1a\x22\x2a\x32\x3a\x42\x4a\x50\x58\x62'
if binr[cursor] not in tags:
continue
while cursor < len(binr) and binr[cursor] in tags:
tags = tags[tags.index(binr[cursor]):]
varint, end = _DecodeVarint(binr, cursor + 1)
cursor = end + varint * (binr[cursor] & 0b111 == 2)
# Parse descriptor
proto = FileDescriptorProto()
proto.ParseFromString(binr[start:cursor])
# Convert to ascii
yield descpb_to_proto(proto)
def DesempacotaMensagem(self, msg):
message = message_pb2.Message()
# Read length
(size, position) = decoder._DecodeVarint(msg, 0)
# Read the message
message.ParseFromString(msg[position:position + size])
return message.arguments
def _decode_varint(serial):
"""Decode varint as integer
"""
try:
i,count =_DecodeVarint(serial,0)
return i, serial[count:]
except IndexError:
return None, serial
def load_inode_section(self):
inode_section = fsimage_pb2.INodeSection()
(len, pos) = decoder._DecodeVarint(self.buf, self._start_pos)
inode_section_bytes = self.buf[pos:pos + len]
inode_section.ParseFromString(inode_section_bytes)
self.num_inodes = inode_section.numInodes
self.last_inode_id = inode_section.lastInodeId
self._cur_pos = pos + len
return inode_section
def read_delimited_message(cls, f):
t = f.tell()
# Though numbers can be longer, only read 4 bytes = 2**27 bytes
b = f.read(4)
l, p = decoder._DecodeVarint(b, 0)
# rewind the file to the correct position
f.seek(t + p)
return cls.FromString(f.read(l))
def op_friend_buf2resp(buf):
res = mm_pb2.oplog_resp()
res.ParseFromString(UnPack(buf))
try:
code,__ = decoder._DecodeVarint(res.res.code, 0)
if code:
logger.info('好友操作失败,错误码:0x{:x},错误信息:{}'.format(code, res.res.msg), 11)
except:
code = -1
logger.info('好友操作失败!', 11)
return code
def set_group_nick_name_buf2resp(buf):
res = mm_pb2.oplog_resp()
res.ParseFromString(UnPack(buf))
try:
code,__ = decoder._DecodeVarint(res.res.code, 0)
if code:
logger.info('设置群昵称失败,错误码:0x{:x},错误信息:{}'.format(code, res.res.msg), 11)
except:
code = -1
logger.info('设置群昵称失败!', 11)
return code
def extract_proto_from_file(filename, descriptor_pool):
with open(filename, "rb") as f:
data = f.read()
offset = 0
PROTO_MARKER = b".proto"
while True:
# Look for ".proto"
suffix_position = data.find(PROTO_MARKER, offset)
if suffix_position == -1:
break
marker_start = data.rfind(b"\x0A", offset, suffix_position)
if marker_start == -1:
# Doesn't look like a proto descriptor
offset = suffix_position + len(PROTO_MARKER)
continue
try:
name_length, new_pos = _DecodeVarint(data, marker_start)
except Exception:
# Expected a VarInt here, so if not, continue
offset = suffix_position + len(PROTO_MARKER)
continue
# Length = 1 byte for the marker (0x0A)
# + length of the varint + length of the descriptor name
expected_length = 1 + (new_pos - marker_start) + name_length + 7
current_length = (suffix_position + len(PROTO_MARKER)) - marker_start
# Huge margin of error here - my calculations above are probably just wrong.
if current_length > expected_length + 30:
offset = suffix_position + len(PROTO_MARKER)
continue
# Split the data starting at the marker byte and try to read it as a
# protobuf stream. Descriptors are stored as c strings in the .pb.cc files.
# They're null-terminated, but can also contain embedded null bytes. Since we
# can't search for the null-terminator explicitly, we parse the string manually
# until we reach a protobuf tag which equals 0 (identifier = 0, wiretype =
# varint), signalling the final null byte of the string. This works because
# there are no 0 tags in a real FileDescriptorProto stream.
descriptor_length = read_until_null_tag(data[marker_start:]) - 1
descriptor_data = data[marker_start:marker_start + descriptor_length]
try:
proto_file = ProtoFile(descriptor_data, descriptor_pool)
if (proto_file.path.endswith(".proto")
and proto_file.path != "google/protobuf/descriptor.proto"):
yield proto_file
except Exception:
pass
offset = marker_start + descriptor_length
def parseFromDelimitedString(obj, buf, offset=0):
"""
Stanford CoreNLP uses the Java "writeDelimitedTo" function, which
writes the size (and offset) of the buffer before writing the object.
This function handles parsing this message starting from offset 0.
@returns how many bytes of @buf were consumed.
"""
size, pos = _DecodeVarint(buf, offset)
obj.ParseFromString(buf[offset + pos:offset + pos + size])
return pos + size
def decode_uvarint(buf, pos):
"""Decode bytearray into a long."""
# Convert buffer to string
if six.PY2:
buf = str(buf)
try:
value, pos = decoder._DecodeVarint(buf, pos)
except (TypeError, IndexError) as exc:
six.raise_from(
DecoderException("Error decoding uvarint from %s..." %
binascii.hexlify(buf[pos:pos + 8])), exc)
return (value, pos)
def proto_load(f):
out = []
while True:
buf = f.read(4)
if not buf:
break
size, pos = _DecodeVarint(buf, 0)
buf = buf[pos:] + f.read(size - 4 + pos)
out.append(fpb.Thing.FromString(buf))
return out
def parse_next_message(self, from_u):
message = None
message_data = ""
message_length, message_start = _DecodeVarint(from_u, 0)
if message_start + message_length > len(from_u):
print "size error"
return
message_data = from_u[message_start:message_start + message_length]
print "vonxc returnvalue parse_next_message ", binascii.hexlify(
message_data)
message = self.deserialize(message_data)
return message
def readDelimited(sock):
"""Read a delmited protobuf from a socket"""
# FIXME - current implementation is very expensive in terms of heap thrash and it will fail for protobufs bigger than 32768 bytes
from google.protobuf.internal import decoder
delimsize = 2
delimbytes = sock.recv(delimsize)
(payloadsize, pos) = decoder._DecodeVarint(delimbytes, 0)
# We now know size of our msg, but we might have unused bytes from the delimeter
numextra = delimsize - pos
numtoread = payloadsize - numextra
payload = delimbytes[pos:] + sock.recv(numtoread, MSG_WAITALL)
msg = Envelope()
#print "payloadsize=%d, pos=%d, numextra=%d, numtoread=%d, paylen=%d" % (payloadsize, pos, numextra, numtoread, len(payload))
msg.ParseFromString(payload)
return msg
def approximate_size(self):
end = False
offset = 0
while not end and offset < len(self.data):
value, offset = _DecodeVarint(self.data, offset)
field = (value & 0xfffffff8) >> 3
wire_type = value & 0x7 # last 3 bits
# If we have seen certain fields already, we might have entered a next encoded protobuffer.
if (field not in self.KNOWN_FIELD_NUMBERS) or \
(field in self.no_double_fields and field in self.seen_fields):
end = True
break
self.seen_fields.append(field)
# Varint
if wire_type == 0:
_, offset = _DecodeVarint(self.data, offset)
# 64-bit
elif wire_type == 1:
offset += 8
# Length-delimited (~string/bytes)
elif wire_type == 2:
value, offset = _DecodeVarint(self.data, offset)
offset += value
# Groups - deprecated
elif wire_type == 3 or wire_type == 4:
continue
# 32-bit
elif wire_type == 5:
offset += 4
else:
end = True
self._size = offset
def parseFromDelimitedString(obj, buf, offset=0):
"""
Stanford CoreNLP uses the Java "writeDelimitedTo" function, which
writes the size (and offset) of the buffer before writing the object.
This function handles parsing this message starting from offset 0.
@returns how many bytes of @buf were consumed.
"""
size, pos = _DecodeVarint(buf, offset)
try:
obj.ParseFromString(buf[offset + pos:offset + pos + size])
except DecodeError as e:
warnings.warn("Failed to decode a serialized output from CoreNLP server. An incomplete or empty object will be returned.", \
RuntimeWarning)
return pos + size
def UnPack(src, key=b''):
if len(src) < 0x20:
raise RuntimeError('Unpack Error!Please check mm protocol!') # 协议需要更新
return b''
if not key:
key = Util.sessionKey
# 解析包头
nCur = 0
if src[nCur] == struct.unpack('>B', b'\xbf')[0]:
nCur += 1 # 跳过协议标志位
nLenHeader = src[nCur] >> 2 # 包头长度
bUseCompressed = (src[nCur] & 0x3 == 1) # 包体是否使用压缩算法:01使用,02不使用
nCur += 1
nDecryptType = src[nCur] >> 4 # 解密算法(固定为AES解密): 05 aes解密 / 07 rsa解密
nLenCookie = src[nCur] & 0xf # cookie长度
nCur += 1
nCur += 4 # 服务器版本(当前固定返回4字节0)
uin = struct.unpack('>i', src[nCur:nCur+4])[0] # uin
nCur += 4
cookie_temp = src[nCur:nCur+nLenCookie] # cookie
if cookie_temp and not(cookie_temp == Util.cookie):
Util.cookie = cookie_temp # 刷新cookie
nCur += nLenCookie
(nCgi, nCur) = decoder._DecodeVarint(src, nCur) # cgi type
(nLenProtobuf, nCur) = decoder._DecodeVarint(src, nCur) # 压缩前protobuf长度
(nLenCompressed, nCur) = decoder._DecodeVarint(src, nCur) # 压缩后protobuf长度
logger.debug('包头长度:{}\n是否使用压缩算法:{}\n解密算法:{}\ncookie长度:{}\nuin:{}\ncookie:{}\ncgi type:{}\nprotobuf长度:{}\n压缩后protobuf长度:{}'.format(
nLenHeader, bUseCompressed, nDecryptType, nLenCookie, uin, str(Util.cookie), nCgi, nLenProtobuf, nLenCompressed))
# 对包体aes解密解压缩
body = src[nLenHeader:] # 取包体数据
if bUseCompressed:
protobufData = Util.decompress_and_aesDecrypt(body, key)
else:
protobufData = Util.aesDecrypt(body, key)
logger.debug('解密后数据:%s' % str(protobufData))
return protobufData
def read_proto(file, message):
"""
read a protobuffer struct from file, the length of the struct is stored as
a varint, then followed by the actual struct data.
@return True success, False for end of file
"""
buf = file.read(8)
if not buf:
return False
result, pos = _DecodeVarint(buf, 0)
buf = buf[pos:] + file.read(result - len(buf) + pos)
message.ParseFromString(buf)
return True
def recvRpcMessage(self, sock):
'''Handle reading an RPC reply from the server.'''
try:
rfile = sock.makefile('r')
""" modified by wuzhw """
recv = rfile.read(1)
(size, position) = decoder._DecodeVarint(recv, 0)
byte_stream = rfile.read(size)
""" Original: """
# byte_stream = rfile.read()
except socket.error:
raise error.IOError("Error reading data from server")
finally:
self.closeSocket(sock)
return byte_stream
def read_delimited(data, metric_type): result = [] start = 0 while len(data) > 0 and start < len(data): (msg_length, hdr_length) = _DecodeVarint(data[start:], 0) # read message as a byte string proto_str = data[start + hdr_length:start + msg_length + hdr_length] # EOF if len(proto_str) == 0: return start = start + msg_length + hdr_length # de-serialize the bytes into a protobuf message msg = metric_type() msg.ParseFromString(proto_str) result.append(convert_protobuf_dict(msg)) return result
def get_n_inodes(self, n, start_pos=None):
if not start_pos:
if not self._cur_pos:
inode_sec = self.load_inode_section()
start_pos = self._cur_pos
else:
start_pos = self._cur_pos
inode_sec = self.inode_section
else:
inode_sec = self.inode_section
new_pos = start_pos
for _ in xrange(n):
(len, pos) = decoder._DecodeVarint(self.buf, new_pos)
inode_bytes = self.buf[pos:pos + len]
inode = inode_sec.INode()
inode.ParseFromString(inode_bytes)
new_pos = pos + len
yield inode
def _parse_protobuf_message(self, message_buffer):
parsed_message = None
remainder = message_buffer
message_data = ""
# 1. decode a varint from the top of the stream
# 2. using that as the length, if there's enough in the buffer, try and
# decode try and decode a VehicleMessage after the varint
# 3. if it worked, great, we're oriented in the stream - continue
# 4. if either couldn't be parsed, skip to the next byte and repeat
while parsed_message is None and len(message_buffer) > 1:
message_length, message_start = _DecodeVarint(message_buffer, 0)
# sanity check to make sure we didn't parse some huge number that's
# clearly not the length prefix
if message_length > self.MAX_PROTOBUF_MESSAGE_LENGTH:
message_buffer = message_buffer[1:]
continue
if message_start + message_length >= len(message_buffer):
break
message_data = message_buffer[message_start:message_start +
message_length]
remainder = message_buffer[message_start + message_length:]
message = openxc_pb2.VehicleMessage()
try:
message.ParseFromString(message_data)
except google.protobuf.message.DecodeError as e:
message_buffer = message_buffer[1:]
except UnicodeDecodeError as e:
LOG.warn("Unable to parse protobuf: %s", e)
else:
parsed_message = self._protobuf_to_dict(message)
if parsed_message is None:
message_buffer = message_buffer[1:]
bytes_received = 0
if parsed_message is not None:
bytes_received = len(message_data)
return parsed_message, remainder, bytes_received
def annotate_proto(self, text, annotators=None):
"""Return a Document protocol buffer from the CoreNLP server, containing annotations of the text.
:param (str) text: text to be annotated
:param (list[str]) annotators: a list of annotator names
:return (CoreNLP_pb2.Document): a Document protocol buffer
"""
properties = {
'annotators': ','.join(annotators or self.default_annotators),
'outputFormat': 'serialized',
'serializer': 'edu.stanford.nlp.pipeline.ProtobufAnnotationSerializer'
}
r = self._request(text, properties)
buffer = r.content # bytes
size, pos = _DecodeVarint(buffer, 0)
buffer = buffer[pos:(pos + size)]
doc = CoreNLP_pb2.Document()
doc.ParseFromString(buffer)
return doc
def _read_varint(self):
"""Read exactly a varint out of the underlying file."""
buf = self._read(8)
(n,l) = _DecodeVarint(buf, 0)
self._unread(buf[l:])
return n
def dataReceived(self, message):
self.logger.info("Processing incoming data of %d bytes" % (len(message)))
self.logger.debug("\n%s" % (hexdump("Msg", message)))
alldata = self._unprocessed + message
self._unprocessed = alldata
while len(self._unprocessed) > 0:
#
# Coming back from the NameNode will only be responses to earlier RPC Requests
# (normally, we just send heartbeats)
# If the NN wants us to do something, it will tack on a command then
#
# To decode it, we have to pick apart a few things
# First is a variable-length encoded integer, which will tell us the size of
# header that will follow. (This is one of the few PB things in Hadoop which are variably-encoded)
#
# Varints are encoded as 7 bits in a byte, with the MSB used as a flag
# to find the stopping position
# Once you get a byte with 0x80 set to 0, you've got all the bytes
# for the encoded integer and you can decode it.
#
# Because dataReceived may give us back a partial read, we have to buffer until
# we've read enough to make progress. I'm not being clever, and we simply buffer
# everything, and reprocess it all as necessary
#
# for the full format, minus the bit about the varint encoded initial transmission, see
# hadoop-common/src/main/proto/RpcPayloadHeader.proto
#
# This is basically the same code that's in the PB decode routine
position = 0
while 1:
b = ord(self._unprocessed[position])
position += 1
if not (b & 0x80):
break
if position == len(self._unprocessed):
return
#
# Unfortunately, the Python implementation doesn't expose
# a decoding method directly for varints.
#
# However, you can use an internal API to do so.
# You get back a length, and where in the stream to start reading from
# Thanks to Evan Jones for the info
# http://comments.gmane.org/gmane.comp.lib.protocol-buffers.general/8223
#
# We could be more clever and decode it in the prior codeblock, but we'll let
# the actual PB code do the decoding
(size, position) = decoder._DecodeVarint(self._unprocessed,0)
# If we don't have enough to decode yet, bail. We'll try again next time.
if (size + position) > len(self._unprocessed):
return
resp = RpcPayloadHeader_pb2.RpcResponseHeaderProto()
resp.ParseFromString(self._unprocessed[position:position+size])
#
# The RpcPayloadHeader will tell us if the call was successful or unsuccessful. If it was
# unsucessful, well, we'll figure that out later
if resp.status == 0:
self.logger.info("Success RPC for call %d (%s)" % (resp.callId, self.calls[resp.callId]["name"]))
#
# Success means that after we've read the header, we can read a 4-byte integer which will give
# us the size of the next block - first, make sure we've got at least another 4 bytes already read
#
messagelen = 0
if(size + position + 4) <= len(self._unprocessed):
remaining = unpack(">I", self._unprocessed[size+position:size+position+4])
messagelen = size + position + 4 + remaining[0]
#print "Still have %d bytes to process, excepting %d, and the total is %d" % (remaining[0], (size+position+4), len(alldata))
#
# OK, we know how much farther we need to read - do we have all of that data yet?
# If we do, look up the function that stashed as the callback, and let it decode whatever comes next
#
if(messagelen) <= len(self._unprocessed):
if "callback" in self.calls[resp.callId]:
self.calls[resp.callId]["callback"](self._unprocessed[position+size+4:])
self.state = False
else:
self.logger.info("No callback for call %d" % (resp.callId))
# We know how big the response is, but we don't have all that data yet
else:
return
# We don't have all four bytes that tell us how big the response data is, so bail and try again
else:
return
#
# We're done with that callback, and have processed all of the data, so free it
#
del self.calls[resp.callId]
self._unprocessed = self._unprocessed[messagelen:]
def process_outgoing_data(self):
message = ""
while True:
# First, loop until we've got the first integer with header length
position = 0
while True:
if position == len(message):
message = message + (yield)
next
b = ord(message[position])
position += 1
if not (b & 0x80):
break
# At this point, we know we've got enough data onhand to decode the length integer, so
# we don't have to flip back to the other function to get more data
(msgsize, msgstart) = decoder._DecodeVarint(message, 0)
# Keep looping until we've got enough data to decode the RPC results
while len(message) - msgstart < msgsize:
message = message + (yield)
resp = RpcPayloadHeader_pb2.RpcResponseHeaderProto()
resp.ParseFromString(message[msgstart : msgstart + msgsize])
message = message[msgsize + msgstart :]
logger.debug("Extracting info for %s from messages, which is\n%s" % (str(resp.callId), str(self.messages)))
elapsed = time.time() - self.messages[resp.callId]["time"]
if resp.status == 0:
logger.info(
"Success for RPC %d (%s) - took %.4f seconds"
% (resp.callId, self.messages[resp.callId]["method"], elapsed)
)
while len(message) < 4:
message = message + (yield)
remaining = unpack(">I", message[0:4])
message = message[4:]
while len(message) < remaining[0]:
message = message + (yield)
logger.debug("Read all of the response bytes, but don't really care what it is")
message = message[remaining[0] :]
else: # resp.status != 0
logger.debug(
"Status for RPC %d (%s) was %d, discarding results"
% (resp.callId, self.messages[resp.callId]["method"], reps.status)
)
# In an error, there are two response strings, each prefixed with 4 byte integers
# TODO - RpcPayloadHeader.proto says "In case of Fatal error then the respose contains the Serverside's IPC version"
while len(message) < 4:
message = message + (yield)
remaining = unpack(">I", message[0:4])
message = message[4:]
if remaining[0] > 0:
while len(message) < remaining[0]:
message = message + (yield)
logger.debug("RPC Failure class: %s" % (str(message[0 : remaining[0]])))
message = message[remaining[0] :]
else:
logger.debug("RPC Failure: Unknown Class")
# Next up, stack trace
while len(message) < 4:
message = message + (yield)
remaining = unpack(">I", message[0:4])
message = message[4:]
if remaining[0] > 0:
while len(message) < remaining[0]:
message = message + (yield)
logger.debug("RPC Failure Stack Trace:\n%s" % (str(message[0 : remaining[0]])))
message = message[remaining[0] :]
else:
logger.debug("RPC Failure: No Stack Trace follows")
if self.messages[resp.callId]:
del self.messages[resp.callId]
def decode_message_size(cls, data):
return protobuf_decoder._DecodeVarint(data, 0)[0]
def decode_string(cls, data):
(size, position) = protobuf_decoder._DecodeVarint(data, 0)
return data[position:position+size].decode('utf-8')
def unpack(data):
'''unpack from delimited data'''
size, position = decoder._DecodeVarint(data, 0)
envelope = wire.Envelope()
envelope.ParseFromString(data[position:position+size])
return envelope
def decode_size_and_position(cls, data):
""" Decode a varint and return the (size, position) """
return protobuf_decoder._DecodeVarint(data, 0)
def _decode_varint(cls, data):
return protobuf_decoder._DecodeVarint(data, 0)[0]
def decode_varint(data):
""" Decode a protobuf varint to an int """
return _DecodeVarint(data, 0)[0]
def decode_delimited(cls, data, typ):
""" Decode a message or value with size information
(used in a delimited communication stream) """
(size, position) = protobuf_decoder._DecodeVarint(data, 0)
return cls.decode(data[position:position+size], typ)
def decode_bytes(data):
return data[_DecodeVarint(data, 0)[1]:]
import keyValue_pb2
HOST = "localhost"
INPORT = 8080
# TODO: Look if AF_INET could be AF_LOCAL and be quicker....
inSock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
inSock.connect((HOST, INPORT))
READING_BYTES = 10
print "Python subprogram started"
while True:
buf = inSock.recv(READING_BYTES)
(size, position) = decoder._DecodeVarint(buf, 0)
print '\n' + "size: "+str(size)+" position: "+str(position) + '\n'
if size == 4294967295: # this is a hack my friend we probably need fixlength types for the length
break
# 1) Reading the keyValue pair from the socket
toRead = size+position-READING_BYTES
buf += inSock.recv(toRead) # this is probably inefficient because the buffer sizes changes all the time
print("bufSize "+str(len(buf)))
kv = keyValue_pb2.KeyValuePair()
kv.ParseFromString(buf[position:position+size])
print("key "+kv.key)
print("value "+kv.value)
def connectGCM(androidId, securityToken):
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
context = ssl.create_default_context()
conn = context.wrap_socket(s, server_hostname=mtalk.server)
conn.connect((mtalk.server, 5228))
conn.sendall(b'\x07')
data = conn.recv(1024)
mtalkpacket = mtalk.Packets()
loginRequestPacket = mtalkpacket.LoginRequestPacket(androidId, securityToken, hex(androidId)[2:])
conn.sendall(loginRequestPacket)
threading.Thread(target=send_heartbeats,
args=(conn, ),
daemon=True, ).start()
lastStreamId = 1
while True:
data = b''
while 1:
data += conn.recv(1024)
if len(data) != 0:
packetlength, varintlength = GoogleProtobufDecoder._DecodeVarint(data, 1)
if (len(data) - varintlength) == packetlength:
break
tag = data[0]
data = data[varintlength:]
print('message parsed with tag: ' + str(tag))
print(data)
if tag == 0x00:
print('Disconnected from GCM')
if tag == 0x03:
# Recieved LoginResponse
loginResponse = GoogleServicesFramework_pb2.LoginResponse()
loginResponse.ParseFromString(data)
if loginResponse.error.code == 0:
print('Login Successful')
elif tag == 0x07:
# Recieved DataMessageStanza
iqStanza = GoogleServicesFramework_pb2.IQStanza()
iqStanza.ParseFromString(data)
elif tag == 0x08:
# Recieved DataMessageStanza
dataMessageStanza = GoogleServicesFramework_pb2.DataMessageStanza()
dataMessageStanza.ParseFromString(data)
if dataMessageStanza.category == app:
for appdata in dataMessageStanza.appdata:
#print(appdata.key + ' - ' + appdata.value)
if appdata.key == 'message':
print('Received GCM message for textsecure')
receive_textsecure.receive_textsecure_message(appdata.value, q)
else:
pass
#if debug:
# for appdata in dataMessageStanza.appdata:
# print(appdata.key + ' - ' + appdata.value)
notificationRequestPacket = mtalkpacket.NotificationPacket(lastStreamId)
conn.sendall(notificationRequestPacket)
lastStreamId += 1
def decode_bytes(cls, data):
(size, position) = protobuf_decoder._DecodeVarint(data, 0)
return data[position:position+size]
exit()
tsf_file = open(sys.argv[1], "rb")
# Read magic number and offset
mnumber = readinsight3._getV(tsf_file, "I", 4)
offset = readinsight3._getV(tsf_file, ">Q", 8)
print("mnumber:", mnumber)
print("offset:", offset)
# Read SpotList message
tsf_file.seek(offset+12)
buffer = tsf_file.read()
(spot_list_size, position) = decoder._DecodeVarint(buffer, 0)
spot_list = TSFProto_pb2.SpotList()
spot_list.ParseFromString(buffer[position:position+spot_list_size])
print("Spot List:", spot_list_size)
print(spot_list)
print("")
# Read the first few spots.
cur_pos = 12
for i in range(2):
tsf_file.seek(cur_pos)
buffer = tsf_file.read(200)
(spot_size, position) = decoder._DecodeVarint(buffer, 0)
