def pack(src, cgi_type, use_compress=0):
# 必要参数合法性判定
if not Util.cookie or not Util.uin or not Util.sessionKey:
return b''
# 压缩加密
len_proto_compressed = len(src)
if use_compress:
(body, len_proto_compressed) = Util.compress_and_aes(src, Util.sessionKey)
else:
body = Util.aes(src, Util.sessionKey)
logger.debug("cgi:{},protobuf数据:{}\n加密后数据:{}".format(cgi_type, Util.b2hex(src), Util.b2hex(body)))
# 封包包头
header = bytearray(0)
header += b'\xbf' # 标志位(可忽略该字节)
header += bytes([0]) # 最后2bit：02--包体不使用压缩算法;前6bit:包头长度,最后计算
header += bytes([((0x5 << 4) + 0xf)]) # 05:AES加密算法 0xf:cookie长度(默认使用15字节长的cookie)
header += struct.pack(">I", define.__CLIENT_VERSION__) # 客户端版本号 网络字节序
header += struct.pack(">i", Util.uin) # uin
header += Util.cookie # coockie
header += encoder._VarintBytes(cgi_type) # cgi type
header += encoder._VarintBytes(len(src)) # body proto压缩前长度
header += encoder._VarintBytes(len_proto_compressed) # body proto压缩后长度
header += bytes([0]*15) # 3个未知变长整数参数,共15字节
header[1] = (len(header) << 2) + (1 if use_compress else 2) # 包头长度
logger.debug("包头数据:{}".format(Util.b2hex(header)))
# 组包
senddata = header + body
return senddata
def getBytes(self, protoPacket = 0):
requestBytes = self.packet_type
if protoPacket == 0:
requestBytes += GoogleProtobufEncoder._VarintBytes(0)
else:
protoRequestBytes = protoPacket.SerializeToString()
requestBytes += GoogleProtobufEncoder._VarintBytes(len(protoRequestBytes))
requestBytes += protoRequestBytes
return requestBytes
def send(msg):
string = msg.SerializeToString()
string = encoder._VarintBytes(len(string)) + string
s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
s.connect("/tmp/slim_socket")
s.send(string)
s.close()
def delimitProtobuf(src):
"""Python protobuf bindings are missing writeDelimited, this is a workaround to add a delimiter"""
from google.protobuf.internal import encoder
serializedMessage = src.SerializeToString()
delimiter = encoder._VarintBytes(len(serializedMessage))
return delimiter + serializedMessage
def send_over_socket(sock, message):
delimiter = encoder._VarintBytes(len(message))
message = delimiter + message
msg_len = len(message)
total_sent = 0
while total_sent < msg_len:
sent = sock.send(message[total_sent:])
if sent == 0:
raise RuntimeError('Socket connection broken')
total_sent = total_sent + sent
def save_protobuf(filename, feature_trajectories):
"""
save a features in the given filename
"""
with open(filename, 'wb') as f:
for features in feature_trajectories:
for fea in features:
serializedMessage = fea.SerializeToString()
delimiter = encoder._VarintBytes(len(serializedMessage))
f.write(delimiter + serializedMessage)
def pack(mtype, request, rid=None):
'''pack request to delimited data'''
envelope = wire.Envelope()
if rid is not None:
envelope.id = rid
envelope.type = mtype
envelope.message = request.SerializeToString()
data = envelope.SerializeToString()
data = encoder._VarintBytes(len(data)) + data
return data
def sendCommand(self, request, cbinfo):
rpchead = RpcPayloadHeader_pb2.RpcPayloadHeaderProto()
# From the Hadoop code, some definitions:
# enum RpcKindProto {
# RPC_BUILTIN = 0; // Used for built in calls by tests
# RPC_WRITABLE = 1; // Use WritableRpcEngine
# RPC_PROTOCOL_BUFFER = 2; // Use ProtobufRpcEngine
# }
# enum RpcPayloadOperationProto {
# RPC_FINAL_PAYLOAD = 0; // The final payload
# RPC_CONTINUATION_PAYLOAD = 1; // not implemented yet
# RPC_CLOSE_CONNECTION = 2; // close the rpc connection
# }
rpchead.rpcKind = 2
rpchead.rpcOp = 0
rpchead.callId = self._nextcallid
self.calls[self._nextcallid] = cbinfo
self._nextcallid += 1
rpcreq = hadoop_rpc_pb2.HadoopRpcRequestProto()
rpcreq.methodName = cbinfo["methodName"]
if request is None:
rpcreq.request = ""
else:
rpcreq.request = request.SerializeToString()
rpcreq.declaringClassProtocolName = "org.apache.hadoop.hdfs.server.protocol.DatanodeProtocol"
rpcreq.clientProtocolVersion = 1
headerout = rpchead.SerializeToString()
headerout = encoder._VarintBytes(len(headerout)) + headerout
reqout = rpcreq.SerializeToString()
reqout = encoder._VarintBytes(len(reqout)) + reqout
buf = headerout + reqout
self.transport.write(pack(">I", len(buf)))
self.transport.write(buf)
def kafka_produce_protobuf_messages(topic, start_index, num_messages):
data = ''
for i in range(start_index, start_index + num_messages):
msg = kafka_pb2.KeyValuePair()
msg.key = i
msg.value = str(i)
serialized_msg = msg.SerializeToString()
data = data + _VarintBytes(len(serialized_msg)) + serialized_msg
producer = KafkaProducer(bootstrap_servers="localhost:9092")
producer.send(topic=topic, value=data)
producer.flush()
print("Produced {} messages for topic {}".format(num_messages, topic))
def create_rpc_header(self):
'''Creates and serializes a delimited RpcPayloadHeaderProto message.'''
rpcheader = rpcheaderproto.RpcPayloadHeaderProto()
rpcheader.rpcKind = 2 # rpcheaderproto.RpcKindProto.Value('RPC_PROTOCOL_BUFFER')
rpcheader.rpcOp = 0 # rpcheaderproto.RpcPayloadOperationProto.Value('RPC_FINAL_PAYLOAD')
rpcheader.callId = self.call_id
self.call_id += 1
# Serialize delimited
s_rpcHeader = rpcheader.SerializeToString()
log_protobuf_message("RpcPayloadHeader (len: %d)" % (len(s_rpcHeader)), rpcheader)
return encoder._VarintBytes(len(s_rpcHeader)) + s_rpcHeader
def create_rpc_request(self, method, request):
'''Wraps the user's request in an HadoopRpcRequestProto message and serializes it delimited.'''
s_request = request.SerializeToString()
log_protobuf_message("Protobuf message", request)
log.debug("Protobuf message bytes (%d): %s" % (len(s_request), format_bytes(s_request)))
rpcRequest = hadoop_rpc.HadoopRpcRequestProto()
rpcRequest.methodName = method.name
rpcRequest.request = s_request
rpcRequest.declaringClassProtocolName = "org.apache.hadoop.hdfs.protocol.ClientProtocol"
rpcRequest.clientProtocolVersion = 1L
# Serialize delimited
s_rpcRequest = rpcRequest.SerializeToString()
log_protobuf_message("RpcRequest (len: %d)" % len(s_rpcRequest), rpcRequest)
return encoder._VarintBytes(len(s_rpcRequest)) + s_rpcRequest
def dumMsgToFileProto(self, message):
"""
Parameters
----------
message : TYPE
DESCRIPTION.
Returns
-------
None.
"""
size = message.ByteSize()
self.DumpfileProto.write(_VarintBytes(size))
self.DumpfileProto.write(message.SerializeToString())
def __dumpMsgToFileProto(self, message):
"""
private function to dump MSG as binaryblob \n for new data packet.
Parameters
----------
message : protobuff message
Data to be dumped.
Returns
-------
None.
"""
size = message.ByteSize()
self.DumpfileProto.write(_VarintBytes(size))
self.DumpfileProto.write(message.SerializeToString())
def create_rpc_request(self, method, request):
'''Wraps the user's request in an HadoopRpcRequestProto message and serializes it delimited.'''
s_request = request.SerializeToString()
log_protobuf_message("Protobuf message", request)
log.debug("Protobuf message bytes (%d): %s" %
(len(s_request), format_bytes(s_request)))
rpcRequest = hadoop_rpc.HadoopRpcRequestProto()
rpcRequest.methodName = method.name
rpcRequest.request = s_request
rpcRequest.declaringClassProtocolName = "org.apache.hadoop.hdfs.protocol.ClientProtocol"
rpcRequest.clientProtocolVersion = 1L
# Serialize delimited
s_rpcRequest = rpcRequest.SerializeToString()
log_protobuf_message("RpcRequest (len: %d)" % len(s_rpcRequest),
rpcRequest)
return encoder._VarintBytes(len(s_rpcRequest)) + s_rpcRequest
def orderHistory(request):
orders = purchase.objects.filter(user=request.user.id)
for order in orders:
command = amazon.Cmd()
query_status = amazon.Query()
query_status.packageid = order.id
command.queries.append(query_status)
serialized_request = command.SerializeToString()
size = command.ByteSize()
web_socket.sendall(_VarintBytes(size) + serialized_request)
# response = recv_from_back(amazon.Resp, web_socket)
# order.status = response.pkgstatus.status
# order.save()
time.sleep(4)
ordersAll = purchase.objects.filter(user=request.user.id)
context = {'historis': ordersAll}
return render(request, 'purchase/orderHistory.html', context)
def to_buffer(self):
# Each message_info as part of the header needs to be updated
# so that its length matches the object contained within.
for obj, message_info in zip(self.objects, self.header.message_infos):
try:
object_length = len(obj.SerializeToString())
provided_length = message_info.length
if object_length != provided_length:
message_info.length = object_length
except EncodeError as e:
raise ValueError(
"Failed to encode object: %s\nObject: '%s'\nMessage info: %s"
% (e, repr(obj), message_info))
return b''.join([
_VarintBytes(self.header.ByteSize()),
self.header.SerializeToString()
] + [obj.SerializeToString() for obj in self.objects])
def single(msgtype, stream=False):
if msgtype == 'cat':
obj = cat_pb2.Cat(is_lazy=bool(random.getrandbits(1)))
elif msgtype == 'dog':
obj = dog_pb2.Dog(age=random.choice(range(0, 20)),
breed=['rottweiler', 'gsd',
'poodle'][random.choice(range(0, 3))],
temperament=['chill', 'aggressive',
'excited'][random.choice(range(0, 3))])
elif msgtype == 'person':
obj = person_pb2.Person(id=random.choice(range(0, 4)),
name=['raffi', 'khosrov',
'vahaken'][random.choice(range(0, 3))])
else:
usage()
obj = obj.SerializeToString()
varint_ = encoder._VarintBytes(len(obj)) if stream else b''
sys.stdout.buffer.write(varint_ + obj)
def sendRpcMessage(self, sock, rpcRequest):
'''Send an RPC request to the server.'''
try:
wfile = sock.makefile('w')
""" modified by wuzhw """
out = rpcRequest.SerializeToString()
# print("request size: %d"%len(out))
out = encoder._VarintBytes(len(out))+out
wfile.write(out)
""" Original: """
# wfile.write(rpcRequest.SerializeToString())
wfile.flush()
sock.shutdown(socket.SHUT_WR)
except socket.error:
self.closeSocket(sock)
raise error.IOError("Error writing data to server")
def serialize_delimited(self) -> bytes:
"""
Write out in delimited format (data is prefixed with the length of the
datastream).
This is useful when you are streaming multiple dataset profile objects
Returns
-------
data : bytes
A sequence of bytes
"""
with io.BytesIO() as f:
protobuf: DatasetProfileMessage = self.to_protobuf()
size = protobuf.ByteSize()
f.write(_VarintBytes(size))
f.write(protobuf.SerializeToString(deterministic=True))
return f.getvalue()
def __gen_datum(self, key: int, as_text=False):
encoded_key = encode_key(key)
record = ''.join(np.random.choice(ALPHABET, size=self.record_size))
master = key % self.num_replicas
if as_text:
datum_tuple = map(str, (encoded_key.decode(), record, master))
datum = ','.join(datum_tuple) + '\n'
else:
datum_proto = Datum()
datum_proto.key = encoded_key.decode()
datum_proto.record = record
datum_proto.master = master
# Size of the datum and the serialized datum
datum = (_VarintBytes(datum_proto.ByteSize()) +
datum_proto.SerializeToString())
return datum
def single(msgtype, stream=False):
if msgtype == "cat":
obj = cat_pb2.Cat(is_lazy=bool(random.getrandbits(1)))
elif msgtype == "dog":
obj = dog_pb2.Dog(
age=random.choice(range(0, 20)),
breed=["rottweiler", "gsd", "poodle"][random.choice(range(0, 3))],
temperament=["chill", "aggressive", "excited"][random.choice(range(0, 3))],
)
elif msgtype == "person":
obj = person_pb2.Person(
id=random.choice(range(0, 4)),
name=["raffi", "khosrov", "vahaken"][random.choice(range(0, 3))],
)
else:
usage()
obj = obj.SerializeToString()
varint_ = encoder._VarintBytes(len(obj)) if stream else b""
sys.stdout.buffer.write(varint_ + obj)
def save_grpc_requests(requests: List[types.InferenceRequest]):
infer_requests = [
converters.ModelInferRequestConverter.from_types(
req, model_name=MODEL_NAME, model_version=MODEL_VERSION
)
for req in requests
]
requests_file_path = os.path.join(DATA_PATH, "grpc-requests.pb")
with open(requests_file_path, "wb") as requests_file:
for req in infer_requests:
# To stream multiple messages we need to prefix each one with its
# size
# https://ghz.sh/docs/options#-b---binary
size = req.ByteSize()
size_varint = _VarintBytes(size)
requests_file.write(size_varint)
serialised = req.SerializeToString()
requests_file.write(serialised)
def __gen_data_per_partition(
self,
partition: int,
keys: list,
as_text: bool,
) -> None:
# Set per-partition seed so that partitions have
# different data. Keys and master are not randomly
# generated so this seed only affects records
np.random.seed(partition)
file_name = os.path.join(self.data_dir,
self.prefix + str(partition) + FILE_EXTENSION)
LOG.info("Generating data for %s", file_name)
mode = 'w' if as_text else 'wb'
part_file = open(file_name, mode)
# Write number of keys in this partition
if as_text:
part_file.write(str(len(keys)) + "\n")
else:
part_file.write(_VarintBytes(len(keys)))
last_time = time.time()
last_index = 0
for i, key in enumerate(keys):
# Generate the datum for this key
datum = self.__gen_datum(key, as_text)
# Append the datum to file
part_file.write(datum)
now = time.time()
if now - last_time >= LOG_EVERY_SEC:
pct = (i) / len(keys) * 100
rate = (i - last_index) / LOG_EVERY_SEC
LOG.info("Progress: %d/%d (%.1f%%). Rate: %d datums/s", i + 1,
len(keys), pct, rate)
last_time = now
last_index = i
part_file.close()
def serializeDelimited(self):
# create outer packet message
packetMessage = protoPackets_pb2.PacketMessage()
packetMessage.type = protoPackets_pb2.PacketMessage.CHECK
packetMessage.transactionID = self.transactionID
# add check-specific content
packetMessage.checkMessage.newBalance = self.newBalance
for (transactionID, amount) in self.balanceUpdates:
x = packetMessage.checkMessage.balanceUpdates.add()
x.transactionID = transactionID
x.amount = amount
print self
serializedMessage = packetMessage.SerializeToString()
delimiter = encoder._VarintBytes(len(serializedMessage))
return delimiter + serializedMessage
def serializeDelimited(self):
# create outer packet message
packetMessage = protoPackets_pb2.PacketMessage()
packetMessage.type = protoPackets_pb2.PacketMessage.CHECK
packetMessage.transactionID = self.transactionID
# add check-specific content
packetMessage.checkMessage.newBalance = self.newBalance
for (transactionID, amount) in self.balanceUpdates:
x = packetMessage.checkMessage.balanceUpdates.add()
x.transactionID = transactionID
x.amount = amount
print self
serializedMessage = packetMessage.SerializeToString()
delimiter = encoder._VarintBytes(len(serializedMessage))
return delimiter + serializedMessage
def write_protobuf(self, protobuf_path: str, delimited_file: bool = True):
"""
Write the dataset profile to disk in binary format
Parameters
----------
protobuf_path : str
local path or any path supported supported by smart_open:
https://github.com/RaRe-Technologies/smart_open#how.
The parent directory must already exist
delimited_file : bool, optional
whether to prefix the data with the length of output or not.
Default is True
"""
with open(protobuf_path, "wb") as f:
msg = self.to_protobuf()
size = msg.ByteSize()
if delimited_file:
f.write(_VarintBytes(size))
f.write(msg.SerializeToString())
def __sendDataMsg(self):
"""
Sends out simulated data.
Returns
-------
None.
"""
tmptime = time.time()
secs = int(np.floor(tmptime))
nsecs = int((tmptime - secs) * 1e9)
# setting up the proto message
protodata = messages_pb2.DataMessage()
protodata.id = self.params["ID"]
protodata.sample_number = self.packetssend
protodata.unix_time = secs
protodata.unix_time_nsecs = nsecs
protodata.time_uncertainty = 1000000
res = 2**self.params["resolutionbit"]
tmp = (nsecs / 1e9) * (res - 1) - res / 2
qunatizedint = int(tmp)
qunatizedfloat = qunatizedint / res * 2
protodata.Data_01 = np.sin(qunatizedfloat * np.pi)
protodata.Data_02 = np.cos(qunatizedfloat * np.pi)
protodata.Data_03 = qunatizedfloat
protodata.Data_04 = abs(qunatizedfloat)
binproto = protodata.SerializeToString()
# add DATA peramble for data Pacet
binarymessage = b"DATA"
binarymessage = binarymessage + _VarintBytes(len(binproto)) + binproto
self.socket.sendto(binarymessage,
(self.params["TargetIp"], self.params["Port"]))
self.packetssend = self.packetssend + 1
if self.packetssend % self.params["UpdateRateHz"] == 0:
print(str(self.packetssend) + " Packets sent")
return
def activate(self):
retry = None
while True:
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as conn:
conn.connect((self.hostname, self.port))
conn.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
conn.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, 1)
conn.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPINTVL, 5)
conn.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPCNT, 5)
while True:
if retry is None:
cmd = self.command_queue.get()
to_send = ClusterCommand()
to_send.type, to_send.data = cmd
else:
to_send = retry
size = to_send.ByteSize()
try:
conn.send(
_VarintBytes(size) + to_send.SerializeToString())
retry = None
except BrokenPipeError:
retry = to_send
break
def __sendDataMsg(self, line):
"""
Sends out data.
Returns
-------
None.
"""
# setting up the proto message
protodata = messages_pb2.DataMessage()
protodata.id = int(line[0])
protodata.sample_number = int(line[1])
protodata.unix_time = int(line[2])
protodata.unix_time_nsecs = int(line[3])
protodata.time_uncertainty = int(line[4])
for i in range(16):
protodata.__setattr__(self.dataFieldNames[i], float(line[5 + i]))
binproto = protodata.SerializeToString()
# add DATA peramble for data Pacet
binarymessage = b"DATA"
binarymessage = binarymessage + _VarintBytes(len(binproto)) + binproto
self.socket.sendto(binarymessage,
(self.params["TargetIp"], self.params["Port"]))
def encode_delimited(cls, x, typ):
""" Encode a message or value with size information
(for use in a delimited communication stream) """
data = cls.encode(x, typ)
delimiter = protobuf_encoder._VarintBytes(len(data))
return delimiter + data
def delimited_value(msg):
# Emits a writeDelimited compatible Protobuf message
o = msg.SerializeToString()
d = encoder._VarintBytes(len(o))
return d + o
def getInfo(protocolName,methodName,requestProto,responseProto):
rpcrequestheader = RpcRequestHeaderProto()
rpcrequestheader.rpcKind = 2
#RpcHeader_pb2.RPC_PROTOCOL_BUFFER
rpcrequestheader.rpcOp = 0
#RpcRequestHeaderProto.RPC_FINAL_PACKET
rpcrequestheader.callId = -3 # During initial connection
# 0 otherwise
# 4 for ping i guess
client_id = str(uuid.uuid4())
rpcrequestheader.clientId = client_id[0:16]
s_rpcrequestheader = rpcrequestheader.SerializeToString()
'''Create and serialize a IpcConnectionContextProto '''
context = IpcConnectionContextProto()
context.userInfo.effectiveUser = "******"
context.protocol = protocolName
#"org.apache.hadoop.mapreduce.v2.api.MRClientProtocolPB"
s_context = context.SerializeToString()
''' Length of the two messages '''
rpcipc_length = len(s_rpcrequestheader) + encoder._VarintSize(len(s_rpcrequestheader)) + len(s_context) + encoder._VarintSize(len(s_context))
''' Send to server in the order given above'''
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
sock.settimeout(10)
sock.connect(("localhost", 8032)) #8020 for name node, 8032 for yarn
sock.send("hrpc") # header
sock.send(struct.pack('B', 9)) # version
sock.send(struct.pack('B', 0x00)) # RPC service class
sock.send(struct.pack('B', 0x00)) # auth none
sock.sendall(struct.pack('!I', rpcipc_length) +
encoder._VarintBytes(len(s_rpcrequestheader)) +
s_rpcrequestheader +
encoder._VarintBytes(len(s_context)) +
s_context)
'''
Create the Hadoop RPC protocol looks like this for sending requests:
When sending requests
+---------------------------------------------------------------------+
| Length of the next three parts (4 bytes/32 bit int) |
+---------------------------------------------------------------------+
| Delimited serialized RpcRequestHeaderProto (varint len + header) |
+---------------------------------------------------------------------+
| Delimited serialized RequestHeaderProto (varint len + header) |
+---------------------------------------------------------------------+
| Delimited serialized Request (varint len + request) |
+---------------------------------------------------------------------+
'''
'''
Steps:
1. create rpcrequestheader
2. create requestheader in which you can mention the name of protocol and method name you want
3. create actual request i guess you can use it to pass parameters
'''
''' we need a rpcrequestheaderproto for every message we send '''
rpcrequestheader = RpcRequestHeaderProto()
rpcrequestheader.rpcKind = 2 #RpcHeader_pb2.RPC_PROTOCOL_BUFFER
rpcrequestheader.rpcOp = 0 #RpcRequestHeaderProto.RPC_FINAL_PACKET
rpcrequestheader.callId = 0 # For all other communication other than initial, 4 for ping
client_id = str(uuid.uuid4())
rpcrequestheader.clientId = client_id[0:16]
s_rpcrequestheader = rpcrequestheader.SerializeToString()
''' ok thats our header'''
'''lets create our requestheaderproto '''
requestheader = RequestHeaderProto()
requestheader.methodName = methodName
#"getClusterNodes"
requestheader.declaringClassProtocolName = protocolName
#"org.apache.hadoop.yarn.api.ApplicationClientProtocolPB"
requestheader.clientProtocolVersion= 1
'''serialize this '''
s_requestheader = requestheader.SerializeToString()
'''Now we need to write our actual request....may be lets start it here '''
#request = protoFile.GetClusterMetricsRequestProto()
s_request = requestProto.SerializeToString()
''' lenght of three messages '''
rpc_message_length = len(s_rpcrequestheader) + encoder._VarintSize(len(s_rpcrequestheader)) + \
len(s_requestheader) + encoder._VarintSize(len(s_requestheader)) + \
len(s_request) + encoder._VarintSize(len(s_request))
'''pack in the above given format and send :) '''
sock.sendall(struct.pack('!I', rpc_message_length) +
encoder._VarintBytes(len(s_rpcrequestheader)) +
s_rpcrequestheader +
encoder._VarintBytes(len(s_requestheader))+
s_requestheader+
encoder._VarintBytes(len(s_request)) +
s_request)
#responseObject = yarn_service_protos_pb2.GetClusterMetricsResponseProto
stream = recv_rpc_message(sock)
parse_response(stream, responseProto)
spot.z = float(i3_block['zc'][i])
spot.intensity = float(i3_block['a'][i])
spot.background = float(i3_block['bg'][i])
spot.width = float(i3_block['w'][i])
spot.a = float(i3_block['ax'][i])
spot.theta = 0.0
spot.x_original = float(i3_block['x'][i])*pix_to_nm
spot.y_original = float(i3_block['y'][i])*pix_to_nm
spot.z_original = float(i3_block['z'][i])
localization_number += 1
out = spot.SerializeToString()
out = encoder._VarintBytes(len(out)) + out
tsf_file.write(out)
i3_block = i3_reader.nextBlock(block_size = 1000, good_only = False)
print("")
print(localization_number, "total localizations")
# Save SpotList.
print("")
print("Saving analysis meta-data")
print(" data set contains", len(channels), "channels", channels)
spot_list = TSFProto_pb2.SpotList()
# FIXME: get a real id..
spot_list.application_id = 1
spot_list.name = os.path.basename(sys.argv[1])
runCommand("heightOn")
runCommand("i2cOn")
ok = runTest("i2cReady " + str(SERIAL_ADDRESS), "ok", "Serial number not ready. I2C lines probably hosed.") and ok
ok = runTest("i2cReady " + str(MEMORY_ADDRESS), "ok", "Memory address not ready. I2C lines probably hosed.") and ok
serial_number = i2cRead(SERIAL_ADDRESS, SERIAL_PAGE, 16)
if serial_number not in ["error", "timeout", "busy"]:
serial_number = base64.urlsafe_b64encode(binascii.unhexlify(serial_number.replace(" ", ""))).strip("=")
# Set manufacturing info into the board info.
board_info.manufacturing_info.test_time = long(startTime)
serialized_board_info = board_info.SerializeToString()
delimiter = encoder._VarintBytes(len(serialized_board_info))
ok = i2cWrite(MEMORY_ADDRESS + 1, MEMORY_PAGE, delimiter + serialized_board_info) and ok
else:
logError("Unable to read EEPROM serial number!")
ok = False
runCommand("i2cOff")
runCommand("heightOff")
else:
# Wait for bootup.
time.sleep(0.5)
testBin = "builds/" + board + "/test.bin"
returncode = flash(testBin)
if returncode != 0:
def write_delimited(self, data):
self.write(encoder._VarintBytes(len(data)))
self.write(data)
def write(p, st):
sm = p.SerializeToString()
l = len(sm)
s = encoder._VarintBytes(len(sm))
st.write(s + sm)
def writeDelimitedTo(message, connection):
message_str = message.SerializeToString()
delimiter = encoder._VarintBytes(len(message_str))
connection.send(delimiter + message_str)
def readBlock(self, length, pool_id, block_id, generation_stamp, offset, check_crc):
'''Send a read request to given block. If we receive a successful response,
we start reading packets.
Send read request:
+---------------------------------------------------------------------+
| Data Transfer Protocol Version, 2 bytes |
+---------------------------------------------------------------------+
| Op code, 1 byte (READ_BLOCK = 81) |
+---------------------------------------------------------------------+
| Delimited serialized OpReadBlockProto (varint len + request) |
+---------------------------------------------------------------------+
Receive response:
+---------------------------------------------------------------------+
| Delimited BlockOpResponseProto (varint len + response) |
+---------------------------------------------------------------------+
Start reading packets. Each packet has the following structure:
+---------------------------------------------------------------------+
| Packet length (4 bytes/32 bit int) |
+---------------------------------------------------------------------+
| Serialized size of header, 2 bytes |
+---------------------------------------------------------------------+
| Packet Header Proto |
+---------------------------------------------------------------------+
| x checksums, 4 bytes each |
+---------------------------------------------------------------------+
| x chunks of payload data |
+---------------------------------------------------------------------+
'''
log.debug("%s sending readBlock request" % self)
# Send version and opcode
self.sock.send(struct.pack('>h', 28))
self.sock.send(struct.pack('b', self.READ_BLOCK))
length = length - offset
# Create and send OpReadBlockProto message
request = datatransfer_proto.OpReadBlockProto()
request.offset = offset
request.len = length
header = request.header
header.clientName = "snakebite"
base_header = header.baseHeader
block = base_header.block
block.poolId = pool_id
block.blockId = block_id
block.generationStamp = generation_stamp
s_request = request.SerializeToString()
log_protobuf_message("OpReadBlockProto:", request)
delimited_request = encoder._VarintBytes(len(s_request)) + s_request
self.sock.send(delimited_request)
byte_stream = RpcBufferedReader(self.sock)
block_op_response_bytes = get_delimited_message_bytes(byte_stream)
block_op_response = datatransfer_proto.BlockOpResponseProto()
block_op_response.ParseFromString(block_op_response_bytes)
log_protobuf_message("BlockOpResponseProto", block_op_response)
checksum_type = block_op_response.readOpChecksumInfo.checksum.type
bytes_per_chunk = block_op_response.readOpChecksumInfo.checksum.bytesPerChecksum
log.debug("Checksum type: %s, bytesPerChecksum: %s" % (checksum_type, bytes_per_chunk))
if checksum_type in [self.CHECKSUM_CRC32C, self.CHECKSUM_CRC32]:
checksum_len = 4
else:
raise Exception("Checksum type %s not implemented" % checksum_type)
total_read = 0
if block_op_response.status == 0: # datatransfer_proto.Status.Value('SUCCESS')
while total_read < length:
log.debug("== Reading next packet")
packet_len = struct.unpack("!I", byte_stream.read(4))[0]
log.debug("Packet length: %s", packet_len)
serialized_size = struct.unpack("!H", byte_stream.read(2))[0]
log.debug("Serialized size: %s", serialized_size)
packet_header_bytes = byte_stream.read(serialized_size)
packet_header = datatransfer_proto.PacketHeaderProto()
packet_header.ParseFromString(packet_header_bytes)
log_protobuf_message("PacketHeaderProto", packet_header)
data_len = packet_header.dataLen
chunks_per_packet = int((data_len + bytes_per_chunk - 1) / bytes_per_chunk)
log.debug("Nr of chunks: %d", chunks_per_packet)
data_len = packet_len - 4 - chunks_per_packet * checksum_len
log.debug("Payload len: %d", data_len)
byte_stream.reset()
# Collect checksums
if check_crc:
checksums = []
for _ in xrange(0, chunks_per_packet):
checksum = self._read_bytes(checksum_len)
checksum = struct.unpack("!I", checksum)[0]
checksums.append(checksum)
else:
self._read_bytes(checksum_len * chunks_per_packet)
# We use a fixed size buffer (a "load") to read only a couple of chunks at once.
bytes_per_load = self.LOAD_SIZE - (self.LOAD_SIZE % bytes_per_chunk)
chunks_per_load = int(bytes_per_load / bytes_per_chunk)
loads_per_packet = int(math.ceil(bytes_per_chunk * chunks_per_packet / bytes_per_load))
read_on_packet = 0
for i in range(loads_per_packet):
load = ''
for j in range(chunks_per_load):
log.debug("Reading chunk %s in load %s:", j, i)
bytes_to_read = min(bytes_per_chunk, data_len - read_on_packet)
chunk = self._read_bytes(bytes_to_read)
if check_crc:
checksum_index = i * chunks_per_load + j
if checksum_index < len(checksums) and crc(chunk) != checksums[checksum_index]:
raise Exception("Checksum doesn't match")
load += chunk
total_read += len(chunk)
read_on_packet += len(chunk)
yield load
# Send ClientReadStatusProto message confirming successful read
request = datatransfer_proto.ClientReadStatusProto()
request.status = 0 # SUCCESS
s_request = request.SerializeToString()
log_protobuf_message("ClientReadStatusProto:", request)
delimited_request = encoder._VarintBytes(len(s_request)) + s_request
self.sock.send(delimited_request)
self._close_socket()
def createMsg():
rpcrequestheader = RpcRequestHeaderProto()
rpcrequestheader.rpcKind = 2
#RpcHeader_pb2.RPC_PROTOCOL_BUFFER
rpcrequestheader.rpcOp = 0
#RpcRequestHeaderProto.RPC_FINAL_PACKET
rpcrequestheader.callId = -3 # During initial connection
# 0 otherwise
# 4 for ping i guess
client_id = str(uuid.uuid4())
rpcrequestheader.clientId = client_id[0:16]
s_rpcrequestheader = rpcrequestheader.SerializeToString()
'''Create and serialize a IpcConnectionContextProto '''
context = IpcConnectionContextProto()
context.userInfo.effectiveUser = "******"
context.protocol = "org.apache.hadoop.hdfs.protocol.ClientProtocol" #"org.apache.hadoop.yarn.protocol.MRClientProtocol" #"org.apache.hadoop.mapred.JobSubmissionProtocol"
s_context = context.SerializeToString()
''' Length of the two messages '''
rpcipc_length = len(s_rpcrequestheader) + encoder._VarintSize(len(s_rpcrequestheader)) + len(s_context) + encoder._VarintSize(len(s_context))
''' Send to server in the order given above'''
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
sock.settimeout(10)
sock.connect(("localhost", 8020))
sock.send("hrpc") # header
sock.send(struct.pack('B', 9)) # version
sock.send(struct.pack('B', 0x00)) # RPC service class
sock.send(struct.pack('B', 0x00)) # auth none
sock.sendall(struct.pack('!I', rpcipc_length) +
encoder._VarintBytes(len(s_rpcrequestheader)) +
s_rpcrequestheader +
encoder._VarintBytes(len(s_context)) +
s_context)
'''
Create the Hadoop RPC protocol looks like this for sending requests:
When sending requests
+---------------------------------------------------------------------+
| Length of the next three parts (4 bytes/32 bit int) |
+---------------------------------------------------------------------+
| Delimited serialized RpcRequestHeaderProto (varint len + header) |
+---------------------------------------------------------------------+
| Delimited serialized RequestHeaderProto (varint len + header) |
+---------------------------------------------------------------------+
| Delimited serialized Request (varint len + request) |
+---------------------------------------------------------------------+
'''
'''
Steps:
1. create rpcrequestheader
2. create requestheader in which you can mention the name of protocol and method name you want
3. create actual request i guess you can use it to pass parameters
'''
''' we need a rpcrequestheaderproto for every message we send '''
rpcrequestheader = RpcRequestHeaderProto()
rpcrequestheader.rpcKind = 2 #RpcHeader_pb2.RPC_PROTOCOL_BUFFER
rpcrequestheader.rpcOp = 0 #RpcRequestHeaderProto.RPC_FINAL_PACKET
rpcrequestheader.callId = 0 # For all other communication other than initial, 4 for ping i guess
client_id = str(uuid.uuid4())
rpcrequestheader.clientId = client_id[0:16]
s_rpcrequestheader = rpcrequestheader.SerializeToString()
''' ok thats our header'''
'''lets create our requestheaderproto '''
requestheader = RequestHeaderProto()
requestheader.methodName = "getServerDefaults" #"getDiagnostics" # "getCounters" #"getJobReport"
#"GetFsStatusRequestProto " # #"getAllJobs" #GetFsStatusRequestProto
requestheader.declaringClassProtocolName = "org.apache.hadoop.hdfs.protocol.ClientProtocol" #"org.apache.hadoop.yarn.protocol.MRClientProtocol"
#"org.apache.hadoop.hdfs.protocol.ClientProtocol"
#"org.apache.hadoop.mapred.JobSubmissionProtocol"
# org.apache.hadoop.hdfs.protocol.ClientProtocol
requestheader.clientProtocolVersion= 1 # not sure what is this 2,28
'''serialize this '''
s_requestheader = requestheader.SerializeToString()
'''Now we need to write our actual request....may be lets start it here '''
#request = mr_service_protos_pb2.GetJobReportRequestProto()
request = ClientNamenodeProtocol_pb2.GetServerDefaultsRequestProto()
s_request = request.SerializeToString() # random shit
''' lenght of three messages '''
rpc_message_length = len(s_rpcrequestheader) + encoder._VarintSize(len(s_rpcrequestheader)) + \
len(s_requestheader) + encoder._VarintSize(len(s_requestheader)) + \
len(s_request) + encoder._VarintSize(len(s_request))
'''pack in the above given format and send :) '''
sock.sendall(struct.pack('!I', rpc_message_length) +
encoder._VarintBytes(len(s_rpcrequestheader)) +
s_rpcrequestheader +
encoder._VarintBytes(len(s_requestheader))+
s_requestheader+
encoder._VarintBytes(len(s_request)) +
s_request)
#responseObject = mr_service_protos_pb2.GetJobReportResponseProto()
responseObject = ClientNamenodeProtocol_pb2.GetServerDefaultsResponseProto()
#get_message(sock,responseObject)
stream = recv_rpc_message(sock)
parse_response(stream, ClientNamenodeProtocol_pb2.GetServerDefaultsResponseProto) #response proto here
print "reading response"
def handle_connection(self):
"""
Run connection handling routine.
"""
context = SSLContext(PROTOCOL_TLS_CLIENT)
context.check_hostname = False
context.verify_mode = CERT_NONE
with create_connection(
(self.server_host, self.server_port)) as self.socket:
with context.wrap_socket(
self.socket, server_hostname=self.server_host) as self.tls:
# Use this buffer for all reads
read_buffer = bytearray(4096)
# Perform SID handshake
rq = RQ_Cvid()
rq.uuid = os.environ["S7S_UUID"]
rq.instance = InstanceType.CLIENT
rq.instance_flavor = InstanceFlavor.CLIENT_BRIGHTSTONE
msg = MSG()
setattr(msg, "payload", rq.SerializeToString())
setattr(msg, "id", randint(0, 65535))
self.tls.write(_VarintBytes(msg.ByteSize()))
self.tls.sendall(msg.SerializeToString())
read = self.tls.recv_into(read_buffer, 4096)
if read == 0:
raise EOFError("")
msg_len, msg_start = _DecodeVarint32(read_buffer, 0)
msg = MSG()
msg.ParseFromString(read_buffer[msg_start:msg_start + msg_len])
rs = RS_Cvid()
rs.ParseFromString(msg.payload)
self.server_cvid = rs.server_cvid
self.sid = rs.sid
# The connection is now connected
with self.connection_state_cv:
self.connection_state = ConnectionState.CONNECTED
self.connection_state_cv.notify()
# Begin accepting messages
while True:
# TODO there may be another message in the read buffer
# Read from the socket
read = self.tls.recv_into(read_buffer, 4096)
if read == 0:
raise EOFError("")
n = 0
while n < read:
msg_len, n = _DecodeVarint32(read_buffer, n)
msg = MSG()
msg.ParseFromString(read_buffer[n:n + msg_len])
# Place message in response map
with self.response_map_cv:
self.response_map[msg.id] = msg
self.response_map_cv.notify()
# The connection is now closed
with self.connection_state_cv:
self.connection_state = ConnectionState.CLOSED
self.connection_state_cv.notify()
print '\n' + "size: "+str(size)+" position: "+str(position) + '\n'
if size == 4294967295: # this is a hack my friend we probably need fixlength types for the length
break
# 1) Reading the keyValue pair from the socket
toRead = size+position-READING_BYTES
buf += inSock.recv(toRead) # this is probably inefficient because the buffer sizes changes all the time
print("bufSize "+str(len(buf)))
kv = keyValue_pb2.KeyValuePair()
kv.ParseFromString(buf[position:position+size])
print("key "+kv.key)
print("value "+kv.value)
kvs = keyValue_pb2.keyValueStream()
kvp = kvs.record.add()
kvp.key = "key1"
kvp.value = "value1"
kvp = kvs.record.add()
kvp.key = "key2"
kvp.value = "value2"
# 2) Reading the keyValue pair from the socket
outBuf = kvs.SerializeToString()
print "Sending back to java- outbuf-len: " + str(len(outBuf))
buf = encoder._VarintBytes(len(outBuf))
inSock.send(buf)
inSock.send(outBuf)
print "Got -1, Finishing python process"
import metric_pb2
from google.protobuf.internal.encoder import _VarintBytes
from google.protobuf.internal.decoder import _DecodeVarint32
with open('out.bin', 'wb') as f:
for i in range(128):
my_metric = metric_pb2.Metric()
my_metric.name = 'sys.cpu'
my_metric.type = 'gauge'
my_metric.value = i
my_metric.tags = str(i) + 'tag'
size = my_metric.ByteSize()
f.write(_VarintBytes(size))
f.write(my_metric.SerializeToString())
with open('out.bin', 'rb') as f:
buf = f.read()
n = 0
while n < len(buf):
msg_len, new_pos = _DecodeVarint32(buf, n)
n = new_pos
msg_buf = buf[n:n + msg_len]
n += msg_len
read_metric = metric_pb2.Metric()
read_metric.ParseFromString(msg_buf)
# do something with read_metric
print(read_metric)
def readBlock(self, length, pool_id, block_id, generation_stamp, offset,
check_crc):
'''Send a read request to given block. If we receive a successful response,
we start reading packets.
Send read request:
+---------------------------------------------------------------------+
| Data Transfer Protocol Version, 2 bytes |
+---------------------------------------------------------------------+
| Op code, 1 byte (READ_BLOCK = 81) |
+---------------------------------------------------------------------+
| Delimited serialized OpReadBlockProto (varint len + request) |
+---------------------------------------------------------------------+
Receive response:
+---------------------------------------------------------------------+
| Delimited BlockOpResponseProto (varint len + response) |
+---------------------------------------------------------------------+
Start reading packets. Each packet has the following structure:
+---------------------------------------------------------------------+
| Packet length (4 bytes/32 bit int) |
+---------------------------------------------------------------------+
| Serialized size of header, 2 bytes |
+---------------------------------------------------------------------+
| Packet Header Proto |
+---------------------------------------------------------------------+
| x checksums, 4 bytes each |
+---------------------------------------------------------------------+
| x chunks of payload data |
+---------------------------------------------------------------------+
'''
log.debug("%s sending readBlock request" % self)
# Send version and opcode
self.sock.send(struct.pack('>h', 28))
self.sock.send(struct.pack('b', self.READ_BLOCK))
length = length - offset
# Create and send OpReadBlockProto message
request = datatransfer_proto.OpReadBlockProto()
request.offset = offset
request.len = length
header = request.header
header.clientName = "snakebite"
base_header = header.baseHeader
block = base_header.block
block.poolId = pool_id
block.blockId = block_id
block.generationStamp = generation_stamp
s_request = request.SerializeToString()
log_protobuf_message("OpReadBlockProto:", request)
delimited_request = encoder._VarintBytes(len(s_request)) + s_request
self.sock.send(delimited_request)
byte_stream = RpcBufferedReader(self.sock)
block_op_response_bytes = get_delimited_message_bytes(byte_stream)
block_op_response = datatransfer_proto.BlockOpResponseProto()
block_op_response.ParseFromString(block_op_response_bytes)
log_protobuf_message("BlockOpResponseProto", block_op_response)
checksum_type = block_op_response.readOpChecksumInfo.checksum.type
bytes_per_chunk = block_op_response.readOpChecksumInfo.checksum.bytesPerChecksum
log.debug("Checksum type: %s, bytesPerChecksum: %s" %
(checksum_type, bytes_per_chunk))
if checksum_type in [self.CHECKSUM_CRC32C, self.CHECKSUM_CRC32]:
checksum_len = 4
else:
raise Exception("Checksum type %s not implemented" % checksum_type)
total_read = 0
if block_op_response.status == 0: # datatransfer_proto.Status.Value('SUCCESS')
while total_read < length:
log.debug("== Reading next packet")
packet_len = struct.unpack("!I", byte_stream.read(4))[0]
log.debug("Packet length: %s", packet_len)
serialized_size = struct.unpack("!H", byte_stream.read(2))[0]
log.debug("Serialized size: %s", serialized_size)
packet_header_bytes = byte_stream.read(serialized_size)
packet_header = datatransfer_proto.PacketHeaderProto()
packet_header.ParseFromString(packet_header_bytes)
log_protobuf_message("PacketHeaderProto", packet_header)
data_len = packet_header.dataLen
chunks_per_packet = int(
(data_len + bytes_per_chunk - 1) / bytes_per_chunk)
log.debug("Nr of chunks: %d", chunks_per_packet)
data_len = packet_len - 4 - chunks_per_packet * checksum_len
log.debug("Payload len: %d", data_len)
byte_stream.reset()
# Collect checksums
if check_crc:
checksums = []
for _ in xrange(0, chunks_per_packet):
checksum = self._read_bytes(checksum_len)
checksum = struct.unpack("!I", checksum)[0]
checksums.append(checksum)
else:
self._read_bytes(checksum_len * chunks_per_packet)
# We use a fixed size buffer (a "load") to read only a couple of chunks at once.
bytes_per_load = self.LOAD_SIZE - (self.LOAD_SIZE %
bytes_per_chunk)
chunks_per_load = int(bytes_per_load / bytes_per_chunk)
loads_per_packet = int(
math.ceil(bytes_per_chunk * chunks_per_packet /
bytes_per_load))
read_on_packet = 0
for i in range(loads_per_packet):
load = ''
for j in range(chunks_per_load):
log.debug("Reading chunk %s in load %s:", j, i)
bytes_to_read = min(bytes_per_chunk,
data_len - read_on_packet)
chunk = self._read_bytes(bytes_to_read)
if check_crc:
checksum_index = i * chunks_per_load + j
if checksum_index < len(checksums) and crc(
chunk) != checksums[checksum_index]:
raise Exception("Checksum doesn't match")
load += chunk
total_read += len(chunk)
read_on_packet += len(chunk)
yield load
# Send ClientReadStatusProto message confirming successful read
request = datatransfer_proto.ClientReadStatusProto()
request.status = 0 # SUCCESS
s_request = request.SerializeToString()
log_protobuf_message("ClientReadStatusProto:", request)
delimited_request = encoder._VarintBytes(
len(s_request)) + s_request
self.sock.send(delimited_request)
self._close_socket()
def serialize_for_stream(self, message):
protobuf_message = ProtobufFormatter.serialize(message)
delimiter = encoder._VarintBytes(len(protobuf_message))
return delimiter + protobuf_message
def login_req2buf(name, password):
# 随机生成16位登录包AesKey
login_aes_key = bytes(''.join(random.sample(string.ascii_letters + string.digits, 16)), encoding="utf8")
# protobuf组包1
accountRequest = mm_pb2.ManualAuthAccountRequest(
aes=mm_pb2.ManualAuthAccountRequest.AesKey(
len=16,
key=login_aes_key
),
ecdh=mm_pb2.ManualAuthAccountRequest.Ecdh(
nid=713,
ecdhKey=mm_pb2.ManualAuthAccountRequest.Ecdh.EcdhKey(
len=len(Util.EcdhPubKey),
key=Util.EcdhPubKey
)
),
userName=name,
password1=Util.GetMd5(password),
password2=Util.GetMd5(password)
)
# protobuf组包2
deviceRequest = mm_pb2.ManualAuthDeviceRequest(
login=mm_pb2.LoginInfo(
aesKey=login_aes_key,
uin=0,
guid=define.__GUID__ + '\0', # guid以\0结尾
clientVer=define.__CLIENT_VERSION__,
androidVer=define.__ANDROID_VER__,
unknown=1,
),
tag2=mm_pb2.ManualAuthDeviceRequest._Tag2(),
imei=define.__IMEI__,
softInfoXml=define.__SOFTINFO__.format(define.__IMEI__, define.__ANDROID_ID__, define.__MANUFACTURER__+" "+define.__MODELNAME__, define.__MOBILE_WIFI_MAC_ADDRESS__,
define.__CLIENT_SEQID_SIGN__, define.__AP_BSSID__, define.__MANUFACTURER__, "taurus", define.__MODELNAME__, define.__IMEI__),
unknown5=0,
clientSeqID=define.__CLIENT_SEQID__,
clientSeqID_sign=define.__CLIENT_SEQID_SIGN__,
loginDeviceName=define.__MANUFACTURER__+" "+define.__MODELNAME__,
deviceInfoXml=define.__DEVICEINFO__.format(
define.__MANUFACTURER__, define.__MODELNAME__),
language=define.__LANGUAGE__,
timeZone="8.00",
unknown13=0,
unknown14=0,
deviceBrand=define.__MANUFACTURER__,
deviceModel=define.__MODELNAME__+"armeabi-v7a",
osType=define.__ANDROID_VER__,
realCountry="cn",
unknown22=2, # Unknown
)
logger.debug("accountData protobuf数据:" + Util.b2hex(accountRequest.SerializeToString()))
logger.debug("deviceData protobuf数据:" + Util.b2hex(deviceRequest.SerializeToString()))
# 加密
reqAccount = Util.compress_and_rsa(accountRequest.SerializeToString())
reqDevice = Util.compress_and_aes(deviceRequest.SerializeToString(), login_aes_key)
logger.debug("加密后数据长度:reqAccount={},reqDevice={}".format(len(reqAccount), len(reqDevice[0])))
logger.debug("加密后reqAccount数据:" + Util.b2hex(reqAccount))
logger.debug("加密后reqDevice数据:" + Util.b2hex(reqDevice[0]))
# 封包包体
subheader = b''
subheader += struct.pack(">I", len(accountRequest.SerializeToString())) # accountData protobuf长度
subheader += struct.pack(">I", len(deviceRequest.SerializeToString())) # deviceData protobuf长度
subheader += struct.pack(">I", len(reqAccount)) # accountData RSA加密后长度
# 包体由头信息、账号密码加密后数据、硬件设备信息加密后数据3部分组成
body = subheader + reqAccount + reqDevice[0]
# 封包包头
header = bytearray(0)
header += bytes([0]) # 最后2bit：02--包体不使用压缩算法;前6bit:包头长度,最后计算
header += bytes([((0x7 << 4) + 0xf)]) # 07:RSA加密算法 0xf:cookie长度
header += struct.pack(">I", define.__CLIENT_VERSION__) # 客户端版本号 网络字节序
header += bytes([0]*4) # uin
header += bytes([0]*15) # coockie
header += encoder._VarintBytes(701) # cgi type
header += encoder._VarintBytes(len(body)) # body 压缩前长度
header += encoder._VarintBytes(len(body)) # body 压缩后长度(登录包不需要压缩body数据)
header += struct.pack(">B", define.__LOGIN_RSA_VER__) # RSA秘钥版本
header += b'\x01\x02' # Unknown Param
header[0] = (len(header) << 2) + 2 # 包头长度
# 组包
logger.debug('包体数据:' + str(body))
logger.debug('包头数据:' + str(header))
senddata = header + body
return (senddata, login_aes_key)
def writeDelimitedTo(message, stream):
message_str = message.SerializeToString()
delimiter = encoder._VarintBytes(len(message_str))
stream.write(delimiter + message_str)
#!/bin/python3 import msg_pb2 import socket import sys from google.protobuf.internal import encoder theip = sys.argv[1] print(theip) print(str(sys.argv)) msg = msg_pb2.PGPMessage() msg.type = msg_pb2.PGPMessage.SG_ACK msg.ack.ip = theip msg.ack.port = 6789 msg.ack.uuid = "123e4567-e89b-12d3-a456-426655440000" msg.ack.name = "username" msgs = msg.SerializeToString() msgl = encoder._VarintBytes(len(msgs)) msgf = msgl + msgs s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) s.sendto(msgf, (theip, 6789))
def sendMsg(self, resp):
#print "Sending info"
data = resp.SerializeToString()
self.con.sendall(encoder._VarintBytes(len(data)) + data)
def proto_length_serializer(data: Any) -> bytes:
f = io.BytesIO()
f.write(_VarintBytes(data.ByteSize()))
f.write(data.SerializeToString())
f.seek(0)
return f.read()
def encode_delimited(cls, x, typ):
""" Encode a message or value with size information
(for use in a delimited communication stream) """
data = cls.encode(x, typ)
delimiter = protobuf_encoder._VarintBytes(len(data))
return delimiter + data
def deliminate_msg(packetMessage):
serializedMessage = packetMessage.SerializeToString()
delimiter = encoder._VarintBytes(len(serializedMessage))
return delimiter + serializedMessage
''' Send to server in the order given above'''
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
sock.settimeout(10)
sock.connect(("localhost", 8020))
sock.send("hrpc") # header
sock.send(struct.pack('B', 9)) # version
sock.send(struct.pack('B', 0x00)) # RPC service class
sock.send(struct.pack('B', 0x00)) # auth none
sock.sendall(struct.pack('!I', rpcipc_length) +
encoder._VarintBytes(len(s_rpcrequestheader)) +
s_rpcrequestheader +
encoder._VarintBytes(len(s_context)) +
s_context)
'''
Create the Hadoop RPC protocol looks like this for sending requests:
When sending requests
+---------------------------------------------------------------------+
| Length of the next three parts (4 bytes/32 bit int) |
+---------------------------------------------------------------------+
| Delimited serialized RpcRequestHeaderProto (varint len + header) |
+---------------------------------------------------------------------+
| Delimited serialized RequestHeaderProto (varint len + header) |
+---------------------------------------------------------------------+
def write(self, message):
serialized = message.SerializeToString()
size = len(serialized)
self._writer.write(_VarintBytes(size))
self._writer.write(serialized)
self._writer.flush()
def send_request_on_socket(request, socket):
serialized_request = request.SerializeToString()
size = request.ByteSize()
socket.sendall(_VarintBytes(size) + serialized_request)
def login_req2buf(name,password):
#随机生成16位登录包AesKey
login_aes_key = bytes(''.join(random.sample(string.ascii_letters + string.digits, 16)), encoding = "utf8")
#protobuf组包1
accountRequest = mm_pb2.ManualAuthAccountRequest(
aes = mm_pb2.ManualAuthAccountRequest.AesKey(
len = 16,
key = login_aes_key
),
ecdh = mm_pb2.ManualAuthAccountRequest.Ecdh(
nid = 713,
ecdhKey = mm_pb2.ManualAuthAccountRequest.Ecdh.EcdhKey(
len = len(Util.EcdhPubKey),
key = Util.EcdhPubKey
)
),
userName = name,
password1 = Util.GetMd5(password),
password2 = Util.GetMd5(password)
)
#protobuf组包2
deviceRequest = mm_pb2.ManualAuthDeviceRequest(
login = mm_pb2.LoginInfo(
aesKey = login_aes_key,
uin = 0,
guid = define.__GUID__ + '\0', #guid以\0结尾
clientVer = define.__CLIENT_VERSION__,
androidVer = define.__ANDROID_VER__,
unknown = 1,
),
tag2 = mm_pb2.ManualAuthDeviceRequest._Tag2(),
imei = define.__IMEI__,
softInfoXml = define.__SOFTINFO__.format(define.__IMEI__,define.__ANDROID_ID__, define.__MANUFACTURER__+" "+define.__MODELNAME__, define.__MOBILE_WIFI_MAC_ADDRESS__, define.__CLIENT_SEQID_SIGN__, define.__AP_BSSID__, define.__MANUFACTURER__,"taurus", define.__MODELNAME__, define.__IMEI__),
unknown5 = 0,
clientSeqID = define.__CLIENT_SEQID__,
clientSeqID_sign = define.__CLIENT_SEQID_SIGN__,
loginDeviceName = define.__MANUFACTURER__+" "+define.__MODELNAME__,
deviceInfoXml = define.__DEVICEINFO__.format(define.__MANUFACTURER__, define.__MODELNAME__),
language = define.__LANGUAGE__,
timeZone = "8.00",
unknown13 = 0,
unknown14 = 0,
deviceBrand = define.__MANUFACTURER__,
deviceModel = define.__MODELNAME__+"armeabi-v7a",
osType = define.__ANDROID_VER__,
realCountry = "cn",
unknown22 = 2, #Unknown
)
logger.debug("accountData protobuf数据:" + str(accountRequest.SerializeToString()))
logger.debug("deviceData protobuf数据:" + str(deviceRequest.SerializeToString()))
#加密
reqAccount = Util.compress_and_rsa(accountRequest.SerializeToString())
reqDevice = Util.compress_and_aes(deviceRequest.SerializeToString(),login_aes_key)
logger.debug("加密后数据长度:reqAccount={},reqDevice={}".format(len(reqAccount),len(reqDevice[0])))
logger.debug("加密后reqAccount数据:" + str(reqAccount))
logger.debug("加密后reqDevice数据:" + str(reqDevice[0]))
#封包包体
subheader = b''
subheader += struct.pack(">I",len(accountRequest.SerializeToString())) #accountData protobuf长度
subheader += struct.pack(">I",len(deviceRequest.SerializeToString())) #deviceData protobuf长度
subheader += struct.pack(">I",len(reqAccount)) #accountData RSA加密后长度
body = subheader + reqAccount + reqDevice[0] #包体由头信息、账号密码加密后数据、硬件设备信息加密后数据3部分组成
#封包包头
header = bytearray(0)
header += bytes([0]) #最后2bit：02--包体不使用压缩算法;前6bit:包头长度,最后计算 #
header += bytes([((0x7<<4) + 0xf)]) #07:RSA加密算法 0xf:cookie长度
header += struct.pack(">I",define.__CLIENT_VERSION__) #客户端版本号 网络字节序
header += bytes([0]*4) #uin
header += bytes([0]*15) #coockie
header += encoder._VarintBytes(701) #cgi type
header += encoder._VarintBytes(len(body)) #body 压缩前长度
header += encoder._VarintBytes(len(body)) #body 压缩后长度(登录包不需要压缩body数据)
header += struct.pack(">B",define.__LOGIN_RSA_VER__) #RSA秘钥版本
header += b'\x01\x02' #Unknown Param
header[0] = (len(header)<<2) + 2 #包头长度
#组包
logger.debug('包体数据:' + str(body))
logger.debug('包头数据:' + str(header))
senddata = header + body
return (senddata,login_aes_key)
def encode_message_with_size(cls, message):
""" Encode a protobuf message, prepended with its size """
data = message.SerializeToString()
size = protobuf_encoder._VarintBytes(len(data))
return size + data
