def __init__(self, msgBytes=None):
if (msgBytes != None):
self.msgBytes = msgBytes
self.bodySize = len(msgBytes)
self.byteBuffer = ByteBuffer(msgBytes)
self.version = self.byteBuffer.get()
self.attribute = self.byteBuffer.get()
self.crc32 = self.byteBuffer.getInt()
self.msgSize = self.bodySize - 6
class Message: def __init__(self,msgBytes = None): if(msgBytes != None): self.msgBytes = msgBytes; self.bodySize = len(msgBytes); self.byteBuffer = ByteBuffer(msgBytes); self.version = self.byteBuffer.get(); self.attribute = self.byteBuffer.get(); self.crc32 = self.byteBuffer.getInt(); self.msgSize = self.bodySize - 6; #6 bytes are version size + attribute size + crc32 bytes
def buildMessage(self, messageStr):
self.byteBuffer = ByteBuffer()
self.byteBuffer.put(1)
#magic
self.byteBuffer.put(0)
#attribute
crcValue = binascii.crc32(messageStr)
self.byteBuffer.putInt(crcValue)
#crc32
self.byteBuffer.putBytes(bytearray(messageStr))
self.messageSize = len(self.byteBuffer.array())
class StringMessageSet(object): """This is a message set""" def __init__(self,allMsgBytes): self.allMsgBytes = allMsgBytes; self.messageSetByteBuffer = ByteBuffer(self.allMsgBytes); self.messageSet = []; def getAllStringMessages(self): while(self.messageSetByteBuffer.remaining() > 0): msgSize = self.messageSetByteBuffer.getInt(); msgBytes = self.messageSetByteBuffer.getBytes(msgSize); self.messageSet.append(StringMessage(msgBytes)); return self.messageSet;
class StringMessageSet(object):
"""This is a message set"""
def __init__(self, allMsgBytes):
self.allMsgBytes = allMsgBytes
self.messageSetByteBuffer = ByteBuffer(self.allMsgBytes)
self.messageSet = []
def getAllStringMessages(self):
while (self.messageSetByteBuffer.remaining() > 0):
msgSize = self.messageSetByteBuffer.getInt()
msgBytes = self.messageSetByteBuffer.getBytes(msgSize)
self.messageSet.append(StringMessage(msgBytes))
return self.messageSet
def getData(self,path):
requestBytes = [0, 0, 0, -1, 0, 0, 0, 1, 0, 0, 0, 4];
pathLen = len(path);
pathLenList = list(struct.unpack("4b",struct.pack("!i",pathLen)));
for pathLenByte in pathLenList:
requestBytes.append(pathLenByte);#add path lenth
pathByteList = list(struct.unpack(str(pathLen)+"b",struct.pack("!" + str(pathLen) + "s",path)));
for pathByte in pathByteList:
requestBytes.append(pathByte);#add path
requestBytes.append(0);#add ifwatch
bodySize = len(requestBytes) - 4;
bodySizeList = list(struct.unpack("4b",struct.pack("!i",bodySize)));
for i in range(4):
requestBytes[i] = bodySizeList[i];
#print requestBytes;
trueBytes = bytearray(requestBytes);
self.zkSocket.sendall(trueBytes);
responseBytes = self.zkSocket.recv(1024);
byteBuffer = ByteBuffer(responseBytes);
byteBuffer.getInt();#msglen
byteBuffer.getInt();#xid
byteBuffer.getLong();#zxid
byteBuffer.getInt();#err
resultValue = byteBuffer.getString();
return resultValue;
def getRequestBytes(self,request):
requestKeySize = jafka_utils.SHORT_SIZE;
totalSize = requestKeySize + request.getSizeInBytes();
byteBuffer = ByteBuffer.allocate(totalSize);
byteBuffer.putInt(totalSize);
byteBuffer.putShort(request.getRequestKey());
byteBuffer.putBytes(request.toBytes());
return byteBuffer.array();
def getRequestBytes(self, request):
requestKeySize = jafka_utils.SHORT_SIZE
totalSize = requestKeySize + request.getSizeInBytes()
byteBuffer = ByteBuffer.allocate(totalSize)
byteBuffer.putInt(totalSize)
byteBuffer.putShort(request.getRequestKey())
byteBuffer.putBytes(request.toBytes())
return byteBuffer.array()
def buildMessage(self,messageStr): self.byteBuffer = ByteBuffer(); self.byteBuffer.put(1);#magic self.byteBuffer.put(0);#attribute crcValue = binascii.crc32(messageStr); self.byteBuffer.putInt(crcValue);#crc32 self.byteBuffer.putBytes(bytearray(messageStr)); self.messageSize = len(self.byteBuffer.array());
def toBytes(self):
byteBuffer = ByteBuffer.allocate(self.getSizeInBytes());
topicLen = len(self.topic);
byteBuffer.putShort(topicLen);
byteBuffer.putBytes(bytearray(self.topic));
byteBuffer.putInt(self.partition);
byteBuffer.putLong(self.offset);
byteBuffer.putInt(self.maxSize);
return byteBuffer.array();
def toBytes(self):
byteBuffer = ByteBuffer.allocate(self.getSizeInBytes())
topicLen = len(self.topic)
byteBuffer.putShort(topicLen)
byteBuffer.putBytes(bytearray(self.topic))
byteBuffer.putInt(self.partition)
byteBuffer.putLong(self.offset)
byteBuffer.putInt(self.maxSize)
return byteBuffer.array()
class StringMessage(Message):
def getMessage(self):
return str(self.byteBuffer.getBytes(self.msgSize))
def buildMessage(self, messageStr):
self.byteBuffer = ByteBuffer()
self.byteBuffer.put(1)
#magic
self.byteBuffer.put(0)
#attribute
crcValue = binascii.crc32(messageStr)
self.byteBuffer.putInt(crcValue)
#crc32
self.byteBuffer.putBytes(bytearray(messageStr))
self.messageSize = len(self.byteBuffer.array())
def toByteArray(self):
messageByteBuffer = ByteBuffer()
messageByteBuffer.putInt(self.messageSize)
messageByteBuffer.putBytes(self.byteBuffer.array())
return messageByteBuffer.array()
class StringMessage(Message): def getMessage(self): return str(self.byteBuffer.getBytes(self.msgSize)); def buildMessage(self,messageStr): self.byteBuffer = ByteBuffer(); self.byteBuffer.put(1);#magic self.byteBuffer.put(0);#attribute crcValue = binascii.crc32(messageStr); self.byteBuffer.putInt(crcValue);#crc32 self.byteBuffer.putBytes(bytearray(messageStr)); self.messageSize = len(self.byteBuffer.array()); def toByteArray(self): messageByteBuffer = ByteBuffer(); messageByteBuffer.putInt(self.messageSize); messageByteBuffer.putBytes(self.byteBuffer.array()); return messageByteBuffer.array();
def sendRequest(self, request):
requestBytes = self.getRequestBytes(request)
self.jafkaSocket.sendall(bytearray(requestBytes))
responseBytes = self.jafkaSocket.recv(64 * 1024)
#It is the same as java client
# print("receive responseBytes:"+str(len(responseBytes)));
responseByteBuffer = ByteBuffer(responseBytes, None)
msgByteSize = responseByteBuffer.getInt()
resultCode = responseByteBuffer.getShort()
if (resultCode != 0):
print("Can not get message success which resultCode:" +
str(resultCode))
return
while (len(responseBytes) != msgByteSize +
4): #if not receive completely we will get when read complete
responseBytes = responseBytes + self.jafkaSocket.recv(64 * 1024)
responseByteBuffer = ByteBuffer(responseBytes, None)
responseByteBuffer.currentIndex = responseByteBuffer.currentIndex + 6
#add 6 is because the responseBytes is new and we don't need read msgByteSize and resultCode
# print("receive responseBytes length:"+str(len(responseBytes)));
# print "receive message size:"+str(msgByteSize);
# print("resultCode:"+str(resultCode));
messageByteBuffer = responseByteBuffer.slice()
return messageByteBuffer
def sendRequest(self,request):
requestBytes = self.getRequestBytes(request);
self.jafkaSocket.sendall(bytearray(requestBytes));
responseBytes = self.jafkaSocket.recv(64 * 1024);#It is the same as java client
# print("receive responseBytes:"+str(len(responseBytes)));
responseByteBuffer = ByteBuffer(responseBytes,None);
msgByteSize = responseByteBuffer.getInt();
resultCode = responseByteBuffer.getShort();
if(resultCode != 0):
print("Can not get message success which resultCode:"+str(resultCode));
return;
while(len(responseBytes) != msgByteSize + 4):#if not receive completely we will get when read complete
responseBytes = responseBytes + self.jafkaSocket.recv(64 * 1024);
responseByteBuffer = ByteBuffer(responseBytes,None);
responseByteBuffer.currentIndex = responseByteBuffer.currentIndex + 6;#add 6 is because the responseBytes is new and we don't need read msgByteSize and resultCode
# print("receive responseBytes length:"+str(len(responseBytes)));
# print "receive message size:"+str(msgByteSize);
# print("resultCode:"+str(resultCode));
messageByteBuffer = responseByteBuffer.slice();
return messageByteBuffer;
def toBytes(self):
self.totalBytes = []
self.totolMessageBytesSize = 0
for message in self.messages:
self.totolMessageBytesSize = self.totolMessageBytesSize + len(
message.toByteArray())
messageBodyByteBuffer = ByteBuffer()
messageBodyByteBuffer.putShort(0)
#requestKey
messageBodyByteBuffer.putShort(len(bytearray(self.topicName)))
messageBodyByteBuffer.putBytes(bytearray(self.topicName))
messageBodyByteBuffer.putInt(-1)
#partition
messageBodyByteBuffer.putInt(self.totolMessageBytesSize)
for message in self.messages:
messageBodyByteBuffer.putBytes(message.toByteArray())
totolByteBuffer = ByteBuffer()
totolByteBuffer.putInt(len(messageBodyByteBuffer.array()))
totolByteBuffer.putBytes(messageBodyByteBuffer.array())
return totolByteBuffer.array()
# request = ProducerRequest("demo",["a","b"]);
# print request.toBytes();
# producter = Producer("localhost",9092);
# producter.connect();
# producter.sendMessage("demo",["liao","haha"]);
def getData(self, path):
requestBytes = [0, 0, 0, -1, 0, 0, 0, 1, 0, 0, 0, 4]
pathLen = len(path)
pathLenList = list(struct.unpack("4b", struct.pack("!i", pathLen)))
for pathLenByte in pathLenList:
requestBytes.append(pathLenByte)
#add path lenth
pathByteList = list(
struct.unpack(
str(pathLen) + "b", struct.pack("!" + str(pathLen) + "s",
path)))
for pathByte in pathByteList:
requestBytes.append(pathByte)
#add path
requestBytes.append(0)
#add ifwatch
bodySize = len(requestBytes) - 4
bodySizeList = list(struct.unpack("4b", struct.pack("!i", bodySize)))
for i in range(4):
requestBytes[i] = bodySizeList[i]
#print requestBytes;
trueBytes = bytearray(requestBytes)
self.zkSocket.sendall(trueBytes)
responseBytes = self.zkSocket.recv(1024)
byteBuffer = ByteBuffer(responseBytes)
byteBuffer.getInt()
#msglen
byteBuffer.getInt()
#xid
byteBuffer.getLong()
#zxid
byteBuffer.getInt()
#err
resultValue = byteBuffer.getString()
return resultValue
def toByteArray(self): messageByteBuffer = ByteBuffer(); messageByteBuffer.putInt(self.messageSize); messageByteBuffer.putBytes(self.byteBuffer.array()); return messageByteBuffer.array();
def __init__(self, allMsgBytes):
self.allMsgBytes = allMsgBytes
self.messageSetByteBuffer = ByteBuffer(self.allMsgBytes)
self.messageSet = []
def toBytes(self):
self.totalBytes = [];
self.totolMessageBytesSize = 0;
for message in self.messages:
self.totolMessageBytesSize = self.totolMessageBytesSize + len(message.toByteArray());
messageBodyByteBuffer = ByteBuffer();
messageBodyByteBuffer.putShort(0);#requestKey
messageBodyByteBuffer.putShort(len(bytearray(self.topicName)));
messageBodyByteBuffer.putBytes(bytearray(self.topicName));
messageBodyByteBuffer.putInt(-1);#partition
messageBodyByteBuffer.putInt(self.totolMessageBytesSize);
for message in self.messages:
messageBodyByteBuffer.putBytes(message.toByteArray());
totolByteBuffer = ByteBuffer();
totolByteBuffer.putInt(len(messageBodyByteBuffer.array()));
totolByteBuffer.putBytes(messageBodyByteBuffer.array());
return totolByteBuffer.array();
# request = ProducerRequest("demo",["a","b"]);
# print request.toBytes();
# producter = Producer("localhost",9092);
# producter.connect();
# producter.sendMessage("demo",["liao","haha"]);
def __init__(self,allMsgBytes): self.allMsgBytes = allMsgBytes; self.messageSetByteBuffer = ByteBuffer(self.allMsgBytes); self.messageSet = [];
def toByteArray(self):
messageByteBuffer = ByteBuffer()
messageByteBuffer.putInt(self.messageSize)
messageByteBuffer.putBytes(self.byteBuffer.array())
return messageByteBuffer.array()