def __init__(self):
self.__dict = defaultdict(None)
self.__codec = String('utf-8')
self.encode = WeakMethod(self._encode_self)
self.__kv_split = Property.KV_SPLIT
self.__newline = Property.NEWLINE
self.encode = WeakMethod(self._encode_self)
self.str_cache = None
class JoyQueueHeader(Header):
MAGIC = ~(0xCAFEBEBE ^ 0xFFFFFFFF)
VERSION = 2
SCHEMA = Schema(('magic', Int32), ('version', Int8), ('identity', Int8),
('requestId', Int32), ('type', Int8), ('send_time', Int64),
('status', Int8), ('error', String('utf-8')))
def __init__(self, magic, version, identity, requestId, type, send_time,
status, error):
self.magic = magic
self.version = version
self.identity = identity
self.requestId = requestId
self.type = type
self.send_time = send_time
self.status = status
self.error = error
self.encode = WeakMethod(self._encode_self)
def _encode_self(self):
# return super()._encode_self()
request_or_response = self.identity & 0b00000001
if request_or_response == 0:
values = (self.magic, self.version, self.identity, self.requestId,
self.type, self.send_time)
base_fields = self.SCHEMA.fields
bits = []
for i in range(len(values)):
bits.append(base_fields[i].encode(values[i]))
return b''.join(bits)
else:
return super().encode()
@classmethod
def defaultHeader(cls, request):
magic = ~(0xCAFEBEBE ^ 0xFFFFFFFF)
version = 2
# request and qos level receive
identity = 0b00000010
requestId = 78237
type = request.TYPE
send_time = int(time.time())
status = 0
error = None
return JoyQueueHeader(magic, version, identity, requestId, type,
send_time, status, error)
from joyqueue.protocol.interface import Request, Response
from joyqueue.protocol.types import Schema, Array, String, ByteString, Int8, Int16, Int32, Int64, Boolean
from joyqueue.protocol.command_key import (FETCH_CLUSTER_REQUEST, FETCH_CLUSTER_RESPONSE,\
ADD_CONNECTION_REQUEST, ADD_CONNECTION_RESPONSE,
ADD_PRODUCER_REQUEST, ADD_PRODUCER_RESPONSE,
ADD_CONSUMER_REQUEST, ADD_CONSUMER_RESPONSE)
from io import BytesIO
from joyqueue.util import WeakMethod
UTF8String = String('utf-8')
UTF8ByteString = ByteString('utf-8')
class MetadataRequest(Request):
# STR = String('utf-8')
SCHEMA = Schema(('topics', Array(UTF8String)),
('app', UTF8String))
TYPE = FETCH_CLUSTER_REQUEST
class Weight(Response):
SCHEMA = Schema(('brokerId', UTF8String),
('weight', Int16))
class ProducerPolicy(Response):
SCHEMA = Schema(('nearby', Boolean),
('single', Boolean),
('archive', Boolean),
('weight', Array(Weight)),
('blackList', Array(UTF8String)),
('timeout', Int32))
def test_array_string():
topics = ['test_topic']
bytes=Array(String()).encode(topics)
print(bytes)
class Message(Request):
SCHEMA = Schema(('length', Int32), ('partition', Int16), ('index', Int64),
('term', Int32), ('magic', Int16), ('system_code', Int16),
('priority', Int8), ('clientIp', Bytes),
('send_time', Int64), ('store_time', Int32),
('body_crc', Int64), ('flag', Int16), ('body', Bytes),
('bussiness_id', ByteString('utf-8')),
('attributes', String('utf-8')), ('extension', Bytes),
('app', ByteString('utf-8')))
CODEC_MASK = 0x07
CODEC_GZIP = 0x01
CODEC_SNAPPY = 0x02
CODEC_LZ4 = 0x03
TIMESTAMP_TYPE_MASK = 0x08
SYSTEM_CODE = 0x000000000000
MAGIC = 0x1234
CLIENT_IP = b'abcdefabcdef'
HEADER_SIZE = 40 # length(4),partition(1),index(8),term(4),system_code(1),
# priority(1),send_time(8),store_time(4),body_crc(8),flag(1)
TYPE = 'Message'
def __init__(self,
body,
bussiness_id,
attributes,
extension,
app,
length=-1,
partition=0,
index=-1,
term=-1,
magic=MAGIC,
system_code=0,
priority=-1,
client_ip=CLIENT_IP,
send_time=int(time.time()) * 1000,
store_time=1,
body_crc=-1,
flag=0):
assert body is None or isinstance(body, bytes), 'value must be bytes'
self.length = length
self.partition = partition
self.index = index
self.term = term
self.magic = magic
self.system_code = system_code
self.priority = priority
self.clientIp = client_ip
self.send_time = send_time
self.store_time = store_time
self.body_crc = body_crc
self.flag = flag
self.body = body
self.bussiness_id = bussiness_id
self.attributes = attributes
self.extension = extension
self.app = app
self.encode = WeakMethod(self._encode_self)
def _encode_self(self, recalc_crc=True):
fields = (self.length, self.partition, self.index, self.term,
self.magic, self.system_code, self.priority, self.clientIp,
self.send_time, self.store_time, self.body_crc, self.flag,
self.body, self.bussiness_id, self.attributes,
self.extension, self.app)
message = self.SCHEMA.encode(fields)
return message
@classmethod
def encode(cls, item):
it = cls(*item)
return it.encode()
@classmethod
def decode(cls, data):
if isinstance(data, bytes):
data = io.BytesIO(data)
# Partial decode required to determine message version
base_fields = cls.SCHEMA.fields[0:7]
length, partition, index, term, magic, system_code, priority = [
field.decode(data) for field in base_fields
]
client_ip = data.read(16)
remaining = cls.SCHEMA.fields[8:]
send_time, store_time, body_crc, flag, body, bussiness_id, attr, extension, app = [
field.decode(data) for field in remaining
]
msg = cls(body, bussiness_id, attr, extension, app, length, partition,
index, term, magic, system_code, priority, client_ip,
send_time, store_time, body_crc, flag)
return msg
def validate_crc(self):
pass
def is_compressed(self):
return self.attributes & self.CODEC_MASK != 0
def decompress(self):
pass
def __hash__(self):
return hash(self._encode_self(recalc_crc=False))