def __init__(self, *array_of):
if len(array_of) > 1:
self.array_of = Schema(*array_of)
elif len(array_of) == 1 and (isinstance(array_of[0], AbstractType)
or issubclass(array_of[0], AbstractType)
or isinstance(array_of[0], Schema)):
self.array_of = array_of[0]
else:
raise ValueError('Array instantiated with no array_of type')
class Application(Model):
SCHEMA = Schema(('username', UTF8String), ('password', UTF8String),
('app', UTF8String), ('token', UTF8String),
('region', UTF8String), ('namespace', UTF8String),
('language', Int8), ('version', UTF8String),
('ip', UTF8String), ('time', Int64), ('sequence', Int64))
TYPE = 'Application'
class ProducerPolicy(Response):
SCHEMA = Schema(('nearby', Boolean),
('single', Boolean),
('archive', Boolean),
('weight', Array(Weight)),
('blackList', Array(UTF8String)),
('timeout', Int32))
TYPE = 'Producer Policy'
class Topic(Response):
SCHEMA = Schema(('topicCode', UTF8String),
('hasProducerPolicy', Boolean),
('producerPolicy', ProducerPolicy),
('hasConsumerPolicy', Boolean),
('consumerPolicy', ConsumerPolicy),
('type', Int8),
('partitionGroups', Array(PartitionGroup)),
('code', Int32))
TYPE = 'Topic'
def __init__(self, topic_code, hasProducerPolicy, producerPolicy,hasConsumerPolicy,consumerPolicy,type,partitionGroups,code):
self.topicCode = topic_code
self.hasProducerPolicy = hasProducerPolicy
self.producerPolicy = producerPolicy
self.hasConsumerPolicy = hasConsumerPolicy
self.consumerPolicy = consumerPolicy
self.type = type
self.partitionGroups = partitionGroups
self.code = code
self.encode = WeakMethod(self._encode_self)
def _encode_self(self):
list = []
list.append(self.SCHEMA.fields[0].encode(self.code))
list.append(self.SCHEMA.fields[1].encode(self.hasProducerPolicy))
if self.hasProducerPolicy:
list.append(self.SCHEMA.fields[2].encode(self.producerPolicy))
list.append(self.SCHEMA.fields[3].encode(self.hasConsumerPolicy))
if self.hasConsumerPolicy:
list.append(self.SCHEMA.fields[4].encode(self.consumerPolicy))
list.append(self.SCHEMA.fields[5].encode(self.type))
list.append(self.SCHEMA.fields[6].encode(self.partitionGroups))
return b''.join(list)
@classmethod
def encode(cls, item):
it = cls(*item)
return it.encode()
@classmethod
def decode(cls, data):
if isinstance(data, bytes):
data = BytesIO(data)
base_fields = cls.SCHEMA.fields[0:2]
topic_code, has_producer_policy = [f.decode(data) for f in base_fields]
producerPolicy = None
consumerPolicy = None
if has_producer_policy:
producerPolicy = cls.SCHEMA.fields[2].decode(data)
has_consumer_policy = cls.SCHEMA.fields[3].decode(data)
if has_consumer_policy:
consumerPolicy = cls.SCHEMA.fields[4].decode(data)
type, partitionGroups, code = [f.decode(data) for f in cls.SCHEMA.fields[5:]]
return cls(topic_code, has_producer_policy, producerPolicy, has_consumer_policy,
consumerPolicy, type, partitionGroups, code)
class FetchMessageRequest(Request):
SCHEMA = Schema(('topics', Array(FetchTopic)), ('app', UTF8String),
('ackTimeout', Int32), ('longPollTimeout', Int32))
TYPE = FETCH_TOPIC_MESSAGE_REQUEST
@classmethod
def default_fetch_message_request(cls, topic, app, batch_size):
topics = [(topic, batch_size)]
fetch_message = (topics, app, 5000, 10000)
return FetchMessageRequest(*fetch_message)
class ProduceMessageRequest(Request):
SCHEMA = Schema(('topics', Array(TopicProduceMessage)),
('app', UTF8String))
TYPE = PRODUCE_MESSAGE_REQUEST
@classmethod
def default_messages_request(cls, topic, producer, msg):
msgs = [(msg, 'deault_bussiness_id', None, None, producer)]
topic_messages = [(topic, None, 5000, 2, msgs)]
return ProduceMessageRequest(topic_messages, producer)
class Broker(Response):
SCHEMA = Schema(('id', Int32),
('host', UTF8String),
('port', Int32),
('dataCenter', UTF8String),
('nearby', Boolean),
('weight', Int32),
('systemCode', Int32),
('permission', Int32))
TYPE = 'Broker'
class ConnectionRequest(Request):
SCHEMA = Schema(('username', UTF8ByteString),
('password', UTF8ByteString),
('app', UTF8ByteString),
('token', UTF8ByteString),
('region', UTF8ByteString),
('namespace', UTF8ByteString),
('language', Int8),
('version', UTF8ByteString),
('ip', UTF8ByteString),
('time', Int64),
('sequence', Int64))
TYPE = ADD_CONNECTION_REQUEST
class StructArray(AbstractType):
def __init__(self, *array_of):
if len(array_of) > 1:
self.array_of = Schema(*array_of)
elif len(array_of) == 1 and (isinstance(array_of[0], AbstractType)
or issubclass(array_of[0], AbstractType)
or isinstance(array_of[0], Schema)):
self.array_of = array_of[0]
else:
raise ValueError('Array instantiated with no array_of type')
"""
list.append(self.array_of(*item).encode())
diff
it = self.array_of(*item)
list.append(it.encode())
"""
def encode(self, items):
if items is None:
return Int16.encode(-1)
list = []
for item in items:
it = self.array_of(*item)
list.append(it.encode())
return b''.join([Int16.encode(len(items))] + list)
def decode(self, data):
length = Int16.decode(data)
if length == -1:
return None
return [self.array_of.decode(data) for _ in range(length)]
def repr(self, list_of_items):
if list_of_items is None:
return 'NULL'
return '[' + ', '.join(
[self.array_of.repr(item) for item in list_of_items]) + ']'
class ConsumerPolicy(Response):
SCHEMA = Schema(('nearby', Boolean),
('pause', Boolean),
('archive', Boolean),
('retry', Boolean),
('seq', Boolean),
('ackTimeout', Int32),
('backSize', Int16),
('concurrent', Boolean),
('concurrentSize', Int32),
('delay', Int32),
('backList', Array(UTF8String)),
('errTimes', Int32),
('maxPartitionNum', Int32),
('readRetryProportion', Int32))
TYPE = 'Consumer policy'
class JoyQueueHeader(Header):
MAGIC = ~(0xCAFEBEBE ^ 0xFFFFFFFF)
VERSION = 2
SCHEMA = Schema(('magic', Int32), ('version', Int8), ('identity', Int8),
('requestId', Int32), ('type', Int8), ('send_time', Int64),
('status', Int8), ('error', String('utf-8')))
def __init__(self, magic, version, identity, requestId, type, send_time,
status, error):
self.magic = magic
self.version = version
self.identity = identity
self.requestId = requestId
self.type = type
self.send_time = send_time
self.status = status
self.error = error
self.encode = WeakMethod(self._encode_self)
def _encode_self(self):
# return super()._encode_self()
request_or_response = self.identity & 0b00000001
if request_or_response == 0:
values = (self.magic, self.version, self.identity, self.requestId,
self.type, self.send_time)
base_fields = self.SCHEMA.fields
bits = []
for i in range(len(values)):
bits.append(base_fields[i].encode(values[i]))
return b''.join(bits)
else:
return super().encode()
@classmethod
def defaultHeader(cls, request):
magic = ~(0xCAFEBEBE ^ 0xFFFFFFFF)
version = 2
# request and qos level receive
identity = 0b00000010
requestId = 78237
type = request.TYPE
send_time = int(time.time())
status = 0
error = None
return JoyQueueHeader(magic, version, identity, requestId, type,
send_time, status, error)
class Model(object):
__metaclass__ = abc.ABCMeta
SCHEMA = Schema()
def __init__(self, *args, **kwargs):
if len(args) == len(self.SCHEMA.fields):
for i, name in enumerate(self.SCHEMA.names):
self.__dict__[name] = args[i]
elif len(args) > 0:
raise ValueError('Args must be empty or mirror schema')
else:
for name in self.SCHEMA.names:
self.__dict__[name] = kwargs.pop(name, None)
if kwargs:
raise ValueError(
'Keyword(s) not in schema %s: %s' %
(list(self.SCHEMA.names), ', '.join(kwargs.keys())))
def get(self, property_key):
return self.__dict__[property_key]
def set(self, key, value):
self.__dict__[key] = value
@abc.abstractproperty
def TYPE(self):
"""
Response type
"""
pass
@abc.abstractproperty
def SCHEMA(self):
"""
Serialize and Deserialize fields and types
"""
pass
def __repr__(self):
key_vals = []
for name in self.SCHEMA.names:
key_vals.append('%s=%s' % (name, self.__dict__[name]))
return self.__class__.__name__ + '(' + ', '.join(key_vals) + ')'
class ConsumePartitionAckResponse(Response):
SCHEMA = Schema(('partition', Int16),
('data', Array(ConsumePartitionIndexAckResponse)))
TYPE = 'Consume partition ack'
class ConsumerResponse(Response):
SCHEMA = Schema(('consumerIds', Array(ConsumerId)))
TYPE = ADD_CONSUMER_RESPONSE
class ConsumeTopicPartitionGroupAckResponse(Response):
SCHEMA = Schema(('topic', UTF8String),
('partitions', Array(ConsumePartitionAckResponse)))
TYPE = 'Consume topic partition group ack'
class ConsumerRequest(Request):
SCHEMA = Schema(('topics', Array(UTF8String)),
('app', UTF8String),
('sequence', Int64))
TYPE = ADD_CONSUMER_REQUEST
class Weight(Response):
SCHEMA = Schema(('brokerId', UTF8String),
('weight', Int16))
class PartitionMetadata(Model):
SCHEMA = Schema(('id', Int16), ('partitionGroupId', Int32),
('topic', UTF8String), ('leader', Broker))
TYPE = 'Application'
class ConsumerId(Response):
SCHEMA = Schema(('topic', UTF8String),
('consumerId', UTF8String))
TYPE = 'Consumer id'
class ProducerRequest(Request):
SCHEMA = Schema(('topics', Array(UTF8String)),
('app', UTF8String),
('sequence', Int64))
TYPE = ADD_PRODUCER_REQUEST
class ProducerId(Response):
SCHEMA = Schema(('topic', UTF8String),
('producerId', UTF8String))
TYPE = 'Producer id'
class ConsumeAck(Request):
SCHEMA = Schema(('topics', Array(ConsumeTopicPartitionGroupAckResponse)),
('app', UTF8String))
TYPE = COMMIT_ACK_REQUEST
class ConnectionResponse(Response):
SCHEMA = Schema(('connectionId', UTF8ByteString),
('notification', UTF8String))
TYPE = ADD_CONNECTION_RESPONSE
class MetadataResponse(Response):
SCHEMA = Schema(('topics', Array(Topic)),
('brokers', Array(Broker)))
TYPE = FETCH_CLUSTER_RESPONSE
class ConsumePartitionAckState(Response):
SCHEMA = Schema(('partition', Int16), ('code', Int8))
TYPE = 'Consume partition ack state'
class ConsumeTopicPartitionAckState(Response):
SCHEMA = Schema(('topic', UTF8String),
('partitionData', Array(ConsumePartitionAckState)))
TYPE = 'Consume topic partition ack state'
class PartitionGroup(Response):
SCHEMA = Schema(('id', Int32),
('leader', Int32),
('partitions', Array(Int16)))
TYPE = 'Partition Group'
class ProducerResponse(Response):
SCHEMA = Schema(('producerIds', Array(ProducerId)))
TYPE = ADD_PRODUCER_RESPONSE
class MetadataRequest(Request):
# STR = String('utf-8')
SCHEMA = Schema(('topics', Array(UTF8String)),
('app', UTF8String))
TYPE = FETCH_CLUSTER_REQUEST
class ConsumeAckResponse(Response):
SCHEMA = Schema(('data', Array(ConsumeTopicPartitionAckState)))
TYPE = COMMIT_ACK_RESPONSE
