def __init__(self, *array_of): if len(array_of) > 1: self.array_of = Schema(*array_of) elif len(array_of) == 1 and (isinstance(array_of[0], AbstractType) or issubclass(array_of[0], AbstractType) or isinstance(array_of[0], Schema)): self.array_of = array_of[0] else: raise ValueError('Array instantiated with no array_of type')
class Application(Model): SCHEMA = Schema(('username', UTF8String), ('password', UTF8String), ('app', UTF8String), ('token', UTF8String), ('region', UTF8String), ('namespace', UTF8String), ('language', Int8), ('version', UTF8String), ('ip', UTF8String), ('time', Int64), ('sequence', Int64)) TYPE = 'Application'
class ProducerPolicy(Response): SCHEMA = Schema(('nearby', Boolean), ('single', Boolean), ('archive', Boolean), ('weight', Array(Weight)), ('blackList', Array(UTF8String)), ('timeout', Int32)) TYPE = 'Producer Policy'
class Topic(Response): SCHEMA = Schema(('topicCode', UTF8String), ('hasProducerPolicy', Boolean), ('producerPolicy', ProducerPolicy), ('hasConsumerPolicy', Boolean), ('consumerPolicy', ConsumerPolicy), ('type', Int8), ('partitionGroups', Array(PartitionGroup)), ('code', Int32)) TYPE = 'Topic' def __init__(self, topic_code, hasProducerPolicy, producerPolicy,hasConsumerPolicy,consumerPolicy,type,partitionGroups,code): self.topicCode = topic_code self.hasProducerPolicy = hasProducerPolicy self.producerPolicy = producerPolicy self.hasConsumerPolicy = hasConsumerPolicy self.consumerPolicy = consumerPolicy self.type = type self.partitionGroups = partitionGroups self.code = code self.encode = WeakMethod(self._encode_self) def _encode_self(self): list = [] list.append(self.SCHEMA.fields[0].encode(self.code)) list.append(self.SCHEMA.fields[1].encode(self.hasProducerPolicy)) if self.hasProducerPolicy: list.append(self.SCHEMA.fields[2].encode(self.producerPolicy)) list.append(self.SCHEMA.fields[3].encode(self.hasConsumerPolicy)) if self.hasConsumerPolicy: list.append(self.SCHEMA.fields[4].encode(self.consumerPolicy)) list.append(self.SCHEMA.fields[5].encode(self.type)) list.append(self.SCHEMA.fields[6].encode(self.partitionGroups)) return b''.join(list) @classmethod def encode(cls, item): it = cls(*item) return it.encode() @classmethod def decode(cls, data): if isinstance(data, bytes): data = BytesIO(data) base_fields = cls.SCHEMA.fields[0:2] topic_code, has_producer_policy = [f.decode(data) for f in base_fields] producerPolicy = None consumerPolicy = None if has_producer_policy: producerPolicy = cls.SCHEMA.fields[2].decode(data) has_consumer_policy = cls.SCHEMA.fields[3].decode(data) if has_consumer_policy: consumerPolicy = cls.SCHEMA.fields[4].decode(data) type, partitionGroups, code = [f.decode(data) for f in cls.SCHEMA.fields[5:]] return cls(topic_code, has_producer_policy, producerPolicy, has_consumer_policy, consumerPolicy, type, partitionGroups, code)
class FetchMessageRequest(Request): SCHEMA = Schema(('topics', Array(FetchTopic)), ('app', UTF8String), ('ackTimeout', Int32), ('longPollTimeout', Int32)) TYPE = FETCH_TOPIC_MESSAGE_REQUEST @classmethod def default_fetch_message_request(cls, topic, app, batch_size): topics = [(topic, batch_size)] fetch_message = (topics, app, 5000, 10000) return FetchMessageRequest(*fetch_message)
class ProduceMessageRequest(Request): SCHEMA = Schema(('topics', Array(TopicProduceMessage)), ('app', UTF8String)) TYPE = PRODUCE_MESSAGE_REQUEST @classmethod def default_messages_request(cls, topic, producer, msg): msgs = [(msg, 'deault_bussiness_id', None, None, producer)] topic_messages = [(topic, None, 5000, 2, msgs)] return ProduceMessageRequest(topic_messages, producer)
class Broker(Response): SCHEMA = Schema(('id', Int32), ('host', UTF8String), ('port', Int32), ('dataCenter', UTF8String), ('nearby', Boolean), ('weight', Int32), ('systemCode', Int32), ('permission', Int32)) TYPE = 'Broker'
class ConnectionRequest(Request): SCHEMA = Schema(('username', UTF8ByteString), ('password', UTF8ByteString), ('app', UTF8ByteString), ('token', UTF8ByteString), ('region', UTF8ByteString), ('namespace', UTF8ByteString), ('language', Int8), ('version', UTF8ByteString), ('ip', UTF8ByteString), ('time', Int64), ('sequence', Int64)) TYPE = ADD_CONNECTION_REQUEST
class StructArray(AbstractType): def __init__(self, *array_of): if len(array_of) > 1: self.array_of = Schema(*array_of) elif len(array_of) == 1 and (isinstance(array_of[0], AbstractType) or issubclass(array_of[0], AbstractType) or isinstance(array_of[0], Schema)): self.array_of = array_of[0] else: raise ValueError('Array instantiated with no array_of type') """ list.append(self.array_of(*item).encode()) diff it = self.array_of(*item) list.append(it.encode()) """ def encode(self, items): if items is None: return Int16.encode(-1) list = [] for item in items: it = self.array_of(*item) list.append(it.encode()) return b''.join([Int16.encode(len(items))] + list) def decode(self, data): length = Int16.decode(data) if length == -1: return None return [self.array_of.decode(data) for _ in range(length)] def repr(self, list_of_items): if list_of_items is None: return 'NULL' return '[' + ', '.join( [self.array_of.repr(item) for item in list_of_items]) + ']'
class ConsumerPolicy(Response): SCHEMA = Schema(('nearby', Boolean), ('pause', Boolean), ('archive', Boolean), ('retry', Boolean), ('seq', Boolean), ('ackTimeout', Int32), ('backSize', Int16), ('concurrent', Boolean), ('concurrentSize', Int32), ('delay', Int32), ('backList', Array(UTF8String)), ('errTimes', Int32), ('maxPartitionNum', Int32), ('readRetryProportion', Int32)) TYPE = 'Consumer policy'
class JoyQueueHeader(Header): MAGIC = ~(0xCAFEBEBE ^ 0xFFFFFFFF) VERSION = 2 SCHEMA = Schema(('magic', Int32), ('version', Int8), ('identity', Int8), ('requestId', Int32), ('type', Int8), ('send_time', Int64), ('status', Int8), ('error', String('utf-8'))) def __init__(self, magic, version, identity, requestId, type, send_time, status, error): self.magic = magic self.version = version self.identity = identity self.requestId = requestId self.type = type self.send_time = send_time self.status = status self.error = error self.encode = WeakMethod(self._encode_self) def _encode_self(self): # return super()._encode_self() request_or_response = self.identity & 0b00000001 if request_or_response == 0: values = (self.magic, self.version, self.identity, self.requestId, self.type, self.send_time) base_fields = self.SCHEMA.fields bits = [] for i in range(len(values)): bits.append(base_fields[i].encode(values[i])) return b''.join(bits) else: return super().encode() @classmethod def defaultHeader(cls, request): magic = ~(0xCAFEBEBE ^ 0xFFFFFFFF) version = 2 # request and qos level receive identity = 0b00000010 requestId = 78237 type = request.TYPE send_time = int(time.time()) status = 0 error = None return JoyQueueHeader(magic, version, identity, requestId, type, send_time, status, error)
class Model(object): __metaclass__ = abc.ABCMeta SCHEMA = Schema() def __init__(self, *args, **kwargs): if len(args) == len(self.SCHEMA.fields): for i, name in enumerate(self.SCHEMA.names): self.__dict__[name] = args[i] elif len(args) > 0: raise ValueError('Args must be empty or mirror schema') else: for name in self.SCHEMA.names: self.__dict__[name] = kwargs.pop(name, None) if kwargs: raise ValueError( 'Keyword(s) not in schema %s: %s' % (list(self.SCHEMA.names), ', '.join(kwargs.keys()))) def get(self, property_key): return self.__dict__[property_key] def set(self, key, value): self.__dict__[key] = value @abc.abstractproperty def TYPE(self): """ Response type """ pass @abc.abstractproperty def SCHEMA(self): """ Serialize and Deserialize fields and types """ pass def __repr__(self): key_vals = [] for name in self.SCHEMA.names: key_vals.append('%s=%s' % (name, self.__dict__[name])) return self.__class__.__name__ + '(' + ', '.join(key_vals) + ')'
class ConsumePartitionAckResponse(Response): SCHEMA = Schema(('partition', Int16), ('data', Array(ConsumePartitionIndexAckResponse))) TYPE = 'Consume partition ack'
class ConsumerResponse(Response): SCHEMA = Schema(('consumerIds', Array(ConsumerId))) TYPE = ADD_CONSUMER_RESPONSE
class ConsumeTopicPartitionGroupAckResponse(Response): SCHEMA = Schema(('topic', UTF8String), ('partitions', Array(ConsumePartitionAckResponse))) TYPE = 'Consume topic partition group ack'
class ConsumerRequest(Request): SCHEMA = Schema(('topics', Array(UTF8String)), ('app', UTF8String), ('sequence', Int64)) TYPE = ADD_CONSUMER_REQUEST
class Weight(Response): SCHEMA = Schema(('brokerId', UTF8String), ('weight', Int16))
class PartitionMetadata(Model): SCHEMA = Schema(('id', Int16), ('partitionGroupId', Int32), ('topic', UTF8String), ('leader', Broker)) TYPE = 'Application'
class ConsumerId(Response): SCHEMA = Schema(('topic', UTF8String), ('consumerId', UTF8String)) TYPE = 'Consumer id'
class ProducerRequest(Request): SCHEMA = Schema(('topics', Array(UTF8String)), ('app', UTF8String), ('sequence', Int64)) TYPE = ADD_PRODUCER_REQUEST
class ProducerId(Response): SCHEMA = Schema(('topic', UTF8String), ('producerId', UTF8String)) TYPE = 'Producer id'
class ConsumeAck(Request): SCHEMA = Schema(('topics', Array(ConsumeTopicPartitionGroupAckResponse)), ('app', UTF8String)) TYPE = COMMIT_ACK_REQUEST
class ConnectionResponse(Response): SCHEMA = Schema(('connectionId', UTF8ByteString), ('notification', UTF8String)) TYPE = ADD_CONNECTION_RESPONSE
class MetadataResponse(Response): SCHEMA = Schema(('topics', Array(Topic)), ('brokers', Array(Broker))) TYPE = FETCH_CLUSTER_RESPONSE
class ConsumePartitionAckState(Response): SCHEMA = Schema(('partition', Int16), ('code', Int8)) TYPE = 'Consume partition ack state'
class ConsumeTopicPartitionAckState(Response): SCHEMA = Schema(('topic', UTF8String), ('partitionData', Array(ConsumePartitionAckState))) TYPE = 'Consume topic partition ack state'
class PartitionGroup(Response): SCHEMA = Schema(('id', Int32), ('leader', Int32), ('partitions', Array(Int16))) TYPE = 'Partition Group'
class ProducerResponse(Response): SCHEMA = Schema(('producerIds', Array(ProducerId))) TYPE = ADD_PRODUCER_RESPONSE
class MetadataRequest(Request): # STR = String('utf-8') SCHEMA = Schema(('topics', Array(UTF8String)), ('app', UTF8String)) TYPE = FETCH_CLUSTER_REQUEST
class ConsumeAckResponse(Response): SCHEMA = Schema(('data', Array(ConsumeTopicPartitionAckState))) TYPE = COMMIT_ACK_RESPONSE