class CreatePartitionsRequest_v0(Request):
API_KEY = 37
API_VERSION = 0
RESPONSE_TYPE = CreatePartitionsResponse_v0
SCHEMA = Schema(('topic_partitions',
Array(('topic', String('utf-8')),
('new_partitions',
Schema(('count', Int32),
('assignment', Array(Int32)))))),
('timeout', Int32), ('validate_only', Boolean))
class ListGroupsResponse_v0(Response):
API_KEY = 16
API_VERSION = 0
SCHEMA = Schema(('error_code', Int16),
('groups',
Array(('group', String('utf-8')),
('protocol_type', String('utf-8')))))
class CreatePartitionsResponse_v0(Response):
API_KEY = 37
API_VERSION = 0
SCHEMA = Schema(('throttle_time_ms', Int32),
('topic_errors',
Array(('topic', String('utf-8')), ('error_code', Int16),
('error_message', String('utf-8')))))
class CreateTopicsResponse_v2(Response):
API_KEY = 19
API_VERSION = 2
SCHEMA = Schema(('throttle_time_ms', Int32),
('topic_error_codes',
Array(('topic', String('utf-8')), ('error_code', Int16),
('error_message', String('utf-8')))))
class ApiVersionResponse_v0(Response):
API_KEY = 18
API_VERSION = 0
SCHEMA = Schema(('error_code', Int16),
('api_versions',
Array(('api_key', Int16), ('min_version', Int16),
('max_version', Int16))))
class ListGroupsResponse_v1(Response):
API_KEY = 16
API_VERSION = 1
SCHEMA = Schema(('throttle_time_ms', Int32), ('error_code', Int16),
('groups',
Array(('group', String('utf-8')),
('protocol_type', String('utf-8')))))
class ProduceResponse_v0(Response):
API_KEY = 0
API_VERSION = 0
SCHEMA = Schema(('topics',
Array(('topic', String('utf-8')),
('partitions',
Array(('partition', Int32), ('error_code', Int16),
('offset', Int64))))))
class RequestHeader(Struct):
SCHEMA = Schema(('api_key', Int16), ('api_version', Int16),
('correlation_id', Int32), ('client_id', String('utf-8')))
def __init__(self, request, correlation_id=0, client_id='kafka-python'):
super(RequestHeader,
self).__init__(request.API_KEY, request.API_VERSION,
correlation_id, client_id)
class MetadataRequest_v0(Request):
API_KEY = 3
API_VERSION = 0
RESPONSE_TYPE = MetadataResponse_v0
SCHEMA = Schema(
('topics', Array(String('utf-8')))
)
ALL_TOPICS = None # Empty Array (len 0) for topics returns all topics
class OffsetCommitResponse_v0(Response):
API_KEY = 8
API_VERSION = 0
SCHEMA = Schema(
('topics',
Array(('topic', String('utf-8')),
('partitions', Array(('partition', Int32),
('error_code', Int16))))))
class OffsetFetchRequest_v0(Request):
API_KEY = 9
API_VERSION = 0 # zookeeper-backed storage
RESPONSE_TYPE = OffsetFetchResponse_v0
SCHEMA = Schema(('consumer_group', String('utf-8')),
('topics',
Array(('topic', String('utf-8')),
('partitions', Array(Int32)))))
class OffsetResponse_v1(Response):
API_KEY = 2
API_VERSION = 1
SCHEMA = Schema(('topics',
Array(('topic', String('utf-8')),
('partitions',
Array(('partition', Int32), ('error_code', Int16),
('timestamp', Int64), ('offset', Int64))))))
class ApiVersionResponse_v1(Response):
API_KEY = 18
API_VERSION = 1
SCHEMA = Schema(('error_code', Int16),
('api_versions',
Array(('api_key', Int16), ('min_version', Int16),
('max_version', Int16))),
('throttle_time_ms', Int32))
class ProduceRequest_v0(ProduceRequest):
API_VERSION = 0
RESPONSE_TYPE = ProduceResponse_v0
SCHEMA = Schema(
('required_acks', Int16), ('timeout', Int32),
('topics',
Array(('topic', String('utf-8')),
('partitions', Array(('partition', Int32),
('messages', Bytes))))))
class FetchResponse_v0(Response):
API_KEY = 1
API_VERSION = 0
SCHEMA = Schema(
('topics',
Array(('topics', String('utf-8')),
('partitions',
Array(('partition', Int32), ('error_code', Int16),
('highwater_offset', Int64), ('message_set', Bytes))))))
class OffsetFetchResponse_v0(Response):
API_KEY = 9
API_VERSION = 0
SCHEMA = Schema(
('topics',
Array(('topic', String('utf-8')),
('partitions',
Array(('partition', Int32), ('offset', Int64),
('metadata', String('utf-8')), ('error_code', Int16))))))
class ProduceResponse_v2(Response):
API_KEY = 0
API_VERSION = 2
SCHEMA = Schema(('topics',
Array(('topic', String('utf-8')),
('partitions',
Array(('partition', Int32), ('error_code', Int16),
('offset', Int64), ('timestamp', Int64))))),
('throttle_time_ms', Int32))
class SyncGroupRequest_v0(Request):
API_KEY = 14
API_VERSION = 0
RESPONSE_TYPE = SyncGroupResponse_v0
SCHEMA = Schema(('group', String('utf-8')), ('generation_id', Int32),
('member_id', String('utf-8')),
('group_assignment',
Array(('member_id', String('utf-8')),
('member_metadata', Bytes))))
class JoinGroupResponse_v0(Response):
API_KEY = 11
API_VERSION = 0
SCHEMA = Schema(
('error_code', Int16), ('generation_id', Int32),
('group_protocol', String('utf-8')), ('leader_id', String('utf-8')),
('member_id', String('utf-8')), ('members',
Array(('member_id', String('utf-8')),
('member_metadata', Bytes))))
class Struct(AbstractType):
SCHEMA = Schema()
def __init__(self, *args, **kwargs):
if len(args) == len(self.SCHEMA.fields):
for i, name in enumerate(self.SCHEMA.names):
self.__dict__[name] = args[i]
elif len(args) > 0:
raise ValueError('Args must be empty or mirror schema')
else:
for name in self.SCHEMA.names:
self.__dict__[name] = kwargs.pop(name, None)
if kwargs:
raise ValueError('Keyword(s) not in schema %s: %s'
% (list(self.SCHEMA.names),
', '.join(kwargs.keys())))
# overloading encode() to support both class and instance
# Without WeakMethod() this creates circular ref, which
# causes instances to "leak" to garbage
self.encode = WeakMethod(self._encode_self)
@classmethod
def encode(cls, item): # pylint: disable=E0202
bits = []
for i, field in enumerate(cls.SCHEMA.fields):
bits.append(field.encode(item[i]))
return b''.join(bits)
def _encode_self(self):
return self.SCHEMA.encode(
[self.__dict__[name] for name in self.SCHEMA.names]
)
@classmethod
def decode(cls, data):
if isinstance(data, bytes):
data = BytesIO(data)
return cls(*[field.decode(data) for field in cls.SCHEMA.fields])
def __repr__(self):
key_vals = []
for name, field in zip(self.SCHEMA.names, self.SCHEMA.fields):
key_vals.append('%s=%s' % (name, field.repr(self.__dict__[name])))
return self.__class__.__name__ + '(' + ', '.join(key_vals) + ')'
def __hash__(self):
return hash(self.encode())
def __eq__(self, other):
if self.SCHEMA != other.SCHEMA:
return False
for attr in self.SCHEMA.names:
if self.__dict__[attr] != other.__dict__[attr]:
return False
return True
class MetadataRequest_v4(Request):
API_KEY = 3
API_VERSION = 4
RESPONSE_TYPE = MetadataResponse_v4
SCHEMA = Schema(
('topics', Array(String('utf-8'))),
('allow_auto_topic_creation', Boolean)
)
ALL_TOPICS = -1 # Null Array (len -1) for topics returns all topics
NO_TOPICS = None # Empty array (len 0) for topics returns no topics
class OffsetCommitRequest_v0(Request):
API_KEY = 8
API_VERSION = 0 # Zookeeper-backed storage
RESPONSE_TYPE = OffsetCommitResponse_v0
SCHEMA = Schema(('consumer_group', String('utf-8')),
('topics',
Array(('topic', String('utf-8')),
('partitions',
Array(('partition', Int32), ('offset', Int64),
('metadata', String('utf-8')))))))
class FetchRequest_v0(Request):
API_KEY = 1
API_VERSION = 0
RESPONSE_TYPE = FetchResponse_v0
SCHEMA = Schema(('replica_id', Int32), ('max_wait_time', Int32),
('min_bytes', Int32),
('topics',
Array(('topic', String('utf-8')),
('partitions',
Array(('partition', Int32), ('offset', Int64),
('max_bytes', Int32))))))
class OffsetRequest_v1(Request):
API_KEY = 2
API_VERSION = 1
RESPONSE_TYPE = OffsetResponse_v1
SCHEMA = Schema(
('replica_id', Int32),
('topics',
Array(('topic', String('utf-8')),
('partitions', Array(('partition', Int32),
('timestamp', Int64))))))
DEFAULTS = {'replica_id': -1}
class JoinGroupRequest_v0(Request):
API_KEY = 11
API_VERSION = 0
RESPONSE_TYPE = JoinGroupResponse_v0
SCHEMA = Schema(('group', String('utf-8')), ('session_timeout', Int32),
('member_id', String('utf-8')),
('protocol_type', String('utf-8')),
('group_protocols',
Array(('protocol_name', String('utf-8')),
('protocol_metadata', Bytes))))
UNKNOWN_MEMBER_ID = ''
class OffsetRequest_v2(Request):
API_KEY = 2
API_VERSION = 2
RESPONSE_TYPE = OffsetResponse_v2
SCHEMA = Schema(
('replica_id', Int32),
('isolation_level', Int8), # <- added isolation_level
('topics',
Array(('topic', String('utf-8')),
('partitions', Array(('partition', Int32),
('timestamp', Int64))))))
DEFAULTS = {'replica_id': -1}
class ConsumerProtocolMemberAssignment(Struct):
SCHEMA = Schema(('version', Int16), ('assignment',
Array(('topic', String('utf-8')),
('partitions', Array(Int32)))),
('user_data', Bytes))
def partitions(self):
return [
TopicPartition(topic, partition) for topic, partitions in
self.assignment # pylint: disable-msg=no-member
for partition in partitions
]
class FetchRequest_v5(Request):
# This may only be used in broker-broker api calls
API_KEY = 1
API_VERSION = 5
RESPONSE_TYPE = FetchResponse_v5
SCHEMA = Schema(
('replica_id', Int32), ('max_wait_time', Int32), ('min_bytes', Int32),
('max_bytes', Int32), ('isolation_level', Int8),
('topics',
Array(('topic', String('utf-8')),
('partitions',
Array(('partition', Int32), ('fetch_offset', Int64),
('log_start_offset', Int64), ('max_bytes', Int32))))))
class FetchRequest_v4(Request):
# Adds isolation_level field
API_KEY = 1
API_VERSION = 4
RESPONSE_TYPE = FetchResponse_v4
SCHEMA = Schema(('replica_id', Int32), ('max_wait_time', Int32),
('min_bytes', Int32), ('max_bytes', Int32),
('isolation_level', Int8),
('topics',
Array(('topic', String('utf-8')),
('partitions',
Array(('partition', Int32), ('offset', Int64),
('max_bytes', Int32))))))
class OffsetCommitRequest_v1(Request):
API_KEY = 8
API_VERSION = 1 # Kafka-backed storage
RESPONSE_TYPE = OffsetCommitResponse_v1
SCHEMA = Schema(('consumer_group', String('utf-8')),
('consumer_group_generation_id', Int32),
('consumer_id', String('utf-8')),
('topics',
Array(('topic', String('utf-8')),
('partitions',
Array(('partition', Int32), ('offset', Int64),
('timestamp', Int64),
('metadata', String('utf-8')))))))
