def send_request_and_get_response(self, request, node_id=None):
"""
Sends a Kafka protocol request and returns
the associated response
"""
if node_id is None:
try:
node_id = self.get_controller()
except UndefinedController:
raise
except Exception as e:
raise KafkaManagerError(
'Cannot determine a controller for your current Kafka '
'server. Is your Kafka server running and available on '
'\'%s\' with security protocol \'%s\'? Are you using the '
'library versions from given \'requirements.txt\'? '
'Exception was: %s' %
(self.client.config['bootstrap_servers'],
self.client.config['security_protocol'], e))
if self.connection_check(node_id):
future = self.client.send(node_id, request)
self.client.poll(future=future)
if future.succeeded():
return future.value
else:
raise KafkaManagerError(
'Error while sending request %s to Kafka server: %s.' %
(request, future.exception))
else:
raise KafkaManagerError(
'Connection is not ready, please check your client '
'and server configurations.')
def get_assignment_for_replica_factor_update_with_zk(
self, topic_name, replica_factor):
"""
Generates a json assignment based on replica_factor given to update
replicas for a topic.
Uses all brokers available and distributes them as replicas using
a round robin method.
"""
all_replicas = []
assign = {'partitions': [], 'version': 1}
if replica_factor > self.get_total_brokers():
raise KafkaManagerError(
'Error while updating topic \'%s\' replication factor : '
'replication factor \'%s\' is more than available brokers '
'\'%s\'' %
(topic_name, replica_factor, self.get_total_brokers()))
else:
for node_id, _, _, _ in self.get_brokers():
all_replicas.append(node_id)
brokers_iterator = itertools.cycle(all_replicas)
for _, part in self.get_partitions_for_topic(topic_name).items():
_, partition, _, _, _, _ = part
assign_tmp = {
'topic': topic_name,
'partition': partition,
'replicas': []
}
for _i in range(replica_factor):
assign_tmp['replicas'].append(next(brokers_iterator))
assign['partitions'].append(assign_tmp)
return bytes(str(json.dumps(assign)).encode('ascii'))
def get_assignment_for_replica_factor_update(self, topic_name,
replica_factor):
"""
Generates a json assignment based on replica_factor given to update
replicas for a topic.
Uses all brokers available and distributes them as replicas using
a round robin method.
"""
all_replicas = []
partitions = []
if replica_factor > self.get_total_brokers():
raise KafkaManagerError(
'Error while updating topic \'%s\' replication factor : '
'replication factor \'%s\' is more than available brokers '
'\'%s\'' %
(topic_name, replica_factor, self.get_total_brokers()))
else:
for node_id, _, _, _ in self.get_brokers():
all_replicas.append(node_id)
brokers_iterator = itertools.cycle(all_replicas)
for _, part in self.get_partitions_for_topic(topic_name).items():
_, partition, _, _, _, _ = part
replicas = []
for _i in range(replica_factor):
replicas.append(next(brokers_iterator))
assign_tmp = (partition, replicas, {})
partitions.append(assign_tmp)
return [(topic_name, partitions, {})]
def describe_acls(self, acl_resource, api_version):
"""Describe a set of ACLs
"""
if api_version < parse_version('2.0.0'):
request = DescribeAclsRequest_v0(
resource_type=acl_resource.resource_type,
resource_name=acl_resource.name,
principal=acl_resource.principal,
host=acl_resource.host,
operation=acl_resource.operation,
permission_type=acl_resource.permission_type)
else:
request = DescribeAclsRequest_v1(
resource_type=acl_resource.resource_type,
resource_name=acl_resource.name,
resource_pattern_type_filter=acl_resource.pattern_type,
principal=acl_resource.principal,
host=acl_resource.host,
operation=acl_resource.operation,
permission_type=acl_resource.permission_type)
response = self.send_request_and_get_response(request)
if response.error_code != self.SUCCESS_CODE:
raise KafkaManagerError(
'Error while describing ACL %s. Error %s: %s.' %
(acl_resource, response.error_code, response.error_message))
return response.resources
def create_topic(self,
name,
partitions,
replica_factor,
replica_assignment=[],
config_entries=[],
timeout=None):
"""
Creates a topic
Usable for Kafka version >= 0.10.1
"""
if timeout is None:
timeout = self.DEFAULT_TIMEOUT
request = CreateTopicsRequest_v0(create_topic_requests=[
(name, partitions, replica_factor, replica_assignment,
config_entries)
],
timeout=timeout)
response = self.send_request_and_get_response(request)
for topic, error_code in response.topic_errors:
if error_code != self.SUCCESS_CODE:
raise KafkaManagerError(
'Error while creating topic %s. '
'Error key is %s, %s.' %
(topic, kafka.errors.for_code(error_code).message,
kafka.errors.for_code(error_code).description))
def update_topic_assignment(self, json_assignment, zknode):
"""
Updates the topic partition assignment using a json assignment
Used when Kafka version < 1.0.0
Requires zk connection.
"""
if not self.zk_client.exists(zknode):
raise KafkaManagerError(
'Error while updating assignment: zk node %s missing. '
'Is the topic name correct?' % (zknode)
)
self.zk_client.set(zknode, json_assignment)
self.refresh()
def update_admin_assignments(self, topics):
"""
Updates the topic replica factor using a json assignment
Cf core/src/main/scala/kafka/admin/ReassignPartitionsCommand.scala#L580
1 - Send AlterReplicaLogDirsRequest to allow broker to create replica in
the right log dir later if the replica has not been created yet.
2 - Create reassignment znode so that controller will send
LeaderAndIsrRequest to create replica in the broker
def path = "/admin/reassign_partitions" ->
zk.create("/admin/reassign_partitions", b"a value")
case class ReplicaAssignment(
@BeanProperty @JsonProperty("topic") topic: String,
@BeanProperty @JsonProperty("partition") partition: Int,
@BeanProperty @JsonProperty("replicas") replicas: java.util.List[Int])
3 - Send AlterReplicaLogDirsRequest again to make sure broker will start
to move replica to the specified log directory.
It may take some time for controller to create replica in the broker
Retry if the replica has not been created.
It may be possible that the node '/admin/reassign_partitions' is already
there for another topic. That's why we need to check for its existence
and wait for its consumption if it is already present.
Requires zk connection.
"""
if (parse_version(self.get_api_version()) >= parse_version('2.4.0')):
assign = []
for name, replica_factor in topics.items():
assign += self.get_assignment_for_replica_factor_update(
name, replica_factor)
request = AlterPartitionReassignmentsRequest_v0(timeout_ms=60000,
topics=assign,
tags={})
self.wait_for_partition_assignement()
self.send_request_and_get_response(request)
self.wait_for_partition_assignement()
elif self.zk_configuration is not None:
try:
json_assignment = (
self.get_assignment_for_replica_factor_update_with_zk(
topics))
self.init_zk_client()
self.wait_for_znode_assignment()
self.zk_client.create(self.ZK_REASSIGN_NODE, json_assignment)
self.wait_for_znode_assignment()
finally:
self.close_zk_client()
else:
raise KafkaManagerError(
'Zookeeper is mandatory for partition assignment when \
using Kafka <= 2.4.0.')
self.refresh()
def describe_acls(self, acl_resource, api_version):
"""Describe a set of ACLs
"""
if api_version < parse_version('2.0.0'):
request = DescribeAclsRequest_v0(
resource_type=acl_resource.resource_type,
resource_name=acl_resource.name,
principal=acl_resource.principal,
host=acl_resource.host,
operation=acl_resource.operation,
permission_type=acl_resource.permission_type)
else:
request = DescribeAclsRequest_v1(
resource_type=acl_resource.resource_type,
resource_name=acl_resource.name,
resource_pattern_type_filter=acl_resource.pattern_type,
principal=acl_resource.principal,
host=acl_resource.host,
operation=acl_resource.operation,
permission_type=acl_resource.permission_type)
response = self.send_request_and_get_response(request)
if response.error_code != self.SUCCESS_CODE:
raise KafkaManagerError(
'Error while describing ACL %s. Error %s: %s.' %
(acl_resource, response.error_code, response.error_message))
acl_list = []
for resources in response.resources:
if api_version < parse_version('2.0.0'):
resource_type, resource_name, acls = resources
resource_pattern_type = ACLPatternType.LITERAL.value
else:
resource_type, resource_name, resource_pattern_type, acls = \
resources
for acl in acls:
principal, host, operation, permission_type = acl
conv_acl = ACLResource(
principal=principal,
host=host,
operation=ACLOperation(operation),
permission_type=ACLPermissionType(permission_type),
name=resource_name,
pattern_type=ACLPatternType(resource_pattern_type),
resource_type=ACLResourceType(resource_type),
)
acl_list.append(conv_acl)
return acl_list
def update_topic_partitions(self, topic_name, partitions):
"""
Updates the topic partitions
Usable for Kafka version >= 1.0.0
Requires to be the sended to the current controller of the Kafka
cluster.
The request requires to precise the total number of partitions and
broker assignment for each new partition without forgeting replica.
See NewPartitions class for explanations
apache/kafka/clients/admin/NewPartitions.java#L53
"""
brokers = []
for node_id, _, _, _ in self.get_brokers():
brokers.append(int(node_id))
brokers_iterator = itertools.cycle(brokers)
topic, _, _, replicas, _, _ = (
self.get_partitions_for_topic(topic_name)[0]
)
total_replica = len(replicas)
old_partition = self.get_total_partitions_for_topic(topic_name)
assignments = []
for _new_partition in range(partitions - old_partition):
assignment = []
for _replica in range(total_replica):
assignment.append(next(brokers_iterator))
assignments.append(assignment)
request = CreatePartitionsRequest_v0(
topic_partitions=[(topic_name, (partitions, assignments))],
timeout=self.DEFAULT_TIMEOUT,
validate_only=False
)
response = self.send_request_and_get_response(request)
for topic, error_code, _error_message in response.topic_errors:
if error_code != self.SUCCESS_CODE:
raise KafkaManagerError(
'Error while updating topic \'%s\' partitions. '
'Error key is %s, %s. Request was %s.' % (
topic, kafka.errors.for_code(error_code).message,
kafka.errors.for_code(error_code).description,
request
)
)
self.refresh()
def is_topic_partitions_need_update(self, topic_name, partitions):
"""
Checks whether topic's partitions need to be updated or not.
"""
total_partitions = self.get_total_partitions_for_topic(topic_name)
need_update = False
if partitions != total_partitions:
if partitions > total_partitions:
# increasing partition number
need_update = True
else:
# decreasing partition number, which is not possible
raise KafkaManagerError(
'Can\'t update \'%s\' topic partition from %s to %s :'
'only increase is possible.' %
(topic_name, total_partitions, partitions))
return need_update
def delete_topic(self, name, timeout=None):
"""
Deletes a topic
Usable for Kafka version >= 0.10.1
Need to know which broker is controller for topic
"""
if timeout is None:
timeout = self.DEFAULT_TIMEOUT
request = DeleteTopicsRequest_v0(topics=[name], timeout=timeout)
response = self.send_request_and_get_response(request)
for topic, error_code in response.topic_error_codes:
if error_code != self.SUCCESS_CODE:
raise KafkaManagerError(
'Error while deleting topic %s. Error key is: %s, %s. '
'Is option \'delete.topic.enable\' set to true on '
' your Kafka server?' %
(topic, kafka.errors.for_code(error_code).message,
kafka.errors.for_code(error_code).description))
def update_topic_configuration(self, topic_name, topic_conf):
"""
Updates the topic configuration
Usable for Kafka version >= 0.11.0
Requires to be the sended to the current controller of the Kafka
cluster.
"""
request = AlterConfigsRequest_v0(resources=[(self.TOPIC_RESOURCE_ID,
topic_name, topic_conf)],
validate_only=False)
response = self.send_request_and_get_response(request)
for error_code, _, _, resource_name in response.resources:
if error_code != self.SUCCESS_CODE:
raise KafkaManagerError(
'Error while updating topic \'%s\' configuration. '
'Error key is %s, %s' %
(resource_name, kafka.errors.for_code(error_code).message,
kafka.errors.for_code(error_code).description))
self.refresh()
def create_acls(self, acl_resources, api_version):
"""Create a set of ACLs"""
if api_version < parse_version('2.0.0'):
request = CreateAclsRequest_v0(creations=[
self._convert_create_acls_resource_request_v0(acl_resource)
for acl_resource in acl_resources
])
else:
request = CreateAclsRequest_v1(creations=[
self._convert_create_acls_resource_request_v1(acl_resource)
for acl_resource in acl_resources
])
response = self.send_request_and_get_response(request)
for error_code, error_message in response.creation_responses:
if error_code != self.SUCCESS_CODE:
raise KafkaManagerError(
'Error while creating ACL %s. Error %s: %s.' %
(acl_resources, error_code, error_message))
def create_topics(self, topics, timeout=None):
"""
Creates a topic
Usable for Kafka version >= 0.10.1
"""
if timeout is None:
timeout = self.DEFAULT_TIMEOUT
request = CreateTopicsRequest_v0(create_topic_requests=[
(topic['name'], topic['partitions'], topic['replica_factor'],
topic['replica_assignment'] if 'replica_assignment' in topic else
[], topic['options'].items() if 'options' in topic else [])
for topic in topics
],
timeout=timeout)
response = self.send_request_and_get_response(request)
for topic, error_code in response.topic_errors:
if error_code != self.SUCCESS_CODE:
raise KafkaManagerError(
'Error while creating topic %s. '
'Error key is %s, %s.' %
(topic, kafka.errors.for_code(error_code).message,
kafka.errors.for_code(error_code).description))
def get_consumer_groups_resource(self):
"""
Return a dict object containing information about consumer groups and
following this structure:
{
"AWESOME_consumer_group_1607465801": {
"coordinator": {
"host": "172.17.0.9",
"nodeId": 1001,
"port": 9092,
"rack": null
},
"error_code": 0,
"group_state": "Empty",
"members": {},
"protocol": "",
"protocol_type": "consumer"
},
"AWESOME_consumer_group_1607466258": {
"coordinator": {
"host": "172.17.0.10",
"nodeId": 1002,
"port": 9092,
"rack": null
},
"error_code": 0,
"group_state": "Stable",
"members": {
"kafka-python-2.0.1-e5500fee-8df9-4f37-bcd7-788522a1c382": {
"client_host": "/172.17.0.1",
"client_id": "kafka-python-2.0.1",
"member_assignment": {
"assignment": {
"test_1607465755": [
0
]
},
"user_data": "",
"version": 0
},
"member_metadata": {
"subscription": [
"test_1607465755"
],
"user_data": "",
"version": 0
}
}
},
"protocol": "range",
"protocol_type": "consumer"
}
}
"""
consumer_groups = {}
for broker in self.get_brokers():
request = ListGroupsRequest_v2()
response = self.send_request_and_get_response(
request, node_id=broker.nodeId)
if response.error_code != self.SUCCESS_CODE:
raise KafkaManagerError(
'Error while list consumer groups of %s. '
'Error key is %s, %s.' %
(broker.nodeId, kafka.errors.for_code(
response.error_code).message,
kafka.errors.for_code(response.error_code).description))
if response.groups:
request = DescribeGroupsRequest_v0(groups=tuple(
[group for group, protocol in response.groups]))
response = self.send_request_and_get_response(
request, node_id=broker.nodeId)
consumer_groups.update(
self.generate_consumer_groups_for_broker(broker, response))
return consumer_groups
