def resize_partitions(self, **kwargs):
"""resize partitions. Noted, kafka currently supports only increasing
the number of partitions, but not decreasing. :param kwargs: examples:
topic_name_1=4, topic_name_2=3, ...
Args:
**kwargs:
"""
new_parts = [
NewPartitions(topic, int(kwargs[topic])) for topic in kwargs
]
# Try switching validate_only to True to only validate the operation
# on the broker but not actually perform it.
fs = self.a.create_partitions(new_parts, validate_only=False)
# Wait for operation to finish.
for topic, f in fs.items():
try:
f.result() # The result itself is None
print(
"Additional partitions created for topic {}".format(topic))
except Exception as e:
print("Failed to add partitions to topic {}: {}".format(
topic, e))
def add_concurrency(self, admin_id: str, message_type: str, \
concurrency_count: int):
"""
Increases the partitions for a message type.
Parameters:
admin_id A String that represents Admin client ID.
message_type This is essentially equivalent to queue/topic name.
For e.g. "Alert"
concurrency_count Integer that represents number of partitions to be
increased.
Note: Number of partitions for a message type can only be increased,
never decreased
"""
admin = self._clients['admin'][admin_id]
new_partition = [NewPartitions(message_type, \
new_total_count=concurrency_count)]
partitions = admin.create_partitions(new_partition)
self._task_status(partitions)
# Waiting for few seconds to complete the partition addition process
for list_retry in range(1, self._max_list_message_type_count+2):
if concurrency_count != len(self._get_metadata(admin)\
[message_type].__dict__['partitions']):
if list_retry > self._max_list_message_type_count:
raise MessageBusError(errno.EINVAL, "Maximum retries \
exceeded to increase concurrency for %s.", \
message_type)
time.sleep(list_retry*1)
continue
else:
break
def create_or_ensure_topic(admin, topic, min_partitions, min_replicas):
metadata = admin.list_topics(timeout=10)
if topic not in metadata.topics:
# Create the topic for the first time
print("Topic %s does not exist - creating..." % (topic))
new_topics = [
NewTopic(topic,
num_partitions=min_partitions,
replication_factor=min_replicas)
]
fs = admin.create_topics(new_topics)
for topic, f in fs.items():
try:
f.result()
print("Topic %s created" % (topic))
except Exception as e:
print("Failed to create topic %s - %s" % (topic, e))
else:
# Check the partitions and replication is at least high enough
if min_partitions > len(metadata.topics[topic].partitions):
new_partitions = [NewPartitions(topic, min_partitions)]
fs = admin.create_partitions(new_partitions)
for topic, f in fs.items():
try:
f.result()
print("Additional partitions for %s created" % (topic))
except Exception as e:
print("Failed to add partitions to topic %s - %s" %
(topic, e))
def increase_parallelism(self, admin_id: str, message_types: list, \
partitions: int):
"""
Increases the partitions for for a list of message types.
Parameters:
admin_id A String that represents Admin client ID.
message_types This is essentially equivalent to the list of
queue/topic name. For e.g. ["Alert"]
partitions Integer that represents number of partitions to be
increased.
Note: Number of partitions for a topic can only be increased, never
decreased
"""
admin = self._clients['admin'][admin_id]
new_partition = [NewPartitions(each_message_type, \
new_total_count=partitions) for each_message_type in message_types]
partitions = admin.create_partitions(new_partition)
self._task_status(partitions)
for each_message_type in message_types:
for list_retry in range(1, self._max_list_message_type_count + 2):
if partitions != len(self._get_metadata(admin)\
[each_message_type].__dict__['partitions']):
if list_retry > self._max_list_message_type_count:
raise MessageBusError(errno.EINVAL, "Maximum retries \
exceeded to increase partition for %s." , \
each_message_type)
time.sleep(list_retry * 1)
continue
else:
break
def _configuire_topic(self, configs):
admin_client = AdminClient(
{'bootstrap.servers': configs['kafka_host']})
sensor_topic_partitions = NewPartitions(
configs['sensor_manager_topic'],
int(configs['sensor_manager_topic_partitions']))
admin_client.create_partitions([sensor_topic_partitions])
def create_partitions(self, topic_partitions):
"""
create partitions for list of topics
:param topic_partitions: list of tuples (topic, number of partitions)
:return:
"""
new_parts = [
NewPartitions(topic, int(new_total_count))
for topic, new_total_count in topic_partitions
]
# Try switching validate_only to True to only validate the operation
# on the broker but not actually perform it.
fs = self.admin_client.create_partitions(new_parts,
validate_only=False)
# Wait for operation to finish.
for topic, f in fs.items():
try:
f.result() # The result itself is None
self.logger.debug(
f"Additional partitions created for topic {topic}")
except Exception as e:
self.logger.error(
f"Failed to add partitions to topic {topic}: {e}")
def modify_part(topic, new_part):
new_parts = [NewPartitions(topic, new_part)]
try:
fs = admin.create_partitions(new_parts, validate_only=False)
y = list(fs.values())
conf = y[0].result()
except Exception:
msg = ("Failed to finalize partition-change for topic %s" \
%(topic)
)
fail_module(msg)
def example_create_partitions(a, topics):
""" create partitions """
new_parts = [NewPartitions(topic, int(new_total_count)) for
topic, new_total_count in zip(topics[0::2], topics[1::2])]
# Try switching validate_only to True to only validate the operation
# on the broker but not actually perform it.
fs = a.create_partitions(new_parts, validate_only=False)
# Wait for operation to finish.
for topic, f in fs.items():
try:
f.result() # The result itself is None
print("Additional partitions created for topic {}".format(topic))
except Exception as e:
print("Failed to add partitions to topic {}: {}".format(topic, e))
def modify_part(topic, new_part):
# type: (str, int)
"""Modify topic-partition.
Keyword arguments:
topic -- topicname
new_part -- new number of partitions
"""
new_parts = [NewPartitions(topic, new_part)]
try:
fs = admin.create_partitions(new_parts, validate_only=False)
y = list(fs.values())
y[0].result()
except KafkaException as e:
msg = ("Failed to finalize partition-change for topic %s: %s" %
(topic, e))
fail_module(msg)
def create_partitions(self, num_partitions, topic_name):
# Create Admin client
self.list_kafka_info()
new_parts = [NewPartitions(topic_name, int(num_partitions))]
# Try switching validate_only to True to only validate the operation
# on the broker but not actually perform it.
fs = self.__client.create_partitions(new_parts, validate_only=False)
# Wait for operation to finish.
for topic, f in fs.items():
try:
f.result() # The result itself is None
print(
"Additional partitions created for topic {}".format(topic))
except Exception as e:
print("Failed to add partitions to topic {}: {}".format(
topic, e))
def add_concurrency(self, admin_id: str, message_type: str,\
concurrency_count: int):
"""
Increases the partitions for a message type.
Parameters:
admin_id A String that represents Admin client ID.
message_type This is essentially equivalent to queue/topic name.
For e.g. "Alert"
concurrency_count Integer that represents number of partitions to be
increased.
Note: Number of partitions for a message type can only be increased,
never decreased
"""
Log.debug(f"Adding concurrency count {concurrency_count} for message"\
f" type {message_type} with admin id {admin_id}")
admin = self._clients['admin'][admin_id]
new_partition = [NewPartitions(message_type,\
new_total_count=concurrency_count)]
partitions = admin.create_partitions(new_partition)
self._task_status(partitions, method='add_concurrency')
# Waiting for few seconds to complete the partition addition process
for list_retry in range(1, self._max_list_message_type_count + 2):
if concurrency_count != len(self._get_metadata(admin)\
[message_type].__dict__['partitions']):
if list_retry > self._max_list_message_type_count:
Log.error(f"MessageBusError: Exceeded retry count "\
f"{list_retry} for creating partitions for "\
f"message_type {message_type}")
raise MessageBusError(errno.E2BIG, "Exceeded retry count" +\
" %d for creating partitions for message_type" +\
" %s.", list_retry, message_type)
time.sleep(list_retry * 1)
continue
else:
break
Log.debug(f"Successfully Increased the partitions for a "\
f"{message_type} to {concurrency_count}")
def test_create_partitions_api():
""" create_partitions() tests, these wont really do anything since there
is no broker configured. """
a = AdminClient({"socket.timeout.ms": 10})
fs = a.create_partitions([NewPartitions("mytopic", 50)])
# ignore the result
with pytest.raises(TypeError):
a.create_partitions(None)
with pytest.raises(Exception):
a.create_partitions("mytopic")
with pytest.raises(Exception):
a.create_partitions([])
with pytest.raises(Exception):
a.create_partitions([None, NewPartitions("mytopic", 2)])
fs = a.create_partitions([
NewPartitions("mytopic", 100),
NewPartitions("other", 3, replica_assignment=[[10, 11], [15, 20]])
])
with pytest.raises(KafkaException):
for f in concurrent.futures.as_completed(iter(fs.values())):
f.result(timeout=1)
fs = a.create_partitions([
NewPartitions("mytopic", 2),
NewPartitions("othertopic", 10),
NewPartitions(
"third", 55, replica_assignment=[[1, 2, 3, 4, 5, 6, 7], [2]])
],
validate_only=True,
request_timeout=0.5,
operation_timeout=300.1)
for f in concurrent.futures.as_completed(iter(fs.values())):
e = f.exception(timeout=1)
assert isinstance(e, KafkaException)
assert e.args[0].code() == KafkaError._TIMED_OUT
def partition_topic(ctx, topic, count, dry_run):
"""Increase the number of partitions for a topic.
"""
client = ctx.parent.obj['client']
new_partitions = [NewPartitions(topic, count)]
# Try switching validate_only to True to only validate the operation
# on the broker but not actually perform it.
fs = client.create_partitions(new_partitions, validate_only=dry_run)
# Wait for operation to finish.
for topic, f in fs.items():
try:
f.result() # The result itself is None
if dry_run:
print('Validated additional partitions for {}'.format(topic))
else:
print(
"Additional partitions created for topic {}".format(topic))
except Exception as e:
print("Failed to add partitions to topic {}: {}".format(topic, e))
def new_partitions(self, topic, **kwargs):
"""Instantiate a NewPartitions object.
- ``topic`` (str): Topic name
"""
return NewPartitions(topic=topic, **kwargs)