def describe(user, output, cluster, namespace):
if output is not None:
if resource_exists("kafkausers", user, cluster, namespace):
os.system(Kubectl().get().kafkausers(user).namespace(
namespace).output(output).build())
else:
if resource_exists("kafkausers", user, cluster, namespace):
os.system(Kubectl().describe().kafkausers(user).namespace(
namespace).build())
def clusters(cluster, list, describe, output, namespace):
"""The kafka cluster(s) to be created, altered or described. """
if list:
os.system(Kubectl().get().kafkas().namespace(namespace).build())
elif describe:
if output is not None:
os.system(Kubectl().get().kafkas(cluster).namespace(
namespace).output(output).build())
else:
os.system(Kubectl().describe().kafkas(cluster).namespace(
namespace).build())
else:
print_missing_options_for_command("clusters")
def alter(topic, partitions, replication_factor, config, delete_config,
cluster, namespace):
if resource_exists("kafkatopics", topic, cluster, namespace):
file = get_resource_as_file("kafkatopics", topic, namespace)
topic_dict = yaml.full_load(file)
if partitions is not None:
topic_dict["spec"]["partitions"] = int(partitions)
if replication_factor is not None:
topic_dict["spec"]["replicas"] = int(replication_factor)
delete_last_applied_configuration(topic_dict)
if len(config) > 0:
if topic_dict["spec"].get("config") is None:
topic_dict["spec"]["config"] = {}
add_resource_kv_config(config, topic_dict["spec"]["config"])
if len(delete_config) > 0:
if topic_dict["spec"].get("config") is not None:
delete_resource_config(delete_config,
topic_dict["spec"]["config"])
topic_yaml = yaml.dump(topic_dict)
topic_temp_file = create_temp_file(topic_yaml)
os.system(Kubectl().apply().from_file(
"{topic_temp_file_path}").namespace(namespace).build().format(
topic_temp_file_path=topic_temp_file.name))
topic_temp_file.close()
else:
print_resource_found_msg(cluster, namespace)
def configs(entity_type, entity_name, add_config, delete_config, describe,
native, cluster, namespace):
"""Add/Remove entity config for a topic, client, user or cluster"""
if len(add_config) > 0 or len(delete_config) > 0:
if entity_type == "topics":
topics_command.alter(entity_name, None, None, add_config,
delete_config, cluster, namespace)
elif entity_type == "users":
users_command.alter(entity_name, None, None, False, False, tuple(),
None, None, None, None, None, add_config,
delete_config, cluster, namespace)
elif entity_type == "clusters":
click.echo("Not implemented")
elif describe:
if entity_type == "topics":
if native:
native_command = "bin/kafka-configs.sh --bootstrap-server {cluster}-kafka-brokers:{port} --entity-type " \
"{entity_type} --entity-name {entity_name} --describe"
os.system(Kubectl().exec(
"-it", cluster + "-kafka-0").container("kafka").namespace(
namespace).exec_command(native_command).build().format(
cluster=cluster,
port=KAFKA_PORT,
entity_type=entity_type,
entity_name=entity_name))
else:
topics_command.describe(entity_name, None, False, cluster,
namespace)
elif entity_type == "users":
click.echo("Not implemented")
elif entity_type == "clusters":
click.echo("Not implemented")
else:
print_missing_options_for_command("configs")
def test_exec(self):
self.assertEqual(
Kubectl().exec(
"-it", "test-pod").container("test-container").namespace(
"test-namespace").exec_command("echo 'test'").build(),
"{kubectl} exec -it test-pod -c test-container -n test-namespace -- bash -c \"echo 'test'\""
.format(kubectl=KUBECTL_PATH))
def test_delete_user(self, mock_os, mock_resource_exists):
mock_resource_exists.return_value = True
result = self.runner.invoke(kfk,
['users', '--delete', '--user', self.user, '-c', self.cluster, '-n',
self.namespace])
assert result.exit_code == 0
mock_os.system.assert_called_with(Kubectl().delete().kafkausers(self.user).namespace(self.namespace).build())
def test_console_producer(self, mock_os):
result = self.runner.invoke(kfk, ['console-producer', '--topic', self.topic, '-c', self.cluster, '-n',
self.namespace])
assert result.exit_code == 0
native_command = "bin/kafka-console-producer.sh --broker-list my-cluster-kafka-brokers:9092 --topic {topic}"
mock_os.system.assert_called_with(
Kubectl().exec("-it", "{cluster}-kafka-0").container("kafka").namespace(self.namespace).exec_command(
native_command).build().format(cluster=self.cluster, topic=self.topic))
def test_describe_cluster(self, mock_os):
result = self.runner.invoke(kfk, [
'clusters', '--describe', '--cluster', self.cluster, '-n',
self.namespace
])
assert result.exit_code == 0
mock_os.system.assert_called_with(Kubectl().describe().kafkas(
self.cluster).namespace(self.namespace).build())
def test_describe_topic_config(self, mock_os, mock_resource_exists):
mock_resource_exists.return_value = True
result = self.runner.invoke(kfk,
['configs', '--describe', '--entity-type', 'topics',
'--entity-name', self.topic, '-c', self.cluster, '-n', self.namespace])
assert result.exit_code == 0
mock_os.system.assert_called_with(
Kubectl().describe().kafkatopics(self.topic).namespace(self.namespace).build())
def describe(topic, output, native, cluster, namespace):
if output is not None:
if resource_exists("kafkatopics", topic, cluster, namespace):
os.system(Kubectl().get().kafkatopics(topic).namespace(
namespace).output(output).build())
else:
if native:
native_command = "bin/kafka-topics.sh --bootstrap-server {cluster}-kafka-bootstrap:9092 --describe " \
"--topic {topic}"
os.system(Kubectl().exec(
"-it", "{cluster}-kafka-0").container("kafka").namespace(
namespace).exec_command(native_command).build().format(
topic=topic, cluster=cluster))
else:
if resource_exists("kafkatopics", topic, cluster, namespace):
os.system(Kubectl().describe().kafkatopics(topic).namespace(
namespace).build())
def test_describe_cluster_output_yaml(self, mock_os):
result = self.runner.invoke(kfk, [
'clusters', '--describe', '--cluster', self.cluster, '-n',
self.namespace, '-o', 'yaml'
])
assert result.exit_code == 0
mock_os.system.assert_called_with(Kubectl().get().kafkas(
self.cluster).namespace(self.namespace).output("yaml").build())
def test_list_topics(self, mock_os):
result = self.runner.invoke(
kfk,
['topics', '--list', '-c', self.cluster, '-n', self.namespace])
assert result.exit_code == 0
mock_os.system.assert_called_with(Kubectl().get().kafkatopics().label(
"strimzi.io/cluster={cluster}").namespace(
self.namespace).build().format(cluster=self.cluster))
def test_describe_topic_output_yaml(self, mock_os, mock_resource_exists):
mock_resource_exists.return_value = True
result = self.runner.invoke(kfk, [
'topics', '--describe', '--topic', self.topic, '-c', self.cluster,
'-n', self.namespace, '-o', 'yaml'
])
assert result.exit_code == 0
mock_os.system.assert_called_with(Kubectl().get().kafkatopics(
self.topic).namespace(self.namespace).output("yaml").build())
def test_describe_topic_config_native(self, mock_os):
result = self.runner.invoke(kfk,
['configs', '--describe', '--entity-type', 'topics',
'--entity-name', self.topic, '--native', '-c', self.cluster, '-n', self.namespace])
assert result.exit_code == 0
native_command = "bin/kafka-configs.sh --bootstrap-server {cluster}-kafka-brokers:9092 --entity-type " \
"topics --entity-name {entity_name} --describe"
mock_os.system.assert_called_with(
Kubectl().exec("-it", "{cluster}-kafka-0").container("kafka").namespace(self.namespace).exec_command(
native_command).build().format(cluster=self.cluster, entity_name=self.topic))
def test_console_consumer_with_from_beginning(self, mock_os):
from_beginning = True
result = self.runner.invoke(kfk, ['console-consumer', '--topic', self.topic, '-c', self.cluster, '-n',
self.namespace, '--from-beginning'])
assert result.exit_code == 0
native_command = "bin/kafka-console-consumer.sh --bootstrap-server my-cluster-kafka-bootstrap:9092 --topic {" \
"topic} {from_beginning}"
mock_os.system.assert_called_with(
Kubectl().exec("-it", "{cluster}-kafka-0").container("kafka").namespace(self.namespace).exec_command(
native_command).build().format(cluster=self.cluster, topic=self.topic,
from_beginning=(from_beginning and '--from-beginning' or '')))
def test_describe_topic_native(self, mock_os):
result = self.runner.invoke(kfk, [
'topics', '--describe', '--topic', self.topic, '-c', self.cluster,
'-n', self.namespace, '--native'
])
assert result.exit_code == 0
native_command = "bin/kafka-topics.sh --bootstrap-server {cluster}-kafka-bootstrap:9092 --describe --topic {" \
"topic}"
mock_os.system.assert_called_with(Kubectl().exec(
"-it", "{cluster}-kafka-0").container("kafka").namespace(
self.namespace).exec_command(native_command).build().format(
topic=self.topic, cluster=self.cluster))
def alter(user, authentication_type, authorization_type, add_acl, delete_acl,
operation_tuple, host, type, resource_type, resource_name,
resource_pattern_type, quota_tuple, delete_quota_tuple, cluster,
namespace):
if resource_exists("kafkausers", user, cluster, namespace):
file = get_resource_as_file("kafkausers", user, namespace)
user_dict = yaml.full_load(file)
if authentication_type is not None:
user_dict["spec"]["authentication"]["type"] = authentication_type
if authorization_type is not None:
if user_dict["spec"].get("authorization") is None:
user_dict["spec"]["authorization"] = {}
user_dict["spec"]["authorization"]["type"] = authorization_type
if add_acl:
if user_dict["spec"].get("authorization") is None:
user_dict["spec"]["authorization"] = {}
add_acl_option(user_dict, operation_tuple, host, type,
resource_type, resource_name, resource_pattern_type)
if delete_acl:
if user_dict["spec"].get("authorization") is None:
user_dict["spec"]["authorization"] = {}
delete_acl_option(user_dict, operation_tuple, host, type,
resource_type, resource_name,
resource_pattern_type)
delete_last_applied_configuration(user_dict)
if len(quota_tuple) > 0:
if user_dict["spec"].get("quotas") is None:
user_dict["spec"]["quotas"] = {}
add_resource_kv_config(quota_tuple, user_dict["spec"]["quotas"],
snake_to_camel_case)
if len(delete_quota_tuple) > 0:
if user_dict["spec"].get("quotas") is not None:
delete_resource_config(delete_quota_tuple,
user_dict["spec"]["quotas"],
snake_to_camel_case)
user_yaml = yaml.dump(user_dict)
user_temp_file = create_temp_file(user_yaml)
os.system(Kubectl().apply().from_file(
"{user_temp_file_path}").namespace(namespace).build().format(
user_temp_file_path=user_temp_file.name))
user_temp_file.close()
else:
print_resource_found_msg(cluster, namespace)
def test_console_producer_with_producer_config(self, mock_os, mock_transfer_file_to_container):
result = self.runner.invoke(kfk, ['console-producer', '--topic', self.topic, '--producer.config',
'files/client.properties', '-c', self.cluster, '-n', self.namespace])
assert result.exit_code == 0
native_command = "bin/kafka-console-producer.sh --broker-list {cluster}-kafka-brokers:9093 --topic {topic} " \
"--producer-property security.protocol=SSL --producer-property " \
"ssl.truststore.location=/tmp/truststore.jks --producer-property " \
"ssl.truststore.password=123456 --producer-property ssl.keystore.location=/tmp/user.p12 " \
"--producer-property ssl.keystore.password=123456;rm -rf /tmp/truststore.jks;rm -rf " \
"/tmp/user.p12;"
mock_os.system.assert_called_with(
Kubectl().exec("-it", "{cluster}-kafka-0").container("kafka").namespace(self.namespace).exec_command(
native_command).build().format(cluster=self.cluster, topic=self.topic))
def console_producer(topic, producer_config, cluster, namespace):
"""The console producer is a tool that reads data from standard input and publish it to Kafka."""
native_command = "bin/kafka-console-producer.sh --broker-list {cluster}-kafka-brokers:{port} --topic {topic}"
pod = cluster + "-kafka-0"
container = "kafka"
if producer_config is not None:
native_command = apply_client_config_from_file(native_command,
producer_config,
"--producer-property",
container, pod,
namespace)
os.system(Kubectl().exec("-it", pod).container(container).namespace(
namespace).exec_command(native_command).build().format(
port=KAFKA_PORT, topic=topic, cluster=cluster))
def console_consumer(topic, consumer_config, from_beginning, cluster,
namespace):
"""The console consumer is a tool that reads data from Kafka and outputs it to standard output."""
native_command = "bin/kafka-console-consumer.sh --bootstrap-server {cluster}-kafka-bootstrap:{port} --topic {" \
"topic} {from_beginning}"
pod = cluster + "-kafka-0"
container = "kafka"
if consumer_config is not None:
native_command = apply_client_config_from_file(native_command,
consumer_config,
"--consumer-property",
container, pod,
namespace)
os.system(Kubectl().exec("-it", pod).container(container).namespace(
namespace).exec_command(native_command).build().format(
port=KAFKA_PORT,
topic=topic,
cluster=cluster,
from_beginning=(from_beginning and '--from-beginning' or '')))
def acls(list, topic, cluster, group, add, allow_principal, deny_principal,
operation_tuple, allow_host, deny_host, resource_pattern_type, remove,
kafka_cluster, namespace):
"""This tool helps to manage ACLs on Kafka."""
if list:
native_command = "bin/kafka-acls.sh --authorizer-properties zookeeper.connect=localhost:12181 --list {topic}" \
"{cluster} {group}"
os.system(Kubectl().exec(
"-it",
"{kafka_cluster}-zookeeper-0").container("zookeeper").namespace(
namespace).exec_command(native_command).build().format(
kafka_cluster=kafka_cluster,
topic=(topic and '--topic ' + topic or ''),
cluster=(cluster and '--cluster ' + cluster or ''),
group=(group and '--group ' + group or '')))
elif add or remove:
alter(topic, cluster, group, add, remove, allow_principal,
deny_principal, operation_tuple, allow_host, deny_host,
resource_pattern_type, kafka_cluster, namespace)
else:
print_missing_options_for_command("acls")
def create(topic, partitions, replication_factor, config, cluster, namespace):
with open('{strimzi_path}/examples/topic/kafka-topic.yaml'.format(
strimzi_path=STRIMZI_PATH).format(
version=STRIMZI_VERSION)) as file:
topic_dict = yaml.full_load(file)
topic_dict["metadata"]["name"] = topic
topic_dict["metadata"]["labels"]["strimzi.io/cluster"] = cluster
topic_dict["spec"]["partitions"] = int(partitions)
topic_dict["spec"]["replicas"] = int(replication_factor)
if len(config) > 0:
if topic_dict["spec"].get("config") is None:
topic_dict["spec"]["config"] = {}
add_resource_kv_config(config, topic_dict["spec"]["config"])
topic_yaml = yaml.dump(topic_dict)
topic_temp_file = create_temp_file(topic_yaml)
os.system(Kubectl().create().from_file(
"{topic_temp_file_path}").namespace(namespace).build().format(
topic_temp_file_path=topic_temp_file.name))
topic_temp_file.close()
def create(user, authentication_type, quota, cluster, namespace):
with open('{strimzi_path}/examples/user/kafka-user.yaml'.format(
strimzi_path=STRIMZI_PATH).format(
version=STRIMZI_VERSION)) as file:
user_dict = yaml.full_load(file)
user_dict["metadata"]["name"] = user
user_dict["metadata"]["labels"]["strimzi.io/cluster"] = cluster
user_dict["spec"]["authentication"]["type"] = authentication_type
del user_dict["spec"]["authorization"]
if len(quota) > 0:
if user_dict["spec"].get("quotas") is None:
user_dict["spec"]["quotas"] = {}
add_resource_kv_config(quota, user_dict["spec"]["quotas"])
user_yaml = yaml.dump(user_dict)
user_temp_file = create_temp_file(user_yaml)
os.system(Kubectl().create().from_file(
"{user_temp_file_path}").namespace(namespace).build().format(
user_temp_file_path=user_temp_file.name))
user_temp_file.close()
def transfer_file_to_container(source_file_path, dest_file_path, container, pod, namespace):
os.system(Kubectl().cp(source_file_path, "{namespace}/{pod}:" + dest_file_path).container(container).build().format(
namespace=namespace, pod=pod))
def delete(cluster, namespace, user):
if resource_exists("kafkausers", user, cluster, namespace):
os.system(
Kubectl().delete().kafkausers(user).namespace(namespace).build())
def test_list_clusters_all_namespaces(self, mock_os):
result = self.runner.invoke(kfk, ['clusters', '--list'])
assert result.exit_code == 0
mock_os.system.assert_called_with(
Kubectl().get().kafkas().namespace().build())
def list(cluster, namespace):
os.system(Kubectl().get().kafkausers().label(
"strimzi.io/cluster={cluster}").namespace(namespace).build().format(
cluster=cluster))
def resource_exists(resource_type, resource_name, cluster, namespace):
return resource_name in os.popen(
Kubectl().get().resource(resource_type).label("strimzi.io/cluster={cluster}").namespace(
namespace).build().format(
cluster=cluster)).read()
def get_resource_yaml(resource_type, resource_name, namespace):
return os.popen(
Kubectl().get().resource(resource_type, resource_name).namespace(namespace).output("yaml").build()).read()
def delete(topic, cluster, namespace):
if resource_exists("kafkatopics", topic, cluster, namespace):
os.system(
Kubectl().delete().kafkatopics(topic).namespace(namespace).build())