def test_authz_acls_required(kafka_client, kafka_server, kerberos):
client_id = kafka_client["id"]
sdk_cmd.resolve_hosts(kafka_client["id"], kafka_client["brokers"])
topic_name = "authz.test"
sdk_cmd.svc_cli(kafka_server["package_name"], kafka_server["service"]["name"],
"topic create {}".format(topic_name),
json=True)
test_utils.wait_for_topic(kafka_server["package_name"], kafka_server["service"]["name"], topic_name)
message = str(uuid.uuid4())
log.info("Writing and reading: Writing to the topic, but not super user")
assert not write_to_topic("authorized", client_id, topic_name, message, kerberos)
log.info("Writing and reading: Writing to the topic, as super user")
assert write_to_topic("super", client_id, topic_name, message, kerberos)
log.info("Writing and reading: Reading from the topic, but not super user")
assert auth.is_not_authorized(read_from_topic("authorized", client_id, topic_name, 1, kerberos))
log.info("Writing and reading: Reading from the topic, as super user")
assert message in read_from_topic("super", client_id, topic_name, 1, kerberos)
zookeeper_endpoint = sdk_cmd.svc_cli(
kafka_server["package_name"],
kafka_server["service"]["name"],
"endpoint zookeeper").strip()
# TODO: If zookeeper has Kerberos enabled, then the environment should be changed
topics.add_acls("authorized", client_id, topic_name, zookeeper_endpoint, env_str=None)
# Send a second message which should not be authorized
second_message = str(uuid.uuid4())
log.info("Writing and reading: Writing to the topic, but not super user")
assert write_to_topic("authorized", client_id, topic_name, second_message, kerberos)
log.info("Writing and reading: Writing to the topic, as super user")
assert write_to_topic("super", client_id, topic_name, second_message, kerberos)
log.info("Writing and reading: Reading from the topic, but not super user")
topic_output = read_from_topic("authorized", client_id, topic_name, 3, kerberos)
assert message in topic_output
assert second_message in topic_output
log.info("Writing and reading: Reading from the topic, as super user")
topic_output = read_from_topic("super", client_id, topic_name, 3, kerberos)
assert message in topic_output
assert second_message in topic_output
# Check that the unauthorized client can still not read or write from the topic.
log.info("Writing and reading: Writing to the topic, but not super user")
assert not write_to_topic("unauthorized", client_id, topic_name, second_message, kerberos)
log.info("Writing and reading: Reading from the topic, but not super user")
assert auth.is_not_authorized(read_from_topic("unauthorized", client_id, topic_name, 1, kerberos))
def client_can_read_and_write(test_id: str,
kafka_client: dict, kafka_server: dict,
endpoint_name: str, krb5: object=None) -> tuple:
client_id = kafka_client["id"]
brokers_list = service_get_brokers(kafka_server, endpoint_name)
broker_hosts = map(lambda b: b.split(":")[0], brokers_list)
brokers = ",".join(brokers_list)
if not sdk_cmd.resolve_hosts(kafka_client["id"], broker_hosts):
log.error("Failed to resolve brokers: %s", broker_hosts)
return False, []
topic_name = kafka_client["env"]["KAFKA_TOPIC"]
sdk_cmd.svc_cli(kafka_server["package_name"], kafka_server["service"]["name"],
"topic create {}".format(topic_name),
json=True)
test_utils.wait_for_topic(kafka_server["package_name"], kafka_server["service"]["name"], topic_name)
message = str(uuid.uuid4())
security_options = {"is-tls": endpoint_name == "broker-tls",
"kerberos": krb5}
write_success = write_to_topic(test_id, client_id, topic_name, message, brokers, security_options)
if write_success:
MESSAGES.append(message)
read_messages = read_from_topic(test_id, client_id, topic_name, len(MESSAGES), brokers, security_options)
read_success = map(lambda m: m in read_messages, MESSAGES)
return write_success, read_success
def wait_for(self, kafka_server: dict, topic_name: str) -> bool:
"""
Wait for the service to be visible from a client perspective.
"""
service = KafkaService(kafka_server)
if not self.brokers:
brokers_list = service.get_brokers_endpoints(
self.get_endpoint_name())
broker_hosts = map(lambda b: b.split(":")[0], brokers_list)
brokers = ",".join(brokers_list)
if not sdk_cmd.resolve_hosts(
self.id, broker_hosts, bootstrap_cmd='/opt/bootstrap'):
log.error("Failed to resolve brokers: %s", broker_hosts)
return False
self.brokers = brokers
return True
if self.topic_name != topic_name:
service.wait_for_topic(topic_name)
self.topic_name = topic_name
return True
def test_client_can_read_and_write(kafka_client, kafka_server, kerberos):
client_id = kafka_client["id"]
sdk_cmd.resolve_hosts(kafka_client["id"], kafka_client["brokers"])
topic_name = "authn.test"
sdk_cmd.svc_cli(kafka_server["package_name"], kafka_server["service"]["name"],
"topic create {}".format(topic_name),
json=True)
test_utils.wait_for_topic(kafka_server["package_name"], kafka_server["service"]["name"], topic_name)
message = str(uuid.uuid4())
assert write_to_topic("client", client_id, topic_name, message, kerberos)
assert message in read_from_topic("client", client_id, topic_name, 1, kerberos)
def test_authn_client_can_read_and_write(kafka_client, service_account, setup_principals):
try:
sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
config.install(
config.PACKAGE_NAME,
config.SERVICE_NAME,
config.DEFAULT_BROKER_COUNT,
additional_options={
"brokers": {
"port_tls": 1030
},
"service": {
"service_account": service_account["name"],
"service_account_secret": service_account["secret"],
"security": {
"transport_encryption": {
"enabled": True
},
"ssl_authentication": {
"enabled": True
}
}
}
})
client_id = kafka_client["id"]
sdk_cmd.resolve_hosts(client_id, kafka_client["brokers"])
sdk_cmd.svc_cli(config.PACKAGE_NAME, config.SERVICE_NAME,
"topic create tls.topic",
json=True)
test_utils.wait_for_topic(config.PACKAGE_NAME, config.SERVICE_NAME, "tls.topic")
message = str(uuid.uuid4())
# Write to the topic
log.info("Writing and reading: Writing to the topic, with authn")
assert write_to_topic("kafka-tester", client_id, "tls.topic", message)
log.info("Writing and reading: reading from the topic, with authn")
# Read from the topic
assert message in read_from_topic("kafka-tester", client_id, "tls.topic", 1)
finally:
sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
def test_client_can_read_and_write(kafka_client, kafka_server, kerberos):
client_id = kafka_client["id"]
sdk_cmd.resolve_hosts(kafka_client["id"], kafka_client["brokers"])
topic_name = "authn.test"
sdk_cmd.svc_cli(kafka_server["package_name"],
kafka_server["service"]["name"],
"topic create {}".format(topic_name),
json=True)
test_utils.wait_for_topic(kafka_server["package_name"],
kafka_server["service"]["name"], topic_name)
message = str(uuid.uuid4())
assert write_to_topic("client", client_id, topic_name, message, kerberos)
assert message in read_from_topic("client", client_id, topic_name, 1,
kerberos)
def wait_for_broker_dns(package_name: str, service_name: str):
brokers = sdk_networks.get_endpoint(package_name, service_name, "broker")
broker_dns = list(map(lambda x: x.split(":")[0], brokers["dns"]))
def get_scheduler_task_id(service_name: str) -> str:
for task in sdk_tasks.get_summary():
if task.name == service_name:
return task.id
scheduler_task_id = get_scheduler_task_id(service_name)
log.info("Scheduler task ID: %s", scheduler_task_id)
log.info("Waiting for brokers: %s", broker_dns)
assert sdk_cmd.resolve_hosts(scheduler_task_id, broker_dns)
def wait_for_broker_dns(package_name: str, service_name: str):
brokers = sdk_cmd.svc_cli(package_name, service_name, "endpoint broker", json=True)
broker_dns = list(map(lambda x: x.split(':')[0], brokers["dns"]))
def get_scheduler_task_id(service_name: str) -> str:
for task in sdk_tasks.get_summary():
if task.name == service_name:
return task.id
scheduler_task_id = get_scheduler_task_id(service_name)
log.info("Scheduler task ID: %s", scheduler_task_id)
log.info("Waiting for brokers: %s", broker_dns)
assert sdk_cmd.resolve_hosts(scheduler_task_id, broker_dns)
def client_can_read_and_write(test_id: str,
kafka_client: dict,
kafka_server: dict,
endpoint_name: str,
krb5: object = None) -> tuple:
client_id = kafka_client["id"]
brokers_list = service_get_brokers(kafka_server, endpoint_name)
broker_hosts = map(lambda b: b.split(":")[0], brokers_list)
brokers = ",".join(brokers_list)
if not sdk_cmd.resolve_hosts(kafka_client["id"], broker_hosts):
log.error("Failed to resolve brokers: %s", broker_hosts)
return False, []
topic_name = kafka_client["env"]["KAFKA_TOPIC"]
sdk_cmd.svc_cli(
kafka_server["package_name"],
kafka_server["service"]["name"],
"topic create {}".format(topic_name),
)
test_utils.wait_for_topic(kafka_server["package_name"],
kafka_server["service"]["name"], topic_name)
message = str(uuid.uuid4())
security_options = {
"is-tls": endpoint_name == "broker-tls",
"kerberos": krb5
}
write_success = write_to_topic(test_id, client_id, topic_name, message,
brokers, security_options)
if write_success:
MESSAGES.append(message)
read_messages = read_from_topic(test_id, client_id, topic_name,
len(MESSAGES), brokers, security_options)
read_success = map(lambda m: m in read_messages, MESSAGES)
return write_success, read_success
def wait_for(self, topic_name: typing.Optional[str] = None) -> bool:
"""
Wait for the service to be visible from a client perspective.
"""
if not self.brokers:
brokers_list = self.kafka_service.get_endpoint_dns(self.get_endpoint_name())
broker_hosts = map(lambda b: b.split(":")[0], brokers_list)
brokers = ",".join(brokers_list)
if not sdk_cmd.resolve_hosts(self.id, broker_hosts, bootstrap_cmd="/opt/bootstrap"):
log.error("Failed to resolve brokers: %s", broker_hosts)
return False
self.brokers = brokers
return True
if topic_name:
self.kafka_service.wait_for_topic(topic_name)
return True
def test_authz_acls_not_required(kafka_client, service_account, setup_principals):
client_id = kafka_client["id"]
try:
sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
config.install(
config.PACKAGE_NAME,
config.SERVICE_NAME,
config.DEFAULT_BROKER_COUNT,
additional_options={
"brokers": {
"port_tls": 1030
},
"service": {
"service_account": service_account["name"],
"service_account_secret": service_account["secret"],
"security": {
"transport_encryption": {
"enabled": True
},
"ssl_authentication": {
"enabled": True
},
"authorization": {
"enabled": True,
"super_users": "User:{}".format("super"),
"allow_everyone_if_no_acl_found": True
}
}
}
})
sdk_cmd.resolve_hosts(client_id, kafka_client["brokers"])
# Create the topic
sdk_cmd.svc_cli(config.PACKAGE_NAME, config.SERVICE_NAME,
"topic create authz.test",
json=True)
test_utils.wait_for_topic(config.PACKAGE_NAME, config.SERVICE_NAME, "authz.test")
super_message = str(uuid.uuid4())
authorized_message = str(uuid.uuid4())
unauthorized_message = str(uuid.uuid4())
log.info("Writing and reading: Writing to the topic, as authorized user")
assert write_to_topic("authorized", client_id, "authz.test", authorized_message)
log.info("Writing and reading: Writing to the topic, as unauthorized user")
assert write_to_topic("unauthorized", client_id, "authz.test", unauthorized_message)
log.info("Writing and reading: Writing to the topic, as super user")
assert write_to_topic("super", client_id, "authz.test", super_message)
log.info("Writing and reading: Reading from the topic, as authorized user")
assert authorized_message in read_from_topic("authorized", client_id, "authz.test", 3)
log.info("Writing and reading: Reading from the topic, as unauthorized user")
assert unauthorized_message in read_from_topic("unauthorized", client_id, "authz.test", 3)
log.info("Writing and reading: Reading from the topic, as super user")
assert super_message in read_from_topic("super", client_id, "authz.test", 3)
log.info("Writing and reading: Adding acl for authorized user")
zookeeper_endpoint = str(sdk_cmd.svc_cli(
config.PACKAGE_NAME,
config.SERVICE_NAME,
"endpoint zookeeper")).strip()
topics.add_acls("authorized", client_id, "authz.test", zookeeper_endpoint, env_str=None)
# Re-roll the messages so we really prove auth is in place.
super_message = str(uuid.uuid4())
authorized_message = str(uuid.uuid4())
unauthorized_message = str(uuid.uuid4())
log.info("Writing and reading: Writing to the topic, as authorized user")
assert write_to_topic("authorized", client_id, "authz.test", authorized_message)
log.info("Writing and reading: Writing to the topic, as unauthorized user")
assert not write_to_topic("unauthorized", client_id, "authz.test", unauthorized_message)
log.info("Writing and reading: Writing to the topic, as super user")
assert write_to_topic("super", client_id, "authz.test", super_message)
log.info("Writing and reading: Reading from the topic, as authorized user")
read_result = read_from_topic("authorized", client_id, "authz.test", 5)
assert authorized_message in read_result and unauthorized_message not in read_result
log.info("Writing and reading: Reading from the topic, as unauthorized user")
assert auth.is_not_authorized(read_from_topic("unauthorized", client_id, "authz.test", 1))
log.info("Writing and reading: Reading from the topic, as super user")
read_result = read_from_topic("super", client_id, "authz.test", 5)
assert super_message in read_result and unauthorized_message not in read_result
finally:
sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)