def test_authz_acls_required(kafka_client, kafka_server, kerberos):
client_id = kafka_client["id"]
sdk_cmd.resolve_hosts(kafka_client["id"], kafka_client["brokers"])
topic_name = "authz.test"
sdk_cmd.svc_cli(kafka_server["package_name"], kafka_server["service"]["name"],
"topic create {}".format(topic_name),
json=True)
test_utils.wait_for_topic(kafka_server["package_name"], kafka_server["service"]["name"], topic_name)
message = str(uuid.uuid4())
log.info("Writing and reading: Writing to the topic, but not super user")
assert not write_to_topic("authorized", client_id, topic_name, message, kerberos)
log.info("Writing and reading: Writing to the topic, as super user")
assert write_to_topic("super", client_id, topic_name, message, kerberos)
log.info("Writing and reading: Reading from the topic, but not super user")
assert auth.is_not_authorized(read_from_topic("authorized", client_id, topic_name, 1, kerberos))
log.info("Writing and reading: Reading from the topic, as super user")
assert message in read_from_topic("super", client_id, topic_name, 1, kerberos)
zookeeper_endpoint = sdk_cmd.svc_cli(
kafka_server["package_name"],
kafka_server["service"]["name"],
"endpoint zookeeper").strip()
# TODO: If zookeeper has Kerberos enabled, then the environment should be changed
topics.add_acls("authorized", client_id, topic_name, zookeeper_endpoint, env_str=None)
# Send a second message which should not be authorized
second_message = str(uuid.uuid4())
log.info("Writing and reading: Writing to the topic, but not super user")
assert write_to_topic("authorized", client_id, topic_name, second_message, kerberos)
log.info("Writing and reading: Writing to the topic, as super user")
assert write_to_topic("super", client_id, topic_name, second_message, kerberos)
log.info("Writing and reading: Reading from the topic, but not super user")
topic_output = read_from_topic("authorized", client_id, topic_name, 3, kerberos)
assert message in topic_output
assert second_message in topic_output
log.info("Writing and reading: Reading from the topic, as super user")
topic_output = read_from_topic("super", client_id, topic_name, 3, kerberos)
assert message in topic_output
assert second_message in topic_output
# Check that the unauthorized client can still not read or write from the topic.
log.info("Writing and reading: Writing to the topic, but not super user")
assert not write_to_topic("unauthorized", client_id, topic_name, second_message, kerberos)
log.info("Writing and reading: Reading from the topic, but not super user")
assert auth.is_not_authorized(read_from_topic("unauthorized", client_id, topic_name, 1, kerberos))
def test_authz_acls_required(kafka_client: client.KafkaClient,
kafka_server: dict,
kerberos: sdk_auth.KerberosEnvironment):
topic_name = "authz.test"
sdk_cmd.svc_cli(
kafka_server["package_name"],
kafka_server["service"]["name"],
"topic create {}".format(topic_name),
)
kafka_client.connect(kafka_server)
# Since no ACLs are specified, only the super user can read and write
for user in ["super"]:
log.info("Checking write / read permissions for user=%s", user)
write_success, read_successes, _ = kafka_client.can_write_and_read(
user, kafka_server, topic_name, kerberos)
assert write_success, "Write failed (user={})".format(user)
assert read_successes, ("Read failed (user={}): "
"MESSAGES={} "
"read_successes={}".format(
user, kafka_client.MESSAGES,
read_successes))
for user in ["authorized", "unauthorized"]:
log.info("Checking lack of write / read permissions for user=%s", user)
write_success, _, read_messages = kafka_client.can_write_and_read(
user, kafka_server, topic_name, kerberos)
assert not write_success, "Write not expected to succeed (user={})".format(
user)
assert auth.is_not_authorized(
read_messages), "Unauthorized expected (user={}".format(user)
log.info("Writing and reading: Adding acl for authorized user")
kafka_client.add_acls("authorized", kafka_server, topic_name)
# After adding ACLs the authorized user and super user should still have access to the topic.
for user in ["authorized", "super"]:
log.info("Checking write / read permissions for user=%s", user)
write_success, read_successes, _ = kafka_client.can_write_and_read(
user, kafka_server, topic_name, kerberos)
assert write_success, "Write failed (user={})".format(user)
assert read_successes, ("Read failed (user={}): "
"MESSAGES={} "
"read_successes={}".format(
user, kafka_client.MESSAGES,
read_successes))
for user in ["unauthorized"]:
log.info("Checking lack of write / read permissions for user=%s", user)
write_success, _, read_messages = kafka_client.can_write_and_read(
user, kafka_server, topic_name, kerberos)
assert not write_success, "Write not expected to succeed (user={})".format(
user)
assert auth.is_not_authorized(
read_messages), "Unauthorized expected (user={}".format(user)
def check_users_are_not_authorized_to_read_and_write(
self, users: typing.List[str], topic_name: str
) -> None:
for user in users:
log.info("Checking lack of write / read permissions for user=%s", user)
write_success, _, read_messages = self.can_write_and_read(user, topic_name)
assert not write_success, "Write not expected to succeed (user={})".format(user)
assert auth.is_not_authorized(read_messages), "Unauthorized expected (user={}".format(
user
)
def test_authz_acls_not_required(kafka_client, service_account,
setup_principals):
client_id = kafka_client["id"]
try:
sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
config.install(config.PACKAGE_NAME,
config.SERVICE_NAME,
config.DEFAULT_BROKER_COUNT,
additional_options={
"brokers": {
"port_tls": 1030
},
"service": {
"service_account": service_account["name"],
"service_account_secret":
service_account["secret"],
"security": {
"transport_encryption": {
"enabled": True
},
"ssl_authentication": {
"enabled": True
},
"authorization": {
"enabled": True,
"super_users":
"User:{}".format("super"),
"allow_everyone_if_no_acl_found": True
}
}
}
})
auth.wait_for_brokers(client_id, kafka_client["brokers"])
# Create the topic
sdk_cmd.svc_cli(config.PACKAGE_NAME,
config.SERVICE_NAME,
"topic create authz.test",
json=True)
test_utils.wait_for_topic(config.PACKAGE_NAME, config.SERVICE_NAME,
"authz.test")
super_message = str(uuid.uuid4())
authorized_message = str(uuid.uuid4())
unauthorized_message = str(uuid.uuid4())
log.info(
"Writing and reading: Writing to the topic, as authorized user")
assert write_to_topic("authorized", client_id, "authz.test",
authorized_message)
log.info(
"Writing and reading: Writing to the topic, as unauthorized user")
assert write_to_topic("unauthorized", client_id, "authz.test",
unauthorized_message)
log.info("Writing and reading: Writing to the topic, as super user")
assert write_to_topic("super", client_id, "authz.test", super_message)
log.info(
"Writing and reading: Reading from the topic, as authorized user")
assert authorized_message in read_from_topic("authorized", client_id,
"authz.test", 3)
log.info(
"Writing and reading: Reading from the topic, as unauthorized user"
)
assert unauthorized_message in read_from_topic("unauthorized",
client_id, "authz.test",
3)
log.info("Writing and reading: Reading from the topic, as super user")
assert super_message in read_from_topic("super", client_id,
"authz.test", 3)
log.info("Writing and reading: Adding acl for authorized user")
zookeeper_endpoint = str(
sdk_cmd.svc_cli(config.PACKAGE_NAME, config.SERVICE_NAME,
"endpoint zookeeper")).strip()
topics.add_acls("authorized",
client_id,
"authz.test",
zookeeper_endpoint,
env_str=None)
# Re-roll the messages so we really prove auth is in place.
super_message = str(uuid.uuid4())
authorized_message = str(uuid.uuid4())
unauthorized_message = str(uuid.uuid4())
log.info(
"Writing and reading: Writing to the topic, as authorized user")
assert write_to_topic("authorized", client_id, "authz.test",
authorized_message)
log.info(
"Writing and reading: Writing to the topic, as unauthorized user")
assert not write_to_topic("unauthorized", client_id, "authz.test",
unauthorized_message)
log.info("Writing and reading: Writing to the topic, as super user")
assert write_to_topic("super", client_id, "authz.test", super_message)
log.info(
"Writing and reading: Reading from the topic, as authorized user")
read_result = read_from_topic("authorized", client_id, "authz.test", 5)
assert authorized_message in read_result and unauthorized_message not in read_result
log.info(
"Writing and reading: Reading from the topic, as unauthorized user"
)
assert auth.is_not_authorized(
read_from_topic("unauthorized", client_id, "authz.test", 1))
log.info("Writing and reading: Reading from the topic, as super user")
read_result = read_from_topic("super", client_id, "authz.test", 5)
assert super_message in read_result and unauthorized_message not in read_result
finally:
sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
def test_authz_acls_not_required(kafka_client, service_account, setup_principals):
client_id = kafka_client["id"]
try:
sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
config.install(
config.PACKAGE_NAME,
config.SERVICE_NAME,
config.DEFAULT_BROKER_COUNT,
additional_options={
"brokers": {
"port_tls": 1030
},
"service": {
"service_account": service_account["name"],
"service_account_secret": service_account["secret"],
"security": {
"transport_encryption": {
"enabled": True
},
"ssl_authentication": {
"enabled": True
},
"authorization": {
"enabled": True,
"super_users": "User:{}".format("super"),
"allow_everyone_if_no_acl_found": True
}
}
}
})
sdk_cmd.resolve_hosts(client_id, kafka_client["brokers"])
# Create the topic
sdk_cmd.svc_cli(config.PACKAGE_NAME, config.SERVICE_NAME,
"topic create authz.test",
json=True)
test_utils.wait_for_topic(config.PACKAGE_NAME, config.SERVICE_NAME, "authz.test")
super_message = str(uuid.uuid4())
authorized_message = str(uuid.uuid4())
unauthorized_message = str(uuid.uuid4())
log.info("Writing and reading: Writing to the topic, as authorized user")
assert write_to_topic("authorized", client_id, "authz.test", authorized_message)
log.info("Writing and reading: Writing to the topic, as unauthorized user")
assert write_to_topic("unauthorized", client_id, "authz.test", unauthorized_message)
log.info("Writing and reading: Writing to the topic, as super user")
assert write_to_topic("super", client_id, "authz.test", super_message)
log.info("Writing and reading: Reading from the topic, as authorized user")
assert authorized_message in read_from_topic("authorized", client_id, "authz.test", 3)
log.info("Writing and reading: Reading from the topic, as unauthorized user")
assert unauthorized_message in read_from_topic("unauthorized", client_id, "authz.test", 3)
log.info("Writing and reading: Reading from the topic, as super user")
assert super_message in read_from_topic("super", client_id, "authz.test", 3)
log.info("Writing and reading: Adding acl for authorized user")
zookeeper_endpoint = str(sdk_cmd.svc_cli(
config.PACKAGE_NAME,
config.SERVICE_NAME,
"endpoint zookeeper")).strip()
topics.add_acls("authorized", client_id, "authz.test", zookeeper_endpoint, env_str=None)
# Re-roll the messages so we really prove auth is in place.
super_message = str(uuid.uuid4())
authorized_message = str(uuid.uuid4())
unauthorized_message = str(uuid.uuid4())
log.info("Writing and reading: Writing to the topic, as authorized user")
assert write_to_topic("authorized", client_id, "authz.test", authorized_message)
log.info("Writing and reading: Writing to the topic, as unauthorized user")
assert not write_to_topic("unauthorized", client_id, "authz.test", unauthorized_message)
log.info("Writing and reading: Writing to the topic, as super user")
assert write_to_topic("super", client_id, "authz.test", super_message)
log.info("Writing and reading: Reading from the topic, as authorized user")
read_result = read_from_topic("authorized", client_id, "authz.test", 5)
assert authorized_message in read_result and unauthorized_message not in read_result
log.info("Writing and reading: Reading from the topic, as unauthorized user")
assert auth.is_not_authorized(read_from_topic("unauthorized", client_id, "authz.test", 1))
log.info("Writing and reading: Reading from the topic, as super user")
read_result = read_from_topic("super", client_id, "authz.test", 5)
assert super_message in read_result and unauthorized_message not in read_result
finally:
sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
def test_authz_acls_required(kafka_client: client.KafkaClient, kerberos,
service_account, setup_principals):
try:
sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
service_options = {
"service": {
"name": config.SERVICE_NAME,
"service_account": service_account["name"],
"service_account_secret": service_account["secret"],
"security": {
"kerberos": {
"enabled": True,
"kdc": {
"hostname": kerberos.get_host(),
"port": int(kerberos.get_port())
},
"realm": kerberos.get_realm(),
"keytab_secret": kerberos.get_keytab_path(),
},
"transport_encryption": {
"enabled":
True,
"ciphers":
"TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_128_CBC_SHA256,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_CBC_SHA256,TLS_DHE_RSA_WITH_AES_128_GCM_SHA256,TLS_DHE_RSA_WITH_AES_128_CBC_SHA256,TLS_DHE_RSA_WITH_AES_256_GCM_SHA384,TLS_DHE_RSA_WITH_AES_256_CBC_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384",
},
"authorization": {
"enabled": True,
"super_users": "User:{}".format("super")
},
},
}
}
config.install(
config.PACKAGE_NAME,
config.SERVICE_NAME,
config.DEFAULT_BROKER_COUNT,
additional_options=service_options,
)
kafka_server = {
**service_options,
**{
"package_name": config.PACKAGE_NAME
}
}
topic_name = "authz.test"
sdk_cmd.svc_cli(
kafka_server["package_name"],
kafka_server["service"]["name"],
"topic create {}".format(topic_name),
parse_json=True,
)
kafka_client.connect()
# Clear the ACLs
kafka_client.remove_acls("authorized", topic_name)
# Since no ACLs are specified, only the super user can read and write
for user in ["super"]:
log.info("Checking write / read permissions for user=%s", user)
write_success, read_successes, _ = kafka_client.can_write_and_read(
user, topic_name)
assert write_success, "Write failed (user={})".format(user)
assert read_successes, ("Read failed (user={}): "
"MESSAGES={} "
"read_successes={}".format(
user, kafka_client.MESSAGES,
read_successes))
for user in ["authorized", "unauthorized"]:
log.info("Checking lack of write / read permissions for user=%s",
user)
write_success, _, read_messages = kafka_client.can_write_and_read(
user, topic_name)
assert not write_success, "Write not expected to succeed (user={})".format(
user)
assert auth.is_not_authorized(
read_messages), "Unauthorized expected (user={}".format(user)
log.info("Writing and reading: Adding acl for authorized user")
kafka_client.add_acls("authorized", topic_name)
# After adding ACLs the authorized user and super user should still have access to the topic.
for user in ["authorized", "super"]:
log.info("Checking write / read permissions for user=%s", user)
write_success, read_successes, _ = kafka_client.can_write_and_read(
user, topic_name)
assert write_success, "Write failed (user={})".format(user)
assert read_successes, ("Read failed (user={}): "
"MESSAGES={} "
"read_successes={}".format(
user, kafka_client.MESSAGES,
read_successes))
for user in ["unauthorized"]:
log.info("Checking lack of write / read permissions for user=%s",
user)
write_success, _, read_messages = kafka_client.can_write_and_read(
user, topic_name)
assert not write_success, "Write not expected to succeed (user={})".format(
user)
assert auth.is_not_authorized(
read_messages), "Unauthorized expected (user={}".format(user)
finally:
sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
def test_authz_acls_not_required(kafka_client, service_account,
setup_principals):
try:
sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
service_options = {
"service": {
"name": config.SERVICE_NAME,
"service_account": service_account["name"],
"service_account_secret": service_account["secret"],
"security": {
"transport_encryption": {
"enabled": True
},
"ssl_authentication": {
"enabled": True
},
"authorization": {
"enabled": True,
"super_users": "User:{}".format("super"),
"allow_everyone_if_no_acl_found": True,
},
},
}
}
config.install(
config.PACKAGE_NAME,
config.SERVICE_NAME,
config.DEFAULT_BROKER_COUNT,
additional_options=service_options,
)
kafka_server = {
**service_options,
**{
"package_name": config.PACKAGE_NAME
}
}
topic_name = "authz.test"
sdk_cmd.svc_cli(
kafka_server["package_name"],
kafka_server["service"]["name"],
"topic create {}".format(topic_name),
parse_json=True,
)
kafka_client.connect()
# Since no ACLs are specified, all users can read and write.
for user in ["authorized", "unauthorized", "super"]:
log.info("Checking write / read permissions for user=%s", user)
write_success, read_successes, _ = kafka_client.can_write_and_read(
user, topic_name)
assert write_success, "Write failed (user={})".format(user)
assert read_successes, ("Read failed (user={}): "
"MESSAGES={} "
"read_successes={}".format(
user, kafka_client.MESSAGES,
read_successes))
log.info("Writing and reading: Adding acl for authorized user")
kafka_client.add_acls("authorized", topic_name)
# After adding ACLs the authorized user and super user should still have access to the topic.
for user in ["authorized", "super"]:
log.info("Checking write / read permissions for user=%s", user)
write_success, read_successes, _ = kafka_client.can_write_and_read(
user, topic_name)
assert write_success, "Write failed (user={})".format(user)
assert read_successes, ("Read failed (user={}): "
"MESSAGES={} "
"read_successes={}".format(
user, kafka_client.MESSAGES,
read_successes))
for user in ["unauthorized"]:
log.info("Checking lack of write / read permissions for user=%s",
user)
write_success, _, read_messages = kafka_client.can_write_and_read(
user, topic_name)
assert not write_success, "Write not expected to succeed (user={})".format(
user)
assert auth.is_not_authorized(
read_messages), "Unauthorized expected (user={}".format(user)
finally:
sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
def test_authz_acls_not_required(kafka_client: client.KafkaClient,
zookeeper_server, kerberos):
try:
zookeeper_dns = sdk_networks.get_endpoint(
zookeeper_server["package_name"],
zookeeper_server["service"]["name"], "clientport")["dns"]
sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
service_options = {
"service": {
"name": config.SERVICE_NAME,
"security": {
"kerberos": {
"enabled": True,
"enabled_for_zookeeper": True,
"kdc": {
"hostname": kerberos.get_host(),
"port": int(kerberos.get_port())
},
"realm": kerberos.get_realm(),
"keytab_secret": kerberos.get_keytab_path(),
},
"authorization": {
"enabled": True,
"super_users": "User:{}".format("super"),
"allow_everyone_if_no_acl_found": True,
},
},
},
"kafka": {
"kafka_zookeeper_uri": ",".join(zookeeper_dns)
},
}
config.install(
config.PACKAGE_NAME,
config.SERVICE_NAME,
config.DEFAULT_BROKER_COUNT,
additional_options=service_options,
)
kafka_server = {
**service_options,
**{
"package_name": config.PACKAGE_NAME
}
}
topic_name = "authz.test"
sdk_cmd.svc_cli(
kafka_server["package_name"],
kafka_server["service"]["name"],
"topic create {}".format(topic_name),
)
kafka_client.connect(kafka_server)
# Clear the ACLs
kafka_client.remove_acls("authorized", kafka_server, topic_name)
# Since no ACLs are specified, all users can read and write.
for user in ["authorized", "unauthorized", "super"]:
log.info("Checking write / read permissions for user=%s", user)
write_success, read_successes, _ = kafka_client.can_write_and_read(
user, kafka_server, topic_name, kerberos)
assert write_success, "Write failed (user={})".format(user)
assert read_successes, ("Read failed (user={}): "
"MESSAGES={} "
"read_successes={}".format(
user, kafka_client.MESSAGES,
read_successes))
log.info("Writing and reading: Adding acl for authorized user")
kafka_client.add_acls("authorized", kafka_server, topic_name)
# After adding ACLs the authorized user and super user should still have access to the topic.
for user in ["authorized", "super"]:
log.info("Checking write / read permissions for user=%s", user)
write_success, read_successes, _ = kafka_client.can_write_and_read(
user, kafka_server, topic_name, kerberos)
assert write_success, "Write failed (user={})".format(user)
assert read_successes, ("Read failed (user={}): "
"MESSAGES={} "
"read_successes={}".format(
user, kafka_client.MESSAGES,
read_successes))
for user in ["unauthorized"]:
log.info("Checking lack of write / read permissions for user=%s",
user)
write_success, _, read_messages = kafka_client.can_write_and_read(
user, kafka_server, topic_name, kerberos)
assert not write_success, "Write not expected to succeed (user={})".format(
user)
assert auth.is_not_authorized(
read_messages), "Unauthorized expected (user={}".format(user)
finally:
# Ensure that we clean up the ZK state.
kafka_client.remove_acls("authorized", kafka_server, topic_name)
sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
def test_authz_acls_required(kafka_client, kafka_server):
client_id = kafka_client["id"]
auth.wait_for_brokers(kafka_client["id"], kafka_client["brokers"])
topic_name = "authz.test"
sdk_cmd.svc_cli(kafka_server["package_name"],
kafka_server["service"]["name"],
"topic create {}".format(topic_name),
json=True)
test_utils.wait_for_topic(kafka_server["package_name"],
kafka_server["service"]["name"], topic_name)
message = str(uuid.uuid4())
log.info("Writing and reading: Writing to the topic, but not super user")
assert not auth.write_to_topic("authorized", client_id, topic_name,
message)
log.info("Writing and reading: Writing to the topic, as super user")
assert auth.write_to_topic("super", client_id, topic_name, message)
log.info("Writing and reading: Reading from the topic, but not super user")
assert auth.is_not_authorized(
auth.read_from_topic("authorized", client_id, topic_name, 1))
log.info("Writing and reading: Reading from the topic, as super user")
assert message in auth.read_from_topic("super", client_id, topic_name, 1)
zookeeper_endpoint = sdk_cmd.svc_cli(kafka_server["package_name"],
kafka_server["service"]["name"],
"endpoint zookeeper").strip()
# TODO: If zookeeper has Kerberos enabled, then the environment should be changed
topics.add_acls("authorized",
client_id,
topic_name,
zookeeper_endpoint,
env_str=None)
# Send a second message which should not be authorized
second_message = str(uuid.uuid4())
log.info("Writing and reading: Writing to the topic, but not super user")
assert auth.write_to_topic("authorized", client_id, topic_name,
second_message)
log.info("Writing and reading: Writing to the topic, as super user")
assert auth.write_to_topic("super", client_id, topic_name, second_message)
log.info("Writing and reading: Reading from the topic, but not super user")
topic_output = auth.read_from_topic("authorized", client_id, topic_name, 3)
assert message in topic_output
assert second_message in topic_output
log.info("Writing and reading: Reading from the topic, as super user")
topic_output = auth.read_from_topic("super", client_id, topic_name, 3)
assert message in topic_output
assert second_message in topic_output
# Check that the unauthorized client can still not read or write from the topic.
log.info("Writing and reading: Writing to the topic, but not super user")
assert not auth.write_to_topic("unauthorized", client_id, topic_name,
second_message)
log.info("Writing and reading: Reading from the topic, but not super user")
assert auth.is_not_authorized(
auth.read_from_topic("unauthorized", client_id, topic_name, 1))
