def test_authn_client_can_read_and_write(kafka_client: client.KafkaClient,
service_account, setup_principals):
try:
sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
service_options = {
"service": {
"name": config.SERVICE_NAME,
"service_account": service_account["name"],
"service_account_secret": service_account["secret"],
"security": {
"transport_encryption": {
"enabled": True
},
"ssl_authentication": {
"enabled": True
},
},
}
}
config.install(
config.PACKAGE_NAME,
config.SERVICE_NAME,
config.DEFAULT_BROKER_COUNT,
additional_options=service_options,
)
topic_name = "tls.topic"
kafka_client.connect(config.DEFAULT_BROKER_COUNT)
kafka_client.create_topic(topic_name)
kafka_client.check_users_can_read_and_write(["kafka-tester"],
topic_name)
finally:
sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
def configure_package(configure_zookeeper):
try:
sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
zookeeper_framework_host = "{}.autoip.dcos.thisdcos.directory:1140".format(
ZK_SERVICE_NAME)
config.install(
config.PACKAGE_NAME,
config.SERVICE_NAME,
config.DEFAULT_BROKER_COUNT,
additional_options={
"kafka": {
"kafka_zookeeper_uri":
"zookeeper-0-server.{host},zookeeper-0-server.{host},zookeeper-0-server.{host}"
.format(host=zookeeper_framework_host)
}
})
# wait for brokers to finish registering before starting tests
test_utils.broker_count_check(config.DEFAULT_BROKER_COUNT,
service_name=config.SERVICE_NAME)
yield # let the test session execute
finally:
sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
def test_authz_acls_not_required(kafka_client: client.KafkaClient, kerberos,
service_account, setup_principals):
try:
sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
service_options = {
"service": {
"name": config.SERVICE_NAME,
"service_account": service_account["name"],
"service_account_secret": service_account["secret"],
"security": {
"kerberos": {
"enabled": True,
"kdc": {
"hostname": kerberos.get_host(),
"port": int(kerberos.get_port())
},
"realm": kerberos.get_realm(),
"keytab_secret": kerberos.get_keytab_path(),
},
"transport_encryption": {
"enabled": True
},
"authorization": {
"enabled": True,
"super_users": "User:{}".format("super"),
"allow_everyone_if_no_acl_found": True,
},
},
}
}
config.install(
config.PACKAGE_NAME,
config.SERVICE_NAME,
config.DEFAULT_BROKER_COUNT,
additional_options=service_options,
)
topic_name = "authz.test"
kafka_client.connect(config.DEFAULT_BROKER_COUNT)
kafka_client.create_topic(topic_name)
# Clear the ACLs
kafka_client.remove_acls("authorized", topic_name)
# Since no ACLs are specified, all users can read and write.
kafka_client.check_users_can_read_and_write(
["authorized", "unauthorized", "super"], topic_name)
log.info("Writing and reading: Adding acl for authorized user")
kafka_client.add_acls("authorized", topic_name)
# After adding ACLs the authorized user and super user should still have access to the topic.
kafka_client.check_users_can_read_and_write(["authorized", "super"],
topic_name)
kafka_client.check_users_are_not_authorized_to_read_and_write(
["unauthorized"], topic_name)
finally:
sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
def kafka_service_tls(service_account):
try:
sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
config.install(
config.PACKAGE_NAME,
config.SERVICE_NAME,
config.DEFAULT_BROKER_COUNT,
additional_options={
"service": {
"service_account": service_account,
"service_account_secret": service_account,
"security": {
"transport_encryption": {
"enabled": True
}
}
}
}
)
sdk_plan.wait_for_completed_deployment(config.SERVICE_NAME)
yield service_account
finally:
sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
def kafka_service_tls(service_account):
try:
sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
config.install(
config.PACKAGE_NAME,
config.SERVICE_NAME,
config.DEFAULT_BROKER_COUNT,
additional_options={
"service": {
"service_account": service_account["name"],
"service_account_secret": service_account["secret"],
"security": {
"transport_encryption": {
"enabled": True
}
}
}
}
)
sdk_plan.wait_for_completed_deployment(config.SERVICE_NAME)
yield service_account
finally:
sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
def kafka_server(zookeeper_server):
try:
# Get the zookeeper DNS values
zookeeper_dns = sdk_cmd.svc_cli(zookeeper_server["package_name"],
zookeeper_server["service"]["name"],
"endpoint clientport", json=True)["dns"]
sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
config.install(
config.PACKAGE_NAME,
config.SERVICE_NAME,
config.DEFAULT_BROKER_COUNT,
additional_options={
"kafka": {
"kafka_zookeeper_uri": ",".join(zookeeper_dns)
}
})
# wait for brokers to finish registering before starting tests
test_utils.broker_count_check(config.DEFAULT_BROKER_COUNT,
service_name=config.SERVICE_NAME)
yield # let the test session execute
finally:
sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
def _configure_kafka_cluster(
kafka_client: client.KafkaClient, zookeeper_service: typing.Dict,
allow_access_if_no_acl: bool) -> client.KafkaClient:
zookeeper_dns = sdk_networks.get_endpoint(
zookeeper_service["package_name"],
zookeeper_service["service"]["name"], "clientport")["dns"]
sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
service_options = _get_service_options(allow_access_if_no_acl,
kafka_client.kerberos,
zookeeper_dns)
config.install(
config.PACKAGE_NAME,
config.SERVICE_NAME,
config.DEFAULT_BROKER_COUNT,
additional_options=service_options,
)
kafka_server = {**service_options, **{"package_name": config.PACKAGE_NAME}}
sdk_cmd.svc_cli(
kafka_server["package_name"],
kafka_server["service"]["name"],
"topic create {}".format(TOPIC_NAME),
)
kafka_client.connect()
# Clear the ACLs
kafka_client.remove_acls("authorized", TOPIC_NAME)
return kafka_client
def kafka_server(zookeeper_server):
try:
# Get the zookeeper DNS values
zookeeper_dns = sdk_cmd.svc_cli(
zookeeper_server["package_name"],
zookeeper_server["service"]["name"],
"endpoint clientport",
parse_json=True,
)[1]["dns"]
sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
config.install(
config.PACKAGE_NAME,
config.SERVICE_NAME,
config.DEFAULT_BROKER_COUNT,
additional_options={
"kafka": {
"kafka_zookeeper_uri": ",".join(zookeeper_dns)
}
},
)
# wait for brokers to finish registering before starting tests
test_utils.broker_count_check(config.DEFAULT_BROKER_COUNT,
service_name=config.SERVICE_NAME)
yield # let the test session execute
finally:
sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
def kafka_server(zookeeper_service, kafka_client):
try:
# Get the zookeeper DNS values
zookeeper_dns = sdk_networks.get_endpoint(
zookeeper_service["package_name"],
zookeeper_service["service"]["name"], "clientport")["dns"]
sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
config.install(
config.PACKAGE_NAME,
config.SERVICE_NAME,
config.DEFAULT_BROKER_COUNT,
additional_options={
"kafka": {
"kafka_zookeeper_uri": ",".join(zookeeper_dns)
}
},
)
# wait for brokers to finish registering before starting tests
kafka_client.connect(config.DEFAULT_BROKER_COUNT)
yield # let the test session execute
finally:
sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
def test_authn_client_can_read_and_write(kafka_client: client.KafkaClient,
service_account, setup_principals):
try:
sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
service_options = {
"service": {
"name": config.SERVICE_NAME,
"service_account": service_account["name"],
"service_account_secret": service_account["secret"],
"security": {
"transport_encryption": {
"enabled":
True,
"ciphers":
"TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_128_CBC_SHA256,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_CBC_SHA256,TLS_DHE_RSA_WITH_AES_128_GCM_SHA256,TLS_DHE_RSA_WITH_AES_128_CBC_SHA256,TLS_DHE_RSA_WITH_AES_256_GCM_SHA384,TLS_DHE_RSA_WITH_AES_256_CBC_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384",
},
"ssl_authentication": {
"enabled": True
},
},
}
}
config.install(
config.PACKAGE_NAME,
config.SERVICE_NAME,
config.DEFAULT_BROKER_COUNT,
additional_options=service_options,
)
kafka_server = {
**service_options,
**{
"package_name": config.PACKAGE_NAME
}
}
topic_name = "tls.topic"
sdk_cmd.svc_cli(
kafka_server["package_name"],
kafka_server["service"]["name"],
"topic create {}".format(topic_name),
parse_json=True,
)
kafka_client.connect()
user = "******"
write_success, read_successes, _ = kafka_client.can_write_and_read(
user, topic_name)
assert write_success, "Write failed (user={})".format(user)
assert read_successes, ("Read failed (user={}): "
"MESSAGES={} "
"read_successes={}".format(
user, kafka_client.MESSAGES,
read_successes))
finally:
sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
def kafka_server(configure_security, kafka_client: client.KafkaClient):
try:
sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
config.install(config.PACKAGE_NAME, config.SERVICE_NAME,
config.DEFAULT_BROKER_COUNT)
kafka_client.connect(config.DEFAULT_BROKER_COUNT)
yield
finally:
sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
def kafka_server(kafka_client: client.KafkaClient, configure_security):
try:
sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
config.install(config.PACKAGE_NAME, config.SERVICE_NAME, config.DEFAULT_BROKER_COUNT)
kafka_client.connect()
yield {"package_name": config.PACKAGE_NAME, "service": {"name": config.SERVICE_NAME}}
finally:
sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
def setup_module(module):
options = {"brokers": {"kill_grace_period": BROKER_KILL_GRACE_PERIOD}}
sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
config.install(config.PACKAGE_NAME,
config.SERVICE_NAME,
config.DEFAULT_BROKER_COUNT,
additional_options=options)
sdk_plan.wait_for_completed_deployment(config.SERVICE_NAME)
def configure_package(configure_security):
try:
install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
config.install(
config.PACKAGE_NAME,
config.SERVICE_NAME,
config.DEFAULT_BROKER_COUNT,
additional_options=sdk_networks.ENABLE_VIRTUAL_NETWORKS_OPTIONS)
yield # let the test session execute
finally:
install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
def configure_package(configure_security):
try:
install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
config.install(
config.PACKAGE_NAME,
config.SERVICE_NAME,
config.DEFAULT_BROKER_COUNT,
additional_options=sdk_networks.ENABLE_VIRTUAL_NETWORKS_OPTIONS)
yield # let the test session execute
finally:
install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
def test_authn_client_can_read_and_write(kafka_client: client.KafkaClient,
service_account, setup_principals):
try:
sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
service_options = {
"service": {
"name": config.SERVICE_NAME,
"service_account": service_account["name"],
"service_account_secret": service_account["secret"],
"security": {
"transport_encryption": {
"enabled": True
},
"ssl_authentication": {
"enabled": True
}
}
}
}
config.install(config.PACKAGE_NAME,
config.SERVICE_NAME,
config.DEFAULT_BROKER_COUNT,
additional_options=service_options)
kafka_server = {
**service_options,
**{
"package_name": config.PACKAGE_NAME
}
}
topic_name = "tls.topic"
sdk_cmd.svc_cli(kafka_server["package_name"],
kafka_server["service"]["name"],
"topic create {}".format(topic_name),
json=True)
kafka_client.connect(kafka_server)
user = "******"
write_success, read_successes, _ = kafka_client.can_write_and_read(
user, kafka_server, topic_name, None)
assert write_success, "Write failed (user={})".format(user)
assert read_successes, "Read failed (user={}): " \
"MESSAGES={} " \
"read_successes={}".format(user,
kafka_client.MESSAGES,
read_successes)
finally:
sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
def test_authz_acls_required(kafka_client: client.KafkaClient, service_account,
setup_principals):
try:
sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
service_options = {
"service": {
"name": config.SERVICE_NAME,
"service_account": service_account["name"],
"service_account_secret": service_account["secret"],
"security": {
"transport_encryption": {
"enabled": True
},
"ssl_authentication": {
"enabled": True
},
"authorization": {
"enabled": True,
"super_users": "User:{}".format("super")
},
},
}
}
config.install(
config.PACKAGE_NAME,
config.SERVICE_NAME,
config.DEFAULT_BROKER_COUNT,
additional_options=service_options,
)
topic_name = "authz.test"
kafka_client.connect(config.DEFAULT_BROKER_COUNT)
kafka_client.create_topic(topic_name)
# Since no ACLs are specified, only the super user can read and write
kafka_client.check_users_can_read_and_write(["super"], topic_name)
kafka_client.check_users_are_not_authorized_to_read_and_write(
["authorized", "unauthorized"], topic_name)
log.info("Writing and reading: Adding acl for authorized user")
kafka_client.add_acls("authorized", topic_name)
# After adding ACLs the authorized user and super user should still have access to the topic.
kafka_client.check_users_can_read_and_write(["authorized", "super"],
topic_name)
kafka_client.check_users_are_not_authorized_to_read_and_write(
["unauthorized"], topic_name)
finally:
sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
def test_authn_client_can_read_and_write(kafka_client, service_account,
setup_principals):
try:
sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
config.install(config.PACKAGE_NAME,
config.SERVICE_NAME,
config.DEFAULT_BROKER_COUNT,
additional_options={
"brokers": {
"port_tls": 1030
},
"service": {
"service_account": service_account["name"],
"service_account_secret":
service_account["secret"],
"security": {
"transport_encryption": {
"enabled": True
},
"ssl_authentication": {
"enabled": True
}
}
}
})
client_id = kafka_client["id"]
auth.wait_for_brokers(client_id, kafka_client["brokers"])
sdk_cmd.svc_cli(config.PACKAGE_NAME,
config.SERVICE_NAME,
"topic create tls.topic",
json=True)
test_utils.wait_for_topic(config.PACKAGE_NAME, config.SERVICE_NAME,
"tls.topic")
message = str(uuid.uuid4())
# Write to the topic
log.info("Writing and reading: Writing to the topic, with authn")
assert write_to_topic("kafka-tester", client_id, "tls.topic", message)
log.info("Writing and reading: reading from the topic, with authn")
# Read from the topic
assert message in read_from_topic("kafka-tester", client_id,
"tls.topic", 1)
finally:
sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
def setup_module(module):
options = {
"brokers": {
"kill_grace_period": BROKER_KILL_GRACE_PERIOD
}
}
sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
config.install(
config.PACKAGE_NAME,
config.SERVICE_NAME,
config.DEFAULT_BROKER_COUNT,
additional_options=options)
sdk_plan.wait_for_completed_deployment(config.SERVICE_NAME)
def test_authn_client_can_read_and_write(kafka_client, service_account, setup_principals):
try:
sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
config.install(
config.PACKAGE_NAME,
config.SERVICE_NAME,
config.DEFAULT_BROKER_COUNT,
additional_options={
"brokers": {
"port_tls": 1030
},
"service": {
"service_account": service_account["name"],
"service_account_secret": service_account["secret"],
"security": {
"transport_encryption": {
"enabled": True
},
"ssl_authentication": {
"enabled": True
}
}
}
})
client_id = kafka_client["id"]
sdk_cmd.resolve_hosts(client_id, kafka_client["brokers"])
sdk_cmd.svc_cli(config.PACKAGE_NAME, config.SERVICE_NAME,
"topic create tls.topic",
json=True)
test_utils.wait_for_topic(config.PACKAGE_NAME, config.SERVICE_NAME, "tls.topic")
message = str(uuid.uuid4())
# Write to the topic
log.info("Writing and reading: Writing to the topic, with authn")
assert write_to_topic("kafka-tester", client_id, "tls.topic", message)
log.info("Writing and reading: reading from the topic, with authn")
# Read from the topic
assert message in read_from_topic("kafka-tester", client_id, "tls.topic", 1)
finally:
sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
def kafka_server(configure_security):
try:
sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
config.install(
config.PACKAGE_NAME,
config.SERVICE_NAME,
config.DEFAULT_BROKER_COUNT)
# wait for brokers to finish registering before starting tests
test_utils.broker_count_check(config.DEFAULT_BROKER_COUNT,
service_name=config.SERVICE_NAME)
# Since the tests below interact with the brokers, ensure that the DNS resolves
test_utils.wait_for_broker_dns(config.PACKAGE_NAME, config.SERVICE_NAME)
yield {"package_name": config.PACKAGE_NAME, "service": {"name": config.SERVICE_NAME}}
finally:
sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
def kafka_server(configure_security):
try:
sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
config.install(
config.PACKAGE_NAME,
config.SERVICE_NAME,
config.DEFAULT_BROKER_COUNT)
# wait for brokers to finish registering before starting tests
test_utils.broker_count_check(config.DEFAULT_BROKER_COUNT,
service_name=config.SERVICE_NAME)
# Since the tests below interact with the brokers, ensure that the DNS resolves
test_utils.wait_for_broker_dns(config.PACKAGE_NAME, config.SERVICE_NAME)
yield {"package_name": config.PACKAGE_NAME, "service": {"name": config.SERVICE_NAME}}
finally:
sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
def _configure_kafka_cluster(
kafka_client: client.KafkaClient, zookeeper_service: typing.Dict,
allow_access_if_no_acl: bool) -> client.KafkaClient:
zookeeper_dns = sdk_networks.get_endpoint(
zookeeper_service["package_name"],
zookeeper_service["service"]["name"], "clientport")["dns"]
sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
service_options = _get_service_options(allow_access_if_no_acl,
kafka_client.kerberos,
zookeeper_dns)
config.install(
config.PACKAGE_NAME,
config.SERVICE_NAME,
config.DEFAULT_BROKER_COUNT,
additional_options=service_options,
)
kafka_client.connect(config.DEFAULT_BROKER_COUNT)
# Clear the ACLs
return kafka_client
def kafka_service_tls(service_account):
config.install(config.PACKAGE_NAME,
config.SERVICE_NAME,
config.DEFAULT_BROKER_COUNT,
additional_options={
"service": {
"service_account": service_account,
"service_account_secret": service_account,
"security": {
"transport_encryption": {
"enabled": True
}
}
}
})
sdk_plan.wait_for_completed_deployment(config.SERVICE_NAME)
# Wait for service health check to pass
shakedown.service_healthy(config.SERVICE_NAME)
yield service_account
sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
def kafka_service_tls(service_account):
config.install(
config.PACKAGE_NAME,
config.SERVICE_NAME,
config.DEFAULT_BROKER_COUNT,
additional_options={
"service": {
"service_account": service_account,
"service_account_secret": service_account,
# Legacy values
"principal": service_account,
"secret_name": service_account,
"tls": True
}
})
sdk_plan.wait_for_completed_deployment(config.SERVICE_NAME)
# Wait for service health check to pass
shakedown.service_healthy(config.SERVICE_NAME)
yield service_account
sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
def test_authz_acls_not_required(kafka_client, service_account,
setup_principals):
client_id = kafka_client["id"]
try:
sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
config.install(config.PACKAGE_NAME,
config.SERVICE_NAME,
config.DEFAULT_BROKER_COUNT,
additional_options={
"brokers": {
"port_tls": 1030
},
"service": {
"service_account": service_account["name"],
"service_account_secret":
service_account["secret"],
"security": {
"transport_encryption": {
"enabled": True
},
"ssl_authentication": {
"enabled": True
},
"authorization": {
"enabled": True,
"super_users":
"User:{}".format("super"),
"allow_everyone_if_no_acl_found": True
}
}
}
})
auth.wait_for_brokers(client_id, kafka_client["brokers"])
# Create the topic
sdk_cmd.svc_cli(config.PACKAGE_NAME,
config.SERVICE_NAME,
"topic create authz.test",
json=True)
test_utils.wait_for_topic(config.PACKAGE_NAME, config.SERVICE_NAME,
"authz.test")
super_message = str(uuid.uuid4())
authorized_message = str(uuid.uuid4())
unauthorized_message = str(uuid.uuid4())
log.info(
"Writing and reading: Writing to the topic, as authorized user")
assert write_to_topic("authorized", client_id, "authz.test",
authorized_message)
log.info(
"Writing and reading: Writing to the topic, as unauthorized user")
assert write_to_topic("unauthorized", client_id, "authz.test",
unauthorized_message)
log.info("Writing and reading: Writing to the topic, as super user")
assert write_to_topic("super", client_id, "authz.test", super_message)
log.info(
"Writing and reading: Reading from the topic, as authorized user")
assert authorized_message in read_from_topic("authorized", client_id,
"authz.test", 3)
log.info(
"Writing and reading: Reading from the topic, as unauthorized user"
)
assert unauthorized_message in read_from_topic("unauthorized",
client_id, "authz.test",
3)
log.info("Writing and reading: Reading from the topic, as super user")
assert super_message in read_from_topic("super", client_id,
"authz.test", 3)
log.info("Writing and reading: Adding acl for authorized user")
zookeeper_endpoint = str(
sdk_cmd.svc_cli(config.PACKAGE_NAME, config.SERVICE_NAME,
"endpoint zookeeper")).strip()
topics.add_acls("authorized",
client_id,
"authz.test",
zookeeper_endpoint,
env_str=None)
# Re-roll the messages so we really prove auth is in place.
super_message = str(uuid.uuid4())
authorized_message = str(uuid.uuid4())
unauthorized_message = str(uuid.uuid4())
log.info(
"Writing and reading: Writing to the topic, as authorized user")
assert write_to_topic("authorized", client_id, "authz.test",
authorized_message)
log.info(
"Writing and reading: Writing to the topic, as unauthorized user")
assert not write_to_topic("unauthorized", client_id, "authz.test",
unauthorized_message)
log.info("Writing and reading: Writing to the topic, as super user")
assert write_to_topic("super", client_id, "authz.test", super_message)
log.info(
"Writing and reading: Reading from the topic, as authorized user")
read_result = read_from_topic("authorized", client_id, "authz.test", 5)
assert authorized_message in read_result and unauthorized_message not in read_result
log.info(
"Writing and reading: Reading from the topic, as unauthorized user"
)
assert auth.is_not_authorized(
read_from_topic("unauthorized", client_id, "authz.test", 1))
log.info("Writing and reading: Reading from the topic, as super user")
read_result = read_from_topic("super", client_id, "authz.test", 5)
assert super_message in read_result and unauthorized_message not in read_result
finally:
sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
def test_authz_acls_not_required(kafka_client, service_account,
setup_principals):
try:
sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
service_options = {
"service": {
"name": config.SERVICE_NAME,
"service_account": service_account["name"],
"service_account_secret": service_account["secret"],
"security": {
"transport_encryption": {
"enabled": True
},
"ssl_authentication": {
"enabled": True
},
"authorization": {
"enabled": True,
"super_users": "User:{}".format("super"),
"allow_everyone_if_no_acl_found": True,
},
},
}
}
config.install(
config.PACKAGE_NAME,
config.SERVICE_NAME,
config.DEFAULT_BROKER_COUNT,
additional_options=service_options,
)
kafka_server = {
**service_options,
**{
"package_name": config.PACKAGE_NAME
}
}
topic_name = "authz.test"
sdk_cmd.svc_cli(
kafka_server["package_name"],
kafka_server["service"]["name"],
"topic create {}".format(topic_name),
)
kafka_client.connect()
# Since no ACLs are specified, all users can read and write.
kafka_client.check_users_can_read_and_write(
["authorized", "unauthorized", "super"], topic_name)
log.info("Writing and reading: Adding acl for authorized user")
kafka_client.add_acls("authorized", topic_name)
# After adding ACLs the authorized user and super user should still have access to the topic.
kafka_client.check_users_can_read_and_write(["authorized", "super"],
topic_name)
kafka_client.check_users_are_not_authorized_to_read_and_write(
["unauthorized"], topic_name)
finally:
sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
def test_authz_acls_not_required(kafka_client, service_account, setup_principals):
client_id = kafka_client["id"]
try:
sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
config.install(
config.PACKAGE_NAME,
config.SERVICE_NAME,
config.DEFAULT_BROKER_COUNT,
additional_options={
"brokers": {
"port_tls": 1030
},
"service": {
"service_account": service_account["name"],
"service_account_secret": service_account["secret"],
"security": {
"transport_encryption": {
"enabled": True
},
"ssl_authentication": {
"enabled": True
},
"authorization": {
"enabled": True,
"super_users": "User:{}".format("super"),
"allow_everyone_if_no_acl_found": True
}
}
}
})
sdk_cmd.resolve_hosts(client_id, kafka_client["brokers"])
# Create the topic
sdk_cmd.svc_cli(config.PACKAGE_NAME, config.SERVICE_NAME,
"topic create authz.test",
json=True)
test_utils.wait_for_topic(config.PACKAGE_NAME, config.SERVICE_NAME, "authz.test")
super_message = str(uuid.uuid4())
authorized_message = str(uuid.uuid4())
unauthorized_message = str(uuid.uuid4())
log.info("Writing and reading: Writing to the topic, as authorized user")
assert write_to_topic("authorized", client_id, "authz.test", authorized_message)
log.info("Writing and reading: Writing to the topic, as unauthorized user")
assert write_to_topic("unauthorized", client_id, "authz.test", unauthorized_message)
log.info("Writing and reading: Writing to the topic, as super user")
assert write_to_topic("super", client_id, "authz.test", super_message)
log.info("Writing and reading: Reading from the topic, as authorized user")
assert authorized_message in read_from_topic("authorized", client_id, "authz.test", 3)
log.info("Writing and reading: Reading from the topic, as unauthorized user")
assert unauthorized_message in read_from_topic("unauthorized", client_id, "authz.test", 3)
log.info("Writing and reading: Reading from the topic, as super user")
assert super_message in read_from_topic("super", client_id, "authz.test", 3)
log.info("Writing and reading: Adding acl for authorized user")
zookeeper_endpoint = str(sdk_cmd.svc_cli(
config.PACKAGE_NAME,
config.SERVICE_NAME,
"endpoint zookeeper")).strip()
topics.add_acls("authorized", client_id, "authz.test", zookeeper_endpoint, env_str=None)
# Re-roll the messages so we really prove auth is in place.
super_message = str(uuid.uuid4())
authorized_message = str(uuid.uuid4())
unauthorized_message = str(uuid.uuid4())
log.info("Writing and reading: Writing to the topic, as authorized user")
assert write_to_topic("authorized", client_id, "authz.test", authorized_message)
log.info("Writing and reading: Writing to the topic, as unauthorized user")
assert not write_to_topic("unauthorized", client_id, "authz.test", unauthorized_message)
log.info("Writing and reading: Writing to the topic, as super user")
assert write_to_topic("super", client_id, "authz.test", super_message)
log.info("Writing and reading: Reading from the topic, as authorized user")
read_result = read_from_topic("authorized", client_id, "authz.test", 5)
assert authorized_message in read_result and unauthorized_message not in read_result
log.info("Writing and reading: Reading from the topic, as unauthorized user")
assert auth.is_not_authorized(read_from_topic("unauthorized", client_id, "authz.test", 1))
log.info("Writing and reading: Reading from the topic, as super user")
read_result = read_from_topic("super", client_id, "authz.test", 5)
assert super_message in read_result and unauthorized_message not in read_result
finally:
sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
def test_authz_acls_required(kafka_client: client.KafkaClient, kerberos,
service_account, setup_principals):
try:
sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
service_options = {
"service": {
"name": config.SERVICE_NAME,
"service_account": service_account["name"],
"service_account_secret": service_account["secret"],
"security": {
"kerberos": {
"enabled": True,
"kdc": {
"hostname": kerberos.get_host(),
"port": int(kerberos.get_port())
},
"realm": kerberos.get_realm(),
"keytab_secret": kerberos.get_keytab_path(),
},
"transport_encryption": {
"enabled":
True,
"ciphers":
"TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_128_CBC_SHA256,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_CBC_SHA256,TLS_DHE_RSA_WITH_AES_128_GCM_SHA256,TLS_DHE_RSA_WITH_AES_128_CBC_SHA256,TLS_DHE_RSA_WITH_AES_256_GCM_SHA384,TLS_DHE_RSA_WITH_AES_256_CBC_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384",
},
"authorization": {
"enabled": True,
"super_users": "User:{}".format("super")
},
},
}
}
config.install(
config.PACKAGE_NAME,
config.SERVICE_NAME,
config.DEFAULT_BROKER_COUNT,
additional_options=service_options,
)
kafka_server = {
**service_options,
**{
"package_name": config.PACKAGE_NAME
}
}
topic_name = "authz.test"
sdk_cmd.svc_cli(
kafka_server["package_name"],
kafka_server["service"]["name"],
"topic create {}".format(topic_name),
parse_json=True,
)
kafka_client.connect()
# Clear the ACLs
kafka_client.remove_acls("authorized", topic_name)
# Since no ACLs are specified, only the super user can read and write
for user in ["super"]:
log.info("Checking write / read permissions for user=%s", user)
write_success, read_successes, _ = kafka_client.can_write_and_read(
user, topic_name)
assert write_success, "Write failed (user={})".format(user)
assert read_successes, ("Read failed (user={}): "
"MESSAGES={} "
"read_successes={}".format(
user, kafka_client.MESSAGES,
read_successes))
for user in ["authorized", "unauthorized"]:
log.info("Checking lack of write / read permissions for user=%s",
user)
write_success, _, read_messages = kafka_client.can_write_and_read(
user, topic_name)
assert not write_success, "Write not expected to succeed (user={})".format(
user)
assert auth.is_not_authorized(
read_messages), "Unauthorized expected (user={}".format(user)
log.info("Writing and reading: Adding acl for authorized user")
kafka_client.add_acls("authorized", topic_name)
# After adding ACLs the authorized user and super user should still have access to the topic.
for user in ["authorized", "super"]:
log.info("Checking write / read permissions for user=%s", user)
write_success, read_successes, _ = kafka_client.can_write_and_read(
user, topic_name)
assert write_success, "Write failed (user={})".format(user)
assert read_successes, ("Read failed (user={}): "
"MESSAGES={} "
"read_successes={}".format(
user, kafka_client.MESSAGES,
read_successes))
for user in ["unauthorized"]:
log.info("Checking lack of write / read permissions for user=%s",
user)
write_success, _, read_messages = kafka_client.can_write_and_read(
user, topic_name)
assert not write_success, "Write not expected to succeed (user={})".format(
user)
assert auth.is_not_authorized(
read_messages), "Unauthorized expected (user={}".format(user)
finally:
sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
def test_authz_acls_not_required(kafka_client, service_account,
setup_principals):
try:
sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
service_options = {
"service": {
"name": config.SERVICE_NAME,
"service_account": service_account["name"],
"service_account_secret": service_account["secret"],
"security": {
"transport_encryption": {
"enabled": True
},
"ssl_authentication": {
"enabled": True
},
"authorization": {
"enabled": True,
"super_users": "User:{}".format("super"),
"allow_everyone_if_no_acl_found": True,
},
},
}
}
config.install(
config.PACKAGE_NAME,
config.SERVICE_NAME,
config.DEFAULT_BROKER_COUNT,
additional_options=service_options,
)
kafka_server = {
**service_options,
**{
"package_name": config.PACKAGE_NAME
}
}
topic_name = "authz.test"
sdk_cmd.svc_cli(
kafka_server["package_name"],
kafka_server["service"]["name"],
"topic create {}".format(topic_name),
parse_json=True,
)
kafka_client.connect()
# Since no ACLs are specified, all users can read and write.
for user in ["authorized", "unauthorized", "super"]:
log.info("Checking write / read permissions for user=%s", user)
write_success, read_successes, _ = kafka_client.can_write_and_read(
user, topic_name)
assert write_success, "Write failed (user={})".format(user)
assert read_successes, ("Read failed (user={}): "
"MESSAGES={} "
"read_successes={}".format(
user, kafka_client.MESSAGES,
read_successes))
log.info("Writing and reading: Adding acl for authorized user")
kafka_client.add_acls("authorized", topic_name)
# After adding ACLs the authorized user and super user should still have access to the topic.
for user in ["authorized", "super"]:
log.info("Checking write / read permissions for user=%s", user)
write_success, read_successes, _ = kafka_client.can_write_and_read(
user, topic_name)
assert write_success, "Write failed (user={})".format(user)
assert read_successes, ("Read failed (user={}): "
"MESSAGES={} "
"read_successes={}".format(
user, kafka_client.MESSAGES,
read_successes))
for user in ["unauthorized"]:
log.info("Checking lack of write / read permissions for user=%s",
user)
write_success, _, read_messages = kafka_client.can_write_and_read(
user, topic_name)
assert not write_success, "Write not expected to succeed (user={})".format(
user)
assert auth.is_not_authorized(
read_messages), "Unauthorized expected (user={}".format(user)
finally:
sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
def test_authz_acls_not_required(kafka_client: client.KafkaClient,
zookeeper_server, kerberos):
try:
zookeeper_dns = sdk_networks.get_endpoint(
zookeeper_server["package_name"],
zookeeper_server["service"]["name"], "clientport")["dns"]
sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
service_options = {
"service": {
"name": config.SERVICE_NAME,
"security": {
"kerberos": {
"enabled": True,
"enabled_for_zookeeper": True,
"kdc": {
"hostname": kerberos.get_host(),
"port": int(kerberos.get_port())
},
"realm": kerberos.get_realm(),
"keytab_secret": kerberos.get_keytab_path(),
},
"authorization": {
"enabled": True,
"super_users": "User:{}".format("super"),
"allow_everyone_if_no_acl_found": True,
},
},
},
"kafka": {
"kafka_zookeeper_uri": ",".join(zookeeper_dns)
},
}
config.install(
config.PACKAGE_NAME,
config.SERVICE_NAME,
config.DEFAULT_BROKER_COUNT,
additional_options=service_options,
)
kafka_server = {
**service_options,
**{
"package_name": config.PACKAGE_NAME
}
}
topic_name = "authz.test"
sdk_cmd.svc_cli(
kafka_server["package_name"],
kafka_server["service"]["name"],
"topic create {}".format(topic_name),
)
kafka_client.connect(kafka_server)
# Clear the ACLs
kafka_client.remove_acls("authorized", kafka_server, topic_name)
# Since no ACLs are specified, all users can read and write.
for user in ["authorized", "unauthorized", "super"]:
log.info("Checking write / read permissions for user=%s", user)
write_success, read_successes, _ = kafka_client.can_write_and_read(
user, kafka_server, topic_name, kerberos)
assert write_success, "Write failed (user={})".format(user)
assert read_successes, ("Read failed (user={}): "
"MESSAGES={} "
"read_successes={}".format(
user, kafka_client.MESSAGES,
read_successes))
log.info("Writing and reading: Adding acl for authorized user")
kafka_client.add_acls("authorized", kafka_server, topic_name)
# After adding ACLs the authorized user and super user should still have access to the topic.
for user in ["authorized", "super"]:
log.info("Checking write / read permissions for user=%s", user)
write_success, read_successes, _ = kafka_client.can_write_and_read(
user, kafka_server, topic_name, kerberos)
assert write_success, "Write failed (user={})".format(user)
assert read_successes, ("Read failed (user={}): "
"MESSAGES={} "
"read_successes={}".format(
user, kafka_client.MESSAGES,
read_successes))
for user in ["unauthorized"]:
log.info("Checking lack of write / read permissions for user=%s",
user)
write_success, _, read_messages = kafka_client.can_write_and_read(
user, kafka_server, topic_name, kerberos)
assert not write_success, "Write not expected to succeed (user={})".format(
user)
assert auth.is_not_authorized(
read_messages), "Unauthorized expected (user={}".format(user)
finally:
# Ensure that we clean up the ZK state.
kafka_client.remove_acls("authorized", kafka_server, topic_name)
sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
