def test_authz_acls_required(kafka_client, kafka_server, kerberos): client_id = kafka_client["id"] sdk_cmd.resolve_hosts(kafka_client["id"], kafka_client["brokers"]) topic_name = "authz.test" sdk_cmd.svc_cli(kafka_server["package_name"], kafka_server["service"]["name"], "topic create {}".format(topic_name), json=True) test_utils.wait_for_topic(kafka_server["package_name"], kafka_server["service"]["name"], topic_name) message = str(uuid.uuid4()) log.info("Writing and reading: Writing to the topic, but not super user") assert not write_to_topic("authorized", client_id, topic_name, message, kerberos) log.info("Writing and reading: Writing to the topic, as super user") assert write_to_topic("super", client_id, topic_name, message, kerberos) log.info("Writing and reading: Reading from the topic, but not super user") assert auth.is_not_authorized(read_from_topic("authorized", client_id, topic_name, 1, kerberos)) log.info("Writing and reading: Reading from the topic, as super user") assert message in read_from_topic("super", client_id, topic_name, 1, kerberos) zookeeper_endpoint = sdk_cmd.svc_cli( kafka_server["package_name"], kafka_server["service"]["name"], "endpoint zookeeper").strip() # TODO: If zookeeper has Kerberos enabled, then the environment should be changed topics.add_acls("authorized", client_id, topic_name, zookeeper_endpoint, env_str=None) # Send a second message which should not be authorized second_message = str(uuid.uuid4()) log.info("Writing and reading: Writing to the topic, but not super user") assert write_to_topic("authorized", client_id, topic_name, second_message, kerberos) log.info("Writing and reading: Writing to the topic, as super user") assert write_to_topic("super", client_id, topic_name, second_message, kerberos) log.info("Writing and reading: Reading from the topic, but not super user") topic_output = read_from_topic("authorized", client_id, topic_name, 3, kerberos) assert message in topic_output assert second_message in topic_output log.info("Writing and reading: Reading from the topic, as super user") topic_output = read_from_topic("super", client_id, topic_name, 3, kerberos) assert message in topic_output assert second_message in topic_output # Check that the unauthorized client can still not read or write from the topic. log.info("Writing and reading: Writing to the topic, but not super user") assert not write_to_topic("unauthorized", client_id, topic_name, second_message, kerberos) log.info("Writing and reading: Reading from the topic, but not super user") assert auth.is_not_authorized(read_from_topic("unauthorized", client_id, topic_name, 1, kerberos))
def client_can_read_and_write(test_id: str, kafka_client: dict, kafka_server: dict, endpoint_name: str, krb5: object=None) -> tuple: client_id = kafka_client["id"] brokers_list = service_get_brokers(kafka_server, endpoint_name) broker_hosts = map(lambda b: b.split(":")[0], brokers_list) brokers = ",".join(brokers_list) if not sdk_cmd.resolve_hosts(kafka_client["id"], broker_hosts): log.error("Failed to resolve brokers: %s", broker_hosts) return False, [] topic_name = kafka_client["env"]["KAFKA_TOPIC"] sdk_cmd.svc_cli(kafka_server["package_name"], kafka_server["service"]["name"], "topic create {}".format(topic_name), json=True) test_utils.wait_for_topic(kafka_server["package_name"], kafka_server["service"]["name"], topic_name) message = str(uuid.uuid4()) security_options = {"is-tls": endpoint_name == "broker-tls", "kerberos": krb5} write_success = write_to_topic(test_id, client_id, topic_name, message, brokers, security_options) if write_success: MESSAGES.append(message) read_messages = read_from_topic(test_id, client_id, topic_name, len(MESSAGES), brokers, security_options) read_success = map(lambda m: m in read_messages, MESSAGES) return write_success, read_success
def wait_for(self, kafka_server: dict, topic_name: str) -> bool: """ Wait for the service to be visible from a client perspective. """ service = KafkaService(kafka_server) if not self.brokers: brokers_list = service.get_brokers_endpoints( self.get_endpoint_name()) broker_hosts = map(lambda b: b.split(":")[0], brokers_list) brokers = ",".join(brokers_list) if not sdk_cmd.resolve_hosts( self.id, broker_hosts, bootstrap_cmd='/opt/bootstrap'): log.error("Failed to resolve brokers: %s", broker_hosts) return False self.brokers = brokers return True if self.topic_name != topic_name: service.wait_for_topic(topic_name) self.topic_name = topic_name return True
def test_client_can_read_and_write(kafka_client, kafka_server, kerberos): client_id = kafka_client["id"] sdk_cmd.resolve_hosts(kafka_client["id"], kafka_client["brokers"]) topic_name = "authn.test" sdk_cmd.svc_cli(kafka_server["package_name"], kafka_server["service"]["name"], "topic create {}".format(topic_name), json=True) test_utils.wait_for_topic(kafka_server["package_name"], kafka_server["service"]["name"], topic_name) message = str(uuid.uuid4()) assert write_to_topic("client", client_id, topic_name, message, kerberos) assert message in read_from_topic("client", client_id, topic_name, 1, kerberos)
def test_authn_client_can_read_and_write(kafka_client, service_account, setup_principals): try: sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME) config.install( config.PACKAGE_NAME, config.SERVICE_NAME, config.DEFAULT_BROKER_COUNT, additional_options={ "brokers": { "port_tls": 1030 }, "service": { "service_account": service_account["name"], "service_account_secret": service_account["secret"], "security": { "transport_encryption": { "enabled": True }, "ssl_authentication": { "enabled": True } } } }) client_id = kafka_client["id"] sdk_cmd.resolve_hosts(client_id, kafka_client["brokers"]) sdk_cmd.svc_cli(config.PACKAGE_NAME, config.SERVICE_NAME, "topic create tls.topic", json=True) test_utils.wait_for_topic(config.PACKAGE_NAME, config.SERVICE_NAME, "tls.topic") message = str(uuid.uuid4()) # Write to the topic log.info("Writing and reading: Writing to the topic, with authn") assert write_to_topic("kafka-tester", client_id, "tls.topic", message) log.info("Writing and reading: reading from the topic, with authn") # Read from the topic assert message in read_from_topic("kafka-tester", client_id, "tls.topic", 1) finally: sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
def test_client_can_read_and_write(kafka_client, kafka_server, kerberos): client_id = kafka_client["id"] sdk_cmd.resolve_hosts(kafka_client["id"], kafka_client["brokers"]) topic_name = "authn.test" sdk_cmd.svc_cli(kafka_server["package_name"], kafka_server["service"]["name"], "topic create {}".format(topic_name), json=True) test_utils.wait_for_topic(kafka_server["package_name"], kafka_server["service"]["name"], topic_name) message = str(uuid.uuid4()) assert write_to_topic("client", client_id, topic_name, message, kerberos) assert message in read_from_topic("client", client_id, topic_name, 1, kerberos)
def wait_for_broker_dns(package_name: str, service_name: str): brokers = sdk_networks.get_endpoint(package_name, service_name, "broker") broker_dns = list(map(lambda x: x.split(":")[0], brokers["dns"])) def get_scheduler_task_id(service_name: str) -> str: for task in sdk_tasks.get_summary(): if task.name == service_name: return task.id scheduler_task_id = get_scheduler_task_id(service_name) log.info("Scheduler task ID: %s", scheduler_task_id) log.info("Waiting for brokers: %s", broker_dns) assert sdk_cmd.resolve_hosts(scheduler_task_id, broker_dns)
def wait_for_broker_dns(package_name: str, service_name: str): brokers = sdk_cmd.svc_cli(package_name, service_name, "endpoint broker", json=True) broker_dns = list(map(lambda x: x.split(':')[0], brokers["dns"])) def get_scheduler_task_id(service_name: str) -> str: for task in sdk_tasks.get_summary(): if task.name == service_name: return task.id scheduler_task_id = get_scheduler_task_id(service_name) log.info("Scheduler task ID: %s", scheduler_task_id) log.info("Waiting for brokers: %s", broker_dns) assert sdk_cmd.resolve_hosts(scheduler_task_id, broker_dns)
def client_can_read_and_write(test_id: str, kafka_client: dict, kafka_server: dict, endpoint_name: str, krb5: object = None) -> tuple: client_id = kafka_client["id"] brokers_list = service_get_brokers(kafka_server, endpoint_name) broker_hosts = map(lambda b: b.split(":")[0], brokers_list) brokers = ",".join(brokers_list) if not sdk_cmd.resolve_hosts(kafka_client["id"], broker_hosts): log.error("Failed to resolve brokers: %s", broker_hosts) return False, [] topic_name = kafka_client["env"]["KAFKA_TOPIC"] sdk_cmd.svc_cli( kafka_server["package_name"], kafka_server["service"]["name"], "topic create {}".format(topic_name), ) test_utils.wait_for_topic(kafka_server["package_name"], kafka_server["service"]["name"], topic_name) message = str(uuid.uuid4()) security_options = { "is-tls": endpoint_name == "broker-tls", "kerberos": krb5 } write_success = write_to_topic(test_id, client_id, topic_name, message, brokers, security_options) if write_success: MESSAGES.append(message) read_messages = read_from_topic(test_id, client_id, topic_name, len(MESSAGES), brokers, security_options) read_success = map(lambda m: m in read_messages, MESSAGES) return write_success, read_success
def wait_for(self, topic_name: typing.Optional[str] = None) -> bool: """ Wait for the service to be visible from a client perspective. """ if not self.brokers: brokers_list = self.kafka_service.get_endpoint_dns(self.get_endpoint_name()) broker_hosts = map(lambda b: b.split(":")[0], brokers_list) brokers = ",".join(brokers_list) if not sdk_cmd.resolve_hosts(self.id, broker_hosts, bootstrap_cmd="/opt/bootstrap"): log.error("Failed to resolve brokers: %s", broker_hosts) return False self.brokers = brokers return True if topic_name: self.kafka_service.wait_for_topic(topic_name) return True
def test_authz_acls_not_required(kafka_client, service_account, setup_principals): client_id = kafka_client["id"] try: sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME) config.install( config.PACKAGE_NAME, config.SERVICE_NAME, config.DEFAULT_BROKER_COUNT, additional_options={ "brokers": { "port_tls": 1030 }, "service": { "service_account": service_account["name"], "service_account_secret": service_account["secret"], "security": { "transport_encryption": { "enabled": True }, "ssl_authentication": { "enabled": True }, "authorization": { "enabled": True, "super_users": "User:{}".format("super"), "allow_everyone_if_no_acl_found": True } } } }) sdk_cmd.resolve_hosts(client_id, kafka_client["brokers"]) # Create the topic sdk_cmd.svc_cli(config.PACKAGE_NAME, config.SERVICE_NAME, "topic create authz.test", json=True) test_utils.wait_for_topic(config.PACKAGE_NAME, config.SERVICE_NAME, "authz.test") super_message = str(uuid.uuid4()) authorized_message = str(uuid.uuid4()) unauthorized_message = str(uuid.uuid4()) log.info("Writing and reading: Writing to the topic, as authorized user") assert write_to_topic("authorized", client_id, "authz.test", authorized_message) log.info("Writing and reading: Writing to the topic, as unauthorized user") assert write_to_topic("unauthorized", client_id, "authz.test", unauthorized_message) log.info("Writing and reading: Writing to the topic, as super user") assert write_to_topic("super", client_id, "authz.test", super_message) log.info("Writing and reading: Reading from the topic, as authorized user") assert authorized_message in read_from_topic("authorized", client_id, "authz.test", 3) log.info("Writing and reading: Reading from the topic, as unauthorized user") assert unauthorized_message in read_from_topic("unauthorized", client_id, "authz.test", 3) log.info("Writing and reading: Reading from the topic, as super user") assert super_message in read_from_topic("super", client_id, "authz.test", 3) log.info("Writing and reading: Adding acl for authorized user") zookeeper_endpoint = str(sdk_cmd.svc_cli( config.PACKAGE_NAME, config.SERVICE_NAME, "endpoint zookeeper")).strip() topics.add_acls("authorized", client_id, "authz.test", zookeeper_endpoint, env_str=None) # Re-roll the messages so we really prove auth is in place. super_message = str(uuid.uuid4()) authorized_message = str(uuid.uuid4()) unauthorized_message = str(uuid.uuid4()) log.info("Writing and reading: Writing to the topic, as authorized user") assert write_to_topic("authorized", client_id, "authz.test", authorized_message) log.info("Writing and reading: Writing to the topic, as unauthorized user") assert not write_to_topic("unauthorized", client_id, "authz.test", unauthorized_message) log.info("Writing and reading: Writing to the topic, as super user") assert write_to_topic("super", client_id, "authz.test", super_message) log.info("Writing and reading: Reading from the topic, as authorized user") read_result = read_from_topic("authorized", client_id, "authz.test", 5) assert authorized_message in read_result and unauthorized_message not in read_result log.info("Writing and reading: Reading from the topic, as unauthorized user") assert auth.is_not_authorized(read_from_topic("unauthorized", client_id, "authz.test", 1)) log.info("Writing and reading: Reading from the topic, as super user") read_result = read_from_topic("super", client_id, "authz.test", 5) assert super_message in read_result and unauthorized_message not in read_result finally: sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)