def kafka_server(service_account, kafka_client: client.KafkaClient): service_options = { "service": { "name": config.SERVICE_NAME, "service_account": service_account["name"], "service_account_secret": service_account["secret"], "security": { "transport_encryption": { "enabled": True } }, } } sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME) try: sdk_install.install( config.PACKAGE_NAME, service_name=config.SERVICE_NAME, expected_running_tasks=config.DEFAULT_TASK_COUNT, additional_options=service_options, timeout_seconds=30 * 60, ) kafka_client.connect(config.DEFAULT_BROKER_COUNT) kafka_client.create_topic(config.DEFAULT_TOPIC_NAME) kafka_client.check_topic_partition_count( config.DEFAULT_TOPIC_NAME, config.DEFAULT_PARTITION_COUNT) yield {**service_options, **{"package_name": config.PACKAGE_NAME}} finally: sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
def test_custom_zookeeper(kafka_client: client.KafkaClient): broker_ids = sdk_tasks.get_task_ids(FOLDERED_NAME, "{}-".format(config.DEFAULT_POD_TYPE)) # create a topic against the default zk: kafka_client.create_topic(config.DEFAULT_TOPIC_NAME) marathon_config = sdk_marathon.get_config(FOLDERED_NAME) # should be using default path when this envvar is empty/unset: assert marathon_config["env"]["KAFKA_ZOOKEEPER_URI"] == "" # use a custom zk path that's WITHIN the 'dcos-service-' path, so that it's automatically cleaned up in uninstall: zk_path = "master.mesos:2181/{}/CUSTOMPATH".format( sdk_utils.get_zk_path(FOLDERED_NAME)) marathon_config["env"]["KAFKA_ZOOKEEPER_URI"] = zk_path sdk_marathon.update_app(marathon_config) sdk_tasks.check_tasks_updated(FOLDERED_NAME, "{}-".format(config.DEFAULT_POD_TYPE), broker_ids) sdk_plan.wait_for_completed_deployment(FOLDERED_NAME) # wait for brokers to finish registering kafka_client.check_broker_count(config.DEFAULT_BROKER_COUNT) zookeeper = sdk_networks.get_endpoint_string(config.PACKAGE_NAME, FOLDERED_NAME, "zookeeper") assert zookeeper == zk_path # topic created earlier against default zk should no longer be present: rc, stdout, _ = sdk_cmd.svc_cli(config.PACKAGE_NAME, FOLDERED_NAME, "topic list") assert rc == 0, "Topic list command failed" assert config.DEFAULT_TOPIC_NAME not in json.loads(stdout)
def test_authn_client_can_read_and_write(kafka_client: client.KafkaClient, service_account, setup_principals): try: sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME) service_options = { "service": { "name": config.SERVICE_NAME, "service_account": service_account["name"], "service_account_secret": service_account["secret"], "security": { "transport_encryption": { "enabled": True }, "ssl_authentication": { "enabled": True }, }, } } config.install( config.PACKAGE_NAME, config.SERVICE_NAME, config.DEFAULT_BROKER_COUNT, additional_options=service_options, ) topic_name = "tls.topic" kafka_client.connect(config.DEFAULT_BROKER_COUNT) kafka_client.create_topic(topic_name) kafka_client.check_users_can_read_and_write(["kafka-tester"], topic_name) finally: sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
def test_authz_acls_not_required(kafka_client: client.KafkaClient, kerberos, service_account, setup_principals): try: sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME) service_options = { "service": { "name": config.SERVICE_NAME, "service_account": service_account["name"], "service_account_secret": service_account["secret"], "security": { "kerberos": { "enabled": True, "kdc": { "hostname": kerberos.get_host(), "port": int(kerberos.get_port()) }, "realm": kerberos.get_realm(), "keytab_secret": kerberos.get_keytab_path(), }, "transport_encryption": { "enabled": True }, "authorization": { "enabled": True, "super_users": "User:{}".format("super"), "allow_everyone_if_no_acl_found": True, }, }, } } config.install( config.PACKAGE_NAME, config.SERVICE_NAME, config.DEFAULT_BROKER_COUNT, additional_options=service_options, ) topic_name = "authz.test" kafka_client.connect(config.DEFAULT_BROKER_COUNT) kafka_client.create_topic(topic_name) # Clear the ACLs kafka_client.remove_acls("authorized", topic_name) # Since no ACLs are specified, all users can read and write. kafka_client.check_users_can_read_and_write( ["authorized", "unauthorized", "super"], topic_name) log.info("Writing and reading: Adding acl for authorized user") kafka_client.add_acls("authorized", topic_name) # After adding ACLs the authorized user and super user should still have access to the topic. kafka_client.check_users_can_read_and_write(["authorized", "super"], topic_name) kafka_client.check_users_are_not_authorized_to_read_and_write( ["unauthorized"], topic_name) finally: sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
def test_authz_acls_required(kafka_client: client.KafkaClient, service_account, setup_principals): try: sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME) service_options = { "service": { "name": config.SERVICE_NAME, "service_account": service_account["name"], "service_account_secret": service_account["secret"], "security": { "transport_encryption": { "enabled": True }, "ssl_authentication": { "enabled": True }, "authorization": { "enabled": True, "super_users": "User:{}".format("super") }, }, } } config.install( config.PACKAGE_NAME, config.SERVICE_NAME, config.DEFAULT_BROKER_COUNT, additional_options=service_options, ) topic_name = "authz.test" kafka_client.connect(config.DEFAULT_BROKER_COUNT) kafka_client.create_topic(topic_name) # Since no ACLs are specified, only the super user can read and write kafka_client.check_users_can_read_and_write(["super"], topic_name) kafka_client.check_users_are_not_authorized_to_read_and_write( ["authorized", "unauthorized"], topic_name) log.info("Writing and reading: Adding acl for authorized user") kafka_client.add_acls("authorized", topic_name) # After adding ACLs the authorized user and super user should still have access to the topic. kafka_client.check_users_can_read_and_write(["authorized", "super"], topic_name) kafka_client.check_users_are_not_authorized_to_read_and_write( ["unauthorized"], topic_name) finally: sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
def test_authz_acls_not_required(kafka_client: client.KafkaClient): topic_name = "authz.test" kafka_client.create_topic(topic_name) # Since no ACLs are specified, all users can read and write. kafka_client.check_users_can_read_and_write( ["authorized", "unauthorized", "super"], topic_name) log.info("Writing and reading: Adding acl for authorized user") kafka_client.add_acls("authorized", topic_name) # After adding ACLs the authorized user and super user should still have access to the topic. kafka_client.check_users_can_read_and_write(["authorized", "super"], topic_name) kafka_client.check_users_are_not_authorized_to_read_and_write( ["unauthorized"], topic_name)
def test_client_can_read_and_write(kafka_client: client.KafkaClient): topic_name = "authn.test" kafka_client.create_topic(topic_name) kafka_client.check_users_can_read_and_write(["client"], topic_name)
def test_topic_offsets_increase_with_writes(kafka_client: client.KafkaClient): package_name = config.PACKAGE_NAME service_name = config.SERVICE_NAME def offset_is_valid(result) -> bool: initial = result[0] offsets = result[1] LOG.info("Checking validity with initial=%s offsets=%s", initial, offsets) has_elements = bool( topics.filter_empty_offsets(offsets, additional=initial)) # The return of this function triggers the restart. return not has_elements @retrying.retry( stop_max_delay=5 * 60 * 1000, wait_exponential_multiplier=1000, wait_exponential_max=60 * 1000, retry_on_result=offset_is_valid, ) def get_offset_change(topic_name, initial_offsets=[]): """ Run: `dcos kafa topic offsets --time="-1"` until the output is not the initial output specified """ LOG.info("Getting offsets for %s", topic_name) rc, stdout, _ = sdk_cmd.svc_cli( package_name, service_name, 'topic offsets --time="-1" {}'.format(topic_name)) assert rc == 0, "Topic offsets failed" offsets = json.loads(stdout) LOG.info("offsets=%s", offsets) return initial_offsets, offsets topic_name = str(uuid.uuid4()) LOG.info("Creating topic: %s", topic_name) kafka_client.create_topic(topic_name) _, offset_info = get_offset_change(topic_name) # offset_info is a list of (partition index, offset) key-value pairs sum the # integer representations of the offsets initial_offset = sum( map(lambda partition: sum(map(int, partition.values())), offset_info)) LOG.info("Initial offset=%s", initial_offset) num_messages = 10 LOG.info("Sending %s messages", num_messages) rc, stdout, _ = sdk_cmd.svc_cli( package_name, service_name, "topic producer_test {} {}".format(topic_name, num_messages)) assert rc == 0, "Producer test failed" write_info = json.loads(stdout) assert len(write_info) == 1 assert write_info["message"].startswith( "Output: {} records sent".format(num_messages)) _, post_write_offset_info = get_offset_change(topic_name, offset_info) post_write_offset = sum( map(lambda partition: sum(map(int, partition.values())), post_write_offset_info)) LOG.info("Post-write offset=%s", post_write_offset) assert post_write_offset > initial_offset
def test_initial_kerberos_off_tls_off_plaintext_off(kafka_client: client.KafkaClient): assert kafka_client.connect(config.DEFAULT_BROKER_COUNT) kafka_client.create_topic(TOPIC_NAME) kafka_client.check_users_can_read_and_write(["default"], TOPIC_NAME)