def test_endpoints():
# check that we can reach the scheduler via admin router, and that returned endpoints are sanitized:
core_site = etree.fromstring(
sdk_networks.get_endpoint_string(config.PACKAGE_NAME, foldered_name, "core-site.xml")
)
check_properties(
core_site,
{"ha.zookeeper.parent-znode": "/{}/hadoop-ha".format(sdk_utils.get_zk_path(foldered_name))},
)
hdfs_site = etree.fromstring(
sdk_networks.get_endpoint_string(config.PACKAGE_NAME, foldered_name, "hdfs-site.xml")
)
expect = {
"dfs.namenode.shared.edits.dir": "qjournal://{}/hdfs".format(
";".join(
[
sdk_hosts.autoip_host(foldered_name, "journal-{}-node".format(i), 8485)
for i in range(3)
]
)
)
}
for i in range(2):
name_node = "name-{}-node".format(i)
expect["dfs.namenode.rpc-address.hdfs.{}".format(name_node)] = sdk_hosts.autoip_host(
foldered_name, name_node, 9001
)
expect["dfs.namenode.http-address.hdfs.{}".format(name_node)] = sdk_hosts.autoip_host(
foldered_name, name_node, 9002
)
check_properties(hdfs_site, expect)
def test_custom_zookeeper(kafka_client: client.KafkaClient):
broker_ids = sdk_tasks.get_task_ids(FOLDERED_NAME,
"{}-".format(config.DEFAULT_POD_TYPE))
# create a topic against the default zk:
kafka_client.create_topic(config.DEFAULT_TOPIC_NAME)
marathon_config = sdk_marathon.get_config(FOLDERED_NAME)
# should be using default path when this envvar is empty/unset:
assert marathon_config["env"]["KAFKA_ZOOKEEPER_URI"] == ""
# use a custom zk path that's WITHIN the 'dcos-service-' path, so that it's automatically cleaned up in uninstall:
zk_path = "master.mesos:2181/{}/CUSTOMPATH".format(
sdk_utils.get_zk_path(FOLDERED_NAME))
marathon_config["env"]["KAFKA_ZOOKEEPER_URI"] = zk_path
sdk_marathon.update_app(marathon_config)
sdk_tasks.check_tasks_updated(FOLDERED_NAME,
"{}-".format(config.DEFAULT_POD_TYPE),
broker_ids)
sdk_plan.wait_for_completed_deployment(FOLDERED_NAME)
# wait for brokers to finish registering
kafka_client.check_broker_count(config.DEFAULT_BROKER_COUNT)
zookeeper = sdk_networks.get_endpoint_string(config.PACKAGE_NAME,
FOLDERED_NAME, "zookeeper")
assert zookeeper == zk_path
# topic created earlier against default zk should no longer be present:
rc, stdout, _ = sdk_cmd.svc_cli(config.PACKAGE_NAME, FOLDERED_NAME,
"topic list")
assert rc == 0, "Topic list command failed"
assert config.DEFAULT_TOPIC_NAME not in json.loads(stdout)
def test_overlay_network_deployment_and_endpoints():
endpoint_names = sdk_networks.get_endpoint_names(config.PACKAGE_NAME, config.SERVICE_NAME)
assert set(["broker", "zookeeper"]) == set(endpoint_names)
sdk_networks.check_endpoint_on_overlay(config.PACKAGE_NAME, config.SERVICE_NAME, "broker", config.DEFAULT_BROKER_COUNT)
zookeeper = sdk_networks.get_endpoint_string(
config.PACKAGE_NAME, config.SERVICE_NAME, "zookeeper"
)
assert zookeeper == "master.mesos:2181/{}".format(sdk_utils.get_zk_path(config.SERVICE_NAME))
def test_custom_zookeeper():
foldered_name = sdk_utils.get_foldered_name(config.SERVICE_NAME)
broker_ids = sdk_tasks.get_task_ids(foldered_name,
"{}-".format(config.DEFAULT_POD_TYPE))
# create a topic against the default zk:
test_utils.create_topic(config.DEFAULT_TOPIC_NAME,
service_name=foldered_name)
marathon_config = sdk_marathon.get_config(foldered_name)
# should be using default path when this envvar is empty/unset:
assert marathon_config["env"]["KAFKA_ZOOKEEPER_URI"] == ""
# use a custom zk path that's WITHIN the 'dcos-service-' path, so that it's automatically cleaned up in uninstall:
zk_path = "master.mesos:2181/{}/CUSTOMPATH".format(
sdk_utils.get_zk_path(foldered_name))
marathon_config["env"]["KAFKA_ZOOKEEPER_URI"] = zk_path
sdk_marathon.update_app(marathon_config)
sdk_tasks.check_tasks_updated(foldered_name,
"{}-".format(config.DEFAULT_POD_TYPE),
broker_ids)
sdk_plan.wait_for_completed_deployment(foldered_name)
# wait for brokers to finish registering
test_utils.broker_count_check(config.DEFAULT_BROKER_COUNT,
service_name=foldered_name)
zookeeper = sdk_networks.get_endpoint_string(config.PACKAGE_NAME,
foldered_name, "zookeeper")
assert zookeeper == zk_path
# topic created earlier against default zk should no longer be present:
rc, stdout, _ = sdk_cmd.svc_cli(config.PACKAGE_NAME, foldered_name,
"topic list")
assert rc == 0, "Topic list command failed"
test_utils.assert_topic_lists_are_equal_without_automatic_topics(
[], json.loads(stdout))
def test_endpoints_zookeeper_default():
zookeeper = sdk_networks.get_endpoint_string(config.PACKAGE_NAME,
FOLDERED_NAME, "zookeeper")
assert zookeeper == "master.mesos:2181/{}".format(
sdk_utils.get_zk_path(FOLDERED_NAME))
def test_endpoints_zookeeper_default():
foldered_name = sdk_utils.get_foldered_name(config.SERVICE_NAME)
zookeeper = sdk_networks.get_endpoint_string(config.PACKAGE_NAME,
foldered_name, "zookeeper")
assert zookeeper == "master.mesos:2181/{}".format(
sdk_utils.get_zk_path(foldered_name))
def get_zookeeper_endpoint(self) -> str:
return sdk_networks.get_endpoint_string(self._package_name,
self._service_name,
"zookeeper")
def test_endpoints_on_overlay():
endpoint_names = sdk_networks.get_endpoint_names(config.PACKAGE_NAME, config.SERVICE_NAME)
assert set(endpoint_names) == set(["hdfs-site.xml", "core-site.xml"])
for endpoint_name in endpoint_names:
# Validate that XML is parseable:
ElementTree.fromstring(sdk_networks.get_endpoint_string(config.PACKAGE_NAME, config.SERVICE_NAME, endpoint_name))
