def test_custom_domain():
task_count = 3
custom_domain = sdk_hosts.get_crypto_id_domain()
sdk_install.install(config.PACKAGE_NAME,
config.SERVICE_NAME,
task_count,
additional_options={
"service": {
"security": {
"custom_domain": custom_domain
}
}
})
# Verify the endpoint entry is correct
assert set(["native-client"]) == set(
sdk_networks.get_endpoint_names(config.PACKAGE_NAME,
config.SERVICE_NAME))
test_endpoint = sdk_networks.get_endpoint(config.PACKAGE_NAME,
config.SERVICE_NAME,
"native-client")
assert set(["address", "dns"]) == set(test_endpoint.keys())
assert len(test_endpoint["address"]) == task_count
# Expect ip:port:
for entry in test_endpoint["address"]:
assert len(entry.split(":")) == 2
assert len(test_endpoint["dns"]) == task_count
# Expect custom domain:
for entry in test_endpoint["dns"]:
assert custom_domain in entry
def test_custom_service_tld():
task_count = 1
custom_tld = sdk_hosts.get_crypto_id_domain()
sdk_install.install(
config.PACKAGE_NAME,
config.SERVICE_NAME,
task_count,
additional_options={
"service": {
"custom_service_tld": custom_tld,
"yaml": "custom_tld"
}
},
)
# Verify the endpoint entry is correct
assert set(["test"]) == set(
sdk_networks.get_endpoint_names(config.PACKAGE_NAME,
config.SERVICE_NAME))
test_endpoint = sdk_networks.get_endpoint(config.PACKAGE_NAME,
config.SERVICE_NAME, "test")
assert set(["address", "dns"]) == set(test_endpoint.keys())
assert len(test_endpoint["address"]) == task_count
# Expect ip:port:
for entry in test_endpoint["address"]:
assert len(entry.split(":")) == 2
assert len(test_endpoint["dns"]) == task_count
# Expect custom tld:
for entry in test_endpoint["dns"]:
assert custom_tld in entry
def test_endpoints():
endpoint_names = sdk_networks.get_endpoint_names(config.PACKAGE_NAME,
config.SERVICE_NAME)
assert set(endpoint_names) == set(["native-client"])
sdk_networks.check_endpoint_on_overlay(config.PACKAGE_NAME,
config.SERVICE_NAME,
"native-client",
config.DEFAULT_TASK_COUNT)
def test_overlay_network_deployment_and_endpoints():
endpoint_names = sdk_networks.get_endpoint_names(config.PACKAGE_NAME, config.SERVICE_NAME)
assert set(["broker", "zookeeper"]) == set(endpoint_names)
sdk_networks.check_endpoint_on_overlay(config.PACKAGE_NAME, config.SERVICE_NAME, "broker", config.DEFAULT_BROKER_COUNT)
zookeeper = sdk_networks.get_endpoint_string(
config.PACKAGE_NAME, config.SERVICE_NAME, "zookeeper"
)
assert zookeeper == "master.mesos:2181/{}".format(sdk_utils.get_zk_path(config.SERVICE_NAME))
def test_tls_endpoints():
endpoint_names = sdk_networks.get_endpoint_names(config.PACKAGE_NAME,
config.SERVICE_NAME)
assert len(endpoint_names) == 2
assert BROKER_TLS_ENDPOINT in endpoint_names
# Test that broker-tls endpoint is available
endpoint_tls = sdk_networks.get_endpoint(config.PACKAGE_NAME,
config.SERVICE_NAME,
BROKER_TLS_ENDPOINT)
assert len(endpoint_tls["dns"]) == config.DEFAULT_BROKER_COUNT
def test_tls_endpoints(kafka_service):
endpoints = sdk_networks.get_endpoint_names(config.PACKAGE_NAME, config.SERVICE_NAME)
assert BROKER_TLS_ENDPOINT in endpoints
# Test that broker-tls endpoint is available
_, endpoint_tls, _ = sdk_cmd.svc_cli(
config.PACKAGE_NAME,
config.SERVICE_NAME,
"endpoints {name}".format(name=BROKER_TLS_ENDPOINT),
parse_json=True,
)
assert len(endpoint_tls["dns"]) == config.DEFAULT_BROKER_COUNT
def service_has_brokers(kafka_server: dict,
endpoint_name: str,
number_of_brokers: int = None) -> bool:
endpoint_names = sdk_networks.get_endpoint_names(
kafka_server["package_name"], kafka_server["service"]["name"])
if endpoint_name not in endpoint_names:
log.error("Expecting endpoint %s. Found %s", endpoint_name,
endpoint_names)
return False
brokers = service_get_brokers(kafka_server, endpoint_name)
return number_of_brokers == len(brokers)
def test_endpoints_on_overlay():
endpoints_on_overlay_to_count = {
"coordinator-http": 1,
"coordinator-transport": 1,
"data-http": 2,
"data-transport": 2,
"master-http": 3,
"master-transport": 3,
}
endpoint_names = sdk_networks.get_endpoint_names(config.PACKAGE_NAME, config.SERVICE_NAME)
assert set(endpoints_on_overlay_to_count.keys()) == set(endpoint_names)
for endpoint_name, expected_count in endpoints_on_overlay_to_count.items():
sdk_networks.check_endpoint_on_overlay(config.PACKAGE_NAME, config.SERVICE_NAME, endpoint_name, expected_count)
def test_endpoints_on_overlay() -> None:
endpoints_on_overlay_to_count = {
"coordinator-http": 1,
"coordinator-transport": 1,
"data-http": 2,
"data-transport": 2,
"master-http": 3,
"master-transport": 3,
}
endpoint_names = sdk_networks.get_endpoint_names(config.PACKAGE_NAME, config.SERVICE_NAME)
assert set(endpoints_on_overlay_to_count.keys()) == set(endpoint_names)
for endpoint_name, expected_count in endpoints_on_overlay_to_count.items():
sdk_networks.check_endpoint_on_overlay(config.PACKAGE_NAME, config.SERVICE_NAME, endpoint_name, expected_count)
def test_overlay_network_deployment_and_endpoints():
# double check
sdk_tasks.check_running(config.SERVICE_NAME, config.DEFAULT_BROKER_COUNT)
endpoints = sdk_networks.get_endpoint_names(config.PACKAGE_NAME,
config.SERVICE_NAME)
assert "broker" in endpoints, "broker is missing from endpoints {}".format(
endpoints)
assert "zookeeper" in endpoints, "zookeeper missing from endpoints {}".format(
endpoints)
broker_endpoints = sdk_networks.get_endpoint(config.PACKAGE_NAME,
config.SERVICE_NAME, "broker")
kafka_networks.check_endpoints_on_overlay(broker_endpoints)
_, zookeeper, _ = sdk_cmd.svc_cli(config.PACKAGE_NAME, config.SERVICE_NAME,
"endpoints zookeeper")
assert zookeeper.rstrip() == "master.mesos:2181/{}".format(
sdk_utils.get_zk_path(config.SERVICE_NAME))
def test_custom_service_tld():
task_count = 1
custom_tld = sdk_hosts.get_crypto_id_domain()
sdk_install.install(
config.PACKAGE_NAME,
config.SERVICE_NAME,
task_count,
additional_options={"service": {"custom_service_tld": custom_tld, "yaml": "custom_tld"}},
)
# Verify the endpoint entry is correct
assert set(["test"]) == set(sdk_networks.get_endpoint_names(config.PACKAGE_NAME, config.SERVICE_NAME))
test_endpoint = sdk_networks.get_endpoint(config.PACKAGE_NAME, config.SERVICE_NAME, "test")
assert set(["address", "dns"]) == set(test_endpoint.keys())
assert len(test_endpoint["address"]) == task_count
# Expect ip:port:
for entry in test_endpoint["address"]:
assert len(entry.split(":")) == 2
assert len(test_endpoint["dns"]) == task_count
# Expect custom tld:
for entry in test_endpoint["dns"]:
assert custom_tld in entry
def test_overlay_network():
"""Verify that the current deploy plan matches the expected plan from the spec."""
deployment_plan = sdk_plan.wait_for_completed_deployment(
config.SERVICE_NAME)
log.info(sdk_plan.plan_string("deploy", deployment_plan))
# test that the tasks are all up, which tests the overlay DNS
framework_tasks = sdk_tasks.get_service_tasks(config.SERVICE_NAME)
expected_running_tasks = [
"overlay-vip-0-server",
"overlay-0-server",
"host-vip-0-server",
"host-0-server",
]
assert set(expected_running_tasks) == set(
[t.name for t in framework_tasks])
for task in framework_tasks:
name = task.name
if name.startswith("host-"):
assert "ports" in task.resources.keys(
), "Task {} should have port resources".format(name)
sdk_networks.check_task_network(name, expected_network_name=None)
elif name.startswith("overlay-"):
assert ("ports" not in task.resources.keys()
), "Task {} should NOT have port resources".format(name)
sdk_networks.check_task_network(name)
else:
assert False, "Unknown task {}".format(name)
endpoints_result = sdk_networks.get_endpoint_names(config.PACKAGE_NAME,
config.SERVICE_NAME)
assert len(endpoints_result) == 2, "Expected 2 endpoints, got: {}".format(
endpoints_result)
overlay_endpoints_result = sdk_networks.get_endpoint(
config.PACKAGE_NAME, config.SERVICE_NAME, "overlay-vip")
assert ("address" in overlay_endpoints_result.keys()
), "overlay endpoints missing 'address': {}".format(
overlay_endpoints_result)
assert len(overlay_endpoints_result["address"]) == 1
assert overlay_endpoints_result["address"][0].startswith("9")
overlay_port = overlay_endpoints_result["address"][0].split(":")[-1]
assert overlay_port == "4044"
assert "dns" in overlay_endpoints_result.keys()
assert len(overlay_endpoints_result["dns"]) == 1
assert overlay_endpoints_result["dns"][0] == sdk_hosts.autoip_host(
config.SERVICE_NAME, "overlay-vip-0-server", 4044)
host_endpoints_result = sdk_networks.get_endpoint(config.PACKAGE_NAME,
config.SERVICE_NAME,
"host-vip")
assert (
"address" in host_endpoints_result.keys()
), "overlay endpoints missing 'address'" "{}".format(host_endpoints_result)
assert len(host_endpoints_result["address"]) == 1
assert host_endpoints_result["address"][0].startswith("10")
host_port = host_endpoints_result["address"][0].split(":")[-1]
assert host_port == "4044"
assert "dns" in host_endpoints_result.keys()
assert len(host_endpoints_result["dns"]) == 1
assert host_endpoints_result["dns"][0] == sdk_hosts.autoip_host(
config.SERVICE_NAME, "host-vip-0-server", 4044)
def test_endpoints_on_overlay():
endpoint_names = sdk_networks.get_endpoint_names(config.PACKAGE_NAME, config.SERVICE_NAME)
assert set(endpoint_names) == set(["hdfs-site.xml", "core-site.xml"])
for endpoint_name in endpoint_names:
# Validate that XML is parseable:
ElementTree.fromstring(sdk_networks.get_endpoint_string(config.PACKAGE_NAME, config.SERVICE_NAME, endpoint_name))
def test_overlay_network():
"""Verify that the current deploy plan matches the expected plan from the spec."""
deployment_plan = sdk_plan.wait_for_completed_deployment(config.SERVICE_NAME)
log.info(sdk_plan.plan_string("deploy", deployment_plan))
# test that the tasks are all up, which tests the overlay DNS
framework_tasks = sdk_tasks.get_service_tasks(config.SERVICE_NAME)
expected_running_tasks = [
"overlay-vip-0-server",
"overlay-0-server",
"host-vip-0-server",
"host-0-server"
]
assert set(expected_running_tasks) == set([t.name for t in framework_tasks])
for task in framework_tasks:
name = task.name
if name.startswith("host-"):
assert "ports" in task.resources.keys(), "Task {} should have port resources".format(
name
)
sdk_networks.check_task_network(name, expected_network_name=None)
elif name.startswith("overlay-"):
assert (
"ports" not in task.resources.keys()
), "Task {} should NOT have port resources".format(
name
)
sdk_networks.check_task_network(name)
else:
assert False, "Unknown task {}".format(name)
endpoints_result = sdk_networks.get_endpoint_names(config.PACKAGE_NAME, config.SERVICE_NAME)
assert len(endpoints_result) == 2, "Expected 2 endpoints, got: {}".format(endpoints_result)
overlay_endpoints_result = sdk_networks.get_endpoint(
config.PACKAGE_NAME, config.SERVICE_NAME, "overlay-vip"
)
assert "address" in overlay_endpoints_result.keys(), (
"overlay endpoints missing 'address': {}".format(overlay_endpoints_result)
)
assert len(overlay_endpoints_result["address"]) == 1
assert overlay_endpoints_result["address"][0].startswith("9")
overlay_port = overlay_endpoints_result["address"][0].split(":")[-1]
assert overlay_port == "4044"
assert "dns" in overlay_endpoints_result.keys()
assert len(overlay_endpoints_result["dns"]) == 1
assert overlay_endpoints_result["dns"][0] == sdk_hosts.autoip_host(
config.SERVICE_NAME, "overlay-vip-0-server", 4044
)
host_endpoints_result = sdk_networks.get_endpoint(
config.PACKAGE_NAME, config.SERVICE_NAME, "host-vip"
)
assert "address" in host_endpoints_result.keys(), (
"overlay endpoints missing 'address'" "{}".format(host_endpoints_result)
)
assert len(host_endpoints_result["address"]) == 1
assert host_endpoints_result["address"][0].startswith("10")
host_port = host_endpoints_result["address"][0].split(":")[-1]
assert host_port == "4044"
assert "dns" in host_endpoints_result.keys()
assert len(host_endpoints_result["dns"]) == 1
assert host_endpoints_result["dns"][0] == sdk_hosts.autoip_host(
config.SERVICE_NAME, "host-vip-0-server", 4044
)
def test_endpoints() -> None:
endpoint_names = sdk_networks.get_endpoint_names(config.PACKAGE_NAME, config.SERVICE_NAME)
assert set(endpoint_names) == set(["native-client"])
sdk_networks.check_endpoint_on_overlay(config.PACKAGE_NAME, config.SERVICE_NAME, "native-client", config.DEFAULT_TASK_COUNT)