def test_endpoints(): # check that we can reach the scheduler via admin router, and that returned endpoints are sanitized: core_site = etree.fromstring( sdk_networks.get_endpoint_string(config.PACKAGE_NAME, foldered_name, "core-site.xml") ) check_properties( core_site, {"ha.zookeeper.parent-znode": "/{}/hadoop-ha".format(sdk_utils.get_zk_path(foldered_name))}, ) hdfs_site = etree.fromstring( sdk_networks.get_endpoint_string(config.PACKAGE_NAME, foldered_name, "hdfs-site.xml") ) expect = { "dfs.namenode.shared.edits.dir": "qjournal://{}/hdfs".format( ";".join( [ sdk_hosts.autoip_host(foldered_name, "journal-{}-node".format(i), 8485) for i in range(3) ] ) ) } for i in range(2): name_node = "name-{}-node".format(i) expect["dfs.namenode.rpc-address.hdfs.{}".format(name_node)] = sdk_hosts.autoip_host( foldered_name, name_node, 9001 ) expect["dfs.namenode.http-address.hdfs.{}".format(name_node)] = sdk_hosts.autoip_host( foldered_name, name_node, 9002 ) check_properties(hdfs_site, expect)
def test_custom_zookeeper(kafka_client: client.KafkaClient): broker_ids = sdk_tasks.get_task_ids(FOLDERED_NAME, "{}-".format(config.DEFAULT_POD_TYPE)) # create a topic against the default zk: kafka_client.create_topic(config.DEFAULT_TOPIC_NAME) marathon_config = sdk_marathon.get_config(FOLDERED_NAME) # should be using default path when this envvar is empty/unset: assert marathon_config["env"]["KAFKA_ZOOKEEPER_URI"] == "" # use a custom zk path that's WITHIN the 'dcos-service-' path, so that it's automatically cleaned up in uninstall: zk_path = "master.mesos:2181/{}/CUSTOMPATH".format( sdk_utils.get_zk_path(FOLDERED_NAME)) marathon_config["env"]["KAFKA_ZOOKEEPER_URI"] = zk_path sdk_marathon.update_app(marathon_config) sdk_tasks.check_tasks_updated(FOLDERED_NAME, "{}-".format(config.DEFAULT_POD_TYPE), broker_ids) sdk_plan.wait_for_completed_deployment(FOLDERED_NAME) # wait for brokers to finish registering kafka_client.check_broker_count(config.DEFAULT_BROKER_COUNT) zookeeper = sdk_networks.get_endpoint_string(config.PACKAGE_NAME, FOLDERED_NAME, "zookeeper") assert zookeeper == zk_path # topic created earlier against default zk should no longer be present: rc, stdout, _ = sdk_cmd.svc_cli(config.PACKAGE_NAME, FOLDERED_NAME, "topic list") assert rc == 0, "Topic list command failed" assert config.DEFAULT_TOPIC_NAME not in json.loads(stdout)
def test_overlay_network_deployment_and_endpoints(): endpoint_names = sdk_networks.get_endpoint_names(config.PACKAGE_NAME, config.SERVICE_NAME) assert set(["broker", "zookeeper"]) == set(endpoint_names) sdk_networks.check_endpoint_on_overlay(config.PACKAGE_NAME, config.SERVICE_NAME, "broker", config.DEFAULT_BROKER_COUNT) zookeeper = sdk_networks.get_endpoint_string( config.PACKAGE_NAME, config.SERVICE_NAME, "zookeeper" ) assert zookeeper == "master.mesos:2181/{}".format(sdk_utils.get_zk_path(config.SERVICE_NAME))
def test_custom_zookeeper(): foldered_name = sdk_utils.get_foldered_name(config.SERVICE_NAME) broker_ids = sdk_tasks.get_task_ids(foldered_name, "{}-".format(config.DEFAULT_POD_TYPE)) # create a topic against the default zk: test_utils.create_topic(config.DEFAULT_TOPIC_NAME, service_name=foldered_name) marathon_config = sdk_marathon.get_config(foldered_name) # should be using default path when this envvar is empty/unset: assert marathon_config["env"]["KAFKA_ZOOKEEPER_URI"] == "" # use a custom zk path that's WITHIN the 'dcos-service-' path, so that it's automatically cleaned up in uninstall: zk_path = "master.mesos:2181/{}/CUSTOMPATH".format( sdk_utils.get_zk_path(foldered_name)) marathon_config["env"]["KAFKA_ZOOKEEPER_URI"] = zk_path sdk_marathon.update_app(marathon_config) sdk_tasks.check_tasks_updated(foldered_name, "{}-".format(config.DEFAULT_POD_TYPE), broker_ids) sdk_plan.wait_for_completed_deployment(foldered_name) # wait for brokers to finish registering test_utils.broker_count_check(config.DEFAULT_BROKER_COUNT, service_name=foldered_name) zookeeper = sdk_networks.get_endpoint_string(config.PACKAGE_NAME, foldered_name, "zookeeper") assert zookeeper == zk_path # topic created earlier against default zk should no longer be present: rc, stdout, _ = sdk_cmd.svc_cli(config.PACKAGE_NAME, foldered_name, "topic list") assert rc == 0, "Topic list command failed" test_utils.assert_topic_lists_are_equal_without_automatic_topics( [], json.loads(stdout))
def test_endpoints_zookeeper_default(): zookeeper = sdk_networks.get_endpoint_string(config.PACKAGE_NAME, FOLDERED_NAME, "zookeeper") assert zookeeper == "master.mesos:2181/{}".format( sdk_utils.get_zk_path(FOLDERED_NAME))
def test_endpoints_zookeeper_default(): foldered_name = sdk_utils.get_foldered_name(config.SERVICE_NAME) zookeeper = sdk_networks.get_endpoint_string(config.PACKAGE_NAME, foldered_name, "zookeeper") assert zookeeper == "master.mesos:2181/{}".format( sdk_utils.get_zk_path(foldered_name))
def get_zookeeper_endpoint(self) -> str: return sdk_networks.get_endpoint_string(self._package_name, self._service_name, "zookeeper")
def test_endpoints_on_overlay(): endpoint_names = sdk_networks.get_endpoint_names(config.PACKAGE_NAME, config.SERVICE_NAME) assert set(endpoint_names) == set(["hdfs-site.xml", "core-site.xml"]) for endpoint_name in endpoint_names: # Validate that XML is parseable: ElementTree.fromstring(sdk_networks.get_endpoint_string(config.PACKAGE_NAME, config.SERVICE_NAME, endpoint_name))