Esempio n. 1
0
    def test__configure_yarn_site(self, mock_get_instances, mock_get_instance):
        mock_get_instances.return_value = [self.in1, self.in2, self.in3]
        mock_get_instance.return_value = self.in1
        bp = ha._configure_yarn_site(self.cluster, copy.deepcopy(self.bp))

        zks = ",".join(["%s:2181" % i.fqdn() for i in mock_get_instances()])
        rm_ids = ",".join([i.instance_name for i in mock_get_instances()])
        result = {
            "yarn-site": {
                "hadoop.registry.rm.enabled":
                "false",
                "yarn.resourcemanager.zk-address":
                zks,
                "yarn.log.server.url":
                "{}:19888/jobhistory/logs/".format(mock_get_instance().fqdn()),
                "yarn.resourcemanager.address":
                "{}:8050".format(mock_get_instances()[0].fqdn()),
                "yarn.resourcemanager.admin.address":
                "{}:8141".format(mock_get_instances()[0].fqdn()),
                "yarn.resourcemanager.cluster-id":
                self.cluster.name,
                "yarn.resourcemanager.ha.automatic-failover.zk-base-path":
                "/yarn-leader-election",
                "yarn.resourcemanager.ha.enabled":
                "true",
                "yarn.resourcemanager.ha.rm-ids":
                rm_ids,
                "yarn.resourcemanager.hostname":
                mock_get_instances()[0].fqdn(),
                "yarn.resourcemanager.recovery.enabled":
                "true",
                "yarn.resourcemanager.resource-tracker.address":
                "{}:8025".format(mock_get_instances()[0].fqdn()),
                "yarn.resourcemanager.scheduler.address":
                "{}:8030".format(mock_get_instances()[0].fqdn()),
                "yarn.resourcemanager.store.class":
                "org.apache.hadoop.yarn.server.resourcemanager.recovery."
                "ZKRMStateStore",
                "yarn.resourcemanager.webapp.address":
                "{}:8088".format(mock_get_instances()[0].fqdn()),
                "yarn.resourcemanager.webapp.https.address":
                "{}:8090".format(mock_get_instances()[0].fqdn()),
                "yarn.timeline-service.address":
                "{}:10200".format(mock_get_instance().fqdn()),
                "yarn.timeline-service.webapp.address":
                "{}:8188".format(mock_get_instance().fqdn()),
                "yarn.timeline-service.webapp.https.address":
                "{}:8190".format(mock_get_instance().fqdn())
            }
        }
        props = result["yarn-site"]
        for i in mock_get_instances():
            props["yarn.resourcemanager.hostname.{}".format(
                i.instance_name)] = i.fqdn()
            props["yarn.resourcemanager.webapp.address.{}".format(
                i.instance_name)] = "{}:8088".format(i.fqdn())
            props["yarn.resourcemanager.webapp.https.address.{}".format(
                i.instance_name)] = "{}:8090".format(i.fqdn())
        self.assertDictEqual(result["yarn-site"], ha._find_yarn_site(bp))
Esempio n. 2
0
    def test__configure_yarn_site(self, mock_get_instances, mock_get_instance):
        mock_get_instances.return_value = [self.in1, self.in2, self.in3]
        mock_get_instance.return_value = self.in1
        bp = ha._configure_yarn_site(self.cluster, copy.deepcopy(self.bp))

        zks = ",".join(["%s:2181" % i.fqdn() for i in mock_get_instances()])
        rm_ids = ",".join([i.instance_name for i in mock_get_instances()])
        result = {
            "yarn-site": {
                "hadoop.registry.rm.enabled": "false",
                "yarn.resourcemanager.zk-address": zks,
                "yarn.log.server.url": "{}:19888/jobhistory/logs/".format(
                    mock_get_instance().fqdn()),
                "yarn.resourcemanager.address": "{}:8050".format(
                    mock_get_instances()[0].fqdn()),
                "yarn.resourcemanager.admin.address": "{}:8141".format(
                    mock_get_instances()[0].fqdn()),
                "yarn.resourcemanager.cluster-id": self.cluster.name,
                "yarn.resourcemanager.ha.automatic-failover.zk-base-path":
                    "/yarn-leader-election",
                "yarn.resourcemanager.ha.enabled": "true",
                "yarn.resourcemanager.ha.rm-ids": rm_ids,
                "yarn.resourcemanager.hostname":
                    mock_get_instances()[0].fqdn(),
                "yarn.resourcemanager.recovery.enabled": "true",
                "yarn.resourcemanager.resource-tracker.address":
                    "{}:8025".format(mock_get_instances()[0].fqdn()),
                "yarn.resourcemanager.scheduler.address": "{}:8030".format(
                    mock_get_instances()[0].fqdn()),
                "yarn.resourcemanager.store.class":
                    "org.apache.hadoop.yarn.server.resourcemanager.recovery."
                    "ZKRMStateStore",
                "yarn.resourcemanager.webapp.address": "{}:8088".format(
                    mock_get_instances()[0].fqdn()),
                "yarn.resourcemanager.webapp.https.address": "{}:8090".format(
                    mock_get_instances()[0].fqdn()),
                "yarn.timeline-service.address": "{}:10200".format(
                    mock_get_instance().fqdn()),
                "yarn.timeline-service.webapp.address": "{}:8188".format(
                    mock_get_instance().fqdn()),
                "yarn.timeline-service.webapp.https.address": "{}:8190".format(
                    mock_get_instance().fqdn())
            }
        }
        props = result["yarn-site"]
        for i in mock_get_instances():
            props["yarn.resourcemanager.hostname.{}".format(
                i.instance_name)] = i.fqdn()
            props["yarn.resourcemanager.webapp.address.{}".format(
                i.instance_name)] = "{}:8088".format(i.fqdn())
            props["yarn.resourcemanager.webapp.https.address.{}".format(
                i.instance_name)] = "{}:8090".format(i.fqdn())
        self.assertEqual(result["yarn-site"], ha._find_yarn_site(bp))
Esempio n. 3
0
    def test__set_zk_quorum(self, mock_get_instances):
        mock_get_instances.return_value = [self.in1, self.in2, self.in3]
        bp = ha._set_zk_quorum(self.cluster, copy.deepcopy(self.bp), ha.CORE_SITE)
        self.assertEqual(
            "{}:2181,{}:2181,{}:2181".format(self.in1.fqdn(), self.in2.fqdn(), self.in3.fqdn()),
            ha._find_core_site(bp)["ha.zookeeper.quorum"],
        )

        bp = ha._set_zk_quorum(self.cluster, copy.deepcopy(self.bp), ha.YARN_SITE)
        self.assertEqual(
            "{}:2181,{}:2181,{}:2181".format(self.in1.fqdn(), self.in2.fqdn(), self.in3.fqdn()),
            ha._find_yarn_site(bp)["hadoop.registry.zk.quorum"],
        )
Esempio n. 4
0
    def test__set_zk_quorum(self, mock_get_instances):
        mock_get_instances.return_value = [self.in1, self.in2, self.in3]
        bp = ha._set_zk_quorum(self.cluster, copy.deepcopy(self.bp),
                               ha.CORE_SITE)
        self.assertEqual(
            "{}:2181,{}:2181,{}:2181".format(
                self.in1.fqdn(), self.in2.fqdn(), self.in3.fqdn()),
            ha._find_core_site(bp)['ha.zookeeper.quorum'])

        bp = ha._set_zk_quorum(self.cluster, copy.deepcopy(self.bp),
                               ha.YARN_SITE)
        self.assertEqual(
            "{}:2181,{}:2181,{}:2181".format(
                self.in1.fqdn(), self.in2.fqdn(), self.in3.fqdn()),
            ha._find_yarn_site(bp)['hadoop.registry.zk.quorum'])