def test_init_client_default(self): client = ambari_client.AmbariClient(self.instance) self.assertEqual(self.http_client, client._http_client) self.assertEqual("http://1.2.3.4:8080/api/v1", client._base_url) self.assertEqual("admin", client._auth.username) self.assertEqual("admin", client._auth.password) self.remote.get_http_client.assert_called_with("8080")
def test_stop_process_on_host(self): client = ambari_client.AmbariClient(self.instance) check_mock = mock.MagicMock() check_mock.status_code = 200 check_mock.text = '{"HostRoles": {"state": "SOME_STATE"}}' self.http_client.get.return_value = check_mock self.http_client.put.return_value = self.good_pending_resp client.wait_ambari_request = mock.MagicMock() instance = mock.MagicMock() instance.fqdn.return_value = "i1" client.stop_process_on_host("cluster_name", instance, "p1") self.http_client.put.assert_called_with( "http://1.2.3.4:8080/api/v1/clusters/" "cluster_name/hosts/i1/host_components/p1", data=jsonutils.dumps({ "HostRoles": { "state": "INSTALLED" }, "RequestInfo": { "context": "Stopping p1" } }), verify=False, auth=client._auth, headers=self.headers)
def test_delete_method(self): client = ambari_client.AmbariClient(self.instance) client.delete("http://spam") self.http_client.delete.assert_called_with("http://spam", verify=False, auth=client._auth, headers=self.headers)
def test_start_process_on_host(self): client = ambari_client.AmbariClient(self.instance) self.http_client.put.return_value = self.good_pending_resp client.wait_ambari_request = mock.MagicMock() instance = mock.MagicMock() instance.fqdn.return_value = "i1" instance.cluster.name = "cl" client.start_service_on_host(instance, "HDFS", 'STATE') self.http_client.put.assert_called_with( "http://1.2.3.4:8080/api/v1/clusters/" "cl/hosts/i1/host_components/HDFS", data=jsonutils.dumps({ "HostRoles": { "state": "STATE" }, "RequestInfo": { "context": "Starting service HDFS, " "moving to state STATE" } }), verify=False, auth=client._auth, headers=self.headers)
def start_cluster(cluster): cl_tmpl = { "blueprint": cluster.name, "default_password": uuidutils.generate_uuid(), "host_groups": [] } for ng in cluster.node_groups: for instance in ng.instances: cl_tmpl["host_groups"].append({ "name": instance.instance_name, "hosts": [{ "fqdn": instance.fqdn() }] }) ambari = plugin_utils.get_instance(cluster, p_common.AMBARI_SERVER) password = cluster.extra["ambari_password"] with ambari_client.AmbariClient(ambari, password=password) as client: req_id = client.create_cluster(cluster.name, cl_tmpl)["id"] while True: status = client.check_request_status(cluster.name, req_id) LOG.debug("Task %s in %s state. Completed %.1f%%" % (status["request_context"], status["request_status"], status["progress_percent"])) if status["request_status"] == "COMPLETED": return if status["request_status"] in ["IN_PROGRESS", "PENDING"]: context.sleep(5) else: raise p_exc.HadoopProvisionError( _("Ambari request in %s state") % status["request_status"])
def test_get_alerts_data(self, mock_check_response): cluster = mock.Mock() cluster.name = "test_cluster" client = ambari_client.AmbariClient(self.instance) # check_response returning empty json mock_check_response.return_value = {} res = client.get_alerts_data(cluster) self.assertEqual(res, []) self.http_client.get.assert_called_once_with( "http://1.2.3.4:8080/api/v1/clusters/test_cluster/alerts?fields=*", verify=False, auth=client._auth, headers=self.headers) mock_check_response.assert_called_once() # check_response returning json with items as key mock_check_response.return_value = {'items': ['item1', 'item2']} res = client.get_alerts_data(cluster) self.assertEqual(res, ['item1', 'item2']) self.http_client.get.assert_called_with( "http://1.2.3.4:8080/api/v1/clusters/test_cluster/alerts?fields=*", verify=False, auth=client._auth, headers=self.headers) self.assertEqual(self.http_client.get.call_count, 2) self.assertEqual(mock_check_response.call_count, 2)
def create_blueprint(cluster): _prepare_ranger(cluster) cluster = conductor.cluster_get(context.ctx(), cluster.id) host_groups = [] for ng in cluster.node_groups: procs = p_common.get_ambari_proc_list(ng) procs.extend(p_common.get_clients(cluster)) for instance in ng.instances: hg = { "name": instance.instance_name, "configurations": configs.get_instance_params(instance), "components": [] } for proc in procs: hg["components"].append({"name": proc}) host_groups.append(hg) bp = { "Blueprints": { "stack_name": "HDP", "stack_version": cluster.hadoop_version }, "host_groups": host_groups, "configurations": configs.get_cluster_params(cluster) } ambari = plugin_utils.get_instance(cluster, p_common.AMBARI_SERVER) password = cluster.extra["ambari_password"] with ambari_client.AmbariClient(ambari, password=password) as client: client.create_blueprint(cluster.name, bp)
def get_alerts_data(self, service=None): if self._data is not None: # return cached data return self._data.get(service, []) if service else self._data self._data = {} self._cluster_services = [] try: ambari = plugin_utils.get_instance(self.cluster, p_common.AMBARI_SERVER) password = self.cluster.extra.get("ambari_password") with client.AmbariClient(ambari, password=password) as ambari: resp = ambari.get_alerts_data(self.cluster) for alert in resp: alert = alert.get('Alert', {}) service = alert.get('service_name').lower() if service not in self._data: self._data[service] = [] self._cluster_services.append(service) self._data[service].append(alert) except Exception as e: prefix = _("Can't get response from Ambari Monitor") msg = _("%(problem)s: %(description)s") % { 'problem': prefix, 'description': six.text_type(e) } # don't put in exception to logs, it will be done by log.exception LOG.exception(prefix) self._exception_store = msg
def manage_config_groups(cluster, instances): groups = [] ambari = plugin_utils.get_instance(cluster, p_common.AMBARI_SERVER) password = cluster.extra["ambari_password"] for instance in instances: groups.extend(configs.get_config_group(instance)) with ambari_client.AmbariClient(ambari, password=password) as client: client.create_config_group(cluster, groups)
def test_put_method(self): client = ambari_client.AmbariClient(self.instance) client.put("http://spam", data="data") self.http_client.put.assert_called_with("http://spam", data="data", verify=False, auth=client._auth, headers=self.headers)
def start_cluster(cluster): ambari_template = _build_ambari_cluster_template(cluster) ambari = plugin_utils.get_instance(cluster, p_common.AMBARI_SERVER) password = cluster.extra["ambari_password"] with ambari_client.AmbariClient(ambari, password=password) as client: req_id = client.create_cluster(cluster.name, ambari_template)["id"] client.wait_ambari_request(req_id, cluster.name)
def test_init_client_manual(self): client = ambari_client.AmbariClient(self.instance, port="1234", username="******", password="******") self.assertEqual("http://1.2.3.4:1234/api/v1", client._base_url) self.assertEqual("user", client._auth.username) self.assertEqual("pass", client._auth.password) self.remote.get_http_client.assert_called_with("1234")
def update_default_ambari_password(cluster): ambari = plugin_utils.get_instance(cluster, p_common.AMBARI_SERVER) new_password = uuidutils.generate_uuid() with ambari_client.AmbariClient(ambari) as client: client.update_user_password("admin", "admin", new_password) extra = cluster.extra.to_dict() if cluster.extra else {} extra["ambari_password"] = new_password ctx = context.ctx() conductor.cluster_update(ctx, cluster, {"extra": extra}) cluster = conductor.cluster_get(ctx, cluster.id)
def _wait_all_processes_removed(cluster, instance): ambari = plugin_utils.get_instance(cluster, p_common.AMBARI_SERVER) password = cluster.extra["ambari_password"] with ambari_client.AmbariClient(ambari, password=password) as client: while True: hdp_processes = client.list_host_processes(cluster.name, instance) if not hdp_processes: return context.sleep(5)
def wait_host_registration(cluster): ambari = plugin_utils.get_instance(cluster, p_common.AMBARI_SERVER) hosts = plugin_utils.get_instances(cluster) password = cluster.extra["ambari_password"] with ambari_client.AmbariClient(ambari, password=password) as client: kwargs = {"client": client, "num_hosts": len(hosts)} poll_utils.poll(_check_host_registration, kwargs=kwargs, timeout=600) registered_hosts = client.get_registered_hosts() registered_host_names = [h["Hosts"]["host_name"] for h in registered_hosts] actual_host_names = [h.fqdn() for h in hosts] if sorted(registered_host_names) != sorted(actual_host_names): raise p_exc.HadoopProvisionError( _("Host registration fails in Ambari"))
def test_wait_ambari_request_error(self, mock_context): client = ambari_client.AmbariClient(self.instance) check_mock = mock.MagicMock() d1 = { "request_context": "r1", "request_status": "ERROR", "progress_percent": 146 } check_mock.return_value = d1 client.check_request_status = check_mock self.assertRaises(p_exc.HadoopProvisionError, client.wait_ambari_request, "id1", "c1")
def test_create_blueprint(self): client = ambari_client.AmbariClient(self.instance) resp = mock.Mock() resp.text = "" resp.status_code = 200 self.http_client.post.return_value = resp client.create_blueprint("cluster_name", {"some": "data"}) self.http_client.post.assert_called_with( "http://1.2.3.4:8080/api/v1/blueprints/cluster_name", data=jsonutils.dumps({"some": "data"}), verify=False, auth=client._auth, headers=self.headers)
def test_import_credential(self): resp = mock.Mock() resp.text = "" resp.status_code = 200 self.http_client.post.return_value = resp client = ambari_client.AmbariClient(self.instance) client.import_credential("test", alias="credential", data={"some": "data"}) self.http_client.post.assert_called_once_with( "http://1.2.3.4:8080/api/v1/clusters/test/credentials/credential", verify=False, data=jsonutils.dumps({"some": "data"}), auth=client._auth, headers=self.headers)
def set_up_hdp_repos(cluster): hdp_repo = configs.get_hdp_repo_url(cluster) hdp_utils_repo = configs.get_hdp_utils_repo_url(cluster) if not hdp_repo and not hdp_utils_repo: return ambari = plugin_utils.get_instance(cluster, p_common.AMBARI_SERVER) password = cluster.extra["ambari_password"] pv = cluster.hadoop_version repos = repo_id_map[pv] with ambari_client.AmbariClient(ambari, password=password) as client: os_type = os_type_map[client.get_host_info(ambari.fqdn())["os_type"]] if hdp_repo: client.set_up_mirror(pv, os_type, repos["HDP"], hdp_repo) if hdp_utils_repo: client.set_up_mirror(pv, os_type, repos["HDP-UTILS"], hdp_utils_repo)
def _remove_services_from_host(cluster, instance): ambari = plugin_utils.get_instance(cluster, p_common.AMBARI_SERVER) password = cluster.extra["ambari_password"] with ambari_client.AmbariClient(ambari, password=password) as client: hdp_processes = client.list_host_processes(cluster.name, instance) for proc in hdp_processes: LOG.debug("Stopping process %s on host %s " % (proc, instance.fqdn())) client.stop_process_on_host(cluster.name, instance, proc) LOG.debug("Removing process %s from host %s " % (proc, instance.fqdn())) client.remove_process_from_host(cluster.name, instance, proc) _wait_all_processes_removed(cluster, instance)
def test_add_host_to_cluster(self): client = ambari_client.AmbariClient(self.instance) resp = mock.Mock() resp.text = "" resp.status_code = 200 self.http_client.post.return_value = resp instance = mock.MagicMock() instance.fqdn.return_value = "i1" instance.cluster.name = "cl" client.add_host_to_cluster(instance) self.http_client.post.assert_called_with( "http://1.2.3.4:8080/api/v1/clusters/cl/hosts/i1", verify=False, auth=client._auth, headers=self.headers)
def test_update_user_password(self): client = ambari_client.AmbariClient(self.instance) resp = mock.Mock() resp.text = "" resp.status_code = 200 self.http_client.put.return_value = resp client.update_user_password("bart", "old_pw", "new_pw") exp_req = jsonutils.dumps( {"Users": { "old_password": "******", "password": "******" }}) self.http_client.put.assert_called_with( "http://1.2.3.4:8080/api/v1/users/bart", data=exp_req, verify=False, auth=client._auth, headers=self.headers)
def test_get_credential(self): resp = mock.Mock() resp.text = "" resp.status_code = 200 self.http_client.get.return_value = resp client = ambari_client.AmbariClient(self.instance) client.get_credential("test", alias="credential") self.http_client.get.assert_called_once_with( "http://1.2.3.4:8080/api/v1/clusters/test/credentials/credential", verify=False, auth=client._auth, headers=self.headers) resp.status_code = 404 self.assertRaises(ambari_client.AmbariNotFound, ambari_client.AmbariClient.check_response, resp, True)
def create_blueprint(cluster): _prepare_ranger(cluster) cluster = conductor.cluster_get(context.ctx(), cluster.id) host_groups = [] for ng in cluster.node_groups: procs = p_common.get_ambari_proc_list(ng) procs.extend(p_common.get_clients(cluster)) for instance in ng.instances: hg = { "name": instance.instance_name, "configurations": configs.get_instance_params(instance), "components": [] } for proc in procs: hg["components"].append({"name": proc}) host_groups.append(hg) bp = { "Blueprints": { "stack_name": "HDP", "stack_version": cluster.hadoop_version }, "host_groups": host_groups, "configurations": configs.get_cluster_params(cluster) } ambari = plugin_utils.get_instance(cluster, p_common.AMBARI_SERVER) password = cluster.extra["ambari_password"] general_configs = cluster.cluster_configs.get("general", {}) if (general_configs.get(p_common.NAMENODE_HA) or general_configs.get(p_common.RESOURCEMANAGER_HA) or general_configs.get(p_common.HBASE_REGIONSERVER_HA)): bp = ha_helper.update_bp_ha_common(cluster, bp) if general_configs.get(p_common.NAMENODE_HA): bp = ha_helper.update_bp_for_namenode_ha(cluster, bp) if general_configs.get(p_common.RESOURCEMANAGER_HA): bp = ha_helper.update_bp_for_resourcemanager_ha(cluster, bp) if general_configs.get(p_common.HBASE_REGIONSERVER_HA): bp = ha_helper.update_bp_for_hbase_ha(cluster, bp) with ambari_client.AmbariClient(ambari, password=password) as client: return client.create_blueprint(cluster.name, bp)
def test_create_cluster(self): client = ambari_client.AmbariClient(self.instance) resp = mock.Mock() resp.text = """{ "Requests": { "id": 1, "status": "InProgress" } }""" resp.status_code = 200 self.http_client.post.return_value = resp req_info = client.create_cluster("cluster_name", {"some": "data"}) self.assertEqual(1, req_info["id"]) self.http_client.post.assert_called_with( "http://1.2.3.4:8080/api/v1/clusters/cluster_name", data=jsonutils.dumps({"some": "data"}), verify=False, auth=client._auth, headers=self.headers)
def test_wait_ambari_request(self, mock_context): client = ambari_client.AmbariClient(self.instance) check_mock = mock.MagicMock() d1 = { "request_context": "r1", "request_status": "PENDING", "progress_percent": 42 } d2 = { "request_context": "r1", "request_status": "COMPLETED", "progress_percent": 100 } check_mock.side_effect = [d1, d2] client.check_request_status = check_mock client.wait_ambari_request("id1", "c1") check_mock.assert_has_calls( [mock.call("c1", "id1"), mock.call("c1", "id1")])
def test_get_registered_hosts(self): client = ambari_client.AmbariClient(self.instance) resp_data = """{ "href" : "http://1.2.3.4:8080/api/v1/hosts", "items" : [ { "href" : "http://1.2.3.4:8080/api/v1/hosts/host1", "Hosts" : { "host_name" : "host1" } }, { "href" : "http://1.2.3.4:8080/api/v1/hosts/host2", "Hosts" : { "host_name" : "host2" } }, { "href" : "http://1.2.3.4:8080/api/v1/hosts/host3", "Hosts" : { "host_name" : "host3" } } ] }""" resp = mock.Mock() resp.text = resp_data resp.status_code = 200 self.http_client.get.return_value = resp hosts = client.get_registered_hosts() self.http_client.get.assert_called_with( "http://1.2.3.4:8080/api/v1/hosts", verify=False, auth=client._auth, headers=self.headers) self.assertEqual(3, len(hosts)) self.assertEqual("host1", hosts[0]["Hosts"]["host_name"]) self.assertEqual("host2", hosts[1]["Hosts"]["host_name"]) self.assertEqual("host3", hosts[2]["Hosts"]["host_name"])
def manage_host_components(cluster, instances): ambari = plugin_utils.get_instance(cluster, p_common.AMBARI_SERVER) password = cluster.extra["ambari_password"] requests_ids = [] with ambari_client.AmbariClient(ambari, password=password) as client: clients = p_common.get_clients(cluster) for instance in instances: services = p_common.get_ambari_proc_list(instance.node_group) services.extend(clients) for service in services: client.add_service_to_host(instance, service) requests_ids.append( client.start_service_on_host(instance, service, 'INSTALLED')) client.wait_ambari_requests(requests_ids, cluster.name) # all services added and installed, let's start them requests_ids = [] for instance in instances: services = p_common.get_ambari_proc_list(instance.node_group) services.extend(p_common.ALL_LIST) for service in services: requests_ids.append( client.start_service_on_host(instance, service, 'STARTED')) client.wait_ambari_requests(requests_ids, cluster.name)
def _get_ambari_client(cluster): ambari = plugin_utils.get_instance(cluster, p_common.AMBARI_SERVER) password = cluster.extra["ambari_password"] return ambari_client.AmbariClient(ambari, password=password)
def _remove_host(cluster, inst): ambari = plugin_utils.get_instance(cluster, p_common.AMBARI_SERVER) password = cluster.extra["ambari_password"] with ambari_client.AmbariClient(ambari, password=password) as client: client.delete_host(cluster.name, inst)