def test_init_client_default(self):
client = ambari_client.AmbariClient(self.instance)
self.assertEqual(self.http_client, client._http_client)
self.assertEqual("http://1.2.3.4:8080/api/v1", client._base_url)
self.assertEqual("admin", client._auth.username)
self.assertEqual("admin", client._auth.password)
self.remote.get_http_client.assert_called_with("8080")
def test_stop_process_on_host(self):
client = ambari_client.AmbariClient(self.instance)
check_mock = mock.MagicMock()
check_mock.status_code = 200
check_mock.text = '{"HostRoles": {"state": "SOME_STATE"}}'
self.http_client.get.return_value = check_mock
self.http_client.put.return_value = self.good_pending_resp
client.wait_ambari_request = mock.MagicMock()
instance = mock.MagicMock()
instance.fqdn.return_value = "i1"
client.stop_process_on_host("cluster_name", instance, "p1")
self.http_client.put.assert_called_with(
"http://1.2.3.4:8080/api/v1/clusters/"
"cluster_name/hosts/i1/host_components/p1",
data=jsonutils.dumps({
"HostRoles": {
"state": "INSTALLED"
},
"RequestInfo": {
"context": "Stopping p1"
}
}),
verify=False,
auth=client._auth,
headers=self.headers)
def test_delete_method(self):
client = ambari_client.AmbariClient(self.instance)
client.delete("http://spam")
self.http_client.delete.assert_called_with("http://spam",
verify=False,
auth=client._auth,
headers=self.headers)
def test_start_process_on_host(self):
client = ambari_client.AmbariClient(self.instance)
self.http_client.put.return_value = self.good_pending_resp
client.wait_ambari_request = mock.MagicMock()
instance = mock.MagicMock()
instance.fqdn.return_value = "i1"
instance.cluster.name = "cl"
client.start_service_on_host(instance, "HDFS", 'STATE')
self.http_client.put.assert_called_with(
"http://1.2.3.4:8080/api/v1/clusters/"
"cl/hosts/i1/host_components/HDFS",
data=jsonutils.dumps({
"HostRoles": {
"state": "STATE"
},
"RequestInfo": {
"context": "Starting service HDFS, "
"moving to state STATE"
}
}),
verify=False,
auth=client._auth,
headers=self.headers)
def start_cluster(cluster):
cl_tmpl = {
"blueprint": cluster.name,
"default_password": uuidutils.generate_uuid(),
"host_groups": []
}
for ng in cluster.node_groups:
for instance in ng.instances:
cl_tmpl["host_groups"].append({
"name": instance.instance_name,
"hosts": [{
"fqdn": instance.fqdn()
}]
})
ambari = plugin_utils.get_instance(cluster, p_common.AMBARI_SERVER)
password = cluster.extra["ambari_password"]
with ambari_client.AmbariClient(ambari, password=password) as client:
req_id = client.create_cluster(cluster.name, cl_tmpl)["id"]
while True:
status = client.check_request_status(cluster.name, req_id)
LOG.debug("Task %s in %s state. Completed %.1f%%" %
(status["request_context"], status["request_status"],
status["progress_percent"]))
if status["request_status"] == "COMPLETED":
return
if status["request_status"] in ["IN_PROGRESS", "PENDING"]:
context.sleep(5)
else:
raise p_exc.HadoopProvisionError(
_("Ambari request in %s state") % status["request_status"])
def test_get_alerts_data(self, mock_check_response):
cluster = mock.Mock()
cluster.name = "test_cluster"
client = ambari_client.AmbariClient(self.instance)
# check_response returning empty json
mock_check_response.return_value = {}
res = client.get_alerts_data(cluster)
self.assertEqual(res, [])
self.http_client.get.assert_called_once_with(
"http://1.2.3.4:8080/api/v1/clusters/test_cluster/alerts?fields=*",
verify=False,
auth=client._auth,
headers=self.headers)
mock_check_response.assert_called_once()
# check_response returning json with items as key
mock_check_response.return_value = {'items': ['item1', 'item2']}
res = client.get_alerts_data(cluster)
self.assertEqual(res, ['item1', 'item2'])
self.http_client.get.assert_called_with(
"http://1.2.3.4:8080/api/v1/clusters/test_cluster/alerts?fields=*",
verify=False,
auth=client._auth,
headers=self.headers)
self.assertEqual(self.http_client.get.call_count, 2)
self.assertEqual(mock_check_response.call_count, 2)
def create_blueprint(cluster):
_prepare_ranger(cluster)
cluster = conductor.cluster_get(context.ctx(), cluster.id)
host_groups = []
for ng in cluster.node_groups:
procs = p_common.get_ambari_proc_list(ng)
procs.extend(p_common.get_clients(cluster))
for instance in ng.instances:
hg = {
"name": instance.instance_name,
"configurations": configs.get_instance_params(instance),
"components": []
}
for proc in procs:
hg["components"].append({"name": proc})
host_groups.append(hg)
bp = {
"Blueprints": {
"stack_name": "HDP",
"stack_version": cluster.hadoop_version
},
"host_groups": host_groups,
"configurations": configs.get_cluster_params(cluster)
}
ambari = plugin_utils.get_instance(cluster, p_common.AMBARI_SERVER)
password = cluster.extra["ambari_password"]
with ambari_client.AmbariClient(ambari, password=password) as client:
client.create_blueprint(cluster.name, bp)
def get_alerts_data(self, service=None):
if self._data is not None:
# return cached data
return self._data.get(service, []) if service else self._data
self._data = {}
self._cluster_services = []
try:
ambari = plugin_utils.get_instance(self.cluster,
p_common.AMBARI_SERVER)
password = self.cluster.extra.get("ambari_password")
with client.AmbariClient(ambari, password=password) as ambari:
resp = ambari.get_alerts_data(self.cluster)
for alert in resp:
alert = alert.get('Alert', {})
service = alert.get('service_name').lower()
if service not in self._data:
self._data[service] = []
self._cluster_services.append(service)
self._data[service].append(alert)
except Exception as e:
prefix = _("Can't get response from Ambari Monitor")
msg = _("%(problem)s: %(description)s") % {
'problem': prefix,
'description': six.text_type(e)
}
# don't put in exception to logs, it will be done by log.exception
LOG.exception(prefix)
self._exception_store = msg
def manage_config_groups(cluster, instances):
groups = []
ambari = plugin_utils.get_instance(cluster, p_common.AMBARI_SERVER)
password = cluster.extra["ambari_password"]
for instance in instances:
groups.extend(configs.get_config_group(instance))
with ambari_client.AmbariClient(ambari, password=password) as client:
client.create_config_group(cluster, groups)
def test_put_method(self):
client = ambari_client.AmbariClient(self.instance)
client.put("http://spam", data="data")
self.http_client.put.assert_called_with("http://spam",
data="data",
verify=False,
auth=client._auth,
headers=self.headers)
def start_cluster(cluster):
ambari_template = _build_ambari_cluster_template(cluster)
ambari = plugin_utils.get_instance(cluster, p_common.AMBARI_SERVER)
password = cluster.extra["ambari_password"]
with ambari_client.AmbariClient(ambari, password=password) as client:
req_id = client.create_cluster(cluster.name, ambari_template)["id"]
client.wait_ambari_request(req_id, cluster.name)
def test_init_client_manual(self):
client = ambari_client.AmbariClient(self.instance,
port="1234",
username="******",
password="******")
self.assertEqual("http://1.2.3.4:1234/api/v1", client._base_url)
self.assertEqual("user", client._auth.username)
self.assertEqual("pass", client._auth.password)
self.remote.get_http_client.assert_called_with("1234")
def update_default_ambari_password(cluster):
ambari = plugin_utils.get_instance(cluster, p_common.AMBARI_SERVER)
new_password = uuidutils.generate_uuid()
with ambari_client.AmbariClient(ambari) as client:
client.update_user_password("admin", "admin", new_password)
extra = cluster.extra.to_dict() if cluster.extra else {}
extra["ambari_password"] = new_password
ctx = context.ctx()
conductor.cluster_update(ctx, cluster, {"extra": extra})
cluster = conductor.cluster_get(ctx, cluster.id)
def _wait_all_processes_removed(cluster, instance):
ambari = plugin_utils.get_instance(cluster, p_common.AMBARI_SERVER)
password = cluster.extra["ambari_password"]
with ambari_client.AmbariClient(ambari, password=password) as client:
while True:
hdp_processes = client.list_host_processes(cluster.name, instance)
if not hdp_processes:
return
context.sleep(5)
def wait_host_registration(cluster):
ambari = plugin_utils.get_instance(cluster, p_common.AMBARI_SERVER)
hosts = plugin_utils.get_instances(cluster)
password = cluster.extra["ambari_password"]
with ambari_client.AmbariClient(ambari, password=password) as client:
kwargs = {"client": client, "num_hosts": len(hosts)}
poll_utils.poll(_check_host_registration, kwargs=kwargs, timeout=600)
registered_hosts = client.get_registered_hosts()
registered_host_names = [h["Hosts"]["host_name"] for h in registered_hosts]
actual_host_names = [h.fqdn() for h in hosts]
if sorted(registered_host_names) != sorted(actual_host_names):
raise p_exc.HadoopProvisionError(
_("Host registration fails in Ambari"))
def test_wait_ambari_request_error(self, mock_context):
client = ambari_client.AmbariClient(self.instance)
check_mock = mock.MagicMock()
d1 = {
"request_context": "r1",
"request_status": "ERROR",
"progress_percent": 146
}
check_mock.return_value = d1
client.check_request_status = check_mock
self.assertRaises(p_exc.HadoopProvisionError,
client.wait_ambari_request, "id1", "c1")
def test_create_blueprint(self):
client = ambari_client.AmbariClient(self.instance)
resp = mock.Mock()
resp.text = ""
resp.status_code = 200
self.http_client.post.return_value = resp
client.create_blueprint("cluster_name", {"some": "data"})
self.http_client.post.assert_called_with(
"http://1.2.3.4:8080/api/v1/blueprints/cluster_name",
data=jsonutils.dumps({"some": "data"}),
verify=False,
auth=client._auth,
headers=self.headers)
def test_import_credential(self):
resp = mock.Mock()
resp.text = ""
resp.status_code = 200
self.http_client.post.return_value = resp
client = ambari_client.AmbariClient(self.instance)
client.import_credential("test",
alias="credential",
data={"some": "data"})
self.http_client.post.assert_called_once_with(
"http://1.2.3.4:8080/api/v1/clusters/test/credentials/credential",
verify=False,
data=jsonutils.dumps({"some": "data"}),
auth=client._auth,
headers=self.headers)
def set_up_hdp_repos(cluster):
hdp_repo = configs.get_hdp_repo_url(cluster)
hdp_utils_repo = configs.get_hdp_utils_repo_url(cluster)
if not hdp_repo and not hdp_utils_repo:
return
ambari = plugin_utils.get_instance(cluster, p_common.AMBARI_SERVER)
password = cluster.extra["ambari_password"]
pv = cluster.hadoop_version
repos = repo_id_map[pv]
with ambari_client.AmbariClient(ambari, password=password) as client:
os_type = os_type_map[client.get_host_info(ambari.fqdn())["os_type"]]
if hdp_repo:
client.set_up_mirror(pv, os_type, repos["HDP"], hdp_repo)
if hdp_utils_repo:
client.set_up_mirror(pv, os_type, repos["HDP-UTILS"],
hdp_utils_repo)
def _remove_services_from_host(cluster, instance):
ambari = plugin_utils.get_instance(cluster, p_common.AMBARI_SERVER)
password = cluster.extra["ambari_password"]
with ambari_client.AmbariClient(ambari, password=password) as client:
hdp_processes = client.list_host_processes(cluster.name, instance)
for proc in hdp_processes:
LOG.debug("Stopping process %s on host %s " %
(proc, instance.fqdn()))
client.stop_process_on_host(cluster.name, instance, proc)
LOG.debug("Removing process %s from host %s " %
(proc, instance.fqdn()))
client.remove_process_from_host(cluster.name, instance, proc)
_wait_all_processes_removed(cluster, instance)
def test_add_host_to_cluster(self):
client = ambari_client.AmbariClient(self.instance)
resp = mock.Mock()
resp.text = ""
resp.status_code = 200
self.http_client.post.return_value = resp
instance = mock.MagicMock()
instance.fqdn.return_value = "i1"
instance.cluster.name = "cl"
client.add_host_to_cluster(instance)
self.http_client.post.assert_called_with(
"http://1.2.3.4:8080/api/v1/clusters/cl/hosts/i1",
verify=False,
auth=client._auth,
headers=self.headers)
def test_update_user_password(self):
client = ambari_client.AmbariClient(self.instance)
resp = mock.Mock()
resp.text = ""
resp.status_code = 200
self.http_client.put.return_value = resp
client.update_user_password("bart", "old_pw", "new_pw")
exp_req = jsonutils.dumps(
{"Users": {
"old_password": "******",
"password": "******"
}})
self.http_client.put.assert_called_with(
"http://1.2.3.4:8080/api/v1/users/bart",
data=exp_req,
verify=False,
auth=client._auth,
headers=self.headers)
def test_get_credential(self):
resp = mock.Mock()
resp.text = ""
resp.status_code = 200
self.http_client.get.return_value = resp
client = ambari_client.AmbariClient(self.instance)
client.get_credential("test", alias="credential")
self.http_client.get.assert_called_once_with(
"http://1.2.3.4:8080/api/v1/clusters/test/credentials/credential",
verify=False,
auth=client._auth,
headers=self.headers)
resp.status_code = 404
self.assertRaises(ambari_client.AmbariNotFound,
ambari_client.AmbariClient.check_response, resp,
True)
def create_blueprint(cluster):
_prepare_ranger(cluster)
cluster = conductor.cluster_get(context.ctx(), cluster.id)
host_groups = []
for ng in cluster.node_groups:
procs = p_common.get_ambari_proc_list(ng)
procs.extend(p_common.get_clients(cluster))
for instance in ng.instances:
hg = {
"name": instance.instance_name,
"configurations": configs.get_instance_params(instance),
"components": []
}
for proc in procs:
hg["components"].append({"name": proc})
host_groups.append(hg)
bp = {
"Blueprints": {
"stack_name": "HDP",
"stack_version": cluster.hadoop_version
},
"host_groups": host_groups,
"configurations": configs.get_cluster_params(cluster)
}
ambari = plugin_utils.get_instance(cluster, p_common.AMBARI_SERVER)
password = cluster.extra["ambari_password"]
general_configs = cluster.cluster_configs.get("general", {})
if (general_configs.get(p_common.NAMENODE_HA)
or general_configs.get(p_common.RESOURCEMANAGER_HA)
or general_configs.get(p_common.HBASE_REGIONSERVER_HA)):
bp = ha_helper.update_bp_ha_common(cluster, bp)
if general_configs.get(p_common.NAMENODE_HA):
bp = ha_helper.update_bp_for_namenode_ha(cluster, bp)
if general_configs.get(p_common.RESOURCEMANAGER_HA):
bp = ha_helper.update_bp_for_resourcemanager_ha(cluster, bp)
if general_configs.get(p_common.HBASE_REGIONSERVER_HA):
bp = ha_helper.update_bp_for_hbase_ha(cluster, bp)
with ambari_client.AmbariClient(ambari, password=password) as client:
return client.create_blueprint(cluster.name, bp)
def test_create_cluster(self):
client = ambari_client.AmbariClient(self.instance)
resp = mock.Mock()
resp.text = """{
"Requests": {
"id": 1,
"status": "InProgress"
}
}"""
resp.status_code = 200
self.http_client.post.return_value = resp
req_info = client.create_cluster("cluster_name", {"some": "data"})
self.assertEqual(1, req_info["id"])
self.http_client.post.assert_called_with(
"http://1.2.3.4:8080/api/v1/clusters/cluster_name",
data=jsonutils.dumps({"some": "data"}),
verify=False,
auth=client._auth,
headers=self.headers)
def test_wait_ambari_request(self, mock_context):
client = ambari_client.AmbariClient(self.instance)
check_mock = mock.MagicMock()
d1 = {
"request_context": "r1",
"request_status": "PENDING",
"progress_percent": 42
}
d2 = {
"request_context": "r1",
"request_status": "COMPLETED",
"progress_percent": 100
}
check_mock.side_effect = [d1, d2]
client.check_request_status = check_mock
client.wait_ambari_request("id1", "c1")
check_mock.assert_has_calls(
[mock.call("c1", "id1"),
mock.call("c1", "id1")])
def test_get_registered_hosts(self):
client = ambari_client.AmbariClient(self.instance)
resp_data = """{
"href" : "http://1.2.3.4:8080/api/v1/hosts",
"items" : [
{
"href" : "http://1.2.3.4:8080/api/v1/hosts/host1",
"Hosts" : {
"host_name" : "host1"
}
},
{
"href" : "http://1.2.3.4:8080/api/v1/hosts/host2",
"Hosts" : {
"host_name" : "host2"
}
},
{
"href" : "http://1.2.3.4:8080/api/v1/hosts/host3",
"Hosts" : {
"host_name" : "host3"
}
}
]
}"""
resp = mock.Mock()
resp.text = resp_data
resp.status_code = 200
self.http_client.get.return_value = resp
hosts = client.get_registered_hosts()
self.http_client.get.assert_called_with(
"http://1.2.3.4:8080/api/v1/hosts",
verify=False,
auth=client._auth,
headers=self.headers)
self.assertEqual(3, len(hosts))
self.assertEqual("host1", hosts[0]["Hosts"]["host_name"])
self.assertEqual("host2", hosts[1]["Hosts"]["host_name"])
self.assertEqual("host3", hosts[2]["Hosts"]["host_name"])
def manage_host_components(cluster, instances):
ambari = plugin_utils.get_instance(cluster, p_common.AMBARI_SERVER)
password = cluster.extra["ambari_password"]
requests_ids = []
with ambari_client.AmbariClient(ambari, password=password) as client:
clients = p_common.get_clients(cluster)
for instance in instances:
services = p_common.get_ambari_proc_list(instance.node_group)
services.extend(clients)
for service in services:
client.add_service_to_host(instance, service)
requests_ids.append(
client.start_service_on_host(instance, service,
'INSTALLED'))
client.wait_ambari_requests(requests_ids, cluster.name)
# all services added and installed, let's start them
requests_ids = []
for instance in instances:
services = p_common.get_ambari_proc_list(instance.node_group)
services.extend(p_common.ALL_LIST)
for service in services:
requests_ids.append(
client.start_service_on_host(instance, service, 'STARTED'))
client.wait_ambari_requests(requests_ids, cluster.name)
def _get_ambari_client(cluster):
ambari = plugin_utils.get_instance(cluster, p_common.AMBARI_SERVER)
password = cluster.extra["ambari_password"]
return ambari_client.AmbariClient(ambari, password=password)
def _remove_host(cluster, inst):
ambari = plugin_utils.get_instance(cluster, p_common.AMBARI_SERVER)
password = cluster.extra["ambari_password"]
with ambari_client.AmbariClient(ambari, password=password) as client:
client.delete_host(cluster.name, inst)
