def test_load_template_use_neutron(self):
"""This test checks Heat cluster template with Neutron enabled.
Two NodeGroups used: 'master' with Ephemeral drive attached and
'worker' with 2 attached volumes 10GB size each
"""
ng1 = tu.make_ng_dict('master', 42, ['namenode'], 1,
floating_ip_pool='floating', image_id=None,
volumes_per_node=0, volumes_size=0, id=1)
ng2 = tu.make_ng_dict('worker', 42, ['datanode'], 1,
floating_ip_pool='floating', image_id=None,
volumes_per_node=2, volumes_size=10, id=2)
cluster = tu.create_cluster("cluster", "tenant1", "general",
"1.2.1", [ng1, ng2],
user_keypair_id='user_key',
neutron_management_network='private_net',
default_image_id='1', anti_affinity=[],
image_id=None)
heat_template = h.ClusterTemplate(cluster)
heat_template.add_node_group_extra(ng1['id'], 1, 'line1\nline2')
heat_template.add_node_group_extra(ng2['id'], 1, 'line2\nline3')
self.override_config("use_neutron", True)
main_template = h._load_template(
'main.heat', {'resources':
heat_template._serialize_resources()})
self.assertEqual(
json.loads(main_template),
json.loads(f.get_file_text(
"tests/unit/resources/"
"test_serialize_resources_use_neutron.heat")))
def test_validate(self):
plugin = p.IDHProvider()
ng_mng = tu.make_ng_dict('mng', 'f1', ['manager'], 1)
ng_nn = tu.make_ng_dict('nn', 'f1', ['namenode'], 1)
ng_jt = tu.make_ng_dict('jt', 'f1', ['jobtracker'], 1)
ng_dn = tu.make_ng_dict('dn', 'f1', ['datanode'], 2)
ng_tt = tu.make_ng_dict('tt', 'f1', ['tasktracker'], 2)
cl = tu.create_cluster('cl1', 't1', 'intel', '2.5.1',
[ng_nn] + [ng_dn])
self.assertRaises(g_ex.InvalidComponentCountException, plugin.validate,
cl)
cl = tu.create_cluster('cl1', 't1', 'intel', '2.5.1', [ng_mng])
self.assertRaises(g_ex.InvalidComponentCountException, plugin.validate,
cl)
cl = tu.create_cluster('cl1', 't1', 'intel', '2.5.1',
[ng_mng] + [ng_nn] * 2)
self.assertRaises(g_ex.InvalidComponentCountException, plugin.validate,
cl)
cl = tu.create_cluster('cl1', 't1', 'intel', '2.5.1',
[ng_mng] + [ng_nn] + [ng_tt])
self.assertRaises(g_ex.RequiredServiceMissingException,
plugin.validate, cl)
cl = tu.create_cluster('cl1', 't1', 'intel', '2.5.1',
[ng_mng] + [ng_nn] + [ng_jt] * 2 + [ng_tt])
self.assertRaises(g_ex.InvalidComponentCountException, plugin.validate,
cl)
def test_load_template_with_anti_affinity_single_ng(self):
"""This test checks Heat cluster template with Neutron enabled
and anti-affinity feature enabled for single node process
in single node group.
"""
ng1 = tu.make_ng_dict('master', 42, ['namenode'], 1,
floating_ip_pool='floating', image_id=None,
volumes_per_node=0, volumes_size=0, id=1)
ng2 = tu.make_ng_dict('worker', 42, ['datanode'], 2,
floating_ip_pool='floating', image_id=None,
volumes_per_node=0, volumes_size=0, id=2)
cluster = tu.create_cluster("cluster", "tenant1", "general",
"1.2.1", [ng1, ng2],
user_keypair_id='user_key',
neutron_management_network='private_net',
default_image_id='1',
anti_affinity=['datanode'], image_id=None)
aa_heat_template = h.ClusterTemplate(cluster)
aa_heat_template.add_node_group_extra(ng1['id'], 1, 'line1\nline2')
aa_heat_template.add_node_group_extra(ng2['id'], 2, 'line2\nline3')
self.override_config("use_neutron", True)
main_template = h._load_template(
'main.heat', {'resources':
aa_heat_template._serialize_resources()})
self.assertEqual(
json.loads(main_template),
json.loads(f.get_file_text(
"tests/unit/resources/"
"test_serialize_resources_aa.heat")))
def test_get_hadoop_ssh_keys(self):
cluster_dict = {
'name': 'cluster1',
'plugin_name': 'mock_plugin',
'hadoop_version': 'mock_version',
'default_image_id': 'initial',
'node_groups': [tu.make_ng_dict("ng1", "f1", ["s1"], 1)]
}
cluster1 = conductor.cluster_create(context.ctx(), cluster_dict)
(private_key1, public_key1) = c_h.get_hadoop_ssh_keys(cluster1)
#should store keys for old cluster
cluster1 = conductor.cluster_get(context.ctx(), cluster1)
(private_key2, public_key2) = c_h.get_hadoop_ssh_keys(cluster1)
self.assertEqual(public_key1, public_key2)
self.assertEqual(private_key1, private_key2)
#should generate new keys for new cluster
cluster_dict.update({'name': 'cluster2'})
cluster2 = conductor.cluster_create(context.ctx(), cluster_dict)
(private_key3, public_key3) = c_h.get_hadoop_ssh_keys(cluster2)
self.assertNotEqual(public_key1, public_key3)
self.assertNotEqual(private_key1, private_key3)
def setUp(self):
i1 = tu.make_inst_dict('i1', 'master')
i2 = tu.make_inst_dict('i2', 'worker1')
i3 = tu.make_inst_dict('i3', 'worker2')
i4 = tu.make_inst_dict('i4', 'worker3')
i5 = tu.make_inst_dict('i5', 'sn')
ng1 = tu.make_ng_dict("master", "f1", ["jt", "nn"], 1, [i1])
ng2 = tu.make_ng_dict("workers", "f1", ["tt", "dn"], 3, [i2, i3, i4])
ng3 = tu.make_ng_dict("sn", "f1", ["dn"], 1, [i5])
self.c1 = tu.create_cluster("cluster1", "tenant1", "general", "1.2.1",
[ng1, ng2, ng3])
self.ng1 = self.c1.node_groups[0]
self.ng2 = self.c1.node_groups[1]
self.ng3 = self.c1.node_groups[2]
def test_validate(self):
self.ng = []
self.ng.append(tu.make_ng_dict("nn", "f1", ["namenode"], 0))
self.ng.append(tu.make_ng_dict("jt", "f1", ["jobtracker"], 0))
self.ng.append(tu.make_ng_dict("tt", "f1", ["tasktracker"], 0))
self.ng.append(tu.make_ng_dict("oozie", "f1", ["oozie"], 0))
self._validate_case(1, 1, 10, 1)
with self.assertRaises(ex.InvalidComponentCountException):
self._validate_case(0, 1, 10, 1)
with self.assertRaises(ex.InvalidComponentCountException):
self._validate_case(2, 1, 10, 1)
with self.assertRaises(ex.RequiredServiceMissingException):
self._validate_case(1, 0, 10, 1)
with self.assertRaises(ex.InvalidComponentCountException):
self._validate_case(1, 2, 10, 1)
with self.assertRaises(ex.InvalidComponentCountException):
self._validate_case(1, 1, 0, 2)
with self.assertRaises(ex.RequiredServiceMissingException):
self._validate_case(1, 0, 0, 1)
def test_check_cluster_scaling_resize_ng(self):
ng1 = tu.make_ng_dict('ng', '42', ['namenode'], 1)
cluster = tu.create_cluster("cluster1",
"tenant1",
"vanilla",
"1.2.1", [ng1],
status='Validating',
id='12321')
self._assert_check_scaling(data={},
cluster=cluster,
expected_message="Cluster cannot be scaled "
"not in 'Active' "
"status. Cluster status: "
"Validating")
cluster = tu.create_cluster("cluster1",
"tenant1",
"vanilla",
"1.2.1", [ng1],
status='Active',
id='12321')
data = {
'resize_node_groups': [{
'name': 'a',
'flavor_id': '42',
'node_processes': ['namenode']
}],
}
self._assert_check_scaling(data=data,
cluster=cluster,
expected_message="Cluster doesn't contain "
"node group with name 'a'")
data.update({
'resize_node_groups': [{
'name': 'a',
'flavor_id': '42',
'node_processes': ['namenode']
}, {
'name': 'a',
'flavor_id': '42',
'node_processes': ['namenode']
}]
})
self._assert_check_scaling(data=data,
cluster=cluster,
expected_message='Duplicates in node '
'group names are detected')
def test_check_cluster_scaling_add_ng(self):
ng1 = tu.make_ng_dict('ng', '42', ['namenode'], 1)
cluster = tu.create_cluster("test-cluster",
"tenant",
"vanilla",
"1.2.1", [ng1],
status='Active',
id='12321')
data = {
'add_node_groups': [{
'name': 'a',
'flavor_id': '42',
'node_processes': ['namenode']
}, {
'name': 'a',
'flavor_id': '42',
'node_processes': ['namenode']
}]
}
self._assert_check_scaling(data=data,
cluster=cluster,
expected_message='Duplicates in node '
'group names are detected')
data = {
'add_node_groups': [
{
'name': 'ng',
'flavor_id': '42',
'node_processes': ['namenode']
},
]
}
self._assert_check_scaling(data=data,
cluster=cluster,
expected_message="Can't add new nodegroup. "
"Cluster already has nodegroup "
"with name 'ng'")
def start_patch(patch_templates=True):
get_clusters_p = mock.patch("savanna.service.api.get_clusters")
get_cluster_p = mock.patch("savanna.service.api.get_cluster")
if patch_templates:
get_ng_templates_p = \
mock.patch("savanna.service.api.get_node_group_templates")
get_ng_template_p = \
mock.patch("savanna.service.api.get_node_group_template")
get_plugins_p = mock.patch("savanna.service.api.get_plugins")
get_plugin_p = \
mock.patch("savanna.plugins.base.PluginManager.get_plugin")
if patch_templates:
get_cl_templates_p = \
mock.patch("savanna.service.api.get_cluster_templates")
get_cl_template_p = \
mock.patch("savanna.service.api.get_cluster_template")
nova_p = mock.patch("savanna.utils.openstack.nova.client")
keystone_p = mock.patch("savanna.utils.openstack.keystone.client")
get_image_p = mock.patch("savanna.service.api.get_image")
get_image = get_image_p.start()
get_clusters = get_clusters_p.start()
get_cluster = get_cluster_p.start()
if patch_templates:
get_ng_templates = get_ng_templates_p.start()
get_ng_template = get_ng_template_p.start()
get_plugins = get_plugins_p.start()
get_plugin = get_plugin_p.start()
if patch_templates:
get_cl_templates = get_cl_templates_p.start()
get_cl_template_p.start()
nova = nova_p.start()
keystone = keystone_p.start()
if patch_templates:
get_cl_templates.return_value = []
nova().flavors.list.side_effect = _get_flavors_list
nova().keypairs.get.side_effect = _get_keypair
nova().networks.find.side_effect = _get_network
class Service:
@property
def name(self):
return 'cinder'
def _services_list():
return [Service()]
keystone().services.list.side_effect = _services_list
class Image:
def __init__(self, name='test'):
self.name = name
@property
def id(self):
if self.name == 'test':
return '550e8400-e29b-41d4-a716-446655440000'
else:
return '813fe450-40d2-4acc-ade5-ea753a1bd5bc'
@property
def tags(self):
if self.name == 'test':
return ['vanilla', '1.2.1']
else:
return ['wrong_tag']
def _get_image(id):
if id == '550e8400-e29b-41d4-a716-446655440000':
return Image()
else:
return Image('wrong_test')
get_image.side_effect = _get_image
nova().images.list_registered.return_value = [Image(),
Image(name='wrong_name')]
ng_dict = tu.make_ng_dict('ng', '42', ['namenode'], 1)
cluster = tu.create_cluster('test', 't', 'vanilla', '1.2.1', [ng_dict],
id=1, status='Active')
# stub clusters list
get_clusters.return_value = [cluster]
get_cluster.return_value = cluster
# stub node templates
if patch_templates:
ngt_dict = {'name': 'test', 'tenant_id': 't', 'flavor_id': '42',
'plugin_name': 'vanilla', 'hadoop_version': '1.2.1',
#'id': '1234321',
'id': '550e8400-e29b-41d4-a716-446655440000',
'node_processes': ['namenode']}
get_ng_templates.return_value = [r.NodeGroupTemplateResource(ngt_dict)]
ct_dict = {'name': 'test', 'tenant_id': 't',
'plugin_name': 'vanilla', 'hadoop_version': '1.2.1'}
get_cl_templates.return_value = [r.ClusterTemplateResource(ct_dict)]
vanilla = plugin.VanillaProvider()
vanilla.name = 'vanilla'
get_plugins.return_value = [vanilla]
def _get_ng_template(id):
for template in get_ng_templates():
if template.id == id:
return template
return None
def _get_plugin(name):
if name == 'vanilla':
return vanilla
return None
get_plugin.side_effect = _get_plugin
if patch_templates:
get_ng_template.side_effect = _get_ng_template
# request data to validate
patchers = [get_clusters_p, get_plugins_p, get_plugin_p,
nova_p, keystone_p, get_image_p]
if patch_templates:
patchers.extend([get_ng_template_p, get_ng_templates_p,
get_cl_template_p, get_cl_templates_p])
return patchers