def test_get_storage_paths(self):
for version in versions:
s = self.get_services_processor(version=version)
service = s.create_service('AMBARI')
ng1 = hdp_test_base.TestNodeGroup(None, None, None)
ng1.ng_storage_paths = ['/mnt', '/volume/disk1']
ng2 = hdp_test_base.TestNodeGroup(None, None, None)
ng2.ng_storage_paths = ['/mnt']
paths = service._get_common_paths([ng1, ng2])
self.assertEqual(['/mnt'], paths)
ng3 = hdp_test_base.TestNodeGroup(None, None, None)
ng1.ng_storage_paths = ['/mnt', '/volume/disk1', '/volume/disk2']
ng2.ng_storage_paths = ['/mnt']
ng3.ng_storage_paths = ['/mnt', '/volume/disk1']
paths = service._get_common_paths([ng1, ng2, ng3])
self.assertEqual(['/mnt'], paths)
ng1.ng_storage_paths = ['/mnt', '/volume/disk1', '/volume/disk2']
ng2.ng_storage_paths = ['/mnt', '/volume/disk1']
ng3.ng_storage_paths = ['/mnt', '/volume/disk1']
paths = service._get_common_paths([ng1, ng2, ng3])
self.assertEqual(['/volume/disk1'], paths)
def _create_hbase_cluster(self, version='1.3.2'):
master_host = hdp_test_base.TestServer('master.novalocal', 'master',
'11111', 3, '111.11.1111',
'222.11.1111')
if version == '1.3.2':
master_ng = hdp_test_base.TestNodeGroup('master', [master_host], [
"NAMENODE", "JOBTRACKER", "SECONDARY_NAMENODE", "TASKTRACKER",
"DATANODE", "AMBARI_SERVER", "ZOOKEEPER_SERVER"
])
elif version == '2.0.6':
master_ng = hdp_test_base.TestNodeGroup('master', [master_host], [
"NAMENODE", "RESOURCEMANAGER", "SECONDARY_NAMENODE",
"NODEMANAGER", "DATANODE", "AMBARI_SERVER", "HISTORYSERVER",
"ZOOKEEPER_SERVER"
])
extra_zk_host = hdp_test_base.TestServer('zk.novalocal', 'zk', '11112',
3, '111.11.1112',
'222.11.1112')
extra_zk_ng = hdp_test_base.TestNodeGroup('zk', [extra_zk_host],
['ZOOKEEPER_SERVER'])
hbase_host = hdp_test_base.TestServer('hbase.novalocal', 'hbase',
'11111', 3, '222.22.2222',
'222.11.1111')
hbase_ng = hdp_test_base.TestNodeGroup('hbase', [hbase_host],
["HBASE_MASTER"])
return hdp_test_base.TestCluster([master_ng, extra_zk_ng, hbase_ng])
def test_hdp2_create_sqoop_service(self, patched):
s = self.get_services_processor('2.0.6')
service = s.create_service('SQOOP')
self.assertEqual('SQOOP', service.name)
expected_configs = set(['global', 'core-site'])
self.assertEqual(expected_configs,
expected_configs & service.configurations)
self.assertFalse(service.is_mandatory())
# ensure that hdfs and mr clients are added implicitly
master_host = hdp_test_base.TestServer('master.novalocal', 'master',
'11111', 3, '111.11.1111',
'222.11.1111')
master_ng = hdp_test_base.TestNodeGroup('master', [master_host], [
"NAMENODE", "RESOURCEMANAGER", "HISTORYSERVER",
"SECONDARY_NAMENODE", "NODEMANAGER", "DATANODE", "AMBARI_SERVER",
"ZOOKEEPER_SERVER"
])
sqoop_host = hdp_test_base.TestServer('sqoop.novalocal', 'sqoop',
'11111', 3, '111.11.1111',
'222.11.1111')
sqoop_ng = hdp_test_base.TestNodeGroup('sqoop', [sqoop_host],
["SQOOP"])
cluster = hdp_test_base.TestCluster([master_ng, sqoop_ng])
cluster_spec = hdp_test_base.create_clusterspec(hdp_version='2.0.6')
cluster_spec.create_operational_config(cluster, [])
components = cluster_spec.get_node_groups_containing_component(
'SQOOP')[0].components
self.assertIn('HDFS_CLIENT', components)
self.assertIn('MAPREDUCE2_CLIENT', components)
def test_get_oozie_server(self):
test_host = base.TestServer('host1', 'test-master', '11111', 3,
'111.11.1111', '222.11.1111')
node_group = base.TestNodeGroup('ng1', [test_host], [
"AMBARI_SERVER", "NAMENODE", "DATANODE", "JOBTRACKER",
"TASKTRACKER", "OOZIE_SERVER"
])
cluster = base.TestCluster([node_group])
cluster.hadoop_version = '2.0.6'
plugin = ap.AmbariPlugin()
self.assertIsNotNone(
plugin.get_edp_engine(cluster,
edp.JOB_TYPE_PIG).get_oozie_server(cluster))
node_group = base.TestNodeGroup('ng1', [test_host], [
"AMBARI_SERVER", "NAMENODE", "DATANODE", "JOBTRACKER",
"TASKTRACKER", "NOT_OOZIE"
])
cluster = base.TestCluster([node_group])
cluster.hadoop_version = '2.0.6'
self.assertIsNone(
plugin.get_edp_engine(cluster,
edp.JOB_TYPE_PIG).get_oozie_server(cluster))
def test_get_storage_paths(self):
for version in versions:
s = self.get_services_processor(version=version)
service = s.create_service('AMBARI')
server1 = hdp_test_base.TestServer('host1', 'test-master', '11111',
3, '1.1.1.1', '2.2.2.2')
server2 = hdp_test_base.TestServer('host2', 'test-slave', '11111',
3, '3.3.3.3', '4.4.4.4')
server3 = hdp_test_base.TestServer('host3', 'another-test',
'11111', 3, '6.6.6.6',
'5.5.5.5')
ng1 = hdp_test_base.TestNodeGroup('ng1', [server1], None)
ng2 = hdp_test_base.TestNodeGroup('ng2', [server2], None)
ng3 = hdp_test_base.TestNodeGroup('ng3', [server3], None)
server1.storage_path = ['/volume/disk1']
server2.storage_path = ['/mnt']
paths = service._get_common_paths([ng1, ng2])
self.assertEqual([], paths)
server1.storage_path = ['/volume/disk1', '/volume/disk2']
server2.storage_path = ['/mnt']
server3.storage_path = ['/volume/disk1']
paths = service._get_common_paths([ng1, ng2, ng3])
self.assertEqual([], paths)
server1.storage_path = ['/volume/disk1', '/volume/disk2']
server2.storage_path = ['/volume/disk1']
server3.storage_path = ['/volume/disk1']
paths = service._get_common_paths([ng1, ng2, ng3])
self.assertEqual(['/volume/disk1'], paths)
def test__get_ambari_info(self, patched):
cluster_config_file = pkg.resource_string(
version.version_info.package,
'plugins/hdp/versions/version_1_3_2/resources/'
'default-cluster.template')
test_host = base.TestServer('host1', 'test-master', '11111', 3,
'111.11.1111', '222.11.1111')
node_group = base.TestNodeGroup('ng1', [test_host], [
"AMBARI_SERVER", "NAMENODE", "DATANODE", "JOBTRACKER",
"TASKTRACKER"
])
cluster = base.TestCluster([node_group])
cluster_config = cs.ClusterSpec(cluster_config_file)
cluster_config.create_operational_config(cluster, [])
plugin = ap.AmbariPlugin()
# change port
cluster_config.configurations['ambari']['server.port'] = '9000'
ambari_info = plugin.get_ambari_info(cluster_config)
self.assertEqual('9000', ambari_info.port)
# remove port
del cluster_config.configurations['ambari']['server.port']
ambari_info = plugin.get_ambari_info(cluster_config)
self.assertEqual('8080', ambari_info.port)
def test_hdp2_hbase_validation(self, patched):
master_host = hdp_test_base.TestServer('master.novalocal', 'master',
'11111', 3, '111.11.1111',
'222.11.1111')
master_ng = hdp_test_base.TestNodeGroup('master', [master_host], [
"NAMENODE", "RESOURCEMANAGER", "SECONDARY_NAMENODE",
"HISTORYSERVER", "NODEMANAGER", "DATANODE", "AMBARI_SERVER",
"ZOOKEEPER_SERVER"
])
hbase_host = hdp_test_base.TestServer('hbase.novalocal', 'hbase',
'11111', 3, '111.11.1111',
'222.11.1111')
hbase_ng = hdp_test_base.TestNodeGroup('hbase', [hbase_host],
["HBASE_MASTER"])
hbase_ng2 = hdp_test_base.TestNodeGroup('hbase2', [hbase_host],
["HBASE_MASTER"])
hbase_client_host = hdp_test_base.TestServer('hbase-client.novalocal',
'hbase-client', '11111',
3, '111.11.1111',
'222.11.1111')
hbase_client_ng = hdp_test_base.TestNodeGroup('hbase-client',
[hbase_client_host],
["HBASE_CLIENT"])
hbase_slave_host = hdp_test_base.TestServer('hbase-rs.novalocal',
'hbase-rs', '11111', 3,
'111.11.1111',
'222.11.1111')
hbase_slave_ng = hdp_test_base.TestNodeGroup('hbase-rs',
[hbase_slave_host],
["HBASE_REGIONSERVER"])
cluster = hdp_test_base.TestCluster([master_ng, hbase_client_ng])
cluster_spec = hdp_test_base.create_clusterspec(hdp_version='2.0.6')
# validation should fail due to lack of hbase master
self.assertRaises(ex.InvalidComponentCountException,
cluster_spec.create_operational_config, cluster, [])
cluster = hdp_test_base.TestCluster(
[master_ng, hbase_client_ng, hbase_slave_ng])
cluster_spec = hdp_test_base.create_clusterspec(hdp_version='2.0.6')
# validation should fail due to lack of hbase master
self.assertRaises(ex.InvalidComponentCountException,
cluster_spec.create_operational_config, cluster, [])
cluster = hdp_test_base.TestCluster(
[master_ng, hbase_client_ng, hbase_ng])
cluster_spec = hdp_test_base.create_clusterspec(hdp_version='2.0.6')
# validation should succeed with hbase master included
cluster_spec.create_operational_config(cluster, [])
cluster = hdp_test_base.TestCluster(
[master_ng, hbase_client_ng, hbase_ng, hbase_ng2])
cluster_spec = hdp_test_base.create_clusterspec(hdp_version='2.0.6')
# validation should fail with multiple hbase master components
self.assertRaises(ex.InvalidComponentCountException,
cluster_spec.create_operational_config, cluster, [])
