def test_get_oozie_server(self):
test_host = base.TestServer('host1', 'test-master', '11111', 3,
'111.11.1111', '222.11.1111')
node_group = base.TestNodeGroup('ng1', [test_host], [
"AMBARI_SERVER", "NAMENODE", "DATANODE", "JOBTRACKER",
"TASKTRACKER", "OOZIE_SERVER"
])
cluster = base.TestCluster([node_group])
cluster.hadoop_version = '2.0.6'
plugin = ap.AmbariPlugin()
self.assertIsNotNone(
plugin.get_edp_engine(cluster,
edp.JOB_TYPE_PIG).get_oozie_server(cluster))
node_group = base.TestNodeGroup('ng1', [test_host], [
"AMBARI_SERVER", "NAMENODE", "DATANODE", "JOBTRACKER",
"TASKTRACKER", "NOT_OOZIE"
])
cluster = base.TestCluster([node_group])
cluster.hadoop_version = '2.0.6'
self.assertIsNone(
plugin.get_edp_engine(cluster,
edp.JOB_TYPE_PIG).get_oozie_server(cluster))
def test__get_ambari_info(self, patched):
cluster_config_file = pkg.resource_string(
version.version_info.package,
'plugins/hdp/versions/version_1_3_2/resources/'
'default-cluster.template')
test_host = base.TestServer('host1', 'test-master', '11111', 3,
'111.11.1111', '222.11.1111')
node_group = base.TestNodeGroup('ng1', [test_host], [
"AMBARI_SERVER", "NAMENODE", "DATANODE", "JOBTRACKER",
"TASKTRACKER"
])
cluster = base.TestCluster([node_group])
cluster_config = cs.ClusterSpec(cluster_config_file)
cluster_config.create_operational_config(cluster, [])
plugin = ap.AmbariPlugin()
# change port
cluster_config.configurations['ambari']['server.port'] = '9000'
ambari_info = plugin.get_ambari_info(cluster_config)
self.assertEqual('9000', ambari_info.port)
# remove port
del cluster_config.configurations['ambari']['server.port']
ambari_info = plugin.get_ambari_info(cluster_config)
self.assertEqual('8080', ambari_info.port)
def test__set_ambari_credentials__admin_only(self, client):
client.side_effect = self._get_test_request
self.requests = []
plugin = ap.AmbariPlugin()
cluster_config_file = pkg.resource_string(
version.version_info.package,
'plugins/hdp/versions/version_1_3_2/resources/'
'default-cluster.template')
cluster_spec = cs.ClusterSpec(cluster_config_file)
ambari_info = ap.AmbariInfo(TestHost('111.11.1111'), '8080', 'admin',
'old-pwd')
plugin._set_ambari_credentials(cluster_spec, ambari_info, '1.3.2')
self.assertEqual(1, len(self.requests))
request = self.requests[0]
self.assertEqual('put', request.method)
self.assertEqual('http://111.11.1111:8080/api/v1/users/admin',
request.url)
self.assertEqual(
'{"Users":{"roles":"admin","password":"******",'
'"old_password":"******"} }', request.data)
self.assertEqual(('admin', 'old-pwd'), request.auth)
self.assertEqual('admin', ambari_info.user)
self.assertEqual('admin', ambari_info.password)
def test_get_node_processes(self):
plugin = ap.AmbariPlugin()
service_components = plugin.get_node_processes('2.0.6')
self.assertEqual(
{
'YARN': ['RESOURCEMANAGER', 'YARN_CLIENT', 'NODEMANAGER'],
'GANGLIA': ['GANGLIA_SERVER'],
'HUE': ['HUE'],
'HIVE': [
'HIVE_SERVER', 'HIVE_METASTORE', 'HIVE_CLIENT',
'MYSQL_SERVER'
],
'OOZIE': ['OOZIE_SERVER', 'OOZIE_CLIENT'],
'HDFS': [
'NAMENODE', 'DATANODE', 'SECONDARY_NAMENODE',
'HDFS_CLIENT', 'JOURNALNODE', 'ZKFC'
],
'SQOOP': ['SQOOP'],
'MAPREDUCE2': ['HISTORYSERVER', 'MAPREDUCE2_CLIENT'],
'ZOOKEEPER': ['ZOOKEEPER_SERVER', 'ZOOKEEPER_CLIENT'],
'HBASE':
['HBASE_MASTER', 'HBASE_REGIONSERVER', 'HBASE_CLIENT'],
'HCATALOG': ['HCAT'],
'NAGIOS': ['NAGIOS_SERVER'],
'AMBARI': ['AMBARI_SERVER'],
'WEBHCAT': ['WEBHCAT_SERVER'],
'PIG': ['PIG']
}, service_components)
def test_edp206_calls_hadoop2_create_dir(self, create_dir):
cluster = base.TestCluster([])
cluster.plugin_name = 'hdp'
cluster.hadoop_version = '2.0.6'
plugin = ap.AmbariPlugin()
plugin.get_edp_engine(cluster, edp.JOB_TYPE_PIG).create_hdfs_dir(
mock.Mock(), '/tmp')
self.assertEqual(1, create_dir.call_count)
def test_get_node_processes(self):
plugin = ap.AmbariPlugin()
service_components = plugin.get_node_processes('1.3.2')
self.assertEqual(13, len(service_components))
components = service_components['HDFS']
self.assertIn('NAMENODE', components)
self.assertIn('DATANODE', components)
self.assertIn('SECONDARY_NAMENODE', components)
self.assertIn('HDFS_CLIENT', components)
components = service_components['MAPREDUCE']
self.assertIn('JOBTRACKER', components)
self.assertIn('TASKTRACKER', components)
self.assertIn('MAPREDUCE_CLIENT', components)
components = service_components['GANGLIA']
self.assertIn('GANGLIA_SERVER', components)
components = service_components['NAGIOS']
self.assertIn('NAGIOS_SERVER', components)
components = service_components['AMBARI']
self.assertIn('AMBARI_SERVER', components)
components = service_components['HCATALOG']
self.assertIn('HCAT', components)
components = service_components['ZOOKEEPER']
self.assertIn('ZOOKEEPER_SERVER', components)
self.assertIn('ZOOKEEPER_CLIENT', components)
components = service_components['HIVE']
self.assertIn('HIVE_SERVER', components)
self.assertIn('HIVE_METASTORE', components)
self.assertIn('HIVE_CLIENT', components)
self.assertIn('MYSQL_SERVER', components)
components = service_components['PIG']
self.assertIn('PIG', components)
components = service_components['WEBHCAT']
self.assertIn('WEBHCAT_SERVER', components)
components = service_components['OOZIE']
self.assertIn('OOZIE_SERVER', components)
self.assertIn('OOZIE_CLIENT', components)
self.assertIn('SQOOP', service_components['SQOOP'])
components = service_components['HBASE']
self.assertIn('HBASE_MASTER', components)
self.assertIn('HBASE_REGIONSERVER', components)
self.assertIn('HBASE_CLIENT', components)
def test_convert(self):
plugin = ap.AmbariPlugin()
cluster_config_file = pkg.resource_string(
version.version_info.package,
'plugins/hdp/versions/version_1_3_2/resources/'
'default-cluster.template')
cluster = plugin.convert(cluster_config_file, 'ambari', '1.3.2',
'test-plugin', create_cluster_template)
normalized_config = cs.ClusterSpec(cluster_config_file).normalize()
self.assertEqual(normalized_config.hadoop_version,
cluster.hadoop_version)
self.assertEqual(len(normalized_config.node_groups),
len(cluster.node_groups))
def test_update_ambari_info_credentials(self):
plugin = ap.AmbariPlugin()
cluster_config_file = pkg.resource_string(
version.version_info.package,
'plugins/hdp/versions/version_1_3_2/resources/'
'default-cluster.template')
cluster_spec = cs.ClusterSpec(cluster_config_file)
ambari_info = ap.AmbariInfo(TestHost('111.11.1111'), '8080', 'admin',
'old-pwd')
plugin._update_ambari_info_credentials(cluster_spec, ambari_info)
self.assertEqual('admin', ambari_info.user)
self.assertEqual('admin', ambari_info.password)
def test__set_ambari_credentials__new_user_with_admin(self, client):
self.requests = []
plugin = ap.AmbariPlugin()
client.side_effect = self._get_test_request
cluster_config_file = pkg.resource_string(
version.version_info.package,
'plugins/hdp/versions/version_1_3_2/resources/'
'default-cluster.template')
cluster_spec = cs.ClusterSpec(cluster_config_file)
for service in cluster_spec.services:
if service.name == 'AMBARI':
new_user = cs.User('test', 'test_pw', ['user'])
service.users.append(new_user)
ambari_info = ap.AmbariInfo(TestHost('111.11.1111'), '8080', 'admin',
'old-pwd')
plugin._set_ambari_credentials(cluster_spec, ambari_info, '1.3.2')
self.assertEqual(2, len(self.requests))
request = self.requests[0]
self.assertEqual('put', request.method)
self.assertEqual('http://111.11.1111:8080/api/v1/users/admin',
request.url)
self.assertEqual(
'{"Users":{"roles":"admin","password":"******",'
'"old_password":"******"} }', request.data)
self.assertEqual(('admin', 'old-pwd'), request.auth)
request = self.requests[1]
self.assertEqual('post', request.method)
self.assertEqual('http://111.11.1111:8080/api/v1/users/test',
request.url)
self.assertEqual('{"Users":{"password":"******","roles":"user"} }',
request.data)
self.assertEqual(('admin', 'admin'), request.auth)
self.assertEqual('admin', ambari_info.user)
self.assertEqual('admin', ambari_info.password)
def test__set_ambari_credentials__new_user_no_admin(self, client):
self.requests = []
plugin = ap.AmbariPlugin()
client.side_effect = self._get_test_request
cluster_config_file = pkg.resource_string(
version.version_info.package,
'plugins/hdp/versions/version_2_0_6/resources/'
'default-cluster.template')
cluster_spec = cs.ClusterSpec(cluster_config_file)
for service in cluster_spec.services:
if service.name == 'AMBARI':
user = service.users[0]
user.name = 'test'
user.password = '******'
ambari_info = ap.AmbariInfo(TestHost('111.11.1111'), '8080', 'admin',
'old-pwd')
plugin._set_ambari_credentials(cluster_spec, ambari_info, '2.0.6')
self.assertEqual(2, len(self.requests))
request = self.requests[0]
self.assertEqual('post', request.method)
self.assertEqual('http://111.11.1111:8080/api/v1/users/test',
request.url)
self.assertEqual(
'{"Users":{"password":"******","roles":"admin"'
'} }', request.data)
self.assertEqual(('admin', 'old-pwd'), request.auth)
request = self.requests[1]
self.assertEqual('delete', request.method)
self.assertEqual('http://111.11.1111:8080/api/v1/users/admin',
request.url)
self.assertIsNone(request.data)
self.assertEqual(('test', 'test_pw'), request.auth)
self.assertEqual('test', ambari_info.user)
self.assertEqual('test_pw', ambari_info.password)
def test__set_ambari_credentials__no_admin_user(self, client):
self.requests = []
plugin = ap.AmbariPlugin()
client.side_effect = self._get_test_request
cluster_config_file = pkg.resource_string(
version.version_info.package,
'plugins/hdp/versions/version_1_3_2/resources/'
'default-cluster.template')
cluster_spec = cs.ClusterSpec(cluster_config_file)
for service in cluster_spec.services:
if service.name == 'AMBARI':
user = service.users[0]
user.name = 'test'
user.password = '******'
user.groups = ['user']
ambari_info = ap.AmbariInfo(TestHost('111.11.1111'), '8080', 'admin',
'old-pwd')
self.assertRaises(
ex.HadoopProvisionError,
plugin._set_ambari_credentials(cluster_spec, ambari_info, '1.3.2'))
