def test__set_ambari_credentials__admin_only(self, client): client.side_effect = self._get_test_request self.requests = [] plugin = ap.AmbariPlugin() cluster_config_file = pkg.resource_string( version.version_info.package, 'plugins/hdp/versions/version_1_3_2/resources/' 'default-cluster.template') cluster_spec = cs.ClusterSpec(cluster_config_file) ambari_info = ap.AmbariInfo(TestHost('111.11.1111'), '8080', 'admin', 'old-pwd') plugin._set_ambari_credentials(cluster_spec, ambari_info, '1.3.2') self.assertEqual(1, len(self.requests)) request = self.requests[0] self.assertEqual('put', request.method) self.assertEqual('http://111.11.1111:8080/api/v1/users/admin', request.url) self.assertEqual( '{"Users":{"roles":"admin","password":"******",' '"old_password":"******"} }', request.data) self.assertEqual(('admin', 'old-pwd'), request.auth) self.assertEqual('admin', ambari_info.user) self.assertEqual('admin', ambari_info.password)
def test_parse_default_with_hosts(self, patched): patched.side_effect = test_get_instance_info cluster_config_file = pkg.resource_string( version.version_info.package, 'plugins/hdp/resources/default-cluster.template') servers = [] server1 = TestServer('host1', 'master', '11111', 3, '111.11.1111', '222.11.1111', node_processes=[ "namenode", "jobtracker", "secondary_namenode", "ganglia_server", "ganglia_monitor", "nagios_server", "AMBARI_SERVER", "ambari_agent" ]) server2 = TestServer('host2', 'slave', '11111', 3, '222.22.2222', '333.22.2222') servers.append(server1) servers.append(server2) cluster = TestCluster() cluster.instances = servers cluster_config = cs.ClusterSpec(cluster_config_file, cluster) self._assert_services(cluster_config.services) self._assert_configurations(cluster_config.configurations) self._assert_host_role_mappings(cluster_config.node_groups) return cluster_config
def test__get_ambari_info(self, patched): cluster_config_file = pkg.resource_string( version.version_info.package, 'plugins/hdp/versions/version_1_3_2/resources/' 'default-cluster.template') test_host = base.TestServer('host1', 'test-master', '11111', 3, '111.11.1111', '222.11.1111') node_group = base.TestNodeGroup('ng1', [test_host], [ "AMBARI_SERVER", "NAMENODE", "DATANODE", "JOBTRACKER", "TASKTRACKER" ]) cluster = base.TestCluster([node_group]) cluster_config = cs.ClusterSpec(cluster_config_file) cluster_config.create_operational_config(cluster, []) plugin = ap.AmbariPlugin() #change port cluster_config.configurations['ambari']['server.port'] = '9000' ambari_info = plugin.get_ambari_info(cluster_config) self.assertEqual('9000', ambari_info.port) #remove port del cluster_config.configurations['ambari']['server.port'] ambari_info = plugin.get_ambari_info(cluster_config) self.assertEqual('8080', ambari_info.port)
def test__set_ambari_credentials__admin_only(self): self.requests = [] plugin = ap.AmbariPlugin() plugin._get_rest_request = self._get_test_request with open( os.path.join(os.path.realpath('../plugins'), 'hdp', 'resources', 'default-cluster.template'), 'r') as f: cluster_spec = cs.ClusterSpec(f.read()) ambari_info = ap.AmbariInfo(TestHost('111.11.1111'), '8080', 'admin', 'old-pwd') plugin._set_ambari_credentials(cluster_spec, ambari_info) self.assertEqual(1, len(self.requests)) request = self.requests[0] self.assertEqual('put', request.method) self.assertEqual('http://111.11.1111:8080/api/v1/users/admin', request.url) self.assertEqual( '{"Users":{"roles":"admin,user","password":"******",' '"old_password":"******"} }', request.data) self.assertEqual(('admin', 'old-pwd'), request.auth) self.assertEqual('admin', ambari_info.user) self.assertEqual('admin', ambari_info.password)
def scale_cluster(self, cluster, instances): processor = self._get_blueprint_processor(cluster) cluster_spec = clusterspec.ClusterSpec( json.dumps(processor.blueprint), cluster=cluster) rpm = self._get_rpm_uri(cluster_spec) servers = [] for instance in instances: host_role = utils.get_host_role(instance) servers.append(h.HadoopServer(instance, cluster_spec.node_groups [host_role], ambari_rpm=rpm)) ambari_info = self.get_ambari_info(cluster_spec, self._get_servers(cluster)) for server in servers: self._spawn('Ambari provisioning thread', server.provision_ambari, ambari_info) self._wait_for_host_registrations(self._get_num_hosts(cluster), ambari_info) # now add the hosts and the component self._add_hosts_and_components(cluster_spec, servers, ambari_info, cluster.name) self._install_and_start_components(cluster.name, servers, ambari_info)
def convert(self, config, plugin_name, version, cluster_template_create): normalized_config = clusterspec.ClusterSpec(config).normalize() #TODO(jspeidel): can we get the name (first arg) from somewhere? node_groups = [] for ng in normalized_config.node_groups: node_group = { "name": ng.name, "flavor_id": ng.flavor, "node_processes": ng.node_processes, "count": ng.count } node_groups.append(node_group) cluster_configs = dict() for entry in normalized_config.cluster_configs: ci = entry.config # get the associated service dictionary target = entry.config.applicable_target service_dict = cluster_configs.get(target, {}) service_dict[ci.name] = entry.value cluster_configs[target] = service_dict ctx = context.ctx() return cluster_template_create( ctx, { "name": uuidutils.generate_uuid(), "plugin_name": plugin_name, "hadoop_version": version, "node_groups": node_groups, "cluster_configs": cluster_configs })
def create_clusterspec(hdp_version='1.3.2'): version_suffix = hdp_version.replace('.', '_') cluster_config_file = pkg.resource_string( version.version_info.package, 'plugins/hdp/versions/version_{0}/resources/' 'default-cluster.template'.format(version_suffix)) return cs.ClusterSpec(cluster_config_file, version=hdp_version)
def get_cluster_spec(self, cluster, user_inputs, scaled_groups=None, cluster_template=None): if cluster_template: cluster_spec = cs.ClusterSpec(cluster_template, '2.0.6') else: cluster_spec = self.get_default_cluster_configuration() cluster_spec.create_operational_config( cluster, user_inputs, scaled_groups) return cluster_spec
def test_ambari_rpm(self): cluster_config_file = pkg.resource_string( version.version_info.package, 'plugins/hdp/resources/default-cluster.template') cluster_config = cs.ClusterSpec(cluster_config_file) self._assert_configurations(cluster_config.configurations) ambari_config = cluster_config.configurations['ambari'] self.assertIsNotNone('no rpm uri found', ambari_config.get('rpm', None))
def test_parse_default(self): cluster_config_file = pkg.resource_string( version.version_info.package, 'plugins/hdp/resources/default-cluster.template') cluster_config = cs.ClusterSpec(cluster_config_file) self._assert_services(cluster_config.services) self._assert_configurations(cluster_config.configurations) self._assert_host_role_mappings(cluster_config.node_groups) return cluster_config
def test_convert(self, ctx_func): plugin = ap.AmbariPlugin() cluster_config_file = pkg.resource_string( version.version_info.package, 'plugins/hdp/versions/1_3_2/resources/default-cluster.template') cluster = plugin.convert(cluster_config_file, 'ambari', '1.3.2', 'test-plugin', create_cluster_template) normalized_config = cs.ClusterSpec(cluster_config_file).normalize() self.assertEqual(normalized_config.hadoop_version, cluster.hadoop_version) self.assertEqual(len(normalized_config.node_groups), len(cluster.node_groups))
def test_update_ambari_info_credentials(self): plugin = ap.AmbariPlugin() cluster_config_file = pkg.resource_string( version.version_info.package, 'plugins/hdp/versions/1_3_2/resources/default-cluster.template') cluster_spec = cs.ClusterSpec(cluster_config_file) ambari_info = ap.AmbariInfo(TestHost('111.11.1111'), '8080', 'admin', 'old-pwd') plugin._update_ambari_info_credentials(cluster_spec, ambari_info) self.assertEqual('admin', ambari_info.user) self.assertEqual('admin', ambari_info.password)
def test_convert(self, ctx_func): plugin = ap.AmbariPlugin() with open( os.path.join(os.path.realpath('../plugins'), 'hdp', 'resources', 'default-cluster.template'), 'r') as f: cluster = plugin.convert(f.read(), 'ambari', '1.3.0', create_cluster_template) with open( os.path.join(os.path.realpath('../plugins'), 'hdp', 'resources', 'default-cluster.template'), 'r') as f: normalized_config = cs.ClusterSpec(f.read()).normalize() self.assertEqual(normalized_config.hadoop_version, cluster.hadoop_version) self.assertEqual(len(normalized_config.node_groups), len(cluster.node_groups))
def test__get_ambari_info(self): cluster_config_file = pkg.resource_string( version.version_info.package, 'plugins/hdp/resources/default-cluster.template') cluster_config = cs.ClusterSpec(cluster_config_file) plugin = ap.AmbariPlugin() #change port cluster_config.configurations['ambari']['server.port'] = '9000' ambari_info = plugin.get_ambari_info( cluster_config, [TestHost('111.11.1111', 'master')]) self.assertEqual('9000', ambari_info.port) #remove port del cluster_config.configurations['ambari']['server.port'] ambari_info = plugin.get_ambari_info( cluster_config, [TestHost('111.11.1111', 'master')]) self.assertEqual('8080', ambari_info.port)
def test__set_ambari_credentials__no_admin_user(self): self.requests = [] plugin = ap.AmbariPlugin() plugin._get_rest_request = self._get_test_request with open( os.path.join(os.path.realpath('../plugins'), 'hdp', 'resources', 'default-cluster.template'), 'r') as f: cluster_spec = cs.ClusterSpec(f.read()) for service in cluster_spec.services: if service.name == 'AMBARI': user = service.users[0] user.name = 'test' user.password = '******' user.groups = ['user'] self.assertRaises( ex.HadoopProvisionError, plugin._set_ambari_credentials(cluster_spec, '111.11.1111'))
def test__set_ambari_credentials__new_user_with_admin(self, client): self.requests = [] plugin = ap.AmbariPlugin() client.side_effect = self._get_test_request cluster_config_file = pkg.resource_string( version.version_info.package, 'plugins/hdp/versions/version_1_3_2/resources/' 'default-cluster.template') cluster_spec = cs.ClusterSpec(cluster_config_file) for service in cluster_spec.services: if service.name == 'AMBARI': new_user = cs.User('test', 'test_pw', ['user']) service.users.append(new_user) ambari_info = ap.AmbariInfo(TestHost('111.11.1111'), '8080', 'admin', 'old-pwd') plugin._set_ambari_credentials(cluster_spec, ambari_info, '1.3.2') self.assertEqual(2, len(self.requests)) request = self.requests[0] self.assertEqual('put', request.method) self.assertEqual('http://111.11.1111:8080/api/v1/users/admin', request.url) self.assertEqual( '{"Users":{"roles":"admin","password":"******",' '"old_password":"******"} }', request.data) self.assertEqual(('admin', 'old-pwd'), request.auth) request = self.requests[1] self.assertEqual('post', request.method) self.assertEqual('http://111.11.1111:8080/api/v1/users/test', request.url) self.assertEqual('{"Users":{"password":"******","roles":"user"} }', request.data) self.assertEqual(('admin', 'admin'), request.auth) self.assertEqual('admin', ambari_info.user) self.assertEqual('admin', ambari_info.password)
def test__set_ambari_credentials__new_user_with_admin(self): self.requests = [] plugin = ap.AmbariPlugin() plugin._get_rest_request = self._get_test_request with open( os.path.join(os.path.realpath('../plugins'), 'hdp', 'resources', 'default-cluster.template'), 'r') as f: cluster_spec = cs.ClusterSpec(f.read()) for service in cluster_spec.services: if service.name == 'AMBARI': new_user = cs.User('test', 'test_pw', ['user']) service.users.append(new_user) ambari_info = ap.AmbariInfo(TestHost('111.11.1111'), '8080', 'admin', 'old-pwd') plugin._set_ambari_credentials(cluster_spec, ambari_info) self.assertEqual(2, len(self.requests)) request = self.requests[0] self.assertEqual('put', request.method) self.assertEqual('http://111.11.1111:8080/api/v1/users/admin', request.url) self.assertEqual( '{"Users":{"roles":"admin,user","password":"******",' '"old_password":"******"} }', request.data) self.assertEqual(('admin', 'old-pwd'), request.auth) request = self.requests[1] self.assertEqual('post', request.method) self.assertEqual('http://111.11.1111:8080/api/v1/users/test', request.url) self.assertEqual('{"Users":{"password":"******","roles":"user"} }', request.data) self.assertEqual(('admin', 'admin'), request.auth) self.assertEqual('admin', ambari_info.user) self.assertEqual('admin', ambari_info.password)
def test_select_correct_server_for_ambari_host(self, patched): patched.side_effect = test_get_instance_info cluster_config_file = pkg.resource_string( version.version_info.package, 'plugins/hdp/resources/default-cluster.template') servers = [] server1 = TestServer('ambari_machine', 'master', '11111', 3, '111.11.1111', '222.11.1111', node_processes=[ "namenode", "jobtracker", "secondary_namenode", "ganglia_server", "ganglia_monitor", "nagios_server", "AMBARI_SERVER", "ambari_agent" ]) server2 = TestServer('host2', 'slave', '11111', 3, '222.22.2222', '333.22.2222', node_processes=[ "datanode", "tasktracker", "ganglia_monitor", "hdfs_client", "mapreduce_client", "ambari_agent" ]) servers.append(server1) servers.append(server2) cluster = TestCluster cluster.instances = servers cluster_config = cs.ClusterSpec(cluster_config_file, cluster) self.assertIn('ambari_machine', cluster_config.str, 'Ambari host not found')
def test__set_ambari_credentials__no_admin_user(self, client): self.requests = [] plugin = ap.AmbariPlugin() client.side_effect = self._get_test_request cluster_config_file = pkg.resource_string( version.version_info.package, 'plugins/hdp/versions/1_3_2/resources/default-cluster.template') cluster_spec = cs.ClusterSpec(cluster_config_file) for service in cluster_spec.services: if service.name == 'AMBARI': user = service.users[0] user.name = 'test' user.password = '******' user.groups = ['user'] ambari_info = ap.AmbariInfo(TestHost('111.11.1111'), '8080', 'admin', 'old-pwd') self.assertRaises( ex.HadoopProvisionError, plugin._set_ambari_credentials(cluster_spec, ambari_info, '1.3.2'))
def create_cluster(self, cluster, cluster_template): if cluster_template is None: raise ValueError('must supply cluster template') cluster_spec = clusterspec.ClusterSpec(cluster_template, cluster=cluster) hosts = self._get_servers(cluster) ambari_host = self._determine_host_for_server_component( 'AMBARI_SERVER', cluster_spec, hosts) self.cluster_name_to_ambari_host_mapping[cluster.name] = ambari_host rpm = self._get_rpm_uri(cluster_spec) servers = [] for host in hosts: host_role = utils.get_host_role(host) servers.append( h.HadoopServer(host, cluster_spec.node_groups[host_role], ambari_rpm=rpm)) provisioned = self._provision_cluster(cluster.name, cluster_spec, ambari_host, servers) if provisioned: installed = self._install_services(cluster.name, ambari_host) if installed: LOG.info("Install of Hadoop stack successful.") # add service urls self._set_cluster_info(cluster, cluster_spec, hosts) else: raise ex.HadoopProvisionError( 'Installation of Hadoop stack failed.') else: raise ex.HadoopProvisionError( 'Provisioning of Hadoop cluster failed.')
def test__set_ambari_credentials__new_user_no_admin(self): self.requests = [] plugin = ap.AmbariPlugin() plugin._get_rest_request = self._get_test_request with open( os.path.join(os.path.realpath('../plugins'), 'hdp', 'resources', 'default-cluster.template'), 'r') as f: cluster_spec = cs.ClusterSpec(f.read()) for service in cluster_spec.services: if service.name == 'AMBARI': user = service.users[0] user.name = 'test' user.password = '******' plugin._set_ambari_credentials(cluster_spec, '111.11.1111') self.assertEqual(2, len(self.requests)) request = self.requests[0] self.assertEqual('post', request.method) self.assertEqual('http://111.11.1111:8080/api/v1/users/test', request.url) self.assertEqual( '{"Users":{"password":"******","roles":"admin,user"' '} }', request.data) self.assertEqual(('admin', 'admin'), request.auth) request = self.requests[1] self.assertEqual('delete', request.method) self.assertEqual('http://111.11.1111:8080/api/v1/users/admin', request.url) self.assertEqual(None, request.data) self.assertEqual(('test', 'test_pw'), request.auth) self.assertEqual('test', plugin.ambari_user) self.assertEqual('test_pw', plugin.ambari_password)
def get_default_cluster_configuration(self): return cs.ClusterSpec(self._get_default_cluster_template(), '2.0.6')
def _get_default_cluster_configuration(self): with open( os.path.join(os.path.dirname(__file__), 'resources', 'default-cluster.template'), 'r') as f: return clusterspec.ClusterSpec(f.read())
def create_clusterspec(): cluster_config_file = pkg.resource_string( version.version_info.package, 'plugins/hdp/versions/1_3_2/resources/default-cluster.template') return cs.ClusterSpec(cluster_config_file)
def test_normalize(self): cluster_config_file = pkg.resource_string( version.version_info.package, 'plugins/hdp/resources/default-cluster.template') cluster_config = cs.ClusterSpec(cluster_config_file) cluster = cluster_config.normalize() configs = cluster.cluster_configs contains_dfs_datanode_http_address = False contains_mapred_jobtracker_taskScheduler = False contains_dfs_include = False for entry in configs: config = entry.config # assert some random configurations across targets if config.name == 'dfs.datanode.http.address': contains_dfs_datanode_http_address = True self.assertEqual('string', config.type) self.assertEqual('0.0.0.0:50075', config.default_value) self.assertEqual('HDFS', config.applicable_target) if config.name == 'mapred.jobtracker.taskScheduler': contains_mapred_jobtracker_taskScheduler = True self.assertEqual('string', config.type) self.assertEqual( 'org.apache.hadoop.mapred.CapacityTaskScheduler', config.default_value) self.assertEqual('MAPREDUCE', config.applicable_target) if config.name == 'dfs_include': contains_dfs_include = True self.assertEqual('string', config.type) self.assertEqual('dfs.include', config.default_value) self.assertEqual('HDFS', config.applicable_target) # print 'Config: name: {0}, type:{1}, # default value:{2}, target:{3}, Value:{4}'.format( # config.name, config.type, # config.default_value, # config.applicable_target, entry.value) self.assertTrue(contains_dfs_datanode_http_address) self.assertTrue(contains_mapred_jobtracker_taskScheduler) self.assertTrue(contains_dfs_include) node_groups = cluster.node_groups self.assertEqual(2, len(node_groups)) contains_master_group = False contains_slave_group = False for i in range(2): node_group = node_groups[i] components = node_group.node_processes if node_group.name == "master": contains_master_group = True self.assertEqual(8, len(components)) self.assertIn('NAMENODE', components) self.assertIn('JOBTRACKER', components) self.assertIn('SECONDARY_NAMENODE', components) self.assertIn('GANGLIA_SERVER', components) self.assertIn('GANGLIA_MONITOR', components) self.assertIn('NAGIOS_SERVER', components) self.assertIn('AMBARI_SERVER', components) self.assertIn('AMBARI_AGENT', components) #TODO(jspeidel): node configs #TODO(jspeidel): vm_requirements elif node_group.name == 'slave': contains_slave_group = True self.assertEqual(6, len(components)) self.assertIn('DATANODE', components) self.assertIn('TASKTRACKER', components) self.assertIn('GANGLIA_MONITOR', components) self.assertIn('HDFS_CLIENT', components) self.assertIn('MAPREDUCE_CLIENT', components) self.assertIn('AMBARI_AGENT', components) #TODO(jspeidel): node configs #TODO(jspeidel): vm requirements else: self.fail('Unexpected node group: {0}'.format(node_group.name)) self.assertTrue(contains_master_group) self.assertTrue(contains_slave_group)