def _get_core_site_props(self, cluster_context): result = {} if cluster_context.is_node_aware: result.update(self._get_core_site_node_aware_props()) for conf in swift_helper.get_swift_configs(): result[conf['name']] = conf['value'] for conf in self._get_impersonation_props(): result[conf['name']] = conf['value'] return result
def _get_core_site_props(self, cluster_context): result = { 'hadoop.proxyuser.mapr.groups': '*', 'hadoop.proxyuser.mapr.hosts': '*', } if cluster_context.is_node_aware: result.update(self._get_core_site_node_aware_props()) for conf in swift_helper.get_swift_configs(): result[conf['name']] = conf['value'] return result
def get_cluster_params(cluster): configs = _create_ambari_configs(cluster.cluster_configs, cluster.hadoop_version) swift_configs = { x["name"]: x["value"] for x in swift_helper.get_swift_configs() } configs.setdefault("core-site", {}) configs["core-site"].update(swift_configs) if utils.get_instance(cluster, common.RANGER_ADMIN): configs.setdefault("admin-properties", {}) configs["admin-properties"]["db_root_password"] = ( cluster.extra["ranger_db_password"]) return _serialize_ambari_configs(configs)
def generate_xml_configs(configs, storage_path, nn_hostname, hadoop_port): if hadoop_port is None: hadoop_port = 8020 cfg = { 'fs.defaultFS': 'hdfs://%s:%s' % (nn_hostname, str(hadoop_port)), 'dfs.namenode.name.dir': extract_hadoop_path(storage_path, '/dfs/nn'), 'dfs.datanode.data.dir': extract_hadoop_path(storage_path, '/dfs/dn'), 'dfs.hosts': '/etc/hadoop/dn.incl', 'dfs.hosts.exclude': '/etc/hadoop/dn.excl' } # inserting user-defined configs for key, value in extract_hadoop_xml_confs(configs): cfg[key] = value # Add the swift defaults if they have not been set by the user swft_def = [] if is_swift_enabled(configs): swft_def = SWIFT_DEFAULTS swift_configs = extract_name_values(swift.get_swift_configs()) for key, value in six.iteritems(swift_configs): if key not in cfg: cfg[key] = value # invoking applied configs to appropriate xml files core_all = CORE_DEFAULT + swft_def if CONF.enable_data_locality: cfg.update(topology.TOPOLOGY_CONFIG) # applying vm awareness configs core_all += topology.vm_awareness_core_config() xml_configs = { 'core-site': utils.create_hadoop_xml(cfg, core_all), 'hdfs-site': utils.create_hadoop_xml(cfg, HDFS_DEFAULT) } return xml_configs
def test_generate_xml_configs(self, auth_url): auth_url.return_value = "http://localhost:5000/v2/" # Make a dict of swift configs to verify generated values swift_vals = c_helper.extract_name_values(swift.get_swift_configs()) # Make sure that all the swift configs are in core-site c = c_helper.generate_xml_configs({}, ['/mnt/one'], 'localhost', None) doc = xml.parseString(c['core-site']) configuration = doc.getElementsByTagName('configuration') properties = utils.get_property_dict(configuration[0]) self.assertDictContainsSubset(swift_vals, properties) # Make sure that user values have precedence over defaults c = c_helper.generate_xml_configs( {'HDFS': { 'fs.swift.service.sahara.tenant': 'fred' }}, ['/mnt/one'], 'localhost', None) doc = xml.parseString(c['core-site']) configuration = doc.getElementsByTagName('configuration') properties = utils.get_property_dict(configuration[0]) mod_swift_vals = copy.copy(swift_vals) mod_swift_vals['fs.swift.service.sahara.tenant'] = 'fred' self.assertDictContainsSubset(mod_swift_vals, properties) # Make sure that swift configs are left out if not enabled c = c_helper.generate_xml_configs( { 'HDFS': { 'fs.swift.service.sahara.tenant': 'fred' }, 'general': { 'Enable Swift': False } }, ['/mnt/one'], 'localhost', None) doc = xml.parseString(c['core-site']) configuration = doc.getElementsByTagName('configuration') properties = utils.get_property_dict(configuration[0]) for key in mod_swift_vals.keys(): self.assertNotIn(key, properties)
def _get_hadoop_configs(pctx, instance): cluster = instance.node_group.cluster nn_hostname = vu.get_instance_hostname(vu.get_namenode(cluster)) dirs = _get_hadoop_dirs(instance) confs = { 'Hadoop': { 'fs.defaultFS': 'hdfs://%s:9000' % nn_hostname }, 'HDFS': { 'dfs.namenode.name.dir': ','.join(dirs['hadoop_name_dirs']), 'dfs.datanode.data.dir': ','.join(dirs['hadoop_data_dirs']), 'dfs.hosts': '%s/dn-include' % HADOOP_CONF_DIR, 'dfs.hosts.exclude': '%s/dn-exclude' % HADOOP_CONF_DIR } } res_hostname = vu.get_instance_hostname(vu.get_resourcemanager(cluster)) if res_hostname: confs['YARN'] = { 'yarn.nodemanager.aux-services': 'mapreduce_shuffle', 'yarn.resourcemanager.hostname': '%s' % res_hostname, 'yarn.resourcemanager.nodes.include-path': '%s/nm-include' % (HADOOP_CONF_DIR), 'yarn.resourcemanager.nodes.exclude-path': '%s/nm-exclude' % (HADOOP_CONF_DIR) } confs['MapReduce'] = {'mapreduce.framework.name': 'yarn'} hs_hostname = vu.get_instance_hostname(vu.get_historyserver(cluster)) if hs_hostname: confs['MapReduce']['mapreduce.jobhistory.address'] = ("%s:10020" % hs_hostname) oozie = vu.get_oozie(cluster) if oozie: hadoop_cfg = { 'hadoop.proxyuser.hadoop.hosts': '*', 'hadoop.proxyuser.hadoop.groups': 'hadoop' } confs['Hadoop'].update(hadoop_cfg) oozie_cfg = oozie_helper.get_oozie_required_xml_configs( HADOOP_CONF_DIR) if config_helper.is_mysql_enabled(pctx, cluster): oozie_cfg.update(oozie_helper.get_oozie_mysql_configs(cluster)) confs['JobFlow'] = oozie_cfg if config_helper.is_swift_enabled(pctx, cluster): swift_configs = {} for config in swift.get_swift_configs(): swift_configs[config['name']] = config['value'] confs['Hadoop'].update(swift_configs) if config_helper.is_data_locality_enabled(pctx, cluster): confs['Hadoop'].update(th.TOPOLOGY_CONFIG) confs['Hadoop'].update( {"topology.script.file.name": HADOOP_CONF_DIR + "/topology.sh"}) hive_hostname = vu.get_instance_hostname(vu.get_hiveserver(cluster)) if hive_hostname: hive_pass = u.get_hive_password(cluster) hive_cfg = { 'hive.warehouse.subdir.inherit.perms': True, 'javax.jdo.option.ConnectionURL': 'jdbc:derby:;databaseName=/opt/hive/metastore_db;create=true' } if config_helper.is_mysql_enabled(pctx, cluster): hive_cfg.update({ 'javax.jdo.option.ConnectionURL': 'jdbc:mysql://%s/metastore' % hive_hostname, 'javax.jdo.option.ConnectionDriverName': 'com.mysql.jdbc.Driver', 'javax.jdo.option.ConnectionUserName': '******', 'javax.jdo.option.ConnectionPassword': hive_pass, 'datanucleus.autoCreateSchema': 'false', 'datanucleus.fixedDatastore': 'true', 'hive.metastore.uris': 'thrift://%s:9083' % hive_hostname, }) proxy_configs = cluster.cluster_configs.get('proxy_configs') if proxy_configs and config_helper.is_swift_enabled(pctx, cluster): hive_cfg.update({ swift.HADOOP_SWIFT_USERNAME: proxy_configs['proxy_username'], swift.HADOOP_SWIFT_PASSWORD: key_manager.get_secret(proxy_configs['proxy_password']), swift.HADOOP_SWIFT_TRUST_ID: proxy_configs['proxy_trust_id'], swift.HADOOP_SWIFT_DOMAIN_NAME: CONF.proxy_user_domain_name }) confs['Hive'] = hive_cfg return confs
def _get_configs(self, service, cluster=None, instance=None): def get_hadoop_dirs(mount_points, suffix): return ','.join([x + suffix for x in mount_points]) all_confs = {} if cluster: zk_count = self.validator.get_inst_count(cluster, 'ZOOKEEPER_SERVER') hbm_count = self.validator.get_inst_count(cluster, 'HBASE_MASTER') snt_count = self.validator.get_inst_count(cluster, 'SENTRY_SERVER') ks_count =\ self.validator.get_inst_count(cluster, 'KEY_VALUE_STORE_INDEXER') kms_count = self.validator.get_inst_count(cluster, 'KMS') imp_count =\ self.validator.get_inst_count(cluster, 'IMPALA_CATALOGSERVER') hive_count = self.validator.get_inst_count(cluster, 'HIVE_METASTORE') slr_count = self.validator.get_inst_count(cluster, 'SOLR_SERVER') sqp_count = self.validator.get_inst_count(cluster, 'SQOOP_SERVER') core_site_safety_valve = '' if self.pu.c_helper.is_swift_enabled(cluster): configs = swift_helper.get_swift_configs() confs = {c['name']: c['value'] for c in configs} core_site_safety_valve = utils.create_elements_xml(confs) all_confs = { 'HDFS': { 'zookeeper_service': self.ZOOKEEPER_SERVICE_NAME if zk_count else '', 'dfs_block_local_path_access_user': '******' if imp_count else '', 'kms_service': self.KMS_SERVICE_NAME if kms_count else '', 'core_site_safety_valve': core_site_safety_valve }, 'HIVE': { 'mapreduce_yarn_service': self.YARN_SERVICE_NAME, 'sentry_service': self.SENTRY_SERVICE_NAME if snt_count else '', 'zookeeper_service': self.ZOOKEEPER_SERVICE_NAME if zk_count else '' }, 'OOZIE': { 'mapreduce_yarn_service': self.YARN_SERVICE_NAME, 'hive_service': self.HIVE_SERVICE_NAME if hive_count else '', 'zookeeper_service': self.ZOOKEEPER_SERVICE_NAME if zk_count else '' }, 'YARN': { 'hdfs_service': self.HDFS_SERVICE_NAME, 'zookeeper_service': self.ZOOKEEPER_SERVICE_NAME if zk_count else '' }, 'HUE': { 'hive_service': self.HIVE_SERVICE_NAME, 'oozie_service': self.OOZIE_SERVICE_NAME, 'sentry_service': self.SENTRY_SERVICE_NAME if snt_count else '', 'solr_service': self.SOLR_SERVICE_NAME if slr_count else '', 'zookeeper_service': self.ZOOKEEPER_SERVICE_NAME if zk_count else '', 'hbase_service': self.HBASE_SERVICE_NAME if hbm_count else '', 'impala_service': self.IMPALA_SERVICE_NAME if imp_count else '', 'sqoop_service': self.SQOOP_SERVICE_NAME if sqp_count else '' }, 'SPARK_ON_YARN': { 'yarn_service': self.YARN_SERVICE_NAME }, 'HBASE': { 'hdfs_service': self.HDFS_SERVICE_NAME, 'zookeeper_service': self.ZOOKEEPER_SERVICE_NAME, 'hbase_enable_indexing': 'true' if ks_count else 'false', 'hbase_enable_replication': 'true' if ks_count else 'false' }, 'FLUME': { 'hdfs_service': self.HDFS_SERVICE_NAME, 'solr_service': self.SOLR_SERVICE_NAME if slr_count else '', 'hbase_service': self.HBASE_SERVICE_NAME if hbm_count else '' }, 'SENTRY': { 'hdfs_service': self.HDFS_SERVICE_NAME, 'sentry_server_config_safety_valve': ( self.c_helper.SENTRY_IMPALA_CLIENT_SAFETY_VALVE if imp_count else '') }, 'SOLR': { 'hdfs_service': self.HDFS_SERVICE_NAME, 'zookeeper_service': self.ZOOKEEPER_SERVICE_NAME }, 'SQOOP': { 'mapreduce_yarn_service': self.YARN_SERVICE_NAME }, 'KS_INDEXER': { 'hbase_service': self.HBASE_SERVICE_NAME, 'solr_service': self.SOLR_SERVICE_NAME }, 'IMPALA': { 'hdfs_service': self.HDFS_SERVICE_NAME, 'hbase_service': self.HBASE_SERVICE_NAME if hbm_count else '', 'hive_service': self.HIVE_SERVICE_NAME, 'sentry_service': self.SENTRY_SERVICE_NAME if snt_count else '', 'zookeeper_service': self.ZOOKEEPER_SERVICE_NAME if zk_count else '' } } hive_confs = { 'HIVE': { 'hive_metastore_database_type': 'postgresql', 'hive_metastore_database_host': self.pu.get_manager(cluster).internal_ip, 'hive_metastore_database_port': '7432', 'hive_metastore_database_password': dh.get_hive_db_password(cluster) } } hue_confs = { 'HUE': { 'hue_webhdfs': self.pu.get_role_name( self.pu.get_namenode(cluster), 'NAMENODE') } } sentry_confs = { 'SENTRY': { 'sentry_server_database_type': 'postgresql', 'sentry_server_database_host': self.pu.get_manager(cluster).internal_ip, 'sentry_server_database_port': '7432', 'sentry_server_database_password': dh.get_sentry_db_password(cluster) } } kafka_confs = { 'KAFKA': { 'zookeeper_service': self.ZOOKEEPER_SERVICE_NAME } } all_confs = utils.merge_configs(all_confs, hue_confs) all_confs = utils.merge_configs(all_confs, hive_confs) all_confs = utils.merge_configs(all_confs, sentry_confs) all_confs = utils.merge_configs(all_confs, kafka_confs) all_confs = utils.merge_configs(all_confs, cluster.cluster_configs) if instance: snt_count = self.validator.get_inst_count(instance.cluster, 'SENTRY_SERVER') paths = instance.storage_paths() instance_default_confs = { 'NAMENODE': { 'dfs_name_dir_list': get_hadoop_dirs(paths, '/fs/nn') }, 'SECONDARYNAMENODE': { 'fs_checkpoint_dir_list': get_hadoop_dirs(paths, '/fs/snn') }, 'DATANODE': { 'dfs_data_dir_list': get_hadoop_dirs(paths, '/fs/dn'), 'dfs_datanode_data_dir_perm': 755, 'dfs_datanode_handler_count': 30 }, 'NODEMANAGER': { 'yarn_nodemanager_local_dirs': get_hadoop_dirs(paths, '/yarn/local'), 'container_executor_allowed_system_users': "nobody,impala,hive,llama,hdfs,yarn,mapred," "spark,oozie", "container_executor_banned_users": "bin" }, 'SERVER': { 'maxSessionTimeout': 60000 }, 'HIVESERVER2': { 'hiveserver2_enable_impersonation': 'false' if snt_count else 'true', 'hive_hs2_config_safety_valve': ( self.c_helper.HIVE_SERVER2_SENTRY_SAFETY_VALVE if snt_count else '') }, 'HIVEMETASTORE': { 'hive_metastore_config_safety_valve': ( self.c_helper.HIVE_METASTORE_SENTRY_SAFETY_VALVE if snt_count else '') } } self._load_version_specific_instance_configs( instance, instance_default_confs) ng_user_confs = self.pu.convert_process_configs( instance.node_group.node_configs) all_confs = utils.merge_configs(all_confs, ng_user_confs) all_confs = utils.merge_configs(all_confs, instance_default_confs) return all_confs.get(service, {})