def _get_configs(self, service, cluster=None, instance=None): def get_hadoop_dirs(mount_points, suffix): return ','.join([x + suffix for x in mount_points]) all_confs = {} if cluster: zk_count = v._get_inst_count(cluster, 'ZOOKEEPER_SERVER') core_site_safety_valve = '' if self.pu.c_helper.is_swift_enabled(cluster): configs = swift_helper.get_swift_configs() confs = {c['name']: c['value'] for c in configs} core_site_safety_valve = xmlutils.create_elements_xml(confs) all_confs = { 'HDFS': { 'zookeeper_service': self.ZOOKEEPER_SERVICE_NAME if zk_count else '', 'core_site_safety_valve': core_site_safety_valve }, 'HIVE': { 'mapreduce_yarn_service': self.YARN_SERVICE_NAME, 'zookeeper_service': self.ZOOKEEPER_SERVICE_NAME if zk_count else '' }, 'OOZIE': { 'mapreduce_yarn_service': self.YARN_SERVICE_NAME, 'zookeeper_service': self.ZOOKEEPER_SERVICE_NAME if zk_count else '' }, 'YARN': { 'hdfs_service': self.HDFS_SERVICE_NAME, 'zookeeper_service': self.ZOOKEEPER_SERVICE_NAME if zk_count else '' }, 'HUE': { 'hive_service': self.HIVE_SERVICE_NAME, 'oozie_service': self.OOZIE_SERVICE_NAME, 'zookeeper_service': self.ZOOKEEPER_SERVICE_NAME if zk_count else '' }, 'SPARK_ON_YARN': { 'yarn_service': self.YARN_SERVICE_NAME }, 'HBASE': { 'hdfs_service': self.HDFS_SERVICE_NAME, 'zookeeper_service': self.ZOOKEEPER_SERVICE_NAME } } hive_confs = { 'HIVE': { 'hive_metastore_database_type': 'postgresql', 'hive_metastore_database_host': self.pu.get_manager(cluster).internal_ip, 'hive_metastore_database_port': '7432', 'hive_metastore_database_password': self.pu.db_helper.get_hive_db_password(cluster) } } hue_confs = { 'HUE': { 'hue_webhdfs': self.pu.get_role_name(self.pu.get_namenode(cluster), 'NAMENODE') } } all_confs = s_cfg.merge_configs(all_confs, hue_confs) all_confs = s_cfg.merge_configs(all_confs, hive_confs) all_confs = s_cfg.merge_configs(all_confs, cluster.cluster_configs) if instance: paths = instance.storage_paths() instance_default_confs = { 'NAMENODE': { 'dfs_name_dir_list': get_hadoop_dirs(paths, '/fs/nn') }, 'SECONDARYNAMENODE': { 'fs_checkpoint_dir_list': get_hadoop_dirs(paths, '/fs/snn') }, 'DATANODE': { 'dfs_data_dir_list': get_hadoop_dirs(paths, '/fs/dn'), 'dfs_datanode_data_dir_perm': 755, 'dfs_datanode_handler_count': 30 }, 'NODEMANAGER': { 'yarn_nodemanager_local_dirs': get_hadoop_dirs(paths, '/yarn/local') }, 'SERVER': { 'maxSessionTimeout': 60000 } } ng_user_confs = self.pu.convert_process_configs( instance.node_group.node_configs) all_confs = s_cfg.merge_configs(all_confs, ng_user_confs) all_confs = s_cfg.merge_configs(all_confs, instance_default_confs) return all_confs.get(service, {})
def _get_configs(self, service, cluster=None, node_group=None): def get_hadoop_dirs(mount_points, suffix): return ','.join([x + suffix for x in mount_points]) all_confs = {} if cluster: zk_count = v._get_inst_count(cluster, 'ZOOKEEPER_SERVER') hbm_count = v._get_inst_count(cluster, 'HBASE_MASTER') snt_count = v._get_inst_count(cluster, 'SENTRY_SERVER') ks_count = v._get_inst_count(cluster, 'KEY_VALUE_STORE_INDEXER') imp_count = v._get_inst_count(cluster, 'IMPALA_CATALOGSERVER') hive_count = v._get_inst_count(cluster, 'HIVE_METASTORE') slr_count = v._get_inst_count(cluster, 'SOLR_SERVER') sqp_count = v._get_inst_count(cluster, 'SQOOP_SERVER') core_site_safety_valve = '' if self.pu.c_helper.is_swift_enabled(cluster): configs = swift_helper.get_swift_configs() confs = {c['name']: c['value'] for c in configs} core_site_safety_valve = xmlutils.create_elements_xml(confs) all_confs = { 'HDFS': { 'zookeeper_service': self.ZOOKEEPER_SERVICE_NAME if zk_count else '', 'dfs_block_local_path_access_user': '******' if imp_count else '', 'core_site_safety_valve': core_site_safety_valve }, 'HIVE': { 'mapreduce_yarn_service': self.YARN_SERVICE_NAME, 'sentry_service': self.SENTRY_SERVICE_NAME if snt_count else '', 'zookeeper_service': self.ZOOKEEPER_SERVICE_NAME if zk_count else '' }, 'OOZIE': { 'mapreduce_yarn_service': self.YARN_SERVICE_NAME, 'hive_service': self.HIVE_SERVICE_NAME if hive_count else '', 'zookeeper_service': self.ZOOKEEPER_SERVICE_NAME if zk_count else '' }, 'YARN': { 'hdfs_service': self.HDFS_SERVICE_NAME, 'zookeeper_service': self.ZOOKEEPER_SERVICE_NAME if zk_count else '' }, 'HUE': { 'hive_service': self.HIVE_SERVICE_NAME, 'oozie_service': self.OOZIE_SERVICE_NAME, 'sentry_service': self.SENTRY_SERVICE_NAME if snt_count else '', 'solr_service': self.SOLR_SERVICE_NAME if slr_count else '', 'zookeeper_service': self.ZOOKEEPER_SERVICE_NAME if zk_count else '', 'hbase_service': self.HBASE_SERVICE_NAME if hbm_count else '', 'impala_service': self.IMPALA_SERVICE_NAME if imp_count else '', 'sqoop_service': self.SQOOP_SERVICE_NAME if sqp_count else '' }, 'SPARK_ON_YARN': { 'yarn_service': self.YARN_SERVICE_NAME }, 'HBASE': { 'hdfs_service': self.HDFS_SERVICE_NAME, 'zookeeper_service': self.ZOOKEEPER_SERVICE_NAME, 'hbase_enable_indexing': 'true' if ks_count else 'false', 'hbase_enable_replication': 'true' if ks_count else 'false' }, 'FLUME': { 'hdfs_service': self.HDFS_SERVICE_NAME, 'solr_service': self.SOLR_SERVICE_NAME if slr_count else '', 'hbase_service': self.HBASE_SERVICE_NAME if hbm_count else '' }, 'SENTRY': { 'hdfs_service': self.HDFS_SERVICE_NAME, 'sentry_server_config_safety_valve': ( c_helper.SENTRY_IMPALA_CLIENT_SAFETY_VALVE if imp_count else '') }, 'SOLR': { 'hdfs_service': self.HDFS_SERVICE_NAME, 'zookeeper_service': self.ZOOKEEPER_SERVICE_NAME }, 'SQOOP': { 'mapreduce_yarn_service': self.YARN_SERVICE_NAME }, 'KS_INDEXER': { 'hbase_service': self.HBASE_SERVICE_NAME, 'solr_service': self.SOLR_SERVICE_NAME }, 'IMPALA': { 'hdfs_service': self.HDFS_SERVICE_NAME, 'hbase_service': self.HBASE_SERVICE_NAME if hbm_count else '', 'hive_service': self.HIVE_SERVICE_NAME, 'sentry_service': self.SENTRY_SERVICE_NAME if snt_count else '', 'zookeeper_service': self.ZOOKEEPER_SERVICE_NAME if zk_count else '' } } hive_confs = { 'HIVE': { 'hive_metastore_database_type': 'postgresql', 'hive_metastore_database_host': self.pu.get_manager(cluster).internal_ip, 'hive_metastore_database_port': '7432', 'hive_metastore_database_password': self.pu.db_helper.get_hive_db_password(cluster) } } hue_confs = { 'HUE': { 'hue_webhdfs': self.pu.get_role_name( self.pu.get_namenode(cluster), 'NAMENODE') } } sentry_confs = { 'SENTRY': { 'sentry_server_database_type': 'postgresql', 'sentry_server_database_host': self.pu.get_manager(cluster).internal_ip, 'sentry_server_database_port': '7432', 'sentry_server_database_password': self.pu.db_helper.get_sentry_db_password(cluster) } } all_confs = _merge_dicts(all_confs, hue_confs) all_confs = _merge_dicts(all_confs, hive_confs) all_confs = _merge_dicts(all_confs, sentry_confs) all_confs = _merge_dicts(all_confs, cluster.cluster_configs) if node_group: paths = node_group.storage_paths() ng_default_confs = { 'NAMENODE': { 'dfs_name_dir_list': get_hadoop_dirs(paths, '/fs/nn') }, 'SECONDARYNAMENODE': { 'fs_checkpoint_dir_list': get_hadoop_dirs(paths, '/fs/snn') }, 'DATANODE': { 'dfs_data_dir_list': get_hadoop_dirs(paths, '/fs/dn'), 'dfs_datanode_data_dir_perm': 755, 'dfs_datanode_handler_count': 30 }, 'NODEMANAGER': { 'yarn_nodemanager_local_dirs': get_hadoop_dirs(paths, '/yarn/local') }, 'SERVER': { 'maxSessionTimeout': 60000 }, 'HIVESERVER2': { 'hiveserver2_enable_impersonation': 'false' if snt_count else 'true', 'hive_hs2_config_safety_valve': ( c_helper.HIVE_SERVER2_SENTRY_SAFETY_VALVE if snt_count else '') }, 'HIVEMETASTORE': { 'hive_metastore_config_safety_valve': ( c_helper.HIVE_METASTORE_SENTRY_SAFETY_VALVE if snt_count else '') } } ng_user_confs = self.pu.convert_process_configs( node_group.node_configs) all_confs = _merge_dicts(all_confs, ng_user_confs) all_confs = _merge_dicts(all_confs, ng_default_confs) return all_confs.get(service, {})
def _get_configs(self, service, cluster=None, node_group=None): def get_hadoop_dirs(mount_points, suffix): return ','.join([x + suffix for x in mount_points]) all_confs = {} if cluster: zk_count = v._get_inst_count(cluster, 'ZOOKEEPER_SERVER') core_site_safety_valve = '' if self.pu.c_helper.is_swift_enabled(cluster): configs = swift_helper.get_swift_configs() confs = dict((c['name'], c['value']) for c in configs) core_site_safety_valve = xmlutils.create_elements_xml(confs) all_confs = { 'HDFS': { 'zookeeper_service': self.ZOOKEEPER_SERVICE_NAME if zk_count else '', 'core_site_safety_valve': core_site_safety_valve }, 'HIVE': { 'mapreduce_yarn_service': self.YARN_SERVICE_NAME, 'zookeeper_service': self.ZOOKEEPER_SERVICE_NAME if zk_count else '' }, 'OOZIE': { 'mapreduce_yarn_service': self.YARN_SERVICE_NAME, 'zookeeper_service': self.ZOOKEEPER_SERVICE_NAME if zk_count else '' }, 'YARN': { 'hdfs_service': self.HDFS_SERVICE_NAME, 'zookeeper_service': self.ZOOKEEPER_SERVICE_NAME if zk_count else '' }, 'HUE': { 'hive_service': self.HIVE_SERVICE_NAME, 'oozie_service': self.OOZIE_SERVICE_NAME, 'zookeeper_service': self.ZOOKEEPER_SERVICE_NAME if zk_count else '' }, 'SPARK_ON_YARN': { 'yarn_service': self.YARN_SERVICE_NAME }, 'HBASE': { 'hdfs_service': self.HDFS_SERVICE_NAME, 'zookeeper_service': self.ZOOKEEPER_SERVICE_NAME } } hive_confs = { 'HIVE': { 'hive_metastore_database_type': 'postgresql', 'hive_metastore_database_host': self.pu.get_manager(cluster).internal_ip, 'hive_metastore_database_port': '7432', 'hive_metastore_database_password': self.pu.db_helper.get_hive_db_password(cluster) } } hue_confs = { 'HUE': { 'hue_webhdfs': self.pu.get_role_name(self.pu.get_namenode(cluster), 'NAMENODE') } } all_confs = _merge_dicts(all_confs, hue_confs) all_confs = _merge_dicts(all_confs, hive_confs) all_confs = _merge_dicts(all_confs, cluster.cluster_configs) if node_group: paths = node_group.storage_paths() ng_default_confs = { 'NAMENODE': { 'dfs_name_dir_list': get_hadoop_dirs(paths, '/fs/nn') }, 'SECONDARYNAMENODE': { 'fs_checkpoint_dir_list': get_hadoop_dirs(paths, '/fs/snn') }, 'DATANODE': { 'dfs_data_dir_list': get_hadoop_dirs(paths, '/fs/dn'), 'dfs_datanode_data_dir_perm': 755, 'dfs_datanode_handler_count': 30 }, 'NODEMANAGER': { 'yarn_nodemanager_local_dirs': get_hadoop_dirs(paths, '/yarn/local') }, 'SERVER': { 'maxSessionTimeout': 60000 } } ng_user_confs = self.pu.convert_process_configs( node_group.node_configs) all_confs = _merge_dicts(all_confs, ng_user_confs) all_confs = _merge_dicts(all_confs, ng_default_confs) return all_confs.get(service, {})
def _get_configs(self, service, cluster=None, instance=None): def get_hadoop_dirs(mount_points, suffix): return ','.join([x + suffix for x in mount_points]) all_confs = {} if cluster: zk_count = self.validator._get_inst_count(cluster, 'ZOOKEEPER_SERVER') hbm_count = self.validator._get_inst_count(cluster, 'HBASE_MASTER') snt_count = self.validator._get_inst_count(cluster, 'SENTRY_SERVER') ks_count =\ self.validator._get_inst_count(cluster, 'KEY_VALUE_STORE_INDEXER') kms_count = self.validator._get_inst_count(cluster, 'KMS') imp_count =\ self.validator._get_inst_count(cluster, 'IMPALA_CATALOGSERVER') hive_count = self.validator._get_inst_count( cluster, 'HIVE_METASTORE') slr_count = self.validator._get_inst_count(cluster, 'SOLR_SERVER') sqp_count = self.validator._get_inst_count(cluster, 'SQOOP_SERVER') core_site_safety_valve = '' if self.pu.c_helper.is_swift_enabled(cluster): configs = swift_helper.get_swift_configs() confs = {c['name']: c['value'] for c in configs} core_site_safety_valve = xmlutils.create_elements_xml(confs) all_confs = { 'HDFS': { 'zookeeper_service': self.ZOOKEEPER_SERVICE_NAME if zk_count else '', 'dfs_block_local_path_access_user': '******' if imp_count else '', 'kms_service': self.KMS_SERVICE_NAME if kms_count else '', 'core_site_safety_valve': core_site_safety_valve }, 'HIVE': { 'mapreduce_yarn_service': self.YARN_SERVICE_NAME, 'sentry_service': self.SENTRY_SERVICE_NAME if snt_count else '', 'zookeeper_service': self.ZOOKEEPER_SERVICE_NAME if zk_count else '' }, 'OOZIE': { 'mapreduce_yarn_service': self.YARN_SERVICE_NAME, 'hive_service': self.HIVE_SERVICE_NAME if hive_count else '', 'zookeeper_service': self.ZOOKEEPER_SERVICE_NAME if zk_count else '' }, 'YARN': { 'hdfs_service': self.HDFS_SERVICE_NAME, 'zookeeper_service': self.ZOOKEEPER_SERVICE_NAME if zk_count else '' }, 'HUE': { 'hive_service': self.HIVE_SERVICE_NAME, 'oozie_service': self.OOZIE_SERVICE_NAME, 'sentry_service': self.SENTRY_SERVICE_NAME if snt_count else '', 'solr_service': self.SOLR_SERVICE_NAME if slr_count else '', 'zookeeper_service': self.ZOOKEEPER_SERVICE_NAME if zk_count else '', 'hbase_service': self.HBASE_SERVICE_NAME if hbm_count else '', 'impala_service': self.IMPALA_SERVICE_NAME if imp_count else '', 'sqoop_service': self.SQOOP_SERVICE_NAME if sqp_count else '' }, 'SPARK_ON_YARN': { 'yarn_service': self.YARN_SERVICE_NAME }, 'HBASE': { 'hdfs_service': self.HDFS_SERVICE_NAME, 'zookeeper_service': self.ZOOKEEPER_SERVICE_NAME, 'hbase_enable_indexing': 'true' if ks_count else 'false', 'hbase_enable_replication': 'true' if ks_count else 'false' }, 'FLUME': { 'hdfs_service': self.HDFS_SERVICE_NAME, 'solr_service': self.SOLR_SERVICE_NAME if slr_count else '', 'hbase_service': self.HBASE_SERVICE_NAME if hbm_count else '' }, 'SENTRY': { 'hdfs_service': self.HDFS_SERVICE_NAME, 'sentry_server_config_safety_valve': (c_helper.SENTRY_IMPALA_CLIENT_SAFETY_VALVE if imp_count else '') }, 'SOLR': { 'hdfs_service': self.HDFS_SERVICE_NAME, 'zookeeper_service': self.ZOOKEEPER_SERVICE_NAME }, 'SQOOP': { 'mapreduce_yarn_service': self.YARN_SERVICE_NAME }, 'KS_INDEXER': { 'hbase_service': self.HBASE_SERVICE_NAME, 'solr_service': self.SOLR_SERVICE_NAME }, 'IMPALA': { 'hdfs_service': self.HDFS_SERVICE_NAME, 'hbase_service': self.HBASE_SERVICE_NAME if hbm_count else '', 'hive_service': self.HIVE_SERVICE_NAME, 'sentry_service': self.SENTRY_SERVICE_NAME if snt_count else '', 'zookeeper_service': self.ZOOKEEPER_SERVICE_NAME if zk_count else '' } } hive_confs = { 'HIVE': { 'hive_metastore_database_type': 'postgresql', 'hive_metastore_database_host': self.pu.get_manager(cluster).internal_ip, 'hive_metastore_database_port': '7432', 'hive_metastore_database_password': self.pu.db_helper.get_hive_db_password(cluster) } } hue_confs = { 'HUE': { 'hue_webhdfs': self.pu.get_role_name(self.pu.get_namenode(cluster), 'NAMENODE') } } sentry_confs = { 'SENTRY': { 'sentry_server_database_type': 'postgresql', 'sentry_server_database_host': self.pu.get_manager(cluster).internal_ip, 'sentry_server_database_port': '7432', 'sentry_server_database_password': self.pu.db_helper.get_sentry_db_password(cluster) } } all_confs = s_cfg.merge_configs(all_confs, hue_confs) all_confs = s_cfg.merge_configs(all_confs, hive_confs) all_confs = s_cfg.merge_configs(all_confs, sentry_confs) all_confs = s_cfg.merge_configs(all_confs, cluster.cluster_configs) if instance: snt_count = self.validator._get_inst_count(instance.cluster, 'SENTRY_SERVER') paths = instance.storage_paths() instance_default_confs = { 'NAMENODE': { 'dfs_name_dir_list': get_hadoop_dirs(paths, '/fs/nn') }, 'SECONDARYNAMENODE': { 'fs_checkpoint_dir_list': get_hadoop_dirs(paths, '/fs/snn') }, 'DATANODE': { 'dfs_data_dir_list': get_hadoop_dirs(paths, '/fs/dn'), 'dfs_datanode_data_dir_perm': 755, 'dfs_datanode_handler_count': 30 }, 'NODEMANAGER': { 'yarn_nodemanager_local_dirs': get_hadoop_dirs(paths, '/yarn/local') }, 'SERVER': { 'maxSessionTimeout': 60000 }, 'HIVESERVER2': { 'hiveserver2_enable_impersonation': 'false' if snt_count else 'true', 'hive_hs2_config_safety_valve': (c_helper.HIVE_SERVER2_SENTRY_SAFETY_VALVE if snt_count else '') }, 'HIVEMETASTORE': { 'hive_metastore_config_safety_valve': (c_helper.HIVE_METASTORE_SENTRY_SAFETY_VALVE if snt_count else '') } } ng_user_confs = self.pu.convert_process_configs( instance.node_group.node_configs) all_confs = s_cfg.merge_configs(all_confs, ng_user_confs) all_confs = s_cfg.merge_configs(all_confs, instance_default_confs) return all_confs.get(service, {})
def _get_configs(self, service, cluster=None, instance=None): def get_hadoop_dirs(mount_points, suffix): return ",".join([x + suffix for x in mount_points]) all_confs = {} if cluster: zk_count = self.validator._get_inst_count(cluster, "ZOOKEEPER_SERVER") core_site_safety_valve = "" if self.pu.c_helper.is_swift_enabled(cluster): configs = swift_helper.get_swift_configs() confs = {c["name"]: c["value"] for c in configs} core_site_safety_valve = xmlutils.create_elements_xml(confs) all_confs = { "HDFS": { "zookeeper_service": self.ZOOKEEPER_SERVICE_NAME if zk_count else "", "core_site_safety_valve": core_site_safety_valve, }, "HIVE": { "mapreduce_yarn_service": self.YARN_SERVICE_NAME, "zookeeper_service": self.ZOOKEEPER_SERVICE_NAME if zk_count else "", }, "OOZIE": { "mapreduce_yarn_service": self.YARN_SERVICE_NAME, "zookeeper_service": self.ZOOKEEPER_SERVICE_NAME if zk_count else "", }, "YARN": { "hdfs_service": self.HDFS_SERVICE_NAME, "zookeeper_service": self.ZOOKEEPER_SERVICE_NAME if zk_count else "", }, "HUE": { "hive_service": self.HIVE_SERVICE_NAME, "oozie_service": self.OOZIE_SERVICE_NAME, "zookeeper_service": self.ZOOKEEPER_SERVICE_NAME if zk_count else "", }, "SPARK_ON_YARN": {"yarn_service": self.YARN_SERVICE_NAME}, "HBASE": {"hdfs_service": self.HDFS_SERVICE_NAME, "zookeeper_service": self.ZOOKEEPER_SERVICE_NAME}, } hive_confs = { "HIVE": { "hive_metastore_database_type": "postgresql", "hive_metastore_database_host": self.pu.get_manager(cluster).internal_ip, "hive_metastore_database_port": "7432", "hive_metastore_database_password": self.pu.db_helper.get_hive_db_password(cluster), } } hue_confs = {"HUE": {"hue_webhdfs": self.pu.get_role_name(self.pu.get_namenode(cluster), "NAMENODE")}} all_confs = s_cfg.merge_configs(all_confs, hue_confs) all_confs = s_cfg.merge_configs(all_confs, hive_confs) all_confs = s_cfg.merge_configs(all_confs, cluster.cluster_configs) if instance: paths = instance.storage_paths() instance_default_confs = { "NAMENODE": {"dfs_name_dir_list": get_hadoop_dirs(paths, "/fs/nn")}, "SECONDARYNAMENODE": {"fs_checkpoint_dir_list": get_hadoop_dirs(paths, "/fs/snn")}, "DATANODE": { "dfs_data_dir_list": get_hadoop_dirs(paths, "/fs/dn"), "dfs_datanode_data_dir_perm": 755, "dfs_datanode_handler_count": 30, }, "NODEMANAGER": {"yarn_nodemanager_local_dirs": get_hadoop_dirs(paths, "/yarn/local")}, "SERVER": {"maxSessionTimeout": 60000}, } ng_user_confs = self.pu.convert_process_configs(instance.node_group.node_configs) all_confs = s_cfg.merge_configs(all_confs, ng_user_confs) all_confs = s_cfg.merge_configs(all_confs, instance_default_confs) return all_confs.get(service, {})
def create_elements_xml(configs, **kwargs): return xmlutils.create_elements_xml(configs)