namenode_host = default("/clusterHostInfo/namenode_host", []) hbase_master_hosts = default("/clusterHostInfo/hbase_master_hosts", []) oozie_servers = default("/clusterHostInfo/oozie_server", []) falcon_server_hosts = default("/clusterHostInfo/falcon_server_hosts", []) ranger_admin_hosts = default("/clusterHostInfo/ranger_admin_hosts", []) has_namenode = not len(namenode_host) == 0 has_ganglia_server = not len(ganglia_server_hosts) == 0 has_tez = 'tez-site' in config['configurations'] has_hbase_masters = not len(hbase_master_hosts) == 0 has_oozie_server = not len(oozie_servers) == 0 has_falcon_server_hosts = not len(falcon_server_hosts) == 0 has_ranger_admin = not len(ranger_admin_hosts) == 0 if has_namenode: hadoop_conf_dir = conf_select.get_hadoop_conf_dir(force_latest_on_upgrade=True) hbase_tmp_dir = "/tmp/hbase-hbase" proxyuser_group = default("/configurations/hadoop-env/proxyuser_group","users") ranger_group = config['configurations']['ranger-env']['ranger_group'] dfs_cluster_administrators_group = config['configurations']['hdfs-site']["dfs.cluster.administrators"] ignore_groupsusers_create = default("/configurations/cluster-env/ignore_groupsusers_create", False) smoke_user_dirs = format("/tmp/hadoop-{smoke_user},/tmp/hsperfdata_{smoke_user},/home/{smoke_user},/tmp/{smoke_user},/tmp/sqoop-{smoke_user}") if has_hbase_masters: hbase_user_dirs = format("/home/{hbase_user},/tmp/{hbase_user},/usr/bin/{hbase_user},/var/log/{hbase_user},{hbase_tmp_dir}") #repo params repo_info = config['hostLevelParams']['repo_info'] service_repo_info = default("/hostLevelParams/service_repo_info",None)
credential_file = format('/etc/ranger/{repo_name}/cred.jceks') if xml_configurations_supported else None #For SQLA explicitly disable audit to DB for Ranger if xa_audit_db_flavor == 'sqla': xa_audit_db_is_enabled = False namenode_hosts = default("/clusterHostInfo/namenode_host", []) has_namenode = not len(namenode_hosts) == 0 hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] if has_namenode else None hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab'] if has_namenode else None hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name'] if has_namenode else None hdfs_site = config['configurations']['hdfs-site'] if has_namenode else None default_fs = config['configurations']['core-site']['fs.defaultFS'] if has_namenode else None hadoop_bin_dir = hdp_select.get_hadoop_dir("bin") if has_namenode else None hadoop_conf_dir = conf_select.get_hadoop_conf_dir() if has_namenode else None kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None)) import functools #create partial functions with common arguments for every HdfsResource call #to create/delete hdfs directory/file/copyfromlocal we need to call params.HdfsResource in code HdfsResource = functools.partial( HdfsResource, user=hdfs_user, security_enabled = security_enabled, keytab = hdfs_user_keytab, kinit_path_local = kinit_path_local, hadoop_bin_dir = hadoop_bin_dir, hadoop_conf_dir = hadoop_conf_dir, principal_name = hdfs_principal_name, hdfs_site = hdfs_site,
from params_windows import * else: from params_linux import * host_sys_prepped = default("/hostLevelParams/host_sys_prepped", False) # server configurations config = Script.get_config() stack_name = default("/hostLevelParams/stack_name", None) # New Cluster Stack Version that is defined during the RESTART of a Rolling Upgrade version = default("/commandParams/version", None) stack_version_unformatted = str(config['hostLevelParams']['stack_version']) hdp_stack_version = format_hdp_stack_version(stack_version_unformatted) #hadoop params hadoop_conf_dir = conf_select.get_hadoop_conf_dir() smokeuser = config['configurations']['cluster-env']['smokeuser'] smokeuser_principal = config['configurations']['cluster-env']['smokeuser_principal_name'] security_enabled = config['configurations']['cluster-env']['security_enabled'] smokeuser_keytab = config['configurations']['cluster-env']['smokeuser_keytab'] kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None)) slider_env_sh_template = config['configurations']['slider-env']['content'] java64_home = config['hostLevelParams']['java_home'] log4j_props = config['configurations']['slider-log4j']['content'] slider_cmd = format("{slider_bin_dir}/slider")
else: daemon_name = 'mysqld' # Security related/required params hostname = config['hostname'] security_enabled = config['configurations']['cluster-env'][ 'security_enabled'] kinit_path_local = get_kinit_path( default('/configurations/kerberos-env/executable_search_paths', None)) tmp_dir = Script.get_tmp_dir() hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] hive_user = config['configurations']['hive-env']['hive_user'] webhcat_user = config['configurations']['hive-env']['webhcat_user'] # default configuration directories hadoop_conf_dir = conf_select.get_hadoop_conf_dir() #hadoop_bin_dir = stack_select.get_hadoop_dir("bin") hadoop_bin_dir = '/usr/lib/hadoop/bin' hive_etc_dir_prefix = "/etc/hive" hive_interactive_etc_dir_prefix = "/etc/hive2" hive_server_conf_dir = "/etc/hive/conf/conf.server" hive_server_interactive_conf_dir = "/etc/hive2/conf/conf.server" webhcat_conf_dir = format("/etc/webhcat/conf") hive_home_dir = format("/usr/lib/hive") hive_conf_dir = format("/etc/hive/conf") hive_client_conf_dir = format("/etc/hive/conf") if check_stack_feature(StackFeature.CONFIG_VERSIONING, stack_version_formatted_major):
xa_audit_hdfs_is_enabled = config['configurations']['ranger-knox-audit']['xasecure.audit.destination.hdfs'] if xml_configurations_supported else None ssl_keystore_password = unicode(config['configurations']['ranger-knox-policymgr-ssl']['xasecure.policymgr.clientssl.keystore.password']) if xml_configurations_supported else None ssl_truststore_password = unicode(config['configurations']['ranger-knox-policymgr-ssl']['xasecure.policymgr.clientssl.truststore.password']) if xml_configurations_supported else None credential_file = format('/etc/ranger/{repo_name}/cred.jceks') if xml_configurations_supported else None #For SQLA explicitly disable audit to DB for Ranger if xa_audit_db_flavor == 'sqla': xa_audit_db_is_enabled = False hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] if has_namenode else None hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab'] if has_namenode else None hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name'] if has_namenode else None hdfs_site = config['configurations']['hdfs-site'] if has_namenode else None default_fs = config['configurations']['core-site']['fs.defaultFS'] if has_namenode else None hadoop_bin_dir = hdp_select.get_hadoop_dir("bin") if has_namenode else None hadoop_conf_dir = conf_select.get_hadoop_conf_dir() if has_namenode else None import functools #create partial functions with common arguments for every HdfsResource call #to create/delete hdfs directory/file/copyfromlocal we need to call params.HdfsResource in code HdfsResource = functools.partial( HdfsResource, user=hdfs_user, hdfs_resource_ignore_file = "/var/lib/ambari-agent/data/.hdfs_resource_ignore", security_enabled = security_enabled, keytab = hdfs_user_keytab, kinit_path_local = kinit_path_local, hadoop_bin_dir = hadoop_bin_dir, hadoop_conf_dir = hadoop_conf_dir, principal_name = hdfs_principal_name, hdfs_site = hdfs_site,
"/configurations/hadoop-env/namenode_opt_maxpermsize", "256m") jtnode_opt_newsize = "200m" jtnode_opt_maxnewsize = "200m" jtnode_heapsize = "1024m" ttnode_heapsize = "1024m" dtnode_heapsize = config['configurations']['hadoop-env']['dtnode_heapsize'] mapred_pid_dir_prefix = default( "/configurations/mapred-env/mapred_pid_dir_prefix", "/var/run/hadoop-mapreduce") mapred_log_dir_prefix = default( "/configurations/mapred-env/mapred_log_dir_prefix", "/var/log/hadoop-mapreduce") #users and groups hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] user_group = config['configurations']['cluster-env']['user_group'] namenode_host = default("/clusterHostInfo/namenode_host", []) has_namenode = not len(namenode_host) == 0 if has_namenode or dfs_type == 'HCFS': hadoop_conf_dir = conf_select.get_hadoop_conf_dir( force_latest_on_upgrade=True) link_configs_lock_file = os.path.join(tmp_dir, "link_configs_lock_file") hdp_select_lock_file = os.path.join(tmp_dir, "hdp_select_lock_file") upgrade_suspended = default("/roleParams/upgrade_suspended", False)