config = Script.get_config() tmp_dir = Script.get_tmp_dir() stack_name = status_params.stack_name stack_root = Script.get_stack_root() stack_version_unformatted = config['hostLevelParams']['stack_version'] stack_version_formatted = format_stack_version(stack_version_unformatted) sysprep_skip_copy_tarballs_hdfs = get_sysprep_skip_copy_tarballs_hdfs() # New Cluster Stack Version that is defined during the RESTART of a Stack Upgrade version = default("/commandParams/version", None) spark_conf = '/etc/spark2/conf' hadoop_conf_dir = conf_select.get_hadoop_conf_dir() hadoop_bin_dir = stack_select.get_hadoop_dir("bin") if stack_version_formatted and check_stack_feature( StackFeature.ROLLING_UPGRADE, stack_version_formatted): hadoop_home = stack_select.get_hadoop_dir("home") spark_conf = format("{stack_root}/current/{component_directory}/conf") spark_log_dir = config['configurations']['spark2-env']['spark_log_dir'] spark_pid_dir = status_params.spark_pid_dir spark_home = format("{stack_root}/current/{component_directory}") spark_daemon_memory = config['configurations']['spark2-env'][ 'spark_daemon_memory'] spark_thrift_server_conf_file = spark_conf + "/spark-thrift-sparkconf.conf" java_home = config['hostLevelParams']['java_home'] hdfs_user = config['configurations']['hadoop-env']['hdfs_user']
dfs_dn_http_addr = default( '/configurations/hdfs-site/dfs.datanode.http.address', None) dfs_dn_https_addr = default( '/configurations/hdfs-site/dfs.datanode.https.address', None) dfs_http_policy = default('/configurations/hdfs-site/dfs.http.policy', None) dfs_dn_ipc_address = config['configurations']['hdfs-site'][ 'dfs.datanode.ipc.address'] secure_dn_ports_are_in_use = False hdfs_tmp_dir = default("/configurations/hadoop-env/hdfs_tmp_dir", "/tmp") namenode_backup_dir = default("/configurations/hadoop-env/namenode_backup_dir", "/tmp/upgrades") # hadoop default parameters mapreduce_libs_path = "/usr/lib/hadoop-mapreduce/*" hadoop_libexec_dir = stack_select.get_hadoop_dir("libexec") hadoop_bin = stack_select.get_hadoop_dir("sbin") hadoop_bin_dir = stack_select.get_hadoop_dir("bin") hadoop_home = stack_select.get_hadoop_dir("home") hadoop_secure_dn_user = hdfs_user hadoop_conf_dir = conf_select.get_hadoop_conf_dir() hadoop_conf_secure_dir = os.path.join(hadoop_conf_dir, "secure") hadoop_lib_home = stack_select.get_hadoop_dir("lib") # hadoop parameters for stacks that support rolling_upgrade if stack_version_formatted and check_stack_feature( StackFeature.ROLLING_UPGRADE, stack_version_formatted): mapreduce_libs_path = format( "{stack_root}/current/hadoop-mapreduce-client/*") if not security_enabled:
livy_livyserver_protocol = 'https' if stack_version_formatted and check_stack_feature(StackFeature.SPARK_LIVY2, stack_version_formatted) and \ len(livy2_hosts) > 0: livy2_livyserver_host = str(livy2_hosts[0]) livy2_livyserver_port = config['configurations']['livy2-conf'][ 'livy.server.port'] if 'livy.keystore' in config['configurations']['livy2-conf']: livy2_livyserver_protocol = 'https' hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] security_enabled = config['configurations']['cluster-env']['security_enabled'] hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab'] kinit_path_local = get_kinit_path( default('/configurations/kerberos-env/executable_search_paths', None)) hadoop_bin_dir = stack_select.get_hadoop_dir("bin") hadoop_conf_dir = conf_select.get_hadoop_conf_dir() hdfs_principal_name = config['configurations']['hadoop-env'][ 'hdfs_principal_name'] hdfs_site = config['configurations']['hdfs-site'] default_fs = config['configurations']['core-site']['fs.defaultFS'] # create partial functions with common arguments for every HdfsResource call # to create hdfs directory we need to call params.HdfsResource in code HdfsResource = functools.partial( HdfsResource, user=hdfs_user, hdfs_resource_ignore_file= "/var/lib/ambari-agent/data/.hdfs_resource_ignore", security_enabled=security_enabled, keytab=hdfs_user_keytab,
ambari_bare_jaas_principal = get_bare_principal(_ambari_principal_name) jdk_location = config['ambariLevelParams']['jdk_location'] namenode_hosts = default("/clusterHostInfo/namenode_hosts", []) has_namenode = not len(namenode_hosts) == 0 hdfs_user = config['configurations']['hadoop-env'][ 'hdfs_user'] if has_namenode else None hdfs_user_keytab = config['configurations']['hadoop-env'][ 'hdfs_user_keytab'] if has_namenode else None hdfs_principal_name = config['configurations']['hadoop-env'][ 'hdfs_principal_name'] if has_namenode else None hdfs_site = config['configurations']['hdfs-site'] if has_namenode else None default_fs = config['configurations']['core-site'][ 'fs.defaultFS'] if has_namenode else None hadoop_bin_dir = stack_select.get_hadoop_dir("bin") if has_namenode else None hadoop_conf_dir = conf_select.get_hadoop_conf_dir() if has_namenode else None kinit_path_local = get_kinit_path( default('/configurations/kerberos-env/executable_search_paths', None)) dfs_type = default("/clusterLevelParams/dfs_type", "") # params from flink-ambari-config flink_install_dir = "/opt/flink" flink_bin_dir = "/opt/flink" flink_numcontainers = config['configurations']['flink-site'][ 'flink_numcontainers'] flink_numberoftaskslots = config['configurations']['flink-site'][ 'flink_numberoftaskslots'] flink_jobmanager_memory = config['configurations']['flink-site'][ 'flink_jobmanager_memory'] flink_container_memory = config['configurations']['flink-site'][
# New Cluster Stack Version that is defined during the RESTART of a Rolling Upgrade version = default("/commandParams/version", None) stack_name = status_params.stack_name stack_name_uppercase = stack_name.upper() upgrade_direction = default("/commandParams/upgrade_direction", None) agent_stack_retry_on_unavailability = config['hostLevelParams']['agent_stack_retry_on_unavailability'] agent_stack_retry_count = expect("/hostLevelParams/agent_stack_retry_count", int) stack_root = status_params.stack_root stack_version_unformatted = status_params.stack_version_unformatted stack_version_formatted = status_params.stack_version_formatted version_for_stack_feature_checks = get_stack_feature_version(config) hadoop_conf_dir = conf_select.get_hadoop_conf_dir() hadoop_bin_dir = stack_select.get_hadoop_dir("bin") hadoop_lib_home = stack_select.get_hadoop_dir("lib") #spark_conf spark_conf_dir = format("{stack_root}/current/spark-client/conf") #hadoop params if stack_version_formatted and check_stack_feature(StackFeature.ROLLING_UPGRADE,stack_version_formatted): stack_version = None upgrade_stack = stack_select._get_upgrade_stack() if upgrade_stack is not None and len(upgrade_stack) == 2 and upgrade_stack[1] is not None: stack_version = upgrade_stack[1] # oozie-server or oozie-client, depending on role oozie_root = status_params.component_directory
hadoop_ssl_cert_ca_verify = config['configurations']['hue-hadoop-site']['ssl_cert_ca_verify'] hadoop_conf_dir = config['configurations']['hue-hadoop-site']['hadoop_conf_dir'] # [filebrowser] filebrowser_archive_upload_tempdir = config['configurations']['hue-hadoop-site']['filebrowser_archive_upload_tempdir'] filebrowser_show_download_button = config['configurations']['hue-hadoop-site']['filebrowser_show_download_button'] filebrowser_show_upload_button = config['configurations']['hue-hadoop-site']['filebrowser_show_upload_button'] # [jobbrowser] jobbrowser_share_jobs = config['configurations']['hue-hadoop-site']['jobbrowser_share_jobs'] jobbrowser_disable_killing_jobs = config['configurations']['hue-hadoop-site']['jobbrowser_disable_killing_jobs'] jobbrowser_log_offset = config['configurations']['hue-hadoop-site']['jobbrowser_log_offset'] # [jobsub] jobsub_local_data_dir = config['configurations']['hue-hadoop-site']['jobsub_sample_data_dir'] jobsub_sample_data_dir = config['configurations']['hue-hadoop-site']['jobsub_sample_data_dir'] hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] hadoop_bin_dir = stack_select.get_hadoop_dir('bin') hadoop_conf_dir = conf_select.get_hadoop_conf_dir() hdfs_site = config['configurations']['hdfs-site'] default_fs = config['configurations']['core-site']['fs.defaultFS'] dfs_type = default("/commandParams/dfs_type", "") hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab'] hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name'] kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None)) # create partial functions with common arguments for every HdfsResource call # to create hdfs directory we need to call params.HdfsResource in code HdfsResource = functools.partial( HdfsResource, user=hdfs_user, hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', security_enabled=security_enabled, keytab=hdfs_user_keytab,
""" if port is not None: return port < 1024 else: return False # hadoop default params mapreduce_libs_path = "/usr/lib/hadoop-mapreduce/*" # upgrades would cause these directories to have a version instead of "current" # which would cause a lot of problems when writing out hadoop-env.sh; instead # force the use of "current" in the hook hdfs_user_nofile_limit = default( "/configurations/hadoop-env/hdfs_user_nofile_limit", "128000") hadoop_home = stack_select.get_hadoop_dir("home") hadoop_libexec_dir = stack_select.get_hadoop_dir("libexec") hadoop_conf_empty_dir = "/etc/hadoop/conf.empty" hadoop_secure_dn_user = hdfs_user hadoop_dir = "/etc/hadoop" versioned_stack_root = '/usr/bgtp/current' hadoop_java_io_tmpdir = os.path.join(tmp_dir, "hadoop_java_io_tmpdir") datanode_max_locked_memory = config['configurations']['hdfs-site'][ 'dfs.datanode.max.locked.memory'] is_datanode_max_locked_memory_set = not is_empty( config['configurations']['hdfs-site']['dfs.datanode.max.locked.memory']) #hadoop params hdfs_log_dir_prefix = config['configurations']['hadoop-env'][ 'hdfs_log_dir_prefix']
'NODEMANAGER' : 'hadoop-yarn-nodemanager', 'RESOURCEMANAGER' : 'hadoop-yarn-resourcemanager', 'YARN_CLIENT' : 'hadoop-yarn-client' } # server configurations config = Script.get_config() tmp_dir = Script.get_tmp_dir() architecture = get_architecture() stack_name = status_params.stack_name stack_root = Script.get_stack_root() tarball_map = default("/configurations/cluster-env/tarball_map", None) config_path = stack_select.get_hadoop_dir("conf") config_dir = os.path.realpath(config_path) # get the correct version to use for checking stack features version_for_stack_feature_checks = get_stack_feature_version(config) # This is expected to be of the form #.#.#.# stack_version_unformatted = config['clusterLevelParams']['stack_version'] stack_version_formatted_major = format_stack_version(stack_version_unformatted) stack_version_formatted = functions.get_stack_version('hadoop-yarn-resourcemanager') major_stack_version = get_major_version(stack_version_formatted_major) stack_supports_ru = check_stack_feature(StackFeature.ROLLING_UPGRADE, version_for_stack_feature_checks) stack_supports_timeline_state_store = check_stack_feature(StackFeature.TIMELINE_STATE_STORE, version_for_stack_feature_checks) # New Cluster Stack Version that is defined during the RESTART of a Stack Upgrade.
from resource_management.libraries.script.script import Script from resource_management.libraries import functions from resource_management.libraries.functions import conf_select from resource_management.libraries.functions import stack_select from resource_management.libraries.functions import format from resource_management.libraries.functions.default import default config = Script.get_config() tmp_dir = Script.get_tmp_dir() mapred_user = config['configurations']['mapred-env']['mapred_user'] yarn_user = config['configurations']['yarn-env']['yarn_user'] yarn_pid_dir_prefix = config['configurations']['yarn-env']['yarn_pid_dir_prefix'] mapred_pid_dir_prefix = config['configurations']['mapred-env']['mapred_pid_dir_prefix'] yarn_pid_dir = format("{yarn_pid_dir_prefix}/{yarn_user}") mapred_pid_dir = format("{mapred_pid_dir_prefix}/{mapred_user}") resourcemanager_pid_file = format("{yarn_pid_dir}/yarn-{yarn_user}-resourcemanager.pid") nodemanager_pid_file = format("{yarn_pid_dir}/yarn-{yarn_user}-nodemanager.pid") yarn_historyserver_pid_file_old = format("{yarn_pid_dir}/yarn-{yarn_user}-historyserver.pid") yarn_historyserver_pid_file = format("{yarn_pid_dir}/yarn-{yarn_user}-timelineserver.pid") # *-historyserver.pid is deprecated mapred_historyserver_pid_file = format("{mapred_pid_dir}/mapred-{mapred_user}-historyserver.pid") hadoop_home = stack_select.get_hadoop_dir("home") hadoop_conf_dir = conf_select.get_hadoop_conf_dir() hostname = config['agentLevelParams']['hostname'] kinit_path_local = functions.get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None)) security_enabled = config['configurations']['cluster-env']['security_enabled'] stack_name = default("/clusterLevelParams/stack_name", None)
""" Returns True if port is root-owned at *nix systems """ if port is not None: return port < 1024 else: return False # hadoop default params mapreduce_libs_path = "/usr/lib/hadoop-mapreduce/*" # upgrades would cause these directories to have a version instead of "current" # which would cause a lot of problems when writing out hadoop-env.sh; instead # force the use of "current" in the hook hadoop_home = stack_select.get_hadoop_dir("home", force_latest_on_upgrade=True) hadoop_libexec_dir = stack_select.get_hadoop_dir("libexec", force_latest_on_upgrade=True) hadoop_conf_empty_dir = "/etc/hadoop/conf.empty" hadoop_secure_dn_user = hdfs_user hadoop_dir = "/etc/hadoop" versioned_stack_root = '/usr/hdp/current' hadoop_java_io_tmpdir = os.path.join(tmp_dir, "hadoop_java_io_tmpdir") datanode_max_locked_memory = config['configurations']['hdfs-site'][ 'dfs.datanode.max.locked.memory'] is_datanode_max_locked_memory_set = not is_empty( config['configurations']['hdfs-site']['dfs.datanode.max.locked.memory']) # HDP 2.2+ params if Script.is_stack_greater_or_equal("2.2"):
# stack version version = default("/commandParams/version", None) stack_version_unformatted = config['hostLevelParams']['stack_version'] stack_version_formatted = format_stack_version(stack_version_unformatted) has_secure_user_auth = False if stack_version_formatted and \ check_stack_feature(StackFeature.ACCUMULO_KERBEROS_USER_AUTH, stack_version_formatted): has_secure_user_auth = True # configuration directories conf_dir = status_params.conf_dir server_conf_dir = status_params.server_conf_dir # service locations hadoop_prefix = stack_select.get_hadoop_dir("home") hadoop_bin_dir = stack_select.get_hadoop_dir("bin") zookeeper_home = format("{stack_root}/current/zookeeper-client") # the configuration direction for HDFS/YARN/MapR is the hadoop config # directory, which is symlinked by hadoop-client only hadoop_conf_dir = conf_select.get_hadoop_conf_dir() # accumulo local directory structure log_dir = config['configurations']['accumulo-env']['accumulo_log_dir'] client_script = format("{stack_root}/current/accumulo-client/bin/accumulo") daemon_script = format("ACCUMULO_CONF_DIR={server_conf_dir} {client_script}") # user and status accumulo_user = status_params.accumulo_user user_group = config['configurations']['cluster-env']['user_group']
def pre_upgrade_deregister(): """ Runs the "hive --service hiveserver2 --deregister <version>" command to de-provision the server in preparation for an upgrade. This will contact ZooKeeper to remove the server so that clients that attempt to connect will be directed to other servers automatically. Once all clients have drained, the server will shutdown automatically; this process could take a very long time. This function will obtain the Kerberos ticket if security is enabled. :return: """ import params Logger.info( 'HiveServer2 executing "deregister" command in preparation for upgrade...' ) if params.security_enabled: kinit_command = format( "{kinit_path_local} -kt {smoke_user_keytab} {smokeuser_principal}; " ) Execute(kinit_command, user=params.smokeuser) # calculate the current hive server version current_hiveserver_version = _get_current_hiveserver_version() if current_hiveserver_version is None: raise Fail( 'Unable to determine the current HiveServer2 version to deregister.' ) # fallback when upgrading because /usr/iop/current/hive-server2/conf/conf.server may not exist hive_server_conf_dir = params.hive_server_conf_dir if not os.path.exists(hive_server_conf_dir): hive_server_conf_dir = "/etc/hive/conf.server" # deregister hive_execute_path = params.execute_path # If upgrading, the upgrade-target hive binary should be used to call the --deregister command. # If downgrading, the downgrade-source hive binary should be used to call the --deregister command. if "upgrade" == params.upgrade_direction: # hive_bin upgrade_target_version = format_stack_version(params.version) if upgrade_target_version and compare_versions(upgrade_target_version, "4.1.0.0") >= 0: upgrade_target_hive_bin = format('/usr/iop/{version}/hive/bin') if (os.pathsep + params.hive_bin) in hive_execute_path: hive_execute_path = hive_execute_path.replace( os.pathsep + params.hive_bin, os.pathsep + upgrade_target_hive_bin) # hadoop_bin_dir upgrade_target_hadoop_bin = stack_select.get_hadoop_dir( "bin", upgrade_stack_only=True) upgrade_source_hadoop_bin = params.hadoop_bin_dir if upgrade_target_hadoop_bin and len( upgrade_target_hadoop_bin) > 0 and ( os.pathsep + upgrade_source_hadoop_bin) in hive_execute_path: hive_execute_path = hive_execute_path.replace( os.pathsep + upgrade_source_hadoop_bin, os.pathsep + upgrade_target_hadoop_bin) command = format( 'hive --config {hive_server_conf_dir} --service hiveserver2 --deregister ' + current_hiveserver_version) Execute(command, user=params.hive_user, path=hive_execute_path, tries=1)
stack_supports_ranger_audit_db = check_stack_feature( StackFeature.RANGER_AUDIT_DB_SUPPORT, version_for_stack_feature_checks) stack_supports_ranger_hive_jdbc_url_change = check_stack_feature( StackFeature.RANGER_HIVE_PLUGIN_JDBC_URL, version_for_stack_feature_checks) stack_supports_atlas_hook_for_hive_interactive = check_stack_feature( StackFeature.HIVE_INTERACTIVE_ATLAS_HOOK_REQUIRED, version_for_stack_feature_checks) stack_supports_hive_interactive_ga = check_stack_feature( StackFeature.HIVE_INTERACTIVE_GA_SUPPORT, version_for_stack_feature_checks) # component ROLE directory (like hive-metastore or hive-server2-hive2) component_directory = status_params.component_directory component_directory_interactive = status_params.component_directory_interactive # used to render hadoop configurations, such as writing out its own mapreduce2 configs hadoop_home = stack_select.get_hadoop_dir("home") hadoop_lib_home = stack_select.get_hadoop_dir("lib") hive_bin = format('{stack_root}/current/{component_directory}/bin') hive_cmd = os.path.join(hive_bin, "hive") hive_schematool_ver_bin = format('{stack_root}/{version}/hive/bin') hive_schematool_bin = format('{stack_root}/current/{component_directory}/bin') hive_lib = format('{stack_root}/current/{component_directory}/lib') hive_version_lib = format('{stack_root}/{version}/hive/lib') hive_var_lib = '/var/lib/hive' hive_user_home_dir = "/home/hive" # starting on stacks where HSI is supported, we need to begin using the 'hive2' schematool hive_server2_hive2_dir = None hive_server2_hive2_lib = None
# Elasticsearch hosts and port management es_cluster_name = config['configurations']['metron-env']['es_cluster_name'] es_hosts = config['configurations']['metron-env']['es_hosts'] es_host_list = es_hosts.split(",") es_http_port = config['configurations']['metron-env']['es_http_port'] es_url = ",".join([host + ":" + es_http_port for host in es_host_list]) es_http_url = es_host_list[0] + ":" + es_http_port es_date_format = config['configurations']['metron-env']['es_date_format'] # hadoop params stack_root = Script.get_stack_root() # This is the cluster group named 'hadoop'. Its membership is the stack process user ids not individual users. # The config name 'user_group' is out of our control and a bit misleading, so it is renamed to 'hadoop_group'. hadoop_group = config['configurations']['cluster-env']['user_group'] hadoop_home_dir = stack_select.get_hadoop_dir("home") hadoop_bin_dir = stack_select.get_hadoop_dir("bin") hadoop_conf_dir = conf_select.get_hadoop_conf_dir() kafka_home = os.path.join(stack_root, "current", "kafka-broker") kafka_bin_dir = os.path.join(kafka_home, "bin") # zookeeper zk_hosts = default("/clusterHostInfo/zookeeper_hosts", []) has_zk_host = not len(zk_hosts) == 0 zookeeper_quorum = None if has_zk_host: if 'zoo.cfg' in config['configurations'] and 'clientPort' in config['configurations']['zoo.cfg']: zookeeper_clientPort = config['configurations']['zoo.cfg']['clientPort'] else: zookeeper_clientPort = '2181' zookeeper_quorum = (':' + zookeeper_clientPort + ',').join(config['clusterHostInfo']['zookeeper_hosts'])
from resource_management.libraries.functions import default from resource_management.libraries.functions import conf_select from resource_management.libraries.functions import stack_select from resource_management.libraries.functions import format_jvm_option from resource_management.core.system import System from ambari_commons.os_check import OSCheck config = Script.get_config() sudo = AMBARI_SUDO_BINARY stack_version_unformatted = str(config['hostLevelParams']['stack_version']) # default hadoop params mapreduce_libs_path = "/usr/lib/hadoop-mapreduce/*" hadoop_libexec_dir = stack_select.get_hadoop_dir("libexec") hadoop_conf_empty_dir = "/etc/hadoop/conf.empty" #security params security_enabled = config['configurations']['cluster-env']['security_enabled'] #java params java_home = config['hostLevelParams']['java_home'] #hadoop params hdfs_log_dir_prefix = config['configurations']['hadoop-env'][ 'hdfs_log_dir_prefix'] hadoop_pid_dir_prefix = config['configurations']['hadoop-env'][ 'hadoop_pid_dir_prefix'] hadoop_root_logger = config['configurations']['hadoop-env'][ 'hadoop_root_logger']
solr_collection_config_dir = map_example_collection['solr_collection_sample_config_directory'] solr_collection_shards = str(map_example_collection['solr_collection_sample_shards']) solr_collection_replicas = str(map_example_collection['solr_collection_sample_replicas']) # Solr security security_enabled = config['configurations']['cluster-env']['security_enabled'] kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None)) # solr + HDFS map_solr_hdfs = config['configurations']['solr-hdfs'] solr_hdfs_enable = bool(map_solr_hdfs['solr_hdfs_enable']) solr_hdfs_prefix = '#' if not solr_hdfs_enable else '' if solr_hdfs_enable: solr_hdfs_directory = map_solr_hdfs['solr_hdfs_directory'] hadoop_bin_dir = stack_select.get_hadoop_dir('bin') hadoop_conf_dir = conf_select.get_hadoop_conf_dir() hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] hdfs_site = config['configurations']['hdfs-site'] hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab'] default_fs = config['configurations']['core-site']['fs.defaultFS'] dfs_type = default('/commandParams/dfs_type', '') hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name'] solr_hdfs_delete_write_lock_files = bool(map_solr_hdfs['solr_hdfs_delete_write_lock_files']) HdfsResource = functools.partial( HdfsResource, user=hdfs_user, hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', security_enabled=security_enabled, keytab=hdfs_user_keytab,
stack_version_unformatted = str(config['hostLevelParams']['stack_version']) # e.g. 2.3.0.0 #hdp_stack_version = format_hdp_stack_version(stack_version_unformatted) # e.g. 2.3.0.0-2130 full_version = default("/commandParams/version", None) hdp_version = full_version stack_root = Script.get_stack_root() hdfs_site = config['configurations']['hdfs-site'] default_fs = config['configurations']['core-site']['fs.defaultFS'] hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab'] hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name'] hadoop_bin_dir = stack_select.get_hadoop_dir("bin") hadoop_conf_dir = conf_select.get_hadoop_conf_dir() security_enabled = config['configurations']['cluster-env']['security_enabled'] kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None)) dfs_type = default("/commandParams/dfs_type", "") splice_pid_file = "/tmp/splice-ambari-master.pid" import functools #create partial functions with common arguments for every HdfsResource call #to create/delete hdfs directory/file/copyfromlocal we need to call params.HdfsResource in code HdfsResource = functools.partial( HdfsResource, user=hdfs_user, hdfs_resource_ignore_file = "/var/lib/ambari-agent/data/.hdfs_resource_ignore", security_enabled = security_enabled,