keytab=hdfs_user_keytab, kinit_path_local=kinit_path_local, hadoop_bin_dir=hadoop_bin_dir, hadoop_conf_dir=hadoop_conf_dir, principal_name=hdfs_principal_name, hdfs_site=hdfs_site, default_fs=default_fs, immutable_paths=get_not_managed_resources(), dfs_type=dfs_type) # The logic for LZO also exists in OOZIE's params.py io_compression_codecs = default( "/configurations/core-site/io.compression.codecs", None) lzo_enabled = io_compression_codecs is not None and "com.hadoop.compression.lzo" in io_compression_codecs.lower( ) lzo_packages = get_lzo_packages(stack_version_unformatted) name_node_params = default("/commandParams/namenode", None) java_home = config['hostLevelParams']['java_home'] java_version = expect("/hostLevelParams/java_version", int) java_exec = format("{java_home}/bin/java") hadoop_heapsize = config['configurations']['hadoop-env']['hadoop_heapsize'] namenode_heapsize = config['configurations']['hadoop-env']['namenode_heapsize'] namenode_opt_newsize = config['configurations']['hadoop-env'][ 'namenode_opt_newsize'] namenode_opt_maxnewsize = config['configurations']['hadoop-env'][ 'namenode_opt_maxnewsize'] namenode_opt_permsize = format_jvm_option( "/configurations/hadoop-env/namenode_opt_permsize", "128m")
def oozie_server_specific(upgrade_type): import params no_op_test = as_user(format( "ls {pid_file} >/dev/null 2>&1 && ps -p `cat {pid_file}` >/dev/null 2>&1" ), user=params.oozie_user) File(params.pid_file, action="delete", not_if=no_op_test) oozie_server_directories = [ format("{oozie_home}/{oozie_tmp_dir}"), params.oozie_pid_dir, params.oozie_log_dir, params.oozie_tmp_dir, params.oozie_data_dir, params.oozie_lib_dir, params.oozie_webapps_dir, params.oozie_webapps_conf_dir, params.oozie_server_dir ] Directory( oozie_server_directories, owner=params.oozie_user, group=params.user_group, mode=0755, create_parents=True, cd_access="a", ) Directory( params.oozie_libext_dir, create_parents=True, ) hashcode_file = format("{oozie_home}/.hashcode") skip_recreate_sharelib = format( "test -f {hashcode_file} && test -d {oozie_home}/share") untar_sharelib = ('tar', '-xvf', format('{oozie_home}/oozie-sharelib.tar.gz'), '-C', params.oozie_home) Execute( untar_sharelib, # time-expensive not_if=format("{no_op_test} || {skip_recreate_sharelib}"), sudo=True, ) configure_cmds = [] # Default to /usr/share/$TARGETSTACK-oozie/ext-2.2.zip as the first path source_ext_zip_paths = get_oozie_ext_zip_source_paths(upgrade_type, params) # Copy the first oozie ext-2.2.zip file that is found. # This uses a list to handle the cases when migrating from some versions of BigInsights to HDP. if source_ext_zip_paths is not None: for source_ext_zip_path in source_ext_zip_paths: if os.path.isfile(source_ext_zip_path): configure_cmds.append( ('cp', source_ext_zip_path, params.oozie_libext_dir)) configure_cmds.append( ('chown', format('{oozie_user}:{user_group}'), format('{oozie_libext_dir}/{ext_js_file}'))) Execute( configure_cmds, not_if=no_op_test, sudo=True, ) break Directory( params.oozie_webapps_conf_dir, owner=params.oozie_user, group=params.user_group, recursive_ownership=True, recursion_follow_links=True, ) # download the database JAR download_database_library_if_needed() #falcon el extension if params.has_falcon_host: Execute(format( '{sudo} cp {falcon_home}/oozie/ext/falcon-oozie-el-extension-*.jar {oozie_libext_dir}' ), not_if=no_op_test) Execute(format( '{sudo} chown {oozie_user}:{user_group} {oozie_libext_dir}/falcon-oozie-el-extension-*.jar' ), not_if=no_op_test) if params.lzo_enabled: all_lzo_packages = get_lzo_packages(params.stack_version_unformatted) Package(all_lzo_packages, retry_on_repo_unavailability=params. agent_stack_retry_on_unavailability, retry_count=params.agent_stack_retry_count) Execute( format( '{sudo} cp {hadoop_lib_home}/hadoop-lzo*.jar {oozie_lib_dir}'), not_if=no_op_test, ) prepare_war(params) File( hashcode_file, mode=0644, ) if params.stack_version_formatted and check_stack_feature( StackFeature.OOZIE_CREATE_HIVE_TEZ_CONFIGS, params.stack_version_formatted): # Create hive-site and tez-site configs for oozie Directory(params.hive_conf_dir, create_parents=True, owner=params.oozie_user, group=params.user_group) if 'hive-site' in params.config['configurations']: hive_site_config = update_credential_provider_path( params.config['configurations']['hive-site'], 'hive-site', os.path.join(params.hive_conf_dir, 'hive-site.jceks'), params.oozie_user, params.user_group) XmlConfig("hive-site.xml", conf_dir=params.hive_conf_dir, configurations=hive_site_config, configuration_attributes=params. config['configuration_attributes']['hive-site'], owner=params.oozie_user, group=params.user_group, mode=0644) if 'tez-site' in params.config['configurations']: XmlConfig( "tez-site.xml", conf_dir=params.hive_conf_dir, configurations=params.config['configurations']['tez-site'], configuration_attributes=params. config['configuration_attributes']['tez-site'], owner=params.oozie_user, group=params.user_group, mode=0664) # If Atlas is also installed, need to generate Atlas Hive hook (hive-atlas-application.properties file) in directory # {stack_root}/{current_version}/atlas/hook/hive/ # Because this is a .properties file instead of an xml file, it will not be read automatically by Oozie. # However, should still save the file on this host so that can upload it to the Oozie Sharelib in DFS. if has_atlas_in_cluster(): atlas_hook_filepath = os.path.join(params.hive_conf_dir, params.atlas_hook_filename) Logger.info( "Has atlas in cluster, will save Atlas Hive hook into location %s" % str(atlas_hook_filepath)) setup_atlas_hook(SERVICE.HIVE, params.hive_atlas_application_properties, atlas_hook_filepath, params.oozie_user, params.user_group) Directory( params.oozie_server_dir, owner=params.oozie_user, group=params.user_group, recursive_ownership=True, ) if params.security_enabled: File(os.path.join(params.conf_dir, 'zkmigrator_jaas.conf'), owner=params.oozie_user, group=params.user_group, content=Template("zkmigrator_jaas.conf.j2"))
hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab'] hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name'] hdfs_site = config['configurations']['hdfs-site'] default_fs = config['configurations']['core-site']['fs.defaultFS'] import functools #create partial functions with common arguments for every HdfsResource call #to create/delete hdfs directory/file/copyfromlocal we need to call params.HdfsResource in code HdfsResource = functools.partial( HdfsResource, user=hdfs_user, hdfs_resource_ignore_file = "/var/lib/ambari-agent/data/.hdfs_resource_ignore", security_enabled = security_enabled, keytab = hdfs_user_keytab, kinit_path_local = kinit_path_local, hadoop_bin_dir = hadoop_bin_dir, hadoop_conf_dir = hadoop_conf_dir, principal_name = hdfs_principal_name, hdfs_site = hdfs_site, default_fs = default_fs ) is_webhdfs_enabled = config['configurations']['hdfs-site']['dfs.webhdfs.enabled'] # The logic for LZO also exists in HDFS' params.py io_compression_codecs = default("/configurations/core-site/io.compression.codecs", None) lzo_enabled = io_compression_codecs is not None and "com.hadoop.compression.lzo" in io_compression_codecs.lower() all_lzo_packages = get_lzo_packages(stack_version_unformatted)
def hdfs(name=None): import params if params.create_lib_snappy_symlinks: install_snappy() # On some OS this folder could be not exists, so we will create it before pushing there files Directory(params.limits_conf_dir, create_parents = True, owner='root', group='root' ) File(os.path.join(params.limits_conf_dir, 'hdfs.conf'), owner='root', group='root', mode=0644, content=Template("hdfs.conf.j2") ) if params.security_enabled: File(os.path.join(params.hadoop_conf_dir, 'hdfs_dn_jaas.conf'), owner=params.hdfs_user, group=params.user_group, content=Template("hdfs_dn_jaas.conf.j2") ) File(os.path.join(params.hadoop_conf_dir, 'hdfs_nn_jaas.conf'), owner=params.hdfs_user, group=params.user_group, content=Template("hdfs_nn_jaas.conf.j2") ) if params.dfs_ha_enabled: File(os.path.join(params.hadoop_conf_dir, 'hdfs_jn_jaas.conf'), owner=params.hdfs_user, group=params.user_group, content=Template("hdfs_jn_jaas.conf.j2") ) tc_mode = 0644 tc_owner = "root" else: tc_mode = None tc_owner = params.hdfs_user if "hadoop-policy" in params.config['configurations']: XmlConfig("hadoop-policy.xml", conf_dir=params.hadoop_conf_dir, configurations=params.config['configurations']['hadoop-policy'], configuration_attributes=params.config['configuration_attributes']['hadoop-policy'], owner=params.hdfs_user, group=params.user_group ) if "ssl-client" in params.config['configurations']: XmlConfig("ssl-client.xml", conf_dir=params.hadoop_conf_dir, configurations=params.config['configurations']['ssl-client'], configuration_attributes=params.config['configuration_attributes']['ssl-client'], owner=params.hdfs_user, group=params.user_group ) Directory(params.hadoop_conf_secure_dir, create_parents = True, owner='root', group=params.user_group, cd_access='a', ) XmlConfig("ssl-client.xml", conf_dir=params.hadoop_conf_secure_dir, configurations=params.config['configurations']['ssl-client'], configuration_attributes=params.config['configuration_attributes']['ssl-client'], owner=params.hdfs_user, group=params.user_group ) if "ssl-server" in params.config['configurations']: XmlConfig("ssl-server.xml", conf_dir=params.hadoop_conf_dir, configurations=params.config['configurations']['ssl-server'], configuration_attributes=params.config['configuration_attributes']['ssl-server'], owner=params.hdfs_user, group=params.user_group ) XmlConfig("hdfs-site.xml", conf_dir=params.hadoop_conf_dir, configurations=params.config['configurations']['hdfs-site'], configuration_attributes=params.config['configuration_attributes']['hdfs-site'], owner=params.hdfs_user, group=params.user_group ) XmlConfig("core-site.xml", conf_dir=params.hadoop_conf_dir, configurations=params.config['configurations']['core-site'], configuration_attributes=params.config['configuration_attributes']['core-site'], owner=params.hdfs_user, group=params.user_group, mode=0644 ) File(os.path.join(params.hadoop_conf_dir, 'slaves'), owner=tc_owner, content=Template("slaves.j2") ) if params.lzo_enabled: lzo_packages = get_lzo_packages(params.stack_version_unformatted) Package(lzo_packages, retry_on_repo_unavailability=params.agent_stack_retry_on_unavailability, retry_count=params.agent_stack_retry_count)