keytab=hdfs_user_keytab,
kinit_path_local=kinit_path_local,
hadoop_bin_dir=hadoop_bin_dir,
hadoop_conf_dir=hadoop_conf_dir,
principal_name=hdfs_principal_name,
hdfs_site=hdfs_site,
default_fs=default_fs,
immutable_paths=get_not_managed_resources(),
dfs_type=dfs_type)
# The logic for LZO also exists in OOZIE's params.py
io_compression_codecs = default(
"/configurations/core-site/io.compression.codecs", None)
lzo_enabled = io_compression_codecs is not None and "com.hadoop.compression.lzo" in io_compression_codecs.lower(
)
lzo_packages = get_lzo_packages(stack_version_unformatted)
name_node_params = default("/commandParams/namenode", None)
java_home = config['hostLevelParams']['java_home']
java_version = expect("/hostLevelParams/java_version", int)
java_exec = format("{java_home}/bin/java")
hadoop_heapsize = config['configurations']['hadoop-env']['hadoop_heapsize']
namenode_heapsize = config['configurations']['hadoop-env']['namenode_heapsize']
namenode_opt_newsize = config['configurations']['hadoop-env'][
'namenode_opt_newsize']
namenode_opt_maxnewsize = config['configurations']['hadoop-env'][
'namenode_opt_maxnewsize']
namenode_opt_permsize = format_jvm_option(
"/configurations/hadoop-env/namenode_opt_permsize", "128m")
def oozie_server_specific(upgrade_type):
import params
no_op_test = as_user(format(
"ls {pid_file} >/dev/null 2>&1 && ps -p `cat {pid_file}` >/dev/null 2>&1"
),
user=params.oozie_user)
File(params.pid_file, action="delete", not_if=no_op_test)
oozie_server_directories = [
format("{oozie_home}/{oozie_tmp_dir}"), params.oozie_pid_dir,
params.oozie_log_dir, params.oozie_tmp_dir, params.oozie_data_dir,
params.oozie_lib_dir, params.oozie_webapps_dir,
params.oozie_webapps_conf_dir, params.oozie_server_dir
]
Directory(
oozie_server_directories,
owner=params.oozie_user,
group=params.user_group,
mode=0755,
create_parents=True,
cd_access="a",
)
Directory(
params.oozie_libext_dir,
create_parents=True,
)
hashcode_file = format("{oozie_home}/.hashcode")
skip_recreate_sharelib = format(
"test -f {hashcode_file} && test -d {oozie_home}/share")
untar_sharelib = ('tar', '-xvf',
format('{oozie_home}/oozie-sharelib.tar.gz'), '-C',
params.oozie_home)
Execute(
untar_sharelib, # time-expensive
not_if=format("{no_op_test} || {skip_recreate_sharelib}"),
sudo=True,
)
configure_cmds = []
# Default to /usr/share/$TARGETSTACK-oozie/ext-2.2.zip as the first path
source_ext_zip_paths = get_oozie_ext_zip_source_paths(upgrade_type, params)
# Copy the first oozie ext-2.2.zip file that is found.
# This uses a list to handle the cases when migrating from some versions of BigInsights to HDP.
if source_ext_zip_paths is not None:
for source_ext_zip_path in source_ext_zip_paths:
if os.path.isfile(source_ext_zip_path):
configure_cmds.append(
('cp', source_ext_zip_path, params.oozie_libext_dir))
configure_cmds.append(
('chown', format('{oozie_user}:{user_group}'),
format('{oozie_libext_dir}/{ext_js_file}')))
Execute(
configure_cmds,
not_if=no_op_test,
sudo=True,
)
break
Directory(
params.oozie_webapps_conf_dir,
owner=params.oozie_user,
group=params.user_group,
recursive_ownership=True,
recursion_follow_links=True,
)
# download the database JAR
download_database_library_if_needed()
#falcon el extension
if params.has_falcon_host:
Execute(format(
'{sudo} cp {falcon_home}/oozie/ext/falcon-oozie-el-extension-*.jar {oozie_libext_dir}'
),
not_if=no_op_test)
Execute(format(
'{sudo} chown {oozie_user}:{user_group} {oozie_libext_dir}/falcon-oozie-el-extension-*.jar'
),
not_if=no_op_test)
if params.lzo_enabled:
all_lzo_packages = get_lzo_packages(params.stack_version_unformatted)
Package(all_lzo_packages,
retry_on_repo_unavailability=params.
agent_stack_retry_on_unavailability,
retry_count=params.agent_stack_retry_count)
Execute(
format(
'{sudo} cp {hadoop_lib_home}/hadoop-lzo*.jar {oozie_lib_dir}'),
not_if=no_op_test,
)
prepare_war(params)
File(
hashcode_file,
mode=0644,
)
if params.stack_version_formatted and check_stack_feature(
StackFeature.OOZIE_CREATE_HIVE_TEZ_CONFIGS,
params.stack_version_formatted):
# Create hive-site and tez-site configs for oozie
Directory(params.hive_conf_dir,
create_parents=True,
owner=params.oozie_user,
group=params.user_group)
if 'hive-site' in params.config['configurations']:
hive_site_config = update_credential_provider_path(
params.config['configurations']['hive-site'], 'hive-site',
os.path.join(params.hive_conf_dir, 'hive-site.jceks'),
params.oozie_user, params.user_group)
XmlConfig("hive-site.xml",
conf_dir=params.hive_conf_dir,
configurations=hive_site_config,
configuration_attributes=params.
config['configuration_attributes']['hive-site'],
owner=params.oozie_user,
group=params.user_group,
mode=0644)
if 'tez-site' in params.config['configurations']:
XmlConfig(
"tez-site.xml",
conf_dir=params.hive_conf_dir,
configurations=params.config['configurations']['tez-site'],
configuration_attributes=params.
config['configuration_attributes']['tez-site'],
owner=params.oozie_user,
group=params.user_group,
mode=0664)
# If Atlas is also installed, need to generate Atlas Hive hook (hive-atlas-application.properties file) in directory
# {stack_root}/{current_version}/atlas/hook/hive/
# Because this is a .properties file instead of an xml file, it will not be read automatically by Oozie.
# However, should still save the file on this host so that can upload it to the Oozie Sharelib in DFS.
if has_atlas_in_cluster():
atlas_hook_filepath = os.path.join(params.hive_conf_dir,
params.atlas_hook_filename)
Logger.info(
"Has atlas in cluster, will save Atlas Hive hook into location %s"
% str(atlas_hook_filepath))
setup_atlas_hook(SERVICE.HIVE,
params.hive_atlas_application_properties,
atlas_hook_filepath, params.oozie_user,
params.user_group)
Directory(
params.oozie_server_dir,
owner=params.oozie_user,
group=params.user_group,
recursive_ownership=True,
)
if params.security_enabled:
File(os.path.join(params.conf_dir, 'zkmigrator_jaas.conf'),
owner=params.oozie_user,
group=params.user_group,
content=Template("zkmigrator_jaas.conf.j2"))
hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab']
hdfs_user = config['configurations']['hadoop-env']['hdfs_user']
hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name']
hdfs_site = config['configurations']['hdfs-site']
default_fs = config['configurations']['core-site']['fs.defaultFS']
import functools
#create partial functions with common arguments for every HdfsResource call
#to create/delete hdfs directory/file/copyfromlocal we need to call params.HdfsResource in code
HdfsResource = functools.partial(
HdfsResource,
user=hdfs_user,
hdfs_resource_ignore_file = "/var/lib/ambari-agent/data/.hdfs_resource_ignore",
security_enabled = security_enabled,
keytab = hdfs_user_keytab,
kinit_path_local = kinit_path_local,
hadoop_bin_dir = hadoop_bin_dir,
hadoop_conf_dir = hadoop_conf_dir,
principal_name = hdfs_principal_name,
hdfs_site = hdfs_site,
default_fs = default_fs
)
is_webhdfs_enabled = config['configurations']['hdfs-site']['dfs.webhdfs.enabled']
# The logic for LZO also exists in HDFS' params.py
io_compression_codecs = default("/configurations/core-site/io.compression.codecs", None)
lzo_enabled = io_compression_codecs is not None and "com.hadoop.compression.lzo" in io_compression_codecs.lower()
all_lzo_packages = get_lzo_packages(stack_version_unformatted)
def hdfs(name=None):
import params
if params.create_lib_snappy_symlinks:
install_snappy()
# On some OS this folder could be not exists, so we will create it before pushing there files
Directory(params.limits_conf_dir,
create_parents = True,
owner='root',
group='root'
)
File(os.path.join(params.limits_conf_dir, 'hdfs.conf'),
owner='root',
group='root',
mode=0644,
content=Template("hdfs.conf.j2")
)
if params.security_enabled:
File(os.path.join(params.hadoop_conf_dir, 'hdfs_dn_jaas.conf'),
owner=params.hdfs_user,
group=params.user_group,
content=Template("hdfs_dn_jaas.conf.j2")
)
File(os.path.join(params.hadoop_conf_dir, 'hdfs_nn_jaas.conf'),
owner=params.hdfs_user,
group=params.user_group,
content=Template("hdfs_nn_jaas.conf.j2")
)
if params.dfs_ha_enabled:
File(os.path.join(params.hadoop_conf_dir, 'hdfs_jn_jaas.conf'),
owner=params.hdfs_user,
group=params.user_group,
content=Template("hdfs_jn_jaas.conf.j2")
)
tc_mode = 0644
tc_owner = "root"
else:
tc_mode = None
tc_owner = params.hdfs_user
if "hadoop-policy" in params.config['configurations']:
XmlConfig("hadoop-policy.xml",
conf_dir=params.hadoop_conf_dir,
configurations=params.config['configurations']['hadoop-policy'],
configuration_attributes=params.config['configuration_attributes']['hadoop-policy'],
owner=params.hdfs_user,
group=params.user_group
)
if "ssl-client" in params.config['configurations']:
XmlConfig("ssl-client.xml",
conf_dir=params.hadoop_conf_dir,
configurations=params.config['configurations']['ssl-client'],
configuration_attributes=params.config['configuration_attributes']['ssl-client'],
owner=params.hdfs_user,
group=params.user_group
)
Directory(params.hadoop_conf_secure_dir,
create_parents = True,
owner='root',
group=params.user_group,
cd_access='a',
)
XmlConfig("ssl-client.xml",
conf_dir=params.hadoop_conf_secure_dir,
configurations=params.config['configurations']['ssl-client'],
configuration_attributes=params.config['configuration_attributes']['ssl-client'],
owner=params.hdfs_user,
group=params.user_group
)
if "ssl-server" in params.config['configurations']:
XmlConfig("ssl-server.xml",
conf_dir=params.hadoop_conf_dir,
configurations=params.config['configurations']['ssl-server'],
configuration_attributes=params.config['configuration_attributes']['ssl-server'],
owner=params.hdfs_user,
group=params.user_group
)
XmlConfig("hdfs-site.xml",
conf_dir=params.hadoop_conf_dir,
configurations=params.config['configurations']['hdfs-site'],
configuration_attributes=params.config['configuration_attributes']['hdfs-site'],
owner=params.hdfs_user,
group=params.user_group
)
XmlConfig("core-site.xml",
conf_dir=params.hadoop_conf_dir,
configurations=params.config['configurations']['core-site'],
configuration_attributes=params.config['configuration_attributes']['core-site'],
owner=params.hdfs_user,
group=params.user_group,
mode=0644
)
File(os.path.join(params.hadoop_conf_dir, 'slaves'),
owner=tc_owner,
content=Template("slaves.j2")
)
if params.lzo_enabled:
lzo_packages = get_lzo_packages(params.stack_version_unformatted)
Package(lzo_packages,
retry_on_repo_unavailability=params.agent_stack_retry_on_unavailability,
retry_count=params.agent_stack_retry_count)
