def prepare_warfile():
"""
Invokes the 'prepare-war' command in Oozie in order to create the WAR.
The prepare-war command uses the input WAR from ${OOZIE_HOME}/oozie.war and
outputs the prepared WAR to ${CATALINA_BASE}/webapps/oozie.war - because of this,
both of these environment variables must point to the upgraded oozie-server path and
not oozie-client since it was not yet updated.
This method will also perform a kinit if necessary.
:return:
"""
import params
# get the kerberos token if necessary to execute commands as oozie
if params.security_enabled:
oozie_principal_with_host = params.oozie_principal.replace("_HOST", params.hostname)
command = format("{kinit_path_local} -kt {oozie_keytab} {oozie_principal_with_host}")
Execute(command, user=params.oozie_user, logoutput=True)
prepare_war(params)
def oozie_server_specific(upgrade_type):
import params
no_op_test = as_user(format("ls {pid_file} >/dev/null 2>&1 && ps -p `cat {pid_file}` >/dev/null 2>&1"), user=params.oozie_user)
File(params.pid_file,
action="delete",
not_if=no_op_test
)
oozie_server_directories = [format("{oozie_home}/{oozie_tmp_dir}"), params.oozie_pid_dir, params.oozie_log_dir, params.oozie_tmp_dir, params.oozie_data_dir, params.oozie_lib_dir, params.oozie_webapps_dir, params.oozie_webapps_conf_dir, params.oozie_server_dir]
Directory( oozie_server_directories,
owner = params.oozie_user,
group = params.user_group,
mode = 0755,
create_parents = True,
cd_access="a",
)
Directory(params.oozie_libext_dir,
create_parents = True,
)
hashcode_file = format("{oozie_home}/.hashcode")
skip_recreate_sharelib = format("test -f {hashcode_file} && test -d {oozie_home}/share")
untar_sharelib = ('tar','-xvf',format('{oozie_home}/oozie-sharelib.tar.gz'),'-C',params.oozie_home)
Execute( untar_sharelib, # time-expensive
not_if = format("{no_op_test} || {skip_recreate_sharelib}"),
sudo = True,
)
configure_cmds = []
# Default to /usr/share/$TARGETSTACK-oozie/ext-2.2.zip as the first path
source_ext_zip_paths = get_oozie_ext_zip_source_paths(upgrade_type, params)
# Copy the first oozie ext-2.2.zip file that is found.
# This uses a list to handle the cases when migrating from some versions of BigInsights to HDP.
if source_ext_zip_paths is not None:
for source_ext_zip_path in source_ext_zip_paths:
if os.path.isfile(source_ext_zip_path):
configure_cmds.append(('cp', source_ext_zip_path, params.oozie_libext_dir))
configure_cmds.append(('chown', format('{oozie_user}:{user_group}'), format('{oozie_libext_dir}/{ext_js_file}')))
Execute(configure_cmds,
not_if=no_op_test,
sudo=True,
)
break
Directory(params.oozie_webapps_conf_dir,
owner = params.oozie_user,
group = params.user_group,
recursive_ownership = True,
recursion_follow_links = True,
)
# download the database JAR
download_database_library_if_needed()
#falcon el extension
if params.has_falcon_host:
Execute(format('{sudo} cp {falcon_home}/oozie/ext/falcon-oozie-el-extension-*.jar {oozie_libext_dir}'),
not_if = no_op_test)
Execute(format('{sudo} chown {oozie_user}:{user_group} {oozie_libext_dir}/falcon-oozie-el-extension-*.jar'),
not_if = no_op_test)
if params.lzo_enabled and len(params.all_lzo_packages) > 0:
Package(params.all_lzo_packages,
retry_on_repo_unavailability=params.agent_stack_retry_on_unavailability,
retry_count=params.agent_stack_retry_count)
Execute(format('{sudo} cp {hadoop_lib_home}/hadoop-lzo*.jar {oozie_lib_dir}'),
not_if = no_op_test,
)
prepare_war(params)
File(hashcode_file,
mode = 0644,
)
if params.stack_version_formatted and check_stack_feature(StackFeature.OOZIE_CREATE_HIVE_TEZ_CONFIGS, params.stack_version_formatted):
# Create hive-site and tez-site configs for oozie
Directory(params.hive_conf_dir,
create_parents = True,
owner = params.oozie_user,
group = params.user_group
)
if 'hive-site' in params.config['configurations']:
hive_site_config = update_credential_provider_path(params.config['configurations']['hive-site'],
'hive-site',
os.path.join(params.hive_conf_dir, 'hive-site.jceks'),
params.oozie_user,
params.user_group
)
XmlConfig("hive-site.xml",
conf_dir=params.hive_conf_dir,
configurations=hive_site_config,
configuration_attributes=params.config['configuration_attributes']['hive-site'],
owner=params.oozie_user,
group=params.user_group,
mode=0644
)
if 'tez-site' in params.config['configurations']:
XmlConfig( "tez-site.xml",
conf_dir = params.hive_conf_dir,
configurations = params.config['configurations']['tez-site'],
configuration_attributes=params.config['configuration_attributes']['tez-site'],
owner = params.oozie_user,
group = params.user_group,
mode = 0664
)
# If Atlas is also installed, need to generate Atlas Hive hook (hive-atlas-application.properties file) in directory
# {stack_root}/{current_version}/atlas/hook/hive/
# Because this is a .properties file instead of an xml file, it will not be read automatically by Oozie.
# However, should still save the file on this host so that can upload it to the Oozie Sharelib in DFS.
if has_atlas_in_cluster():
atlas_hook_filepath = os.path.join(params.hive_conf_dir, params.atlas_hook_filename)
Logger.info("Has atlas in cluster, will save Atlas Hive hook into location %s" % str(atlas_hook_filepath))
setup_atlas_hook(SERVICE.HIVE, params.hive_atlas_application_properties, atlas_hook_filepath, params.oozie_user, params.user_group)
Directory(params.oozie_server_dir,
owner = params.oozie_user,
group = params.user_group,
recursive_ownership = True,
)
if params.security_enabled:
File(os.path.join(params.conf_dir, 'zkmigrator_jaas.conf'),
owner=params.oozie_user,
group=params.user_group,
content=Template("zkmigrator_jaas.conf.j2")
)
hashcode_file = format("{oozie_home}/.hashcode")
skip_recreate_sharelib = format(
"test -f {hashcode_file} && test -d {oozie_home}/share")
Execute(
configure_cmds,
not_if=format("{no_op_test} || {skip_recreate_sharelib}"),
sudo=True,
)
File(
hashcode_file,
mode=0644,
)
prepare_war(params)
oozie_shared_lib = format("/usr/hdp/{stack_version}/oozie/share")
oozie_user = '******'
oozie_hdfs_user_dir = format("{hdfs_path_prefix}/user/{oozie_user}")
kinit_if_needed = ''
create_yarn_service_tarball()
if options.upgrade:
Logger.info("Skipping uploading oozie shared lib during upgrade")
else:
params.HdfsResource(format("{oozie_hdfs_user_dir}/share/"),
action="delete_on_execute",
type='directory')
def oozie_server_specific():
import params
no_op_test = as_user(format(
"ls {pid_file} >/dev/null 2>&1 && ps -p `cat {pid_file}` >/dev/null 2>&1"
),
user=params.oozie_user)
File(params.pid_file, action="delete", not_if=no_op_test)
oozie_server_directories = [
format("{oozie_home}/{oozie_tmp_dir}"), params.oozie_pid_dir,
params.oozie_log_dir, params.oozie_tmp_dir, params.oozie_data_dir,
params.oozie_lib_dir, params.oozie_webapps_dir,
params.oozie_webapps_conf_dir, params.oozie_server_dir
]
Directory(
oozie_server_directories,
owner=params.oozie_user,
group=params.user_group,
mode=0755,
create_parents=True,
cd_access="a",
)
Directory(
params.oozie_libext_dir,
create_parents=True,
)
hashcode_file = format("{oozie_home}/.hashcode")
skip_recreate_sharelib = format(
"test -f {hashcode_file} && test -d {oozie_home}/share")
untar_sharelib = ('tar', '-xvf',
format('{oozie_home}/oozie-sharelib.tar.gz'), '-C',
params.oozie_home)
Execute(
untar_sharelib, # time-expensive
not_if=format("{no_op_test} || {skip_recreate_sharelib}"),
sudo=True,
)
configure_cmds = []
configure_cmds.append(('cp', params.ext_js_path, params.oozie_libext_dir))
configure_cmds.append(('chown', format('{oozie_user}:{user_group}'),
format('{oozie_libext_dir}/{ext_js_file}')))
Execute(
configure_cmds,
not_if=no_op_test,
sudo=True,
)
Directory(
params.oozie_webapps_conf_dir,
owner=params.oozie_user,
group=params.user_group,
recursive_ownership=True,
recursion_follow_links=True,
)
# download the database JAR
download_database_library_if_needed()
#falcon el extension
if params.has_falcon_host:
Execute(format(
'{sudo} cp {falcon_home}/oozie/ext/falcon-oozie-el-extension-*.jar {oozie_libext_dir}'
),
not_if=no_op_test)
Execute(format(
'{sudo} chown {oozie_user}:{user_group} {oozie_libext_dir}/falcon-oozie-el-extension-*.jar'
),
not_if=no_op_test)
if params.lzo_enabled and len(params.all_lzo_packages) > 0:
Package(params.all_lzo_packages,
retry_on_repo_unavailability=params.
agent_stack_retry_on_unavailability,
retry_count=params.agent_stack_retry_count)
Execute(
format(
'{sudo} cp {hadoop_lib_home}/hadoop-lzo*.jar {oozie_lib_dir}'),
not_if=no_op_test,
)
prepare_war(params)
File(
hashcode_file,
mode=0644,
)
if params.stack_version_formatted and check_stack_feature(
StackFeature.OOZIE_CREATE_HIVE_TEZ_CONFIGS,
params.stack_version_formatted):
# Create hive-site and tez-site configs for oozie
Directory(params.hive_conf_dir,
create_parents=True,
owner=params.oozie_user,
group=params.user_group)
if 'hive-site' in params.config['configurations']:
XmlConfig(
"hive-site.xml",
conf_dir=params.hive_conf_dir,
configurations=params.config['configurations']['hive-site'],
configuration_attributes=params.
config['configuration_attributes']['hive-site'],
owner=params.oozie_user,
group=params.user_group,
mode=0644)
if 'tez-site' in params.config['configurations']:
XmlConfig(
"tez-site.xml",
conf_dir=params.hive_conf_dir,
configurations=params.config['configurations']['tez-site'],
configuration_attributes=params.
config['configuration_attributes']['tez-site'],
owner=params.oozie_user,
group=params.user_group,
mode=0664)
Directory(
params.oozie_server_dir,
owner=params.oozie_user,
group=params.user_group,
recursive_ownership=True,
)
