def configure(self, env, upgrade_type=None):
import params
# The configure command doesn't actually receive the upgrade_type from Script.py, so get it from the config dictionary
if upgrade_type is None:
upgrade_type = Script.get_upgrade_type(
default("/commandParams/upgrade_type", ""))
if upgrade_type is not None and params.upgrade_direction == Direction.UPGRADE and params.version is not None:
Logger.info(
format(
"Configuring Oozie during upgrade type: {upgrade_type}, direction: {params.upgrade_direction}, and version {params.version}"
))
if params.version and check_stack_feature(
StackFeature.ROLLING_UPGRADE, params.version):
# In order for the "<stack-root>/current/oozie-<client/server>" point to the new version of
# oozie, we need to create the symlinks both for server and client.
# This is required as both need to be pointing to new installed oozie version.
# Sets the symlink : eg: <stack-root>/current/oozie-client -> <stack-root>/a.b.c.d-<version>/oozie
stack_select.select("oozie-client", params.version)
# Sets the symlink : eg: <stack-root>/current/oozie-server -> <stack-root>/a.b.c.d-<version>/oozie
stack_select.select("oozie-server", params.version)
if params.version and check_stack_feature(
StackFeature.CONFIG_VERSIONING, params.version):
conf_select.select(params.stack_name, "oozie", params.version)
env.set_params(params)
oozie(is_server=True, upgrade_type=upgrade_type)
def pre_upgrade_restart(self, env, upgrade_type=None):
"""
Performs the tasks surrounding the Oozie startup when a rolling upgrade
is in progress. This includes backing up the configuration, updating
the database, preparing the WAR, and installing the sharelib in HDFS.
:param env:
:return:
"""
import params
env.set_params(params)
# this function should not execute if the version can't be determined or
# is not at least IOP 4.0.0.0
if not params.version or compare_versions(
format_stack_version(params.version), '4.0.0.0') < 0:
return
Logger.info("Executing Oozie Server Rolling Upgrade pre-restart")
oozie_server_upgrade.backup_configuration()
stack_select.select_packages(params.version)
#Execute(format("iop-select set oozie-server {version}"))
oozie_server_upgrade.restore_configuration()
#oozie_server_upgrade.prepare_libext_directory()
oozie_server_upgrade.upgrade_oozie()
def pre_upgrade_restart(self, env, upgrade_type=None):
"""
Performs the tasks that should be done before an upgrade of oozie. This includes:
- backing up configurations
- running <stack-selector-tool> and <conf-selector-tool>
- restoring configurations
- preparing the libext directory
:param env:
:return:
"""
import params
env.set_params(params)
# this function should not execute if the version can't be determined or
# the stack does not support rolling upgrade
if not (params.version and check_stack_feature(
StackFeature.ROLLING_UPGRADE, params.version)):
return
Logger.info("Executing Oozie Server Stack Upgrade pre-restart")
if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE,
params.version):
conf_select.select(params.stack_name, "oozie", params.version)
stack_select.select("oozie-server", params.version)
OozieUpgrade.prepare_libext_directory(upgrade_type=upgrade_type)
def pre_upgrade_restart(self, env, upgrade_type=None):
"""
Performs the tasks that should be done before an upgrade of oozie. This includes:
- backing up configurations
- running hdp-select and conf-select
- restoring configurations
- preparing the libext directory
:param env:
:return:
"""
import params
env.set_params(params)
# this function should not execute if the version can't be determined or
# is not at least HDP 2.2.0.0
if not params.version or compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') < 0:
return
Logger.info("Executing Oozie Server Stack Upgrade pre-restart")
OozieUpgrade.backup_configuration()
if params.version and compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') >= 0:
conf_select.select(params.stack_name, "oozie", params.version)
hdp_select.select("oozie-server", params.version)
OozieUpgrade.restore_configuration()
OozieUpgrade.prepare_libext_directory()
def configure(self, env, upgrade_type=None):
import params
# The configure command doesn't actually receive the upgrade_type from Script.py, so get it from the config dictionary
if upgrade_type is None:
restart_type = default("/commandParams/restart_type", "")
if restart_type.lower() == "rolling_upgrade":
upgrade_type = UPGRADE_TYPE_ROLLING
elif restart_type.lower() == "nonrolling_upgrade":
upgrade_type = UPGRADE_TYPE_NON_ROLLING
if upgrade_type is not None and params.upgrade_direction == Direction.UPGRADE and params.version is not None:
Logger.info(format("Configuring Oozie during upgrade type: {upgrade_type}, direction: {params.upgrade_direction}, and version {params.version}"))
if compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') >= 0:
# In order for the "/usr/hdp/current/oozie-<client/server>" point to the new version of
# oozie, we need to create the symlinks both for server and client.
# This is required as both need to be pointing to new installed oozie version.
# Sets the symlink : eg: /usr/hdp/current/oozie-client -> /usr/hdp/2.3.x.y-<version>/oozie
hdp_select.select("oozie-client", params.version)
# Sets the symlink : eg: /usr/hdp/current/oozie-server -> /usr/hdp/2.3.x.y-<version>/oozie
hdp_select.select("oozie-server", params.version)
if compare_versions(format_hdp_stack_version(params.version), '2.3.0.0') >= 0:
conf_select.select(params.stack_name, "oozie", params.version)
env.set_params(params)
oozie(is_server=True)
def pre_upgrade_restart(self, env, upgrade_type=None):
"""
Performs the tasks that should be done before an upgrade of oozie. This includes:
- backing up configurations
- running hdp-select and conf-select
- restoring configurations
- preparing the libext directory
:param env:
:return:
"""
import params
env.set_params(params)
# this function should not execute if the version can't be determined or
# is not at least HDP 2.2.0.0
if not params.version or compare_versions(format_stack_version(params.version), '2.2.0.0') < 0:
return
Logger.info("Executing Oozie Server Stack Upgrade pre-restart")
OozieUpgrade.backup_configuration()
if params.version and compare_versions(format_stack_version(params.version), '2.2.0.0') >= 0:
conf_select.select(params.stack_name, "oozie", params.version)
stack_select.select("oozie-server", params.version)
OozieUpgrade.restore_configuration()
OozieUpgrade.prepare_libext_directory()
def configure(self, env, upgrade_type=None):
import params
# The configure command doesn't actually receive the upgrade_type from Script.py, so get it from the config dictionary
if upgrade_type is None:
restart_type = default("/commandParams/restart_type", "")
if restart_type.lower() == "rolling_upgrade":
upgrade_type = UPGRADE_TYPE_ROLLING
elif restart_type.lower() == "nonrolling_upgrade":
upgrade_type = UPGRADE_TYPE_NON_ROLLING
if upgrade_type is not None and params.upgrade_direction == Direction.UPGRADE and params.version is not None:
Logger.info(format("Configuring Oozie during upgrade type: {upgrade_type}, direction: {params.upgrade_direction}, and version {params.version}"))
if compare_versions(format_stack_version(params.version), '2.2.0.0') >= 0:
# In order for the "/usr/hdp/current/oozie-<client/server>" point to the new version of
# oozie, we need to create the symlinks both for server and client.
# This is required as both need to be pointing to new installed oozie version.
# Sets the symlink : eg: /usr/hdp/current/oozie-client -> /usr/hdp/2.3.x.y-<version>/oozie
stack_select.select("oozie-client", params.version)
# Sets the symlink : eg: /usr/hdp/current/oozie-server -> /usr/hdp/2.3.x.y-<version>/oozie
stack_select.select("oozie-server", params.version)
if compare_versions(format_stack_version(params.version), '2.3.0.0') >= 0:
conf_select.select(params.stack_name, "oozie", params.version)
env.set_params(params)
oozie(is_server=True)
def pre_rolling_restart(self, env):
"""
Performs the tasks surrounding the Oozie startup when a rolling upgrade
is in progress. This includes backing up the configuration, updating
the database, preparing the WAR, and installing the sharelib in HDFS.
:param env:
:return:
"""
import params
env.set_params(params)
# this function should not execute if the version can't be determined or
# is not at least HDP 2.2.0.0
if not params.version or compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') < 0:
return
Logger.info("Executing Oozie Server Rolling Upgrade pre-restart")
oozie_server_upgrade.backup_configuration()
conf_select.select(params.stack_name, "oozie", params.version)
hdp_select.select("oozie-server", params.version)
oozie_server_upgrade.restore_configuration()
oozie_server_upgrade.prepare_libext_directory()
oozie_server_upgrade.upgrade_oozie()
def pre_rolling_restart(self, env):
"""
Performs the tasks surrounding the Oozie startup when a rolling upgrade
is in progress. This includes backing up the configuration, updating
the database, preparing the WAR, and installing the sharelib in HDFS.
:param env:
:return:
"""
import params
env.set_params(params)
# this function should not execute if the version can't be determined or
# is not at least HDP 2.2.0.0
if not params.version or compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') < 0:
return
Logger.info("Executing Oozie Server Rolling Upgrade pre-restart")
oozie_server_upgrade.backup_configuration()
Execute(format("hdp-select set oozie-server {version}"))
oozie_server_upgrade.restore_configuration()
oozie_server_upgrade.prepare_libext_directory()
oozie_server_upgrade.upgrade_oozie()
def disable_security(self, env):
import params
if not params.stack_supports_zk_security:
Logger.info("Stack doesn't support zookeeper security")
return
if not params.zk_connection_string:
Logger.info("No zookeeper connection string. Skipping reverting ACL")
return
zkmigrator = ZkMigrator(params.zk_connection_string, params.java_exec, params.java64_home, params.jaas_file, params.oozie_user)
zkmigrator.set_acls(params.zk_namespace if params.zk_namespace.startswith('/') else '/' + params.zk_namespace, 'world:anyone:crdwa')
def configure(self, env, upgrade_type=None):
import params
# The configure command doesn't actually receive the upgrade_type from Script.py, so get it from the config dictionary
if upgrade_type is None:
upgrade_type = Script.get_upgrade_type(
default("/commandParams/upgrade_type", ""))
if upgrade_type is not None and params.upgrade_direction == Direction.UPGRADE and params.version is not None:
Logger.info(
format(
"Configuring Oozie during upgrade type: {upgrade_type}, direction: {params.upgrade_direction}, and version {params.version}"
))
if params.version and check_stack_feature(
StackFeature.ROLLING_UPGRADE, params.version):
stack_select.select_packages(params.version)
env.set_params(params)
oozie(is_server=True, upgrade_type=upgrade_type)
def update_credential_provider_path(config_type, dest_provider_path):
import params
# Get the path to the provider <config_type>.jceks
if HADOOP_CREDENTIAL_PROVIDER_PROPERTY_NAME in params.config['configurations'][config_type]:
provider_paths = params.config['configurations'][config_type][HADOOP_CREDENTIAL_PROVIDER_PROPERTY_NAME].split(',')
for path_index in range(len(provider_paths)):
provider_path = provider_paths[path_index]
if config_type == os.path.splitext(os.path.basename(provider_path))[0]:
src_provider_path = provider_path[len('jceks://file'):]
Logger.info('src_provider_path={0}, dest_provider_path{1}'.format(src_provider_path, dest_provider_path))
File(dest_provider_path,
owner = params.oozie_user,
group = params.user_group,
mode = 0640,
content = StaticFile(src_provider_path)
)
provider_paths[path_index] = 'jceks://file{0}'.format(dest_provider_path)
# make a copy of the config dictionary since it cannot be modified
config = params.config['configurations'][config_type].copy()
config[HADOOP_CREDENTIAL_PROVIDER_PROPERTY_NAME] = ','.join(provider_paths)
return config
return params.config['configurations'][config_type]
def copy_atlas_hive_hook_to_dfs_share_lib(upgrade_type=None, upgrade_direction=None):
"""
If the Atlas Hive Hook direcotry is present, Atlas is installed, and this is the first Oozie Server,
then copy the entire contents of that directory to the Oozie Sharelib in DFS, e.g.,
/usr/$stack/$current_version/atlas/hook/hive/ -> hdfs:///user/oozie/share/lib/lib_$timetamp/hive
:param upgrade_type: If in the middle of a stack upgrade, the type as UPGRADE_TYPE_ROLLING or UPGRADE_TYPE_NON_ROLLING
:param upgrade_direction: If in the middle of a stack upgrade, the direction as Direction.UPGRADE or Direction.DOWNGRADE.
"""
import params
# Calculate the effective version since this code can also be called during EU/RU in the upgrade direction.
effective_version = params.stack_version_formatted if upgrade_type is None else format_stack_version(params.version)
if not check_stack_feature(StackFeature.ATLAS_HOOK_SUPPORT, effective_version):
return
# Important that oozie_server_hostnames is sorted by name so that this only runs on a single Oozie server.
if not (len(params.oozie_server_hostnames) > 0 and params.hostname == params.oozie_server_hostnames[0]):
Logger.debug("Will not attempt to copy Atlas Hive hook to DFS since this is not the first Oozie Server "
"sorted by hostname.")
return
if not has_atlas_in_cluster():
Logger.debug("Will not attempt to copy Atlas Hve hook to DFS since Atlas is not installed on the cluster.")
return
if upgrade_type is not None and upgrade_direction == Direction.DOWNGRADE:
Logger.debug("Will not attempt to copy Atlas Hve hook to DFS since in the middle of Rolling/Express upgrade "
"and performing a Downgrade.")
return
current_version = get_current_version()
atlas_hive_hook_dir = format("{stack_root}/{current_version}/atlas/hook/hive/")
if not os.path.exists(atlas_hive_hook_dir):
Logger.error(format("ERROR. Atlas is installed in cluster but this Oozie server doesn't "
"contain directory {atlas_hive_hook_dir}"))
return
atlas_hive_hook_impl_dir = os.path.join(atlas_hive_hook_dir, "atlas-hive-plugin-impl")
num_files = len([name for name in os.listdir(atlas_hive_hook_impl_dir) if os.path.exists(os.path.join(atlas_hive_hook_impl_dir, name))])
Logger.info("Found %d files/directories inside Atlas Hive hook impl directory %s"% (num_files, atlas_hive_hook_impl_dir))
# This can return over 100 files, so take the first 5 lines after "Available ShareLib"
# Use -oozie http(s):localhost:{oozie_server_admin_port}/oozie as oozie-env does not export OOZIE_URL
command = format(r'source {conf_dir}/oozie-env.sh ; oozie admin -oozie {oozie_base_url} -shareliblist hive | grep "\[Available ShareLib\]" -A 5')
code, out = checked_call(command, user=params.oozie_user, tries=10, try_sleep=5, logoutput=True)
hive_sharelib_dir = __parse_sharelib_from_output(out)
if hive_sharelib_dir is None:
raise Fail("Could not parse Hive sharelib from output.")
Logger.info(format("Parsed Hive sharelib = {hive_sharelib_dir} and will attempt to copy/replace {num_files} files to it from {atlas_hive_hook_impl_dir}"))
params.HdfsResource(hive_sharelib_dir,
type="directory",
action="create_on_execute",
source=atlas_hive_hook_impl_dir,
user=params.hdfs_user,
owner=params.oozie_user,
group=params.hdfs_user,
mode=0755,
recursive_chown=True,
recursive_chmod=True,
replace_existing_files=True
)
Logger.info("Copying Atlas Hive hook properties file to Oozie Sharelib in DFS.")
atlas_hook_filepath_source = os.path.join(params.hive_conf_dir, params.atlas_hook_filename)
atlas_hook_file_path_dest_in_dfs = os.path.join(hive_sharelib_dir, params.atlas_hook_filename)
params.HdfsResource(atlas_hook_file_path_dest_in_dfs,
type="file",
source=atlas_hook_filepath_source,
action="create_on_execute",
owner=params.oozie_user,
group=params.hdfs_user,
mode=0755,
replace_existing_files=True
)
params.HdfsResource(None, action="execute")
# Update the sharelib after making any changes
# Use -oozie http(s):localhost:{oozie_server_admin_port}/oozie as oozie-env does not export OOZIE_URL
Execute(format("source {conf_dir}/oozie-env.sh ; oozie admin -oozie {oozie_base_url} -sharelibupdate"),
user=params.oozie_user,
tries=5,
try_sleep=5,
logoutput=True,
)
def oozie_server_specific(upgrade_type):
import params
no_op_test = as_user(format("ls {pid_file} >/dev/null 2>&1 && ps -p `cat {pid_file}` >/dev/null 2>&1"), user=params.oozie_user)
File(params.pid_file,
action="delete",
not_if=no_op_test
)
oozie_server_directories = [format("{oozie_home}/{oozie_tmp_dir}"), params.oozie_pid_dir, params.oozie_log_dir, params.oozie_tmp_dir, params.oozie_data_dir, params.oozie_lib_dir, params.oozie_webapps_dir, params.oozie_webapps_conf_dir, params.oozie_server_dir]
Directory( oozie_server_directories,
owner = params.oozie_user,
group = params.user_group,
mode = 0755,
create_parents = True,
cd_access="a",
)
Directory(params.oozie_libext_dir,
create_parents = True,
)
hashcode_file = format("{oozie_home}/.hashcode")
skip_recreate_sharelib = format("test -f {hashcode_file} && test -d {oozie_home}/share")
untar_sharelib = ('tar','-xvf',format('{oozie_home}/oozie-sharelib.tar.gz'),'-C',params.oozie_home)
Execute( untar_sharelib, # time-expensive
not_if = format("{no_op_test} || {skip_recreate_sharelib}"),
sudo = True,
)
configure_cmds = []
# Default to /usr/share/$TARGETSTACK-oozie/ext-2.2.zip as the first path
source_ext_zip_paths = get_oozie_ext_zip_source_paths(upgrade_type, params)
# Copy the first oozie ext-2.2.zip file that is found.
# This uses a list to handle the cases when migrating from some versions of BigInsights to HDP.
if source_ext_zip_paths is not None:
for source_ext_zip_path in source_ext_zip_paths:
if os.path.isfile(source_ext_zip_path):
configure_cmds.append(('cp', source_ext_zip_path, params.oozie_libext_dir))
configure_cmds.append(('chown', format('{oozie_user}:{user_group}'), format('{oozie_libext_dir}/{ext_js_file}')))
Execute(configure_cmds,
not_if=no_op_test,
sudo=True,
)
break
Directory(params.oozie_webapps_conf_dir,
owner = params.oozie_user,
group = params.user_group,
recursive_ownership = True,
recursion_follow_links = True,
)
# download the database JAR
download_database_library_if_needed()
#falcon el extension
if params.has_falcon_host:
Execute(format('{sudo} cp {falcon_home}/oozie/ext/falcon-oozie-el-extension-*.jar {oozie_libext_dir}'),
not_if = no_op_test)
Execute(format('{sudo} chown {oozie_user}:{user_group} {oozie_libext_dir}/falcon-oozie-el-extension-*.jar'),
not_if = no_op_test)
if params.lzo_enabled and len(params.all_lzo_packages) > 0:
Package(params.all_lzo_packages,
retry_on_repo_unavailability=params.agent_stack_retry_on_unavailability,
retry_count=params.agent_stack_retry_count)
Execute(format('{sudo} cp {hadoop_lib_home}/hadoop-lzo*.jar {oozie_lib_dir}'),
not_if = no_op_test,
)
prepare_war(params)
File(hashcode_file,
mode = 0644,
)
if params.stack_version_formatted and check_stack_feature(StackFeature.OOZIE_CREATE_HIVE_TEZ_CONFIGS, params.stack_version_formatted):
# Create hive-site and tez-site configs for oozie
Directory(params.hive_conf_dir,
create_parents = True,
owner = params.oozie_user,
group = params.user_group
)
if 'hive-site' in params.config['configurations']:
hive_site_config = update_credential_provider_path(params.config['configurations']['hive-site'],
'hive-site',
os.path.join(params.hive_conf_dir, 'hive-site.jceks'),
params.oozie_user,
params.user_group
)
XmlConfig("hive-site.xml",
conf_dir=params.hive_conf_dir,
configurations=hive_site_config,
configuration_attributes=params.config['configuration_attributes']['hive-site'],
owner=params.oozie_user,
group=params.user_group,
mode=0644
)
if 'tez-site' in params.config['configurations']:
XmlConfig( "tez-site.xml",
conf_dir = params.hive_conf_dir,
configurations = params.config['configurations']['tez-site'],
configuration_attributes=params.config['configuration_attributes']['tez-site'],
owner = params.oozie_user,
group = params.user_group,
mode = 0664
)
# If Atlas is also installed, need to generate Atlas Hive hook (hive-atlas-application.properties file) in directory
# {stack_root}/{current_version}/atlas/hook/hive/
# Because this is a .properties file instead of an xml file, it will not be read automatically by Oozie.
# However, should still save the file on this host so that can upload it to the Oozie Sharelib in DFS.
if has_atlas_in_cluster():
atlas_hook_filepath = os.path.join(params.hive_conf_dir, params.atlas_hook_filename)
Logger.info("Has atlas in cluster, will save Atlas Hive hook into location %s" % str(atlas_hook_filepath))
setup_atlas_hook(SERVICE.HIVE, params.hive_atlas_application_properties, atlas_hook_filepath, params.oozie_user, params.user_group)
Directory(params.oozie_server_dir,
owner = params.oozie_user,
group = params.user_group,
recursive_ownership = True,
)
if params.security_enabled:
File(os.path.join(params.conf_dir, 'zkmigrator_jaas.conf'),
owner=params.oozie_user,
group=params.user_group,
content=Template("zkmigrator_jaas.conf.j2")
)
