def configure(self, env, upgrade_type=None):
import params
# The configure command doesn't actually receive the upgrade_type from Script.py, so get it from the config dictionary
if upgrade_type is None:
restart_type = default("/commandParams/restart_type", "")
if restart_type.lower() == "rolling_upgrade":
upgrade_type = UPGRADE_TYPE_ROLLING
elif restart_type.lower() == "nonrolling_upgrade":
upgrade_type = UPGRADE_TYPE_NON_ROLLING
if upgrade_type is not None and params.upgrade_direction == Direction.UPGRADE and params.version is not None:
Logger.info(format("Configuring Oozie during upgrade type: {upgrade_type}, direction: {params.upgrade_direction}, and version {params.version}"))
if compare_versions(format_stack_version(params.version), '2.2.0.0') >= 0:
# In order for the "/usr/hdp/current/oozie-<client/server>" point to the new version of
# oozie, we need to create the symlinks both for server and client.
# This is required as both need to be pointing to new installed oozie version.
# Sets the symlink : eg: /usr/hdp/current/oozie-client -> /usr/hdp/2.3.x.y-<version>/oozie
stack_select.select("oozie-client", params.version)
# Sets the symlink : eg: /usr/hdp/current/oozie-server -> /usr/hdp/2.3.x.y-<version>/oozie
stack_select.select("oozie-server", params.version)
if compare_versions(format_stack_version(params.version), '2.3.0.0') >= 0:
conf_select.select(params.stack_name, "oozie", params.version)
env.set_params(params)
oozie(is_server=True)
def pre_upgrade_restart(self, env, upgrade_type=None):
"""
Performs the tasks that should be done before an upgrade of oozie. This includes:
- backing up configurations
- running hdp-select and conf-select
- restoring configurations
- preparing the libext directory
:param env:
:return:
"""
import params
env.set_params(params)
# this function should not execute if the version can't be determined or
# is not at least HDP 2.2.0.0
if not params.version or compare_versions(format_stack_version(params.version), '2.2.0.0') < 0:
return
Logger.info("Executing Oozie Server Stack Upgrade pre-restart")
OozieUpgrade.backup_configuration()
if params.version and compare_versions(format_stack_version(params.version), '2.2.0.0') >= 0:
conf_select.select(params.stack_name, "oozie", params.version)
stack_select.select("oozie-server", params.version)
OozieUpgrade.restore_configuration()
OozieUpgrade.prepare_libext_directory()
def pre_upgrade_restart(self, env, upgrade_type=None):
"""
Performs the tasks surrounding the Oozie startup when a rolling upgrade
is in progress. This includes backing up the configuration, updating
the database, preparing the WAR, and installing the sharelib in HDFS.
:param env:
:return:
"""
import params
env.set_params(params)
# this function should not execute if the version can't be determined or
# is not at least IOP 4.0.0.0
if not params.version or compare_versions(
format_stack_version(params.version), '4.0.0.0') < 0:
return
Logger.info("Executing Oozie Server Rolling Upgrade pre-restart")
oozie_server_upgrade.backup_configuration()
stack_select.select_packages(params.version)
#Execute(format("iop-select set oozie-server {version}"))
oozie_server_upgrade.restore_configuration()
#oozie_server_upgrade.prepare_libext_directory()
oozie_server_upgrade.upgrade_oozie()
def prepare_libext_directory():
"""
Creates /usr/iop/current/oozie/libext-customer and recursively sets
777 permissions on it and its parents.
:return:
"""
import params
# some versions of IOP don't need the lzo compression libraries
target_version_needs_compression_libraries = compare_versions(
format_stack_version(params.version), '4.0.0.0') >= 0
if not os.path.isdir(params.oozie_libext_customer_dir):
os.makedirs(params.oozie_libext_customer_dir, 0o777)
# ensure that it's rwx for all
os.chmod(params.oozie_libext_customer_dir, 0o777)
# get all hadooplzo* JAR files
# stack-select set hadoop-client has not run yet, therefore we cannot use
# /usr/iop/current/hadoop-client ; we must use params.version directly
# however, this only works when upgrading beyond 4.0.0.0; don't do this
# for downgrade to 4.0.0.0 since hadoop-lzo will not be present
# This can also be called during a Downgrade.
# When a version is Intalled, it is responsible for downloading the hadoop-lzo packages
# if lzo is enabled.
if params.lzo_enabled and (params.upgrade_direction == Direction.UPGRADE
or target_version_needs_compression_libraries):
hadoop_lzo_pattern = 'hadoop-lzo*.jar'
hadoop_client_new_lib_dir = format("/usr/iop/{version}/hadoop/lib")
files = glob.iglob(
os.path.join(hadoop_client_new_lib_dir, hadoop_lzo_pattern))
if not files:
raise Fail("There are no files at {0} matching {1}".format(
hadoop_client_new_lib_dir, hadoop_lzo_pattern))
# copy files into libext
files_copied = False
for file in files:
if os.path.isfile(file):
Logger.info("Copying {0} to {1}".format(
str(file), params.oozie_libext_customer_dir))
shutil.copy2(file, params.oozie_libext_customer_dir)
files_copied = True
if not files_copied:
raise Fail("There are no files at {0} matching {1}".format(
hadoop_client_new_lib_dir, hadoop_lzo_pattern))
# copy ext ZIP to customer dir
oozie_ext_zip_file = '/usr/share/IOP-oozie/ext-2.2.zip'
if not os.path.isfile(oozie_ext_zip_file):
raise Fail("Unable to copy {0} because it does not exist".format(
oozie_ext_zip_file))
Logger.info("Copying {0} to {1}".format(oozie_ext_zip_file,
params.oozie_libext_customer_dir))
shutil.copy2(oozie_ext_zip_file, params.oozie_libext_customer_dir)
def configure(self, env, upgrade_type=None):
import params
#TODO: needed?
if upgrade_type == "nonrolling" and params.upgrade_direction == Direction.UPGRADE and \
params.version and compare_versions(format_stack_version(params.version), '4.1.0.0') >= 0:
# In order for the "/usr/hdp/current/oozie-<client/server>" point to the new version of
# oozie, we need to create the symlinks both for server and client.
# This is required as both need to be pointing to new installed oozie version.
# Sets the symlink : eg: /usr/hdp/current/oozie-client -> /usr/hdp/2.3.x.y-<version>/oozie
# Sets the symlink : eg: /usr/hdp/current/oozie-server -> /usr/hdp/2.3.x.y-<version>/oozie
stack_select.select_packages(params.version)
env.set_params(params)
oozie(is_server=True)
def prepare_libext_directory():
"""
Performs the following actions on libext:
- creates <stack-root>/current/oozie/libext and recursively
- set 777 permissions on it and its parents.
- downloads JDBC driver JAR if needed
- copies Falcon JAR for the Oozie WAR if needed
"""
import params
# some versions of HDP don't need the lzo compression libraries
target_version_needs_compression_libraries = compare_versions(
format_stack_version(params.version), '2.2.1.0') >= 0
# ensure the directory exists
Directory(params.oozie_libext_dir, mode=0777)
# get all hadooplzo* JAR files
# <stack-selector-tool> set hadoop-client has not run yet, therefore we cannot use
# <stack-root>/current/hadoop-client ; we must use params.version directly
# however, this only works when upgrading beyond 2.2.0.0; don't do this
# for downgrade to 2.2.0.0 since hadoop-lzo will not be present
# This can also be called during a Downgrade.
# When a version is Intalled, it is responsible for downloading the hadoop-lzo packages
# if lzo is enabled.
if params.lzo_enabled and (
params.upgrade_direction == Direction.UPGRADE
or target_version_needs_compression_libraries):
hadoop_lzo_pattern = 'hadoop-lzo*.jar'
hadoop_client_new_lib_dir = format("/usr/hdp/{version}/hadoop/lib")
files = glob.iglob(
os.path.join(hadoop_client_new_lib_dir, hadoop_lzo_pattern))
if not files:
raise Fail("There are no files at {0} matching {1}".format(
hadoop_client_new_lib_dir, hadoop_lzo_pattern))
# copy files into libext
files_copied = False
for file in files:
if os.path.isfile(file):
Logger.info("Copying {0} to {1}".format(
str(file), params.oozie_libext_dir))
shutil.copy2(file, params.oozie_libext_dir)
files_copied = True
if not files_copied:
raise Fail("There are no files at {0} matching {1}".format(
hadoop_client_new_lib_dir, hadoop_lzo_pattern))
# copy ext ZIP to libext dir
oozie_ext_zip_file = '/usr/share/HDP-oozie/ext-2.2.zip'
# something like <stack-root>/current/oozie-server/libext/ext-2.2.zip
oozie_ext_zip_target_path = os.path.join(params.oozie_libext_dir,
"ext-2.2.zip")
if not os.path.isfile(oozie_ext_zip_file):
raise Fail("Unable to copy {0} because it does not exist".format(
oozie_ext_zip_file))
Logger.info("Copying {0} to {1}".format(oozie_ext_zip_file,
params.oozie_libext_dir))
Execute(("cp", oozie_ext_zip_file, params.oozie_libext_dir), sudo=True)
Execute(("chown", format("{oozie_user}:{user_group}"),
oozie_ext_zip_target_path),
sudo=True)
File(oozie_ext_zip_target_path, mode=0644)
# Redownload jdbc driver to a new current location
oozie.download_database_library_if_needed()
# get the upgrade version in the event that it's needed
upgrade_stack = stack_select._get_upgrade_stack()
if upgrade_stack is None or len(
upgrade_stack) < 2 or upgrade_stack[1] is None:
raise Fail(
"Unable to determine the stack that is being upgraded to or downgraded to."
)
# something like 2.3.0.0-1234
stack_version = upgrade_stack[1]
# copy the Falcon JAR if needed; falcon has not upgraded yet, so we must
# use the versioned falcon directory
if params.has_falcon_host:
versioned_falcon_jar_directory = "/usr/hdp/{0}/falcon/oozie/ext/falcon-oozie-el-extension-*.jar".format(
stack_version)
Logger.info("Copying {0} to {1}".format(
versioned_falcon_jar_directory, params.oozie_libext_dir))
Execute(
format(
'{sudo} cp {versioned_falcon_jar_directory} {oozie_libext_dir}'
))
Execute(
format(
'{sudo} chown {oozie_user}:{user_group} {oozie_libext_dir}/falcon-oozie-el-extension-*.jar'
))